Merge pull request #6 from nlesc-dirac/linesearch_upgrade

keep sign of curvature

Author: SarodYatawatta

kan_pde.py

@@ -4,6 +4,7 @@
 
 from kan import KAN
 from lbfgsb import LBFGSB
+from lbfgsnew import LBFGSNew
 import torch
 import matplotlib.pyplot as plt
 from torch import autograd
@@ -22,7 +23,7 @@
 np_b = 21 # number of boundary points (along each dimension)
 ranges = [-1, 1]
 
-model = KAN(width=[2,2,1], grid=5, k=3, grid_eps=1.0, noise_scale_base=0.25, device=mydevice)
+model = KAN(width=[2,2,1], grid=5, k=3, grid_eps=1.0, device=mydevice)
 
 # get all parameters (all may not be trainable)
 n_params = sum([np.prod(p.size()) for p in model.parameters()])
@@ -70,6 +71,7 @@ def _func_sum(x):
 def train():
     # try running with batch_mode=True and batch_mode=False (both should work)
     optimizer = LBFGSB(model.parameters(), lower_bound=x_l, upper_bound=x_u, history_size=10,  tolerance_grad=1e-32, tolerance_change=1e-32, batch_mode=True, cost_use_gradient=True)
+    #optimizer = LBFGSNew(model.parameters(), history_size=10,  tolerance_grad=1e-32, tolerance_change=1e-32, batch_mode=True, cost_use_gradient=True)
 
     pbar = tqdm(range(steps), desc='description')
 

lbfgsb.py

@@ -556,24 +556,25 @@ def step(self, closure):
             y=g-g_old
             x=torch.cat(self._copy_params_out(),0)
             s=x-x_old
-            curv=abs(torch.dot(s,y))
+            curv=(torch.dot(s,y))
             n_iter +=1
             state['n_iter'] +=1
 
 
             batch_changed=batch_mode and (n_iter==1 and state['n_iter']>1)
             if batch_changed:
                 tmp_grad_1=g_old.clone(memory_format=torch.contiguous_format)
-                tmp_grad_1.add_(self.running_avg,alpha=-1.0)
+                tmp_grad_1.add_(self.running_avg,alpha=-1.0) # grad-oldmean
                 self.running_avg.add_(tmp_grad_1,alpha=1.0/state['n_iter'])
                 tmp_grad_2=g_old.clone(memory_format=torch.contiguous_format)
-                tmp_grad_2.add_(self.running_avg,alpha=-1.0)
-                self.running_avg_sq.addcmul_(tmp_grad_2,tmp_grad_1,value=1)
+                tmp_grad_2.add_(self.running_avg,alpha=-1.0) # grad-newmean
+                self.running_avg_sq.addcmul_(tmp_grad_2,tmp_grad_1,value=1) # # +(grad-newmean)(grad-oldmean)
                 self.alphabar=1.0/(1.0+self.running_avg_sq.sum()/((state['n_iter']-1)*g_old.norm().item()))
 
 
             if (curv<self._eps):
-                print('Warning: negative curvature detected, skipping update')
+                if be_verbose:
+                   print('Warning: negative curvature detected, skipping update')
                 n_iter+=1
                 continue
             # in batch mode, do not update Y and S if the batch has changed
@@ -601,9 +602,6 @@ def step(self, closure):
                 self._M=torch.linalg.pinv(MM) 
 
 
-
-       
-
         if be_verbose and (n_iter==max_iter):
             print('Reached maximum number of iterations,  stopping')