fix warning with detach()

Author: SarodYatawatta

lbfgsb.py

@@ -369,15 +369,15 @@ def _linesearch_backtrack(self, closure, f_old, gk, pk, alphabar):
         x0list=self._copy_params_out()
         xk=[x.clone() for x in x0list]
         self._add_grad(alphak,pk)
-        f_new=float(closure())
+        f_new=float(closure().detach())
         s=gk
         prodterm=c1*s.dot(pk)
         ci=0
         while (ci<citer and (math.isnan(f_new) or f_new>f_old+alphak*prodterm)):
             alphak=0.5*alphak
             self._copy_params_in(xk)
             self._add_grad(alphak,pk)
-            f_new=float(closure())
+            f_new=float(closure().detach())
             ci=ci+1
 
         self._copy_params_in(xk)
@@ -414,7 +414,7 @@ def _strong_wolfe(self, closure, f0, g0, p):
             # x=x0+alpha_i*p
             self._copy_params_in(x0)
             self._add_grad(alpha_i,p)
-            f_i=float(closure())
+            f_i=float(closure().detach())
             if (f_i>f0+c1*dphi0) or ((i>1) and (f_i>f_im1)):
                 alpha=self._alpha_zoom(closure,x0,f0,g0,p,alpha_im1,alpha_i)
                 break
@@ -462,12 +462,12 @@ def _alpha_zoom(self, closure, x0, f0, g0, p, alpha_lo, alpha_hi):
             # x=x0+alpha_i*p
             self._copy_params_in(x0)
             self._add_grad(alpha_i,p)
-            f_i=float(closure())
+            f_i=float(closure().detach())
             g_i=self._gather_flat_grad()
             # x_lo=x0+alpha_lo*p
             self._copy_params_in(x0)
             self._add_grad(alpha_lo,p)
-            f_lo=float(closure())
+            f_lo=float(closure().detach())
             if ((f_i>f0+c1*alpha_i*dphi0) or (f_i>=f_lo)):
                 alpha_hi=alpha_i
             else:
@@ -516,7 +516,7 @@ def step(self, closure):
 
         # evaluate initial f(x) and df/dx
         orig_loss = closure()
-        f= float(orig_loss)
+        f= float(orig_loss.detach())
         current_evals = 1
         state['func_evals'] += 1
 
@@ -551,7 +551,7 @@ def step(self, closure):
 
             self._add_grad(alpha,p)
 
-            f=float(closure())
+            f=float(closure().detach())
             g=self._gather_flat_grad()
             y=g-g_old
             x=torch.cat(self._copy_params_out(),0)

lbfgsnew.py

@@ -138,10 +138,10 @@ def _linesearch_backtrack(self,closure,pk,gk,alphabar):
         xk=self._copy_params_out()
 
 
-        f_old=float(closure())
+        f_old=float(closure().detach())
         # param = param + alphak * pk
         self._add_grad(alphak, pk)
-        f_new=float(closure())
+        f_new=float(closure().detach())
 
         # prod = c1 * ( alphak ) * gk^T pk = alphak * prodterm
         s=gk
@@ -155,7 +155,7 @@ def _linesearch_backtrack(self,closure,pk,gk,alphabar):
            alphak=0.5*alphak
            self._copy_params_in(xk)
            self._add_grad(alphak, pk)
-           f_new=float(closure())
+           f_new=float(closure().detach())
            if be_verbose:
              print('LN %d alpha=%f fnew=%f fold=%f'%(ci,alphak,f_new,f_old))
            ci=ci+1
@@ -165,14 +165,14 @@ def _linesearch_backtrack(self,closure,pk,gk,alphabar):
           alphak1=-alphabar
           self._copy_params_in(xk)
           self._add_grad(alphak1, pk)
-          f_new1=float(closure())
+          f_new1=float(closure().detach())
           if be_verbose:
             print('NLN fnew=%f'%f_new1)
           while (ci<citer and (math.isnan(f_new1) or  f_new1 > f_old + alphak1*prodterm)):
              alphak1=0.5*alphak1
              self._copy_params_in(xk)
              self._add_grad(alphak1, pk)
-             f_new1=float(closure())
+             f_new1=float(closure().detach())
              if be_verbose:
                print('NLN %d alpha=%f fnew=%f fold=%f'%(ci,alphak1,f_new1,f_old))
              ci=ci+1
@@ -215,15 +215,15 @@ def _linesearch_cubic(self,closure,pk,step):
         # make a copy of original params
         xk=self._copy_params_out()
 
-        phi_0=float(closure())
+        phi_0=float(closure().detach())
         tol=min(phi_0*0.01,1e-6)
 
         # xp <- xk+step. pk
         self._add_grad(step, pk) #FF param = param + t * grad 
-        p01=float(closure())
+        p01=float(closure().detach())
         # xp <- xk-step. pk
         self._add_grad(-2.0*step, pk) #FF param = param - t * grad 
-        p02=float(closure())
+        p02=float(closure().detach())
 
         ##print("p01="+str(p01)+" p02="+str(p02))
         gphi_0=(p01-p02)/(2.0*step)
@@ -251,7 +251,7 @@ def _linesearch_cubic(self,closure,pk,step):
           self._copy_params_in(xk) # original
           # xp <- xk+alphai. pk
           self._add_grad(alphai, pk) #
-          phi_alphai=float(closure())
+          phi_alphai=float(closure().detach())
           if phi_alphai<tol:
              alphak=alphai 
              if be_verbose:
@@ -270,10 +270,10 @@ def _linesearch_cubic(self,closure,pk,step):
           # note that self._params already is xk+alphai. pk, so only add the missing term
           # xp <- xk+(alphai+step). pk
           self._add_grad(step, pk) #FF param = param - t * grad 
-          p01=float(closure())
+          p01=float(closure().detach())
           # xp <- xk+(alphai-step). pk
           self._add_grad(-2.0*step, pk) #FF param = param - t * grad 
-          p02=float(closure())
+          p02=float(closure().detach())
           gphi_i=(p01-p02)/(2.0*step);
 
           if (abs(gphi_i)<=-sigma*gphi_0):
@@ -338,24 +338,24 @@ def _cubic_interpolate(self,closure,xk,pk,a,b,step):
 
         # xp <- xk+a. pk
         self._add_grad(a, pk) #FF param = param + t * grad 
-        f0=float(closure())
+        f0=float(closure().detach())
         # xp <- xk+(a+step). pk
         self._add_grad(step, pk) #FF param = param + t * grad 
-        p01=float(closure())
+        p01=float(closure().detach())
         # xp <- xk+(a-step). pk
         self._add_grad(-2.0*step, pk) #FF param = param - t * grad 
-        p02=float(closure())
+        p02=float(closure().detach())
         f0d=(p01-p02)/(2.0*step)
 
         # xp <- xk+b. pk
         self._add_grad(-a+step+b, pk) #FF param = param + t * grad 
-        f1=float(closure())
+        f1=float(closure().detach())
         # xp <- xk+(b+step). pk
         self._add_grad(step, pk) #FF param = param + t * grad 
-        p01=float(closure())
+        p01=float(closure().detach())
         # xp <- xk+(b-step). pk
         self._add_grad(-2.0*step, pk) #FF param = param - t * grad 
-        p02=float(closure())
+        p02=float(closure().detach())
         f1d=(p01-p02)/(2.0*step)
 
         closure_evals=6
@@ -375,7 +375,7 @@ def _cubic_interpolate(self,closure,xk,pk,a,b,step):
            else:
              # xp <- xk+(a+z0*(b-a))*pk
              self._add_grad(-b+step+a+z0*(b-a), pk) #FF param = param + t * grad 
-             fz0=float(closure())
+             fz0=float(closure().detach())
              closure_evals +=1
 
            # update state
@@ -443,12 +443,12 @@ def _linesearch_zoom(self,closure,xk,pk,a,b,phi_0,gphi_0,sigma,rho,t1,t2,t3,step
            self._copy_params_in(xk)
            # xp <- xk+alphaj. pk
            self._add_grad(alphaj, pk) #FF param = param + t * grad 
-           phi_j=float(closure())
+           phi_j=float(closure().detach())
 
            # evaluate phi(aj)
            # xp <- xk+aj. pk
            self._add_grad(-alphaj+aj, pk) #FF param = param + t * grad 
-           phi_aj=float(closure())
+           phi_aj=float(closure().detach())
 
            closure_evals +=2
 
@@ -458,10 +458,10 @@ def _linesearch_zoom(self,closure,xk,pk,a,b,phi_0,gphi_0,sigma,rho,t1,t2,t3,step
               # evaluate grad(alphaj)
               # xp <- xk+(alphaj+step). pk
               self._add_grad(-aj+alphaj+step, pk) #FF param = param + t * grad 
-              p01=float(closure())
+              p01=float(closure().detach())
               # xp <- xk+(alphaj-step). pk
               self._add_grad(-2.0*step, pk) #FF param = param + t * grad 
-              p02=float(closure())
+              p02=float(closure().detach())
               gphi_j=(p01-p02)/(2.0*step)
 
 
@@ -526,7 +526,7 @@ def step(self, closure):
 
         # evaluate initial f(x) and df/dx
         orig_loss = closure()
-        loss = float(orig_loss)
+        loss = float(orig_loss.detach())
         current_evals = 1
         state['func_evals'] += 1
 
@@ -707,7 +707,7 @@ def step(self, closure):
                     # re-evaluate function only if not in last iteration
                     # the reason we do this: in a stochastic setting,
                     # no use to re-evaluate that function here
-                    loss = float(closure())
+                    loss = float(closure().detach())
                     flat_grad = self._gather_flat_grad()
                     abs_grad_sum = flat_grad.abs().sum()
                     if math.isnan(abs_grad_sum):