Optimize Adam and AdaMax Optimizer

Author: andrewdalpino

CHANGELOG.md

@@ -1,3 +1,6 @@
+- 2.3.3
+    - Optimize Adam and AdaMax Optimizers
+
 - 2.3.2
     - Update PHP Stemmer to version 3
 

src/NeuralNet/Optimizers/AdaMax.php

@@ -82,8 +82,10 @@ public function step(Parameter $param, Tensor $gradient) : Tensor
     {
         [$velocity, $norm] = $this->cache[$param->id()];
 
-        $velocity = $velocity->multiply(1.0 - $this->momentumDecay)
-            ->add($gradient->multiply($this->momentumDecay));
+        $vHat = $gradient->subtract($velocity)
+            ->multiply($this->momentumDecay);
+
+        $velocity = $velocity->add($vHat);
 
         $norm = $norm->multiply(1.0 - $this->normDecay);
 

src/NeuralNet/Optimizers/Adam.php

@@ -122,11 +122,15 @@ public function step(Parameter $param, Tensor $gradient) : Tensor
     {
         [$velocity, $norm] = $this->cache[$param->id()];
 
-        $velocity = $velocity->multiply(1.0 - $this->momentumDecay)
-            ->add($gradient->multiply($this->momentumDecay));
+        $vHat = $gradient->subtract($velocity)
+            ->multiply($this->momentumDecay);
 
-        $norm = $norm->multiply(1.0 - $this->normDecay)
-            ->add($gradient->square()->multiply($this->normDecay));
+        $velocity = $velocity->add($vHat);
+
+        $nHat = $gradient->square()->subtract($norm)
+            ->multiply($this->normDecay);
+
+        $norm = $norm->add($nHat);
 
         $this->cache[$param->id()] = [$velocity, $norm];
 

src/NeuralNet/Snapshotter.php

@@ -54,7 +54,7 @@ public function step(Parameter $param, Tensor $gradient, array $expected) : void
 
         $step = $this->optimizer->step($param, $gradient);
 
-        $this->assertEquals($expected, $step->asArray());
+        $this->assertEqualsWithDelta($expected, $step->asArray(), 1e-8);
     }
 
     /**