From 078d54c2d5489f4717c53ba2b996f1a2b333f8d9 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 4 Jul 2019 12:13:04 +0200 Subject: [PATCH 01/13] cleanup SP test --- .../unit/algorithms/SpatialPoolerTest.cpp | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index 667a40dd3b..8dea4b7300 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -239,7 +239,10 @@ void setup(SpatialPooler& sp, UInt numIn, UInt numCols, Real sparsity = 0.5f) { TEST(SpatialPoolerTest, testUpdateInhibitionRadius) { SpatialPooler sp; vector colDim, inputDim; - colDim.push_back(57); + + //test for global inhibition, this is trivial + { + colDim.push_back(57); //max SP dimension colDim.push_back(31); colDim.push_back(2); inputDim.push_back(1); @@ -248,9 +251,13 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) { EXPECT_NO_THROW(sp.initialize(inputDim, colDim)); sp.setGlobalInhibition(true); - ASSERT_EQ(sp.getInhibitionRadius(), 57u); + ASSERT_TRUE(sp.getInhibitionRadius() == 57) << "In global inh radius must match max dimension"; + } - //test 2 - local inhibition radius + //tests for local inhibition + UInt numInputs = 3; + UInt numCols = 12; + { colDim.clear(); inputDim.clear(); // avgColumnsPerInput = 4 @@ -266,12 +273,12 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) { Real permArr[] = {1, 1, 1}; sp.setPermanence(i, permArr); } - UInt trueInhibitionRadius = 6; // ((3 * 4) - 1)/2 => round up sp.updateInhibitionRadius_(); - ASSERT_EQ(trueInhibitionRadius, sp.getInhibitionRadius()); + ASSERT_EQ(6u, sp.getInhibitionRadius()); + } - //test 3 + { colDim.clear(); inputDim.clear(); // avgColumnsPerInput = 1.2 @@ -289,12 +296,11 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) { } sp.setPermanence(i, permArr); } - trueInhibitionRadius = 1; sp.updateInhibitionRadius_(); - ASSERT_EQ(trueInhibitionRadius, sp.getInhibitionRadius()); - + ASSERT_EQ(1u, sp.getInhibitionRadius()); + } - //test 4 + { colDim.clear(); inputDim.clear(); // avgColumnsPerInput = 2.4 @@ -309,10 +315,10 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) { Real permArr[] = {1, 1, 0, 0, 0}; sp.setPermanence(i, permArr); } - trueInhibitionRadius = 2; // ((2.4 * 2) - 1)/2 => round up sp.updateInhibitionRadius_(); - ASSERT_EQ(trueInhibitionRadius, sp.getInhibitionRadius()); + ASSERT_EQ(2u, sp.getInhibitionRadius()); + } } TEST(SpatialPoolerTest, testUpdateMinDutyCycles) { From e08aacddd022ccb384d281daa9639c0f0b1c3c23 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 4 Jul 2019 08:14:48 +0200 Subject: [PATCH 02/13] SP: use log boosting does not require parameter --- CommonCompilerConfig.cmake | 2 +- src/htm/algorithms/SpatialPooler.cpp | 3 ++- .../unit/algorithms/SpatialPoolerTest.cpp | 26 +++++-------------- 3 files changed, 10 insertions(+), 21 deletions(-) diff --git a/CommonCompilerConfig.cmake b/CommonCompilerConfig.cmake index 1b858299db..54ec86897e 100644 --- a/CommonCompilerConfig.cmake +++ b/CommonCompilerConfig.cmake @@ -350,7 +350,7 @@ else() set(optimization_flags_cc ${optimization_flags_cc} -pipe -O3) set(optimization_flags_lt ${optimization_flags_lt} -O3) if(NOT ${CMAKE_SYSTEM_PROCESSOR} STREQUAL "armv7l") - set(optimization_flags_cc ${optimization_flags_cc} -mtune=generic) + set(optimization_flags_cc ${optimization_flags_cc} -march=native) endif() if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" AND NOT MINGW) # NOTE -flto must go together in both cc and ld flags; also, it's presently incompatible diff --git a/src/htm/algorithms/SpatialPooler.cpp b/src/htm/algorithms/SpatialPooler.cpp index d529ba2d3f..768365c0de 100644 --- a/src/htm/algorithms/SpatialPooler.cpp +++ b/src/htm/algorithms/SpatialPooler.cpp @@ -756,7 +756,8 @@ void applyBoosting_(const UInt i, const Real boost, vector& output) { if(boost < htm::Epsilon) return; //skip for disabled boosting - output[i] = exp((targetDensity - actualDensity[i]) * boost); //TODO doc this code + //output[i] = exp((targetDensity - actualDensity[i]) * boost); //exponential boosting, default for Numenta + output[i] = log(actualDensity[i]) / log(targetDensity); } diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index 8dea4b7300..b61d2c46c6 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -991,24 +991,11 @@ TEST(SpatialPoolerTest, testUpdateBoostFactors) { sp.getBoostFactors(resultBoostFactors1.data()); ASSERT_TRUE(check_vector_eq(trueBoostFactors1, resultBoostFactors1)); - Real32 initActiveDutyCycles2[] = {0.1f, 0.3f, 0.02f, 0.04f, 0.7f, 0.12f}; - Real32 initBoostFactors2[] = {0, 0, 0, 0, 0, 0}; - vector trueBoostFactors2 = {3.10599f, 0.42035f, 6.91251f, - 5.65949f, 0.00769898f, 2.54297f}; - vector resultBoostFactors2(6, 0); - sp.setGlobalInhibition(false); - sp.setBoostStrength(10); - sp.setBoostFactors(initBoostFactors2); - sp.setActiveDutyCycles(initActiveDutyCycles2); - sp.updateBoostFactors_(); - sp.getBoostFactors(resultBoostFactors2.data()); - - ASSERT_TRUE(check_vector_eq(trueBoostFactors2, resultBoostFactors2)); Real32 initActiveDutyCycles3[] = {0.1f, 0.3f, 0.02f, 0.04f, 0.7f, 0.12f}; Real initBoostFactors3[] = {0, 0, 0, 0, 0, 0}; - vector trueBoostFactors3 = {1.25441f, 0.840857f, 1.47207f, - 1.41435f, 0.377822f, 1.20523f}; + vector trueBoostFactors3 = {1.49044f, 0.779321f, 2.53222f, + 2.08355f, 0.230873f, 1.37243f}; vector resultBoostFactors3(6, 0); sp.setWrapAround(true); sp.setGlobalInhibition(false); @@ -1021,10 +1008,11 @@ TEST(SpatialPoolerTest, testUpdateBoostFactors) { ASSERT_TRUE(check_vector_eq(trueBoostFactors3, resultBoostFactors3)); + Real32 initActiveDutyCycles4[] = {0.1f, 0.3f, 0.02f, 0.04f, 0.7f, 0.12f}; Real32 initBoostFactors4[] = {0, 0, 0, 0, 0, 0}; - vector trueBoostFactors4 = {1.94773f, 0.263597f, 4.33476f, - 3.549f, 0.00482795f, 1.59467f}; + vector trueBoostFactors4 = {1.2851f, 0.67195f, 2.18334f, + 1.79649f, 0.199064f, 1.18334f}; vector resultBoostFactors4(6, 0); sp.setGlobalInhibition(true); sp.setBoostStrength(10); @@ -2065,7 +2053,7 @@ TEST(SpatialPoolerTest, ExactOutput) { // Silver is an SDR that is loaded by direct initalization from a vector. SDR silver_sdr({ 200 }); SDR_sparse_t data = { - 4, 64, 74, 78, 85, 113, 125, 126, 127, 153 + 12, 51, 81, 113, 129, 133, 172, 175, 178, 190 }; silver_sdr.setSparse(data); @@ -2073,7 +2061,7 @@ TEST(SpatialPoolerTest, ExactOutput) { // Gold tests initalizing an SDR from a manually created string in JSON format. // hint: you can generate this string using // silver_sdr.save(std::cout, JSON); - string gold = "{\"dimensions\": [200],\"sparse\": [4, 64, 74, 78, 85, 113, 125, 126, 127, 153]}"; + string gold = "{\"dimensions\": [200],\"sparse\": [12, 51, 81, 113, 129, 133, 172, 175, 178, 190]}"; std::stringstream gold_stream( gold ); SDR gold_sdr; gold_sdr.load( gold_stream, JSON ); From 1de1a7d3319b6226b1a1433f4c1fad20ca02309f Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 4 Jul 2019 12:34:49 +0200 Subject: [PATCH 03/13] fix bug in a test compared old/tested arrays, not the new values --- src/test/unit/algorithms/SpatialPoolerTest.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index b61d2c46c6..afd9d0cbf9 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -1022,7 +1022,7 @@ TEST(SpatialPoolerTest, testUpdateBoostFactors) { sp.updateBoostFactors_(); sp.getBoostFactors(resultBoostFactors4.data()); - ASSERT_TRUE(check_vector_eq(trueBoostFactors3, resultBoostFactors3)); + ASSERT_TRUE(check_vector_eq(trueBoostFactors4, resultBoostFactors4)); } From d505b698a5408cc78dfaecfa4a97c356abbb5ba5 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Thu, 4 Jul 2019 13:40:46 +0200 Subject: [PATCH 04/13] SP fix boosting if strength == 0.0 --- src/examples/mnist/MNIST_SP.cpp | 4 ++-- src/htm/algorithms/SpatialPooler.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/examples/mnist/MNIST_SP.cpp b/src/examples/mnist/MNIST_SP.cpp index 8bd2d4677b..d43375b673 100644 --- a/src/examples/mnist/MNIST_SP.cpp +++ b/src/examples/mnist/MNIST_SP.cpp @@ -76,7 +76,7 @@ class MNIST { void setup() { input.initialize({28, 28,1}); - columns.initialize({28, 28, 8}); //1D vs 2D no big difference, 2D seems more natural for the problem. Speed-----, Results+++++++++; #columns HIGHEST impact. + columns.initialize({28, 28, 16}); //1D vs 2D no big difference, 2D seems more natural for the problem. Speed-----, Results+++++++++; #columns HIGHEST impact. sp.initialize( /* inputDimensions */ input.dimensions, /* columnDimensions */ columns.dimensions, @@ -90,7 +90,7 @@ void setup() { /* synPermConnected */ 0.5f, //no difference, let's leave at 0.5 in the middle /* minPctOverlapDutyCycles */ 0.2f, //speed of re-learning? /* dutyCyclePeriod */ 1402, - /* boostStrength */ 2.0f, // Boosting does help, but entropy is high, on MNIST it does not matter, for learning with TM prefer boosting off (=0.0), or "neutral"=1.0 + /* boostStrength */ 0.0f, // Boosting does help, but entropy is high, on MNIST it does not matter, for learning with TM prefer boosting off (=0.0), or "neutral"=1.0 /* seed */ 4u, /* spVerbosity */ 1u, /* wrapAround */ true); // does not matter (helps slightly) diff --git a/src/htm/algorithms/SpatialPooler.cpp b/src/htm/algorithms/SpatialPooler.cpp index 768365c0de..c2d281df12 100644 --- a/src/htm/algorithms/SpatialPooler.cpp +++ b/src/htm/algorithms/SpatialPooler.cpp @@ -742,6 +742,7 @@ void SpatialPooler::updateDutyCyclesHelper_(vector &dutyCycles, void SpatialPooler::updateBoostFactors_() { + if(boostStrength_ < htm::Epsilon) return; //skip for disabled boosting if (globalInhibition_) { updateBoostFactorsGlobal_(); } else { @@ -755,7 +756,6 @@ void applyBoosting_(const UInt i, const vector& actualDensity, const Real boost, vector& output) { - if(boost < htm::Epsilon) return; //skip for disabled boosting //output[i] = exp((targetDensity - actualDensity[i]) * boost); //exponential boosting, default for Numenta output[i] = log(actualDensity[i]) / log(targetDensity); } From 43c97df027b9cd8758c0d701af3aa20ad857dbc9 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Sat, 6 Jul 2019 22:24:43 +0200 Subject: [PATCH 05/13] fix experiment, boosting was off! now logboost has 96.2% --- src/examples/mnist/MNIST_SP.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/examples/mnist/MNIST_SP.cpp b/src/examples/mnist/MNIST_SP.cpp index d43375b673..2aa91ce706 100644 --- a/src/examples/mnist/MNIST_SP.cpp +++ b/src/examples/mnist/MNIST_SP.cpp @@ -90,7 +90,7 @@ void setup() { /* synPermConnected */ 0.5f, //no difference, let's leave at 0.5 in the middle /* minPctOverlapDutyCycles */ 0.2f, //speed of re-learning? /* dutyCyclePeriod */ 1402, - /* boostStrength */ 0.0f, // Boosting does help, but entropy is high, on MNIST it does not matter, for learning with TM prefer boosting off (=0.0), or "neutral"=1.0 + /* boostStrength */ 7.0f, // Boosting does help, but entropy is high, on MNIST it does not matter, for learning with TM prefer boosting off (=0.0), or "neutral"=1.0 /* seed */ 4u, /* spVerbosity */ 1u, /* wrapAround */ true); // does not matter (helps slightly) From 5c03ec95c594b1b22802034ee7b0d075ddfb8fb7 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Sat, 6 Jul 2019 22:51:41 +0200 Subject: [PATCH 06/13] SP optimized log boost --- src/htm/algorithms/SpatialPooler.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/htm/algorithms/SpatialPooler.cpp b/src/htm/algorithms/SpatialPooler.cpp index c2d281df12..e0613080cc 100644 --- a/src/htm/algorithms/SpatialPooler.cpp +++ b/src/htm/algorithms/SpatialPooler.cpp @@ -751,22 +751,24 @@ void SpatialPooler::updateBoostFactors_() { } -void applyBoosting_(const UInt i, +void applyBoosting_(const UInt numColumns, const Real targetDensity, const vector& actualDensity, const Real boost, vector& output) { - //output[i] = exp((targetDensity - actualDensity[i]) * boost); //exponential boosting, default for Numenta - output[i] = log(actualDensity[i]) / log(targetDensity); + + const Real LOG_TARGET_SPARSITY = log2(targetDensity); + for(UInt i=0; i< numColumns; i++) { + //output[i] = exp((targetDensity - actualDensity[i]) * boost); //exponential boosting, default for Numenta + output[i] = log2(actualDensity[i]) / LOG_TARGET_SPARSITY; + } } void SpatialPooler::updateBoostFactorsGlobal_() { const Real targetDensity = localAreaDensity_; - for (UInt i = 0; i < numColumns_; ++i) { - applyBoosting_(i, targetDensity, activeDutyCycles_, boostStrength_, boostFactors_); - } + applyBoosting_(numColumns_, targetDensity, activeDutyCycles_, boostStrength_, boostFactors_); } @@ -788,7 +790,7 @@ void SpatialPooler::updateBoostFactorsLocal_() { } const Real targetDensity = localActivityDensity / numNeighbors; - applyBoosting_(i, targetDensity, activeDutyCycles_, boostStrength_, boostFactors_); + //FIXMEapplyBoosting_(i, targetDensity, activeDutyCycles_, boostStrength_, boostFactors_); } } From f8ab3b5646b063e040314bbee89023037d4e84ac Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Sat, 6 Jul 2019 22:51:53 +0200 Subject: [PATCH 07/13] Revert "SP optimized log boost" This reverts commit c473c4d947bb9ede9a09c56da5d89e492f94da6a. --- src/htm/algorithms/SpatialPooler.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/src/htm/algorithms/SpatialPooler.cpp b/src/htm/algorithms/SpatialPooler.cpp index e0613080cc..c2d281df12 100644 --- a/src/htm/algorithms/SpatialPooler.cpp +++ b/src/htm/algorithms/SpatialPooler.cpp @@ -751,24 +751,22 @@ void SpatialPooler::updateBoostFactors_() { } -void applyBoosting_(const UInt numColumns, +void applyBoosting_(const UInt i, const Real targetDensity, const vector& actualDensity, const Real boost, vector& output) { - - const Real LOG_TARGET_SPARSITY = log2(targetDensity); - for(UInt i=0; i< numColumns; i++) { - //output[i] = exp((targetDensity - actualDensity[i]) * boost); //exponential boosting, default for Numenta - output[i] = log2(actualDensity[i]) / LOG_TARGET_SPARSITY; - } + //output[i] = exp((targetDensity - actualDensity[i]) * boost); //exponential boosting, default for Numenta + output[i] = log(actualDensity[i]) / log(targetDensity); } void SpatialPooler::updateBoostFactorsGlobal_() { const Real targetDensity = localAreaDensity_; - applyBoosting_(numColumns_, targetDensity, activeDutyCycles_, boostStrength_, boostFactors_); + for (UInt i = 0; i < numColumns_; ++i) { + applyBoosting_(i, targetDensity, activeDutyCycles_, boostStrength_, boostFactors_); + } } @@ -790,7 +788,7 @@ void SpatialPooler::updateBoostFactorsLocal_() { } const Real targetDensity = localActivityDensity / numNeighbors; - //FIXMEapplyBoosting_(i, targetDensity, activeDutyCycles_, boostStrength_, boostFactors_); + applyBoosting_(i, targetDensity, activeDutyCycles_, boostStrength_, boostFactors_); } } From f8fa30f7bb329ba75d64edaffc075e491cc90a08 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Sat, 6 Jul 2019 23:30:58 +0200 Subject: [PATCH 08/13] SP implement log, exp, none boosting methods --- src/examples/mnist/MNIST_SP.cpp | 4 ++-- src/htm/algorithms/SpatialPooler.cpp | 10 ++++++++-- src/htm/algorithms/SpatialPooler.hpp | 26 +++++++++++++++++--------- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/src/examples/mnist/MNIST_SP.cpp b/src/examples/mnist/MNIST_SP.cpp index 2aa91ce706..13d5defcef 100644 --- a/src/examples/mnist/MNIST_SP.cpp +++ b/src/examples/mnist/MNIST_SP.cpp @@ -75,8 +75,8 @@ class MNIST { void setup() { - input.initialize({28, 28,1}); - columns.initialize({28, 28, 16}); //1D vs 2D no big difference, 2D seems more natural for the problem. Speed-----, Results+++++++++; #columns HIGHEST impact. + input.initialize({28, 28, 1}); + columns.initialize({28, 28, 8}); //1D vs 2D no big difference, 2D seems more natural for the problem. Speed-----, Results+++++++++; #columns HIGHEST impact. sp.initialize( /* inputDimensions */ input.dimensions, /* columnDimensions */ columns.dimensions, diff --git a/src/htm/algorithms/SpatialPooler.cpp b/src/htm/algorithms/SpatialPooler.cpp index c2d281df12..af40f4e336 100644 --- a/src/htm/algorithms/SpatialPooler.cpp +++ b/src/htm/algorithms/SpatialPooler.cpp @@ -756,8 +756,14 @@ void applyBoosting_(const UInt i, const vector& actualDensity, const Real boost, vector& output) { - //output[i] = exp((targetDensity - actualDensity[i]) * boost); //exponential boosting, default for Numenta - output[i] = log(actualDensity[i]) / log(targetDensity); + + if(boost == SpatialPooler::BOOSTING_DISABLED) { //boosting disabled, skip + return; + } else if(boost == SpatialPooler::BOOSTING_LOG) { + output[i] = log2(actualDensity[i]) / log2(targetDensity); + } else { //exponential boost + output[i] = exp((targetDensity - actualDensity[i]) * boost); + } } diff --git a/src/htm/algorithms/SpatialPooler.hpp b/src/htm/algorithms/SpatialPooler.hpp index 459e2ed735..8238917d7b 100644 --- a/src/htm/algorithms/SpatialPooler.hpp +++ b/src/htm/algorithms/SpatialPooler.hpp @@ -61,8 +61,8 @@ using namespace std; class SpatialPooler : public Serializable { public: - - const Real MAX_LOCALAREADENSITY = 0.5f; //require atleast 2 areas + static const constexpr Real BOOSTING_DISABLED = 0.0f; + static const constexpr Real BOOSTING_LOG = -1234.0f; SpatialPooler(); SpatialPooler(const vector inputDimensions, const vector columnDimensions, @@ -176,13 +176,21 @@ class SpatialPooler : public Serializable boost. Shorter values make it potentially more unstable and likely to oscillate. - @param boostStrength A number greater or equal than 0, used to - control boosting strength. - No boosting is applied if it is set to 0.0, (runs faster due to skipped code). - The strength of boosting increases as a function of boostStrength. - Boosting encourages columns to have similar activeDutyCycles as their - neighbors, which will lead to more efficient use of columns. However, - too much boosting may also lead to instability of SP outputs. + @param boostStrength A number used to control boosting strength coeficient, + and mode of operation based on (reserved) values: + - `== SpatialPooler::BOOSTING_DISABLED`: + No boosting is applied if it is set, (runs faster due to skipped code). + - `== SpatialPooler::BOOSTING_LOG`: + Logarithmic boosting is used. + - `> 0.0`: + Exponential boosting (default in Numenta) selected. `boostStrength` is + used as a multiplacation constant. + The strength of boosting increases as a function of `boostStrength`. + + Boosting encourages columns to have similar `activeDutyCycles` (aka. activation + frequencies) as their neighbors, which will lead to more efficient use of columns. + This helps achieving the target sparsity of the output. + However, too much boosting may also lead to instability of SP outputs. @param seed Seed for our random number generator. If seed is < 0 From 2985813180406b14bfc8495ec84fdde6ccbf00ce Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Sun, 7 Jul 2019 00:36:35 +0200 Subject: [PATCH 09/13] review --- src/htm/algorithms/SpatialPooler.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/htm/algorithms/SpatialPooler.cpp b/src/htm/algorithms/SpatialPooler.cpp index af40f4e336..bcbfa3f916 100644 --- a/src/htm/algorithms/SpatialPooler.cpp +++ b/src/htm/algorithms/SpatialPooler.cpp @@ -742,8 +742,8 @@ void SpatialPooler::updateDutyCyclesHelper_(vector &dutyCycles, void SpatialPooler::updateBoostFactors_() { - if(boostStrength_ < htm::Epsilon) return; //skip for disabled boosting - if (globalInhibition_) { + if(boostStrength_ == SpatialPooler::BOOSTING_DISABLED) return; + else if (globalInhibition_) { updateBoostFactorsGlobal_(); } else { updateBoostFactorsLocal_(); @@ -757,13 +757,11 @@ void applyBoosting_(const UInt i, const Real boost, vector& output) { - if(boost == SpatialPooler::BOOSTING_DISABLED) { //boosting disabled, skip - return; - } else if(boost == SpatialPooler::BOOSTING_LOG) { + if(boost == SpatialPooler::BOOSTING_LOG) { //logarithmic boosting output[i] = log2(actualDensity[i]) / log2(targetDensity); - } else { //exponential boost + } else if(boost > 0) { //exponential boost output[i] = exp((targetDensity - actualDensity[i]) * boost); - } + } //else: BOOSTING_DISABLED } From 5ca1d257519542090b9fd45432aaa17b8052d41e Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 9 Jul 2019 12:13:48 +0200 Subject: [PATCH 10/13] code cleanup --- src/examples/mnist/MNIST_SP.cpp | 2 +- src/htm/algorithms/SpatialPooler.cpp | 12 ++++++------ src/htm/algorithms/SpatialPooler.hpp | 15 +++++++++++---- src/htm/types/Types.hpp | 2 +- 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/examples/mnist/MNIST_SP.cpp b/src/examples/mnist/MNIST_SP.cpp index 13d5defcef..df70e65648 100644 --- a/src/examples/mnist/MNIST_SP.cpp +++ b/src/examples/mnist/MNIST_SP.cpp @@ -90,7 +90,7 @@ void setup() { /* synPermConnected */ 0.5f, //no difference, let's leave at 0.5 in the middle /* minPctOverlapDutyCycles */ 0.2f, //speed of re-learning? /* dutyCyclePeriod */ 1402, - /* boostStrength */ 7.0f, // Boosting does help, but entropy is high, on MNIST it does not matter, for learning with TM prefer boosting off (=0.0), or "neutral"=1.0 + /* boostStrength */ 7.0f, // Boosting does help, but entropy is high, on MNIST it does not matter, for learning with TM prefer boosting off (BOOSTING_DISABLED), or "neutral"=1.0 /* seed */ 4u, /* spVerbosity */ 1u, /* wrapAround */ true); // does not matter (helps slightly) diff --git a/src/htm/algorithms/SpatialPooler.cpp b/src/htm/algorithms/SpatialPooler.cpp index bcbfa3f916..bd5b786f84 100644 --- a/src/htm/algorithms/SpatialPooler.cpp +++ b/src/htm/algorithms/SpatialPooler.cpp @@ -756,12 +756,12 @@ void applyBoosting_(const UInt i, const vector& actualDensity, const Real boost, vector& output) { - - if(boost == SpatialPooler::BOOSTING_LOG) { //logarithmic boosting + if(boost == SpatialPooler::BOOSTING_DISABLED) output[i] = actualDensity[i]; //no change + else if(boost == SpatialPooler::BOOSTING_LOG) { //logarithmic boosting output[i] = log2(actualDensity[i]) / log2(targetDensity); - } else if(boost > 0) { //exponential boost + } else if(boost >= SpatialPooler::BOOSTING_EXP) { //exponential boost output[i] = exp((targetDensity - actualDensity[i]) * boost); - } //else: BOOSTING_DISABLED + } else NTA_THROW << "Invalid boost mode! " << boost; } @@ -780,12 +780,12 @@ void SpatialPooler::updateBoostFactorsLocal_() { Real localActivityDensity = 0.0f; if (wrapAround_) { - for(auto neighbor: WrappingNeighborhood(i, inhibitionRadius_, columnDimensions_)) { + for(const auto neighbor: WrappingNeighborhood(i, inhibitionRadius_, columnDimensions_)) { localActivityDensity += activeDutyCycles_[neighbor]; numNeighbors += 1; } } else { - for(auto neighbor: Neighborhood(i, inhibitionRadius_, columnDimensions_)) { + for(const auto neighbor: Neighborhood(i, inhibitionRadius_, columnDimensions_)) { localActivityDensity += activeDutyCycles_[neighbor]; numNeighbors += 1; } diff --git a/src/htm/algorithms/SpatialPooler.hpp b/src/htm/algorithms/SpatialPooler.hpp index 8238917d7b..2109c877b9 100644 --- a/src/htm/algorithms/SpatialPooler.hpp +++ b/src/htm/algorithms/SpatialPooler.hpp @@ -61,12 +61,16 @@ using namespace std; class SpatialPooler : public Serializable { public: + //boosting modes: static const constexpr Real BOOSTING_DISABLED = 0.0f; - static const constexpr Real BOOSTING_LOG = -1234.0f; + static const constexpr Real BOOSTING_LOG = -1.0f * htm::Epsilon; //negative value closest to 0, so it's possible to search in parameter optimization to try all 3 combinations of boosting. + static const constexpr Real BOOSTING_EXP = 1.0f * htm::Epsilon; //any value > BOOSTING_DISABLED enables the exponential boosting mode SpatialPooler(); - SpatialPooler(const vector inputDimensions, const vector columnDimensions, - UInt potentialRadius = 16u, Real potentialPct = 0.5f, + SpatialPooler(const vector inputDimensions, + const vector columnDimensions, + UInt potentialRadius = 16u, + Real potentialPct = 0.5f, bool globalInhibition = true, Real localAreaDensity = 0.05f, //5% UInt stimulusThreshold = 0u, @@ -75,7 +79,7 @@ class SpatialPooler : public Serializable Real synPermConnected = 0.1f, Real minPctOverlapDutyCycles = 0.001f, UInt dutyCyclePeriod = 1000u, - Real boostStrength = 0.0f, + Real boostStrength = BOOSTING_DISABLED, Int seed = 1, UInt spVerbosity = 0u, bool wrapAround = true); @@ -192,6 +196,9 @@ class SpatialPooler : public Serializable This helps achieving the target sparsity of the output. However, too much boosting may also lead to instability of SP outputs. + Notes: + - Log boosting does not require a parameter, but it is slower than exp. + @param seed Seed for our random number generator. If seed is < 0 a randomly generated seed is used. The behavior of the spatial diff --git a/src/htm/types/Types.hpp b/src/htm/types/Types.hpp index 60e37d1846..361c20654d 100644 --- a/src/htm/types/Types.hpp +++ b/src/htm/types/Types.hpp @@ -130,7 +130,7 @@ typedef std::size_t Size; * numeric_limits::epsilon() == 1.19209e-7 * numeric_limits::epsilon() == 2.22045e-16 */ -static const htm::Real32 Epsilon = htm::Real(1e-6); +static const constexpr htm::Real32 Epsilon = htm::Real(1e-6); /** * Represents a signed integer. From cb8576792b7dbeba893bde505c81e8adb3b260a4 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Tue, 9 Jul 2019 12:38:19 +0200 Subject: [PATCH 11/13] SP: make initialize and constructor default params match! --- src/htm/algorithms/SpatialPooler.hpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/htm/algorithms/SpatialPooler.hpp b/src/htm/algorithms/SpatialPooler.hpp index 2109c877b9..65f008a4db 100644 --- a/src/htm/algorithms/SpatialPooler.hpp +++ b/src/htm/algorithms/SpatialPooler.hpp @@ -212,17 +212,22 @@ class SpatialPooler : public Serializable */ virtual void - initialize(const vector& inputDimensions, - const vector& columnDimensions, + initialize(const vector inputDimensions, + const vector columnDimensions, UInt potentialRadius = 16u, Real potentialPct = 0.5f, bool globalInhibition = true, Real localAreaDensity = 0.05f, - UInt stimulusThreshold = 0u, - Real synPermInactiveDec = 0.01f, Real synPermActiveInc = 0.1f, - Real synPermConnected = 0.1f, Real minPctOverlapDutyCycles = 0.001f, - UInt dutyCyclePeriod = 1000u, Real boostStrength = 0.0f, - Int seed = 1, UInt spVerbosity = 0u, bool wrapAround = true); + UInt stimulusThreshold = 0u, + Real synPermInactiveDec = 0.008f, + Real synPermActiveInc = 0.05f, + Real synPermConnected = 0.1f, + Real minPctOverlapDutyCycles = 0.001f, + UInt dutyCyclePeriod = 1000u, + Real boostStrength = BOOSTING_DISABLED, + Int seed = 1, + UInt spVerbosity = 0u, + bool wrapAround = true); /** From c2660a215f26cd3088d976411c07fa019b29eaf9 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Fri, 2 Aug 2019 17:29:51 +0200 Subject: [PATCH 12/13] merge fixups --- src/htm/algorithms/SpatialPooler.cpp | 3 +-- src/htm/algorithms/SpatialPooler.hpp | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/htm/algorithms/SpatialPooler.cpp b/src/htm/algorithms/SpatialPooler.cpp index bd5b786f84..bad25cc34b 100644 --- a/src/htm/algorithms/SpatialPooler.cpp +++ b/src/htm/algorithms/SpatialPooler.cpp @@ -71,7 +71,7 @@ SpatialPooler::SpatialPooler() { } SpatialPooler::SpatialPooler( - const vector inputDimensions, const vector columnDimensions, + const vector& inputDimensions, const vector& columnDimensions, UInt potentialRadius, Real potentialPct, bool globalInhibition, Real localAreaDensity, UInt stimulusThreshold, Real synPermInactiveDec, Real synPermActiveInc, @@ -401,7 +401,6 @@ void SpatialPooler::initialize( // 1D input produces 1D output; 2D => 2D, etc. //TODO allow nD -> mD conversion NTA_CHECK(inputDimensions_.size() == columnDimensions_.size()); - NTA_CHECK(localAreaDensity > 0 && localAreaDensity <= MAX_LOCALAREADENSITY); setLocalAreaDensity(localAreaDensity); rng_ = Random(seed); diff --git a/src/htm/algorithms/SpatialPooler.hpp b/src/htm/algorithms/SpatialPooler.hpp index 65f008a4db..bebebd0352 100644 --- a/src/htm/algorithms/SpatialPooler.hpp +++ b/src/htm/algorithms/SpatialPooler.hpp @@ -67,8 +67,8 @@ class SpatialPooler : public Serializable static const constexpr Real BOOSTING_EXP = 1.0f * htm::Epsilon; //any value > BOOSTING_DISABLED enables the exponential boosting mode SpatialPooler(); - SpatialPooler(const vector inputDimensions, - const vector columnDimensions, + SpatialPooler(const vector& inputDimensions, + const vector& columnDimensions, UInt potentialRadius = 16u, Real potentialPct = 0.5f, bool globalInhibition = true, @@ -212,8 +212,8 @@ class SpatialPooler : public Serializable */ virtual void - initialize(const vector inputDimensions, - const vector columnDimensions, + initialize(const vector& inputDimensions, + const vector& columnDimensions, UInt potentialRadius = 16u, Real potentialPct = 0.5f, bool globalInhibition = true, From 8ebb3e00da2473d806ea2beaa7af1f7129e4a650 Mon Sep 17 00:00:00 2001 From: Marek Otahal Date: Fri, 2 Aug 2019 22:39:01 +0200 Subject: [PATCH 13/13] fix exact output --- src/test/unit/algorithms/SpatialPoolerTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index afd9d0cbf9..422b7062be 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -2053,7 +2053,7 @@ TEST(SpatialPoolerTest, ExactOutput) { // Silver is an SDR that is loaded by direct initalization from a vector. SDR silver_sdr({ 200 }); SDR_sparse_t data = { - 12, 51, 81, 113, 129, 133, 172, 175, 178, 190 + 4, 64, 74, 78, 85, 113, 125, 126, 127, 153 }; silver_sdr.setSparse(data); @@ -2061,7 +2061,7 @@ TEST(SpatialPoolerTest, ExactOutput) { // Gold tests initalizing an SDR from a manually created string in JSON format. // hint: you can generate this string using // silver_sdr.save(std::cout, JSON); - string gold = "{\"dimensions\": [200],\"sparse\": [12, 51, 81, 113, 129, 133, 172, 175, 178, 190]}"; + string gold = "{\"dimensions\": [200],\"sparse\": [4, 64, 74, 78, 85, 113, 125, 126, 127, 153]}"; std::stringstream gold_stream( gold ); SDR gold_sdr; gold_sdr.load( gold_stream, JSON );