diff --git a/CommonCompilerConfig.cmake b/CommonCompilerConfig.cmake index 1b858299db..54ec86897e 100644 --- a/CommonCompilerConfig.cmake +++ b/CommonCompilerConfig.cmake @@ -350,7 +350,7 @@ else() set(optimization_flags_cc ${optimization_flags_cc} -pipe -O3) set(optimization_flags_lt ${optimization_flags_lt} -O3) if(NOT ${CMAKE_SYSTEM_PROCESSOR} STREQUAL "armv7l") - set(optimization_flags_cc ${optimization_flags_cc} -mtune=generic) + set(optimization_flags_cc ${optimization_flags_cc} -march=native) endif() if(${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU" AND NOT MINGW) # NOTE -flto must go together in both cc and ld flags; also, it's presently incompatible diff --git a/src/examples/mnist/MNIST_SP.cpp b/src/examples/mnist/MNIST_SP.cpp index 8bd2d4677b..df70e65648 100644 --- a/src/examples/mnist/MNIST_SP.cpp +++ b/src/examples/mnist/MNIST_SP.cpp @@ -75,7 +75,7 @@ class MNIST { void setup() { - input.initialize({28, 28,1}); + input.initialize({28, 28, 1}); columns.initialize({28, 28, 8}); //1D vs 2D no big difference, 2D seems more natural for the problem. Speed-----, Results+++++++++; #columns HIGHEST impact. sp.initialize( /* inputDimensions */ input.dimensions, @@ -90,7 +90,7 @@ void setup() { /* synPermConnected */ 0.5f, //no difference, let's leave at 0.5 in the middle /* minPctOverlapDutyCycles */ 0.2f, //speed of re-learning? /* dutyCyclePeriod */ 1402, - /* boostStrength */ 2.0f, // Boosting does help, but entropy is high, on MNIST it does not matter, for learning with TM prefer boosting off (=0.0), or "neutral"=1.0 + /* boostStrength */ 7.0f, // Boosting does help, but entropy is high, on MNIST it does not matter, for learning with TM prefer boosting off (BOOSTING_DISABLED), or "neutral"=1.0 /* seed */ 4u, /* spVerbosity */ 1u, /* wrapAround */ true); // does not matter (helps slightly) diff --git a/src/htm/algorithms/SpatialPooler.cpp b/src/htm/algorithms/SpatialPooler.cpp index d529ba2d3f..bad25cc34b 100644 --- a/src/htm/algorithms/SpatialPooler.cpp +++ b/src/htm/algorithms/SpatialPooler.cpp @@ -71,7 +71,7 @@ SpatialPooler::SpatialPooler() { } SpatialPooler::SpatialPooler( - const vector inputDimensions, const vector columnDimensions, + const vector& inputDimensions, const vector& columnDimensions, UInt potentialRadius, Real potentialPct, bool globalInhibition, Real localAreaDensity, UInt stimulusThreshold, Real synPermInactiveDec, Real synPermActiveInc, @@ -401,7 +401,6 @@ void SpatialPooler::initialize( // 1D input produces 1D output; 2D => 2D, etc. //TODO allow nD -> mD conversion NTA_CHECK(inputDimensions_.size() == columnDimensions_.size()); - NTA_CHECK(localAreaDensity > 0 && localAreaDensity <= MAX_LOCALAREADENSITY); setLocalAreaDensity(localAreaDensity); rng_ = Random(seed); @@ -742,7 +741,8 @@ void SpatialPooler::updateDutyCyclesHelper_(vector &dutyCycles, void SpatialPooler::updateBoostFactors_() { - if (globalInhibition_) { + if(boostStrength_ == SpatialPooler::BOOSTING_DISABLED) return; + else if (globalInhibition_) { updateBoostFactorsGlobal_(); } else { updateBoostFactorsLocal_(); @@ -755,8 +755,12 @@ void applyBoosting_(const UInt i, const vector& actualDensity, const Real boost, vector& output) { - if(boost < htm::Epsilon) return; //skip for disabled boosting - output[i] = exp((targetDensity - actualDensity[i]) * boost); //TODO doc this code + if(boost == SpatialPooler::BOOSTING_DISABLED) output[i] = actualDensity[i]; //no change + else if(boost == SpatialPooler::BOOSTING_LOG) { //logarithmic boosting + output[i] = log2(actualDensity[i]) / log2(targetDensity); + } else if(boost >= SpatialPooler::BOOSTING_EXP) { //exponential boost + output[i] = exp((targetDensity - actualDensity[i]) * boost); + } else NTA_THROW << "Invalid boost mode! " << boost; } @@ -775,12 +779,12 @@ void SpatialPooler::updateBoostFactorsLocal_() { Real localActivityDensity = 0.0f; if (wrapAround_) { - for(auto neighbor: WrappingNeighborhood(i, inhibitionRadius_, columnDimensions_)) { + for(const auto neighbor: WrappingNeighborhood(i, inhibitionRadius_, columnDimensions_)) { localActivityDensity += activeDutyCycles_[neighbor]; numNeighbors += 1; } } else { - for(auto neighbor: Neighborhood(i, inhibitionRadius_, columnDimensions_)) { + for(const auto neighbor: Neighborhood(i, inhibitionRadius_, columnDimensions_)) { localActivityDensity += activeDutyCycles_[neighbor]; numNeighbors += 1; } diff --git a/src/htm/algorithms/SpatialPooler.hpp b/src/htm/algorithms/SpatialPooler.hpp index 459e2ed735..bebebd0352 100644 --- a/src/htm/algorithms/SpatialPooler.hpp +++ b/src/htm/algorithms/SpatialPooler.hpp @@ -61,12 +61,16 @@ using namespace std; class SpatialPooler : public Serializable { public: - - const Real MAX_LOCALAREADENSITY = 0.5f; //require atleast 2 areas + //boosting modes: + static const constexpr Real BOOSTING_DISABLED = 0.0f; + static const constexpr Real BOOSTING_LOG = -1.0f * htm::Epsilon; //negative value closest to 0, so it's possible to search in parameter optimization to try all 3 combinations of boosting. + static const constexpr Real BOOSTING_EXP = 1.0f * htm::Epsilon; //any value > BOOSTING_DISABLED enables the exponential boosting mode SpatialPooler(); - SpatialPooler(const vector inputDimensions, const vector columnDimensions, - UInt potentialRadius = 16u, Real potentialPct = 0.5f, + SpatialPooler(const vector& inputDimensions, + const vector& columnDimensions, + UInt potentialRadius = 16u, + Real potentialPct = 0.5f, bool globalInhibition = true, Real localAreaDensity = 0.05f, //5% UInt stimulusThreshold = 0u, @@ -75,7 +79,7 @@ class SpatialPooler : public Serializable Real synPermConnected = 0.1f, Real minPctOverlapDutyCycles = 0.001f, UInt dutyCyclePeriod = 1000u, - Real boostStrength = 0.0f, + Real boostStrength = BOOSTING_DISABLED, Int seed = 1, UInt spVerbosity = 0u, bool wrapAround = true); @@ -176,13 +180,24 @@ class SpatialPooler : public Serializable boost. Shorter values make it potentially more unstable and likely to oscillate. - @param boostStrength A number greater or equal than 0, used to - control boosting strength. - No boosting is applied if it is set to 0.0, (runs faster due to skipped code). - The strength of boosting increases as a function of boostStrength. - Boosting encourages columns to have similar activeDutyCycles as their - neighbors, which will lead to more efficient use of columns. However, - too much boosting may also lead to instability of SP outputs. + @param boostStrength A number used to control boosting strength coeficient, + and mode of operation based on (reserved) values: + - `== SpatialPooler::BOOSTING_DISABLED`: + No boosting is applied if it is set, (runs faster due to skipped code). + - `== SpatialPooler::BOOSTING_LOG`: + Logarithmic boosting is used. + - `> 0.0`: + Exponential boosting (default in Numenta) selected. `boostStrength` is + used as a multiplacation constant. + The strength of boosting increases as a function of `boostStrength`. + + Boosting encourages columns to have similar `activeDutyCycles` (aka. activation + frequencies) as their neighbors, which will lead to more efficient use of columns. + This helps achieving the target sparsity of the output. + However, too much boosting may also lead to instability of SP outputs. + + Notes: + - Log boosting does not require a parameter, but it is slower than exp. @param seed Seed for our random number generator. If seed is < 0 @@ -203,11 +218,16 @@ class SpatialPooler : public Serializable Real potentialPct = 0.5f, bool globalInhibition = true, Real localAreaDensity = 0.05f, - UInt stimulusThreshold = 0u, - Real synPermInactiveDec = 0.01f, Real synPermActiveInc = 0.1f, - Real synPermConnected = 0.1f, Real minPctOverlapDutyCycles = 0.001f, - UInt dutyCyclePeriod = 1000u, Real boostStrength = 0.0f, - Int seed = 1, UInt spVerbosity = 0u, bool wrapAround = true); + UInt stimulusThreshold = 0u, + Real synPermInactiveDec = 0.008f, + Real synPermActiveInc = 0.05f, + Real synPermConnected = 0.1f, + Real minPctOverlapDutyCycles = 0.001f, + UInt dutyCyclePeriod = 1000u, + Real boostStrength = BOOSTING_DISABLED, + Int seed = 1, + UInt spVerbosity = 0u, + bool wrapAround = true); /** diff --git a/src/htm/types/Types.hpp b/src/htm/types/Types.hpp index 60e37d1846..361c20654d 100644 --- a/src/htm/types/Types.hpp +++ b/src/htm/types/Types.hpp @@ -130,7 +130,7 @@ typedef std::size_t Size; * numeric_limits::epsilon() == 1.19209e-7 * numeric_limits::epsilon() == 2.22045e-16 */ -static const htm::Real32 Epsilon = htm::Real(1e-6); +static const constexpr htm::Real32 Epsilon = htm::Real(1e-6); /** * Represents a signed integer. diff --git a/src/test/unit/algorithms/SpatialPoolerTest.cpp b/src/test/unit/algorithms/SpatialPoolerTest.cpp index 667a40dd3b..422b7062be 100644 --- a/src/test/unit/algorithms/SpatialPoolerTest.cpp +++ b/src/test/unit/algorithms/SpatialPoolerTest.cpp @@ -239,7 +239,10 @@ void setup(SpatialPooler& sp, UInt numIn, UInt numCols, Real sparsity = 0.5f) { TEST(SpatialPoolerTest, testUpdateInhibitionRadius) { SpatialPooler sp; vector colDim, inputDim; - colDim.push_back(57); + + //test for global inhibition, this is trivial + { + colDim.push_back(57); //max SP dimension colDim.push_back(31); colDim.push_back(2); inputDim.push_back(1); @@ -248,9 +251,13 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) { EXPECT_NO_THROW(sp.initialize(inputDim, colDim)); sp.setGlobalInhibition(true); - ASSERT_EQ(sp.getInhibitionRadius(), 57u); + ASSERT_TRUE(sp.getInhibitionRadius() == 57) << "In global inh radius must match max dimension"; + } - //test 2 - local inhibition radius + //tests for local inhibition + UInt numInputs = 3; + UInt numCols = 12; + { colDim.clear(); inputDim.clear(); // avgColumnsPerInput = 4 @@ -266,12 +273,12 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) { Real permArr[] = {1, 1, 1}; sp.setPermanence(i, permArr); } - UInt trueInhibitionRadius = 6; // ((3 * 4) - 1)/2 => round up sp.updateInhibitionRadius_(); - ASSERT_EQ(trueInhibitionRadius, sp.getInhibitionRadius()); + ASSERT_EQ(6u, sp.getInhibitionRadius()); + } - //test 3 + { colDim.clear(); inputDim.clear(); // avgColumnsPerInput = 1.2 @@ -289,12 +296,11 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) { } sp.setPermanence(i, permArr); } - trueInhibitionRadius = 1; sp.updateInhibitionRadius_(); - ASSERT_EQ(trueInhibitionRadius, sp.getInhibitionRadius()); - + ASSERT_EQ(1u, sp.getInhibitionRadius()); + } - //test 4 + { colDim.clear(); inputDim.clear(); // avgColumnsPerInput = 2.4 @@ -309,10 +315,10 @@ TEST(SpatialPoolerTest, testUpdateInhibitionRadius) { Real permArr[] = {1, 1, 0, 0, 0}; sp.setPermanence(i, permArr); } - trueInhibitionRadius = 2; // ((2.4 * 2) - 1)/2 => round up sp.updateInhibitionRadius_(); - ASSERT_EQ(trueInhibitionRadius, sp.getInhibitionRadius()); + ASSERT_EQ(2u, sp.getInhibitionRadius()); + } } TEST(SpatialPoolerTest, testUpdateMinDutyCycles) { @@ -985,24 +991,11 @@ TEST(SpatialPoolerTest, testUpdateBoostFactors) { sp.getBoostFactors(resultBoostFactors1.data()); ASSERT_TRUE(check_vector_eq(trueBoostFactors1, resultBoostFactors1)); - Real32 initActiveDutyCycles2[] = {0.1f, 0.3f, 0.02f, 0.04f, 0.7f, 0.12f}; - Real32 initBoostFactors2[] = {0, 0, 0, 0, 0, 0}; - vector trueBoostFactors2 = {3.10599f, 0.42035f, 6.91251f, - 5.65949f, 0.00769898f, 2.54297f}; - vector resultBoostFactors2(6, 0); - sp.setGlobalInhibition(false); - sp.setBoostStrength(10); - sp.setBoostFactors(initBoostFactors2); - sp.setActiveDutyCycles(initActiveDutyCycles2); - sp.updateBoostFactors_(); - sp.getBoostFactors(resultBoostFactors2.data()); - - ASSERT_TRUE(check_vector_eq(trueBoostFactors2, resultBoostFactors2)); Real32 initActiveDutyCycles3[] = {0.1f, 0.3f, 0.02f, 0.04f, 0.7f, 0.12f}; Real initBoostFactors3[] = {0, 0, 0, 0, 0, 0}; - vector trueBoostFactors3 = {1.25441f, 0.840857f, 1.47207f, - 1.41435f, 0.377822f, 1.20523f}; + vector trueBoostFactors3 = {1.49044f, 0.779321f, 2.53222f, + 2.08355f, 0.230873f, 1.37243f}; vector resultBoostFactors3(6, 0); sp.setWrapAround(true); sp.setGlobalInhibition(false); @@ -1015,10 +1008,11 @@ TEST(SpatialPoolerTest, testUpdateBoostFactors) { ASSERT_TRUE(check_vector_eq(trueBoostFactors3, resultBoostFactors3)); + Real32 initActiveDutyCycles4[] = {0.1f, 0.3f, 0.02f, 0.04f, 0.7f, 0.12f}; Real32 initBoostFactors4[] = {0, 0, 0, 0, 0, 0}; - vector trueBoostFactors4 = {1.94773f, 0.263597f, 4.33476f, - 3.549f, 0.00482795f, 1.59467f}; + vector trueBoostFactors4 = {1.2851f, 0.67195f, 2.18334f, + 1.79649f, 0.199064f, 1.18334f}; vector resultBoostFactors4(6, 0); sp.setGlobalInhibition(true); sp.setBoostStrength(10); @@ -1028,7 +1022,7 @@ TEST(SpatialPoolerTest, testUpdateBoostFactors) { sp.updateBoostFactors_(); sp.getBoostFactors(resultBoostFactors4.data()); - ASSERT_TRUE(check_vector_eq(trueBoostFactors3, resultBoostFactors3)); + ASSERT_TRUE(check_vector_eq(trueBoostFactors4, resultBoostFactors4)); }