add new testing utilities for random dataset generation and add randomized stress test

Author: paulbkoch

shared/libebm/tests/boosting_unusual_inputs.cpp

@@ -2060,3 +2060,59 @@ TEST_CASE("lossguide, boosting, regression") {
    termScore = test.GetCurrentTermScore(0, {0}, 0);
    CHECK_APPROX(termScore, 0.40592050000000002);
 }
+
+TEST_CASE("stress test, boosting, regression") {
+   const IntEbm cClasses = Task_Regression;
+   auto rng = MakeRng(0);
+   const IntEbm cTrainSamples = 200;
+   const IntEbm cValidationSamples = 100;
+   const std::vector<FeatureTest> features = {
+         FeatureTest(10, false, false, false),
+         FeatureTest(10, false, false, true),
+         FeatureTest(10, false, true, false),
+         FeatureTest(10, false, true, true),
+         FeatureTest(10, true, false, false),
+         FeatureTest(10, true, false, true),
+         FeatureTest(10, true, true, false),
+         FeatureTest(10, true, true, true),
+   };
+   auto terms = MakeMains(features);
+   terms.push_back({0, 0});
+   if(2 <= features.size()) {
+      terms.push_back({0, 1});
+      terms.push_back({1, 0});
+   }
+   if(3 <= features.size()) {
+      // terms.push_back({0, 1, 2}); // TODO: enable when fast enough
+   }
+   if(4 <= features.size()) {
+      // terms.push_back({0, 1, 2, 3}); // TODO: enable when fast enough
+   }
+   const auto train = MakeRandomDataset(rng, cClasses, cTrainSamples, features);
+   const auto validation = MakeRandomDataset(rng, cClasses, cValidationSamples, features);
+   const size_t cRounds = 1000;
+   const IntEbm boostFlags = TermBoostFlags_Default;
+
+   TestBoost test = TestBoost(cClasses,
+         features,
+         terms,
+         train,
+         validation,
+         k_countInnerBagsDefault,
+         k_testCreateBoosterFlags_Default,
+         AccelerationFlags_NONE);
+
+   double validationMetric = 0;
+   for(size_t iRound = 0; iRound < cRounds; ++iRound) {
+      for(IntEbm iTerm = 0; iTerm < static_cast<IntEbm>(terms.size()); ++iTerm) {
+         // we allow 1 cut more than the number of bins to test excessive leaves.
+         const IntEbm cLeaves = 1 + TestRand(rng, features[terms[iTerm][0]].CountRealBins() + 1);
+         const std::vector<IntEbm> leaves(terms[iTerm].size(), cLeaves);
+         validationMetric =
+               test.Boost(iTerm, boostFlags, k_learningRateDefault, 1, 0.0, 0.0, 0.0, 0.0, 10.0, 6, 0.75, leaves)
+                     .validationMetric;
+      }
+   }
+
+   CHECK(validationMetric == 15092.795219174546);
+}

shared/libebm/tests/libebm_test.cpp

@@ -1004,6 +1004,43 @@ extern void DisplayCuts(IntEbm countSamples,
    std::cout << std::endl << std::endl;
 }
 
+extern std::vector<TestSample> MakeRandomDataset(std::vector<unsigned char>& rng,
+      const IntEbm cClasses,
+      const size_t cSamples,
+      const std::vector<FeatureTest>& features) {
+   std::vector<TestSample> samples;
+
+   for(size_t iSample = 0; iSample < cSamples; ++iSample) {
+      std::vector<IntEbm> sampleBinIndexes;
+      for(const FeatureTest& feature : features) {
+         IntEbm iBin = TestRand(rng, feature.CountRealBins());
+         if(!feature.m_bMissing) {
+            ++iBin;
+         }
+         sampleBinIndexes.push_back(iBin);
+      }
+
+      double target;
+      if(Task_GeneralClassification <= cClasses) {
+         target = static_cast<double>(TestRand(rng, cClasses));
+      } else {
+         target = TestRand(rng);
+      }
+
+      samples.push_back(TestSample(sampleBinIndexes, target));
+   }
+   return samples;
+}
+
+extern std::vector<std::vector<IntEbm>> MakeMains(const std::vector<FeatureTest>& features) {
+   const IntEbm cFeatures = static_cast<IntEbm>(features.size());
+   std::vector<std::vector<IntEbm>> termFeatures;
+   for(IntEbm iFeature = 0; iFeature < cFeatures; ++iFeature) {
+      termFeatures.push_back({iFeature});
+   }
+   return termFeatures;
+}
+
 int main() {
    SetLogCallback(&LogCallback);
    SetTraceLevel(Trace_Verbose);

shared/libebm/tests/libebm_test.hpp

@@ -179,6 +179,8 @@ class FeatureTest final {
    const bool m_bUnseen;
    const bool m_bNominal;
 
+   inline IntEbm CountRealBins() const { return m_countBins - (m_bMissing ? 0 : 1) - (m_bUnseen ? 0 : 1); }
+
    inline FeatureTest(
          const IntEbm countBins, const bool bMissing = true, const bool bUnseen = true, const bool bNominal = false) :
          m_countBins(countBins), m_bMissing(bMissing), m_bUnseen(bUnseen), m_bNominal(bNominal) {}
@@ -536,4 +538,30 @@ void DisplayCuts(IntEbm countSamples,
       double minFeatureVal,
       double maxFeatureVal);
 
+std::vector<TestSample> MakeRandomDataset(std::vector<unsigned char>& rng,
+      const IntEbm cClasses,
+      const size_t cSamples,
+      const std::vector<FeatureTest>& features);
+
+std::vector<std::vector<IntEbm>> MakeMains(const std::vector<FeatureTest>& features);
+
+inline static std::vector<unsigned char> MakeRng(const SeedEbm seed) {
+   std::vector<unsigned char> rng(static_cast<size_t>(MeasureRNG()));
+   InitRNG(seed, &rng[0]);
+   return rng;
+}
+
+inline IntEbm TestRand(std::vector<unsigned char>& rng, const IntEbm count) {
+   // this isn't balanced, but good enough for tests
+   SeedEbm randomNum;
+   GenerateSeed(&rng[0], &randomNum);
+   return static_cast<IntEbm>(static_cast<USeedEbm>(randomNum) % static_cast<USeedEbm>(count));
+}
+
+inline double TestRand(std::vector<unsigned char>& rng) {
+   double ret;
+   GenerateGaussianRandom(&rng[0], 100.0, 1, &ret);
+   return ret;
+}
+
 #endif // LIBEBM_TEST_HPP