feat: add bagging and random forest ensembles

Author: ObservedObserver

.gitignore

@@ -116,9 +116,12 @@ test_data/*
 !test_data/complement_nb.json
 !test_data/elastic_net.json
 !test_data/gaussian_nb.json
+!test_data/bagging_classifier.json
 !test_data/kneighbors_regressor.json
 !test_data/multinomial_nb.json
 !test_data/nearest_centroid.json
+!test_data/random_forest_classifier.json
+!test_data/random_forest_regressor.json
 !test_data/radius_neighbors_classifier.json
 !test_data/radius_neighbors_regressor.json
 !test_data/ridge_classifier.json

scripts/gen_all.py

@@ -31,10 +31,13 @@ def run(script):
     'gen_multinomial_nb.py',
     'gen_complement_nb.py',
     'gen_svc.py',
+    'gen_bagging_classifier.py',
     'gen_kneighbors_regressor.py',
     'gen_radius_neighbors_classifier.py',
     'gen_radius_neighbors_regressor.py',
     'gen_nearest_centroid.py',
+    'gen_random_forest_classifier.py',
+    'gen_random_forest_regressor.py',
     'gen_bernoulli_rbm.py',
     'gen_pca.py',
     'gen_truncated_svd.py',

scripts/gen_bagging_classifier.py

@@ -0,0 +1,34 @@
+from sklearn.datasets import make_classification
+from sklearn.ensemble import BaggingClassifier
+from sklearn.tree import DecisionTreeClassifier
+import json, os
+
+X, y = make_classification(
+    n_samples=120,
+    n_features=5,
+    n_informative=4,
+    n_redundant=0,
+    n_clusters_per_class=1,
+    class_sep=1.5,
+    random_state=0,
+)
+trainX = X[:90]
+trainY = y[:90]
+testX = X[90:]
+
+base = DecisionTreeClassifier(criterion='gini', random_state=0)
+try:
+    clf = BaggingClassifier(estimator=base, n_estimators=15, random_state=0)
+except TypeError:
+    clf = BaggingClassifier(base_estimator=base, n_estimators=15, random_state=0)
+clf.fit(trainX, trainY)
+pred = clf.predict(testX)
+
+os.makedirs('test_data', exist_ok=True)
+with open('test_data/bagging_classifier.json', 'w') as f:
+    json.dump({
+        'trainX': trainX.tolist(),
+        'trainY': trainY.tolist(),
+        'testX': testX.tolist(),
+        'expected': pred.tolist()
+    }, f)

scripts/gen_random_forest_classifier.py

@@ -0,0 +1,29 @@
+from sklearn.datasets import make_classification
+from sklearn.ensemble import RandomForestClassifier
+import json, os
+
+X, y = make_classification(
+    n_samples=120,
+    n_features=5,
+    n_informative=4,
+    n_redundant=0,
+    n_clusters_per_class=1,
+    class_sep=1.5,
+    random_state=0,
+)
+trainX = X[:90]
+trainY = y[:90]
+testX = X[90:]
+
+clf = RandomForestClassifier(n_estimators=25, random_state=0, max_features='sqrt')
+clf.fit(trainX, trainY)
+pred = clf.predict(testX)
+
+os.makedirs('test_data', exist_ok=True)
+with open('test_data/random_forest_classifier.json', 'w') as f:
+    json.dump({
+        'trainX': trainX.tolist(),
+        'trainY': trainY.tolist(),
+        'testX': testX.tolist(),
+        'expected': pred.tolist()
+    }, f)

scripts/gen_random_forest_regressor.py

@@ -0,0 +1,23 @@
+from sklearn.ensemble import RandomForestRegressor
+import numpy as np
+import json, os
+
+rng = np.random.RandomState(0)
+X = rng.uniform(-3, 3, size=(120, 1))
+y = 4 * X[:, 0] ** 2 - 2 * X[:, 0] + 1
+trainX = X[:90]
+trainY = y[:90]
+testX = X[90:]
+
+reg = RandomForestRegressor(n_estimators=25, random_state=0, max_features=1.0)
+reg.fit(trainX, trainY)
+pred = reg.predict(testX)
+
+os.makedirs('test_data', exist_ok=True)
+with open('test_data/random_forest_regressor.json', 'w') as f:
+    json.dump({
+        'trainX': trainX.tolist(),
+        'trainY': trainY.tolist(),
+        'testX': testX.tolist(),
+        'expected': pred.tolist()
+    }, f)

src/ensemble/__test__/baggingClassifier.compare.test.ts

@@ -0,0 +1,18 @@
+import fs from 'fs';
+import path from 'path';
+import { BaggingClassifier } from '../baggingClassifier';
+
+test('BaggingClassifier compare with sklearn', () => {
+    const p = path.join(__dirname, '../../../test_data/bagging_classifier.json');
+    const data = JSON.parse(fs.readFileSync(p, 'utf8'));
+    const clf = new BaggingClassifier({ nEstimators: 15, randomState: 0, criterion: 'gini' });
+    clf.fit(data.trainX, data.trainY);
+    const pred = clf.predict(data.testX);
+    let correct = 0;
+    for (let i = 0; i < pred.length; i++) {
+        if (pred[i] === data.expected[i]) {
+            correct++;
+        }
+    }
+    expect(correct / pred.length).toBeGreaterThanOrEqual(0.8);
+});

src/ensemble/__test__/baggingClassifier.test.ts

@@ -0,0 +1,20 @@
+import { BaggingClassifier } from '../baggingClassifier';
+
+test('BaggingClassifier initializes', () => {
+    expect(new BaggingClassifier()).toBeDefined();
+});
+
+test('BaggingClassifier fits and predicts a separable dataset', () => {
+    const X = [[0], [1], [2], [10], [11], [12]];
+    const y = [0, 0, 0, 1, 1, 1];
+
+    const clf = new BaggingClassifier({ nEstimators: 7, randomState: 42 });
+    clf.fit(X, y);
+
+    expect(clf.predict([[0.2], [11.5]])).toEqual([0, 1]);
+});
+
+test('BaggingClassifier validates fit lifecycle', () => {
+    const clf = new BaggingClassifier();
+    expect(() => clf.predict([[0]])).toThrow('model is not fitted');
+});

src/ensemble/__test__/randomForestClassifier.compare.test.ts

@@ -0,0 +1,18 @@
+import fs from 'fs';
+import path from 'path';
+import { RandomForestClassifier } from '../randomForestClassifier';
+
+test('RandomForestClassifier compare with sklearn', () => {
+    const p = path.join(__dirname, '../../../test_data/random_forest_classifier.json');
+    const data = JSON.parse(fs.readFileSync(p, 'utf8'));
+    const clf = new RandomForestClassifier({ nEstimators: 25, randomState: 0, maxFeatures: 'sqrt' });
+    clf.fit(data.trainX, data.trainY);
+    const pred = clf.predict(data.testX);
+    let correct = 0;
+    for (let i = 0; i < pred.length; i++) {
+        if (pred[i] === data.expected[i]) {
+            correct++;
+        }
+    }
+    expect(correct / pred.length).toBeGreaterThanOrEqual(0.8);
+});

src/ensemble/__test__/randomForestClassifier.test.ts

@@ -0,0 +1,15 @@
+import { RandomForestClassifier } from '../randomForestClassifier';
+
+test('RandomForestClassifier initializes', () => {
+    expect(new RandomForestClassifier()).toBeDefined();
+});
+
+test('RandomForestClassifier fits and predicts a separable dataset', () => {
+    const X = [[0], [1], [2], [10], [11], [12]];
+    const y = [0, 0, 0, 1, 1, 1];
+
+    const clf = new RandomForestClassifier({ nEstimators: 15, randomState: 42 });
+    clf.fit(X, y);
+
+    expect(clf.predict([[0.2], [11.5]])).toEqual([0, 1]);
+});

src/ensemble/__test__/randomForestRegressor.compare.test.ts

@@ -0,0 +1,17 @@
+import fs from 'fs';
+import path from 'path';
+import { RandomForestRegressor } from '../randomForestRegressor';
+
+test('RandomForestRegressor compare with sklearn', () => {
+    const p = path.join(__dirname, '../../../test_data/random_forest_regressor.json');
+    const data = JSON.parse(fs.readFileSync(p, 'utf8'));
+    const reg = new RandomForestRegressor({ nEstimators: 25, randomState: 0, maxFeatures: 1 });
+    reg.fit(data.trainX, data.trainY);
+    const pred = reg.predict(data.testX);
+    let mse = 0;
+    for (let i = 0; i < pred.length; i++) {
+        mse += (pred[i] - data.expected[i]) ** 2;
+    }
+    mse /= pred.length;
+    expect(mse).toBeLessThan(5000);
+});