Add logistic regression algorithm

docs/_sidebar.md

@@ -18,6 +18,7 @@
 
     + [Linear](apis/linear/index.md)
         + [Linear Regression](apis/linear/linearRegression.md)
+        + [Logistic Regression](apis/linear/logisticRegression.md)
 
     + [Decomposition](apis/decomposition.md)
         + [PCA](apis/decomposition/pca.md)

docs/apis/linear/index.md

@@ -1,3 +1,4 @@
 # Linear
 
 - [LinearRegression](linearRegression.md)
+- [LogisticRegression](logisticRegression.md)

docs/apis/linear/logisticRegression.md

@@ -0,0 +1,16 @@
+## LogisticRegression
+
+```ts
+interface LogisticRegressionProps {
+    learningRate?: number;
+    maxIter?: number;
+}
+constructor(props: LogisticRegressionProps = {})
+```
+
+### Example
+```ts
+const clf = new LogisticRegression();
+clf.fit(trainX, trainY);
+const result = clf.predict(testX);
+```

scripts/gen_all.py

@@ -15,6 +15,7 @@ def run(script):
     'gen_mean_shift.py',
     'gen_dbscan.py',
     'gen_optics.py',
+    'gen_logistic_regression.py',
     'gen_svc.py',
     'gen_pca.py'
 ]

scripts/gen_logistic_regression.py

@@ -0,0 +1,20 @@
+from sklearn.datasets import make_classification
+from sklearn.linear_model import LogisticRegression
+import json, os
+
+X, y = make_classification(n_samples=60, n_features=5, n_informative=3,
+                           n_classes=2, random_state=0)
+clf = LogisticRegression(max_iter=1000)
+clf.fit(X, y)
+X_test, _ = make_classification(n_samples=10, n_features=5, n_informative=3,
+                                n_classes=2, random_state=1)
+pred = clf.predict(X_test)
+
+os.makedirs('test_data', exist_ok=True)
+with open('test_data/logistic_regression.json', 'w') as f:
+    json.dump({
+        'trainX': X.tolist(),
+        'trainY': y.tolist(),
+        'testX': X_test.tolist(),
+        'expected': pred.tolist()
+    }, f)

src/linear/__test__/logisticRegression.compare.test.ts

@@ -0,0 +1,12 @@
+import { LogisticRegression } from '../logisticRegression';
+import fs from 'fs';
+import path from 'path';
+
+test('compare with sklearn', () => {
+    const p = path.join(__dirname, '../../../test_data/logistic_regression.json');
+    const data = JSON.parse(fs.readFileSync(p, 'utf8'));
+    const lr = new LogisticRegression({ learningRate: 0.1, maxIter: 1000 });
+    lr.fit(data.trainX, data.trainY);
+    const pred = lr.predict(data.testX);
+    expect(pred).toEqual(data.expected);
+});

src/linear/__test__/logisticRegression.test.ts

@@ -0,0 +1,15 @@
+import { LogisticRegression } from '../logisticRegression';
+
+test('init', () => {
+    const lr = new LogisticRegression();
+    expect(lr).toBeDefined();
+});
+
+test('simple classification', () => {
+    const X = [[0], [1], [2], [3]];
+    const Y = [0, 0, 1, 1];
+    const lr = new LogisticRegression({ learningRate: 0.5, maxIter: 200 });
+    lr.fit(X, Y);
+    const pred = lr.predict(X);
+    expect(pred).toEqual(Y);
+});

src/linear/index.ts

@@ -1 +1,2 @@
 export { LinearRegression } from './linearRegression';
+export { LogisticRegression } from './logisticRegression';

src/linear/logisticRegression.ts

@@ -0,0 +1,67 @@
+import { ClassifierBase } from '../base';
+
+function sigmoid(z: number): number {
+    return 1 / (1 + Math.exp(-z));
+}
+
+export interface LogisticRegressionProps {
+    learningRate?: number;
+    maxIter?: number;
+}
+
+export class LogisticRegression extends ClassifierBase {
+    private weights: number[];
+    private bias: number;
+    private learningRate: number;
+    private maxIter: number;
+
+    constructor(props: LogisticRegressionProps = {}) {
+        super();
+        const { learningRate = 0.1, maxIter = 1000 } = props;
+        this.learningRate = learningRate;
+        this.maxIter = maxIter;
+        this.weights = [];
+        this.bias = 0;
+    }
+
+    public fit(trainX: number[][], trainY: number[]): void {
+        const nFeatures = trainX[0].length;
+        this.weights = new Array(nFeatures).fill(0);
+        this.bias = 0;
+        for (let iter = 0; iter < this.maxIter; iter++) {
+            const gradW = new Array(nFeatures).fill(0);
+            let gradB = 0;
+            for (let i = 0; i < trainX.length; i++) {
+                const x = trainX[i];
+                const y = trainY[i];
+                let z = this.bias;
+                for (let j = 0; j < nFeatures; j++) {
+                    z += this.weights[j] * x[j];
+                }
+                const pred = sigmoid(z);
+                const diff = pred - y;
+                for (let j = 0; j < nFeatures; j++) {
+                    gradW[j] += diff * x[j];
+                }
+                gradB += diff;
+            }
+            for (let j = 0; j < nFeatures; j++) {
+                this.weights[j] -= this.learningRate * gradW[j] / trainX.length;
+            }
+            this.bias -= this.learningRate * gradB / trainX.length;
+        }
+    }
+
+    public predict(testX: number[][]): number[] {
+        const results: number[] = [];
+        for (const x of testX) {
+            let z = this.bias;
+            for (let j = 0; j < x.length; j++) {
+                z += this.weights[j] * x[j];
+            }
+            const p = sigmoid(z);
+            results.push(p >= 0.5 ? 1 : 0);
+        }
+        return results;
+    }
+}