Integrate multinomial logistic regression.

nums/models/glms.py

@@ -33,7 +33,7 @@
 #
 # The link function is expressed as follows.
 # E(Y | X) = mu
-# Define the linear predictor eta = X.T @ beta
+# Define the linear predictor eta = X @ beta
 # Define g as the link function, so that g(mu) = eta
 # E(Y | X) = g^{-1}(eta)
 #
@@ -62,7 +62,35 @@
 #   g(mu) = (b')^{-1}(mu) = ln(mu/(1-mu)) = ln(p/(1-p)) = theta(p)
 
 
-class GLM(object):
+class Model(object):
+    def forward(self, X, beta=None):
+        raise NotImplementedError()
+
+    def objective(
+        self,
+        X: BlockArray,
+        y: BlockArray,
+        beta: BlockArray = None,
+        mu: BlockArray = None,
+    ):
+        raise NotImplementedError()
+
+    def gradient(
+        self,
+        X: BlockArray,
+        y: BlockArray,
+        mu: BlockArray = None,
+        beta: BlockArray = None,
+    ):
+        # gradient w.r.t. beta.
+        raise NotImplementedError()
+
+    def hessian(self, X: BlockArray, y: BlockArray, mu: BlockArray = None):
+        # Hessian w.r.t. beta.
+        raise NotImplementedError()
+
+
+class GLM(Model):
     def __init__(
         self,
         penalty="none",
@@ -207,29 +235,6 @@ def predict(self, X):
     def link_inv(self, eta: BlockArray):
         raise NotImplementedError()
 
-    def objective(
-        self,
-        X: BlockArray,
-        y: BlockArray,
-        beta: BlockArray = None,
-        mu: BlockArray = None,
-    ):
-        raise NotImplementedError()
-
-    def gradient(
-        self,
-        X: BlockArray,
-        y: BlockArray,
-        mu: BlockArray = None,
-        beta: BlockArray = None,
-    ):
-        # gradient w.r.t. beta.
-        raise NotImplementedError()
-
-    def hessian(self, X: BlockArray, y: BlockArray, mu: BlockArray = None):
-        # Hessian w.r.t. beta.
-        raise NotImplementedError()
-
     def deviance(self, y, y_pred):
         raise NotImplementedError()
 

nums/models/lbfgs.py

@@ -19,14 +19,14 @@
 import numpy as np
 from nums.core.array.application import ArrayApplication
 from nums.core.application_manager import instance as _instance
-from nums.models.glms import GLM
+from nums.models.glms import Model
 
 
 # Based on Nocedal and Wright, chapters 2, 3, 6 and 7.
 
 
 class BackTrackingLineSearch(object):
-    def __init__(self, model: GLM):
+    def __init__(self, model: Model):
         self.app = _instance()
         self.model = model
 
@@ -64,9 +64,9 @@ def __init__(self, k, s, y):
 
 
 class LBFGS(object):
-    def __init__(self, model: GLM, m=10, max_iter=100, thresh=1e-4, dtype=np.float64):
+    def __init__(self, model: Model, m=10, max_iter=100, thresh=1e-4, dtype=np.float64):
         self.app: ArrayApplication = _instance()
-        self.model: GLM = model
+        self.model: Model = model
         self.m = m
         self.max_iter = max_iter
         self.thresh = thresh