[Feature]: Meta OrdinalClassifier estimator (#611)

* implementation of main methods

* docstrings

* demo notebook

* tests,docs,api change

* unit tests

* docs

* change calibration api

* feedback adjustments

Author: FBruzzesi

docs/_scripts/meta-models.py

@@ -67,9 +67,9 @@
 plt.clf()
 
 # --8<-- [start:cross-validation-no-refit]
-# %%time 
+# %%time
 
-# Train an original model 
+# Train an original model
 orig_model = LogisticRegression(solver="lbfgs")
 orig_model.fit(X, y)
 
@@ -111,7 +111,7 @@
 
 def plot_model(model):
     df = load_chicken(as_frame=True)
-    
+
     _ = model.fit(df[["diet", "time"]], df["weight"])
     metric_df = (df[["diet", "time", "weight"]]
         .assign(pred=lambda d: model.predict(d[["diet", "time"]]))
@@ -280,7 +280,7 @@ def plot_model(model):
 
 
 # --8<-- [start:decay-functions]
-from sklego.meta._decay_utils import exponential_decay, linear_decay, sigmoid_decay, stepwise_decay 
+from sklego.meta._decay_utils import exponential_decay, linear_decay, sigmoid_decay, stepwise_decay
 
 fig = plt.figure(figsize=(12, 6))
 
@@ -312,13 +312,13 @@ def plot_model(model):
 np.random.seed(42)
 
 n1, n2, n3 = 100, 500, 50
-X = np.concatenate([np.random.normal(0, 1, (n1, 2)), 
+X = np.concatenate([np.random.normal(0, 1, (n1, 2)),
                     np.random.normal(2, 1, (n2, 2)),
-                    np.random.normal(3, 1, (n3, 2))], 
+                    np.random.normal(3, 1, (n3, 2))],
                    axis=0)
-y = np.concatenate([np.zeros((n1, 1)), 
+y = np.concatenate([np.zeros((n1, 1)),
                     np.ones((n2, 1)),
-                    np.zeros((n3, 1))], 
+                    np.zeros((n3, 1))],
                    axis=0).reshape(-1)
 plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap);
 # --8<-- [end:make-blobs]
@@ -360,7 +360,7 @@ def false_negatives(mod, x, y):
 cf_mod = ConfusionBalancer(LogisticRegression(solver="lbfgs", max_iter=1000), alpha=1.0)
 
 grid = GridSearchCV(
-    cf_mod, 
+    cf_mod,
     param_grid={"alpha": np.linspace(-1.0, 3.0, 31)},
     scoring={
         "accuracy": make_scorer(accuracy_score),
@@ -464,4 +464,49 @@ def false_negatives(mod, x, y):
 
 from sklearn.utils import estimator_html_repr
 with open(_static_path / "outlier-classifier-stacking.html", "w") as f:
-    f.write(estimator_html_repr(stacker))
+    f.write(estimator_html_repr(stacker))
+
+# --8<-- [start:ordinal-classifier-data]
+import pandas as pd
+
+url = "https://stats.idre.ucla.edu/stat/data/ologit.dta"
+df = pd.read_stata(url).assign(apply_codes = lambda t: t["apply"].cat.codes)
+
+target = "apply_codes"
+features = [c for c in df.columns if c not in {target, "apply"}]
+
+X, y = df[features].to_numpy(), df[target].to_numpy()
+df.head()
+# --8<-- [end:ordinal-classifier-data]
+
+with open(_static_path / "ordinal_data.md", "w") as f:
+    f.write(df.head().to_markdown(index=False))
+
+# --8<-- [start:ordinal-classifier]
+from sklearn.linear_model import LogisticRegression
+from sklego.meta import OrdinalClassifier
+
+ord_clf = OrdinalClassifier(LogisticRegression(), n_jobs=-1, use_calibration=False)
+_ = ord_clf.fit(X, y)
+ord_clf.predict_proba(X[0])
+# --8<-- [end:ordinal-classifier]
+
+print(ord_clf.predict_proba(X[0]))
+
+# --8<-- [start:ordinal-classifier-with-calibration]
+from sklearn.calibration import CalibratedClassifierCV
+from sklearn.linear_model import LogisticRegression
+from sklego.meta import OrdinalClassifier
+
+calibration_kwargs = {...}
+
+ord_clf = OrdinalClassifier(
+    estimator=LogisticRegression(),
+    use_calibration=True,
+    calibration_kwargs=calibration_kwargs
+)
+
+# This is equivalent to:
+estimator = CalibratedClassifierCV(LogisticRegression(), **calibration_kwargs)
+ord_clf = OrdinalClassifier(estimator)
+# --8<-- [end:ordinal-classifier-with-calibration]

docs/_static/meta-models/ordinal-classification.png

@@ -0,0 +1,7 @@
+| apply           |   pared |   public |   gpa |   apply_codes |
+|:----------------|--------:|---------:|------:|--------------:|
+| very likely     |       0 |        0 |  3.26 |             2 |
+| somewhat likely |       1 |        0 |  3.21 |             1 |
+| unlikely        |       1 |        1 |  3.94 |             0 |
+| somewhat likely |       0 |        0 |  2.81 |             1 |
+| somewhat likely |       0 |        0 |  2.53 |             1 |

docs/_static/meta-models/ordinal_data.md

@@ -25,6 +25,11 @@
         show_root_full_path: true
         show_root_heading: true
 
+::: sklego.meta.ordinal_classification.OrdinalClassifier
+    options:
+        show_root_full_path: true
+        show_root_heading: true
+
 ::: sklego.meta.outlier_classifier.OutlierClassifier
     options:
         show_root_full_path: true

docs/api/meta.md

@@ -100,7 +100,6 @@ The image below demonstrates what will happen.
 
 ![grouped](../_static/meta-models/grouped-df.png)
 
-
 We train 5 models in total because the model will also train a fallback automatically (you can turn this off via `use_fallback=False`).
 
 The idea behind the fallback is that we can predict something if there is a group at prediction time which is unseen during training.
@@ -291,6 +290,7 @@ We'll perform an optimistic demonstration below.
     ```py
     --8<-- "docs/_scripts/meta-models.py:confusion-balancer-results"
     ```
+
 It seems that we can pick a value for $\alpha$ such that the confusion matrix is balanced. there's also a modest increase in accuracy for this balancing moment.
 
 It should be emphasized though that this feature is **experimental**. There have been dataset/model combinations where this effect seems to work very well while there have also been situations where this trick does not work at all.
@@ -327,7 +327,7 @@ ZIR (RFC+RFR) r²: 0.8992404366385873
 RFR r²: 0.8516522752031502
 ```
 
-## OutlierClassifier
+## Outlier Classifier
 
 Outlier models are unsupervised so they don't have `predict_proba` or `score` methods.
 
@@ -381,6 +381,66 @@ The `OutlierClassifier` can be combined with any classification model in the `St
 
 --8<-- "docs/_static/meta-models/outlier-classifier-stacking.html"
 
+## Ordinal Classification
+
+Ordinal classification (sometimes also referred to as Ordinal Regression) involves predicting an ordinal target variable, where the classes have a meaningful order.
+Examples of this kind of problem are: predicting customer satisfaction on a scale from 1 to 5, predicting the severity of a disease, predicting the quality of a product, etc.
+
+The [`OrdinalClassifier`][ordinal-classifier-api] is a meta-model that can be used to transform any classifier into an ordinal classifier by fitting N-1 binary classifiers, each handling a specific class boundary, namely: $P(y <= 1), P(y <= 2), ..., P(y <= N-1)$.
+
+This implementation is based on the paper [A simple approach to ordinal classification][ordinal-classification-paper] and it allows to predict the ordinal probabilities of each sample belonging to a particular class.
+
+??? tip "Graphical representation"
+    An image (from the paper itself) is worth a thousand words:
+    ![ordinal-classification](../_static/meta-models/ordinal-classification.png)
+
+!!! note "mord library"
+    If you are looking for a library that implements other ordinal classification algorithms, you can have a look at the [mord][mord] library.
+
+```py title="Ordinal Data"
+--8<-- "docs/_scripts/meta-models.py:ordinal-classifier-data"
+```
+
+--8<-- "docs/_static/meta-models/ordinal_data.md"
+
+Description of the dataset from [statsmodels tutorial][statsmodels-ordinal-regression]:
+
+> This dataset is about the probability for undergraduate students to apply to graduate school given three exogenous variables:
+>
+> - their grade point average (`gpa`), a float between 0 and 4.
+> - `pared`, a binary that indicates if at least one parent went to graduate school.
+> - `public`, a binary that indicates if the current undergraduate institution of the student is > public or private.
+>
+> `apply`, the target variable is categorical with ordered categories: "unlikely" < "somewhat likely" < "very likely".
+>
+> [...]
+>
+> For more details see the the Documentation of OrderedModel, [the UCLA webpage][ucla-webpage].
+
+The only transformation we are applying to the data is to convert the target variable to an ordinal categorical variable by mapping the ordered categories to integers using their (pandas) category codes.
+
+We are now ready to train a [`OrdinalClassifier`][ordinal-classifier-api] on this dataset:
+
+```py title="OrdinalClassifier"
+--8<-- "docs/_scripts/meta-models.py:ordinal-classifier"
+```
+
+> [[0.54883853 0.36225347 0.088908]]
+
+### Probability Calibration
+
+The `OrdinalClassifier` emphasizes the importance of proper probability estimates for its functionality. It is recommended to use the [`CalibratedClassifierCV`][calibrated-classifier-api] class from scikit-learn to calibrate the probabilities of the binary classifiers.
+
+Probability calibration is _not_ enabled by default, but we provide a convenient keyword argument `use_calibration` to enable it as follows:
+
+```py title="OrdinalClassifier with probability calibration"
+--8<-- "docs/_scripts/meta-models.py:ordinal-classifier-with-calibration"
+```
+
+### Computation Time
+
+As a meta-estimator, the `OrdinalClassifier` fits N-1 binary classifiers, which may be computationally expensive, especially with a large number of samples, features, or a complex classifier.
+
 [thresholder-api]: ../../api/meta#sklego.meta.thresholder.Thresholder
 [grouped-predictor-api]: ../../api/meta#sklego.meta.grouped_predictor.GroupedPredictor
 [grouped-transformer-api]: ../../api/meta#sklego.meta.grouped_transformer.GroupedTransformer
@@ -389,8 +449,14 @@ The `OutlierClassifier` can be combined with any classification model in the `St
 [confusion-balancer-api]: ../../api/meta#sklego.meta.confusion_balancer.ConfusionBalancer
 [zero-inflated-api]: ../../api/meta#sklego.meta.zero_inflated_regressor.ZeroInflatedRegressor
 [outlier-classifier-api]: ../../api/meta#sklego.meta.outlier_classifier.OutlierClassifier
+[ordinal-classifier-api]: ../../api/meta#sklego.meta.ordinal_classification.OrdinalClassifier
 
 [standard-scaler-api]: https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.StandardScaler.html
 [stacking-classifier-api]: https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.StackingClassifier.html#sklearn.ensemble.StackingClassifier
 [dummy-regressor-api]: https://scikit-learn.org/stable/modules/generated/sklearn.dummy.DummyRegressor.html
 [imb-learn]: https://imbalanced-learn.org/stable/
+[ordinal-classification-paper]: https://www.cs.waikato.ac.nz/~eibe/pubs/ordinal_tech_report.pdf
+[mord]: https://pythonhosted.org/mord/
+[statsmodels-ordinal-regression]: https://www.statsmodels.org/dev/examples/notebooks/generated/ordinal_regression.html
+[ucla-webpage]: https://stats.oarc.ucla.edu/r/dae/ordinal-logistic-regression/
+[calibrated-classifier-api]: https://scikit-learn.org/stable/modules/generated/sklearn.calibration.CalibratedClassifierCV.html

docs/user-guide/meta-models.md

@@ -95,6 +95,7 @@ sklego = ["data/*.zip"]
 [tool.ruff]
 line-length = 120
 extend-select = ["I"]
+exclude = ["docs"]
 
 [tool.pytest.ini_options]
 markers = [

pyproject.toml

@@ -5,6 +5,7 @@
     "GroupedEstimator",
     "GroupedPredictor",
     "GroupedTransformer",
+    "OrdinalClassifier",
     "OutlierRemover",
     "SubjectiveClassifier",
     "Thresholder",
@@ -19,6 +20,7 @@
 from sklego.meta.grouped_estimator import GroupedEstimator
 from sklego.meta.grouped_predictor import GroupedPredictor
 from sklego.meta.grouped_transformer import GroupedTransformer
+from sklego.meta.ordinal_classification import OrdinalClassifier
 from sklego.meta.outlier_classifier import OutlierClassifier
 from sklego.meta.outlier_remover import OutlierRemover
 from sklego.meta.regression_outlier_detector import RegressionOutlierDetector