[DOC] TabICL tabular foundation model

CHANGELOG.rst

@@ -26,5 +26,6 @@ under development (0.3.2.dev0)
 - :bdg-danger:`Fix` Fixed unnecessary copy operations of X when only a slice view is needed (:gh:`646` by `Marc Hulcelle`_).
 - :bdg-secondary:`Maint` remove pinned poosh dependency (problem with somato dataset solved) (:gh:`670` by `Joseph Paillard`_).
 - :bdg-secondary:`Maint` remove extra term in variance of X-residual (DOCRT). See  [Reid et al., A Study of Error Variance Estimation in Lasso Regression 2016](https://arxiv.org/pdf/1311.5274) for reference. (:gh:`649` by `Joseph Paillard`_).
+- :bdg-success:`Feature` add leave-one-covariate-in (LOCI) method (:gh:`679` by `Marc Hulcelle`).
 - :bdg-danger:`Fix` fixed deprecated n_alphas with sklearn LassoCV, as well as deprecated penalty for LogisticRegressionCV (:gh:`690` by `Marc Hulcelle`)
 - :bdg-secondary:`Maint`: Fix conditional sampling test by verifying that sampler produces diverse samples. (:gh:`692` by `Joseph Paillard`_).

docs/tools/references.bib

@@ -483,3 +483,10 @@ @article{zhang2014confidence
   volume    = {76},
   year      = {2014}
 }
+
+@inproceedings{qu2025tabicl,
+  title={Tab{ICL}: {A} Tabular Foundation Model for In-Context Learning on Large Data},
+  author={Qu, Jingang and Holzm{\"u}ller, David and Varoquaux, Ga{\"e}l and Le Morvan, Marine},
+  booktitle={International Conference on Machine Learning},
+  year={2025}
+}

examples/plot_advanced_tabicl.py

@@ -0,0 +1,87 @@
+"""
+Tabular Foundation Model TabICL
+================================
+
+In this example, we demonstrate how to use a tabular foundation model such as
+TabICL [:footcite:t:`qu2025tabicl`] with a straightforward example on the California
+Housing regression dataset.
+"""
+
+# %%
+# Loading data and TabICL
+# -----------------------
+# We start by loading data from the California Housing dataset and fitting the
+# TabICL model. If this is the first use, the model will be downloaded and cached
+# for future use. The main advantage is that the model does not require
+# hyperparameter tuning as a pre-trained transformer model. We recommend switching
+# to GPU through the adequate class parameter for datasets that go over 10k
+# samples depending on the number of features due to computation time increase.
+
+from sklearn.datasets import fetch_california_housing
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import StandardScaler
+from sklearn.utils import resample
+from tabicl import TabICLRegressor
+
+dataset = fetch_california_housing()
+X, y, feat_names = dataset.data, dataset.target, dataset.feature_names
+X, y = resample(
+    X, y, replace=False, n_samples=1000, stratify=y, random_state=0
+)
+X_train, X_test, y_train, y_test = train_test_split(
+    X, y, test_size=0.2, random_state=0
+)
+
+model = make_pipeline(StandardScaler(), TabICLRegressor(device="cpu"))
+model.fit(X_train, y_train)
+print(f"Model accuracy on the test data {model.score(X_test, y_test):.3}.")
+
+# %%
+# For the moment, TabICL can only be used with PFI or CFI. TabICL only works with
+# integer seeding, which is currently not supported by D0CRT.
+#
+# TabICL interfaces with HiDimStat the same way as any other Scikit-learn estimator,
+# which lets us compute feature importance as easily as follows:
+
+import pandas as pd
+from sklearn.metrics import mean_squared_error
+
+from hidimstat import CFI
+
+cfi = CFI(estimator=model, method="predict", loss=mean_squared_error)
+cfi.fit(X_train, y_train)
+importance = cfi.importance(X_test, y_test)
+selection = cfi.fdr_selection(fdr=0.1)
+
+# %%
+# Let's wrap the results in a Pandas dataframe and plot the importance
+# and the selected variables.
+
+df = pd.DataFrame(
+    {
+        "feature": feat_names,
+        "importance": importance,
+        "selected": selection,
+    }
+).sort_values("importance", ascending=False)
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+ax = sns.barplot(
+    data=df,
+    y="feature",
+    x="importance",
+    hue="selected",
+    palette="muted",
+    orient="h",
+)
+sns.despine()
+plt.show()
+
+# %%
+# As you can see from the timer under, even with 1000 samples and 9 features,
+# running TabICL on the CPU is slow. We recommend to use it on larger datasets,
+# be it in terms of samples and/or features and to run it on GPU for faster
+# inferences.

examples/plot_loci.py

@@ -0,0 +1,162 @@
+"""
+Leave-One-Covariate-In (LOCI) feature importance with different regression models
+==================================================================================
+
+This example demonstrates how to compare LOCI feature importance [:footcite:t:`Williamson_General_2023`] across different
+predictive models on the same regression dataset. LOCI is model-agnostic and can be
+applied to any predictive model. Here, we use a linear model, a random forest, a neural
+network, and a support vector machine. We compare the models based on their predictive
+performance (R2 score) and the LOCI feature importance they yield.
+"""
+
+# %%
+# Loading and preparing the data
+# ------------------------------
+# We begin by simulating a regression dataset with 10 features, 5 of which
+# are in the support set, meaning they contribute to generating the outcome. In this example,
+# we use a simulated dataset to have access to the true support set of features and
+# evaluate how well the different models identify these important features.
+# The data is then split into training and test sets. These sets are used both to fit
+# the predictive models and within the LOCI procedure, which refits models on subsets
+# of features that exclude the feature of interest.
+
+from sklearn.datasets import make_regression
+from sklearn.model_selection import train_test_split
+
+X, y, beta = make_regression(
+    n_samples=300,
+    n_features=10,
+    n_informative=5,
+    random_state=0,
+    coef=True,
+    noise=4.0,
+)
+
+# We convert the coefficients of the data-generating process into a binary array
+# indicating the true support set of features.
+beta = beta != 0
+
+X_train, X_test, y_train, y_test = train_test_split(
+    X,
+    y,
+    test_size=0.2,
+    random_state=0,
+    shuffle=True,
+)
+
+# %%
+# Fitting models and computing LOCI feature importance
+# ----------------------------------------------------
+# We define a list of predictive models to compare. We use RidgeCV for linear
+# regression, RandomForestRegressor for a tree-based model, MLPRegressor for a
+# neural network, and SVR for a support vector machine, with RBF kernel. We then fit
+# each model on the training data, compute the LOCI feature importance on the test
+# data, and store the results in a DataFrame for comparison.
+# Beware on the choice of hyperparameters of certain models to avoid over-fitting.
+
+import pandas as pd
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.linear_model import RidgeCV
+from sklearn.metrics import mean_squared_error
+from sklearn.neural_network import MLPRegressor
+from sklearn.svm import SVR
+
+from hidimstat import LOCI
+
+models_list = [
+    RidgeCV(),
+    MLPRegressor(
+        hidden_layer_sizes=(8),
+        random_state=0,
+        max_iter=500,
+        learning_rate_init=0.1,
+    ),
+    SVR(kernel="linear"),
+    RandomForestRegressor(
+        n_estimators=50, max_depth=3, min_samples_leaf=10, random_state=0
+    ),
+]
+
+df_list = []
+for model in models_list:
+    # Fit the full model
+    model = model.fit(X_train, y_train)
+    loci = LOCI(model, method="predict", loss=mean_squared_error)
+    # Refit the model on a single feature / group of feature, and compute LOCI
+    # importance. This process is repeated for all features / groups of features to assess
+    # their individual contributions.
+    loci.fit(X_train, y_train)
+    importances = loci.importance(X_test, y_test)
+    df_list.append(
+        pd.DataFrame(
+            {
+                "feature": list(range(X.shape[1])),
+                "importance": importances,
+                "model": model.__class__.__name__,
+                "R2 score": model.score(X_test, y_test),
+            }
+        )
+    )
+
+
+# %%
+# The predictive performance of the models can be compared using their R2 scores.
+# This helps assess how effectively each model captures the underlying data-generating
+# process.
+
+df_plot = pd.concat(df_list)
+df_plot.groupby("model").mean()["R2 score"].to_frame()
+
+# %%
+# Visualization of LOCI feature importance
+# ----------------------------------------
+# Finally, we visualize the LOCI feature importance for each model using a horizontal
+# bar plot. The true support features are highlighted in the plot with a green shaded
+# background.
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+ax = sns.barplot(
+    data=df_plot,
+    y="feature",
+    x="importance",
+    hue="model",
+    palette="muted",
+    orient="h",
+)
+sns.despine()
+
+for i, support in enumerate(beta):
+    if support != 0:
+        ax.axhspan(
+            i - 0.45,
+            i + 0.45,
+            color="tab:olive",
+            alpha=0.3,
+            zorder=-1,
+            label="True Support" if i == 1 else None,
+        )
+ax.legend()
+plt.show()
+
+# %%
+# The plot shows that the different models all identify the true support features and
+# assign them higher importance scores. However, the magnitude of the importance scores
+# varies across models. It can be observed that models with a greater predictive
+# performance (higher R2 score) tend to assign higher importance scores to the true
+# support features.
+
+# %%
+# Conclusion
+# ----------
+# LOCI is a simple and easily-interpretable method that can be used
+# in parallel to LOCO to obtain a different interpretation of variable importance.
+# Please keep in mind that models are fit on single features, which can lead to
+# overfitting depending on the hyperparameters. One way to counteract this is to
+# use LOCICV which can reduce noise for high-variance or unregularized models.
+
+# %%
+# References
+# ----------
+# .. footbibliography::

pyproject.toml

@@ -57,6 +57,7 @@ doc = [
     "sphinx-gallery         >= 0.17.0",
     "sphinx-prompt          >= 1.9",
     "tqdm                   >= 4.66.3",
+    "tabicl                 >= 2.1.0",
     "mne                    >= 1.7.0",
     "sphinx-copybutton      >= 0.5.2",
     "sphinx-design          >= 0.5.0",

src/hidimstat/__init__.py

@@ -6,6 +6,7 @@
 from .distilled_conditional_randomization_test import D0CRT, d0crt_importance
 from .ensemble_clustered_inference import CluDL, EnCluDL
 from .knockoffs import ModelXKnockoff, model_x_knockoff_importance
+from .leave_one_covariate_in import LOCI, LOCICV, loci_importance
 from .leave_one_covariate_out import LOCO, LOCOCV, loco_importance
 from .permutation_feature_importance import PFI, PFICV, pfi_importance
 
@@ -18,6 +19,8 @@
     "CFI",
     "CFICV",
     "D0CRT",
+    "LOCI",
+    "LOCICV",
     "LOCO",
     "LOCOCV",
     "PFI",
@@ -29,6 +32,7 @@
     "cfi_importance",
     "d0crt_importance",
     "desparsified_lasso_importance",
+    "loci_importance",
     "loco_importance",
     "model_x_knockoff_importance",
     "pfi_importance",