plot_advanced_tabicl.py

"""
Tabular Foundation Model TabICL
================================

In this example, we demonstrate how to use a tabular foundation model such as
TabICL [:footcite:t:`qu2025tabicl`] with a straightforward example on the California
Housing regression dataset.
"""

import pandas as pd
from sklearn.datasets import fetch_california_housing
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.utils import resample
from tabicl import TabICLRegressor

from hidimstat import CFI

# %%
# Loading data and TabICL
# -----------------------
# We start by loading data from the California Housing dataset and fitting the
# TabICL model. If this is the first use, the model will be downloaded and cached
# for future use. The main advantage is that the model does not require
# hyperparameter tuning as a pre-trained transformer model. We recommend switching
# to GPU through the adequate class parameter for datasets that go over 10k
# samples depending on the number of features due to computation time increase.

dataset = fetch_california_housing()
X, y, feat_names = dataset.data, dataset.target, dataset.feature_names
X, y = resample(
    X, y, replace=False, n_samples=1000, stratify=y, random_state=0
)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=0
)

model = make_pipeline(StandardScaler(), TabICLRegressor(device="cpu"))
model.fit(X_train, y_train)

# %%
# For the moment, TabICL can only be used with PFI or CFI. TabICL only works with
# integer seeding, which is currently not supported by D0CRT.
#
# TabICL interfaces with HiDimStat the same way as any other Scikit-learn estimator,
# which lets us compute feature importance as easily as follows:

cfi = CFI(estimator=model, method="predict", loss=mean_squared_error)
cfi.fit(X_train, y_train)
importances = cfi.importance(X_test, y_test)
selection = cfi.fdr_selection(fdr=0.1)

df = pd.DataFrame(
    {
        "feature": feat_names,
        "importance": importances,
        "selected": selection,
    }
).sort_values("importance", ascending=False)

import matplotlib.pyplot as plt
import seaborn as sns

ax = sns.barplot(
    data=df,
    y="feature",
    x="importance",
    hue="selected",
    palette="muted",
    orient="h",
)
sns.despine()
plt.show()

# %%
# As you can see from the timer under, even with 1000 samples and 9 features,
# running TabICL on the CPU is slow. We recommend to use it on larger datasets,
# be it in terms of samples and/or features and to run it on GPU for faster
# inferences. Due to technical limitations, this is something we cannot showcase
# here.