From 22f5c0eaea1614f023304988b09f206c36e22b37 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 2 Apr 2025 11:19:25 +0200 Subject: [PATCH 1/5] Feature branch to update to 1.6 From a1f0aef167e5c5c6f651539ba00ed62f627fb48c Mon Sep 17 00:00:00 2001 From: Arturo Amor <86408019+ArturoAmorQ@users.noreply.github.com> Date: Wed, 2 Apr 2025 18:23:50 +0200 Subject: [PATCH 2/5] MNT Fix several FutureWarnings (#810) --- notebooks/datasets_bike_rides.ipynb | 2 +- notebooks/ensemble_adaboost.ipynb | 2 +- notebooks/linear_models_regularization.ipynb | 6 +++--- notebooks/parameter_tuning_nested.ipynb | 1 + notebooks/parameter_tuning_randomized_search.ipynb | 1 + python_scripts/datasets_bike_rides.py | 2 +- python_scripts/ensemble_adaboost.py | 2 +- python_scripts/linear_models_regularization.py | 6 +++--- python_scripts/parameter_tuning_grid_search.py | 3 +++ python_scripts/parameter_tuning_nested.py | 1 + python_scripts/parameter_tuning_randomized_search.py | 1 + 11 files changed, 17 insertions(+), 10 deletions(-) diff --git a/notebooks/datasets_bike_rides.ipynb b/notebooks/datasets_bike_rides.ipynb index c4cb53450..504fdc716 100644 --- a/notebooks/datasets_bike_rides.ipynb +++ b/notebooks/datasets_bike_rides.ipynb @@ -271,7 +271,7 @@ "metadata": {}, "outputs": [], "source": [ - "data_ride.resample(\"60S\").mean().plot()\n", + "data_ride.resample(\"60s\").mean().plot()\n", "plt.legend(bbox_to_anchor=(1.05, 1), loc=\"upper left\")\n", "_ = plt.title(\"Sensor values for different cyclist measurements\")" ] diff --git a/notebooks/ensemble_adaboost.ipynb b/notebooks/ensemble_adaboost.ipynb index 90617c972..8daa73a05 100644 --- a/notebooks/ensemble_adaboost.ipynb +++ b/notebooks/ensemble_adaboost.ipynb @@ -271,7 +271,7 @@ "\n", "estimator = DecisionTreeClassifier(max_depth=3, random_state=0)\n", "adaboost = AdaBoostClassifier(\n", - " estimator=estimator, n_estimators=3, algorithm=\"SAMME\", random_state=0\n", + " estimator=estimator, n_estimators=3, random_state=0\n", ")\n", "adaboost.fit(data, target)" ] diff --git a/notebooks/linear_models_regularization.ipynb b/notebooks/linear_models_regularization.ipynb index fc1129695..4325bd10c 100644 --- a/notebooks/linear_models_regularization.ipynb +++ b/notebooks/linear_models_regularization.ipynb @@ -618,7 +618,7 @@ "ridge = make_pipeline(\n", " MinMaxScaler(),\n", " PolynomialFeatures(degree=2, include_bias=False),\n", - " RidgeCV(alphas=alphas, store_cv_values=True),\n", + " RidgeCV(alphas=alphas, store_cv_results=True),\n", ")" ] }, @@ -677,7 +677,7 @@ "It indicates that our model is not overfitting.\n", "\n", "When fitting the ridge regressor, we also requested to store the error found\n", - "during cross-validation (by setting the parameter `store_cv_values=True`). We\n", + "during cross-validation (by setting the parameter `store_cv_results=True`). We\n", "can plot the mean squared error for the different `alphas` regularization\n", "strengths that we tried. The error bars represent one standard deviation of the\n", "average mean square error across folds for a given value of `alpha`." @@ -690,7 +690,7 @@ "outputs": [], "source": [ "mse_alphas = [\n", - " est[-1].cv_values_.mean(axis=0) for est in cv_results[\"estimator\"]\n", + " est[-1].cv_results_.mean(axis=0) for est in cv_results[\"estimator\"]\n", "]\n", "cv_alphas = pd.DataFrame(mse_alphas, columns=alphas)\n", "cv_alphas = cv_alphas.aggregate([\"mean\", \"std\"]).T\n", diff --git a/notebooks/parameter_tuning_nested.ipynb b/notebooks/parameter_tuning_nested.ipynb index fb43c8145..6cd9bda14 100644 --- a/notebooks/parameter_tuning_nested.ipynb +++ b/notebooks/parameter_tuning_nested.ipynb @@ -70,6 +70,7 @@ " (\"cat_preprocessor\", categorical_preprocessor, categorical_columns),\n", " ],\n", " remainder=\"passthrough\",\n", + " force_int_remainder_cols=False, # Silence a warning in scikit-learn v1.6.\n", ")" ] }, diff --git a/notebooks/parameter_tuning_randomized_search.ipynb b/notebooks/parameter_tuning_randomized_search.ipynb index 94bc085dc..8c9aea809 100644 --- a/notebooks/parameter_tuning_randomized_search.ipynb +++ b/notebooks/parameter_tuning_randomized_search.ipynb @@ -121,6 +121,7 @@ "preprocessor = ColumnTransformer(\n", " [(\"cat_preprocessor\", categorical_preprocessor, categorical_columns)],\n", " remainder=\"passthrough\",\n", + " force_int_remainder_cols=False, # Silence a warning in scikit-learn v1.6.\n", ")" ] }, diff --git a/python_scripts/datasets_bike_rides.py b/python_scripts/datasets_bike_rides.py index 7944a0723..9cb2ec77a 100644 --- a/python_scripts/datasets_bike_rides.py +++ b/python_scripts/datasets_bike_rides.py @@ -155,7 +155,7 @@ # smoother visualization. # %% -data_ride.resample("60S").mean().plot() +data_ride.resample("60s").mean().plot() plt.legend(bbox_to_anchor=(1.05, 1), loc="upper left") _ = plt.title("Sensor values for different cyclist measurements") diff --git a/python_scripts/ensemble_adaboost.py b/python_scripts/ensemble_adaboost.py index 982084aed..25e4bcb16 100644 --- a/python_scripts/ensemble_adaboost.py +++ b/python_scripts/ensemble_adaboost.py @@ -190,7 +190,7 @@ estimator = DecisionTreeClassifier(max_depth=3, random_state=0) adaboost = AdaBoostClassifier( - estimator=estimator, n_estimators=3, algorithm="SAMME", random_state=0 + estimator=estimator, n_estimators=3, random_state=0 ) adaboost.fit(data, target) diff --git a/python_scripts/linear_models_regularization.py b/python_scripts/linear_models_regularization.py index 1b221e856..b4c8b3228 100644 --- a/python_scripts/linear_models_regularization.py +++ b/python_scripts/linear_models_regularization.py @@ -421,7 +421,7 @@ ridge = make_pipeline( MinMaxScaler(), PolynomialFeatures(degree=2, include_bias=False), - RidgeCV(alphas=alphas, store_cv_values=True), + RidgeCV(alphas=alphas, store_cv_results=True), ) # %% @@ -458,14 +458,14 @@ # It indicates that our model is not overfitting. # # When fitting the ridge regressor, we also requested to store the error found -# during cross-validation (by setting the parameter `store_cv_values=True`). We +# during cross-validation (by setting the parameter `store_cv_results=True`). We # can plot the mean squared error for the different `alphas` regularization # strengths that we tried. The error bars represent one standard deviation of the # average mean square error across folds for a given value of `alpha`. # %% mse_alphas = [ - est[-1].cv_values_.mean(axis=0) for est in cv_results["estimator"] + est[-1].cv_results_.mean(axis=0) for est in cv_results["estimator"] ] cv_alphas = pd.DataFrame(mse_alphas, columns=alphas) cv_alphas = cv_alphas.aggregate(["mean", "std"]).T diff --git a/python_scripts/parameter_tuning_grid_search.py b/python_scripts/parameter_tuning_grid_search.py index 79e738395..a3d156940 100644 --- a/python_scripts/parameter_tuning_grid_search.py +++ b/python_scripts/parameter_tuning_grid_search.py @@ -89,6 +89,9 @@ preprocessor = ColumnTransformer( [("cat_preprocessor", categorical_preprocessor, categorical_columns)], remainder="passthrough", + # Silence a deprecation warning in scikit-learn v1.6 related to how the + # ColumnTransformer stores an attribute that we do not use in this notebook + force_int_remainder_cols=False, ) # %% [markdown] diff --git a/python_scripts/parameter_tuning_nested.py b/python_scripts/parameter_tuning_nested.py index 5c13cd28d..d447cb997 100644 --- a/python_scripts/parameter_tuning_nested.py +++ b/python_scripts/parameter_tuning_nested.py @@ -56,6 +56,7 @@ ("cat_preprocessor", categorical_preprocessor, categorical_columns), ], remainder="passthrough", + force_int_remainder_cols=False, # Silence a warning in scikit-learn v1.6. ) # %% diff --git a/python_scripts/parameter_tuning_randomized_search.py b/python_scripts/parameter_tuning_randomized_search.py index 0bcd4761d..2b1cd19c5 100644 --- a/python_scripts/parameter_tuning_randomized_search.py +++ b/python_scripts/parameter_tuning_randomized_search.py @@ -73,6 +73,7 @@ preprocessor = ColumnTransformer( [("cat_preprocessor", categorical_preprocessor, categorical_columns)], remainder="passthrough", + force_int_remainder_cols=False, # Silence a warning in scikit-learn v1.6. ) # %% From 57d821e9c482e1232b779890239fb0c25b122740 Mon Sep 17 00:00:00 2001 From: SebastienMelo <125986598+SebastienMelo@users.noreply.github.com> Date: Thu, 3 Apr 2025 11:57:50 +0200 Subject: [PATCH 3/5] MTN Wrap up quiz sklearn 1.6 verification (#817) --- .gitignore | 1 + Makefile | 5 ++ build_tools/generate-wrap-up.py | 107 ++++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 build_tools/generate-wrap-up.py diff --git a/.gitignore b/.gitignore index 4c0d2950f..5c6b71928 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # exlude datasets and externals notebooks/datasets notebooks/joblib/ +wrap-up/ # jupyter-book jupyter-book/_build diff --git a/Makefile b/Makefile index e3fc7ef67..c8013ac45 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ PYTHON_SCRIPTS_DIR = python_scripts NOTEBOOKS_DIR = notebooks JUPYTER_BOOK_DIR = jupyter-book +WRAP_UP_DIR = wrap-up JUPYTER_KERNEL := python3 MINIMAL_NOTEBOOK_FILES = $(shell ls $(PYTHON_SCRIPTS_DIR)/*.py | perl -pe "s@$(PYTHON_SCRIPTS_DIR)@$(NOTEBOOKS_DIR)@" | perl -pe "s@\.py@.ipynb@") @@ -37,6 +38,10 @@ quizzes: full-index: python build_tools/generate-index.py +run-code-in-wrap-up-quizzes: + python build_tools/generate-wrap-up.py $(GITLAB_REPO_JUPYTERBOOK_DIR) $(WRAP_UP_DIR) + jupytext --execute --to notebook $(WRAP_UP_DIR)/*.py + $(JUPYTER_BOOK_DIR): jupyter-book build $(JUPYTER_BOOK_DIR) rm -rf $(JUPYTER_BOOK_DIR)/_build/html/{slides,figures} && cp -r slides figures $(JUPYTER_BOOK_DIR)/_build/html diff --git a/build_tools/generate-wrap-up.py b/build_tools/generate-wrap-up.py new file mode 100644 index 000000000..0edc807c4 --- /dev/null +++ b/build_tools/generate-wrap-up.py @@ -0,0 +1,107 @@ +import sys +import os +import glob + + +def extract_python_code_blocks(md_file_path): + """ + Extract Python code blocks from a markdown file. + + Args: + md_file_path (str): Path to the markdown file + + Returns: + list: List of extracted Python code blocks + """ + code_blocks = [] + in_python_block = False + current_block = [] + + with open(md_file_path, "r", encoding="utf-8") as file: + for line in file: + line = line.rstrip("\n") + + if line.strip() == "```python": + in_python_block = True + current_block = [] + elif line.strip() == "```" and in_python_block: + in_python_block = False + code_blocks.append("\n".join(current_block)) + elif in_python_block: + current_block.append(line) + + return code_blocks + + +def write_jupyter_notebook_file( + code_blocks, output_file="notebook_from_md.py" +): + """ + Writes extracted code blocks to a Python file formatted as Jupyter notebook cells. + + Args: + code_blocks (list): List of code blocks to write + output_file (str): Path to the output file + """ + with open(output_file, "w", encoding="utf-8") as file: + file.write( + "# %% [markdown] \n # ## Notebook generated from Markdown file\n\n" + ) + + for i, block in enumerate(code_blocks, 1): + file.write(f"# %% [markdown]\n# ## Cell {i}\n\n# %%\n{block}\n\n") + + print( + f"Successfully wrote {len(code_blocks)} code cells to" + f" {output_file}" + ) + + +def process_quiz_files(input_path, output_dir): + """ + Process all wrap_up_quiz files in the input path and convert them to notebooks. + + Args: + input_path (str): Path to look for wrap_up_quiz files in subfolders + output_dir (str): Directory to write the generated notebooks + """ + # Create output directory if it doesn't exist + if not os.path.exists(output_dir): + os.makedirs(output_dir) + print(f"Created output directory: {output_dir}") + + # Find all files containing "wrap_up_quiz" in their name in the input path subfolders + quiz_files = glob.glob( + f"{input_path}/**/*wrap_up_quiz*.md", recursive=True + ) + + if not quiz_files: + print(f"No wrap_up_quiz.md files found in {input_path} subfolders.") + return + + print(f"Found {len(quiz_files)} wrap_up_quiz files to process.") + + # Process each file + for md_file_path in quiz_files: + print(f"\nProcessing: {md_file_path}") + + # Extract code blocks + code_blocks = extract_python_code_blocks(md_file_path) + + # Generate output filename + subfolder = md_file_path.split(os.sep)[3] # Get subfolder name + output_file = os.path.join(output_dir, f"{subfolder}_wrap_up_quiz.py") + + # Display results and write notebook file + if code_blocks: + print(f"Found {len(code_blocks)} Python code blocks") + write_jupyter_notebook_file(code_blocks, output_file=output_file) + else: + print(f"No Python code blocks found in {md_file_path}.") + + +if __name__ == "__main__": + input_path = sys.argv[1] + output_dir = sys.argv[2] + + process_quiz_files(input_path, output_dir) From f01f612e57680b9ff9b44bc78faee33158050386 Mon Sep 17 00:00:00 2001 From: Arturo Amor <86408019+ArturoAmorQ@users.noreply.github.com> Date: Thu, 3 Apr 2025 14:08:53 +0200 Subject: [PATCH 4/5] MAINT Use class_of_interest in DecisionBoundaryDisplay (#772) --- notebooks/trees_ex_01.ipynb | 6 +-- notebooks/trees_sol_01.ipynb | 97 +++++++++++++++------------------- python_scripts/trees_ex_01.py | 6 +-- python_scripts/trees_sol_01.py | 97 +++++++++++++++------------------- 4 files changed, 90 insertions(+), 116 deletions(-) diff --git a/notebooks/trees_ex_01.ipynb b/notebooks/trees_ex_01.ipynb index a2abdea01..cae6407bc 100644 --- a/notebooks/trees_ex_01.ipynb +++ b/notebooks/trees_ex_01.ipynb @@ -83,9 +83,9 @@ "
\n", "

Warning

\n", "

At this time, it is not possible to use response_method=\"predict_proba\" for\n", - "multiclass problems. This is a planned feature for a future version of\n", - "scikit-learn. In the mean time, you can use response_method=\"predict\"\n", - "instead.

\n", + "multiclass problems on a single plot. This is a planned feature for a future\n", + "version of scikit-learn. In the mean time, you can use\n", + "response_method=\"predict\" instead.

\n", "
" ] }, diff --git a/notebooks/trees_sol_01.ipynb b/notebooks/trees_sol_01.ipynb index 2ce0c1b8b..8ca3f8ec2 100644 --- a/notebooks/trees_sol_01.ipynb +++ b/notebooks/trees_sol_01.ipynb @@ -87,9 +87,9 @@ "
\n", "

Warning

\n", "

At this time, it is not possible to use response_method=\"predict_proba\" for\n", - "multiclass problems. This is a planned feature for a future version of\n", - "scikit-learn. In the mean time, you can use response_method=\"predict\"\n", - "instead.

\n", + "multiclass problems on a single plot. This is a planned feature for a future\n", + "version of scikit-learn. In the mean time, you can use\n", + "response_method=\"predict\" instead.

\n", "
" ] }, @@ -212,12 +212,14 @@ "except that for a K-class problem you have K probability outputs for each\n", "data point. Visualizing all these on a single plot can quickly become tricky\n", "to interpret. It is then common to instead produce K separate plots, one for\n", - "each class, in a one-vs-rest (or one-vs-all) fashion.\n", + "each class, in a one-vs-rest (or one-vs-all) fashion. This can be achieved by\n", + "calling `DecisionBoundaryDisplay` several times, once for each class, and\n", + "passing the `class_of_interest` parameter to the function.\n", "\n", - "For example, in the plot below, the first plot on the left shows in yellow the\n", - "certainty on classifying a data point as belonging to the \"Adelie\" class. In\n", - "the same plot, the spectre from green to purple represents the certainty of\n", - "**not** belonging to the \"Adelie\" class. The same logic applies to the other\n", + "For example, in the plot below, the first plot on the left shows the\n", + "certainty of classifying a data point as belonging to the \"Adelie\" class. The\n", + "darker the color, the more certain the model is that a given point in the\n", + "feature space belongs to a given class. The same logic applies to the other\n", "plots in the figure." ] }, @@ -231,48 +233,38 @@ }, "outputs": [], "source": [ - "import numpy as np\n", - "\n", - "xx = np.linspace(30, 60, 100)\n", - "yy = np.linspace(10, 23, 100)\n", - "xx, yy = np.meshgrid(xx, yy)\n", - "Xfull = pd.DataFrame(\n", - " {\"Culmen Length (mm)\": xx.ravel(), \"Culmen Depth (mm)\": yy.ravel()}\n", - ")\n", - "\n", - "probas = tree.predict_proba(Xfull)\n", - "n_classes = len(np.unique(tree.classes_))\n", + "from matplotlib import cm\n", "\n", "_, axs = plt.subplots(ncols=3, nrows=1, sharey=True, figsize=(12, 5))\n", - "plt.suptitle(\"Predicted probabilities for decision tree model\", y=0.8)\n", + "plt.suptitle(\"Predicted probabilities for decision tree model\", y=1.05)\n", + "plt.subplots_adjust(bottom=0.45)\n", "\n", - "for class_of_interest in range(n_classes):\n", - " axs[class_of_interest].set_title(\n", - " f\"Class {tree.classes_[class_of_interest]}\"\n", - " )\n", - " imshow_handle = axs[class_of_interest].imshow(\n", - " probas[:, class_of_interest].reshape((100, 100)),\n", - " extent=(30, 60, 10, 23),\n", - " vmin=0.0,\n", - " vmax=1.0,\n", - " origin=\"lower\",\n", - " cmap=\"viridis\",\n", + "for idx, (class_of_interest, ax) in enumerate(zip(tree.classes_, axs)):\n", + " ax.set_title(f\"Class {class_of_interest}\")\n", + " DecisionBoundaryDisplay.from_estimator(\n", + " tree,\n", + " data_test,\n", + " response_method=\"predict_proba\",\n", + " class_of_interest=class_of_interest,\n", + " ax=ax,\n", + " vmin=0,\n", + " vmax=1,\n", + " cmap=\"Blues\",\n", " )\n", - " axs[class_of_interest].set_xlabel(\"Culmen Length (mm)\")\n", - " if class_of_interest == 0:\n", - " axs[class_of_interest].set_ylabel(\"Culmen Depth (mm)\")\n", - " idx = target_test == tree.classes_[class_of_interest]\n", - " axs[class_of_interest].scatter(\n", - " data_test[\"Culmen Length (mm)\"].loc[idx],\n", - " data_test[\"Culmen Depth (mm)\"].loc[idx],\n", + " ax.scatter(\n", + " data_test[\"Culmen Length (mm)\"].loc[target_test == class_of_interest],\n", + " data_test[\"Culmen Depth (mm)\"].loc[target_test == class_of_interest],\n", " marker=\"o\",\n", " c=\"w\",\n", " edgecolor=\"k\",\n", " )\n", + " ax.set_xlabel(\"Culmen Length (mm)\")\n", + " if idx == 0:\n", + " ax.set_ylabel(\"Culmen Depth (mm)\")\n", "\n", - "ax = plt.axes([0.15, 0.04, 0.7, 0.05])\n", - "plt.colorbar(imshow_handle, cax=ax, orientation=\"horizontal\")\n", - "_ = plt.title(\"Probability\")" + "ax = plt.axes([0.15, 0.14, 0.7, 0.05])\n", + "plt.colorbar(cm.ScalarMappable(cmap=\"Blues\"), cax=ax, orientation=\"horizontal\")\n", + "_ = ax.set_title(\"Predicted class membership probability\")" ] }, { @@ -283,22 +275,17 @@ ] }, "source": [ + "\n", "
\n", "

Note

\n", - "

You may have noticed that we are no longer using a diverging colormap. Indeed,\n", - "the chance level for a one-vs-rest binarization of the multi-class\n", - "classification problem is almost never at predicted probability of 0.5. So\n", - "using a colormap with a neutral white at 0.5 might give a false impression on\n", - "the certainty.

\n", - "
\n", - "\n", - "In future versions of scikit-learn `DecisionBoundaryDisplay` will support a\n", - "`class_of_interest` parameter that will allow in particular for a\n", - "visualization of `predict_proba` in multi-class settings.\n", - "\n", - "We also plan to make it possible to visualize the `predict_proba` values for\n", - "the class with the maximum predicted probability (without having to pass a\n", - "given a fixed `class_of_interest` value)." + "

You may notice that we do not use a diverging colormap (2 color gradients with\n", + "white in the middle). Indeed, in a multiclass setting, 0.5 is not a\n", + "meaningful value, hence using white as the center of the colormap is not\n", + "appropriate. Instead, we use a sequential colormap, where the color intensity\n", + "indicates the certainty of the classification. The darker the color, the more\n", + "certain the model is that a given point in the feature space belongs to a\n", + "given class.

\n", + "" ] } ], diff --git a/python_scripts/trees_ex_01.py b/python_scripts/trees_ex_01.py index b3aa90a10..51068532b 100644 --- a/python_scripts/trees_ex_01.py +++ b/python_scripts/trees_ex_01.py @@ -59,9 +59,9 @@ # # ```{warning} # At this time, it is not possible to use `response_method="predict_proba"` for -# multiclass problems. This is a planned feature for a future version of -# scikit-learn. In the mean time, you can use `response_method="predict"` -# instead. +# multiclass problems on a single plot. This is a planned feature for a future +# version of scikit-learn. In the mean time, you can use +# `response_method="predict"` instead. # ``` # %% diff --git a/python_scripts/trees_sol_01.py b/python_scripts/trees_sol_01.py index e97b7e8b2..a600e9fe3 100644 --- a/python_scripts/trees_sol_01.py +++ b/python_scripts/trees_sol_01.py @@ -57,9 +57,9 @@ # # ```{warning} # At this time, it is not possible to use `response_method="predict_proba"` for -# multiclass problems. This is a planned feature for a future version of -# scikit-learn. In the mean time, you can use `response_method="predict"` -# instead. +# multiclass problems on a single plot. This is a planned feature for a future +# version of scikit-learn. In the mean time, you can use +# `response_method="predict"` instead. # ``` # %% @@ -140,71 +140,58 @@ # except that for a K-class problem you have K probability outputs for each # data point. Visualizing all these on a single plot can quickly become tricky # to interpret. It is then common to instead produce K separate plots, one for -# each class, in a one-vs-rest (or one-vs-all) fashion. +# each class, in a one-vs-rest (or one-vs-all) fashion. This can be achieved by +# calling `DecisionBoundaryDisplay` several times, once for each class, and +# passing the `class_of_interest` parameter to the function. # -# For example, in the plot below, the first plot on the left shows in yellow the -# certainty on classifying a data point as belonging to the "Adelie" class. In -# the same plot, the spectre from green to purple represents the certainty of -# **not** belonging to the "Adelie" class. The same logic applies to the other +# For example, in the plot below, the first plot on the left shows the +# certainty of classifying a data point as belonging to the "Adelie" class. The +# darker the color, the more certain the model is that a given point in the +# feature space belongs to a given class. The same logic applies to the other # plots in the figure. # %% tags=["solution"] -import numpy as np - -xx = np.linspace(30, 60, 100) -yy = np.linspace(10, 23, 100) -xx, yy = np.meshgrid(xx, yy) -Xfull = pd.DataFrame( - {"Culmen Length (mm)": xx.ravel(), "Culmen Depth (mm)": yy.ravel()} -) - -probas = tree.predict_proba(Xfull) -n_classes = len(np.unique(tree.classes_)) +from matplotlib import cm _, axs = plt.subplots(ncols=3, nrows=1, sharey=True, figsize=(12, 5)) -plt.suptitle("Predicted probabilities for decision tree model", y=0.8) - -for class_of_interest in range(n_classes): - axs[class_of_interest].set_title( - f"Class {tree.classes_[class_of_interest]}" +plt.suptitle("Predicted probabilities for decision tree model", y=1.05) +plt.subplots_adjust(bottom=0.45) + +for idx, (class_of_interest, ax) in enumerate(zip(tree.classes_, axs)): + ax.set_title(f"Class {class_of_interest}") + DecisionBoundaryDisplay.from_estimator( + tree, + data_test, + response_method="predict_proba", + class_of_interest=class_of_interest, + ax=ax, + vmin=0, + vmax=1, + cmap="Blues", ) - imshow_handle = axs[class_of_interest].imshow( - probas[:, class_of_interest].reshape((100, 100)), - extent=(30, 60, 10, 23), - vmin=0.0, - vmax=1.0, - origin="lower", - cmap="viridis", - ) - axs[class_of_interest].set_xlabel("Culmen Length (mm)") - if class_of_interest == 0: - axs[class_of_interest].set_ylabel("Culmen Depth (mm)") - idx = target_test == tree.classes_[class_of_interest] - axs[class_of_interest].scatter( - data_test["Culmen Length (mm)"].loc[idx], - data_test["Culmen Depth (mm)"].loc[idx], + ax.scatter( + data_test["Culmen Length (mm)"].loc[target_test == class_of_interest], + data_test["Culmen Depth (mm)"].loc[target_test == class_of_interest], marker="o", c="w", edgecolor="k", ) + ax.set_xlabel("Culmen Length (mm)") + if idx == 0: + ax.set_ylabel("Culmen Depth (mm)") -ax = plt.axes([0.15, 0.04, 0.7, 0.05]) -plt.colorbar(imshow_handle, cax=ax, orientation="horizontal") -_ = plt.title("Probability") +ax = plt.axes([0.15, 0.14, 0.7, 0.05]) +plt.colorbar(cm.ScalarMappable(cmap="Blues"), cax=ax, orientation="horizontal") +_ = ax.set_title("Predicted class membership probability") # %% [markdown] tags=["solution"] +# # ```{note} -# You may have noticed that we are no longer using a diverging colormap. Indeed, -# the chance level for a one-vs-rest binarization of the multi-class -# classification problem is almost never at predicted probability of 0.5. So -# using a colormap with a neutral white at 0.5 might give a false impression on -# the certainty. +# You may notice that we do not use a diverging colormap (2 color gradients with +# white in the middle). Indeed, in a multiclass setting, 0.5 is not a +# meaningful value, hence using white as the center of the colormap is not +# appropriate. Instead, we use a sequential colormap, where the color intensity +# indicates the certainty of the classification. The darker the color, the more +# certain the model is that a given point in the feature space belongs to a +# given class. # ``` -# -# In future versions of scikit-learn `DecisionBoundaryDisplay` will support a -# `class_of_interest` parameter that will allow in particular for a -# visualization of `predict_proba` in multi-class settings. -# -# We also plan to make it possible to visualize the `predict_proba` values for -# the class with the maximum predicted probability (without having to pass a -# given a fixed `class_of_interest` value). From a9ca76b61a20759c88b316ef5e3914caa75e0fc7 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 27 May 2025 14:43:11 +0200 Subject: [PATCH 5/5] Resync everything --- notebooks/03_categorical_pipeline_ex_02.ipynb | 21 +--------------- notebooks/cross_validation_grouping.ipynb | 7 +++--- notebooks/ensemble_ex_03.ipynb | 24 ++++++++++++++++--- notebooks/parameter_tuning_grid_search.ipynb | 3 +++ .../01_tabular_data_exploration_ex_01.py | 2 +- python_scripts/02_numerical_pipeline_ex_00.py | 2 +- python_scripts/02_numerical_pipeline_ex_01.py | 2 +- .../03_categorical_pipeline_ex_01.py | 2 +- .../03_categorical_pipeline_ex_02.py | 23 ++---------------- python_scripts/cross_validation_ex_01.py | 2 +- python_scripts/cross_validation_ex_02.py | 2 +- python_scripts/ensemble_ex_01.py | 2 +- python_scripts/ensemble_ex_02.py | 2 +- python_scripts/ensemble_ex_03.py | 15 ++++++++---- python_scripts/ensemble_ex_04.py | 2 +- python_scripts/feature_selection_ex_01.py | 2 +- python_scripts/linear_models_ex_01.py | 2 +- python_scripts/linear_models_ex_02.py | 2 +- python_scripts/linear_models_ex_03.py | 2 +- python_scripts/linear_models_ex_04.py | 2 +- python_scripts/metrics_ex_01.py | 2 +- python_scripts/metrics_ex_02.py | 2 +- python_scripts/parameter_tuning_ex_02.py | 2 +- python_scripts/parameter_tuning_ex_03.py | 2 +- python_scripts/trees_ex_01.py | 2 +- python_scripts/trees_ex_02.py | 2 +- 26 files changed, 62 insertions(+), 71 deletions(-) diff --git a/notebooks/03_categorical_pipeline_ex_02.ipynb b/notebooks/03_categorical_pipeline_ex_02.ipynb index 8a6d52408..d3bc42d33 100644 --- a/notebooks/03_categorical_pipeline_ex_02.ipynb +++ b/notebooks/03_categorical_pipeline_ex_02.ipynb @@ -160,26 +160,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Analysis\n", - "\n", - "From an accuracy point of view, the result is almost exactly the same. The\n", - "reason is that `HistGradientBoostingClassifier` is expressive and robust\n", - "enough to deal with misleading ordering of integer coded categories (which was\n", - "not the case for linear models).\n", - "\n", - "However from a computation point of view, the training time is much longer:\n", - "this is caused by the fact that `OneHotEncoder` generates more features than\n", - "`OrdinalEncoder`; for each unique categorical value a column is created.\n", - "\n", - "Note that the current implementation `HistGradientBoostingClassifier` is still\n", - "incomplete, and once sparse representation are handled correctly, training\n", - "time might improve with such kinds of encodings.\n", - "\n", - "The main take away message is that arbitrary integer coding of categories is\n", - "perfectly fine for `HistGradientBoostingClassifier` and yields fast training\n", - "times.\n", - "\n", - "Which encoder should I use?\n", + "## Which encoder should I use?\n", "\n", "| | Meaningful order | Non-meaningful order |\n", "| ---------------- | ----------------------------- | -------------------- |\n", diff --git a/notebooks/cross_validation_grouping.ipynb b/notebooks/cross_validation_grouping.ipynb index 705db1ca9..df19e4db0 100644 --- a/notebooks/cross_validation_grouping.ipynb +++ b/notebooks/cross_validation_grouping.ipynb @@ -189,9 +189,10 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "If we read carefully, 13 writers wrote the digits of our dataset, accounting\n", - "for a total amount of 1797 samples. Thus, a writer wrote several times the\n", - "same numbers. Let's suppose that the writer samples are grouped. Subsequently,\n", + "If we read carefully, `load_digits` loads a copy of the **test set** of the\n", + "UCI ML hand-written digits dataset, which consists of 1797 images by\n", + "**13 different writers**. Thus, each writer wrote several times the same\n", + "numbers. Let's suppose the dataset is ordered by writer. Subsequently,\n", "not shuffling the data will keep all writer samples together either in the\n", "training or the testing sets. Mixing the data will break this structure, and\n", "therefore digits written by the same writer will be available in both the\n", diff --git a/notebooks/ensemble_ex_03.ipynb b/notebooks/ensemble_ex_03.ipynb index ae3607697..b1e9b2fc2 100644 --- a/notebooks/ensemble_ex_03.ipynb +++ b/notebooks/ensemble_ex_03.ipynb @@ -107,6 +107,24 @@ "ensemble. However, the scores reach a plateau where adding new trees just\n", "makes fitting and scoring slower.\n", "\n", + "Now repeat the analysis for the gradient boosting model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "lines_to_next_cell": 2 + }, + "outputs": [], + "source": [ + "# Write your code here." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ "Gradient boosting models overfit when the number of trees is too large. To\n", "avoid adding a new unnecessary tree, unlike random-forest gradient-boosting\n", "offers an early-stopping option. Internally, the algorithm uses an\n", @@ -115,9 +133,9 @@ "improving for several iterations, it stops adding trees.\n", "\n", "Now, create a gradient-boosting model with `n_estimators=1_000`. This number\n", - "of trees is certainly too large. Change the parameter `n_iter_no_change`\n", - "such that the gradient boosting fitting stops after adding 5 trees to avoid\n", - "deterioration of the overall generalization performance." + "of trees is certainly too large as we have seen above. Change the parameter\n", + "`n_iter_no_change` such that the gradient boosting fitting stops after adding\n", + "5 trees to avoid deterioration of the overall generalization performance." ] }, { diff --git a/notebooks/parameter_tuning_grid_search.ipynb b/notebooks/parameter_tuning_grid_search.ipynb index a71dba64c..eec36cd72 100644 --- a/notebooks/parameter_tuning_grid_search.ipynb +++ b/notebooks/parameter_tuning_grid_search.ipynb @@ -157,6 +157,9 @@ "preprocessor = ColumnTransformer(\n", " [(\"cat_preprocessor\", categorical_preprocessor, categorical_columns)],\n", " remainder=\"passthrough\",\n", + " # Silence a deprecation warning in scikit-learn v1.6 related to how the\n", + " # ColumnTransformer stores an attribute that we do not use in this notebook\n", + " force_int_remainder_cols=False,\n", ")" ] }, diff --git a/python_scripts/01_tabular_data_exploration_ex_01.py b/python_scripts/01_tabular_data_exploration_ex_01.py index 374efeda5..f40d4aa6a 100644 --- a/python_scripts/01_tabular_data_exploration_ex_01.py +++ b/python_scripts/01_tabular_data_exploration_ex_01.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/02_numerical_pipeline_ex_00.py b/python_scripts/02_numerical_pipeline_ex_00.py index 389640dcb..41629385f 100644 --- a/python_scripts/02_numerical_pipeline_ex_00.py +++ b/python_scripts/02_numerical_pipeline_ex_00.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/02_numerical_pipeline_ex_01.py b/python_scripts/02_numerical_pipeline_ex_01.py index 6f24974d2..7c55ed83e 100644 --- a/python_scripts/02_numerical_pipeline_ex_01.py +++ b/python_scripts/02_numerical_pipeline_ex_01.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/03_categorical_pipeline_ex_01.py b/python_scripts/03_categorical_pipeline_ex_01.py index bd407bf9d..7003939a4 100644 --- a/python_scripts/03_categorical_pipeline_ex_01.py +++ b/python_scripts/03_categorical_pipeline_ex_01.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/03_categorical_pipeline_ex_02.py b/python_scripts/03_categorical_pipeline_ex_02.py index c7f160c2d..670e26105 100644 --- a/python_scripts/03_categorical_pipeline_ex_02.py +++ b/python_scripts/03_categorical_pipeline_ex_02.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 @@ -112,26 +112,7 @@ # Write your code here. # %% [markdown] -# ### Analysis -# -# From an accuracy point of view, the result is almost exactly the same. The -# reason is that `HistGradientBoostingClassifier` is expressive and robust -# enough to deal with misleading ordering of integer coded categories (which was -# not the case for linear models). -# -# However from a computation point of view, the training time is much longer: -# this is caused by the fact that `OneHotEncoder` generates more features than -# `OrdinalEncoder`; for each unique categorical value a column is created. -# -# Note that the current implementation `HistGradientBoostingClassifier` is still -# incomplete, and once sparse representation are handled correctly, training -# time might improve with such kinds of encodings. -# -# The main take away message is that arbitrary integer coding of categories is -# perfectly fine for `HistGradientBoostingClassifier` and yields fast training -# times. - -# Which encoder should I use? +# ## Which encoder should I use? # # | | Meaningful order | Non-meaningful order | # | ---------------- | ----------------------------- | -------------------- | diff --git a/python_scripts/cross_validation_ex_01.py b/python_scripts/cross_validation_ex_01.py index 9b9844024..64f1baada 100644 --- a/python_scripts/cross_validation_ex_01.py +++ b/python_scripts/cross_validation_ex_01.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/cross_validation_ex_02.py b/python_scripts/cross_validation_ex_02.py index 804495a9e..7e3b31e91 100644 --- a/python_scripts/cross_validation_ex_02.py +++ b/python_scripts/cross_validation_ex_02.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/ensemble_ex_01.py b/python_scripts/ensemble_ex_01.py index 8f7031361..62888fc26 100644 --- a/python_scripts/ensemble_ex_01.py +++ b/python_scripts/ensemble_ex_01.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/ensemble_ex_02.py b/python_scripts/ensemble_ex_02.py index 32679a245..9c7c2e364 100644 --- a/python_scripts/ensemble_ex_02.py +++ b/python_scripts/ensemble_ex_02.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/ensemble_ex_03.py b/python_scripts/ensemble_ex_03.py index 4b303b5db..12d601078 100644 --- a/python_scripts/ensemble_ex_03.py +++ b/python_scripts/ensemble_ex_03.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 @@ -70,6 +70,13 @@ # ensemble. However, the scores reach a plateau where adding new trees just # makes fitting and scoring slower. # +# Now repeat the analysis for the gradient boosting model. + +# %% +# Write your code here. + + +# %% [markdown] # Gradient boosting models overfit when the number of trees is too large. To # avoid adding a new unnecessary tree, unlike random-forest gradient-boosting # offers an early-stopping option. Internally, the algorithm uses an @@ -78,9 +85,9 @@ # improving for several iterations, it stops adding trees. # # Now, create a gradient-boosting model with `n_estimators=1_000`. This number -# of trees is certainly too large. Change the parameter `n_iter_no_change` -# such that the gradient boosting fitting stops after adding 5 trees to avoid -# deterioration of the overall generalization performance. +# of trees is certainly too large as we have seen above. Change the parameter +# `n_iter_no_change` such that the gradient boosting fitting stops after adding +# 5 trees to avoid deterioration of the overall generalization performance. # %% # Write your code here. diff --git a/python_scripts/ensemble_ex_04.py b/python_scripts/ensemble_ex_04.py index 126f0eff4..ed481b49a 100644 --- a/python_scripts/ensemble_ex_04.py +++ b/python_scripts/ensemble_ex_04.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/feature_selection_ex_01.py b/python_scripts/feature_selection_ex_01.py index bba499ba6..ff40075b6 100644 --- a/python_scripts/feature_selection_ex_01.py +++ b/python_scripts/feature_selection_ex_01.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/linear_models_ex_01.py b/python_scripts/linear_models_ex_01.py index 8f9a040f5..4ddf7b808 100644 --- a/python_scripts/linear_models_ex_01.py +++ b/python_scripts/linear_models_ex_01.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/linear_models_ex_02.py b/python_scripts/linear_models_ex_02.py index c43ea69be..ae667c160 100644 --- a/python_scripts/linear_models_ex_02.py +++ b/python_scripts/linear_models_ex_02.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/linear_models_ex_03.py b/python_scripts/linear_models_ex_03.py index e89f9ea9e..3458dd2cd 100644 --- a/python_scripts/linear_models_ex_03.py +++ b/python_scripts/linear_models_ex_03.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/linear_models_ex_04.py b/python_scripts/linear_models_ex_04.py index c6f3fc87b..247d2acf3 100644 --- a/python_scripts/linear_models_ex_04.py +++ b/python_scripts/linear_models_ex_04.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/metrics_ex_01.py b/python_scripts/metrics_ex_01.py index 005a6ba54..e4b111e01 100644 --- a/python_scripts/metrics_ex_01.py +++ b/python_scripts/metrics_ex_01.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/metrics_ex_02.py b/python_scripts/metrics_ex_02.py index 2b4837cbf..17a594e98 100644 --- a/python_scripts/metrics_ex_02.py +++ b/python_scripts/metrics_ex_02.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/parameter_tuning_ex_02.py b/python_scripts/parameter_tuning_ex_02.py index efa307de1..f53ddb36a 100644 --- a/python_scripts/parameter_tuning_ex_02.py +++ b/python_scripts/parameter_tuning_ex_02.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/parameter_tuning_ex_03.py b/python_scripts/parameter_tuning_ex_03.py index 5e316eddf..95709912e 100644 --- a/python_scripts/parameter_tuning_ex_03.py +++ b/python_scripts/parameter_tuning_ex_03.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/trees_ex_01.py b/python_scripts/trees_ex_01.py index 51068532b..4967554f4 100644 --- a/python_scripts/trees_ex_01.py +++ b/python_scripts/trees_ex_01.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3 diff --git a/python_scripts/trees_ex_02.py b/python_scripts/trees_ex_02.py index 565db09fc..d11f967a9 100644 --- a/python_scripts/trees_ex_02.py +++ b/python_scripts/trees_ex_02.py @@ -5,7 +5,7 @@ # extension: .py # format_name: percent # format_version: '1.3' -# jupytext_version: 1.16.7 +# jupytext_version: 1.17.1 # kernelspec: # display_name: Python 3 # name: python3