hyperparameter_optimization/hyperparameter_optimization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3d8b9b58",
   "metadata": {
    "papermill": {
     "duration": 0.005595,
     "end_time": "2022-09-08T14:41:08.005868",
     "exception": false,
     "start_time": "2022-09-08T14:41:08.000273",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Hyperparameter Optimization Example"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39796de7",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/cleanlab/examples/blob/master/hyperparameter_optimization/hyperparameter_optimization.ipynb) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d9624cf9",
   "metadata": {},
   "source": [
    "This example will show you the main hyper-parameters for CleanLearning. There are only two!\n",
    "\n",
    "1. `filter_by` : str (default: `'prune_by_noise_rate'`), Method used for pruning.\n",
    "    * Values: [`'prune_by_class'`, `'prune_by_noise_rate'`, or `'both'`]. \n",
    "    * `'prune_by_noise_rate'`: works by removing examples with *high probability* of being mislabeled for every non-diagonal in the prune_counts_matrix (see filter.py).\n",
    "    * `'prune_by_class'`: works by removing the examples with *smallest probability* of belonging to their given class label for every class.\n",
    "    * `'both'`: Finds the examples satisfying (1) AND (2) and removes their set conjunction. \n",
    "\n",
    "\n",
    "2. converge_latent_estimates : bool (Default: False)\n",
    "    * If true, forces numerical consistency of latent estimates. Each is estimated independently, but they are related mathematically with closed form  equivalences. This will iteratively enforce mathematically consistency."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61b288e9",
   "metadata": {},
   "source": [
    "Please install the dependencies specified in this [requirements.txt](https://github.com/cleanlab/examples/blob/master/hyperparameter_optimization/requirements.txt) file before running the notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "89454c70",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:41:08.012229Z",
     "iopub.status.busy": "2022-09-08T14:41:08.011971Z",
     "iopub.status.idle": "2022-09-08T14:41:08.404151Z",
     "shell.execute_reply": "2022-09-08T14:41:08.403829Z"
    },
    "papermill": {
     "duration": 0.396508,
     "end_time": "2022-09-08T14:41:08.405291",
     "exception": false,
     "start_time": "2022-09-08T14:41:08.008783",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from cleanlab.classification import CleanLearning\n",
    "from cleanlab.benchmarking.noise_generation import generate_noise_matrix_from_trace\n",
    "from cleanlab.benchmarking.noise_generation import generate_noisy_labels\n",
    "from cleanlab.internal.util import print_noise_matrix\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import numpy as np\n",
    "import copy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "96441cf1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:41:08.407738Z",
     "iopub.status.busy": "2022-09-08T14:41:08.407582Z",
     "iopub.status.idle": "2022-09-08T14:41:08.409865Z",
     "shell.execute_reply": "2022-09-08T14:41:08.409604Z"
    },
    "papermill": {
     "duration": 0.004526,
     "end_time": "2022-09-08T14:41:08.410823",
     "exception": false,
     "start_time": "2022-09-08T14:41:08.406297",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def make_linear_dataset(n_classes=3, n_samples=300):\n",
    "    X, y = make_classification(\n",
    "        n_samples=n_samples,\n",
    "        n_features=2,\n",
    "        n_redundant=0,\n",
    "        n_informative=2,\n",
    "        random_state=1,\n",
    "        n_clusters_per_class=1,\n",
    "        n_classes=n_classes,\n",
    "    )\n",
    "    rng = np.random.RandomState(2)\n",
    "    X += 2 * rng.uniform(size=X.shape)\n",
    "    return (X, y)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b3d700da",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:41:08.412920Z",
     "iopub.status.busy": "2022-09-08T14:41:08.412821Z",
     "iopub.status.idle": "2022-09-08T14:41:08.414566Z",
     "shell.execute_reply": "2022-09-08T14:41:08.414350Z"
    },
    "papermill": {
     "duration": 0.003726,
     "end_time": "2022-09-08T14:41:08.415374",
     "exception": false,
     "start_time": "2022-09-08T14:41:08.411648",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# hyper-parameters\n",
    "param_grid = {\n",
    "    \"find_label_issues_kwargs\": [\n",
    "        {\"filter_by\": \"prune_by_noise_rate\"},\n",
    "        {\"filter_by\": \"prune_by_class\"},\n",
    "        {\"filter_by\": \"both\"},\n",
    "    ],\n",
    "    \"converge_latent_estimates\": [True, False],\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "21da0eb4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:41:08.417611Z",
     "iopub.status.busy": "2022-09-08T14:41:08.417434Z",
     "iopub.status.idle": "2022-09-08T14:41:08.420512Z",
     "shell.execute_reply": "2022-09-08T14:41:08.420296Z"
    },
    "papermill": {
     "duration": 0.005187,
     "end_time": "2022-09-08T14:41:08.421331",
     "exception": false,
     "start_time": "2022-09-08T14:41:08.416144",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Set the sparsity of the noise matrix.\n",
    "frac_zero_noise_rates = 0.0  # Consider increasing to 0.5\n",
    "# A proxy for the fraction of labels that are correct.\n",
    "avg_trace = 0.65  # ~35% wrong labels. Increasing makes the problem easier.\n",
    "# Amount of data for each dataset.\n",
    "dataset_size = 250  # Try 250 or 400 to use less or more data.\n",
    "num_classes = 3\n",
    "\n",
    "ds = make_linear_dataset(n_classes=num_classes, n_samples=num_classes * dataset_size)\n",
    "X, y = ds\n",
    "X = StandardScaler().fit_transform(X)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=0)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae3518f1-196a-45de-af76-0bfe9b91437b",
   "metadata": {
    "papermill": {
     "duration": 0.000769,
     "end_time": "2022-09-08T14:41:08.422884",
     "exception": false,
     "start_time": "2022-09-08T14:41:08.422115",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## Run hyper-parameter search with sklearn GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9f6a1a63",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:41:08.424794Z",
     "iopub.status.busy": "2022-09-08T14:41:08.424698Z",
     "iopub.status.idle": "2022-09-08T14:41:47.958680Z",
     "shell.execute_reply": "2022-09-08T14:41:47.958320Z"
    },
    "papermill": {
     "duration": 39.53664,
     "end_time": "2022-09-08T14:41:47.960252",
     "exception": false,
     "start_time": "2022-09-08T14:41:08.423612",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " =========== \n",
      " Naive Bayes \n",
      " ===========\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(given_label|true_label) of shape (3, 3)\n",
      " p(s|y)\ty=0\ty=1\ty=2\n",
      "\t---\t---\t---\n",
      "s=0 |\t0.52\t0.1\t0.34\n",
      "s=1 |\t0.2\t0.82\t0.05\n",
      "s=2 |\t0.28\t0.07\t0.61\n",
      "\tTrace(matrix) = 1.95\n",
      "\n",
      "Accuracy with default parameters: 0.65\n",
      "Accuracy with optimized parameters: 0.7\n",
      "\n",
      "Optimal parameter settings using Naive Bayes\n",
      "--------------------------------------------\n",
      "cv : None\n",
      "error_score : nan\n",
      "estimator__clf__priors : None\n",
      "estimator__clf__var_smoothing : 1e-09\n",
      "estimator__clf : GaussianNB()\n",
      "estimator__converge_latent_estimates : False\n",
      "estimator__cv_n_folds : 5\n",
      "estimator__find_label_issues_kwargs : {}\n",
      "estimator__label_quality_scores_kwargs : {}\n",
      "estimator__pulearning : None\n",
      "estimator__seed : None\n",
      "estimator__verbose : False\n",
      "estimator : CleanLearning(clf=GaussianNB())\n",
      "n_jobs : None\n",
      "param_grid : {'find_label_issues_kwargs': [{'filter_by': 'prune_by_noise_rate'}, {'filter_by': 'prune_by_class'}, {'filter_by': 'both'}], 'converge_latent_estimates': [True, False]}\n",
      "pre_dispatch : 2*n_jobs\n",
      "refit : True\n",
      "return_train_score : False\n",
      "scoring : None\n",
      "verbose : 0\n",
      "\n",
      " =================== \n",
      " Logistic Regression \n",
      " ===================\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(given_label|true_label) of shape (3, 3)\n",
      " p(s|y)\ty=0\ty=1\ty=2\n",
      "\t---\t---\t---\n",
      "s=0 |\t0.52\t0.1\t0.34\n",
      "s=1 |\t0.2\t0.82\t0.05\n",
      "s=2 |\t0.28\t0.07\t0.61\n",
      "\tTrace(matrix) = 1.95\n",
      "\n",
      "Accuracy with default parameters: 0.65\n",
      "Accuracy with optimized parameters: 0.71\n",
      "\n",
      "Optimal parameter settings using Logistic Regression\n",
      "----------------------------------------------------\n",
      "cv : None\n",
      "error_score : nan\n",
      "estimator__clf__C : 1.0\n",
      "estimator__clf__class_weight : None\n",
      "estimator__clf__dual : False\n",
      "estimator__clf__fit_intercept : True\n",
      "estimator__clf__intercept_scaling : 1\n",
      "estimator__clf__l1_ratio : None\n",
      "estimator__clf__max_iter : 100\n",
      "estimator__clf__multi_class : auto\n",
      "estimator__clf__n_jobs : None\n",
      "estimator__clf__penalty : l2\n",
      "estimator__clf__random_state : 0\n",
      "estimator__clf__solver : lbfgs\n",
      "estimator__clf__tol : 0.0001\n",
      "estimator__clf__verbose : 0\n",
      "estimator__clf__warm_start : False\n",
      "estimator__clf : LogisticRegression(random_state=0)\n",
      "estimator__converge_latent_estimates : False\n",
      "estimator__cv_n_folds : 5\n",
      "estimator__find_label_issues_kwargs : {}\n",
      "estimator__label_quality_scores_kwargs : {}\n",
      "estimator__pulearning : None\n",
      "estimator__seed : None\n",
      "estimator__verbose : False\n",
      "estimator : CleanLearning(clf=LogisticRegression(random_state=0))\n",
      "n_jobs : None\n",
      "param_grid : {'find_label_issues_kwargs': [{'filter_by': 'prune_by_noise_rate'}, {'filter_by': 'prune_by_class'}, {'filter_by': 'both'}], 'converge_latent_estimates': [True, False]}\n",
      "pre_dispatch : 2*n_jobs\n",
      "refit : True\n",
      "return_train_score : False\n",
      "scoring : None\n",
      "verbose : 0\n"
     ]
    }
   ],
   "source": [
    "for name, clf in [\n",
    "    (\n",
    "        \"Naive Bayes\",\n",
    "        GaussianNB(),\n",
    "    ),\n",
    "    (\n",
    "        \"Logistic Regression\",\n",
    "        LogisticRegression(random_state=0, solver=\"lbfgs\", multi_class=\"auto\"),\n",
    "    ),\n",
    "]:\n",
    "    print(\"\\n\", \"=\" * len(name), \"\\n\", name, \"\\n\", \"=\" * len(name))\n",
    "    np.random.seed(seed=0)\n",
    "    clf_copy = copy.deepcopy(clf)\n",
    "    # Compute p(y=k), the ground truth class prior on the labels.\n",
    "    py = np.bincount(y_train) / float(len(y_train))\n",
    "    # Generate the noisy channel to characterize the label errors.\n",
    "    noise_matrix = generate_noise_matrix_from_trace(\n",
    "        K=num_classes,\n",
    "        trace=num_classes * avg_trace,\n",
    "        py=py,\n",
    "        frac_zero_noise_rates=frac_zero_noise_rates,\n",
    "    )\n",
    "    print_noise_matrix(noise_matrix)\n",
    "\n",
    "    # Create the noisy labels. This method is exact w.r.t. the noise_matrix.\n",
    "    y_train_with_errors = generate_noisy_labels(y_train, noise_matrix)\n",
    "\n",
    "    # Run GridSearch with Cross-Validation\n",
    "    lnl_cv = GridSearchCV(\n",
    "        estimator=CleanLearning(clf, verbose=False),\n",
    "        param_grid=param_grid,\n",
    "    )\n",
    "    lnl_cv.fit(X=X_train, y=y_train_with_errors)\n",
    "\n",
    "    # Also compute the test score with default parameters\n",
    "    clf_copy.fit(X_train, y_train_with_errors)\n",
    "    score_opt = lnl_cv.score(X_test, y_test)\n",
    "    score_default = clf_copy.score(X_test, y_test)\n",
    "    print(\"Accuracy with default parameters:\", np.round(score_default, 2))\n",
    "    print(\"Accuracy with optimized parameters:\", np.round(score_opt, 2))\n",
    "    print()\n",
    "    s = \"Optimal parameter settings using {}\".format(name)\n",
    "    print(s)\n",
    "    print(\"-\" * len(s))\n",
    "    for key in lnl_cv.get_params().keys():\n",
    "        print(key, \":\", lnl_cv.get_params()[key])\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 43.15714,
   "end_time": "2022-09-08T14:41:50.581548",
   "environment_variables": {},
   "exception": null,
   "input_path": "./3_hyperparameter_optimization/hyperparameter_optimization.ipynb",
   "output_path": "./3_hyperparameter_optimization/hyperparameter_optimization.ipynb",
   "parameters": {},
   "start_time": "2022-09-08T14:41:07.424408",
   "version": "2.3.4"
  },
  "vscode": {
   "interpreter": {
    "hash": "00885e89789f58e60dbba52a405dc834aaf92411914fde0d391f9b48289a0610"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}