visualizing_confident_learning/visualizing_confident_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ff3d320c",
   "metadata": {
    "papermill": {
     "duration": 0.008428,
     "end_time": "2022-09-08T14:01:03.837284",
     "exception": false,
     "start_time": "2022-09-08T14:01:03.828856",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Visualizing Confident Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af5d86d5",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/cleanlab/examples/blob/master/visualizing_confident_learning/visualizing_confident_learning.ipynb)     "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "117c4fc1",
   "metadata": {},
   "source": [
    "In this example we can see how `cleanlab` estimates parameters of the label error distribution (noise matrix)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "92cbd279",
   "metadata": {},
   "source": [
    "Please install the dependencies specified in this [requirements.txt](https://github.com/cleanlab/examples/blob/master/visualizing_confident_learning/requirements.txt) file before running the notebook.    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "19b26df6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:01:03.850098Z",
     "iopub.status.busy": "2022-09-08T14:01:03.849356Z",
     "iopub.status.idle": "2022-09-08T14:01:04.270477Z",
     "shell.execute_reply": "2022-09-08T14:01:04.270072Z"
    },
    "papermill": {
     "duration": 0.42798,
     "end_time": "2022-09-08T14:01:04.271719",
     "exception": false,
     "start_time": "2022-09-08T14:01:03.843739",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from numpy.random import multivariate_normal\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "import numpy as np\n",
    "\n",
    "import cleanlab\n",
    "from cleanlab.benchmarking.noise_generation import (\n",
    "    generate_noise_matrix_from_trace,\n",
    "    generate_noisy_labels,\n",
    ")\n",
    "from cleanlab.internal.util import print_noise_matrix\n",
    "from cleanlab.count import estimate_confident_joint_and_cv_pred_proba, estimate_latent\n",
    "from cleanlab.filter import find_label_issues\n",
    "from cleanlab.classification import CleanLearning\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4858a4b8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:01:04.274595Z",
     "iopub.status.busy": "2022-09-08T14:01:04.274435Z",
     "iopub.status.idle": "2022-09-08T14:01:04.279195Z",
     "shell.execute_reply": "2022-09-08T14:01:04.278893Z"
    },
    "papermill": {
     "duration": 0.007189,
     "end_time": "2022-09-08T14:01:04.280129",
     "exception": false,
     "start_time": "2022-09-08T14:01:04.272940",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def show_decision_boundary(clf, title, show_noise=True):\n",
    "    # Plot the decision boundary. For that, we will assign a color to each\n",
    "    # point in the mesh [x_min, x_max]x[y_min, y_max].\n",
    "    h = 0.01  # step size in the mesh\n",
    "    x_min, x_max = X_train[:, 0].min() - 0.5, X_train[:, 0].max() + 0.5\n",
    "    y_min, y_max = X_train[:, 1].min() - 0.5, X_train[:, 1].max() + 0.5\n",
    "    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))\n",
    "    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])\n",
    "\n",
    "    _ = plt.figure(figsize=(15, 12))\n",
    "    plt.axis(\"off\")\n",
    "\n",
    "    # Put the result into a color plot\n",
    "    Z = Z.reshape(xx.shape)\n",
    "    plt.pcolormesh(xx, yy, Z, alpha=0.015)\n",
    "\n",
    "    # Plot the distribution for viewing.\n",
    "    if show_noise:\n",
    "        # Plot noisy labels are circles around label errors\n",
    "        for k in range(K):\n",
    "            X_k = X_train[y_train == k]  # data for class k\n",
    "            _ = plt.scatter(\n",
    "                X_k[:, 0],\n",
    "                X_k[:, 1],\n",
    "                color=[color_map[noisy_k] for noisy_k in labels[y_train == k]],\n",
    "                s=150,\n",
    "                marker=r\"${a}$\".format(a=str(k)),\n",
    "                linewidth=1,\n",
    "            )\n",
    "        _ = plt.scatter(\n",
    "            X_train[:, 0][labels != y_train],\n",
    "            X_train[:, 1][labels != y_train],\n",
    "            s=400,\n",
    "            facecolors=\"none\",\n",
    "            edgecolors=\"black\",\n",
    "            linewidth=0.8,\n",
    "        )\n",
    "    else:\n",
    "        # Plot the actual labels.\n",
    "        for k in range(K):\n",
    "            X_k = X_train[y_train == k]  # data for class k\n",
    "            plt.scatter(\n",
    "                X_k[:, 0],\n",
    "                X_k[:, 1],\n",
    "                color=colors[k],\n",
    "                s=150,\n",
    "                marker=r\"${a}$\".format(a=str(k)),\n",
    "                linewidth=1,\n",
    "            )\n",
    "    plt.title(title, fontsize=25)\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a614e513",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:01:04.282701Z",
     "iopub.status.busy": "2022-09-08T14:01:04.282590Z",
     "iopub.status.idle": "2022-09-08T14:01:04.867249Z",
     "shell.execute_reply": "2022-09-08T14:01:04.866713Z"
    },
    "papermill": {
     "duration": 0.587304,
     "end_time": "2022-09-08T14:01:04.868437",
     "exception": false,
     "start_time": "2022-09-08T14:01:04.281133",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "seed = (\n",
    "    1  # Seeded for reproducibility - remove to created random noise and distributions.\n",
    ")\n",
    "np.random.seed(seed=seed)\n",
    "\n",
    "means = [[3, 2], [7, 7], [0, 8]]\n",
    "covs = [[[5, -1.5], [-1.5, 1]], [[1, 0.5], [0.5, 4]], [[5, 1], [1, 5]]]\n",
    "\n",
    "K = len(means)  # number of classes\n",
    "sizes = [800, 400, 400]\n",
    "data = []\n",
    "labels = []\n",
    "test_data = []\n",
    "test_labels = []\n",
    "\n",
    "for idx in range(len(means)):\n",
    "    data.append(multivariate_normal(mean=means[idx], cov=covs[idx], size=sizes[idx]))\n",
    "    test_data.append(\n",
    "        multivariate_normal(mean=means[idx], cov=covs[idx], size=sizes[idx])\n",
    "    )\n",
    "    labels.append(np.array([idx for i in range(sizes[idx])]))\n",
    "    test_labels.append(np.array([idx for i in range(sizes[idx])]))\n",
    "X_train = np.vstack(data)\n",
    "y_train = np.hstack(labels)\n",
    "X_test = np.vstack(test_data)\n",
    "y_test = np.hstack(test_labels)\n",
    "\n",
    "# Compute p(y=k)\n",
    "py = np.bincount(y_train) / float(len(y_train))\n",
    "\n",
    "noise_matrix = generate_noise_matrix_from_trace(\n",
    "    K,\n",
    "    trace=1.5,\n",
    "    py=py,\n",
    "    valid_noise_matrix=True,\n",
    ")\n",
    "\n",
    "# Generate our noisy labels using the noise_marix.\n",
    "labels = generate_noisy_labels(y_train, noise_matrix)\n",
    "ps = np.bincount(labels) / float(len(labels))\n",
    "\n",
    "confident_joint, pred_probs = estimate_confident_joint_and_cv_pred_proba(X_train, labels, seed=seed)\n",
    "est_py, est_noise_matrix, est_inverse_noise_matrix = estimate_latent(confident_joint, labels)\n",
    "idx_errors = find_label_issues(labels=labels, pred_probs=pred_probs)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "778a90b8",
   "metadata": {
    "papermill": {
     "duration": 0.001018,
     "end_time": "2022-09-08T14:01:04.870717",
     "exception": false,
     "start_time": "2022-09-08T14:01:04.869699",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "#### To show off the power of **cleanlab**, we've chosen an example of multiclass learning with noisy labels in which over 50% of the training labels are wrong.\n",
    "Toggle the ```trace``` parameter in ```generate_noise_matrix_from_trace``` above to try out different amounts of noise. Note, as we prove in our paper, learning becomes impossible if the ```trace <= 1```, so choose a value greater than 1, but less than, or equal to, the number of classes (3)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "47545a51",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:01:04.873370Z",
     "iopub.status.busy": "2022-09-08T14:01:04.873154Z",
     "iopub.status.idle": "2022-09-08T14:01:05.422627Z",
     "shell.execute_reply": "2022-09-08T14:01:05.422269Z"
    },
    "papermill": {
     "duration": 0.552024,
     "end_time": "2022-09-08T14:01:05.423659",
     "exception": false,
     "start_time": "2022-09-08T14:01:04.871635",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Plotting is only supported in an iPython interface.\n",
      "The actual, latent, underlying noise matrix.\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(given_label|true_label) of shape (3, 3)\n",
      " p(s|y)\ty=0\ty=1\ty=2\n",
      "\t---\t---\t---\n",
      "s=0 |\t0.4\t0.21\t0.47\n",
      "s=1 |\t0.25\t0.63\t0.07\n",
      "s=2 |\t0.35\t0.15\t0.47\n",
      "\tTrace(matrix) = 1.5\n",
      "\n",
      "Our estimate of the noise matrix.\n",
      "\n",
      " Noise Matrix (aka Noisy Channel) P(given_label|true_label) of shape (3, 3)\n",
      " p(s|y)\ty=0\ty=1\ty=2\n",
      "\t---\t---\t---\n",
      "s=0 |\t0.42\t0.27\t0.49\n",
      "s=1 |\t0.19\t0.52\t0.06\n",
      "s=2 |\t0.39\t0.21\t0.45\n",
      "\tTrace(matrix) = 1.39\n",
      "\n",
      "\n",
      "The actual, latent, underlying joint distribution matrix.\n",
      "\n",
      " Joint Label Noise Distribution Matrix P(given_label, true_label) of shape (3, 3)\n",
      " p(s,y)\ty=0\ty=1\ty=2\n",
      "\t---\t---\t---\n",
      "s=0 |\t0.2\t0.05\t0.12\n",
      "s=1 |\t0.12\t0.16\t0.02\n",
      "s=2 |\t0.18\t0.04\t0.12\n",
      "\tTrace(matrix) = 0.48\n",
      "\n",
      "Our estimate of the joint distribution matrix.\n",
      "\n",
      " Joint Label Noise Distribution Matrix P(given_label, true_label) of shape (3, 3)\n",
      " p(s,y)\ty=0\ty=1\ty=2\n",
      "\t---\t---\t---\n",
      "s=0 |\t0.23\t0.1\t0.04\n",
      "s=1 |\t0.11\t0.19\t0.0\n",
      "s=2 |\t0.21\t0.08\t0.04\n",
      "\tTrace(matrix) = 0.46\n",
      "\n",
      "Accuracy Comparison\n",
      "-------------------\n",
      "Logistic regression: 0.69875\n",
      "Logistic regression (+rankpruning): 0.75625\n",
      "Fit on denoised data without re-weighting: 0.76\n",
      "Plotting is only supported in an iPython interface.\n"
     ]
    }
   ],
   "source": [
    "est_joint = cleanlab.count.estimate_joint(\n",
    "    labels=labels,\n",
    "    pred_probs=pred_probs,\n",
    "    confident_joint=confident_joint, \n",
    ")\n",
    "true_joint_distribution_of_label_errors = (noise_matrix * py)\n",
    "percent_error_str = 'Percent of training examples that have wrong labels: ' + \\\n",
    "      str(int(round(100 - 100*true_joint_distribution_of_label_errors.trace()))) + \"%\"\n",
    "\n",
    "colors = [(31 / 255., 119 / 255., 180 / 255.), (255 / 255., 127 / 255., 14 / 255.), (44 / 255., 160 / 255., 44 / 255.)]\n",
    "color_map = dict(zip(range(len(colors)), colors))\n",
    "try:\n",
    "# Plot the distribution for your viewing.\n",
    "    % matplotlib inline\n",
    "    from matplotlib import pyplot as plt\n",
    "    _ = plt.figure(figsize=(15, 12))\n",
    "    _ = plt.axis('off')\n",
    "    for k in range(K):\n",
    "        X_k = X_train[y_train==k] # data for class k\n",
    "        _ = plt.scatter(X_k[:,0], X_k[:, 1], color=colors[k], s=150, marker=r\"${a}$\".format(a=str(k)), linewidth=1)\n",
    "    _ = plt.title(\"Original (unobserved) distribution, without any label errors.\", fontsize=30)\n",
    "\n",
    "    print(\"\\n\\n\\n\\n\")\n",
    "\n",
    "    # Plot the noisy distribution for viewing.\n",
    "    _ = plt.figure(figsize=(15, 12))\n",
    "    _ = plt.axis('off')\n",
    "    for k in range(K):\n",
    "        X_k = X_train[y_train == k] # data for class k\n",
    "        _ = plt.scatter(X_k[:,0], X_k[:, 1], color=[color_map[noisy_k] for noisy_k in labels[y_train==k]], s=150, marker=r\"${a}$\".format(a=str(k)), linewidth=1)\n",
    "    _ = plt.scatter(X_train[:,0][labels != y_train], X_train[:,1][labels != y_train], s=400, facecolors='none', edgecolors='black', linewidth=2, alpha = 0.5)\n",
    "    _ = plt.title('Observed distribution, with label errors circled.\\nColors are the given labels, the numbers are latent.\\n'+percent_error_str, fontsize=30)\n",
    "    plt.show()\n",
    "\n",
    "    print(\"\\n\\n\\n\\n\")\n",
    "\n",
    "    # Plot the noisy distribution for viewing.\n",
    "    _ = plt.figure(figsize=(15, 12))\n",
    "    _ = plt.axis('off')\n",
    "    for k in range(K):\n",
    "        X_k = X_train[idx_errors & (y_train == k)] # data for class k\n",
    "        _ = plt.scatter(X_k[:,0], X_k[:, 1], color=[color_map[noisy_k] for noisy_k in labels[y_train==k]], s=150, marker=r\"${a}$\".format(a=str(k)), linewidth=1)\n",
    "    _ = plt.scatter(X_train[:,0][labels != y_train], X_train[:,1][labels != y_train], s=400, facecolors='none', edgecolors='black', linewidth=2, alpha = 0.5)\n",
    "    _ = plt.title('Label errors detected using confident learning.\\nEmpty circles show undetected label errors.\\nUncircled data depicts false positives.', fontsize=30)\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "    print(\"\\n\\n\\n\\n\")\n",
    "\n",
    "    _ = plt.figure(figsize=(15, 12))\n",
    "    _ = plt.axis('off')\n",
    "    for k in range(K):\n",
    "        X_k = X_train[~idx_errors & (y_train == k)] # data for class k\n",
    "        _ = plt.scatter(X_k[:,0], X_k[:, 1], color=[color_map[noisy_k] for noisy_k in labels[y_train==k]], s=150, marker=r\"${a}$\".format(a=str(k)), linewidth=1)\n",
    "    _ = plt.scatter(X_train[~idx_errors][:,0][labels[~idx_errors] != y_train[~idx_errors]], X_train[~idx_errors][:,1][labels[~idx_errors] != y_train[~idx_errors]], s=400, facecolors='none', edgecolors='black', linewidth=2, alpha = 0.5)\n",
    "    _ = plt.title('Dataset after pruning detected label errors.', fontsize=30)\n",
    "    plt.show()\n",
    "except:\n",
    "    print(\"Plotting is only supported in an iPython interface.\")\n",
    "\n",
    "print('The actual, latent, underlying noise matrix.')\n",
    "print_noise_matrix(noise_matrix)\n",
    "print('Our estimate of the noise matrix.')\n",
    "print_noise_matrix(est_noise_matrix)\n",
    "print()\n",
    "print('The actual, latent, underlying joint distribution matrix.')\n",
    "cleanlab.internal.util.print_joint_matrix(true_joint_distribution_of_label_errors)\n",
    "print('Our estimate of the joint distribution matrix.')\n",
    "cleanlab.internal.util.print_joint_matrix(est_joint)\n",
    "print(\"Accuracy Comparison\")\n",
    "print(\"-------------------\")\n",
    "clf = LogisticRegression(solver = 'lbfgs', multi_class = 'auto')\n",
    "baseline_score = accuracy_score(y_test, clf.fit(X_train, labels).predict(X_test))\n",
    "print(\"Logistic regression:\", baseline_score)\n",
    "cl = CleanLearning(seed=seed, verbose=False)\n",
    "cl_score = accuracy_score(y_test, cl.fit(X_train, labels, pred_probs=pred_probs).predict(X_test))\n",
    "print(\"Logistic regression (+rankpruning):\", cl_score)\n",
    "diff = cl_score - baseline_score\n",
    "clf = LogisticRegression(solver = 'lbfgs', multi_class = 'auto')\n",
    "print('Fit on denoised data without re-weighting:', accuracy_score(y_test, clf.fit(X_train[~idx_errors], labels[~idx_errors]).predict(X_test)))\n",
    "\n",
    "\n",
    "\n",
    "try:\n",
    "    % matplotlib inline\n",
    "    from matplotlib import pyplot as plt\n",
    "    \n",
    "    print(\"\\n\\n\\n\\n\\n\\n\")\n",
    "    \n",
    "    clf = LogisticRegression(solver = 'lbfgs', multi_class = 'auto')\n",
    "    _ = clf.fit(X_train, labels)\n",
    "    show_decision_boundary(clf, 'Decision boundary for logistic regression trained with noisy labels.\\n Test Accuracy: ' + str(round(baseline_score, 3)))\n",
    "\n",
    "    _ = clf.fit(X_train, y_train)\n",
    "    max_score = accuracy_score(y_test, clf.predict(X_test))\n",
    "    show_decision_boundary(clf, 'Decision boundary for logistic regression trained with no label errors.\\n Test Accuracy: ' + str(round(max_score, 3)), show_noise = False)\n",
    "\n",
    "    show_decision_boundary(cl.clf, 'Decision boundary for LogisticRegression (+rankpruning) trained with noisy labels.\\n Test Accuracy: ' + str(round(cl_score, 3)))\n",
    "except:\n",
    "    print(\"Plotting is only supported in an iPython interface.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "146e088d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:01:05.426717Z",
     "iopub.status.busy": "2022-09-08T14:01:05.426590Z",
     "iopub.status.idle": "2022-09-08T14:01:11.458170Z",
     "shell.execute_reply": "2022-09-08T14:01:11.457811Z"
    },
    "papermill": {
     "duration": 6.034239,
     "end_time": "2022-09-08T14:01:11.459224",
     "exception": false,
     "start_time": "2022-09-08T14:01:05.424985",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Param settings: {'find_label_issues_kwargs': {'filter_by': 'prune_by_noise_rate'}, 'converge_latent_estimates': True}\n",
      "Accuracy (using confident learning):\t 0.76 \n",
      "\n",
      "The actual, latent, underlying noise matrix:\n",
      " Noise Matrix (aka Noisy Channel) P(given_label|true_label) of shape (3, 3)\n",
      " p(s|y)\ty=0\ty=1\ty=2\n",
      "\t---\t---\t---\n",
      "s=0 |\t0.4\t0.21\t0.47\n",
      "s=1 |\t0.25\t0.63\t0.07\n",
      "s=2 |\t0.35\t0.15\t0.47\n",
      "\tTrace(matrix) = 1.5\n",
      "\n",
      "CleanLearning best estimate of the noise matrix:\n",
      " Noise Matrix (aka Noisy Channel) P(given_label|true_label) of shape (3, 3)\n",
      " p(s|y)\ty=0\ty=1\ty=2\n",
      "\t---\t---\t---\n",
      "s=0 |\t0.41\t0.27\t0.54\n",
      "s=1 |\t0.19\t0.52\t0.04\n",
      "s=2 |\t0.4\t0.21\t0.41\n",
      "\tTrace(matrix) = 1.34\n",
      "\n",
      "Param settings: {'find_label_issues_kwargs': {'filter_by': 'prune_by_noise_rate'}, 'converge_latent_estimates': False}\n",
      "Accuracy (using confident learning):\t 0.75 \n",
      "\n",
      "Param settings: {'find_label_issues_kwargs': {'filter_by': 'both'}, 'converge_latent_estimates': False}\n",
      "Accuracy (using confident learning):\t 0.71 \n",
      "\n",
      "Param settings: {'find_label_issues_kwargs': {'filter_by': 'both'}, 'converge_latent_estimates': True}\n",
      "Accuracy (using confident learning):\t 0.71 \n",
      "\n",
      "Param settings: {'find_label_issues_kwargs': {'filter_by': 'prune_by_class'}, 'converge_latent_estimates': False}\n",
      "Accuracy (using confident learning):\t 0.67 \n",
      "\n",
      "Param settings: {'find_label_issues_kwargs': {'filter_by': 'prune_by_class'}, 'converge_latent_estimates': True}\n",
      "Accuracy (using confident learning):\t 0.67 \n",
      "\n",
      "The actual, latent, underlying noise matrix:\n",
      " Noise Matrix (aka Noisy Channel) P(given_label|true_label) of shape (3, 3)\n",
      " p(s|y)\ty=0\ty=1\ty=2\n",
      "\t---\t---\t---\n",
      "s=0 |\t0.4\t0.21\t0.47\n",
      "s=1 |\t0.25\t0.63\t0.07\n",
      "s=2 |\t0.35\t0.15\t0.47\n",
      "\tTrace(matrix) = 1.5\n",
      "\n",
      "CleanLearning best estimate of the noise matrix:\n",
      " Noise Matrix (aka Noisy Channel) P(given_label|true_label) of shape (3, 3)\n",
      " p(s|y)\ty=0\ty=1\ty=2\n",
      "\t---\t---\t---\n",
      "s=0 |\t0.43\t0.27\t0.44\n",
      "s=1 |\t0.19\t0.52\t0.06\n",
      "s=2 |\t0.38\t0.22\t0.5\n",
      "\tTrace(matrix) = 1.45\n",
      "\n"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "    \"find_label_issues_kwargs\": [\n",
    "        {\"filter_by\": \"prune_by_noise_rate\"},\n",
    "        {\"filter_by\": \"prune_by_class\"},\n",
    "        {\"filter_by\": \"both\"},\n",
    "    ],\n",
    "    \"converge_latent_estimates\": [True, False],\n",
    "}\n",
    "\n",
    "# Fit CleanLearning across all parameter settings.\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "\n",
    "params = ParameterGrid(param_grid)\n",
    "scores = []\n",
    "for param in params:\n",
    "    clf = LogisticRegression(solver=\"lbfgs\", multi_class=\"auto\")\n",
    "    cl = CleanLearning(clf=clf, verbose=False, **param)\n",
    "    _ = cl.fit(X_train, labels)  # labels is the noisy y_train labels\n",
    "    scores.append(accuracy_score(cl.predict(X_test), y_test))\n",
    "\n",
    "# Print results sorted from best to least\n",
    "for i in np.argsort(scores)[::-1]:\n",
    "    print(\"Param settings:\", params[i])\n",
    "    print(\"Accuracy (using confident learning):\\t\", round(scores[i], 2), \"\\n\")\n",
    "\n",
    "    # Print noise matrix for highest/lowest scoring models\n",
    "    if i == np.argmax(scores) or i == np.argmin(scores):\n",
    "        # Retrain with best parameters and show noise matrix estimation\n",
    "        clf = LogisticRegression(solver=\"lbfgs\", multi_class=\"auto\")\n",
    "        cl = CleanLearning(clf=clf, verbose=False, **param)\n",
    "        _ = cl.fit(X_train, labels)  # labels is the noisy y_train labels\n",
    "        print(\"The actual, latent, underlying noise matrix:\", end=\"\")\n",
    "        print_noise_matrix(noise_matrix)\n",
    "        print(\"CleanLearning best estimate of the noise matrix:\", end=\"\")\n",
    "        print_noise_matrix(cl.noise_matrix)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74d653b5",
   "metadata": {
    "papermill": {
     "duration": 0.001171,
     "end_time": "2022-09-08T14:01:11.461925",
     "exception": false,
     "start_time": "2022-09-08T14:01:11.460754",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "### In the above example, notice the robustness to hyper-parameter choice and the stability of the algorithms across parameters. No setting of parameters dramatically affects the results. In fact, in certain non-trivial cases, we can prove that certain settings of parameters are equivalent.\n",
    "\n",
    "### In summary, the default setting of parameters tends to work well, but optimize across their settings freely."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a1a974b4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-09-08T14:01:11.464688Z",
     "iopub.status.busy": "2022-09-08T14:01:11.464557Z",
     "iopub.status.idle": "2022-09-08T14:01:11.467286Z",
     "shell.execute_reply": "2022-09-08T14:01:11.467006Z"
    },
    "papermill": {
     "duration": 0.005106,
     "end_time": "2022-09-08T14:01:11.468086",
     "exception": false,
     "start_time": "2022-09-08T14:01:11.462980",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.231875 0.095625 0.0425  ]\n",
      " [0.1075   0.186875 0.005   ]\n",
      " [0.214375 0.076875 0.039375]]\n",
      "\n",
      "The above output should look like this:\n",
      "[[0.231875 0.095625 0.0425  ]\n",
      " [0.1075   0.186875 0.005   ]\n",
      " [0.214375 0.076875 0.039375]]\n"
     ]
    }
   ],
   "source": [
    "joint = cleanlab.count.estimate_joint(\n",
    "    labels=labels, pred_probs=pred_probs, confident_joint=confident_joint\n",
    ")\n",
    "print(joint)\n",
    "\n",
    "print(\"\\nThe above output should look like this:\")\n",
    "print(confident_joint / len(labels))\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 10.944413,
   "end_time": "2022-09-08T14:01:14.090055",
   "environment_variables": {},
   "exception": null,
   "input_path": "./5_visualizing_confident_learning/visualizing_confident_learning.ipynb",
   "output_path": "./5_visualizing_confident_learning/visualizing_confident_learning.ipynb",
   "parameters": {},
   "start_time": "2022-09-08T14:01:03.145642",
   "version": "2.3.4"
  },
  "vscode": {
   "interpreter": {
    "hash": "00885e89789f58e60dbba52a405dc834aaf92411914fde0d391f9b48289a0610"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}