modeling_with_probs.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "modeling_with_probs.ipynb",
      "provenance": [],
      "toc_visible": true,
      "collapsed_sections": [
        "2ChbdIRvlihf"
      ],
      "authorship_tag": "ABX9TyO5MOBg8GW2ZvibCjJUPwwk",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/dinbav/SpreadingKnowledge/blob/master/modeling_with_probs.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##installs"
      ],
      "metadata": {
        "id": "2ChbdIRvlihf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install CatBoost"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AN3u8du8liOd",
        "outputId": "8e186cb8-19bf-4683-8934-ae7723ac0e77"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: CatBoost in /usr/local/lib/python3.7/dist-packages (1.0.4)\n",
            "Requirement already satisfied: pandas>=0.24.0 in /usr/local/lib/python3.7/dist-packages (from CatBoost) (1.3.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from CatBoost) (1.15.0)\n",
            "Requirement already satisfied: numpy>=1.16.0 in /usr/local/lib/python3.7/dist-packages (from CatBoost) (1.21.5)\n",
            "Requirement already satisfied: plotly in /usr/local/lib/python3.7/dist-packages (from CatBoost) (5.5.0)\n",
            "Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packages (from CatBoost) (3.2.2)\n",
            "Requirement already satisfied: graphviz in /usr/local/lib/python3.7/dist-packages (from CatBoost) (0.10.1)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from CatBoost) (1.4.1)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.24.0->CatBoost) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.24.0->CatBoost) (2018.9)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->CatBoost) (1.4.0)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib->CatBoost) (0.11.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib->CatBoost) (3.0.7)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from kiwisolver>=1.0.1->matplotlib->CatBoost) (3.10.0.2)\n",
            "Requirement already satisfied: tenacity>=6.2.0 in /usr/local/lib/python3.7/dist-packages (from plotly->CatBoost) (8.0.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install category_encoders"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ELBkkz4advGw",
        "outputId": "be764cf7-cd95-4b8c-dce2-3b127cad2c7a"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: category_encoders in /usr/local/lib/python3.7/dist-packages (2.4.0)\n",
            "Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from category_encoders) (1.4.1)\n",
            "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.7/dist-packages (from category_encoders) (1.0.2)\n",
            "Requirement already satisfied: pandas>=0.21.1 in /usr/local/lib/python3.7/dist-packages (from category_encoders) (1.3.5)\n",
            "Requirement already satisfied: statsmodels>=0.9.0 in /usr/local/lib/python3.7/dist-packages (from category_encoders) (0.10.2)\n",
            "Requirement already satisfied: patsy>=0.5.1 in /usr/local/lib/python3.7/dist-packages (from category_encoders) (0.5.2)\n",
            "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.7/dist-packages (from category_encoders) (1.21.5)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.21.1->category_encoders) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.21.1->category_encoders) (2018.9)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from patsy>=0.5.1->category_encoders) (1.15.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.20.0->category_encoders) (1.1.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn>=0.20.0->category_encoders) (3.1.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install shap"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z4VEXHwOdvDr",
        "outputId": "d7284f4b-afe1-468c-d4a2-79f9794463ab"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: shap in /usr/local/lib/python3.7/dist-packages (0.40.0)\n",
            "Requirement already satisfied: slicer==0.0.7 in /usr/local/lib/python3.7/dist-packages (from shap) (0.0.7)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from shap) (1.4.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from shap) (1.3.5)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from shap) (1.21.5)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from shap) (1.3.0)\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.7/dist-packages (from shap) (0.51.2)\n",
            "Requirement already satisfied: tqdm>4.25.0 in /usr/local/lib/python3.7/dist-packages (from shap) (4.63.0)\n",
            "Requirement already satisfied: packaging>20.9 in /usr/local/lib/python3.7/dist-packages (from shap) (21.3)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.7/dist-packages (from shap) (1.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>20.9->shap) (3.0.7)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from numba->shap) (57.4.0)\n",
            "Requirement already satisfied: llvmlite<0.35,>=0.34.0.dev0 in /usr/local/lib/python3.7/dist-packages (from numba->shap) (0.34.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->shap) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.7/dist-packages (from pandas->shap) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->shap) (1.15.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->shap) (3.1.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn->shap) (1.1.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install mlflow"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G7qGRj8NdvBI",
        "outputId": "43e7e5e0-3b75-46fe-e8e8-505b76649a1c"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: mlflow in /usr/local/lib/python3.7/dist-packages (1.24.0)\n",
            "Requirement already satisfied: protobuf>=3.7.0 in /usr/local/lib/python3.7/dist-packages (from mlflow) (3.17.3)\n",
            "Requirement already satisfied: sqlalchemy in /usr/local/lib/python3.7/dist-packages (from mlflow) (1.4.32)\n",
            "Requirement already satisfied: entrypoints in /usr/local/lib/python3.7/dist-packages (from mlflow) (0.4)\n",
            "Requirement already satisfied: Flask in /usr/local/lib/python3.7/dist-packages (from mlflow) (1.1.4)\n",
            "Requirement already satisfied: pytz in /usr/local/lib/python3.7/dist-packages (from mlflow) (2018.9)\n",
            "Requirement already satisfied: databricks-cli>=0.8.7 in /usr/local/lib/python3.7/dist-packages (from mlflow) (0.16.4)\n",
            "Requirement already satisfied: requests>=2.17.3 in /usr/local/lib/python3.7/dist-packages (from mlflow) (2.23.0)\n",
            "Requirement already satisfied: gitpython>=2.1.0 in /usr/local/lib/python3.7/dist-packages (from mlflow) (3.1.27)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from mlflow) (1.21.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from mlflow) (21.3)\n",
            "Requirement already satisfied: alembic in /usr/local/lib/python3.7/dist-packages (from mlflow) (1.7.7)\n",
            "Requirement already satisfied: docker>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from mlflow) (5.0.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from mlflow) (1.3.5)\n",
            "Requirement already satisfied: gunicorn in /usr/local/lib/python3.7/dist-packages (from mlflow) (20.1.0)\n",
            "Requirement already satisfied: prometheus-flask-exporter in /usr/local/lib/python3.7/dist-packages (from mlflow) (0.19.0)\n",
            "Requirement already satisfied: sqlparse>=0.3.1 in /usr/local/lib/python3.7/dist-packages (from mlflow) (0.4.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from mlflow) (6.0)\n",
            "Requirement already satisfied: cloudpickle in /usr/local/lib/python3.7/dist-packages (from mlflow) (1.3.0)\n",
            "Requirement already satisfied: querystring-parser in /usr/local/lib/python3.7/dist-packages (from mlflow) (1.2.4)\n",
            "Requirement already satisfied: importlib-metadata!=4.7.0,>=3.7.0 in /usr/local/lib/python3.7/dist-packages (from mlflow) (4.11.3)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from mlflow) (1.4.1)\n",
            "Requirement already satisfied: click>=7.0 in /usr/local/lib/python3.7/dist-packages (from mlflow) (7.1.2)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.7/dist-packages (from databricks-cli>=0.8.7->mlflow) (1.15.0)\n",
            "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.7/dist-packages (from databricks-cli>=0.8.7->mlflow) (0.8.9)\n",
            "Requirement already satisfied: websocket-client>=0.32.0 in /usr/local/lib/python3.7/dist-packages (from docker>=4.0.0->mlflow) (1.3.2)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.7/dist-packages (from gitpython>=2.1.0->mlflow) (4.0.9)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from gitpython>=2.1.0->mlflow) (3.10.0.2)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.7/dist-packages (from gitdb<5,>=4.0.1->gitpython>=2.1.0->mlflow) (5.0.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata!=4.7.0,>=3.7.0->mlflow) (3.7.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.17.3->mlflow) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.17.3->mlflow) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.17.3->mlflow) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.17.3->mlflow) (2021.10.8)\n",
            "Requirement already satisfied: Mako in /usr/local/lib/python3.7/dist-packages (from alembic->mlflow) (1.2.0)\n",
            "Requirement already satisfied: importlib-resources in /usr/local/lib/python3.7/dist-packages (from alembic->mlflow) (5.4.0)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.7/dist-packages (from sqlalchemy->mlflow) (1.1.2)\n",
            "Requirement already satisfied: Jinja2<3.0,>=2.10.1 in /usr/local/lib/python3.7/dist-packages (from Flask->mlflow) (2.11.3)\n",
            "Requirement already satisfied: itsdangerous<2.0,>=0.24 in /usr/local/lib/python3.7/dist-packages (from Flask->mlflow) (1.1.0)\n",
            "Requirement already satisfied: Werkzeug<2.0,>=0.15 in /usr/local/lib/python3.7/dist-packages (from Flask->mlflow) (1.0.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from Jinja2<3.0,>=2.10.1->Flask->mlflow) (2.0.1)\n",
            "Requirement already satisfied: setuptools>=3.0 in /usr/local/lib/python3.7/dist-packages (from gunicorn->mlflow) (57.4.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->mlflow) (3.0.7)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->mlflow) (2.8.2)\n",
            "Requirement already satisfied: prometheus-client in /usr/local/lib/python3.7/dist-packages (from prometheus-flask-exporter->mlflow) (0.13.1)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "eciDGx7xdu71"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##imports"
      ],
      "metadata": {
        "id": "3i5Hp6PJlec3"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "qEMQhfTqlQ8m"
      },
      "outputs": [],
      "source": [
        "from sklearn.pipeline import make_pipeline, Pipeline\n",
        "from sklearn.metrics import precision_recall_curve\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn import model_selection\n",
        "from sklearn import metrics\n",
        "from sklearn.preprocessing import StandardScaler#, PolynomialFeatures, MinMaxScaler, Normalizer\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import shap\n",
        "import mlflow\n",
        "from sklearn import preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n"
      ],
      "metadata": {
        "id": "1V8KfBIvlRzu"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from catboost import CatBoostClassifier, Pool, cv\n",
        "import category_encoders as ce"
      ],
      "metadata": {
        "id": "vdwQWrAolRuk"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pd.set_option('display.max_columns', 500)\n",
        "pd.set_option('display.max_rows', 500)"
      ],
      "metadata": {
        "id": "VRrfraaBlRoW"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "##Functions"
      ],
      "metadata": {
        "id": "__R5OySHQtiu"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "def extract_threshold_given_recall(y_test, probabilities, given_recall):\n",
        "    p, r, t = precision_recall_curve(y_test, probabilities)\n",
        "    idx = np.abs(r - given_recall).argmin()\n",
        "    return t[idx]"
      ],
      "metadata": {
        "id": "ezMMKtNBRAEB"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def find_best_threshold(thresholds, fpr, tpr, manual_th=0.5):\n",
        "    \"\"\"\n",
        "    find the best threshold from the roc curve. by finding the threshold\n",
        "    for the point which is closest to (fpr=0,tpr=1)\n",
        "    :param thresholds: cut off threshold range for predict probabilities\n",
        "    :param fpr: false positive rate\n",
        "    :param tpr: true positive rate\n",
        "    :param manual_th: manuel threshold to replace calculated threshold\n",
        "    :return: the best threshold which trying to minimize fpr and maximize tpr\n",
        "    \"\"\"\n",
        "    if manual_th < 1 and manual_th > 0:\n",
        "        return manual_th\n",
        "    else:\n",
        "        fpr_tpr = pd.DataFrame({'thresholds': thresholds, 'fpr': fpr, 'tpr': tpr})\n",
        "        fpr_tpr['dist'] = (fpr_tpr['fpr']) ** 2 + (fpr_tpr['tpr'] - 1) ** 2\n",
        "\n",
        "    return fpr_tpr.iloc[fpr_tpr.dist.idxmin(), 0]\n"
      ],
      "metadata": {
        "id": "sXbWO23CRW07"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def print_cm(cm, labels, hide_zeroes=False, hide_diagonal=False, hide_threshold=None):\n",
        "\n",
        "    \"\"\"\n",
        "    Pretty print for confusion matrix\n",
        "    :param cm:\n",
        "    :param labels:\n",
        "    :param hide_zeroes:\n",
        "    :param hide_diagonal:\n",
        "    :param hide_threshold:\n",
        "    :return: pretty print for confusion matrix\n",
        "    \"\"\"\n",
        "\n",
        "    colwidth = 6\n",
        "    empty_cell = \" \" * colwidth\n",
        "    # Begin CHANGES\n",
        "    fst_empty_cell = (colwidth - 3) // 2 * \" \" + \"T\\P\" + (colwidth - 3) // 2 * \" \"\n",
        "    if len(fst_empty_cell) < len(empty_cell):\n",
        "        fst_empty_cell = \" \" * (len(empty_cell) - len(fst_empty_cell)) + fst_empty_cell\n",
        "    # Print header\n",
        "    print(\"    \" + fst_empty_cell, end=\" \")\n",
        "    # End CHANGES\n",
        "    for label in labels:\n",
        "        print(\"%{0}s\".format(colwidth) % label, end=\" \")\n",
        "    print()\n",
        "    # Print rows\n",
        "    for i, label1 in enumerate(labels):\n",
        "        print(\"    %{0}s\".format(colwidth) % label1, end=\" \")\n",
        "        for j in range(len(labels)):\n",
        "            cell = \"%{0}.0f\".format(colwidth) % cm[i, j]\n",
        "            if hide_zeroes:\n",
        "                cell = cell if float(cm[i, j]) != 0 else empty_cell\n",
        "            if hide_diagonal:\n",
        "                cell = cell if i != j else empty_cell\n",
        "            if hide_threshold:\n",
        "                cell = cell if cm[i, j] > hide_threshold else empty_cell\n",
        "            print(cell, end=\" \")\n",
        "        print()"
      ],
      "metadata": {
        "id": "toSQEcE6Ra1f"
      },
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def plot_model(model, X_train, y_train, X_test, y_test):\n",
        "\n",
        "  probabilities = model.predict_proba(X_test)[:, 1]\n",
        "\n",
        "\n",
        "  fpr, tpr, thresholds = metrics.roc_curve(y_test, probabilities)    \n",
        "  plt.figure()\n",
        "  plt.plot(fpr, tpr, label='test')\n",
        "  roc_auc_test=round(metrics.roc_auc_score(y_test, probabilities), 4)\n",
        "  train_probabilities = model.predict_proba(X_train)[:,1]\n",
        "  fpr, tpr, thresholds = metrics.roc_curve(y_train, train_probabilities)\n",
        "  \n",
        "  plt.plot(fpr, tpr, label='train')\n",
        "  plt.plot([0, 1], [0, 1], 'r--', label='random guess')\n",
        "  plt.title(\"Area under the ROC = {}\".format(roc_auc_test), fontsize=18)\n",
        "  plt.legend()\n",
        "  plt.show()\n",
        "\n",
        "  rcl_per_disp = metrics.plot_precision_recall_curve(model, X_test, y_test)\n",
        "  plt.show()\n",
        "\n",
        "  roc_disp = metrics.plot_roc_curve(model, X_test, y_test)\n",
        "  plt.cla()\n",
        "  plt.clf()\n",
        "  plt.close()\n",
        "\n",
        "  X_train = pd.DataFrame(X_train)\n",
        "\n",
        "  xtrain_samp = X_train.sample(frac = 0.5, random_state=1)\n",
        "  %time \n",
        "  shap_values = shap.TreeExplainer(model).shap_values(xtrain_samp)\n",
        "  shap.summary_plot(shap_values, xtrain_samp)"
      ],
      "metadata": {
        "id": "C1Wf0QXURghr"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def model_probs_preds(probabilities, threshold, y_test):\n",
        "    predictions = probabilities > threshold\n",
        "    print('****************************************************')\n",
        "\n",
        "    labels = [0, 1]\n",
        "    cm = metrics.confusion_matrix(y_test, predictions)\n",
        "\n",
        "    print('Confusion Matrix=')\n",
        "    print_cm(cm, labels)\n",
        "    print('classification_report=')\n",
        "    print(metrics.classification_report(y_test, predictions, digits=3, target_names=['0', '1']))\n",
        "    print('FPR:', \"{0:.2%}\".format(round(metrics.confusion_matrix(y_test, predictions)[0, 1] /\n",
        "                                         (metrics.confusion_matrix(y_test, predictions)[0, 1]\n",
        "                                          + metrics.confusion_matrix(y_test, predictions)[0, 0]), 4)))\n",
        "    roc_auc_test=round(metrics.roc_auc_score(y_test, probabilities), 4)\n",
        "    print('AUC:', \"{0:.2%}\".format(roc_auc_test))\n",
        "    print('Accuracy Score:', \"{0:.2%}\".format(round(metrics.accuracy_score(y_test, predictions), 4)))\n",
        "    print('Best Threshold:', \"{0:.2%}\".format(round(threshold, 4)))\n",
        "    print('****************************************************')"
      ],
      "metadata": {
        "id": "CNaQCTlyRhDi"
      },
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_model_results_for_recall(model, X_test, y_test, X_train, y_train, given_recall, with_plots = True):\n",
        "    \"\"\"\n",
        "    Get a summary of model results as follow:\n",
        "    ****************************************************\n",
        "    Confusion Matrix=\n",
        "          T\\P       0      1\n",
        "             0 148328  27376\n",
        "             1   5397  25233\n",
        "    classification_report=\n",
        "                  precision    recall  f1-score   support\n",
        "               0      0.965     0.844     0.901    175704\n",
        "               1      0.480     0.824     0.606     30630\n",
        "        accuracy                          0.841    206334\n",
        "       macro avg      0.722     0.834     0.753    206334\n",
        "    weighted avg      0.893     0.841     0.857    206334\n",
        "    FPR: 15.58%\n",
        "    AUC: 91.53%\n",
        "    Accuracy Score: 84.12%\n",
        "    Best Threshold: 48.55%\n",
        "    Traffic to buy: 0.12%\n",
        "    ****************************************************\n",
        "    :param model:\n",
        "    :param X_test:\n",
        "    :param y_test:\n",
        "    :param X_train:\n",
        "    :param y_train:\n",
        "    :param manual_th:\n",
        "    :param with_plots:\n",
        "    :return: printed summary of model results\n",
        "    \"\"\"\n",
        "\n",
        "    probabilities = model.predict_proba(X_test)[:, 1]\n",
        "    #fpr, tpr, thresholds = metrics.roc_curve(y_test, probabilities)\n",
        "    threshold = extract_threshold_given_recall(y_test, probabilities, given_recall)\n",
        "\n",
        "    predictions = probabilities > threshold\n",
        "    print('****************************************************')\n",
        "\n",
        "    \n",
        "    labels = [0, 1]\n",
        "    cm = metrics.confusion_matrix(y_test, predictions)\n",
        "    tn, fp, fn, tp = cm.ravel()\n",
        "    traffic_to_buy = (fp + tp)/(tn + fp + fn + tp)\n",
        "\n",
        "    print('Confusion Matrix=')\n",
        "    print_cm(cm, labels)\n",
        "    print('classification_report=')\n",
        "    print(metrics.classification_report(y_test, predictions, digits=3, target_names=['0', '1']))\n",
        "    print('FPR:', \"{0:.2%}\".format(round(metrics.confusion_matrix(y_test, predictions)[0, 1] /\n",
        "                                         (metrics.confusion_matrix(y_test, predictions)[0, 1]\n",
        "                                          + metrics.confusion_matrix(y_test, predictions)[0, 0]), 4)))\n",
        "    roc_auc_test=round(metrics.roc_auc_score(y_test, probabilities), 4)\n",
        "    print('AUC:', \"{0:.2%}\".format(roc_auc_test))\n",
        "    print('Accuracy Score:', \"{0:.2%}\".format(round(metrics.accuracy_score(y_test, predictions), 4)))\n",
        "    print('Best Threshold:', \"{0:.2%}\".format(round(threshold, 4)))\n",
        "    print('Traffic to buy:', \"{0:.2%}\".format(round(traffic_to_buy, 4)))\n",
        "    print('****************************************************')\n",
        "\n",
        "    \n",
        "\n",
        "    \n",
        "    if with_plots:\n",
        "      \n",
        "\n",
        "      plot_model(model, X_train, y_train, X_test, y_test)  \n",
        "\n",
        "    return df_with_target_probs(model, X_train, y_train, X_test, y_test)   "
      ],
      "metadata": {
        "id": "xiLKCyxxXc98"
      },
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def get_model_results(model, X_test, y_test, X_train, y_train, manual_th=-9, with_plots = True):\n",
        "    \"\"\"\n",
        "    Get a summary of model results as follow:\n",
        "    ****************************************************\n",
        "    Confusion Matrix=\n",
        "          T\\P       0      1\n",
        "             0 148328  27376\n",
        "             1   5397  25233\n",
        "    classification_report=\n",
        "                  precision    recall  f1-score   support\n",
        "               0      0.965     0.844     0.901    175704\n",
        "               1      0.480     0.824     0.606     30630\n",
        "        accuracy                          0.841    206334\n",
        "       macro avg      0.722     0.834     0.753    206334\n",
        "    weighted avg      0.893     0.841     0.857    206334\n",
        "    FPR: 15.58%\n",
        "    AUC: 91.53%\n",
        "    Accuracy Score: 84.12%\n",
        "    Best Threshold: 48.55%\n",
        "    ****************************************************\n",
        "    :param model:\n",
        "    :param X_test:\n",
        "    :param y_test:\n",
        "    :param X_train:\n",
        "    :param y_train:\n",
        "    :param manual_th:\n",
        "    :param with_plots:\n",
        "    :return: printed summary of model results\n",
        "    \"\"\"\n",
        "\n",
        "    probabilities = model.predict_proba(X_test)[:, 1]\n",
        "    fpr, tpr, thresholds = metrics.roc_curve(y_test, probabilities)\n",
        "    threshold = find_best_threshold(thresholds, fpr, tpr,manual_th)\n",
        "    \n",
        "    model_probs_preds(probabilities, threshold, y_test) \n",
        "\n",
        "\n",
        "    \n",
        "    if with_plots:\n",
        "\n",
        "      plot_model(model, X_train, y_train, X_test, y_test)\n",
        "\n",
        "    df = df_with_target_probs(model, X_train, y_train, X_test, y_test)\n",
        "    return df\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "JTC2JnsVPObo"
      },
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "ZnZwjkIJQfpx"
      },
      "outputs": [],
      "source": [
        "def df_with_target_probs(model, X_train, y_train, X_test, y_test):\n",
        "  train = pd.DataFrame(X_train)\n",
        "  test = pd.DataFrame(X_test)\n",
        "  \n",
        "  test['probabilities'] = pd.Series(model.predict_proba(X_test)[:,1])\n",
        "  train['probabilities'] = pd.Series(model.predict_proba(X_train)[:,1])\n",
        "\n",
        "\n",
        "\n",
        "  train['y_true'] =  pd.Series(y_train.flatten())\n",
        "  test['y_true'] =  pd.Series(y_test.flatten())\n",
        "\n",
        "  \n",
        "  test['y_pred'] =  pd.Series(model.predict(X_train))\n",
        "\n",
        "  df = train.append(test).sort_index()\n",
        "\n",
        "  return df\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## How to use"
      ],
      "metadata": {
        "id": "r0zSGEJcG5X-"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Creating a dataframe with the probabilities "
      ],
      "metadata": {
        "id": "MyeKjmFqfzAX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.datasets import load_breast_cancer\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "dataset = load_breast_cancer()\n",
        "X = dataset.data\n",
        "y = dataset.target\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
        "\n",
        "RFmodel = RandomForestClassifier(n_estimators=100)  # Choose a model here\n",
        "\n",
        "\n",
        "RFmodel.fit(X_train, y_train)\n",
        "\n",
        "df = df_with_target_probs(RFmodel, X_train, y_train, X_test, y_test)\n",
        "\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        },
        "id": "lP1NR4uQwagK",
        "outputId": "cc5d5482-8284-44fe-fd3e-09e048abc197"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       0      1       2       3        4        5         6         7       8  \\\n",
              "0  11.85  17.46   75.54   432.7  0.08372  0.05642  0.026880  0.022800  0.1875   \n",
              "0  13.40  20.52   88.64   556.7  0.11060  0.14690  0.144500  0.081720  0.2116   \n",
              "1  13.21  25.25   84.10   537.9  0.08791  0.05205  0.027720  0.020680  0.1619   \n",
              "1  11.22  19.86   71.94   387.3  0.10540  0.06779  0.005006  0.007583  0.1940   \n",
              "2  20.13  28.25  131.20  1261.0  0.09780  0.10340  0.144000  0.097910  0.1752   \n",
              "\n",
              "         9      10      11     12     13        14        15        16  \\\n",
              "0  0.05715  0.2070  1.2380  1.234  13.88  0.007595  0.015000  0.014120   \n",
              "0  0.07325  0.3906  0.9306  3.093  33.67  0.005414  0.022650  0.034520   \n",
              "1  0.05584  0.2084  1.3500  1.314  17.58  0.005768  0.008082  0.015100   \n",
              "1  0.06028  0.2976  1.9660  1.959  19.62  0.012890  0.011040  0.003297   \n",
              "2  0.05533  0.7655  2.4630  5.203  99.04  0.005769  0.024230  0.039500   \n",
              "\n",
              "         17       18        19     20     21      22      23      24       25  \\\n",
              "0  0.008578  0.01792  0.001784  13.06  25.75   84.35   517.8  0.1369  0.17580   \n",
              "0  0.013340  0.01705  0.004005  16.41  29.66  113.30   844.4  0.1574  0.38560   \n",
              "1  0.006451  0.01347  0.001828  14.35  34.23   91.29   632.9  0.1289  0.10630   \n",
              "1  0.004967  0.04243  0.001963  11.98  25.78   76.91   436.1  0.1424  0.09669   \n",
              "2  0.016780  0.01898  0.002498  23.69  38.25  155.00  1731.0  0.1166  0.19220   \n",
              "\n",
              "        26       27      28       29  probabilities  y_true  y_pred  \n",
              "0  0.13160  0.09140  0.3101  0.07007           1.00       1     NaN  \n",
              "0  0.51060  0.20510  0.3585  0.11090           0.00       0     1.0  \n",
              "1  0.13900  0.06005  0.2444  0.06788           0.94       1     1.0  \n",
              "1  0.01335  0.02022  0.3292  0.06522           1.00       1     NaN  \n",
              "2  0.32150  0.16280  0.2572  0.06637           0.00       0     NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d7858284-ec46-4757-b9b7-2d587a1cb00c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>probabilities</th>\n",
              "      <th>y_true</th>\n",
              "      <th>y_pred</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>11.85</td>\n",
              "      <td>17.46</td>\n",
              "      <td>75.54</td>\n",
              "      <td>432.7</td>\n",
              "      <td>0.08372</td>\n",
              "      <td>0.05642</td>\n",
              "      <td>0.026880</td>\n",
              "      <td>0.022800</td>\n",
              "      <td>0.1875</td>\n",
              "      <td>0.05715</td>\n",
              "      <td>0.2070</td>\n",
              "      <td>1.2380</td>\n",
              "      <td>1.234</td>\n",
              "      <td>13.88</td>\n",
              "      <td>0.007595</td>\n",
              "      <td>0.015000</td>\n",
              "      <td>0.014120</td>\n",
              "      <td>0.008578</td>\n",
              "      <td>0.01792</td>\n",
              "      <td>0.001784</td>\n",
              "      <td>13.06</td>\n",
              "      <td>25.75</td>\n",
              "      <td>84.35</td>\n",
              "      <td>517.8</td>\n",
              "      <td>0.1369</td>\n",
              "      <td>0.17580</td>\n",
              "      <td>0.13160</td>\n",
              "      <td>0.09140</td>\n",
              "      <td>0.3101</td>\n",
              "      <td>0.07007</td>\n",
              "      <td>1.00</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>13.40</td>\n",
              "      <td>20.52</td>\n",
              "      <td>88.64</td>\n",
              "      <td>556.7</td>\n",
              "      <td>0.11060</td>\n",
              "      <td>0.14690</td>\n",
              "      <td>0.144500</td>\n",
              "      <td>0.081720</td>\n",
              "      <td>0.2116</td>\n",
              "      <td>0.07325</td>\n",
              "      <td>0.3906</td>\n",
              "      <td>0.9306</td>\n",
              "      <td>3.093</td>\n",
              "      <td>33.67</td>\n",
              "      <td>0.005414</td>\n",
              "      <td>0.022650</td>\n",
              "      <td>0.034520</td>\n",
              "      <td>0.013340</td>\n",
              "      <td>0.01705</td>\n",
              "      <td>0.004005</td>\n",
              "      <td>16.41</td>\n",
              "      <td>29.66</td>\n",
              "      <td>113.30</td>\n",
              "      <td>844.4</td>\n",
              "      <td>0.1574</td>\n",
              "      <td>0.38560</td>\n",
              "      <td>0.51060</td>\n",
              "      <td>0.20510</td>\n",
              "      <td>0.3585</td>\n",
              "      <td>0.11090</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>13.21</td>\n",
              "      <td>25.25</td>\n",
              "      <td>84.10</td>\n",
              "      <td>537.9</td>\n",
              "      <td>0.08791</td>\n",
              "      <td>0.05205</td>\n",
              "      <td>0.027720</td>\n",
              "      <td>0.020680</td>\n",
              "      <td>0.1619</td>\n",
              "      <td>0.05584</td>\n",
              "      <td>0.2084</td>\n",
              "      <td>1.3500</td>\n",
              "      <td>1.314</td>\n",
              "      <td>17.58</td>\n",
              "      <td>0.005768</td>\n",
              "      <td>0.008082</td>\n",
              "      <td>0.015100</td>\n",
              "      <td>0.006451</td>\n",
              "      <td>0.01347</td>\n",
              "      <td>0.001828</td>\n",
              "      <td>14.35</td>\n",
              "      <td>34.23</td>\n",
              "      <td>91.29</td>\n",
              "      <td>632.9</td>\n",
              "      <td>0.1289</td>\n",
              "      <td>0.10630</td>\n",
              "      <td>0.13900</td>\n",
              "      <td>0.06005</td>\n",
              "      <td>0.2444</td>\n",
              "      <td>0.06788</td>\n",
              "      <td>0.94</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>11.22</td>\n",
              "      <td>19.86</td>\n",
              "      <td>71.94</td>\n",
              "      <td>387.3</td>\n",
              "      <td>0.10540</td>\n",
              "      <td>0.06779</td>\n",
              "      <td>0.005006</td>\n",
              "      <td>0.007583</td>\n",
              "      <td>0.1940</td>\n",
              "      <td>0.06028</td>\n",
              "      <td>0.2976</td>\n",
              "      <td>1.9660</td>\n",
              "      <td>1.959</td>\n",
              "      <td>19.62</td>\n",
              "      <td>0.012890</td>\n",
              "      <td>0.011040</td>\n",
              "      <td>0.003297</td>\n",
              "      <td>0.004967</td>\n",
              "      <td>0.04243</td>\n",
              "      <td>0.001963</td>\n",
              "      <td>11.98</td>\n",
              "      <td>25.78</td>\n",
              "      <td>76.91</td>\n",
              "      <td>436.1</td>\n",
              "      <td>0.1424</td>\n",
              "      <td>0.09669</td>\n",
              "      <td>0.01335</td>\n",
              "      <td>0.02022</td>\n",
              "      <td>0.3292</td>\n",
              "      <td>0.06522</td>\n",
              "      <td>1.00</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>20.13</td>\n",
              "      <td>28.25</td>\n",
              "      <td>131.20</td>\n",
              "      <td>1261.0</td>\n",
              "      <td>0.09780</td>\n",
              "      <td>0.10340</td>\n",
              "      <td>0.144000</td>\n",
              "      <td>0.097910</td>\n",
              "      <td>0.1752</td>\n",
              "      <td>0.05533</td>\n",
              "      <td>0.7655</td>\n",
              "      <td>2.4630</td>\n",
              "      <td>5.203</td>\n",
              "      <td>99.04</td>\n",
              "      <td>0.005769</td>\n",
              "      <td>0.024230</td>\n",
              "      <td>0.039500</td>\n",
              "      <td>0.016780</td>\n",
              "      <td>0.01898</td>\n",
              "      <td>0.002498</td>\n",
              "      <td>23.69</td>\n",
              "      <td>38.25</td>\n",
              "      <td>155.00</td>\n",
              "      <td>1731.0</td>\n",
              "      <td>0.1166</td>\n",
              "      <td>0.19220</td>\n",
              "      <td>0.32150</td>\n",
              "      <td>0.16280</td>\n",
              "      <td>0.2572</td>\n",
              "      <td>0.06637</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d7858284-ec46-4757-b9b7-2d587a1cb00c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d7858284-ec46-4757-b9b7-2d587a1cb00c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d7858284-ec46-4757-b9b7-2d587a1cb00c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Plotting model evaluation plots Creating and retrieving a dataframe with the probabilities"
      ],
      "metadata": {
        "id": "pEeM9PJTf_Fq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.datasets import load_breast_cancer\n",
        "\n",
        "\n",
        "dataset = load_breast_cancer()\n",
        "X = dataset.data\n",
        "y = dataset.target\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
        "\n",
        "\n",
        "RFmodel = RandomForestClassifier(n_estimators=100)  # Choose a model here\n",
        "RFmodel.fit(X_train, y_train)\n",
        "\n",
        "df_probs = get_model_results(RFmodel, X_test, y_test, X_train, y_train, manual_th=-9)\n",
        "\n",
        "df_probs.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "9N6p5agoxjo3",
        "outputId": "a6dda6ed-20af-4bcc-88f3-5ac098a7d66b"
      },
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "****************************************************\n",
            "Confusion Matrix=\n",
            "      T\\P       0      1 \n",
            "         0     52      1 \n",
            "         1      3     87 \n",
            "classification_report=\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0      0.945     0.981     0.963        53\n",
            "           1      0.989     0.967     0.978        90\n",
            "\n",
            "    accuracy                          0.972       143\n",
            "   macro avg      0.967     0.974     0.970       143\n",
            "weighted avg      0.973     0.972     0.972       143\n",
            "\n",
            "FPR: 1.89%\n",
            "AUC: 99.66%\n",
            "Accuracy Score: 97.20%\n",
            "Best Threshold: 47.00%\n",
            "****************************************************\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEMCAYAAADUEk3/AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwURfrH8c8DAgFFZQFZ5UZRTgUMeCAoC3KIiuCuPxUPVERFd1lvXO+LRUXFA0RUxNtlPdkVV1cXRFSEABEl3MgRQC45Re76/VEdGIdJMiGTdGbyfb9e88pMdXXn6TmeqamurjbnHCIikvzKhB2AiIgkhhK6iEiKUEIXEUkRSugiIilCCV1EJEUooYuIpAgldCkwMxttZiVmvKuZ9TEzZ2ZnhB2LSJiU0COYWRUz+zVIDpeGHY/sY2YtzOw+M6sXdiwAwXsk8rbdzBaY2VAzq5rHevXNbLiZzQ/eaxvM7Bsz+6uZVchjvSpmdo+ZTQ3W2WFm2Wb2rpn1MjMrmj3Nm5lVMLMHzOzH4DlYaGZ3mVm5AmzjaDN7w8xWRTyP95tZWoy6ZmbXmtmMiOfvP2Z2ch7br2VmI81sabD9n8zsYzNrEqPuQWb2FzObbma/mNnG4P418T8r4Tko7ABKmN5ABeBH4ErgtXDDkQgtgHuBCcDiUCPZJxN4PLhfBegMDAA6mVkr59yOyMpm1gN4C9gNvAJ8B1QCzgKeBC4zs27OuVVR67UBPgSOAMYCbwCbgJrBuu8C1wPDi2Af8/MPoAcwCvgGOAV4EDgG6JPfymbWKFjvIGAY/rN3CnA3cFLwfET+GhwOXIt/H9yGf/76AV+YWRfn3ISo7bcEPgM2BzEuBX4HpAPVo+qWxz+/HfDP8YggroZA3fyfihLAOadbcANmAJ8DfwH2AA0KsG7lsOMvxudptH/rJHSbFYGD8ljeB3DAGQVZVoTPgQP+HaP8/WDZ+VHlzYBfgWXA0THWuzpY7wvAIsp/D6wCNgKn5RJLF+DCEN4HZwUxPx5V/nhQfmoc2/gg+KydGlV+R7CNSyLKWgRlH0c9R4cDPwHzgDIR5WnA/OBzfWgcsTwI7AI6FPdzmbDXJOwASsoNaBW8WS4DqgE7gIdyqeuCpNYRmARsASZELE8PPthrge3AXODO6IQFtAm2Mw/Yim9FfAX0LEDcDhgdo3y/JAfcF5QdBwwCsoP4vgPOirGNNOAxYEWQjKbgW6GjiZHQ8S2Z14CVwfO3OFj/4Kh6o4M4quNbTauCD3W9XPYxJ+7o2+ioff0DcAuwMNivecDluWyzE/ApsAHYBswEri3g8x4roV8fLBsYVf5uUL7f8xxRZ0xQ5+yIspzkGHdsxfiZeT2IrXZUee2gfHgc29gIzIlRfmSwjc8jym4Myvb78gKeD5a1iyi7LCg7J3hcAaiQSxwH43/1vBs8NpKwkaYul32uwifmd51zv5jZv4HLzewe59yeGPXTgfOBF/A/nwEws+7Ae8AC/IfxZ/xPyAfwLYw/RWyjJ9AI/0FeAlQFLgfeM7Pezrk3E7uLe70C7ASGAOWBvwIfmNmxzrnFEfXeAs4D/gV8Ahwd7NuP0Rs0sxOB/+ET5PPAcuAE/K+dtmZ2unNuZ9Rq/8W3rB7Ef6C25BLve/gPeD/8F9HsoHxhVL1B+Jb+8/iEfh0w2swWOOe+ioi1H/7n9GTgYeAX4EzgOTM72jl3ay5xxOPo4O/PEf8vDegOZDvnxuWx7gv498f5wL+DsvPxX46v5LZSvMysWgGqb4zxekVrDSx3zi2LLHTOLTOzFcHy/FTAN2ai5ZS1MTNzPstWiFoWq/7JwJfB/bOCvxvMbCJwGr4bPhP/hftJxPrtgMrANDN7Ct/leoiZrcW/Lvc453bFsT/hCvsbpSTc8C3R9US0dPH9gg7oFqN+TguxU4zt/ARMZP/WeE7r4oyIsoNjbLsSvkWfFWfsB9JC/ze//cnaOij/e0RZ51jbxid4R1QLHd/Kn0NUqwb/peWAPhFlo4Oy1wvwGu23PzGWzQDKR5TXxCf2tyLKjsS3yN+MsZ2n8P3b+Xa1Bf/vE/yvuWr4PuPrg/+3GTgiom7zoP7YfLb5u6BeRvC4cvB4ZoLe57F+5eR22+95jrG9zcC3uSybAqyIYxvT8L/+fp/b+wz4XVB2TvB4aFRdA6YHy56OKJ8RlK3GN0ouwPe/Lwte504RdQdE1M3GNwYuwB+7cMAriXgNivqmFrrXC98PF9kKGgeswX9Tfxxjne+cc59FlZ0J1MD3/x0eNfBgHPAEPlFOAHDO/ZKz0Mwq4VuXhm/pXmtmhzrnNh3wXuXuKRe8i4M4pprZFnyXSY7zgr+PRa7onPvAzObiu21yYm8OHI8/aFkharTGJHwLOKerJtKQQu5HtOEu4kCkc265mc3jt/v1R3xL76UYLdZ/4X9RdAJGxvH/OuPfI5GmA/2dc6sjyg4N/m7MZ3s5r/VhUesl6j1wZgHqfhdHnUr4L7BYtgXL8/M4/gDkh2Z2G76b7iT8l+tOoFywnZ/xn8MsoH/wC+C9YNlN+GMUOTHlqBz8nQOcm/OeN7PPg+08jD9gGln3d0BT59zc4PEYMxuPP2A92DmX8+uwRFJC967CfzCzzeyYiPJPgT+ZWTXn3NqodebF2E7j4O+oPP5XjZw7ZnYE8BD+18ARMeoeTuI+zJEWxShbh+/yydEA368daz9nE5HQ2bff9we3WGrEKIu17cLIbb/qRjzOiTX6yzhSrFhj+Ra4C/8lXAefWGrhu0giRSfq3EQn/pz1KseoW2AxGiCFtZV93SDR0ojdNfIbzrk3g2GeDxI0dPDP3yB8N1VrgufBObfLzLrhG16PBDfwxz8G4r8cIj8vvwZ/X41qwMw3s6+BdmZ2cNCwyqk7OSKZ53gVOCO4KaGXZGZWHz9Mycg9wVwCDI0qi/VmzWmS34of0hbLiuD/Gv4LozG+NZKB/yDvBq4ALqZw5wnk9druzqX8QMcy56z3OPCfXOqsjy5wzuX7gS+gePYr5/5l+IO3scT6YohlbWSSNLP3ge+Bd82sqXMuJ0nMx7dkW+azvVbB3+8BnHObzWwJ0MjMKkZs74CY2e8LUP1nFzXsMoYV+G6tWGrij6Pkyzn3jJmNxHdNVQBmOec2mNn1wMrIX6nOuaVABzOrA9QD1jnnZplZ/6DKnIhNZ+Nb7j/F+Lcr8e+Fw/C/ILOD8tzqgh+aWqKV+oSOT56GHza2Icbyh/DdLtEJPZb5wd9f4mgNHY8/aPiAc+7eyAVm1jeO/5XjZ/zPxGgNCrCNWBbhv1COBWZFLWsc9Thnv3cXQSswh8u/SlxyYl2b6Fidcz+b2V34X2g34luZOOe2mdnHwHlm1tU5l9uXXs7r/l5E2XvBti4lvm6gvOT2BRZLB/a1mHMzFehtZrVdxIFRM6sNHIUf0x0X59x2fKMmZxs548RfyqX+UvyY8hxn4X9RRh7onAJ0xf9qilYLP0Tx54i65FEXfP96yRZ2J36YN3zCWkoeB53w/cIOaB1RltuByIPxQ/CWEBzIiVpekeCgIb7l4PAJPbJOM3xrzpHLML6o+t/gu4sqRZRVwX94czsout928X2XEyIed4m1n8Q4KIr/Qvwe/wtjvwOK+IbD7yIej45cP87X6vzg//aKsaxP9L5GLJsALI54XAvfvzsFqBij/mHkMrQtqp4j9rDFg/Cjb34mYuwz/gt8W/DeiPX8X0nu49BX4xsbp+QSS2fiGIeOPzYQ761KHNvrTt7j0E+LKm9EjDH4Mbabhj/2sg1oFEf9c3N5rx6HT9oZRAxSwDekdgOfRtWfhP9SaBVRVhbftbYTqFOQ92wYt9LeQu+MHzMbsxUQeBefCK/Ct0hy5fxwx8vwJ0vMNbNR+OGLh+PfzL3woz4m4PviZgG3BQdE5+Jbw9fgk+OJce7Ds/jxwP8zs9eC/3U1PnEU5Cd29L58Ymb/wg/d/B2+K+XoIL4f2HcQCudczlQJ/wNmBvs9C3+A6phgv+9g/4OiBTEV/2G708yq4H8m/+ic+7aA+5VtZtcBLwKzg+dsCb412Bz/hdWEAzwb1fl+3r/jh7oNwPcN45ybaWa98a/V92Y2mn1ninbDtyQzgQtckEmC9X4ys7Pxoy0mmdkH+FFUm/Ct4K744XjXxRFbon+RfBQM773JzA5j35miV+FHME2KWmU2wRdaToGZNcW/L/6N7/aogR+6ezRwhXMusgsFM3sJ34DIxPd7n4Y/w3sq/vmOjG+umT2Kf+99YWZv43/N/gXfZXpLVHx/xg95/MzMnsYff/k//PkiDzj/q6BkC/sbJcwb8E/8N3vzfOrNxbeQKgaPY7bQI+o3w39wl+MP8KwCvsafzhzZUq0bxLAG/wabgk/49xFnCz3Yzq34D8p2/IfmSvIetrjfdolqoQdlFfGtrZ+I78Siuvjx3YuD/V6HH5b2dyJOPslt/Tj283L86IQdka9BrH2NWGcCES30iPK2+JO/VgfbWwGMB24G0uKIJWYLPVhWLng91gOHRS07JniOFuJboJvw4+FvzOv/4hPRvew71rIDnwDfwY/gCOszlIbvllwcvP8WBe/zcrk8Z4ujymoEr0N2sE9r8N1MbXL5f9cE76mNwXvye+BvxPi1FbFOP/yX57bgNXkPP5IlVt3j8V1FOSeczSBiyG1Jv1mwEyIikuQ026KISIpQQhcRSRFK6CIiKUIJXUQkRYQ2bLFatWquXr16Yf17EZGkNG3atLXOueqxloWW0OvVq0dGRkb+FUVEZK9gOoiY1OUiIpIilNBFRFKEErqISIpQQhcRSRFK6CIiKSLfhG5mo8xstZn9kMtyM7OnzWyBmc00s1ax6omISNGKp4U+Gj9FZ2664a/Z2BA/q9lzhQ9LREQKKt9x6M65iWZWL48qPdh3zb7JZna4mR3pnCvI1VHil/EyfP9OkWy6oFZt3sbaLbldI1dE5Ldsx24O+mUnP9dvycn9X0j49hPRh14TWBbxOJtcrjNoZv3MLMPMMtasib5Yepy+fweWRM+bH461W7azdUdul7EUEdnn4IUbOOaZTGq/MQf2FM205cV6pqhzbiTBdRHT09MPfI/qngZXfJSosPL15rdL+TBz/+vdZu3YRJMjD+Uf15xSbLGISJLZsAFuvRVGvQjHHAMvvsjJp59eJP8qES305fjLuOWoRZxX+04WH2YuJ2vlpv3Kmxx5KD1a5HbRcxEp9XbvhlNPhVGj4LbbYOZMKKJkDolpoY8Fbgiu13cSsLHI+s9DpJa4iMRt3Tr43e+gbFl4+GGoXRvS04v83+ab0M3sLeAMoJqZZeOva1gOwDk3AhgHnIW/GPJW4IqiCjYvuXWLJELWSt+1IiKSJ+fgjTdgwAAYPBiuvhp69iy2fx/PKJeL8lnugOsTFtEByukWKYrEq64VEcnXsmVw7bUwbhycfDK0bVvsIYQ2fW5RULeIiITirbfgmmt8n/nQoXDDDb67pZilVEIXEQlFlSpw0kkwciTUrx9aGEroIiIFtWsXPPkk7NgBd94JXbtCly5gFmpYmpxLRKQgvvvO95HnDEN0wSk1ISdzUEIXEYnP9u1w991++OGyZfDPf8Lbb5eIRJ5DCV1EJB7z58Mjj8DFF0NWFvzxjyUqmYP60EVEcrdlC3z4IfTuDc2awZw50KBB2FHlSi10EZFY/vtfaN4cLr0UZs/2ZSU4mYMSuojIb61fD1ddBZ07Q/ny8MUX0Lhx2FHFJem6XHLmIH/g+W9+U67T80Wk0Hbv9md4zpsHd9wB99wDaWlhRxW3pEvouc1BrtPzReSArV27bzKtQYOgTh1olXxX00y6hA5QqXxZneIvIoXnHLz2Gvz1r34yrX794Lzzwo7qgKkPXURKpyVLoFs3uPxy30fevn3YERWaErqIlD6vv+6HIU6aBM88A19+CY0ahR1VoSVll4uISKFUr+4Pfj7/PNStG3Y0CaOELiKpb+dOePxx//fuu/1EWp07l7gzPQtLXS4iktpmzPBT295xhz9lvwRNppVoSugikpq2bYO//Q1at4YVK+Ddd/2FKFIwkedQQheR1LRgAQwZApdd5k/d79Ur7IiKnPrQRSR1bNkC77/v519p1gzmzg31CkLFTS10EUkNn3wCTZv6ceU5k2mVomQOSugikuzWrfNJvGtXqFTJjylPksm0Ek1dLiKSvHIm01qwwF/b8667kmoyrURTQheR5LNmDVSt6ifTeuQRf3JQixZhRxU6dbmISPJwDl5+GY49Fl54wZf16KFkHlBCF5HksHixP8Pzyiv9lYQ6dAg7ohJHCV1ESr7XXvPDEL/5BoYPhwkTfCtdfkN96CJS8tWo4ae3HTHCX3xCYlJCF5GSZ+dOePRRP4rlnnv8RFqdO4cdVYmnLhcRKVmmT/fzr9x1lz/TM2cyLcmXErqIlAy//goDB0KbNrBqlT+F/403UnoyrUSLK6GbWVczm2tmC8xsYIzldcxsvJnNMLOZZnZW4kMVkZS2aBE88QT06eOnuU3ia3uGJd+EbmZlgWFAN6AJcJGZNYmqdhcwxjnXErgQGJ7oQEUkBW3aBKNH+/tNm8L8+fDii1ClSqhhJat4WuhtgAXOuUXOuR3A20CPqDoOODS4fxiwInEhikhKGjfOD0W86qp9k2ml0OXgwhBPQq8JLIt4nB2URboPuMTMsoFxwJ9jbcjM+plZhpllrFmz5gDCFZGkt3atn962e3eoXBm++qrUTqaVaIk6KHoRMNo5Vws4C3jNzPbbtnNupHMu3TmXXr169QT9axFJGjmTab39th+OOH06nHxy2FGljHjGoS8Hakc8rhWURboK6ArgnPvGzNKAasDqRAQpIklu1SqoXt1PpjVkiO9aOf74sKNKOfG00KcCDc2svpmVxx/0HBtVZynQEcDMGgNpgPpUREo75+Cll+C442DkSF92zjlK5kUk34TunNsF3AB8AszGj2aZZWYPmNm5QbWbgavN7DvgLaCPczobQKRUW7QIOnWCvn39bIidOoUdUcqL69R/59w4/MHOyLJ7Iu5nAW0TG5qIJK1XXoH+/X0Xy4gRcPXVUEbnMRY1zeUiIol31FHwhz/Ac89BrVphR1NqKKGLSOHt2AGDB8OePXDffXDmmf4mxUq/gUSkcKZOhRNPhHvv9f3mOnwWGiV0ETkwW7fCLbf4ceTr18PYsfDqq5pMK0RK6CJyYH78EZ55xh/wnDXLD0eUUKkPXUTit3EjvPceXHGFn0xrwQKoXTv/9aRYqIUuIvH56COfxPv2hTlzfJmSeYmihC4ieVuzBnr3hrPP9tPafvMNNGoUdlQSg7pcRCR3u3fDaaf5/vL77/dXFCpfPuyoJBdK6CKyv59+giOO8Gd6Pv441Kvn5y6XEk1dLiKyz5498PzzcOyx/i/4rhYl86SghC4i3oIF0LEjXHsttG4NXbqEHZEUkBK6iMDLL0Pz5v6CEy+8AJ99Bg0ahB2VFJD60EUE6tTxLfJhw6Bm9BUmJVkooYuURtu3w9//7vvMH3jAd7V07Bh2VFJI6nIRKW2+/dZPpnX//bB0qSbTSiFK6CKlxS+/wE03wSmn+FP4//1vGD1ak2mlECV0kdJiyRIYPtyPYpk1C7p3DzsiSTD1oYuksg0b4J13/PwrTZr4oYm6glDKUgtdJFV9+KFP4tdeu28yLSXzlKaELpJqVq+GCy+E886D6tVh8mRNplVKqMtFJJXs3g1t2/rRKw89BLfdBuXKhR2VFBMldJFUsGIF/P73fjKtp57yk2k1aRJ2VFLM1OUiksz27IHnnvNdKiNG+LKzzlIyL6WU0EWS1bx50KED9O8PJ50E3bqFHZGETAldJBm99BKccALMnAmjRsGnn0L9+mFHJSFTH7pIMqpXz7fIhw2DI48MOxopIZTQRZLB9u3w4IP+/kMPaTItiUldLiIl3ddfQ4sW8PDDsHKlJtOSXCmhi5RUW7bAgAH+Is1bt8J//uP7zjWZluQiroRuZl3NbK6ZLTCzgbnUucDMssxslpm9mdgwRUqhpUv9dT2vvx5++EGXhJN85duHbmZlgWHAmUA2MNXMxjrnsiLqNATuANo659ab2RFFFbBISlu/Hv75T+jXz48lX7QIjjoq7KgkScTTQm8DLHDOLXLO7QDeBnpE1bkaGOacWw/gnFud2DBFSoH33/dJvH9/mDvXlymZSwHEk9BrAssiHmcHZZGOBY41s6/MbLKZdY21ITPrZ2YZZpaxZs2aA4tYJNX89BP86U/Qq5c/fX/KFDjuuLCjkiSUqGGLBwENgTOAWsBEM2vunNsQWck5NxIYCZCenq5D9SK7d0O7drBsGQwaBLfcosm05IDFk9CXA7UjHtcKyiJlA98653YCP5rZPHyCn5qQKEVSTXa2704pWxaeftqf5akpbqWQ4ulymQo0NLP6ZlYeuBAYG1XnA3zrHDOrhu+CWZTAOEVSw5498MwzPnk/95wv69ZNyVwSIt+E7pzbBdwAfALMBsY452aZ2QNmdm5Q7RNgnZllAeOBW51z64oqaJGkNGcOtG8Pf/mLH1t+9tlhRyQpJq4+dOfcOGBcVNk9EfcdcFNwE5FoL74IN9wAlSrBK6/ApZfqBCFJOM3lIlIcjj4azjkHnn0WatQIOxpJUUroIkVh2zZ44AF/f9AgP295hw7hxiQpT3O5iCTaV1/5ybT+/ndYs0aTaUmxUUIXSZTNm+HPf/bjyrdvh08+gRdeUF+5FBsldJFEyc72Bz///Gf4/nvo3DnsiKSUUR+6SGGsWwdjxsB110Hjxn4yLV1BSEKiFrrIgXAO3nnHT6b1l7/sm0xLyVxCpIQuUlArV8L55/sJtWrXhowMTaYlJYK6XEQKImcyreXL4dFH4cYb4SB9jKRk0DtRJB7LlkHNmn4yrWHD/GRaxx4bdlQiv6EuF5G87N7tZ0OMnEyrSxclcymR1EIXyc3s2XDVVfDNN35GxHPOCTsikTyphS4Sy8iR/mzPefPgtdfgo4+gTp2woxLJk1roIrE0bAg9e/ruliN0zXNJDkroIgC//gr33edP0x88WJNpSVJSl4vIxIlwwgl+GOLGjZpMS5KWErqUXps2Qf/+cPrpfjTL55/7kSyaTEuSlBK6lF4rVsDo0XDTTTBzJvzhD2FHJFIo6kOX0mXtWj+ZVv/+fmz5jz/qCkKSMtRCl9LBOfjHP/xkWn/9qx+OCErmklKU0CX1rVgB550HF14IdevCtGk601NSkrpcJLXt3g3t2/vJtIYMgQEDNJmWpCy9syU1LVkCtWr5ybSGD4cGDeCYY8KOSqRIqctFUsvu3fDEE/7qQTmTaXXurGQupYJa6JI6fvjBT6Y1ZQqcfbbvNxcpRdRCl9QwYgS0auWv6fnmmzB2rO9yESlFlNAlueWcpt+4sb8kXFYWXHSRzvaUUkldLpKctm6Fe+7xBz0fecSfvn/66WFHJRIqtdAl+UyYAMcfD48/Dlu2aDItkYASuiSPjRvhmmv2TWv7v//563uqe0UEUEKXZLJyJbz+Otxyi59MS/OVi/xGXAndzLqa2VwzW2BmA/Ood76ZOTNLT1yIUqqtWQPPPOPvN2oEixfDY49BpUqhhiVSEuWb0M2sLDAM6AY0AS4ysyYx6lUGBgDfJjpIKYWc88MPGzeGm2/eN5lW9erhxiVSgsXTQm8DLHDOLXLO7QDeBnrEqPcg8AiwLYHxSWm0bBmccw707u3P8JwxQ5NpicQhnoReE1gW8Tg7KNvLzFoBtZ1zH+W1ITPrZ2YZZpaxZs2aAgcrpcCuXXDGGTB+PDz5JHz1FTRtGnZUIkmh0OPQzawM8ATQJ7+6zrmRwEiA9PR0jTWTfRYvhtq1/UyIzz/vJ9Nq0CDsqESSSjwt9OVA7YjHtYKyHJWBZsAEM1sMnAyM1YFRicuuXX5a28aN/ayIAJ06KZmLHIB4WuhTgYZmVh+fyC8ELs5Z6JzbCFTLeWxmE4BbnHMZiQ1VUs7MmX4yrYwM6NEDzj8/7IhEklq+LXTn3C7gBuATYDYwxjk3y8weMLNzizpASVHDh8OJJ/p5y//xD3j/fTjqqLCjEklqcfWhO+fGAeOiyu7Jpe4ZhQ9LUpZz/szOZs38JeGefBKqVct/PRHJlybnkuLxyy9w113+oOdjj/nLwrVvH3ZUIilFp/5L0fv8c2jeHIYOhe3bNZmWSBFRQpeis2ED9O3rR60cdBBMnAhPP63JtESKiBK6FJ1Vq+Dtt+H22+G776Bdu7AjEklp6kOXxMpJ4gMGwHHH+ROGdNBTpFiohS6J4Zyf2rZJE7jtNpg/35crmYsUGyV0KbylS6F7d7j0Ut8qz8yEhg3Djkqk1FGXixROzmRaq1f7A579+/vrfIpIsVNClwOzaBHUretHr7zwAhx9NNSrF3ZUIqWaulykYHbtgkce8X3lw4b5so4dlcxFSgC10CV+mZl+Mq3p06FnT/jTn8KOSEQiqIUu8Xn2WWjdGpYvh3fegffegyOPDDsqEYmghC55yzlN//jj/SXhsrI0za1ICaUuF4ltyxa4804oV85fgEKTaYmUeGqhy/4+/dRPb/vMM7BzpybTEkkSSuiyz/r1cMUV0KULpKX5ybSeekqTaYkkCSV02Wf1an/A8447/IiW004LOyIRKQD1oZd2P/0Eb70FN964bzKtqlXDjkpEDoBa6KWVc/DKK/4EoTvu2DeZlpK5SNJSQi+NFi+Grl2hTx+f0DWZlkhKUJdLabNrF3ToAGvX+lP3r70Wyuh7XSQVKKGXFgsWQP36fjKtUaOgQQM/uZaIpAw1zVLdzp0waBA0bbpvMq0OHZTMRVKQWuipbPp0P5lWZqafSOv//i/siESkCKmFnqqefhratPHDEt97D8aMgRo1wo5KRIqQEnqqyTlNv2VLuOwyP5lWz57hxiQixUJdLqli82Y/nrxCBXj8cWjXzt9EpNRQCz0V/Oc/fjKt4cN9C12TaYmUSkroyWzdOrj8cujWDQ4+GL76Cp54QpNpiZRSSujJbN06eP99uPtumDEDTjkl7IhEJERx9aGbWVfgKaAs8KJzbnDU8puAvsAuYA1wpXNuSYJjFYCVK+GNN+Dmm+HYY2HJEqhSJeyoJIns3LmT7Oxstm3bFj4NQ4sAABGhSURBVHYokoe0tDRq1apFuXLl4l4n34RuZmWBYcCZQDYw1czGOueyIqrNANKdc1vN7DrgUUCDnhPJOXj5ZbjpJti+HXr08POvKJlLAWVnZ1O5cmXq1auHqXuuRHLOsW7dOrKzs6lfv37c68XT5dIGWOCcW+Sc2wG8DfSI+ufjnXNbg4eTgVpxRyD5+/FH6NzZnyR0wgnw3XeaTEsO2LZt26hataqSeQlmZlStWrXAv6Li6XKpCSyLeJwNnJRH/auAj2MtMLN+QD+AOnXqxBliKbdrF/zhD76//LnnoF8/TaYlhaZkXvIdyGuU0HHoZnYJkA6cHmu5c24kMBIgPT1dY+vyMn++n0DroIN8V8vRR0Pt2mFHJSIlWDxNveVAZCapFZT9hpl1Au4EznXObU9MeKXQzp3w0EN+XPmzz/qyM85QMpeUsWHDBoYPH35A6w4dOpStW7fmX7GUiiehTwUamll9MysPXAiMjaxgZi2B5/HJfHXiwywlMjIgPd0PQ+zVCy66KOyIRBJOCb3o5Nvl4pzbZWY3AJ/ghy2Ocs7NMrMHgAzn3FjgMeAQ4J9Bv89S59y5RRh36nnqKT+C5fe/hw8/hHP19EnRu/9fs8hasSmh22xy1KHce07TXJcPHDiQhQsX0qJFC84880yOOOIIxowZw/bt2+nZsyf3338/v/zyCxdccAHZ2dns3r2bu+++m1WrVrFixQo6dOhAtWrVGD9+fELjTgVx9aE758YB46LK7om43ynBcZUezvkzO9PT/SiWRx+Fww8POyqRIjN48GB++OEHMjMz+fTTT3nnnXeYMmUKzjnOPfdcJk6cyJo1azjqqKP46KOPANi4cSOHHXYYTzzxBOPHj6datWoh70XJpMm5wrJpE9x+O6SlwZNPQtu2/iZSjPJqSReHTz/9lE8//ZSWLVsCsGXLFubPn0+7du24+eabuf322zn77LNpp4nm4qKEHoZx4+Caa2DFCt/NktNKFyllnHPccccdXHPNNfstmz59OuPGjeOuu+6iY8eO3HPPPTG2IJE0oLk4rV0Ll1wC3bvDYYfB11/DY48pmUupUrlyZTZv3gxAly5dGDVqFFu2bAFg+fLlrF69mhUrVlCpUiUuueQSbr31VqZPn77furI/tdCL0/r18K9/wb33wt/+BuXLhx2RSLGrWrUqbdu2pVmzZnTr1o2LL76YU4KJ5Q455BBef/11FixYwK233kqZMmUoV64czz33HAD9+vWja9euHHXUUTooGoO5kObOTk9PdxkZGQVeb9ag0wBo+rdJiQ6paCxf7ifTuvVW3xLfsEEHPSVUs2fPpnHjxmGHIXGI9VqZ2TTnXHqs+upyKSrOwQsvQJMmcN99sHChL1cyF5EiooReFBYuhI4d/bwrrVrBzJlwzDFhRyUiKU596Im2a5dP5j//DM8/D337ajItESkWSuiJMneun0DroIPglVf8/VqaRVhEio+ajoW1Ywfcfz80bw7Dhvmy009XMheRYqcWemFMmeJP1//hB7j4YujdO+yIRKQUUwv9QA0d6i/KnDO2/I03QPNLiOTrQGdbPOuss9iwYUMRRJQ6lNALKmfcfps2cPXVMGsWnH12uDGJJJHcEvquXbvyXG/cuHEcrmG/eVKXS7w2boTbboOKFX3r/NRT/U0kmX08EH76PrHb/H1z6DY418WR0+eWK1eOtLQ0qlSpwpw5c5g3bx7nnXcey5YtY9u2bQwYMIB+/foBUK9ePTIyMtiyZQvdunXjtNNO4+uvv6ZmzZp8+OGHVKxYMbH7kYTUQo/Hv/7lTxB68UWoUGFfK11ECmzw4MEcffTRZGZm8thjjzF9+nSeeuop5s2bB8CoUaOYNm0aGRkZPP3006xbt26/bcyfP5/rr7+eWbNmcfjhh/Puu+8W926USGqh52XNGhgwAN56y49i+eADaN067KhEEiePlnRxadOmDfXr19/7+Omnn+b9998HYNmyZcyfP5+qVav+Zp369evTokULAE488UQWL15cbPGWZEroedm40U91e//9MHCgJtMSKQIHH3zw3vsTJkzgs88+45tvvqFSpUqcccYZbNu2bb91KlSosPd+2bJl+fXXX4sl1pJOCT3asmXw+us+gR9zDCxZ4qe6FZGEyGsK3I0bN1KlShUqVarEnDlzmDx5cjFHl9yU0HPs2QMjR/oDn7t3w5/+5BO6krlIQkVOn1uxYkVq1Kixd1nXrl0ZMWIEjRs35rjjjuPkk08OMdLko4QOMH++H4L4xRd+HpaRI6FBg7CjEklZb775ZszyChUq8PHHH8dcltNPXq1aNX744Ye95bfcckvC40tWSui7dsGZZ/p5yl96Ca64QlcQEpGkVHoT+uzZ0LChn0zrtdf8ZFpHHRV2VCIiB6z0jUPfvt1fAu744+HZZ31Zu3ZK5iKS9EpXC33yZD+ZVlYWXHqpv4mIpIjS00J//HF/qv7mzX5s+auvQtTJCiIiySz1E/qePf7vKafAtdf6qW67dQs3JhGRIpC6CX3DBt+9MmCAf3zqqTB8OBx6aLhxiUjC1atXj7Vr14YdRuhSM6F/8IGfTOuVV6ByZU2mJVJCOefYk/MrWgottRL66tVwwQXQsyfUqOGvKDRokMaVi+TljDP2v+XMV751a+zlo0f75WvX7r8sH4sXL+a4447jsssuo1mzZixbtozrrruO9PR0mjZtyr333ru3br169bj33ntp1aoVzZs3Z86cOQCsW7eOzp0707RpU/r27YuLaLQ98cQTNGvWjGbNmjF06NC9/7NRo0b06dOHY489lt69e/PZZ5/Rtm1bGjZsyJQpU/aLc+vWrVxwwQU0adKEnj17ctJJJ5GRkQHAIYccsrfeO++8Q58+fQBYs2YN559/Pq1bt6Z169Z89dVXAHzxxRe0aNGCFi1a0LJlSzZv3szKlStp3749LVq0oFmzZnz55Zf5Pnf5Sa2EvmkT/Pe/8PDDPpm3ahV2RCISw/z58+nfvz+zZs2ibt26PPzww2RkZDBz5ky++OILZs6cubdutWrVmD59Otdddx1DhgwB4P777+e0005j1qxZ9OzZk6VLlwIwbdo0Xn75Zb799lsmT57MCy+8wIwZMwBYsGABN998M3PmzGHOnDm8+eabTJo0iSFDhjBo0KD9Yhw+fDhVqlQhKyuLBx98kGnTpuW7XwMGDODGG29k6tSpvPvuu/Tt2xeAIUOGMGzYMDIzM/nyyy+pWLEib775Jl26dCEzM5Pvvvtu7+yRhZH8wxaXLvUnBv3tb37ulaVLfTeLiMRnwoTcl1WqlPfyatXyXp6LunXr/maeljFjxjBy5Eh27drFypUrycrK4vjjjwegV69egJ8m97333gNg4sSJe+93796dKlWqADBp0iR69uy5dwbHXr168eWXX3LuuedSv359mjdvDkDTpk3p2LEjZkbz5s1jTr87adIkBgTH4Jo1a7Y3nrx89tlnZGVl7X28adMmtmzZQtu2bbnpppvo3bs3vXr1olatWrRu3Zorr7ySnTt3ct555yUkocfVQjezrmY218wWmNnAGMsrmNk/guXfmlm9QkeWnz17/M/Cpk19t8rChb5cyVykxIucMvfHH39kyJAhfP7558ycOZPu3bv/ZsrcnKlyy5Ytm+9l6vISOeVumTJl9j4uU6ZMgbdrEd24kbHu2bOHyZMnk5mZSWZmJsuXL+eQQw5h4MCBvPjii/z666+0bduWOXPm0L59eyZOnEjNmjXp06cPr7766gHv2979iiPwssAwoBvQBLjIzJpEVbsKWO+cOwZ4Enik0JHlofyaoF/v+uv9cMRZs3zrXESSzqZNmzj44IM57LDDWLVqVa6Tc0Vq37793gm+Pv74Y9avXw9Au3bt+OCDD9i6dSu//PIL77//Pu3atTuguNq2bcuYMWMAyMrK4vvv912qr0aNGsyePZs9e/bsvRgHQOfOnXnmmWf2Ps7MzARg4cKFNG/enNtvv53WrVszZ84clixZQo0aNbj66qvp27cv06dPP6A4I8XT5dIGWOCcWwRgZm8DPYCsiDo9gPuC++8Az5qZOVcEw0t2O+qOzgIqwssvw+WX66CnSBI74YQTaNmyJY0aNaJ27dq0bds233XuvfdeLrroIpo2bcqpp55KnTp1AGjVqhV9+vShTZs2APTt25eWLVse0BWN+vfvz+WXX06TJk1o1KgRTZs25bBgOu3Bgwdz9tlnU716ddLT09myZQvgr7Z0/fXXc/zxx7Nr1y7at2/PiBEjGDp0KOPHj6dMmTI0bdqUbt268fbbb/PYY49Rrlw5DjnkkIS00C2/nGtmfwS6Ouf6Bo8vBU5yzt0QUeeHoE528HhhUGdt1Lb6Af0A6tSpc+KSJUsKHPDk4VdTecFPNL11JBx5ZIHXFyntZs+eTePGjcMOo8TbvXs3O3fuJC0tjYULF9KpUyfmzp1L+WK8clms18rMpjnn0mPVL9aDos65kcBIgPT09ANqvZ/c/4WExiQiEsvWrVvp0KEDO3fuxDnH8OHDizWZH4h4EvpyoHbE41pBWaw62WZ2EHAYsP+lukVEkkTlypX3jjtPFvGMcpkKNDSz+mZWHrgQGBtVZyxweXD/j8D/iqT/XEQSQh/Pku9AXqN8E7pzbhdwA/AJMBsY45ybZWYPmNm5QbWXgKpmtgC4CdhvaKOIlAxpaWmsW7dOSb0Ec86xbt060tLSCrRevgdFi0p6erpLtp8zIqlg586dZGdn/2b8tJQ8aWlp1KpVi3Llyv2mvMQcFBWR8JUrV4769euHHYYUgdSay0VEpBRTQhcRSRFK6CIiKSK0g6JmtgYo+KmiXjWgtF2eRPtcOmifS4fC7HNd51z1WAtCS+iFYWYZuR3lTVXa59JB+1w6FNU+q8tFRCRFKKGLiKSIZE3oI8MOIATa59JB+1w6FMk+J2UfuoiI7C9ZW+giIhJFCV1EJEWU6IReIi9OXcTi2OebzCzLzGaa2edmVjeMOBMpv32OqHe+mTkzS/ohbvHss5ldELzWs8zszeKOMdHieG/XMbPxZjYjeH+fFUaciWJmo8xsdXBFt1jLzcyeDp6PmWbWqtD/1DlXIm9AWWAh0AAoD3wHNImq0x8YEdy/EPhH2HEXwz53ACoF968rDfsc1KsMTAQmA+lhx10Mr3NDYAZQJXh8RNhxF8M+jwSuC+43ARaHHXch97k90Ar4IZflZwEfAwacDHxb2P9Zklvoey9O7ZzbAeRcnDpSD+CV4P47QEezpL5idL777Jwb75zbGjycjL+CVDKL53UGeBB4BEiFOV/j2eergWHOufUAzrnVxRxjosWzzw44NLh/GLCiGONLOOfcRODnPKr0AF513mTgcDMr1IWSS3JCrwksi3icHZTFrOP8hTg2AlWLJbqiEc8+R7oK/w2fzPLd5+CnaG3n3EfFGVgRiud1PhY41sy+MrPJZta12KIrGvHs833AJWaWDYwD/lw8oYWmoJ/3fGk+9CRlZpcA6cDpYcdSlMysDPAE0CfkUIrbQfhulzPwv8Immllz59yGUKMqWhcBo51zj5vZKcBrZtbMObcn7MCSRUluoRfk4tSkyMWp49lnzKwTcCdwrnNuezHFVlTy2+fKQDNggpktxvc1jk3yA6PxvM7ZwFjn3E7n3I/APHyCT1bx7PNVwBgA59w3QBp+EqtUFdfnvSBKckIvjRenznefzawl8Dw+mSd7vyrks8/OuY3OuWrOuXrOuXr44wbnOueS+fqF8by3P8C3zjGzavgumEXFGWSCxbPPS4GOAGbWGJ/Q1xRrlMVrLHBZMNrlZGCjc25lobYY9pHgfI4Sn4VvmSwE7gzKHsB/oMG/4P8EFgBTgAZhx1wM+/wZsArIDG5jw465qPc5qu4EknyUS5yvs+G7mrKA74ELw465GPa5CfAVfgRMJtA57JgLub9vASuBnfhfXFcB1wLXRrzGw4Ln4/tEvK916r+ISIooyV0uIiJSAEroIiIpQgldRCRFKKGLiKQIJXQRkRShhC4ikiKU0EVEUsT/A6afmaXy7nZrAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Function plot_precision_recall_curve is deprecated; Function `plot_precision_recall_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: PrecisionRecallDisplay.from_predictions or PrecisionRecallDisplay.from_estimator.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZhV1Znv8e+PgjDIlAjaNqCAEJkEDChB9AqmjcZuUTGOiYqJmhjUXLvVG2OCSmJIHGJiay5iBNTY4hgtI0qMoqYjokUzKNDKIMZCb8QBxEaEwvf+sXdVDsWhzimqTp0afp/nqaf2XmsP7yqoes9ea++1FRGYmZlV16rYAZiZWePkBGFmZlk5QZiZWVZOEGZmlpUThJmZZdW62AHUl27dukXv3r2LHYaZWZOycOHC9yKie7a6ZpMgevfuTVlZWbHDMDNrUiS9uas6dzGZmVlWThBmZpaVE4SZmWXlBGFmZlk5QZiZWVYFSxCSZkh6V9Kru6iXpJslrZK0VNKXMurOlrQy/Tq7UDGamdmuFfIKYhZwTA31XwP6p1/nA/8XQNIXgKuAUcAhwFWSPl/AOM3MLIuCPQcREc9L6l3DJscDd0Uy3/iLkrpK2gcYCzwVER8ASHqKJNHcW6hYr3lsGcvf/qhQhzezFu744T04Y9S+xQ6j1oo5BtEDeCtjvTwt21X5TiSdL6lMUtn69esLFqiZ2e5a/s5HPLp4XbHD2C1N+knqiJgOTAcYOXLkbr/56KrjBtdbTGZmmU69bX6xQ9htxbyCWAf0yljvmZbtqtzMzBpQMRNEKXBWejfTl4GNEfEOMBf4qqTPp4PTX03LzMysARWsi0nSvSQDzt0klZPcmdQGICKmAXOAY4FVwGbgnLTuA0k/AV5ODzWlcsDazMwaTiHvYjo9R30Ak3ZRNwOYUYi4zMwsP36S2szMsnKCMDOzrJwgzMwsKycIMzPLygnCzMyycoIwM7OsnCDMzCwrJwgzM8vKCcLMzLJygjAzs6ycIMzMLCsnCDMzy8oJwszMsnKCMDOzrJwgzMwsKycIMzPLygnCzMyycoIwM7OsnCDMzCwrJwgzM8vKCcLMzLJygjAzs6wKmiAkHSPpNUmrJP0gS/1+kp6WtFTSs5J6ZtT9QtKr6dephYzTzMx2VrAEIakEuBX4GjAIOF3SoGqb3QDcFRFDgSnA1HTffwa+BAwHRgGXSupcqFjNzGxnhbyCOARYFRFrImIrMBs4vto2g4Bn0uV5GfWDgOcjoiIi/gdYChxTwFjNzKyaQiaIHsBbGevlaVmmJcCEdPlEoJOkPdPyYyR1kNQNGAf0KmCsZmZWTbEHqS8FjpC0CDgCWAdsj4g/AnOAF4B7gfnA9uo7SzpfUpmksvXr1zdg2GZmzV8hE8Q6dvzU3zMtqxIRb0fEhIg4CLgyLduQfr82IoZHxFGAgNernyAipkfEyIgY2b1790K1w8ysRWqdz0aS9gLGAP8IfAK8CpRFxGc17PYy0F9SH5LEcBpwRrXjdgM+SI9zBTAjLS8BukbE+5KGAkOBP9amYWZmTdl/LPgrjy7++2fq44f34IxR+zZoDDUmCEnjgB8AXwAWAe8C7YATgP0lPQjcGBEfVd83IiokXQjMBUqAGRGxTNIUkuRSCowFpkoK4HlgUrp7G+DPkgA+Ar4ZERV1bayZWWNSPQlkWvDGBwCM6vMFlr+T/IltVAkCOBY4LyL+Wr1CUmvgX4CjgIey7RwRc0jGEjLLJmcsPwg8mGW/LSR3MpmZNWmVf+hPvW3+LutG9fnCTnWj+nyh6qoh274NocYEERGX1VBXATxS7xGZmbUQmUmgMcprDCIbSedExMz6DMbMrLk5rF83AH537qgiR1J7u50ggGsAJwgzsxo0xcRQKdcg9dJdVQF71384ZmbWWOS6gtgbOBr4sFq5SB5iMzOzZipXgvgD0DEiFlevkPRsQSIyM7NGIdddTN+uoe6MXdWZmVnTV+y5mMzMrJFygjAzs6ycIMzMLCsnCDMzyyrvBCFpek3rZmbWvNTmCuK2HOtmZtaM5J0gImJhTetmZta85Jpq4zEgdlUfEePrPSIzM2sUcj1JfUODRGFmZo1Oriepn6tcltQe2DciXit4VGZmVnR5jUFIOg5YDDyZrg+XVFrIwMzMrLjyHaS+GjgE2ACQTt7Xp0AxmZlZI5BvgtgWERurle1y8NrMzJq+fN8ot0zSGUCJpP7Axfh9EGZmzVq+VxAXAYOBT4F7gY+A/12ooMzMrPjyuoKIiM3AlZJ+kazGpsKGZWZmxZbvXUwHS3oFWAq8ImmJpBF57HeMpNckrZL0gyz1+0l6WtJSSc9K6plRd52kZZJWSLpZkmrTMDMzq5t8u5juAL4XEb0jojcwCZhZ0w6SSoBbga8Bg4DTJQ2qttkNwF0RMRSYAkxN9z0UGAMMBYYABwNH5BmrmZnVg3wTxPaI+HPlSkT8J1CRY59DgFURsSYitgKzgeOrbTMIeCZdnpdRH0A74HNAW6AN8Lc8YzUzs3pQY4KQ9CVJXwKek3SbpLGSjpD0G+DZHMfuAbyVsV6elmVaAkxIl08EOknaMyLmkySMd9KvuRGxIkt850sqk1S2fv36HOGYmVlt5BqkvrHa+lUZy/XxHMSlwC2SJgLPA+uA7ZL6AQOByjGJpyQdnnkVAxAR04HpACNHjvRzGWZm9SjXXEzj6nDsdUCvjPWeaVnm8d8mvYKQ1BE4KSI2SDoPeDEiPk7rngBGAzskCDMzK5x8H5RD0j+TPAvRrrIsIqbUsMvLQH9JfUgSw2nAGdWO2Q34ICI+A64AZqRVfwXOkzQVEMkA9a/yjdXMzOou39tcpwGnkjwwJ+BkYL+a9omICuBCYC6wArg/IpZJmiKp8j0SY4HXJL0O7A1cm5Y/CKwGXiEZp1gSEY/Vol1mZlZH+V5BHBoRQyUtjYhrJN0IPJFrp4iYA8ypVjY5Y/lBkmRQfb/twHfyjM3MzAog39tcP0m/b5b0j8A2YJ/ChGRmZo1BvlcQf5DUFbge+C+SO5h+W7CozMys6PKdi+kn6eJDkv4AtMsy/beZmTUjNSYISRNqqCMiHq7/kMzMrDHIdQVxXA11AThBmJk1U7kelDunoQIxM7PGJd+7mMzMrIVxgjAzs6ycIMzMLKt8p9roIOnHkm5P1/tL+pfChmZmZsWU7xXETOBTkhlVIZl876cFicjMzBqFfBPE/hFxHckUG0TEZpJJ+8zMrJnKN0FsldSe9CVBkvYnuaIwM7NmKt+5mK4GngR6SboHGANMLFBMZmbWCOQ7F9MfJS0EvkzStfT9iHivoJGZmVlR5ZUgJD0G/AdQGhH/U9iQzMysMch3DOIG4HBguaQHJX1dUrtcO5mZWdOVbxfTc8BzkkqAI4HzSN4f3bmAsZmZWRHlO0hNehfTcSTvpv4ScGehgjIzs+LLdwzifuAQkjuZbgGei4jPChmYmZkVV75XEHcAp0fE9kIGY2ZmjUeuN8odGRHPAHsAx0s7PjztN8qZmTVfue5iOiL9flyWr5yT9Uk6RtJrklZJ+kGW+v0kPS1pqaRnJfVMy8dJWpzxtUXSCbVqmZmZ1UmuN8pdlS5OiYg3Musk9alp3/SOp1uBo4By4GVJpRGxPGOzG4C7IuJOSUcCU4EzI2IeMDw9zheAVcAf82+WmZnVVb7PQTyUpezBHPscAqyKiDURsRWYDRxfbZtBwDPp8rws9QBfB55IJwg0M7MGkmsMYgAwGOgiaUJGVWcg14NyPYC3MtbLgVHVtlkCTAB+DZwIdJK0Z0S8n7HNacAvdxHf+cD5APvuu2+OcMzMrDZy3cV0AMlYQ1eScYdKm0gelqurS4FbJE0Enid5z0TVnVKS9gEOBOZm2zkipgPTAUaOHBn1EI+ZmaVyjUE8CjwqaXREzK/lsdcBvTLWe6Zlmcd/m+QKAkkdgZMiYkPGJqcAv4+IbbU8t5mZ1VGuLqbL0xcFnSHp9Or1EXFxDbu/DPRPB7PXkXQVnVHt+N2AD9KH7q4gmb4j0+lpuZmZNbBcXUwr0u9ltT1wRFRIupCke6gEmBERyyRNAcoiohQYC0yVFCRdTJMq95fUm+QK5LnantvMzOouVxfTY+n3qnmXJLUCOkbER7kOHhFzgDnVyiZnLD/ILu6Gioi1JAPdZmZWBHnd5irpPyR1lrQH8CrJtN+XFTY0MzMrpnyfgxiUXjGcADwB9AHOLFhUZmZWdPkmiDaS2pAkiNL0riLfVmpm1ozlmyBuA9aSTNr3vKT9gJxjEGZm1nTl+0a5m4GbM4relDSuMCGZmVljkO8gdRdJv5RUln7dSHI1YWZmzVS+XUwzSKbXOCX9+giYWaigzMys+PJ9o9z+EXFSxvo1khYXIiAzM2sc8r2C+ETSYZUrksYAnxQmJDMzawzyvYL4LnCXpC7p+ofA2YUJyczMGoOcCULScKAfyWR76wDymWbDzMyathq7mCRNBu4HTgIeB051cjAzaxlyXUGcCgyPiM2S9gSeBG4vfFhmZlZsuQapP618F3T6GtB8B7XNzKyJy3UF0VdSabosYP+MdSJifMEiMzOzosqVII6vtn5DoQIxM7PGJdcLg/w2NzOzFirXXUyPSTouneq7el1fSVMkfatw4ZmZWbHk6mI6D/hX4FeSPgDWA+2A3sBq4JaIeLSgEZqZWVHk6mL6f8DlwOWSegP7kEyx8Xrl3U1mZtY85TvVBhGxluSlQWZm1gL4uQYzM8uqoAlC0jGSXpO0StIPstTvJ+lpSUslPSupZ0bdvpL+KGmFpOVpF5eZmTWQgiUISSXArcDXgEHA6ZIGVdvsBuCuiBgKTAGmZtTdBVwfEQOBQ4B3CxWrmZntLN9Xjo6R9JSk1yWtkfSGpDU5djsEWBURayJiKzCbnR+8GwQ8ky7Pq6xPE0nriHgKICI+9qC4mVnDyvcK4g7gl8BhwMHAyPR7TXoAb2Wsl6dlmZYAE9LlE4FO6aSAXwQ2SHpY0iJJ16dXJDuQdH7le7LXr1+fZ1PMzCwf+SaIjRHxRES8GxHvV37Vw/kvBY6QtAg4guR9E9tJ7q46PK0/GOgLTKy+c0RMj4iRETGye/fu9RCOmZlVyvc213mSrgceBj6tLIyI/6phn3VAr4z1nmlZlYh4m/QKQlJH4KSI2CCpHFgcEWvSukeAL5NcyZiZWQPIN0GMSr+PzCgL4Mga9nkZ6C+pD0liOA04I3MDSd2ADyLiM+AKYEbGvl0ldY+I9el5yvKM1czM6kFeCSIixtX2wBFRIelCYC5QAsyIiGWSpgBlEVEKjAWmSgrgeWBSuu92SZcCT0sSsBC/qMjMrEHllSAkdQGuAv5XWvQcMCUiNta0X0TMAeZUK5ucsfwg8OAu9n0KGJpPfGZmVv/yHaSeAWwCTkm/PgJmFiooMzMrvnzHIPaPiJMy1q+RtLgQAZmZWeOQ7xXEJ5IOq1yRNIZkVlczM2um8r2CuAC4Mx2LEPABWZ5LMDOz5iPfu5gWA8MkdU7XPypoVGZmVnQ1JghJ34yI30n612rlAETELwsYm5mZFVGuK4g90u+dCh2ImZk1LrleOXpb+v2ahgnHzMwai3yn+75OUmdJbdIX/KyX9M1CB2dmZsWT722uX00Hpv+F5L3U/YDLChWUmZkVX74JorIr6p+BB3JNsWFmZk1fvs9B/EHSf5M8HHeBpO7AlsKFZWZmxZbXFURE/AA4FBgZEduA/2Hn14eamVkzkus5iCMj4hlJEzLKMjd5uFCBmZlZceXqYjoCeAY4Lktd4ARhZtZs5XoO4qr0+zkNE46ZmTUW+T4H8TNJXTPWPy/pp4ULy8zMii3f21y/FhEbKlci4kPg2MKEZGZmjUG+CaJEUtvKFUntgbY1bG9mZk1cvs9B3AM8LanyNaPnAHcWJiQzM2sM8n0fxC8kLQH+KS36SUTMLVxYZmZWbPleQQCsACoi4k+SOkjqFBGbChWYmZkVV753MZ0HPAjclhb1AB4pVFBmZlZ8+Q5STwLGAB8BRMRKYK9cO0k6RtJrklZJ+kGW+v3S6cOXSnpWUs+Muu2SFqdfpXnGaWZm9STfLqZPI2Jr5TQbklqTPEm9S5JKgFuBo4By4GVJpRGxPGOzG4C7IuJOSUcCU4Ez07pPImJ4/k0xM7P6lO8VxHOSfgi0l3QU8ADwWI59DgFWRcSaiNgKzGbnCf4GkUzlATAvS72ZmRVJvgni/wDrgVeA7wBzgB/l2KcH8FbGenlalmkJUDkR4IlAJ0l7puvtJJVJelHSCdlOIOn8dJuy9evX59kUMzPLR84uprSraFlEDABur+fzXwrcImki8DywDtie1u0XEesk9QWekfRKRKzO3DkipgPTAUaOHFljl5eZmdVOzgQREdvTgeZ9I+KvtTj2OqBXxnrPtCzz2G+TXkFI6gicVDmlR0SsS7+vkfQscBCwQ4IwM7PCyXeQ+vPAMkkvkbwsCICIGF/DPi8D/SX1IUkMpwFnZG4gqRvwQUR8BlwBzEjLPw9sjohP023GANflGauZmdWDfBPEj2t74IiokHQhMBcoAWZExDJJU4CyiCgFxgJTJQVJF9OkdPeBwG2SPiMZJ/l5tbufzMyswHK9Ua4d8F2gH8kA9R0RUZHvwSNiDsmAdmbZ5IzlB0kewKu+3wvAgfmex8zM6l+uu5juBEaSJIevATcWPCIzM2sUcnUxDYqIAwEk3QG8VPiQzMysMch1BbGtcqE2XUtmZtb05bqCGCbpo3RZJE9Sf5QuR0R0Lmh0ZmbGgjc+AODU2+ZXlR0/vAdnjNq3oOetMUFERElBz25mZrW2/J3kc3tRE4SZmTUe931nNLDjlUQhOUGYmTVyh/XrVpTzOkGYmTVyvzt3VFHOm+9srmZm1sI4QZiZWVZOEGZmlpUThJmZZeUEYWZmWTlBmJlZVk4QZmaWlROEmZll5QRhZmZZOUGYmVlWThBmZpaVE4SZmWXlBGFmZlk5QZiZWVYFne5b0jHAr4ES4LcR8fNq9fsBM4DuwAfANyOiPKO+M7AceCQiLqzt+bdt20Z5eTlbtmypQyvMLFO7du3o2bMnbdq0KXYoVmAFSxCSSoBbgaOAcuBlSaURsTxjsxuAuyLiTklHAlOBMzPqfwI8v7sxlJeX06lTJ3r37o2k3T2MmaUigvfff5/y8nL69OlT7HCswArZxXQIsCoi1kTEVmA2cHy1bQYBz6TL8zLrJY0A9gb+uLsBbNmyhT333NPJwayeSGLPPff0VXkLUcgE0QN4K2O9PC3LtASYkC6fCHSStKekVsCNwKU1nUDS+ZLKJJWtX79+V9vsTuxmtgv+nWo5ij1IfSlwhKRFwBHAOmA78D1gTuZ4RDYRMT0iRkbEyO7duxc+WjOzFqSQCWId0CtjvWdaViUi3o6ICRFxEHBlWrYBGA1cKGktyTjFWZJ2GOBuKkpKShg+fDhDhgzhuOOOY8OGDfVy3FmzZnHhhbUet8+qd+/eHHjggQwfPpzhw4fzwgsv1Mtxq1u8eDFz5szZoeyJJ55g5MiRDBo0iIMOOoh/+7d/A+Dqq6/mhhtuqLdzH3rooVXLl112GYMHD+ayyy5j2rRp3HXXXXU69qJFi/j2t7+9Q9kJJ5zAl7/85R3Krr76anr06FH1/6G0tLRO5wW48sor6dWrFx07dqxxu6lTp9KvXz8OOOAA5s6dW1X+5JNPcsABB9CvXz9+/vO//4qddtpprFy5ss7xWRMXEQX5IhkAXwP0AT5H0p00uNo23YBW6fK1wJQsx5kI3JLrfCNGjIjqli9fvlNZQ9tjjz2qls8666z46U9/Wi/HnTlzZkyaNKlejrXffvvF+vXra73ftm3barV99ZhfeeWV6Nu3b6xYsSIiIioqKuI3v/lNRERcddVVcf3119c6pnx07tw5KioqdmvfbG3++te/HosXL65a//DDD6Nnz54xYMCAWL16dVV5ZpuWL18ee+65Z2zfvn234qg0f/78ePvtt3f4f1bdsmXLYujQobFly5ZYs2ZN9O3bNyoqKqKioiL69u0bq1evjk8//TSGDh0ay5Yti4iIZ599Ns4999xdHrMx/G61ZKdMeyFOmfZCvRwLKItd/F0t2F1MEVEh6UJgLsltrjMiYpmkKWlApcBYYKqkILlbaVKh4rnmsWUsf/ujej3moH/szFXHDc57+9GjR7N06VIAXnrpJb7//e+zZcsW2rdvz8yZMznggAOYNWsWpaWlbN68mdWrV3PiiSdy3XXXATBz5kymTp1K165dGTZsGG3btgVg7dq1fOtb3+K9996je/fuzJw5k3333ZeJEyfSvn17Fi1axLvvvsuMGTO46667mD9/PqNGjWLWrFm7jLWmY7Zr145FixYxZswYJk2axKRJk1i/fj0dOnTg9ttvZ8CAATzwwANcc801lJSU0KVLF/70pz8xefJkPvnkE/7zP/+TK664gscff5wrr7ySAQMGAMnV1gUXXLBTLLfffjvTp09n69at9OvXj7vvvpsOHTrsdI7nn3+eZcuWcc4557B161Y+++wzHnroIfr370/Hjh35+OOPGT9+PB9//DEjRozgiiuuYMWKFXTs2JFLL72U1atXZ21L9Tb/8pe/rIpt06ZNLF26lGHDhlWVPfzwwxx33HHsvffezJ49mx/+8Ic7tWngwIG0bt2a9957j7322ivv/0PVVb9KyebRRx/ltNNOo23btvTp04d+/frx0ksvAdCvXz/69u0LJFcNjz76KIMGDeLwww9n4sSJVFRU0Lp1Qe+Gt0asoP/yETEHmFOtbHLG8oPAgzmOMQuYVYDwGtT27dt5+umnq7oiBgwYwJ///Gdat27Nn/70J374wx/y0EMPAUlXzKJFi2jbti0HHHAAF110Ea1bt+aqq65i4cKFdOnShXHjxnHQQQcBcNFFF3H22Wdz9tlnM2PGDC6++GIeeeQRAD788EPmz59PaWkp48eP5y9/+Qu//e1vOfjgg1m8eDHDhw8HYNy4cZSUlNC2bVsWLFhQ4zHLy8t54YUXKCkp4Stf+QrTpk2jf//+LFiwgO9973s888wzTJkyhblz59KjRw82bNjA5z73OaZMmUJZWRm33HILAL/4xS+qupRqMmHCBM477zwAfvSjH3HHHXdw0UUX7XQOgGnTpvH973+fb3zjG2zdupXt27fvcKzS0lI6duzI4sWLgaTbp9L555+ftS3V25yprKyMIUOG7FB27733MnnyZPbee29OOumkrAliwYIFtGrViupjZ/PmzeOSSy7ZafsOHTrsdtffunXrdkgkPXv2ZN26pLe3V69eO5QvWLAAgFatWtGvXz+WLFnCiBEjduu81vS1mI8GtfmkX58++eQThg8fzrp16xg4cCBHHXUUABs3buTss89m5cqVSGLbtm1V+3zlK1+hS5cuAAwaNIg333yT9957j7Fjx1b9QTn11FN5/fXXAZg/fz4PP/wwAGeeeSaXX3551bGOO+44JHHggQey9957c+CBBwIwePBg1q5dW5Ug5s2bR7du3ar2q+mYJ598MiUlJXz88ce88MILnHzyyVV1n376KQBjxoxh4sSJnHLKKUyYMIG6ePXVV/nRj37Ehg0b+Pjjjzn66KN3eY7Ro0dz7bXXUl5ezoQJE+jfv39e56ipLZltru6dd97Z4Y/83/72N1auXMlhhx2GJNq0acOrr75alURuuukmfve739GpUyfuu+++ne4IGjduXFXyKra99tqLt99+2wmiBWsxCaJY2rdvz+LFi9m8eTNHH300t956KxdffDE//vGPGTduHL///e9Zu3YtY8eOrdqnsusIkm6XioqK3T5/5bFatWq1w3FbtWq128fdY489APjss8/o2rVr1j9o06ZNY8GCBTz++OOMGDGChQsX7rTN4MGDWbhw4Q7dM9lMnDiRRx55hGHDhjFr1iyeffbZXZ7jjDPOYNSoUTz++OMce+yx3HbbbRx55JE521RTWzLbXF379u13eCbg/vvv58MPP6x6iOyjjz7i3nvv5dprrwXgkksu4dJLd333diGuIHr06MFbb/39jvPy8nJ69EjuON9VOVDV/WmNz4I3PgDg1NvmA7Xv7s5XsW9zbTE6dOjAzTffzI033khFRQUbN26s+mWsaSyg0qhRo3juued4//332bZtGw888EBV3aGHHsrs2bMBuOeeezj88MPrHG8+x+zcuTN9+vSpiiUiWLJkCQCrV69m1KhRTJkyhe7du/PWW2/RqVMnNm3aVLX/ZZddxs9+9rOqK6HPPvuMadOm7XSeTZs2sc8++7Bt2zbuueeeqvJs51izZg19+/bl4osv5vjjj68a88mlprbUZODAgaxatapq/d577+XJJ59k7dq1rF27loULF1b9HPNReQVR/asud5aNHz+e2bNn8+mnn/LGG2+wcuVKDjnkEA4++GBWrlzJG2+8wdatW5k9ezbjx4+v2u/111/fqfvMWhYniAZ00EEHMXToUO69914uv/xyrrjiCg466KC8Psnvs88+XH311YwePZoxY8YwcODAqrp///d/Z+bMmQwdOpS7776bX//613WONd9j3nPPPdxxxx0MGzaMwYMH8+ijjwLJH/8DDzyQIUOGcOihhzJs2DDGjRvH8uXLGT58OPfddx9Dhw7lV7/6FaeffjoDBw5kyJAhrFmzZqdz/OQnP2HUqFGMGTOmakB7V+e4//77GTJkCMOHD+fVV1/lrLPOyrvNu2pLTQYMGMDGjRvZtGkTa9eu5c0339yhv79Pnz506dKlqm+/vl1++eX07NmTzZs307Nnz6oxldLSUiZPTob7Bg8ezCmnnMKgQYM45phjuPXWWykpKaF169bccsstHH300QwcOJBTTjmFwYOTT6F/+9vfaN++Pf/wD/9QkLitftz3ndHc953RBetCV3KXU9M3cuTIKCsr26FsxYoVO/whNSuEm266iU6dOnHuuecWO5R6c9NNN9G5c+ednu+o5N+t4vrmb5MPHL87d1SdjyVpYUSMzFbnMQizOrrgggt26PJrDrp27cqZZ56Ze0MrivpIDPlwgjCro3bt2jW7P6bnnHNOsUOwRqDZj0E0ly40s8bCv1MtR7NOEO3ateP999/3f2izehLp+58q9mAAAAipSURBVCDatWtX7FCsATTrLqaePXtSXl7OrqYCN7Paq3yjnDV/zTpBtGnTxm+9MjPbTc26i8nMzHafE4SZmWXlBGFmZlk1myepJa0H3qzDIboB79VTOE1FS2tzS2svuM0tRV3avF9EZH1nc7NJEHUlqWxXj5s3Vy2tzS2tveA2txSFarO7mMzMLCsnCDMzy8oJ4u+mFzuAImhpbW5p7QW3uaUoSJs9BmFmZln5CsLMzLJygjAzs6xaVIKQdIyk1yStkvSDLPVtJd2X1i+Q1Lvho6xfebT5XyUtl7RU0tOS9itGnPUpV5sztjtJUkhq8rdE5tNmSaek/9bLJP1HQ8dY3/L4v72vpHmSFqX/v48tRpz1RdIMSe9KenUX9ZJ0c/rzWCrpS3U+aUS0iC+gBFgN9AU+BywBBlXb5nvAtHT5NOC+YsfdAG0eB3RIly9oCW1Ot+sEPA+8CIwsdtwN8O/cH1gEfD5d36vYcTdAm6cDF6TLg4C1xY67jm3+X8CXgFd3UX8s8AQg4MvAgrqesyVdQRwCrIqINRGxFZgNHF9tm+OBO9PlB4GvSFIDxljfcrY5IuZFxOZ09UWgqc/jnM+/M8BPgF8AWxoyuALJp83nAbdGxIcAEfFuA8dY3/JpcwCd0+UuwNsNGF+9i4jngQ9q2OR44K5IvAh0lbRPXc7ZkhJED+CtjPXytCzrNhFRAWwE9myQ6AojnzZn+jbJJ5CmLGeb00vvXhHxeEMGVkD5/Dt/EfiipL9IelHSMQ0WXWHk0+argW9KKgfmABc1TGhFU9vf95ya9fsgLH+SvgmMBI4odiyFJKkV8EtgYpFDaWitSbqZxpJcJT4v6cCI2FDUqArrdGBWRNwoaTRwt6QhEfFZsQNrKlrSFcQ6oFfGes+0LOs2klqTXJa+3yDRFUY+bUbSPwFXAuMj4tMGiq1QcrW5EzAEeFbSWpK+2tImPlCdz79zOVAaEdsi4g3gdZKE0VTl0+ZvA/cDRMR8oB3JpHbNVV6/77XRkhLEy0B/SX0kfY5kELq02jalwNnp8teBZyId/WmicrZZ0kHAbSTJoan3S0OONkfExojoFhG9I6I3ybjL+IgoK0649SKf/9uPkFw9IKkbSZfTmoYMsp7l0+a/Al8BkDSQJEE05/cPlwJnpXczfRnYGBHv1OWALaaLKSIqJF0IzCW5A2JGRCyTNAUoi4hS4A6Sy9BVJINBpxUv4rrLs83XAx2BB9Lx+L9GxPiiBV1Heba5WcmzzXOBr0paDmwHLouIJnt1nGeb/w24XdIlJAPWE5vyBz5J95Ik+W7puMpVQBuAiJhGMs5yLLAK2AycU+dzNuGfl5mZFVBL6mIyM7NacIIwM7OsnCDMzCwrJwgzM8vKCcLMzLJygrCikbRd0mJJr0p6TFLXej7+2vSefyR9vItt2kt6TlKJpN6SPkljWi5pWvrkdW3OOVLSzenyWEmHZtR9V9JZdWlTepyrJV2aY5tZkr5ei2P23tUsodW2u1bSW9V/npIulPStfM9nTYMThBXTJxExPCKGkDx3MqkIMXwLeDgitqfrqyNiODCUZAbQE2pzsIgoi4iL09WxwKEZddMi4q66h1xUj5FMlFfdDJr/XEctjhOENRbzSScWk7S/pCclLZT0Z0kD0vK9Jf1e0pL069C0/JF022WSzq/leb8BPFq9MJ2s8QWgX/rp+hn9/Z0Z+6bnPTm9+lki6fm0bKykPyh5l8h3gUvSK5LDKz/5Sxog6aXKc6XHfyVdHpFe0SyUNDfXbJySzpP0chrDQ5I6ZFT/k6QySa9L+pd0+xJJ16f7LJX0ndr8sCLixWxP56YzAq+VlC15WBPlBGFFJ6mEZEqEyqecpwMXRcQI4FLgN2n5zcBzETGMZF78ZWn5t9JtRwIXS8prBt50ioa+EbE2S12HNKZXgH8H7oyIocA9aRwAk4Gj03h2ePo8PeY04Kb0KunPGXX/DXxOUp+06FTgPklt0nN9PW3PDODaHM14OCIOTmNYQTL/UKXeJJ/2/xmYJqldWr8xIg4GDgbOy4ijsu3/KGlOjvNmUwYcvhv7WSPVYqbasEapvaTFJFcOK4CnJHUk6ZapnPoDoG36/UjgLIC0S2hjWn6xpBPT5V4kk9DlM41EN6D6bKb7pzEF8GhEPCHpbmBCWn83cF26/BdglqT7gYfzOF+m+0kSw8/T76cCB5BMJPhU2vYSINdcOkMk/RToSjJlytzMc6Qzl66UtAYYAHwVGJoxPtGF5Of1euVOEfE2yZQNtfVueg5rJpwgrJg+iYjh6af1uSRjELOADek4QE6SxgL/BIyOiM2SniWZlC2v82fZdnW+546I70oaRfIJfaGkEXmeF+A+kiT4cHKoWCnpQGBZRIyuxXFmASdExBJJE0kn5KsMsXrIJG8buygiMhMJqp/X67Yj+ZlaM+EuJiu6tP/6YpLJ1TYDb0g6Gareszss3fRpkteiVvaldyH5BPxhmhwGkEzfne95PwRK0q6XmrzA3ydu/Abw5zSG/SNiQURMJpkltFe1/TaRTC+e7dyrSSbN+zFJsgB4Deiu5N0FSGojaXCO2DoB76TdU9+oVneypFaS9id5NedrJIn4gnR7JH1R0h45zpGvLwI574SypsMJwhqFiFgELCV5ycs3gG9LWkIyzlD5KsnvA+PSAd2FJHcZPQm0lrSCpLvmxVqe+o/AYTm2uQg4R9JS4Mw0DoDrJb2S3h76Asl7kTM9BpxYOUid5bj3Ad/k7+8s2Eoyzfwv0rYvJuMuqF34MbCApLvrv6vV/RV4ieQtgd+NiC3Ab4HlwH+lcd9GtZ6EmsYgJF2nZCbRDpLKJV2dUT0GeCpHvNaEeDZXa9GUvH70kog4s9ixNGVK3ivyr/45Ni++grAWLSL+C5iX3kllu68bydWMNSO+gjAzs6x8BWFmZlk5QZiZWVZOEGZmlpUThJmZZeUEYWZmWf1/RIecAulJYhwAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Function plot_roc_curve is deprecated; Function :func:`plot_roc_curve` is deprecated in 1.0 and will be removed in 1.2. Use one of the class methods: :meth:`sklearn.metric.RocCurveDisplay.from_predictions` or :meth:`sklearn.metric.RocCurveDisplay.from_estimator`.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "CPU times: user 5 µs, sys: 1 µs, total: 6 µs\n",
            "Wall time: 10.5 µs\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x684 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeQAAAI0CAYAAADMeFpLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZwdVZ34/c8hG0I6CwEHJUCECCMuQTjMDL9hR0GUZlgSRyQgiw4M8CCjBPwFxKioIAk+mREQmZEgBhkGNLFZBMX4qAw6HqPIKmtAHZawhcTIklDPH1UdK00vN+nqvpXbn/fr1a++Vafq1Dl1q+p7T20nZFmGJElqro2aXQBJkmRAliSpFgzIkiTVgAFZkqQaMCBLklQDw5u58I6Ojqy9vb2ZRZAkaTCFnhJsIUuSVAMGZEmSasCALElSDRiQJUmqAQOyJEk1YECWJKkGDMiSJNWAAVmSpBowIEuSVAMGZEmSasCALElSDRiQJUmqAQOyJEk1YECWJKkGDMiSJNVAyLKseQufvap5C5daTDZjarOLILWWbMFA5Gp/yJIk1ZkBWZKkGjAgS5JUAwZkSZJqwIAsSVINDO9rghjjBcDBwNbACuBG4KyU0nNF+jHAScDbgNXAL4EzU0p3DVShJUlqNY20kFcD04EJwBRgIjCvlN4GfKYYvxWwGLg1xrhJpSWVJKmFrfNzyDHG9wHXppTG9JC+MfBnYNeU0uJeF+5zyFJlfA5ZqtgG8Bzy/sCdfaSvBB5cj7wlSRqS+ryGXBZjPIL8evHePaTvAFwBfDKltLz/xZMkaWhouIUcY5wGXA4c0t2p6BjjTsAiYHZK6WvVFVGSpNbXUECOMR4HXAa0p5QWdZO+C/Bj4PyU0pcrLaEkSUNAnwE5xngaMBs4MKV0ezfpfw/cBpydUvq36osoSVLra+Qa8lxgFbAoxrhmZEppdPHxPGAs8JUY41dK8x2UUvppVQWVJKmV9RmQU0o93qJdpO9bXXEkSRqafHWmJEk1YECWJKkG1vlNXVXq6OjI2tvbm7Z8SZIGWaVv6pIkSRUzIEuSVAMGZEmSasCALElSDRiQJUmqAQOyJEk10NTHnsLsVc1buDSIshlTm10EaXBkC5pdgrrzsSdJkurMgCxJUg0YkCVJqgEDsiRJNdBn94sxxguAg4GtgRXAjcBZKaXnivRpwGeArYpZ7gHOTin9fwNSYkmSWlAjLeTVwHRgAjAFmAjMK6X/HHhvSml8Mc2/AjfFGMdVW1RJklpXny3klNLM0uDSGONc4NpS+u9L6YE8gG9C3qJ+oaJySpLU0voMyN3YH7izPCLGuA3wW6CNvNV9TUrprv4XT5KkoWGdAnKM8QjgJGDv8viU0uPAuBjjpsA0YFRlJZQkaQho+C7r4uaty4FDUkqLu5smpfSnlNI84OMxxgOrKaIkSa2voYAcYzwOuAxoTyktamCW4cBb+1MwSZKGkkYeezqN/LGmA1NKv+wm/Rjgv4FHgE2BfwG2AX5UbVElSWpdjVxDngusAhbFGNeMTCmNLj7uAHwe2BxYSX5z1wdSSvdWW1RJklqXvT1Jg8DenjRk2NtTX+ztSZKkOjMgS5JUA009Zd3R0ZG1t7c3bfmSJA0yT1lLklRnBmRJkmrAgCxJUg0YkCVJqgEDsiRJNWBAliSpBgzIkiTVgK/O1JDgqys1aHx1pHrnc8iSJNWZAVmSpBowIEuSVAMGZEmSamB4XxPEGC8ADga2BlYANwJnpZSeK9K/BkzvMtumwCdTShdVW1xJklpTnwEZWE0ecO8GxgHfBOYBhwCklE4CTuqcOMb4XuAm4JqKyypJUsvqMyCnlGaWBpfGGOcC1/Yyy4lAR0rpf/tbOEmShopGWshd7Q/c2V1CjHFL4B+AD/SnUJIkDTXrFJBjjEeQn57eu4dJTgAeB37Qz3JJkjSkNHyXdYxxGnA5cEhKaXE36RsBHwO+nlLyDVySJK2DhgJyjPE44DKgPaW0qIfJ3ge8CfhGRWWTJGnI6DMgxxhPA2YDB6aUbu9l0hOB76SUllZVOEmShopGriHPBVYBi2KMa0amlEZ3fo4xbkV+I9f+VRdQkqShoJHHnnrsmaI0zR8byUuSJHXPV2dKklQDBmRJkmogZFnznlDq6OjI2tvbm7Z8SZIGWY+XgW0hS5JUAwZkSZJqwIAsSVINGJAlSaoBA7IkSTVgQJYkqQaa+thTmL3KXqH0OtmMqc0uguooW9DsEkhV8LEnSZLqzIAsSVINGJAlSaoBA7IkSTVgQJYkqQYq6cM4xngPsG1p1DBgY2DXlNLiKpYhSVIrqyQgp5TeXh6OMX4BONRgLElSYyo/ZR1jHA4cD1xWdd6SJLWqgbiGfCgwFvjmAOQtSVJLGoiAfCLwnymlFwYgb0mSWlIl15A7xRi3B/YHdq8yX0mSWl3VLeQTgTtTSr+oOF9JklpaZQE5xjgSOBb4WlV5SpI0VFTZQj6c/Nnj+RXmKUnSkFDZNeSU0jXANVXlJ0nSUOKrMyVJqoGQZVnTFt7R0ZG1t7c3bfmSJA2y0FOCLWRJkmrAgCxJUg0YkCVJqgEDsiRJNWBAliSpBgzIkiTVgAFZkqQaaOpzyGH2quYtXAMmmzG12UVQ1bIFzS6B1Cp8DlmSpDozIEuSVAMGZEmSasCALElSDfTZ/WKM8QLgYGBrYAVwI3BWSum50jTHAJ8B3gTcBZycUvrVgJRYkqQW1EgLeTUwHZgATAEmAvM6E2OMewCXAv8MjAeuB26KMY6purCSJLWqPlvIKaWZpcGlMca5wLWlcR8DvpNSuhUgxnghcCpwGHBlhWWVJKllrc815P2BO0vDU4A1p6dTShnw62K8JElqQJ8t5LIY4xHAScDepdFtwLIuk74AeMpakqQGNdxCjjFOAy4HDkkpLS4lLQfGdpl8HPBi/4snSdLQ0FBAjjEeB1wGtKeUFnVJvhPYpTRtAHZm7dPakiSpF4089nQa+SNNB6aUftnNJJcD348xXgn8FDgN2Bj4bpUFlSSplTVyDXkusApYFGNcMzKlNLr4/7MY48nkgbnzOeT3p5Q8ZS1JUoPs7UmVs7enFmRvT1JV7O1JkqQ6MyBLklQDTT1l3dHRkbW3tzdt+ZIkDTJPWUuSVGcGZEmSasCALElSDRiQJUmqAQOyJEk1YECWJKkGfFPXEOHbswaRb7WS1DMfe5Ikqc4MyJIk1YABWZKkGjAgS5JUAwZkSZJqYHhfE8QYLwAOBrYGVgA3AmellJ4rTbM9MBvYrxh1H7BnSunVykssSVILaqSFvBqYDkwApgATgXmdiTHGLYCfAncC2wCbAacW80mSpAb02UJOKc0sDS6NMc4Fri2N+wTweEppVnm2aoonSdLQ0GdA7sb+5K3hTvsCv48x3gjsDvwBuCClNL+C8kmSNCSs001dMcYjgJOAj5dGbw4cDlwBvBH4JPAfMcY9qiqkJEmtruEWcoxxGnAZcEhKaXEpaTlwR0rpumL4BzHG7wOHAD+rrKSSJLWwhlrIMcbjyINxe0ppUZfk3wDdvZPa91RLktSgRh57Og34DHBgSumX3UxyGfDTGOOhwPeAvYEDgAuqLKgkSa2skVPWc4FVwKIY45qRKaXRxf+fxxg/TB6A5wOPAh9JKd1RfXElSWpNdr84RNj94iCy+0VJPbP7RUmS6syALElSDTT1lHVHR0fW3t7etOVLkjTIPGUtSVKdGZAlSaoBA7IkSTVgQJYkqQYMyJIk1YABWZKkGvBNXRsA37K1DnxLlqR687EnSZLqzIAsSVINGJAlSaoBA7IkSTXQSH/IfYoxXgAcDGwNrABuBM5KKT1XRf6SJLW6qlrIq4HpwARgCjARmFdR3pIktbxKWsgppZmlwaUxxrnAtVXkLUnSUDBQ15D3B+4coLwlSWo5lbSQy2KMRwAnAXtXnbckSa2q0hZyjHEacDlwSEppcZV5S5LUyioLyDHG44DLgPaU0qKq8pUkaSioJCDHGE8DZgMHppRuryJPSZKGkqquIc8FVgGLYoxrRqaURleUvyRJLa2qx5567L1CkiT1zVdnSpJUAwZkSZJqIGRZ1rSFd3R0ZO3t7U1bviRJg6zHS7y2kCVJqgEDsiRJNWBAliSpBgzIkiTVgAFZkqQaMCBLklQDBmRJkmqgqc8hh9mrmrfwmstmTG12EeorW9DsEkjS+vI5ZEmS6syALElSDRiQJUmqAQOyJEk10FB/yDHGDwGnAFOATVJKw0tpewJzgUnAMOBh4LyU0ncqL60kSS2q0Rby88AlwOndpP0OOAyYAIwrpvlWjPFtlZRQkqQhoKEWckrpFoAY4z7dpD3d+TnGuBHwGnmgnwzcV0kpJUlqcQ0F5EbEGF8ANi3y/Alwa1V5S5LU6iq7qSulNA4YTX76+iZgVVV5S5LU6iq9yzql9HJKaQGwN/DRKvOWJKmVDdRjT8OBtw5Q3pIktZxGH3saBowARhbDGxdJLwOHAw+Q38A1HDga2A+4sOrCSpLUqhq9qeto4IrS8J+L/28B3gScX/x/hfwxqCNTSj+oqpCSJLU6e3uqKXt76oW9PUnacNnbkyRJdWZAliSpBpp6yrqjoyNrb29v2vIlSRpknrKWJKnODMiSJNWAAVmSpBowIEuSVAMGZEmSasCALElSDfimroq03Ju1fBuWJA0EH3uSJKnODMiSJNWAAVmSpBowIEuSVAMGZEmSamB4VRnFGL8AfBiYALwE/AT4RErp8aqWIUlSq6qyhXwVsHNKaQwwCXgcuKbC/CVJalmVtZBTSveXBgPwGrBjVflLktTKKgvIADHGDwOXAmOAVcAnqsxfkqRWVelNXSmlq1NKY4E3AbOAu6rMX5KkVjUgd1mnlJ4ELgduiDFuNhDLkCSplQzkY0/DgU2BNw/gMiRJagmVXEOOMW4EnAxcm1J6OsY4Efg3YAlwf2/zSpKkalvI7wfujjH+CfgFsBJ4T0ppVYXLkCSpJVXSQk4pvUYekCVJ0nrw1ZmSJNVAyLKsaQvv6OjI2tvbm7Z8SZIGWegpwRayJEk1YECWJKkGDMiSJNWAAVmSpBowIEuSVAMGZEmSasCALElSDTT1OeQwe1XzFl6hbMbUZheh/7IFzS6BJA0FPocsSVKdGZAlSaoBA7IkSTVgQJYkqQYq6X4RIMb4HuA84B3AS8C1KaWTq8pfkqRWVklAjjHuA1wHfBToIL+LbKcq8pYkaSioqoX8JeBrKaXrSuMWV5S3JEktr98BOca4KfA3wO0xxsXANsDdwBkppdTf/CVJGgqquKlrfJHPkcCxwJuBW4GbYozjKshfkqSWV0VAXl78vyKl9NuU0ivkp7BHAP+ngvwlSWp5/Q7IKaVlwBKg62sws27GSZKkblR1U9clwMdjjN8GHgA+AbwM/HdF+UuS1NKqCsizgTbgR8DGwK+Bg4rWsyRJ6oO9PVXA3p4kSQ2ytydJkurMgCxJUg009ZR1R0dH1t7e3rTlS5I0yDxlLUlSnRmQJUmqAQOyJEk1YECWJKkGDMiSJNWAAVmSpBrwTV192GDewuWbtiRpQ+BjT5Ik1ZkBWZKkGjAgS5JUAwZkSZJqwIAsSVINDK8ikxjjMOB84FhgY+BW4MSU0jNV5C9JUqurqoX8KeAfgL8FJhbjrqoob0mSWl5VAfmfgAtSSo+klJYBZwLvizFuW1H+kiS1tH4H5BjjOGAb4Fed41JKDwMvAlP6m78kSUNBFS3ktuL/si7jXwDGVJC/JEktr4qAvLz4P7bL+HHkrWRJktSHfgfklNILwOPALp3jYozbkbeOf9vf/CVJGgoqeewJ+DpwVoxxEfAscAFwS0ppSUX5S5LU0qoKyOcD44FfAqOAHwDTK8pbkqSWV0lATimtBs4o/iRJ0jry1ZmSJNVAyLKsaQvv6OjI2tvbm7Z8SZIGWegpwRayJEk1YECWJKkGDMiSJNWAAVmSpBowIEuSVAMGZEmSasCALElSDTT1OeQwe1XzFl7IZkxtdhEgW9DsEkiSBofPIUuSVGcGZEmSasCALElSDRiQJUmqgUq6X4wxfgg4BZgCbJJSqqqfZUmShoSqAufzwCXAG4CvV5SnJElDRiUBOaV0C0CMcZ8q8pMkaajxGrIkSTVgQJYkqQYMyJIk1YABWZKkGqjqsadhwAhgZDG8cZH0ckqp6e+rliSp7qpqIR8N/Bm4BRhWfP4zsG1F+UuS1NKqeuxpHjCvirwkSRqKvIYsSVINGJAlSaqBkGXNu+eqo6Mja29vb9ryJUkaZKGnBFvIkiTVgAFZkqQaMCBLklQDBmRJkmrAgCxJUg0YkCVJqoGmPvYUZq9q2sKzGVObtOAFzVmuJKkOfOxJkqQ6MyBLklQDBmRJkmrAgCxJUg0YkCVJqoE++0OOMV4AHAxsDawAbgTOSik9V6RPAc4H3g38FbBnSulnA1ZiSZJaUCMt5NXAdGACMAWYCMwrpb8CfIc8aEuSpPXQZws5pTSzNLg0xjgXuLaUfh9wH0CMsfICSpI0FKzPNeT9gTurLogkSUNZny3kshjjEcBJwN4DUxxJkoamhlvIMcZpwOXAISmlxQNXJEmShp6GAnKM8TjgMqA9pbRoYIskSdLQ08hjT6cBnwEOTCn9spv0AIwqjRoZY9wYeDWltLqykkqS1MIauYY8F1gFLCrfRZ1SGl183BZ4tDT9bcX/41j78ShJktSDRh576rGrqCJ9Cb10JyVJkvrmqzMlSaoBA7IkSTUQsixr2sI7Ojqy9vb2pi1fkqRB1uMlXlvIkiTVgAFZkqQaMCBLklQDBmRJkmrAgCxJUg0YkCVJqoGmPvYUZq8a9IVnM6YO9iKLBS9oznIlSXXiY0+SJNWZAVmSpBowIEuSVAMGZEmSaqCR/pCJMX4IOAWYAmySUhpeSns/cAbwLmAYcDcwM6X00+qLK0lSa2q0hfw8cAlwejdp44F/AyYDWwBXAzfHGLeupISSJA0BDbWQU0q3AMQY9+kmbX6XUZfGGD8D7Ab8vr8FlCRpKKj8GnKM8Z3A5sBdVectSVKrqjQgxxjfCFwPzE4pPVhl3pIktbLKAnKM8c3AIuBW4P9Wla8kSUNBQ9eQ+xJjnATcBnw3pXRGFXlKkjSUNPrY0zBgBDCyGN64SHoZ2BH4ITAvpXTOQBRSkqRW12gL+WjgitLwn4v/bwHOArYCTo8xlh+LOrGbO7AlSVI37O1p0BZsb0+SJHt7kiSp1gzIkiTVQFNPWXd0dGTt7e1NW74kSYPMU9aSJNWZAVmSpBowIEuSVAMGZEmSasCALElSDRiQJUmqAQOyJEk14KszB2WhvjZTkgT4HLIkSfVmQJYkqQYMyJIk1YABWZKkGhje1wQxxguAg4GtgRXAjcBZKaXnivRjgW8AK0uzdaSUjqy8tJIktag+AzKwGpgO3A2MA74JzAMOKU3zSEppcuWlkyRpiOgzIKeUZpYGl8YY5wLXDlyRJEkaehppIXe1P3Bnl3FbxxifBF4Fbgf+b0rp0f4WTpKkoWKdbuqKMR4BnAR8vDT6J8A7gTcDuwEvAT+IMW5aVSElSWp1DbeQY4zTgMuAQ1JKizvHp5QeKU32ZIzxY8Ay4O+A26oqqCRJrayhFnKM8TjyYNyeUlrUx+RZ8dfj68EkSdLaGnns6TTgM8CBKaVfdpP+AfJryn8ExgNfAp4Bfl5tUSVJal2NnLKeC6wCFsUY14xMKY0uPu4DXA6MBV4kv6nrvSmlFZWWVJKkFmZvT4OyUHt7kiQB9vYkSVK9GZAlSaqBpp6y7ujoyNrb25u2fEmSBpmnrCVJqjMDsiRJNWBAliSpBgzIkiTVgAFZkqQaMCBLklQDQ+ZNXYP+hi7fziVJej0fe5Ikqc4MyJIk1YABWZKkGjAgS5JUAwZkSZJqYHhfE8QYLwAOBrYGVgA3AmellJ4r0ocBXwQ+DIwDlgCfTSldN0BlliSp5TTSQl4NTAcmAFOAicC8UvopwNHAe4AxwKeBq2OMf11pSSVJamF9tpBTSjNLg0tjjHOBa0vjJgM/Tin9rhheEGN8FngHcH9lJZUkqYX1GZC7sT9wZ2n4cmB+jHEn4HfAYUW+P+l/8SRJGhrWKSDHGI8ATgL2Lo1+BPgpcDfwGvAycHRK6emqCilJUqtr+C7rGOM08tbwISmlxaWkS4B3A28BRgLvBb4WYzygyoJKktTKGmohxxiPA+YA7Sml27sk7wpcnFJ6rBj+7xjjT4H3A7dWVlJJklpYny3kGONpwGzgwG6CMcDtwFExxq2K6f8W2Af4VYXllCSppTXSQp4LrAIWxRjXjEwpjS4+zgC+DPxPjHEM8BQwJ6V0VcVllSSpZdn94oAt0O4XJUmvY/eLkiTVWVNbyB0dHVl7e3vTli9J0iCzhSxJUp0ZkCVJqgEDsiRJNWBAliSpBgzIkiTVgAFZkqQaMCBLklQDQ+JNXYP6li7f0CVJ6pnPIUuSVGcGZEmSasCALElSDRiQJUmqgUb6Q+5TjHEecBTwcmn0mSmlS6rIX5KkVldJQC5cmVL6aIX5SZI0ZHjKWpKkGqiyhXxEjPFw4BlgIfDZlNKKCvOXJKllVdVC/jfgr4HNgcOAvYHLK8pbkqSWNyBv6oox/j3wY2B0SunlnqbzTV2SpCFm0N/U9VpfC5YkSX9RSUCOMX4oxjiu+PxWYA7wvZTSS1XkL0lSq6uqhXwS8EiM8U/ArcDPgeMqyluSpJZXyV3WKaV9qshHkqShyueQJUmqAQOyJEk1MCCPPTWqo6Mja29vb9ryJUkaZIP+2JMkSVoHBmRJkmrAgCxJUg0YkCVJqgEDsiRJNWBAliSpBpr62FPL9fZkT0+SpN752JMkSXVmQJYkqQYMyJIk1YABWZKkGjAgS5JUAw31hxxj/BBwCjAF2CSlNLxL+vuAOcB2wMPAJ1JKt1ZcVkmSWlajLeTngUuA07smxBi3A74DfAkYW/z/boxxUkVllCSp5TXUQk4p3QIQY9ynm+SPAL9KKX2rGJ4fYzypGP/ZKgopSVKrq+Ia8hTgV13GLS7GS5KkBlQRkNuAZV3GvQCMqSBvSZKGhCoC8nLya8dl44AXK8hbkqQhoYqAfCewS5dx7y7GS5KkBjT62NMwYAQwshjeuEh6GfgmMCPGeCRwHTAV2BU4pvLSSpLUohptIR8N/Bm4BRhWfP4zsG1K6WHgcOAc8tPU5wCHpZSWVF5aSZJalN0vVrogu1+UJPXK7hclSaozA7IkSTXQ1FPWHR0dWXt7e9OWL0nSIPOUtSRJdWZAliSpBgzIkiTVgAFZkqQaMCBLklQDBmRJkmqgJd7UVembuHzbliRp4PjYkyRJdWZAliSpBgzIkiTVgAFZkqQaGN7XBDHGC4CDga2BFcCNwFkpped6mPZM4OiU0rcqLqskSS2rkRbyamA6MAGYAkwE5nWdKMb4N8BBwBMVlk+SpCGhzxZySmlmaXBpjHEucG15mhjjKOA/gH8Cvl1pCSVJGgLW5xry/sCdXcbNAn6UUrqj3yWSJGkI6rOFXBZjPAI4Cdi7NC4C04Cdqy2aJElDR8Mt5BjjNOBy4JCU0uJi3EjgCuCUlNKKgSmiJEmtr6GAHGM8DrgMaE8pLSolvRl4OzA/xvhMjPEZ8ruxL40xzq+8tJIktahGHns6DfgMcGBK6Zddkn8PbNNl3B3Al4GrKymhJElDQCPXkOcCq4BF+eXiXEppdEppNfCH8sQxxtXA8ymlZ6ssqCRJrayRx5567Jmih+knrXdpJEkaonx1piRJNWBAliRtkGbNmsX06dObXYzKrNNzyFX73o43097e3v+MzljQ/zwkSYTZqwY0/+yMdQs7V199NRdddBH3338/bW1t7Lzzzpx99tnsscceA1TCnn36059mwYIF3HfffZxzzjnMmjWr0vxtIUuSaumiiy7i9NNPZ+bMmTz11FM8/vjjnHzyySxcuLAp5Zk8eTJf/vKX+cAHPjAg+RuQJUm1s2zZMs4991wuvvhiDj/8cDbddFNGjBhBe3s7F154YbfzTJs2jS233JKxY8ey1157cc8996xJu+mmm9hpp51oa2tjq622Yvbs2QA888wzHHzwwYwbN47NNtuMPffck9dee63b/D/ykY9w0EEH0dbWVn2FMSBLkmrojjvu4KWXXuKwww5reJ6DDjqIBx98kKeffppddtmFo446ak3aCSecwGWXXcby5cu5++672W+//QCYM2cOEydOZOnSpTz11FN88YtfJIR1erioMk29hixJUneeffZZNt98c4YPbzxMHX/88Ws+z5o1i/Hjx7Ns2TLGjh3LiBEjuPfee5kyZQrjx49n/PjxAIwYMYInnniCxx57jMmTJ7PnnntWXpdG2UKWJNXOhAkTeOaZZ1i1qrGbzFavXs2nPvUptt9+e8aMGcOkSZOA/JQ0wPXXX89NN93Etttuy957780dd+SdE86YMYPJkydzwAEHsN1223H++ecPSH0aYUCWJNXO7rvvzqhRo1iwoLGnaK6++moWLlzID3/4Q5YtW8aSJUsAyLIMgN12242FCxfy9NNPc+ihh/LBD34QgLa2NubMmcMjjzzC9773PS666CJuu+22AalTXwzIkqTaGTt2LJ/73Oc45ZRTWLBgAStXruTVV1/l5ptv5swzz3zd9MuXL2fUqFFMmDCBlStXMnPmzDVpr7zyCvPnz2fZsmWMGDGCMWPGsNFGefi74YYbeOihh8iyjLFjxzJs2LA1aV29+uqrvPTSS7z22musWrWKl156idWrV1dW56ZeQz7kdwfB7xo7HZHNmNpYppnPJEvS+lrX54QH0ic/+Um23HJLzjvvPI466ija2trYddddOfvss1837THHHMMtt9zCVlttxWabbcbnP/95Lr300jXpV111FaeeeiqrV69mxx13ZP78vEPCBx98kFNPPZWlS5cyfvx4Tj75ZPbdd99uy/Oxj32MK6+8cs3wF77wBa644gqOPfbYSuobOpvzzRBmr2p44QZkSVIL6PEWbk9ZS5JUAwZkSZJqwIAsSVINVHr1Psa4EfAzYHdg65TSH6rMX5KkVlV1C/lfgJUV5ylJUsurLCDHGHcATgbOqCpPSZKGikoCcnGq+hvkwfiFKvKUJGkoqV01ID8AABe+SURBVKqF/HHgyZTSdyvKT5KkIaXfATnGOBn4JHBq/4sjSVJjZs2axfTp05tdjMpUcZf1HsAWwN0xRvhLkP9tjPGclNIlFSxDkjQYwqEDm/86vk3x6quv5qKLLuL++++nra2NnXfembPPPps99thjgArYsyVLlnDcccfxi1/8gm222YavfvWrvOc976ks/ypOWV8LbA/sXPy9vxh/APDNCvKXJA1BF110EaeffjozZ87kqaee4vHHH+fkk09m4cKFTSnPkUceybvf/W6effZZvvCFLzB16lSWLl1aWf79DsgppZUppT90/gFPFklPppRW9Dd/SdLQs2zZMs4991wuvvhiDj/8cDbddFNGjBhBe3s7F154YbfzTJs2jS233JKxY8ey1157cc8996xJu+mmm9hpp51oa2tjq622Yvbs2UDeX/LBBx/MuHHj2Gyzzdhzzz157bXXXpf3Aw88wOLFi/nsZz/LG97wBo444gje+c53cv3111dW58q79UgpLaGXl2dLktSXO+64g5deeonDDjus4XkOOuggvvGNbzBy5EjOOussjjrqKH7zm98AcMIJJ3Dttdey55578vzzz/Poo48CMGfOHCZOnLimpfvzn/+cEF4fwu655x6222472tra1oybMmXKWkG/v3x1piSpdp599lk233xzhg9vvN14/PHH09bWxqhRo5g1axZ33nkny5YtA2DEiBHce++9vPjii4wfP55ddtllzfgnnniCxx57jBEjRrDnnnt2G5BXrFjB2LFj1xo3duxYli9f3o9ars2ALEmqnQkTJvDMM8+watWqhqZfvXo1n/rUp9h+++0ZM2YMkyZNAvJT0gDXX389N910E9tuuy177703d9xxBwAzZsxg8uTJHHDAAWy33Xacf/753eY/evRoXnzxxbXGvfjii2u1mPurqT1Rf2/Hm2lvb29s4jPs51iShordd9+dUaNGsWDBAqZOndrn9FdffTULFy7khz/8IZMmTWLZsmWMHz+eLMsA2G233Vi4cCGvvvoqX/3qV/ngBz/I73//e9ra2pgzZw5z5szh7rvvZr/99mO33XZj//33Xyv/t7/97TzyyCMsX758TRC+8847+fCHP1xZnW0hS5JqZ+zYsXzuc5/jlFNOYcGCBaxcuZJXX32Vm2++mTPPPPN10y9fvpxRo0YxYcIEVq5cycyZM9ekvfLKK8yfP59ly5YxYsQIxowZw0Yb5eHvhhtu4KGHHiLLMsaOHcuwYcPWpJXtsMMO7Lzzznz2s5/lpZde4rvf/S6//e1vOeKIIyqrc1NbyJKkmlnH54QH0ic/+Um23HJLzjvvPI466ija2trYddddOfvss1837THHHMMtt9zCVlttxWabbcbnP/95Lr300jXpV111FaeeeiqrV69mxx13ZP78+QA8+OCDnHrqqSxdupTx48dz8skns++++3ZbnmuuuYZjjz2W8ePHs80223DdddexxRZbVFbf0Nmcb4aOjo6s4VPWkiRt+Hp8CslT1pIk1YABWZKkGjAgS5JUAwZkSZJqwIAsSVINGJAlSaoBA7IkSTVgQJYkqQYMyJIk1YABWZKkGjAgS5JUAwZkSZJqoKmdS4waNeruV1555aWmFWCQDB8+fPNVq1Y90+xyDKShUEewnq3GeraWDaSez2RZ9r5uU7Isa9rfrrvumpq5fOtpHa2n9Wx2Gayn9ez885S1JEk1YECWJKkGmh2Qv97k5Q+WoVDPoVBHsJ6txnq2lg26nk29qUuSJOWa3UKWJEkYkCVJqoXhVWcYY9wBuBKYADwLHJNSerDLNMOAfwXeB2TA+Smlf+8rrU4qqOengQ8Bq4FXgZkppVsGrwaN6W89S9PsCPwauCSldMZglH1dVFHPGOMHgU8DoUh/T0rpqcGpQWMq2G7fCFwBbA2MABYBp6WUVg1aJfrQYB0PAL4IvBP4t/I22WLHoN7q2UrHoB7rWZqm1scgGJgW8teAi1NKOwAXA5d1M81RwGTgrcDuwKwY46QG0uqkv/X8H2C3lNK7gOOB/4wxvmHAS73u+lvPzgPcZcCCAS/t+utXPWOMEZgFvDel9A5gD2DZwBd7nfX3+5wJ3Fdst+8CdgUOH+hCr6NG6vgI8FHgwm7SWukY1Fs9W+kY1Fs9N5RjULUBufj1vAvw7WLUt4FdYoxbdJn0H4HLU0qvpZSWkq+kaQ2k1UIV9Uwp3ZJSWllM91vyVtWEAS/8Oqjo+wT4FHAD8MAAF3m9VFTPfwFmp5SeBEgpLUsp1eotdBXVMwPaYowbAaOAkcAfB7zwDWq0jimlh1JKvwG6a9m3zDGot3q20jGoj+8Tan4M6lR1C3lr4I8ppdUAxf//LcaXbQM8Vhp+vDRNb2l1UUU9y44BHk4p/WEAytof/a5njHEKcCDwlQEv7fqr4vvcCdguxviTGOPiGOM5McYwwOVeV1XU8/PADsATwJPALSml2wey0Ouo0Tr2ppWOQY3a0I9BPdpAjkGAN3U1XYxxb/KD3JHNLkvVYowjyJ8LPKlzh2phw8hP4b4X2Bs4CDi6qSUaGNPIW1NvArYC9ooxTm1ukdQfHoPqo+qA/Htgq+J8fed5+zcX48seB7YtDW9Tmqa3tLqoop7EGHcHvgUcmlL63YCWeP30t55vArYHbooxLgFOBz4WY6zbw/tVbbfXpZReTiktBxYCfzOgpV53VdTz/wHmF6dzl5HXc98BLfW6abSOvWmlY1CvWugY1JMN5RgEVByQU0pPA7/hL7+0jgR+XVyHKfsv8pWyUXEt4FDgugbSaqGKesYYdwP+E5iaUlo8OCVfN/2tZ0rp8ZTS5imlSSmlScD/S35t7p8GqQoNqWi7vRo4IMYYil/l+wN3DnzpG1dRPR8lv/uYGONI4D3A3QNd9katQx1700rHoB612DGop/k3iGNQp8ofewJOAq6MMZ4LPE9+bYIY403AuSmlBFwF/C3Qeev651JKjxafe0urk/7W8xLgDcBl+Q26ABydUrprkMrfqP7Wc0PR33peA0TgXuA14BbgPwav+A3rbz1PB74WY7yL/DT9IuDyQSx/I/qsY4xxD/LvbAwQYowfAk4oHvvZULbn/tazZY5BfdRzg+GrMyVJqgFv6pIkqQYMyJIk1YABWZKkGjAgS5JUAwZkSZJqwIC8DkIIB4YQfloa3ieEsKSJRRo0IYR5IYTKerwJIUwKIWSl4S1CCI+FEDZvYN6TQghXVVWWDUEIYc8QwgvNLsdQFEKYvi77edX7ino3UPvGenzv54cQPt+fZRqQGxRCCOTvQv1MH9P9cwjh7hDCiyGE50MIKYTwj6X0JSGE6d3M97rxIfdAkdfoLmn7hBCyEMKK4u9/QwhXhBA2619NmyPLsqXkL9foa/1uCnyOvGelISPLsp9mWTau2eXoSQhhVgjhh80ux1AwUOs6hPDjEMI5Vec70LruG03cFi8ATgkhbLW+GRiQG3cAec82i3qaIIRwJHlAOQEYS/6Kt38hf5h9fewLbEf+oonu3jO7Osuy0VmWjSbv7m938jfRbKi+ARwXQhjTyzTTgbuyLHt4kMq0lhDCsBCC+42ktWRZ9jxwM3Di+uZRywNL0Vo8J4SwqGj93RVCeFcI4cgQwkMhhGUhhH8PIQwvzbNNCOG6EMKTIYQnQghfDyG0ldK/GEJ4pMjv4RDC6aW0SUVr8+gQwr0hhOUhhFtDCG8qFetQ4IdZ729S+T/AT7Is+0WW+3Px6+3W9VwVJwLfJ39zUK9fcpZlj5B3L/burmkhhOHFOjm0y/h5IYQris/7hxB+UbTql4YQrgkhvLGn5RXra4/S8D4hhFWl4eEhhJlFC/+FEMLtIYTYfW5r6vAg8Az56xh7cijwgy5l+XgI4f7ie3s8hPClEMKwIu3CEMKCLtPvU0y7aTH8jhDCLUW9O+cfUaR1bhsnhBDuBVYCbwwhfCiEcGdx9uKJEMJlnfkV820ZQugottUHivmzEMKk0jQfK86mLAsh/DqEcEBPle5m/c4LIVwVQvhGsX7/WOwfO4cQflnUb1EI4c2leZaEEM4NIfys2A9SCGG3Unqv20AIYUTxnf6uyP/hEMLUkJ8BmgnsE/5yxma7Huqxd7GMZcV3dmIpbZ8QwqoQwj8WeS8LIVxb3o+7yW99jhXvCiH8qKjnI8X8w0rpf1OsmxUhhJ+R/yguL3OTEMLsEMKjIYTnQgjfDyFM7qmM3ZR5QgjhmyE/Vj0ZQrgylM5shS5ny0rb4MSe1nUI4diivmcV2+PTIYQ53WzHE0v5HhtCeKj4/FVgT+DTRZ7dvtc65K3P20IIFxTbyLMhhE+EELYt1unyEMKvQghvK83Tr32ltK1fXtrWX7fdFJ97XT9d6rLWpYWKvvcfkB+j1k+WZbX7A5aQv7bubcAI8pefP0zea8em5C97fxo4qph+Y+Ah8lOZbwDGAzcB3yjlOZ28xRqA/YA/AwcWaZPI+3m9Adic/PVrtwOXl+b/BXBal3LuAywpDU8DXgLOI3+X8bge6ja9r/HAFsDL5J2/v7so365dlr2qNDwZ+F25zl3y/zKwoDQ8GlgB7FkM7wHsRv461S2BnwDfLk0/D/j30nAG7NFLeb5QrLPtyF+xeAJ5sB1fXufdlLMDOK+XbeMp4JAu444A3lJ8t+8upjmxSNsJeAXYojT9lcB/FJ/fCDxL/oNnJHkPRgk4t8u2cVuxXkYW9TkIeDv5j9rJ5K/M/FJpGbcB1xfb0huBHxf5TCrSP0a+zU4p8nh/8X1M7qHeXdfvPPJt+APF/CcV838PmAhsAvyItbfhJeRd1+1a1ONTwFJgTIPbwAVFPd9VrOuJwLuKtFnkP1h726/fUpT52GIZfwc8B0wr1TEjf+XoaOCvyI8DZ1d4rBhbbB+fJu/P+W3kndvPKKU/W6ybkcX6eJK19/P55MeKvyqm+SxwPzCiu32lmzJ/n3w7H1/83Qjc2MuxYFKxXib2tK6LdfoqcDH5MXB78r5/Z3aXR2meh0rDPwbO6eM7nFUs56P8ZT9YDfywy3fwg9I8/d1X5pFvN4cUeRxelGHbHvaNntbPQ13Grfmeqvjei2l2JT+jObK39djj+l2fmQb6r9ggZ5SG3198QeWD6rXAV4rPU4GHu+SxK3lAG9bDMq4DvtxlY92tlH4K8OvS8APAsV3y2Kf8hRXjDga+Q77TryY/xf2OLnX7E/BCl7/XWHsnPJP8QNK5ky8GLuuy7KyY93nyl/5/jW5+BBTTv408ML2xGD4eeKCX7+Bg4OnuNt5iuMeATH6wXg7s1SXPuzrrSM8BeT5wSS/legXYp4/tZzZwbWn4F8C/FJ/bivX/98XwGcCPusx/BMXOW9o29upjmacC/1N8nljMs10pfX/WPsjcDRzTJY8Oejgg0n1ALh/ENynyn1YadzJrb8NLgM+XhgN5z0Yf7msbKKZdAXygh2ln0XdAngnc3mXcl4BbumzT5f38QuC7veS5hHU7VnyYvKegUEo/Efhd8fmoYp2U079AsZ+T/2DPgG1K6RsByyj2B3oJyOSNggx4a2ncjsW4N5XqtD4B+WVgk9K4j1Ls413zKM2zPgH5ni7jnu7mO3i+wn1lHqVtvRi3FPiHHvaNntZPbwG53997Me6txXRv7G099vQ3EJ1LVOWJ0ueV5NdLl3YZ13kq6y3ANuH1d9pl5L/0/xhCOI28VTKR/ODyBvKbiHpa5p9K+UMe9Hq7tpkvMMtuIP8VRQjhr8lf4H5DCOEtWfGNkbfevlWeL5Tu5gshhKKs38qy7NVi9H8A54cQzsiybHkxbnXW4I0+WZbdF0JYTH6m4CLgOOCK0jJ3Bb5I3mLbhHwdje4mq0ZsXszbEUp3UpP/ep7Y/SxrjCH/cdGT130PIb92/wny1vhw8l+vPy9NcgXwz+Q35X0Q+EOWZbcXaW8B/r7LthPIf/2XLemyzPcC5wJ/Td7SGkZ+YIK8lQ35Dt6p3OF953IvDiH8a2nccGBdOohfs71mWbYy32xet990Pd27pDRPFkJ4nOI76WMb2IK8xfnAOpSvq615/Xf7MPAPpeGu+3nX/bA763Ks2Bp4rLQvdpahs8P7id2kl8v8luL/b4v13WlEKY/edE5TzvPhUtoTrL+nsyxbWRpeQt/72/roWsaV9LLdVbCvdLfMRraLdVHV9z6GvzSU1lktryGvh8fIfwmO6/K3cZZlfwwh/D356bYTgc2LINZBfsBp1K/JT382LMuy+8mDwLbkp6YatR/5qZ3jO68zkZ8eGU3+C399XQEcW1z3+Dvgm6W0a8hb4TtkWTaGvjsrX0F+gO705tLnZ8h3mPd0+T42zbLs/D7yfQf5uu7JWt9DCGFr8lNk55G3MMaSn7Yrf7fXADuEEHYh/6V8RSntMfJf0+Vyjs3yG+XKXistcySwoMh3m2J9nVVa5h+L/9uU5i9/7lzu8V2WOzrLsn/upe5VmNT5ofjhtw1/+RHQ2zawlPxA+9Ye8n2th/Flvy8vv7Adg9vX8O+BbcPaR9VyGf7YTfqk0ufOYPHWLt/dJlmWfbvB5XfNc7suacvped+Cntf1G0MIm3Qpd+d32/kjfn3yXW8V7Svrqrt6dF2nsHb9q/re30F+BuGV9Sl4qwTkG4CRIb/hpC3ktgohHFakjyE/fbwUyEIIHyC/rrEuFpCfSulRCOH4EMK0UDxLW9xAcRJwb5Zlz63Dsk4kv37318DOxd87yANJf/rxvIY80P8r+TWeP5bSxpCfflkeQtiG/FpKb34FfCSEMLK4+eITnQnFr8y5wOwQwlsBQgijQ/4cd9eDwBrFD4UtyK9H9WQBa9/0NZp8O14KvBpC+Dvg6PIMWZa9AHyXPGj/Hfk15E7fBGLx3W0cQtiouAnkfb2UYST5L/3nsyz7cwhhJ/LTcJ3L+wP56b/zi+1xC6Dr4yRfAWaF/CasEEJ4Qwhhj+KsykA6PoSwS8hv9plB3hK+sUjrcRsovtNLgC+H/Ca4EPKbjN5VTPIk+Vmqkb0s+9vAriGEY0J+09/fkG/rg9lN5Y3k393MYtvdkTxAdJbhBvJtakbIb2Lbhfz+BwCyLHua/MzaJaF4vCWEMC6EcFjo8mhid7Is+1/gVmBOMd94YA5wc5Zlna3AXwFHFvvMFuTXu8t6WtcbARcU29J25JdjriyW+yzFj8CQPynwTvKzcF3zbfjmtAZVsa+sq+7Wz2/If7AcXOzjhwF7ldKr+t7fS36MWi8tEZCL0zT7kbec7ic/qNxGHsgg75v2m8D/kLfeppIfoNfFLcCqEMI+vUzzPPmp0ftCCH8iv3b5Avm1uIaE/K7WQ4HZWZY9Wf4jb+W/O/Rxt3JPsixbRl7vg8gfMSr7J/JrTsvJr4H/Vx/ZnUq+8z5Hfo1uXpf0zwALgYUhhBfJb7w5id63ueOBeUU5e3IVMKU44JBl2X2lZb1AHkS6a6lcQV7vW0oHPor1ui/5Ol9C/h1+ly53WJZlWbaC/Hv+cghhBXmLvOvljw+TB7s/kN8g2Lk+Xy7yuJz8RrsrimU+Tn7gHdFL3avwdfIfZM8D/0h+Tbhzffe1DZxN/l0vKKb5MX85gP8XeQvvyZDfCfuWLvOSZdmj5NcXTyW/geYq4NNZll1bVeX6UtT1APIfdU/xl2PDRUX6C+Q3yv0j+Tr6V+DSLtl8jPwGyh+HEJaT3xsxjfxUZSOmk6+/35Efr16g6OO3cA55A+IJ8nV8TZf5e1rXj5Fvb4+SH3u+T76NdfoI+bFoWVHfrj+EvkL+4/SFEMI9DdalV1XsK+vhdesnyx+T/Dj59v8c8D7yG8k6y9nv7z2EMI58+/7aepbb/pDXRdFqmpll2V7F8D7kAWRSM8u1ISpa1Y9mWRaK4S3I726OXa7/dTfvSeQ3ZR3d23R1EkI4kPxHwxuyJu10Ib9P4Zyu9y9owxdCOJb8u626hTvo6rCvrI8QwpfI719Y7xZ+nW/qqp0sy75P/qtTFSuC8LYNTvs1+vErdDCEEHYmv5Z1F/kNIecB/7khHWCkwdAq+0qWZf+3v3m0xCnrJlrChv1mrGZ6gfxGtVY1nvy07wrgZ8BvyU+ZSVqb+0rBU9aSJNWALWRJkmrAgCxJUg0YkCVJqgEDsiRJNWBAliSpBv5/Vh4ghnyVD50AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "       0      1       2       3        4        5         6         7       8  \\\n",
              "0  11.85  17.46   75.54   432.7  0.08372  0.05642  0.026880  0.022800  0.1875   \n",
              "0  13.40  20.52   88.64   556.7  0.11060  0.14690  0.144500  0.081720  0.2116   \n",
              "1  13.21  25.25   84.10   537.9  0.08791  0.05205  0.027720  0.020680  0.1619   \n",
              "1  11.22  19.86   71.94   387.3  0.10540  0.06779  0.005006  0.007583  0.1940   \n",
              "2  20.13  28.25  131.20  1261.0  0.09780  0.10340  0.144000  0.097910  0.1752   \n",
              "\n",
              "         9      10      11     12     13        14        15        16  \\\n",
              "0  0.05715  0.2070  1.2380  1.234  13.88  0.007595  0.015000  0.014120   \n",
              "0  0.07325  0.3906  0.9306  3.093  33.67  0.005414  0.022650  0.034520   \n",
              "1  0.05584  0.2084  1.3500  1.314  17.58  0.005768  0.008082  0.015100   \n",
              "1  0.06028  0.2976  1.9660  1.959  19.62  0.012890  0.011040  0.003297   \n",
              "2  0.05533  0.7655  2.4630  5.203  99.04  0.005769  0.024230  0.039500   \n",
              "\n",
              "         17       18        19     20     21      22      23      24       25  \\\n",
              "0  0.008578  0.01792  0.001784  13.06  25.75   84.35   517.8  0.1369  0.17580   \n",
              "0  0.013340  0.01705  0.004005  16.41  29.66  113.30   844.4  0.1574  0.38560   \n",
              "1  0.006451  0.01347  0.001828  14.35  34.23   91.29   632.9  0.1289  0.10630   \n",
              "1  0.004967  0.04243  0.001963  11.98  25.78   76.91   436.1  0.1424  0.09669   \n",
              "2  0.016780  0.01898  0.002498  23.69  38.25  155.00  1731.0  0.1166  0.19220   \n",
              "\n",
              "        26       27      28       29  probabilities  y_true  y_pred  \n",
              "0  0.13160  0.09140  0.3101  0.07007           1.00       1     NaN  \n",
              "0  0.51060  0.20510  0.3585  0.11090           0.05       0     1.0  \n",
              "1  0.13900  0.06005  0.2444  0.06788           0.96       1     1.0  \n",
              "1  0.01335  0.02022  0.3292  0.06522           1.00       1     NaN  \n",
              "2  0.32150  0.16280  0.2572  0.06637           0.00       0     NaN  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-599d20a8-1065-4426-89d6-0a24e45825b2\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "      <th>probabilities</th>\n",
              "      <th>y_true</th>\n",
              "      <th>y_pred</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>11.85</td>\n",
              "      <td>17.46</td>\n",
              "      <td>75.54</td>\n",
              "      <td>432.7</td>\n",
              "      <td>0.08372</td>\n",
              "      <td>0.05642</td>\n",
              "      <td>0.026880</td>\n",
              "      <td>0.022800</td>\n",
              "      <td>0.1875</td>\n",
              "      <td>0.05715</td>\n",
              "      <td>0.2070</td>\n",
              "      <td>1.2380</td>\n",
              "      <td>1.234</td>\n",
              "      <td>13.88</td>\n",
              "      <td>0.007595</td>\n",
              "      <td>0.015000</td>\n",
              "      <td>0.014120</td>\n",
              "      <td>0.008578</td>\n",
              "      <td>0.01792</td>\n",
              "      <td>0.001784</td>\n",
              "      <td>13.06</td>\n",
              "      <td>25.75</td>\n",
              "      <td>84.35</td>\n",
              "      <td>517.8</td>\n",
              "      <td>0.1369</td>\n",
              "      <td>0.17580</td>\n",
              "      <td>0.13160</td>\n",
              "      <td>0.09140</td>\n",
              "      <td>0.3101</td>\n",
              "      <td>0.07007</td>\n",
              "      <td>1.00</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>13.40</td>\n",
              "      <td>20.52</td>\n",
              "      <td>88.64</td>\n",
              "      <td>556.7</td>\n",
              "      <td>0.11060</td>\n",
              "      <td>0.14690</td>\n",
              "      <td>0.144500</td>\n",
              "      <td>0.081720</td>\n",
              "      <td>0.2116</td>\n",
              "      <td>0.07325</td>\n",
              "      <td>0.3906</td>\n",
              "      <td>0.9306</td>\n",
              "      <td>3.093</td>\n",
              "      <td>33.67</td>\n",
              "      <td>0.005414</td>\n",
              "      <td>0.022650</td>\n",
              "      <td>0.034520</td>\n",
              "      <td>0.013340</td>\n",
              "      <td>0.01705</td>\n",
              "      <td>0.004005</td>\n",
              "      <td>16.41</td>\n",
              "      <td>29.66</td>\n",
              "      <td>113.30</td>\n",
              "      <td>844.4</td>\n",
              "      <td>0.1574</td>\n",
              "      <td>0.38560</td>\n",
              "      <td>0.51060</td>\n",
              "      <td>0.20510</td>\n",
              "      <td>0.3585</td>\n",
              "      <td>0.11090</td>\n",
              "      <td>0.05</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>13.21</td>\n",
              "      <td>25.25</td>\n",
              "      <td>84.10</td>\n",
              "      <td>537.9</td>\n",
              "      <td>0.08791</td>\n",
              "      <td>0.05205</td>\n",
              "      <td>0.027720</td>\n",
              "      <td>0.020680</td>\n",
              "      <td>0.1619</td>\n",
              "      <td>0.05584</td>\n",
              "      <td>0.2084</td>\n",
              "      <td>1.3500</td>\n",
              "      <td>1.314</td>\n",
              "      <td>17.58</td>\n",
              "      <td>0.005768</td>\n",
              "      <td>0.008082</td>\n",
              "      <td>0.015100</td>\n",
              "      <td>0.006451</td>\n",
              "      <td>0.01347</td>\n",
              "      <td>0.001828</td>\n",
              "      <td>14.35</td>\n",
              "      <td>34.23</td>\n",
              "      <td>91.29</td>\n",
              "      <td>632.9</td>\n",
              "      <td>0.1289</td>\n",
              "      <td>0.10630</td>\n",
              "      <td>0.13900</td>\n",
              "      <td>0.06005</td>\n",
              "      <td>0.2444</td>\n",
              "      <td>0.06788</td>\n",
              "      <td>0.96</td>\n",
              "      <td>1</td>\n",
              "      <td>1.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>11.22</td>\n",
              "      <td>19.86</td>\n",
              "      <td>71.94</td>\n",
              "      <td>387.3</td>\n",
              "      <td>0.10540</td>\n",
              "      <td>0.06779</td>\n",
              "      <td>0.005006</td>\n",
              "      <td>0.007583</td>\n",
              "      <td>0.1940</td>\n",
              "      <td>0.06028</td>\n",
              "      <td>0.2976</td>\n",
              "      <td>1.9660</td>\n",
              "      <td>1.959</td>\n",
              "      <td>19.62</td>\n",
              "      <td>0.012890</td>\n",
              "      <td>0.011040</td>\n",
              "      <td>0.003297</td>\n",
              "      <td>0.004967</td>\n",
              "      <td>0.04243</td>\n",
              "      <td>0.001963</td>\n",
              "      <td>11.98</td>\n",
              "      <td>25.78</td>\n",
              "      <td>76.91</td>\n",
              "      <td>436.1</td>\n",
              "      <td>0.1424</td>\n",
              "      <td>0.09669</td>\n",
              "      <td>0.01335</td>\n",
              "      <td>0.02022</td>\n",
              "      <td>0.3292</td>\n",
              "      <td>0.06522</td>\n",
              "      <td>1.00</td>\n",
              "      <td>1</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>20.13</td>\n",
              "      <td>28.25</td>\n",
              "      <td>131.20</td>\n",
              "      <td>1261.0</td>\n",
              "      <td>0.09780</td>\n",
              "      <td>0.10340</td>\n",
              "      <td>0.144000</td>\n",
              "      <td>0.097910</td>\n",
              "      <td>0.1752</td>\n",
              "      <td>0.05533</td>\n",
              "      <td>0.7655</td>\n",
              "      <td>2.4630</td>\n",
              "      <td>5.203</td>\n",
              "      <td>99.04</td>\n",
              "      <td>0.005769</td>\n",
              "      <td>0.024230</td>\n",
              "      <td>0.039500</td>\n",
              "      <td>0.016780</td>\n",
              "      <td>0.01898</td>\n",
              "      <td>0.002498</td>\n",
              "      <td>23.69</td>\n",
              "      <td>38.25</td>\n",
              "      <td>155.00</td>\n",
              "      <td>1731.0</td>\n",
              "      <td>0.1166</td>\n",
              "      <td>0.19220</td>\n",
              "      <td>0.32150</td>\n",
              "      <td>0.16280</td>\n",
              "      <td>0.2572</td>\n",
              "      <td>0.06637</td>\n",
              "      <td>0.00</td>\n",
              "      <td>0</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-599d20a8-1065-4426-89d6-0a24e45825b2')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-599d20a8-1065-4426-89d6-0a24e45825b2 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-599d20a8-1065-4426-89d6-0a24e45825b2');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    }
  ]
}