quora_bert_final.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "id": "1580a1dd",
      "metadata": {
        "papermill": {
          "duration": 0.010417,
          "end_time": "2022-07-21T14:06:34.550388",
          "exception": false,
          "start_time": "2022-07-21T14:06:34.539971",
          "status": "completed"
        },
        "tags": [],
        "id": "1580a1dd"
      },
      "source": [
        "Import all the necessary libraries."
      ]
    },
    {
      "cell_type": "markdown",
      "id": "88166764",
      "metadata": {
        "id": "88166764"
      },
      "source": [
        "# Identify question pairs that have the same intent\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "33cac2c0",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:34.571305Z",
          "iopub.status.busy": "2022-07-21T14:06:34.570342Z",
          "iopub.status.idle": "2022-07-21T14:06:37.562183Z",
          "shell.execute_reply": "2022-07-21T14:06:37.561220Z"
        },
        "papermill": {
          "duration": 3.005212,
          "end_time": "2022-07-21T14:06:37.564773",
          "exception": false,
          "start_time": "2022-07-21T14:06:34.559561",
          "status": "completed"
        },
        "tags": [],
        "id": "33cac2c0"
      },
      "outputs": [],
      "source": [
        "import logging\n",
        "logging.basicConfig(level=logging.INFO)\n",
        "import torch\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "from sklearn.model_selection import train_test_split\n",
        "import torch\n",
        "from transformers import BertTokenizer, BertForSequenceClassification\n",
        "from tqdm import tqdm, trange\n",
        "import pandas as pd\n",
        "import io\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from transformers import BertTokenizerFast\n",
        "from datasets import Dataset\n",
        "import datasets\n",
        "from datasets import Dataset, DatasetDict\n",
        "from sklearn.model_selection import train_test_split"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "5acee71a",
      "metadata": {
        "papermill": {
          "duration": 0.009905,
          "end_time": "2022-07-21T14:06:37.584883",
          "exception": false,
          "start_time": "2022-07-21T14:06:37.574978",
          "status": "completed"
        },
        "tags": [],
        "id": "5acee71a"
      },
      "source": [
        "Load the data and check the split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0693d25a",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:37.604406Z",
          "iopub.status.busy": "2022-07-21T14:06:37.603935Z",
          "iopub.status.idle": "2022-07-21T14:06:39.877722Z",
          "shell.execute_reply": "2022-07-21T14:06:39.874903Z"
        },
        "papermill": {
          "duration": 2.287216,
          "end_time": "2022-07-21T14:06:39.881172",
          "exception": false,
          "start_time": "2022-07-21T14:06:37.593956",
          "status": "completed"
        },
        "tags": [],
        "id": "0693d25a",
        "outputId": "54697360-2726-4d3a-af9a-e62b92c05edd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fraction of non duplicate question pairs.\n",
            "0.630799407351708\n"
          ]
        }
      ],
      "source": [
        "data = pd.read_csv('../input/quora-question-pairs/train.csv.zip')\n",
        "data = data.dropna()\n",
        "total=len(data.index)\n",
        "print(\"Fraction of non duplicate question pairs.\")\n",
        "print(len(data[data['is_duplicate']==0].index)/total)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "e425bbf7",
      "metadata": {
        "papermill": {
          "duration": 0.01605,
          "end_time": "2022-07-21T14:06:39.913173",
          "exception": false,
          "start_time": "2022-07-21T14:06:39.897123",
          "status": "completed"
        },
        "tags": [],
        "id": "e425bbf7"
      },
      "source": [
        "Split the data and create Dataset dict. Dataset dict is necessary for us to applyt .withtransform described later"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b0cd7992",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:39.945104Z",
          "iopub.status.busy": "2022-07-21T14:06:39.944620Z",
          "iopub.status.idle": "2022-07-21T14:06:40.290023Z",
          "shell.execute_reply": "2022-07-21T14:06:40.289039Z"
        },
        "papermill": {
          "duration": 0.363864,
          "end_time": "2022-07-21T14:06:40.292364",
          "exception": false,
          "start_time": "2022-07-21T14:06:39.928500",
          "status": "completed"
        },
        "tags": [],
        "id": "b0cd7992"
      },
      "outputs": [],
      "source": [
        "train, validation = train_test_split(data, test_size=0.2,random_state=42)\n",
        "tds = Dataset.from_pandas(train)\n",
        "vds = Dataset.from_pandas(validation)\n",
        "ds = DatasetDict()\n",
        "\n",
        "ds['train'] = tds\n",
        "ds['validation'] = vds"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "c261c09e",
      "metadata": {
        "papermill": {
          "duration": 0.00902,
          "end_time": "2022-07-21T14:06:40.310913",
          "exception": false,
          "start_time": "2022-07-21T14:06:40.301893",
          "status": "completed"
        },
        "tags": [],
        "id": "c261c09e"
      },
      "source": [
        "Check the dataset dict"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "04421c59",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:40.330775Z",
          "iopub.status.busy": "2022-07-21T14:06:40.330212Z",
          "iopub.status.idle": "2022-07-21T14:06:40.339032Z",
          "shell.execute_reply": "2022-07-21T14:06:40.338057Z"
        },
        "papermill": {
          "duration": 0.021377,
          "end_time": "2022-07-21T14:06:40.341293",
          "exception": false,
          "start_time": "2022-07-21T14:06:40.319916",
          "status": "completed"
        },
        "tags": [],
        "id": "04421c59",
        "outputId": "45a03bcf-12e6-474e-d929-a30d22f90fb1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "DatasetDict({\n",
              "    train: Dataset({\n",
              "        features: ['id', 'qid1', 'qid2', 'question1', 'question2', 'is_duplicate', '__index_level_0__'],\n",
              "        num_rows: 323429\n",
              "    })\n",
              "    validation: Dataset({\n",
              "        features: ['id', 'qid1', 'qid2', 'question1', 'question2', 'is_duplicate', '__index_level_0__'],\n",
              "        num_rows: 80858\n",
              "    })\n",
              "})"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "ds"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "acfbdfe7",
      "metadata": {
        "papermill": {
          "duration": 0.009178,
          "end_time": "2022-07-21T14:06:40.359664",
          "exception": false,
          "start_time": "2022-07-21T14:06:40.350486",
          "status": "completed"
        },
        "tags": [],
        "id": "acfbdfe7"
      },
      "source": [
        "Load bert default tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ca11a69c",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:40.379578Z",
          "iopub.status.busy": "2022-07-21T14:06:40.378865Z",
          "iopub.status.idle": "2022-07-21T14:06:53.510029Z",
          "shell.execute_reply": "2022-07-21T14:06:53.506707Z"
        },
        "papermill": {
          "duration": 13.145252,
          "end_time": "2022-07-21T14:06:53.514086",
          "exception": false,
          "start_time": "2022-07-21T14:06:40.368834",
          "status": "completed"
        },
        "tags": [],
        "id": "ca11a69c",
        "outputId": "79e365e1-ef50-467a-c2c3-773df9ec5c5d",
        "colab": {
          "referenced_widgets": [
            "0bc41fc527ee49a2b3659f9e95b8f618",
            "c5b957ef4d794a5b85eca43c57b2dad6",
            "71997b999c1f4da58dda19d9ef0c35cf",
            "3d7bddcd97014d4b99e5cdd6cfdcd996"
          ]
        }
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0bc41fc527ee49a2b3659f9e95b8f618",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c5b957ef4d794a5b85eca43c57b2dad6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "71997b999c1f4da58dda19d9ef0c35cf",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/455k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3d7bddcd97014d4b99e5cdd6cfdcd996",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "1aa0f234",
      "metadata": {
        "papermill": {
          "duration": 0.017668,
          "end_time": "2022-07-21T14:06:53.549430",
          "exception": false,
          "start_time": "2022-07-21T14:06:53.531762",
          "status": "completed"
        },
        "tags": [],
        "id": "1aa0f234"
      },
      "source": [
        "Check the tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1b50eb60",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:53.581753Z",
          "iopub.status.busy": "2022-07-21T14:06:53.581345Z",
          "iopub.status.idle": "2022-07-21T14:06:53.601992Z",
          "shell.execute_reply": "2022-07-21T14:06:53.601118Z"
        },
        "papermill": {
          "duration": 0.040658,
          "end_time": "2022-07-21T14:06:53.605580",
          "exception": false,
          "start_time": "2022-07-21T14:06:53.564922",
          "status": "completed"
        },
        "tags": [],
        "id": "1b50eb60"
      },
      "outputs": [],
      "source": [
        "train_encodings = tokenizer(ds['train'][:2]['question1'], ds['train'][:2]['question2'], max_length=128, truncation=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "f3deb8b7",
      "metadata": {
        "papermill": {
          "duration": 0.014943,
          "end_time": "2022-07-21T14:06:53.637358",
          "exception": false,
          "start_time": "2022-07-21T14:06:53.622415",
          "status": "completed"
        },
        "tags": [],
        "id": "f3deb8b7"
      },
      "source": [
        "See the output"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c8dcaf70",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:53.669314Z",
          "iopub.status.busy": "2022-07-21T14:06:53.668967Z",
          "iopub.status.idle": "2022-07-21T14:06:53.675233Z",
          "shell.execute_reply": "2022-07-21T14:06:53.674398Z"
        },
        "papermill": {
          "duration": 0.028254,
          "end_time": "2022-07-21T14:06:53.681096",
          "exception": false,
          "start_time": "2022-07-21T14:06:53.652842",
          "status": "completed"
        },
        "tags": [],
        "id": "c8dcaf70",
        "outputId": "de089773-2a46-4b0d-ee3a-52ceecb83d0a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "{'input_ids': [[101, 2634, 1024, 2054, 2024, 3105, 7047, 1998, 2925, 7047, 2005, 2659, 1039, 21600, 2050, 2030, 7665, 7017, 3330, 2493, 1999, 2634, 1006, 1019, 1011, 1020, 1007, 1029, 102, 5841, 1998, 10922, 1999, 2634, 1024, 1045, 2572, 2747, 1999, 2026, 2353, 2095, 1997, 3330, 1998, 1045, 2031, 1037, 1039, 21600, 2050, 1997, 1021, 1012, 1020, 2007, 2028, 2067, 21197, 1006, 2044, 1018, 13609, 2015, 1007, 1012, 1045, 2031, 2787, 2008, 1045, 2215, 2000, 6148, 1037, 3105, 1999, 1037, 3813, 2066, 14163, 13201, 2030, 2079, 15038, 1012, 2029, 5724, 2003, 2488, 1998, 2054, 2024, 1996, 3316, 2008, 1045, 2064, 6611, 2005, 1999, 2344, 2000, 2131, 1037, 11519, 3105, 1029, 102], [101, 2029, 2003, 2088, 1005, 1055, 2190, 6298, 3185, 1029, 102, 2054, 2003, 1996, 2190, 6298, 3185, 1997, 2035, 2051, 1029, 102]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}"
            ]
          },
          "execution_count": 7,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "train_encodings"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "c881ffdf",
      "metadata": {
        "papermill": {
          "duration": 0.015314,
          "end_time": "2022-07-21T14:06:53.711719",
          "exception": false,
          "start_time": "2022-07-21T14:06:53.696405",
          "status": "completed"
        },
        "tags": [],
        "id": "c881ffdf"
      },
      "source": [
        "Decode the an encoder token collection"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "89820222",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:53.744274Z",
          "iopub.status.busy": "2022-07-21T14:06:53.743933Z",
          "iopub.status.idle": "2022-07-21T14:06:57.892270Z",
          "shell.execute_reply": "2022-07-21T14:06:57.891280Z"
        },
        "papermill": {
          "duration": 4.166869,
          "end_time": "2022-07-21T14:06:57.894512",
          "exception": false,
          "start_time": "2022-07-21T14:06:53.727643",
          "status": "completed"
        },
        "tags": [],
        "id": "89820222",
        "outputId": "67fb1066-8810-4df0-9635-c2f83a57897d"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "\"[CLS] which is world's best romantic movie? [SEP] what is the best romantic movie of all time? [SEP]\""
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tokenizer.decode(train_encodings[\"input_ids\"][1])"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "077149ba",
      "metadata": {
        "papermill": {
          "duration": 0.010341,
          "end_time": "2022-07-21T14:06:57.914982",
          "exception": false,
          "start_time": "2022-07-21T14:06:57.904641",
          "status": "completed"
        },
        "tags": [],
        "id": "077149ba"
      },
      "source": [
        "This function will take in a batch of data and return the tranform the data before returning the transformed data. This acts like a virtual dataset , which gets formed only when we access certain elements. The elements we access forms the batch. The batch will be defined by the user for example prepared_ds[:2] means accessing first two elements of the virtual dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "659d4853",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:57.938806Z",
          "iopub.status.busy": "2022-07-21T14:06:57.937418Z",
          "iopub.status.idle": "2022-07-21T14:06:57.947142Z",
          "shell.execute_reply": "2022-07-21T14:06:57.946308Z"
        },
        "papermill": {
          "duration": 0.023776,
          "end_time": "2022-07-21T14:06:57.949050",
          "exception": false,
          "start_time": "2022-07-21T14:06:57.925274",
          "status": "completed"
        },
        "tags": [],
        "id": "659d4853"
      },
      "outputs": [],
      "source": [
        "def transform(batch):\n",
        "    inputs = tokenizer(list(zip(batch['question1'], batch['question2'])), padding='max_length', max_length=128, truncation=True)\n",
        "    inputs['labels'] = batch['is_duplicate']\n",
        "    return inputs\n",
        "prepared_ds = ds.with_transform(transform)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "2c822d18",
      "metadata": {
        "papermill": {
          "duration": 0.009698,
          "end_time": "2022-07-21T14:06:57.968894",
          "exception": false,
          "start_time": "2022-07-21T14:06:57.959196",
          "status": "completed"
        },
        "tags": [],
        "id": "2c822d18"
      },
      "source": [
        "Testing the virtual dataset."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "fec222fa",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:57.990702Z",
          "iopub.status.busy": "2022-07-21T14:06:57.990401Z",
          "iopub.status.idle": "2022-07-21T14:06:57.998127Z",
          "shell.execute_reply": "2022-07-21T14:06:57.997183Z"
        },
        "papermill": {
          "duration": 0.0213,
          "end_time": "2022-07-21T14:06:58.000038",
          "exception": false,
          "start_time": "2022-07-21T14:06:57.978738",
          "status": "completed"
        },
        "tags": [],
        "id": "fec222fa",
        "outputId": "23425a76-e6f2-450b-fb9e-265d5b1d6fbd"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "\"[CLS] why do they study human anatomy? [SEP] what's the best way to study human anatomy? [SEP] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD] [PAD]\""
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "tokenizer.decode(prepared_ds['train'][1000][\"input_ids\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "7c8b7fd7",
      "metadata": {
        "papermill": {
          "duration": 0.01039,
          "end_time": "2022-07-21T14:06:58.020431",
          "exception": false,
          "start_time": "2022-07-21T14:06:58.010041",
          "status": "completed"
        },
        "tags": [],
        "id": "7c8b7fd7"
      },
      "source": [
        "Gpu/cpu handler"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3802fa6f",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:58.042850Z",
          "iopub.status.busy": "2022-07-21T14:06:58.042113Z",
          "iopub.status.idle": "2022-07-21T14:06:58.101467Z",
          "shell.execute_reply": "2022-07-21T14:06:58.099970Z"
        },
        "papermill": {
          "duration": 0.072921,
          "end_time": "2022-07-21T14:06:58.103704",
          "exception": false,
          "start_time": "2022-07-21T14:06:58.030783",
          "status": "completed"
        },
        "tags": [],
        "id": "3802fa6f"
      },
      "outputs": [],
      "source": [
        "cuda=torch.device('cuda')\n",
        "cpu=torch.device('cpu')\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "if device==cuda:\n",
        "    n_gpu = torch.cuda.device_count()\n",
        "    torch.cuda.get_device_name(0)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "8c3b1cf2",
      "metadata": {
        "papermill": {
          "duration": 0.010066,
          "end_time": "2022-07-21T14:06:58.124134",
          "exception": false,
          "start_time": "2022-07-21T14:06:58.114068",
          "status": "completed"
        },
        "tags": [],
        "id": "8c3b1cf2"
      },
      "source": [
        "The model.forward requires inputs of batch of data to be in the form of datapoints as rows stacked one top of another. This collate_fn does the online tranformation whenever dataloader asks for a batch of data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1d66f957",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:58.145688Z",
          "iopub.status.busy": "2022-07-21T14:06:58.144946Z",
          "iopub.status.idle": "2022-07-21T14:06:58.151956Z",
          "shell.execute_reply": "2022-07-21T14:06:58.151104Z"
        },
        "papermill": {
          "duration": 0.019774,
          "end_time": "2022-07-21T14:06:58.153870",
          "exception": false,
          "start_time": "2022-07-21T14:06:58.134096",
          "status": "completed"
        },
        "tags": [],
        "id": "1d66f957"
      },
      "outputs": [],
      "source": [
        "def collate_fn(batch):\n",
        "    return {\n",
        "        'input_ids': torch.stack([torch.tensor(x['input_ids']) for x in batch]).to(device),\n",
        "        'token_type_ids': torch.stack([torch.tensor(x['token_type_ids']) for x in batch]).to(device),\n",
        "        'attention_mask': torch.stack([torch.tensor(x['attention_mask']) for x in batch]).to(device),\n",
        "        'labels': torch.tensor([x['labels'] for x in batch]).to(device)\n",
        "    }"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "90c30e1c",
      "metadata": {
        "papermill": {
          "duration": 0.01035,
          "end_time": "2022-07-21T14:06:58.174292",
          "exception": false,
          "start_time": "2022-07-21T14:06:58.163942",
          "status": "completed"
        },
        "tags": [],
        "id": "90c30e1c"
      },
      "source": [
        "Load only the needed data in the memory."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "89aa1e87",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:58.195714Z",
          "iopub.status.busy": "2022-07-21T14:06:58.195149Z",
          "iopub.status.idle": "2022-07-21T14:06:58.200535Z",
          "shell.execute_reply": "2022-07-21T14:06:58.199714Z"
        },
        "papermill": {
          "duration": 0.018094,
          "end_time": "2022-07-21T14:06:58.202414",
          "exception": false,
          "start_time": "2022-07-21T14:06:58.184320",
          "status": "completed"
        },
        "tags": [],
        "id": "89aa1e87"
      },
      "outputs": [],
      "source": [
        "\n",
        "train_dataloader = DataLoader(prepared_ds['train'],  batch_size=32,collate_fn=collate_fn ,drop_last=True)\n",
        "validation_dataloader= DataLoader(prepared_ds['train'],  batch_size=8,collate_fn=collate_fn ,drop_last=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "786483e1",
      "metadata": {
        "papermill": {
          "duration": 0.01072,
          "end_time": "2022-07-21T14:06:58.223389",
          "exception": false,
          "start_time": "2022-07-21T14:06:58.212669",
          "status": "completed"
        },
        "tags": [],
        "id": "786483e1"
      },
      "source": [
        "Test the dataloader."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ed8d6855",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:06:58.244751Z",
          "iopub.status.busy": "2022-07-21T14:06:58.244053Z",
          "iopub.status.idle": "2022-07-21T14:07:02.302609Z",
          "shell.execute_reply": "2022-07-21T14:07:02.301604Z"
        },
        "papermill": {
          "duration": 4.071768,
          "end_time": "2022-07-21T14:07:02.305115",
          "exception": false,
          "start_time": "2022-07-21T14:06:58.233347",
          "status": "completed"
        },
        "tags": [],
        "id": "ed8d6855"
      },
      "outputs": [],
      "source": [
        "a=next(iter(train_dataloader))"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "85ff6c39",
      "metadata": {
        "papermill": {
          "duration": 0.009971,
          "end_time": "2022-07-21T14:07:02.325655",
          "exception": false,
          "start_time": "2022-07-21T14:07:02.315684",
          "status": "completed"
        },
        "tags": [],
        "id": "85ff6c39"
      },
      "source": [
        "Load the pretrained model. Here the model is Bert base."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e36f4422",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:07:02.348603Z",
          "iopub.status.busy": "2022-07-21T14:07:02.347761Z",
          "iopub.status.idle": "2022-07-21T14:07:30.328093Z",
          "shell.execute_reply": "2022-07-21T14:07:30.327077Z"
        },
        "papermill": {
          "duration": 27.99452,
          "end_time": "2022-07-21T14:07:30.330713",
          "exception": false,
          "start_time": "2022-07-21T14:07:02.336193",
          "status": "completed"
        },
        "tags": [],
        "id": "e36f4422",
        "outputId": "13f8e9eb-7031-43bd-ed44-71915720b2d0",
        "colab": {
          "referenced_widgets": [
            "da92078e8c8f46f39f70afd061162fae"
          ]
        }
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "da92078e8c8f46f39f70afd061162fae",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/420M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.weight']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        }
      ],
      "source": [
        "model = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=2)\n",
        "model.save_pretrained('/kaggle/working/utsav/bert_uncased_quora')\n",
        "model = BertForSequenceClassification.from_pretrained('/kaggle/working/utsav/bert_uncased_quora')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8517ef63",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:07:30.353647Z",
          "iopub.status.busy": "2022-07-21T14:07:30.353346Z",
          "iopub.status.idle": "2022-07-21T14:07:30.474641Z",
          "shell.execute_reply": "2022-07-21T14:07:30.473730Z"
        },
        "papermill": {
          "duration": 0.135176,
          "end_time": "2022-07-21T14:07:30.476748",
          "exception": false,
          "start_time": "2022-07-21T14:07:30.341572",
          "status": "completed"
        },
        "tags": [],
        "id": "8517ef63"
      },
      "outputs": [],
      "source": [
        "if device==cuda:\n",
        "    model.cuda()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "90c8024e",
      "metadata": {
        "papermill": {
          "duration": 0.010174,
          "end_time": "2022-07-21T14:07:30.497441",
          "exception": false,
          "start_time": "2022-07-21T14:07:30.487267",
          "status": "completed"
        },
        "tags": [],
        "id": "90c8024e"
      },
      "source": [
        "Test the model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3eecf2bc",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:07:30.519980Z",
          "iopub.status.busy": "2022-07-21T14:07:30.519193Z",
          "iopub.status.idle": "2022-07-21T14:07:31.407383Z",
          "shell.execute_reply": "2022-07-21T14:07:31.406330Z"
        },
        "papermill": {
          "duration": 0.90208,
          "end_time": "2022-07-21T14:07:31.409812",
          "exception": false,
          "start_time": "2022-07-21T14:07:30.507732",
          "status": "completed"
        },
        "tags": [],
        "id": "3eecf2bc",
        "outputId": "7cd83d6c-f94c-4c75-8ecc-900b548bc3f6"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "SequenceClassifierOutput(loss=tensor(0.6623, device='cuda:0', grad_fn=<NllLossBackward0>), logits=tensor([[ 0.2918, -0.2705],\n",
              "        [ 0.3501, -0.2778],\n",
              "        [ 0.3128, -0.2652],\n",
              "        [ 0.3697, -0.2650],\n",
              "        [ 0.3045, -0.2712],\n",
              "        [ 0.2934, -0.2684],\n",
              "        [ 0.4794, -0.2628],\n",
              "        [ 0.3360, -0.2756],\n",
              "        [ 0.3325, -0.2680],\n",
              "        [ 0.2826, -0.2786],\n",
              "        [ 0.3155, -0.2570],\n",
              "        [ 0.3243, -0.2784],\n",
              "        [ 0.3202, -0.2764],\n",
              "        [ 0.3329, -0.2619],\n",
              "        [ 0.3789, -0.2670],\n",
              "        [ 0.3442, -0.2711],\n",
              "        [ 0.3233, -0.2767],\n",
              "        [ 0.3102, -0.2630],\n",
              "        [ 0.3027, -0.2731],\n",
              "        [ 0.2932, -0.2651],\n",
              "        [ 0.2824, -0.2675],\n",
              "        [ 0.3384, -0.2752],\n",
              "        [ 0.3355, -0.2688],\n",
              "        [ 0.2977, -0.2582],\n",
              "        [ 0.3310, -0.2654],\n",
              "        [ 0.3554, -0.2793],\n",
              "        [ 0.2997, -0.2617],\n",
              "        [ 0.3203, -0.2641],\n",
              "        [ 0.3197, -0.2722],\n",
              "        [ 0.3205, -0.2743],\n",
              "        [ 0.3172, -0.2654],\n",
              "        [ 0.3541, -0.2894]], device='cuda:0', grad_fn=<AddmmBackward0>), hidden_states=None, attentions=None)"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model(a['input_ids'], token_type_ids=a['token_type_ids'], attention_mask=a['attention_mask'], labels=a['labels'])"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "b79d7935",
      "metadata": {
        "papermill": {
          "duration": 0.010413,
          "end_time": "2022-07-21T14:07:31.431288",
          "exception": false,
          "start_time": "2022-07-21T14:07:31.420875",
          "status": "completed"
        },
        "tags": [],
        "id": "b79d7935"
      },
      "source": [
        "Group the parameters , so that we don't apply weight decay to these parameters."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5a490f5e",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:07:31.453728Z",
          "iopub.status.busy": "2022-07-21T14:07:31.453104Z",
          "iopub.status.idle": "2022-07-21T14:07:31.460571Z",
          "shell.execute_reply": "2022-07-21T14:07:31.459626Z"
        },
        "papermill": {
          "duration": 0.020868,
          "end_time": "2022-07-21T14:07:31.462593",
          "exception": false,
          "start_time": "2022-07-21T14:07:31.441725",
          "status": "completed"
        },
        "tags": [],
        "id": "5a490f5e"
      },
      "outputs": [],
      "source": [
        "param_optimizer = list(model.named_parameters())\n",
        "no_decay = ['bias', 'gamma', 'beta']\n",
        "optimizer_grouped_parameters = [{'params': [p for n, p in param_optimizer if all(nd not in n for nd in no_decay)], 'weight_decay_rate': 0.01}, {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay_rate': 0.0}]"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "08cd3958",
      "metadata": {
        "papermill": {
          "duration": 0.010812,
          "end_time": "2022-07-21T14:07:31.484410",
          "exception": false,
          "start_time": "2022-07-21T14:07:31.473598",
          "status": "completed"
        },
        "tags": [],
        "id": "08cd3958"
      },
      "source": [
        "Create the optimizer for bert."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5269ddd9",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:07:31.506384Z",
          "iopub.status.busy": "2022-07-21T14:07:31.506128Z",
          "iopub.status.idle": "2022-07-21T14:07:43.872268Z",
          "shell.execute_reply": "2022-07-21T14:07:43.871136Z"
        },
        "papermill": {
          "duration": 12.380233,
          "end_time": "2022-07-21T14:07:43.875127",
          "exception": false,
          "start_time": "2022-07-21T14:07:31.494894",
          "status": "completed"
        },
        "tags": [],
        "id": "5269ddd9",
        "outputId": "b1876bc1-c726-4403-b3dc-11aacd0720c1"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
          ]
        }
      ],
      "source": [
        "%%capture\n",
        "!pip install pytorch-pretrained-bert"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6d104ea5",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:07:43.899172Z",
          "iopub.status.busy": "2022-07-21T14:07:43.898352Z",
          "iopub.status.idle": "2022-07-21T14:07:43.915657Z",
          "shell.execute_reply": "2022-07-21T14:07:43.914826Z"
        },
        "papermill": {
          "duration": 0.031114,
          "end_time": "2022-07-21T14:07:43.917630",
          "exception": false,
          "start_time": "2022-07-21T14:07:43.886516",
          "status": "completed"
        },
        "tags": [],
        "id": "6d104ea5"
      },
      "outputs": [],
      "source": [
        "from pytorch_pretrained_bert import BertAdam\n",
        "optimizer = BertAdam(optimizer_grouped_parameters,lr=2e-5,warmup=.1)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "9301f90d",
      "metadata": {
        "papermill": {
          "duration": 0.010569,
          "end_time": "2022-07-21T14:07:43.938846",
          "exception": false,
          "start_time": "2022-07-21T14:07:43.928277",
          "status": "completed"
        },
        "tags": [],
        "id": "9301f90d"
      },
      "source": [
        "Helper function to check accuracy."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f712f651",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:07:43.962509Z",
          "iopub.status.busy": "2022-07-21T14:07:43.961776Z",
          "iopub.status.idle": "2022-07-21T14:07:43.967128Z",
          "shell.execute_reply": "2022-07-21T14:07:43.966336Z"
        },
        "papermill": {
          "duration": 0.019307,
          "end_time": "2022-07-21T14:07:43.969572",
          "exception": false,
          "start_time": "2022-07-21T14:07:43.950265",
          "status": "completed"
        },
        "tags": [],
        "id": "f712f651"
      },
      "outputs": [],
      "source": [
        "\n",
        "def accuracy(predicted_distribution, true_label):\n",
        "    predicted_label=np.argmax(predicted_distribution, axis=1)\n",
        "    return np.sum(predicted_label.flatten() == true_label.flatten()) / len(true_label.flatten())"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "6abb4b45",
      "metadata": {
        "papermill": {
          "duration": 0.011024,
          "end_time": "2022-07-21T14:07:43.991082",
          "exception": false,
          "start_time": "2022-07-21T14:07:43.980058",
          "status": "completed"
        },
        "tags": [],
        "id": "6abb4b45"
      },
      "source": [
        "Training loop."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "014a77cb",
      "metadata": {
        "papermill": {
          "duration": 0.01063,
          "end_time": "2022-07-21T14:07:44.012678",
          "exception": false,
          "start_time": "2022-07-21T14:07:44.002048",
          "status": "completed"
        },
        "tags": [],
        "id": "014a77cb"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "b663ac9e",
      "metadata": {
        "papermill": {
          "duration": 0.010665,
          "end_time": "2022-07-21T14:07:44.034507",
          "exception": false,
          "start_time": "2022-07-21T14:07:44.023842",
          "status": "completed"
        },
        "tags": [],
        "id": "b663ac9e"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c925ce9a",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T14:07:44.056760Z",
          "iopub.status.busy": "2022-07-21T14:07:44.056485Z",
          "iopub.status.idle": "2022-07-21T16:28:54.191499Z",
          "shell.execute_reply": "2022-07-21T16:28:54.190393Z"
        },
        "papermill": {
          "duration": 8470.149077,
          "end_time": "2022-07-21T16:28:54.194068",
          "exception": false,
          "start_time": "2022-07-21T14:07:44.044991",
          "status": "completed"
        },
        "tags": [],
        "id": "c925ce9a",
        "outputId": "0e2097c0-4b64-4b78-86a1-d485ac5dcfc6"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch:   0%|          | 0/2 [00:00<?, ?it/s]/opt/conda/lib/python3.7/site-packages/pytorch_pretrained_bert/optimization.py:275: UserWarning: This overload of add_ is deprecated:\n",
            "\tadd_(Number alpha, Tensor other)\n",
            "Consider using one of the following signatures instead:\n",
            "\tadd_(Tensor other, *, Number alpha) (Triggered internally at  /usr/local/src/pytorch/torch/csrc/utils/python_arg_parser.cpp:1055.)\n",
            "  next_m.mul_(beta1).add_(1 - beta1, grad)\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train loss: 0.6747491359710693\n",
            "Train loss: 0.5472003678874214\n",
            "Train loss: 0.5084767538813216\n",
            "Train loss: 0.4887447274998573\n",
            "Train loss: 0.4731966943812192\n",
            "Train loss: 0.4625732322593411\n",
            "Train loss: 0.4514341955871233\n",
            "Train loss: 0.4432584316006061\n",
            "Train loss: 0.4370120427060812\n",
            "Train loss: 0.42960275362321193\n",
            "Train loss: 0.4229158838460972\n",
            "Train loss: 0.4164017536064151\n",
            "Train loss: 0.4124621583162795\n",
            "Train loss: 0.4087802881144451\n",
            "Train loss: 0.40421793937980915\n",
            "Train loss: 0.3998031539669996\n",
            "Train loss: 0.39417326926421853\n",
            "Train loss: 0.3910481544022067\n",
            "Train loss: 0.3877770238778844\n",
            "Train loss: 0.38428715650954415\n",
            "Train loss: 0.3811478722302512\n",
            "Train loss: 0.3775077939196634\n",
            "Train loss: 0.3757044928167853\n",
            "Train loss: 0.37371186858633193\n",
            "Train loss: 0.3721031927438092\n",
            "Train loss: 0.36941189318299533\n",
            "Train loss: 0.36686243307631733\n",
            "Train loss: 0.3649489373018326\n",
            "Train loss: 0.3633448027183396\n",
            "Train loss: 0.3616773231004444\n",
            "Train loss: 0.3601237803781998\n",
            "Train loss: 0.358325750202719\n",
            "Train loss: 0.3567044536226133\n",
            "Train loss: 0.35458567723410667\n",
            "Train loss: 0.35338710036848403\n",
            "Train loss: 0.35172246055402473\n",
            "Train loss: 0.34994088560839354\n",
            "Train loss: 0.3490562526113083\n",
            "Train loss: 0.3477353354902745\n",
            "Train loss: 0.3459944677708914\n",
            "Train loss: 0.3444119133083143\n",
            "Train loss: 0.3428850416561883\n",
            "Train loss: 0.34138853547190573\n",
            "Train loss: 0.3403472568691678\n",
            "Train loss: 0.33917938257802266\n",
            "Train loss: 0.3385242189805631\n",
            "Train loss: 0.3375897518359327\n",
            "Train loss: 0.3365220274887955\n",
            "Train loss: 0.3357351016446442\n",
            "Train loss: 0.3350160939946172\n",
            "Train loss: 0.3343607581566391\n",
            "Train loss: 0.333104480194522\n",
            "Train loss: 0.33216046513278674\n",
            "Train loss: 0.3309743366566295\n",
            "Train loss: 0.3300931323964581\n",
            "Train loss: 0.3293026019212138\n",
            "Train loss: 0.32856696371738664\n",
            "Train loss: 0.3276315435696602\n",
            "Train loss: 0.32662537655574947\n",
            "Train loss: 0.32566465706219005\n",
            "Train loss: 0.3248555272944251\n",
            "Train loss: 0.3239974386488155\n",
            "Train loss: 0.3233910449338217\n",
            "Train loss: 0.32270533354665304\n",
            "Train loss: 0.3221332876221535\n",
            "Train loss: 0.3214645931946188\n",
            "Train loss: 0.3209306332136696\n",
            "Train loss: 0.3200579701379829\n",
            "Train loss: 0.3192913377767906\n",
            "Train loss: 0.3185995447246549\n",
            "Train loss: 0.31800893909666234\n",
            "Train loss: 0.3171881886141147\n",
            "Train loss: 0.316419080023947\n",
            "Train loss: 0.3156020893203675\n",
            "Train loss: 0.31482770840890695\n",
            "Train loss: 0.3143464665228709\n",
            "Train loss: 0.31367476817248385\n",
            "Train loss: 0.31313645201056145\n",
            "Train loss: 0.31247560127971724\n",
            "Train loss: 0.312167497103523\n",
            "Train loss: 0.3115451830219156\n",
            "Train loss: 0.311053706193475\n",
            "Train loss: 0.3102899668435035\n",
            "Train loss: 0.3098355832264777\n",
            "Train loss: 0.30940364570147494\n",
            "Train loss: 0.3088755100722727\n",
            "Train loss: 0.3081499331059802\n",
            "Train loss: 0.30773534802635394\n",
            "Train loss: 0.3071244678360273\n",
            "Train loss: 0.30673951852793385\n",
            "Train loss: 0.3062551347233596\n",
            "Train loss: 0.3057957027885609\n",
            "Train loss: 0.3052065240047364\n",
            "Train loss: 0.3048709133977775\n",
            "Train loss: 0.3044545175772172\n",
            "Train loss: 0.3038369993491118\n",
            "Train loss: 0.3034903224250722\n",
            "Train loss: 0.3031190059715823\n",
            "Train loss: 0.3027280114690706\n",
            "Train loss: 0.30232791265488007\n",
            "Train loss: 0.30184096753531703\n",
            "Train loss: 0.30137507025764293\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch:  50%|█████     | 1/2 [1:10:33<1:10:33, 4233.26s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train loss: 0.5451766848564148\n",
            "Train loss: 0.2346865550273716\n",
            "Train loss: 0.2293884817195769\n",
            "Train loss: 0.2304873500279216\n",
            "Train loss: 0.23001869405594252\n",
            "Train loss: 0.23059904216946717\n",
            "Train loss: 0.2308927182225935\n",
            "Train loss: 0.2294831455774212\n",
            "Train loss: 0.2296299049991198\n",
            "Train loss: 0.2273212332179259\n",
            "Train loss: 0.22666803610461814\n",
            "Train loss: 0.2259679102016567\n",
            "Train loss: 0.2252525506505462\n",
            "Train loss: 0.224660979513706\n",
            "Train loss: 0.22249250435705784\n",
            "Train loss: 0.2221410984271451\n",
            "Train loss: 0.2201393289534581\n",
            "Train loss: 0.21990146614317682\n",
            "Train loss: 0.21899945686040223\n",
            "Train loss: 0.21839111454212654\n",
            "Train loss: 0.21744673843516493\n",
            "Train loss: 0.21622279091963792\n",
            "Train loss: 0.21633734377266087\n",
            "Train loss: 0.21576271725106969\n",
            "Train loss: 0.21603889704473636\n",
            "Train loss: 0.21475114170737455\n",
            "Train loss: 0.214042699504436\n",
            "Train loss: 0.21326076294033783\n",
            "Train loss: 0.21345242063200015\n",
            "Train loss: 0.212816795967301\n",
            "Train loss: 0.2128259799064884\n",
            "Train loss: 0.21227125049067463\n",
            "Train loss: 0.21217687502887445\n",
            "Train loss: 0.21158491444746605\n",
            "Train loss: 0.21167753351169347\n",
            "Train loss: 0.21096731578067723\n",
            "Train loss: 0.21029457637470683\n",
            "Train loss: 0.2103975913951652\n",
            "Train loss: 0.20994569466960677\n",
            "Train loss: 0.20936971163136994\n",
            "Train loss: 0.20869554928230422\n",
            "Train loss: 0.20802863773664332\n",
            "Train loss: 0.20753787969161686\n",
            "Train loss: 0.20723722423943247\n",
            "Train loss: 0.20686374741194397\n",
            "Train loss: 0.20672419805835351\n",
            "Train loss: 0.20660873971979882\n",
            "Train loss: 0.20641812417831315\n",
            "Train loss: 0.2062826390783412\n",
            "Train loss: 0.20625616752798523\n",
            "Train loss: 0.20619072790736795\n",
            "Train loss: 0.20584150259794734\n",
            "Train loss: 0.20539861561655653\n",
            "Train loss: 0.20485760300151246\n",
            "Train loss: 0.20442264893783163\n",
            "Train loss: 0.20419286028867178\n",
            "Train loss: 0.20394447738984145\n",
            "Train loss: 0.20354805261137152\n",
            "Train loss: 0.20306698554685929\n",
            "Train loss: 0.20257463441223256\n",
            "Train loss: 0.20216315058524797\n",
            "Train loss: 0.20176007295687742\n",
            "Train loss: 0.2016921616015949\n",
            "Train loss: 0.2015131075169322\n",
            "Train loss: 0.20122091665106245\n",
            "Train loss: 0.20089109318491047\n",
            "Train loss: 0.20077937848046545\n",
            "Train loss: 0.2005196320641563\n",
            "Train loss: 0.20021625887479322\n",
            "Train loss: 0.1999572464650225\n",
            "Train loss: 0.19978363975885724\n",
            "Train loss: 0.19932112645097114\n",
            "Train loss: 0.19895957039203302\n",
            "Train loss: 0.1986305687182114\n",
            "Train loss: 0.19827136063993323\n",
            "Train loss: 0.19813680324523175\n",
            "Train loss: 0.19782255223240724\n",
            "Train loss: 0.19755804006780894\n",
            "Train loss: 0.19728382797108301\n",
            "Train loss: 0.19720917863450937\n",
            "Train loss: 0.19694406667382267\n",
            "Train loss: 0.1968866024865786\n",
            "Train loss: 0.19661708359152658\n",
            "Train loss: 0.19646572867454842\n",
            "Train loss: 0.19631665558910644\n",
            "Train loss: 0.19596454368941701\n",
            "Train loss: 0.19558939626666713\n",
            "Train loss: 0.19547285543001805\n",
            "Train loss: 0.1951293074616369\n",
            "Train loss: 0.1951235810995986\n",
            "Train loss: 0.19488601446062337\n",
            "Train loss: 0.19478426867503082\n",
            "Train loss: 0.19452980458562322\n",
            "Train loss: 0.19457760189217613\n",
            "Train loss: 0.1944842645128716\n",
            "Train loss: 0.1940069208853829\n",
            "Train loss: 0.19377569231754596\n",
            "Train loss: 0.19358443260387181\n",
            "Train loss: 0.19344531984956256\n",
            "Train loss: 0.19327253179692214\n",
            "Train loss: 0.1930499922330693\n",
            "Train loss: 0.19280234654155118\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Epoch: 100%|██████████| 2/2 [2:21:09<00:00, 4234.67s/it]\n"
          ]
        }
      ],
      "source": [
        "def train(epochs = 2):\n",
        "    for _ in trange(epochs, desc=\"Epoch\"):\n",
        "      cumulative_loss = 0\n",
        "      number_of_batch = 0\n",
        "      for step, batch in enumerate(train_dataloader):\n",
        "        optimizer.zero_grad()\n",
        "        output=model.forward(batch['input_ids'], token_type_ids=batch['token_type_ids'], attention_mask=batch['attention_mask'], labels=batch['labels'])\n",
        "        loss = output.loss\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        cumulative_loss += loss.item()\n",
        "        number_of_batch += 1\n",
        "        if (step%100==0):\n",
        "          print(f\"Train loss: {cumulative_loss / number_of_batch}\")\n",
        "    model.save_pretrained('/kaggle/working/utsav/bert_uncased_quora')\n",
        "model.train()\n",
        "train()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "2ea6d8b8",
      "metadata": {
        "papermill": {
          "duration": 0.022017,
          "end_time": "2022-07-21T16:28:54.238786",
          "exception": false,
          "start_time": "2022-07-21T16:28:54.216769",
          "status": "completed"
        },
        "tags": [],
        "id": "2ea6d8b8"
      },
      "source": [
        "Load the model again for testing."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "1e08f9f5",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T16:28:54.284861Z",
          "iopub.status.busy": "2022-07-21T16:28:54.284530Z",
          "iopub.status.idle": "2022-07-21T16:28:55.441502Z",
          "shell.execute_reply": "2022-07-21T16:28:55.440486Z"
        },
        "papermill": {
          "duration": 1.183179,
          "end_time": "2022-07-21T16:28:55.444024",
          "exception": false,
          "start_time": "2022-07-21T16:28:54.260845",
          "status": "completed"
        },
        "tags": [],
        "id": "1e08f9f5"
      },
      "outputs": [],
      "source": [
        "model = BertForSequenceClassification.from_pretrained('/kaggle/working/utsav/bert_uncased_quora')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "941566c5",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T16:28:55.491760Z",
          "iopub.status.busy": "2022-07-21T16:28:55.490906Z",
          "iopub.status.idle": "2022-07-21T16:28:55.609311Z",
          "shell.execute_reply": "2022-07-21T16:28:55.608255Z"
        },
        "papermill": {
          "duration": 0.144823,
          "end_time": "2022-07-21T16:28:55.612319",
          "exception": false,
          "start_time": "2022-07-21T16:28:55.467496",
          "status": "completed"
        },
        "tags": [],
        "id": "941566c5",
        "outputId": "c5726845-4d5a-4ce3-a16f-7998a3b6f288"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.cuda()\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7689f46b",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T16:28:55.705354Z",
          "iopub.status.busy": "2022-07-21T16:28:55.704864Z",
          "iopub.status.idle": "2022-07-21T16:28:55.834617Z",
          "shell.execute_reply": "2022-07-21T16:28:55.833507Z"
        },
        "papermill": {
          "duration": 0.185094,
          "end_time": "2022-07-21T16:28:55.837455",
          "exception": false,
          "start_time": "2022-07-21T16:28:55.652361",
          "status": "completed"
        },
        "tags": [],
        "id": "7689f46b",
        "outputId": "525fabeb-c50b-4377-b818-8deb9f3dd65c"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "SequenceClassifierOutput(loss=tensor(0.3572, device='cuda:0', grad_fn=<NllLossBackward0>), logits=tensor([[ 3.5324, -4.3893],\n",
              "        [-3.2461,  2.9924],\n",
              "        [-0.6646,  0.2634],\n",
              "        [ 3.8006, -4.4894],\n",
              "        [-0.0651, -0.4878],\n",
              "        [-2.0623,  1.7214],\n",
              "        [ 5.3005, -5.5592],\n",
              "        [ 3.5004, -4.6342],\n",
              "        [ 0.7742, -1.4620],\n",
              "        [ 5.3522, -5.6255],\n",
              "        [-3.1062,  2.9500],\n",
              "        [ 3.9453, -4.9249],\n",
              "        [-1.8012,  1.4391],\n",
              "        [ 3.1127, -4.1602],\n",
              "        [-3.5179,  3.2688],\n",
              "        [-0.8664,  0.4685],\n",
              "        [-1.8550,  1.4805],\n",
              "        [ 3.2760, -4.4044],\n",
              "        [-1.7896,  1.4809],\n",
              "        [ 2.9638, -4.0874],\n",
              "        [ 0.5332, -1.0197],\n",
              "        [-2.9439,  2.5944],\n",
              "        [ 0.7045, -1.6593],\n",
              "        [ 2.4276, -3.5577],\n",
              "        [ 0.0999, -0.5319],\n",
              "        [-2.0110,  1.6630],\n",
              "        [ 3.1663, -4.3434],\n",
              "        [ 1.3723, -2.1011],\n",
              "        [-0.9945,  0.5945],\n",
              "        [ 4.8769, -5.4918],\n",
              "        [-1.1205,  0.7428],\n",
              "        [ 1.3958, -2.3469]], device='cuda:0', grad_fn=<AddmmBackward0>), hidden_states=None, attentions=None)"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model(a['input_ids'], token_type_ids=a['token_type_ids'], attention_mask=a['attention_mask'], labels=a['labels'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "0d71fb95",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T16:28:55.918976Z",
          "iopub.status.busy": "2022-07-21T16:28:55.918558Z",
          "iopub.status.idle": "2022-07-21T16:28:55.929710Z",
          "shell.execute_reply": "2022-07-21T16:28:55.928822Z"
        },
        "papermill": {
          "duration": 0.054639,
          "end_time": "2022-07-21T16:28:55.932351",
          "exception": false,
          "start_time": "2022-07-21T16:28:55.877712",
          "status": "completed"
        },
        "tags": [],
        "id": "0d71fb95"
      },
      "outputs": [],
      "source": [
        "def validate():\n",
        "  model.eval()\n",
        "  cumulated_accuracy = 0\n",
        "  number_of_batch= 0\n",
        "  for batch in validation_dataloader:\n",
        "    with torch.no_grad():\n",
        "      output=model.forward(batch['input_ids'], token_type_ids=batch['token_type_ids'], attention_mask=batch['attention_mask'], labels=batch['labels'])\n",
        "      logits = output.logits\n",
        "    logits = logits.detach().cpu().numpy()\n",
        "    label_ids = batch['labels'].to('cpu').numpy()\n",
        "    batch_accuracy = accuracy(logits, label_ids)\n",
        "    cumulated_accuracy += batch_accuracy\n",
        "    number_of_batch += 1\n",
        "  print(f\"Validation Accuracy: {cumulated_accuracy / number_of_batch}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8cda9834",
      "metadata": {
        "execution": {
          "iopub.execute_input": "2022-07-21T16:28:56.010304Z",
          "iopub.status.busy": "2022-07-21T16:28:56.009901Z",
          "iopub.status.idle": "2022-07-21T16:53:05.168416Z",
          "shell.execute_reply": "2022-07-21T16:53:05.167481Z"
        },
        "papermill": {
          "duration": 1449.225337,
          "end_time": "2022-07-21T16:53:05.195569",
          "exception": false,
          "start_time": "2022-07-21T16:28:55.970232",
          "status": "completed"
        },
        "tags": [],
        "id": "8cda9834",
        "outputId": "3eb0964a-8c4e-42b8-8f26-48cedbdd6c15"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Validation Accuracy: 0.9621209310378945\n"
          ]
        }
      ],
      "source": [
        "model.eval()\n",
        "validate()"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.12"
    },
    "papermill": {
      "default_parameters": {},
      "duration": 10001.506728,
      "end_time": "2022-07-21T16:53:08.157716",
      "environment_variables": {},
      "exception": null,
      "input_path": "__notebook__.ipynb",
      "output_path": "__notebook__.ipynb",
      "parameters": {},
      "start_time": "2022-07-21T14:06:26.650988",
      "version": "2.3.4"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}