Vanilla_Enocder_Decoder_Architecture.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Vanilla Enocder Decoder Architecture.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyPYHZusOCFyQIEtgfGLr5I+",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/akashe/Python-Code-Generation/blob/main/Vanilla_Enocder_Decoder_Architecture.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U94udjj8Ztoh"
      },
      "source": [
        "#### Getting data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NXcGChd1W_Bu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4859ee4c-67a8-41dc-d620-4205da03a512"
      },
      "source": [
        "! git clone https://github.com/akashe/Python-Code-Generation"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'Python-Code-Generation'...\n",
            "remote: Enumerating objects: 58, done.\u001b[K\n",
            "remote: Counting objects: 100% (58/58), done.\u001b[K\n",
            "remote: Compressing objects: 100% (46/46), done.\u001b[K\n",
            "remote: Total 58 (delta 23), reused 36 (delta 9), pack-reused 0\u001b[K\n",
            "Unpacking objects: 100% (58/58), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "43npsWgPUpFY"
      },
      "source": [
        "import sys\r\n",
        "sys.path.append(\"/content/Python-Code-Generation/\")"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7kUxSfC6SavI"
      },
      "source": [
        "from data_processing import getTokenizer,getData"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-adCp6fFXSnB"
      },
      "source": [
        "questions, answers = getData(\"/content/Python-Code-Generation/data/english_python_data_pruned.txt\")"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VK3S-crQY0ln",
        "outputId": "1d3f2ae6-ec82-407e-cb3a-1e50a5a75832"
      },
      "source": [
        "questions[100],answers[100]"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('write a python program to break when the num is perfectly divisible',\n",
              " 'i = 1\\nwhile True:\\n\\tif i%3 == 0:\\n\\t\\tbreak\\n\\tprint(i)\\n\\ti+= 1\\n')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4OG9NDa0ZKW4",
        "outputId": "548eb5d3-efbd-4a6c-d9a4-cc2dcbb89cf5"
      },
      "source": [
        "test_tokenizer = getTokenizer(answers[100])\r\n",
        "print(test_tokenizer)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['i', '=', '1', '\\n', 'while', 'True', ':', '\\n', '\\t', 'if', 'i', '%', '3', '==', '0', ':', '\\n', '\\t\\t', 'break', '\\n', '', 'print', '(', 'i', ')', '\\n', 'i', '+=', '1', '\\n', '']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "foHgoFJyY-vo",
        "outputId": "24f44c00-bce8-49b8-c4cc-01fc19cf3ef2"
      },
      "source": [
        "len(questions),len(answers)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3101, 3101)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e8pO61R-ZzKr"
      },
      "source": [
        "#### Transfomer model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l2pRkYbUZ3fT"
      },
      "source": [
        "\r\n",
        "import torch\r\n",
        "import torch.nn as nn\r\n",
        "import torch.optim as optim\r\n",
        "\r\n",
        "import torchtext\r\n",
        "from torchtext.legacy import data\r\n",
        "from torchtext.legacy.datasets import Multi30k\r\n",
        "from torchtext.legacy.data import Field, BucketIterator\r\n",
        "\r\n",
        "import matplotlib.pyplot as plt\r\n",
        "import matplotlib.ticker as ticker\r\n",
        "\r\n",
        "import spacy\r\n",
        "import numpy as np\r\n",
        "\r\n",
        "import random\r\n",
        "import math\r\n",
        "import time\r\n",
        "import pickle"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kL5c1WIIZ8HU"
      },
      "source": [
        "SEED = 1007\r\n",
        "\r\n",
        "random.seed(SEED)\r\n",
        "np.random.seed(SEED)\r\n",
        "torch.manual_seed(SEED)\r\n",
        "torch.cuda.manual_seed(SEED)\r\n",
        "torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NA9IKiSMaAMe",
        "outputId": "3df5a517-487e-4dda-cd74-398f05d4cc8e"
      },
      "source": [
        "!python -m spacy download en"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: en_core_web_sm==2.2.5 from https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.5/en_core_web_sm-2.2.5.tar.gz#egg=en_core_web_sm==2.2.5 in /usr/local/lib/python3.7/dist-packages (2.2.5)\n",
            "Requirement already satisfied: spacy>=2.2.2 in /usr/local/lib/python3.7/dist-packages (from en_core_web_sm==2.2.5) (2.2.4)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (3.0.5)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (7.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (4.41.1)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (0.4.1)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (0.8.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (2.23.0)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.5)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (54.0.0)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.1.3)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (2.0.5)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.19.5)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy>=2.2.2->en_core_web_sm==2.2.5) (1.0.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_sm==2.2.5) (2020.12.5)\n",
            "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.7.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_sm==2.2.5) (3.7.4.3)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('en_core_web_sm')\n",
            "\u001b[38;5;2m✔ Linking successful\u001b[0m\n",
            "/usr/local/lib/python3.7/dist-packages/en_core_web_sm -->\n",
            "/usr/local/lib/python3.7/dist-packages/spacy/data/en\n",
            "You can now load the model via spacy.load('en')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oM1mLuqKaIz0"
      },
      "source": [
        "spacy_en = spacy.load('en')"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S_ZAuM-9aNIX"
      },
      "source": [
        "def tokenize_en(text):\r\n",
        "    \"\"\"\r\n",
        "    Tokenizes English text from a string into a list of strings\r\n",
        "    \"\"\"\r\n",
        "    return [tok.text for tok in spacy_en.tokenizer(text)]"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1a6YAiCPaSAj"
      },
      "source": [
        "SRC = Field(tokenize = tokenize_en, \r\n",
        "            init_token = '<sos>', \r\n",
        "            eos_token = '<eos>', \r\n",
        "            lower = True, \r\n",
        "            batch_first = True)\r\n",
        "\r\n",
        "TRG = Field(tokenize = getTokenizer, \r\n",
        "            init_token = '<sos>', \r\n",
        "            eos_token = '<eos>', \r\n",
        "            lower = False, \r\n",
        "            batch_first = True)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "guwyZixya9RW",
        "outputId": "df931bb0-eba6-49ac-8523-2f1f8e6a21a7"
      },
      "source": [
        "fields = [('src', SRC), ('trg', TRG)]\r\n",
        "\r\n",
        "Examples = [data.Example.fromlist([i,j], fields) for i,j in zip(questions,answers)]\r\n",
        "Dataset = data.Dataset(Examples, fields)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Error in tokenization\n",
            "Error in tokenization\n",
            "Error in tokenization\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R3pBSBsVbgsh"
      },
      "source": [
        "train_data,valid_data = Dataset.split(split_ratio=[0.85,0.15])"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g06uLc1TbmSj"
      },
      "source": [
        "SRC.build_vocab(train_data, min_freq = 2)\r\n",
        "TRG.build_vocab(train_data, min_freq = 2)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cHT0iPaSfbRU",
        "outputId": "21c61a36-5b6a-4f6b-8b6d-9698308ecb96"
      },
      "source": [
        "len(SRC.vocab)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1151"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "17rVyfToe24k"
      },
      "source": [
        "# Dumps dicts\r\n",
        "with open(\"/content/SRC_TRG_stio\",\"wb\") as f:\r\n",
        "  pickle.dump(SRC.vocab.stoi,f)\r\n",
        "with open(\"/content/SRC_TRG_itos\",\"wb\") as f:\r\n",
        "  pickle.dump(TRG.vocab.itos,f)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "49ooxB_gbsne",
        "outputId": "d3ac10b4-45a8-4359-b265-7e8f041f010b"
      },
      "source": [
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\r\n",
        "device"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gRMN8gGIbt7N"
      },
      "source": [
        "BATCH_SIZE = 32\r\n",
        "\r\n",
        "train_iterator, valid_iterator = BucketIterator.splits(\r\n",
        "    (train_data, valid_data), \r\n",
        "     batch_size = BATCH_SIZE,\r\n",
        "     sort_key = lambda x: len(x.trg),\r\n",
        "     device = device)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ByLoYmXcBD4"
      },
      "source": [
        "class PositionalEncodingComponent(nn.Module):\r\n",
        "  '''\r\n",
        "  Class to encode positional information to tokens.\r\n",
        "  \r\n",
        "\r\n",
        "  '''\r\n",
        "  def __init__(self,hid_dim,device,dropout=0.2,max_len=5000):\r\n",
        "    super().__init__()\r\n",
        "\r\n",
        "    assert hid_dim%2==0 # If not, it will result error in allocation to positional_encodings[:,1::2] later\r\n",
        "\r\n",
        "    self.dropout = nn.Dropout(dropout)\r\n",
        "\r\n",
        "    self.positional_encodings = torch.zeros(max_len,hid_dim)\r\n",
        "\r\n",
        "    pos = torch.arange(0,max_len).unsqueeze(1) # pos : [max_len,1]\r\n",
        "    div_term  = torch.exp(-torch.arange(0,hid_dim,2)*math.log(10000.0)/hid_dim) # Calculating value of 1/(10000^(2i/hid_dim)) in log space and then exponentiating it\r\n",
        "    # div_term: [hid_dim//2]\r\n",
        "\r\n",
        "    self.positional_encodings[:,0::2] = torch.sin(pos*div_term) # pos*div_term [max_len,hid_dim//2]\r\n",
        "    self.positional_encodings[:,1::2] = torch.cos(pos*div_term) \r\n",
        "\r\n",
        "    self.positional_encodings = self.positional_encodings.unsqueeze(0) # To account for batch_size in inputs\r\n",
        "\r\n",
        "    self.device = device\r\n",
        "\r\n",
        "  def forward(self,x):\r\n",
        "    x = x + self.positional_encodings[:,:x.size(1)].detach().to(self.device)\r\n",
        "    return self.dropout(x)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XOQC1vbJcFEo"
      },
      "source": [
        "class FeedForwardComponent(nn.Module):\r\n",
        "  '''\r\n",
        "  Class for pointwise feed forward connections\r\n",
        "  '''\r\n",
        "  def __init__(self,hid_dim,pf_dim,dropout):\r\n",
        "    super().__init__()\r\n",
        "\r\n",
        "    self.dropout = nn.Dropout(dropout)\r\n",
        "\r\n",
        "    self.fc1 = nn.Linear(hid_dim,pf_dim)\r\n",
        "    self.fc2 = nn.Linear(pf_dim,hid_dim)\r\n",
        "\r\n",
        "  def forward(self,x):\r\n",
        "\r\n",
        "    # x : [batch_size,seq_len,hid_dim]\r\n",
        "    x = self.dropout(torch.relu(self.fc1(x)))\r\n",
        "\r\n",
        "    # x : [batch_size,seq_len,pf_dim]\r\n",
        "    x = self.fc2(x)\r\n",
        "\r\n",
        "    # x : [batch_size,seq_len,hid_dim]\r\n",
        "    return x"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lffcJcl-cYLq"
      },
      "source": [
        "class MultiHeadedAttentionComponent(nn.Module):\r\n",
        "  '''\r\n",
        "  Multiheaded attention Component. This implementation also supports mask. \r\n",
        "  The reason for mask that in Decoder, we don't want attention mechanism to get\r\n",
        "  important information from future tokens.\r\n",
        "  '''\r\n",
        "  def __init__(self,hid_dim, n_heads, dropout, device):\r\n",
        "    super().__init__()\r\n",
        "\r\n",
        "    assert hid_dim % n_heads == 0 # Since we split hid_dims into n_heads\r\n",
        "\r\n",
        "    self.hid_dim = hid_dim\r\n",
        "    self.n_heads = n_heads # no of heads in 'multiheaded' attention\r\n",
        "    self.head_dim = hid_dim//n_heads # dims of each head\r\n",
        "\r\n",
        "    # Transformation from source vector to query vector\r\n",
        "    self.fc_q = nn.Linear(hid_dim,hid_dim)\r\n",
        "\r\n",
        "    # Transformation from source vector to key vector\r\n",
        "    self.fc_k = nn.Linear(hid_dim,hid_dim)\r\n",
        "\r\n",
        "    # Transformation from source vector to value vector\r\n",
        "    self.fc_v = nn.Linear(hid_dim,hid_dim)\r\n",
        "\r\n",
        "    self.fc_o = nn.Linear(hid_dim,hid_dim)\r\n",
        "\r\n",
        "    self.dropout = nn.Dropout(dropout)\r\n",
        "\r\n",
        "    # Used in self attention for smoother gradients\r\n",
        "    self.scale = torch.sqrt(torch.FloatTensor([self.head_dim])).to(device)\r\n",
        "\r\n",
        "  def forward(self,query,key,value,mask=None):\r\n",
        "\r\n",
        "    #query : [batch_size, query_len, hid_dim]\r\n",
        "    #key : [batch_size, key_len, hid_dim]\r\n",
        "    #value : [batch_size, value_len, hid_dim]\r\n",
        "\r\n",
        "    batch_size = query.shape[0]\r\n",
        "\r\n",
        "    # Transforming quey,key,values\r\n",
        "    Q = self.fc_q(query)\r\n",
        "    K = self.fc_k(key)\r\n",
        "    V = self.fc_v(value)\r\n",
        "\r\n",
        "    #Q : [batch_size, query_len, hid_dim]\r\n",
        "    #K : [batch_size, key_len, hid_dim]\r\n",
        "    #V : [batch_size, value_len,hid_dim]\r\n",
        "\r\n",
        "    # Changing shapes to acocmadate n_heads information\r\n",
        "    Q = Q.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\r\n",
        "    K = K.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\r\n",
        "    V = V.view(batch_size, -1, self.n_heads, self.head_dim).permute(0, 2, 1, 3)\r\n",
        "\r\n",
        "    #Q : [batch_size, n_heads, query_len, head_dim]\r\n",
        "    #K : [batch_size, n_heads, key_len, head_dim]\r\n",
        "    #V : [batch_size, n_heads, value_len, head_dim]\r\n",
        "\r\n",
        "    # Calculating alpha\r\n",
        "    score = torch.matmul(Q,K.permute(0,1,3,2))/self.scale\r\n",
        "    # score : [batch_size, n_heads, query_len, key_len]\r\n",
        "\r\n",
        "    if mask is not None:\r\n",
        "      score = score.masked_fill(mask==0,-1e10)\r\n",
        "\r\n",
        "    alpha = torch.softmax(score,dim=-1)\r\n",
        "    # alpha : [batch_size, n_heads, query_len, key_len]\r\n",
        "\r\n",
        "    # Get the final self-attention  vector\r\n",
        "    x = torch.matmul(self.dropout(alpha),V)\r\n",
        "    # x : [batch_size, n_heads, query_len, head_dim]\r\n",
        "\r\n",
        "    # Reshaping self attention vector to concatenate\r\n",
        "    x = x.permute(0,2,1,3).contiguous()\r\n",
        "    # x : [batch_size, query_len, n_heads, head_dim]\r\n",
        "\r\n",
        "    x = x.view(batch_size,-1,self.hid_dim)\r\n",
        "    # x: [batch_size, query_len, hid_dim]\r\n",
        "\r\n",
        "    # Transforming concatenated outputs \r\n",
        "    x = self.fc_o(x)\r\n",
        "    #x : [batch_size, query_len, hid_dim] \r\n",
        "\r\n",
        "    return x, alpha"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hDPR180Oci6d"
      },
      "source": [
        "class EncoderLayer(nn.Module):  \r\n",
        "  '''\r\n",
        "  Operations of a single layer in an Encoder. An Encoder employs multiple such layers. Each layer contains:\r\n",
        "  1) multihead attention, folllowed by\r\n",
        "  2) LayerNorm of addition of multihead attention output and input to the layer, followed by\r\n",
        "  3) FeedForward connections, followed by\r\n",
        "  4) LayerNorm of addition of FeedForward outputs and output of previous layerNorm.\r\n",
        "  '''\r\n",
        "  def __init__(self, hid_dim,n_heads,pf_dim,dropout,device):\r\n",
        "    super().__init__()\r\n",
        "    \r\n",
        "    self.self_attn_layer_norm = nn. LayerNorm(hid_dim) #Layer norm after self-attention\r\n",
        "    self.ff_layer_norm = nn.LayerNorm(hid_dim) # Layer norm after FeedForward component\r\n",
        "\r\n",
        "    self.self_attention = MultiHeadedAttentionComponent(hid_dim,n_heads,dropout,device)\r\n",
        "    self.feed_forward = FeedForwardComponent(hid_dim,pf_dim,dropout)\r\n",
        "\r\n",
        "    self.dropout = nn.Dropout(dropout)\r\n",
        "    \r\n",
        "  def forward(self,src,src_mask):\r\n",
        "    \r\n",
        "    # src : [batch_size, src_len, hid_dim]\r\n",
        "    # src_mask : [batch_size, 1, 1, src_len]\r\n",
        "\r\n",
        "    # get self-attention\r\n",
        "    _src, _ = self.self_attention(src,src,src,src_mask)\r\n",
        "\r\n",
        "    # LayerNorm after dropout\r\n",
        "    src = self.self_attn_layer_norm(src + self.dropout(_src))\r\n",
        "    # src : [batch_size, src_len, hid_dim]\r\n",
        "\r\n",
        "    # FeedForward\r\n",
        "    _src = self.feed_forward(src)\r\n",
        "\r\n",
        "    # layerNorm after dropout\r\n",
        "    src = self.ff_layer_norm(src + self.dropout(_src))\r\n",
        "    # src: [batch_size, src_len, hid_dim]\r\n",
        "\r\n",
        "    return src"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Guap3xIzcvcW"
      },
      "source": [
        "class DecoderLayer(nn.Module):\r\n",
        "  '''\r\n",
        "  Operations of a single layer in an Decoder. An Decoder employs multiple such layers. Each layer contains:\r\n",
        "  1) masked decoder self attention, followed by\r\n",
        "  2) LayerNorm of addition of previous attention output and input to the layer,, followed by\r\n",
        "  3) encoder self attention, followed by\r\n",
        "  4) LayerNorm of addition of result of encoder self attention and its input, followed by\r\n",
        "  5) FeedForward connections, followed by\r\n",
        "  6) LayerNorm of addition of Feedforward results and its input.\r\n",
        "  '''\r\n",
        "  def __init__(self,hid_dim,n_heads,pf_dim,dropout,device):\r\n",
        "    super().__init__()\r\n",
        "\r\n",
        "    self.self_attn_layer_norm = nn.LayerNorm(hid_dim)\r\n",
        "    self.enc_attn_layer_norm = nn.LayerNorm(hid_dim)\r\n",
        "    self.ff_layer_norm = nn.LayerNorm(hid_dim)\r\n",
        "\r\n",
        "    # decoder self attention\r\n",
        "    self.self_attention = MultiHeadedAttentionComponent(hid_dim,n_heads,dropout,device)\r\n",
        "\r\n",
        "    # encoder attention\r\n",
        "    self.encoder_attention = MultiHeadedAttentionComponent(hid_dim,n_heads,dropout,device)\r\n",
        "\r\n",
        "    # FeedForward\r\n",
        "    self.feed_forward = FeedForwardComponent(hid_dim,pf_dim,dropout)\r\n",
        "\r\n",
        "    self.dropout = nn.Dropout(dropout)\r\n",
        "\r\n",
        "  def forward(self,trg, enc_src,trg_mask,src_mask):\r\n",
        "\r\n",
        "    #trg : [batch_size, trg_len, hid_dim]\r\n",
        "    #enc_src : [batch_size, src_len, hid_dim]\r\n",
        "    #trg_mask : [batch_size, 1, trg_len, trg_len]\r\n",
        "    #src_mask : [batch_size, 1, 1, src_len]\r\n",
        "\r\n",
        "    '''\r\n",
        "    Decoder self-attention\r\n",
        "    trg_mask is to force decoder to look only into past tokens and not get information from future tokens.\r\n",
        "    Since we apply mask before doing softmax, the final self attention vector gets no information from future tokens.\r\n",
        "    '''\r\n",
        "    _trg, _ = self.self_attention(trg,trg,trg,trg_mask)\r\n",
        "\r\n",
        "    # LayerNorm and dropout with resdiual connection\r\n",
        "    trg = self.self_attn_layer_norm(trg + self.dropout(_trg))\r\n",
        "    # trg : [batch_size, trg_len, hid_dim]\r\n",
        "\r\n",
        "    '''\r\n",
        "    Encoder attention:\r\n",
        "    Query: trg\r\n",
        "    key: enc_src\r\n",
        "    Value : enc_src\r\n",
        "    Why? \r\n",
        "    the idea here is to extract information from encoder outputs. So we use decoder self-attention as a query to find important values from enc_src\r\n",
        "    and that is why we use src_mask, to avoid getting information from enc_src positions where it is equal to pad-id\r\n",
        "    After we get necessary infromation from encoder outputs we add them back to decoder self-attention.\r\n",
        "    '''\r\n",
        "    _trg, encoder_attn_alpha = self.encoder_attention(trg,enc_src,enc_src,src_mask)\r\n",
        "\r\n",
        "        # LayerNorm , residual connection and dropout\r\n",
        "    trg = self.enc_attn_layer_norm(trg + self.dropout(_trg))\r\n",
        "    # trg : [ batch_size, trg_len, hid_dim]\r\n",
        "\r\n",
        "    # Feed Forward\r\n",
        "    _trg = self.feed_forward(trg)\r\n",
        "\r\n",
        "    # LayerNorm, residual connection and dropout\r\n",
        "    trg = self.ff_layer_norm(trg + self.dropout(_trg))\r\n",
        "\r\n",
        "    return trg, encoder_attn_alpha"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Vz_6uSaFc7rn"
      },
      "source": [
        "class Encoder(nn.Module):\r\n",
        "  '''\r\n",
        "  An encoder, creates token embeddings and position embeddings and passes them through multiple encoder layers\r\n",
        "  '''\r\n",
        "  def __init__(self,input_dim,hid_dim,n_layers,n_heads,pf_dim,dropout,device,max_length = 5000):\r\n",
        "    super().__init__()\r\n",
        "    self.device = device\r\n",
        "\r\n",
        "    self.tok_embedding = nn.Embedding(input_dim,hid_dim)\r\n",
        "    self.pos_embedding = PositionalEncodingComponent(hid_dim,device,dropout,max_length)\r\n",
        "\r\n",
        "    # encoder layers\r\n",
        "    self.layers = nn.ModuleList([EncoderLayer(hid_dim,n_heads,pf_dim,dropout,device) for _ in range(n_layers)])\r\n",
        "\r\n",
        "    self.dropout = nn.Dropout(dropout)\r\n",
        "\r\n",
        "    self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)\r\n",
        "\r\n",
        "  def forward(self,src,src_mask):\r\n",
        "\r\n",
        "    # src : [batch_size, src_len]\r\n",
        "    # src_mask : [batch_size,1,1,src_len]\r\n",
        "\r\n",
        "    batch_size = src.shape[0]\r\n",
        "    src_len = src.shape[1]\r\n",
        "\r\n",
        "    tok_embeddings = self.tok_embedding(src)*self.scale\r\n",
        "\r\n",
        "    # token plus position embeddings\r\n",
        "    src  = self.pos_embedding(tok_embeddings)\r\n",
        "\r\n",
        "    for layer in self.layers:\r\n",
        "      src = layer(src,src_mask)\r\n",
        "    # src : [batch_size, src_len, hid_dim]\r\n",
        "\r\n",
        "    return src"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fVnPu_cedHuM"
      },
      "source": [
        "class Decoder(nn.Module):\r\n",
        "  '''\r\n",
        "  An decoder, creates token embeddings and position embeddings and passes them through multiple decoder layers\r\n",
        "  '''\r\n",
        "  def __init__(self,output_dim,hid_dim,n_layers,n_heads,pf_dim,dropout,device,max_length= 5000):\r\n",
        "    super().__init__()\r\n",
        "\r\n",
        "    self.device = device\r\n",
        "\r\n",
        "    self.tok_embedding = nn.Embedding(output_dim,hid_dim)\r\n",
        "    self.pos_embedding = PositionalEncodingComponent(hid_dim,device,dropout,max_length)\r\n",
        "\r\n",
        "    # decoder layers\r\n",
        "    self.layers = nn.ModuleList([DecoderLayer(hid_dim,n_heads,pf_dim,dropout,device) for _ in range(n_layers)])\r\n",
        "\r\n",
        "    # convert decoder outputs to real outputs\r\n",
        "    self.fc_out = nn.Linear(hid_dim,output_dim)\r\n",
        "\r\n",
        "    self.dropout = nn.Dropout(dropout)\r\n",
        "\r\n",
        "    self.scale = torch.sqrt(torch.FloatTensor([hid_dim])).to(device)\r\n",
        "\r\n",
        "  def forward(self, trg, enc_src,trg_mask,src_mask):\r\n",
        "    \r\n",
        "    #trg : [batch_size, trg_len]\r\n",
        "    #enc_src : [batch_size, src_len, hid_dim]\r\n",
        "    #trg_mask : [batch_size, 1, trg_len, trg_len]\r\n",
        "    #src_mask : [batch_size, 1, 1, src_len]\r\n",
        "\r\n",
        "    batch_size = trg.shape[0]\r\n",
        "    trg_len = trg.shape[1]\r\n",
        "\r\n",
        "    tok_embeddings = self.tok_embedding(trg)*self.scale\r\n",
        "\r\n",
        "    # token plus pos embeddings\r\n",
        "    trg = self.pos_embedding(tok_embeddings)\r\n",
        "    # trg : [batch_size, trg_len, hid_dim]\r\n",
        "\r\n",
        "    # Pass trg thorugh decoder layers\r\n",
        "    for layer in self.layers:\r\n",
        "      trg, encoder_attention = layer(trg,enc_src,trg_mask,src_mask)\r\n",
        "    \r\n",
        "    # trg : [batch_size,trg_len,hid_dim]\r\n",
        "    # encoder_attention :  [batch_size, n_head,trg_len, src_len]\r\n",
        "\r\n",
        "    # Convert to outputs\r\n",
        "    output = self.fc_out(trg)\r\n",
        "    # output : [batch_size, trg_len, output_dim]\r\n",
        "    \r\n",
        "    return output, encoder_attention"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lS8s1DWqdNbK"
      },
      "source": [
        "\r\n",
        "class Seq2Seq(nn.Module):\r\n",
        "  def __init__(self, encoder, decoder, src_pad_idx, trg_pad_idx, device):\r\n",
        "    super().__init__()\r\n",
        "    self.encoder = encoder\r\n",
        "    self.decoder = decoder\r\n",
        "    self.src_pad_idx = src_pad_idx\r\n",
        "    self.trg_pad_idx = trg_pad_idx\r\n",
        "    self.device = device\r\n",
        "\r\n",
        "  def make_src_mask(self,src):\r\n",
        "    # src : [batch_size, src_len]\r\n",
        "\r\n",
        "    # Masking pad values\r\n",
        "    src_mask = (src != self.src_pad_idx).unsqueeze(1).unsqueeze(2)\r\n",
        "    # src_mask : [batch_size,1,1,src_len]\r\n",
        "\r\n",
        "    return src_mask\r\n",
        "\r\n",
        "  def make_trg_mask(self,trg):\r\n",
        "    # trg : [batch_size, trg_len]\r\n",
        "\r\n",
        "    # Masking pad values\r\n",
        "    trg_pad_mask = (trg != self.trg_pad_idx).unsqueeze(1).unsqueeze(2)\r\n",
        "    # trg_pad_mask : [batch_size,1,1, trg_len]\r\n",
        "\r\n",
        "    # Masking future values\r\n",
        "    trg_len = trg.shape[1]\r\n",
        "    trg_sub_mask = torch.tril(torch.ones((trg_len,trg_len),device= self.device)).bool()\r\n",
        "    # trg_sub_mask : [trg_len, trg_len]\r\n",
        "\r\n",
        "    # combine both masks\r\n",
        "    trg_mask = trg_pad_mask & trg_sub_mask\r\n",
        "    # trg_mask = [batch_size,1,trg_len,trg_len]\r\n",
        "\r\n",
        "    return trg_mask\r\n",
        "\r\n",
        "  def forward(self,src,trg):\r\n",
        "\r\n",
        "    # src : [batch_size, src_len]\r\n",
        "    # trg : [batch_size, trg_len]\r\n",
        "\r\n",
        "    src_mask = self.make_src_mask(src)\r\n",
        "    trg_mask = self.make_trg_mask(trg)\r\n",
        "\r\n",
        "    # src_mask : [ batch_size, 1,1,src_len]\r\n",
        "    # trg_mask : [batch_size, 1, trg_len, trg_len]\r\n",
        "\r\n",
        "    enc_src = self.encoder(src,src_mask)\r\n",
        "    #enc_src : [batch_size, src_len, hid_dim]\r\n",
        "\r\n",
        "    output, encoder_decoder_attention = self.decoder(trg,enc_src,trg_mask,src_mask)\r\n",
        "    # output : [batch_size, trg_len, output_dim]\r\n",
        "    # encoder_decoder_attention : [batch_size, n_heads, trg_len, src_len]\r\n",
        "\r\n",
        "    return output, encoder_decoder_attention"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VotguMUgdaO-"
      },
      "source": [
        "INPUT_DIM = len(SRC.vocab)\r\n",
        "OUTPUT_DIM = len(TRG.vocab)\r\n",
        "HID_DIM = 256\r\n",
        "ENC_LAYERS = 3\r\n",
        "DEC_LAYERS = 3\r\n",
        "ENC_HEADS = 8\r\n",
        "DEC_HEADS = 8\r\n",
        "ENC_PF_DIM = 512\r\n",
        "DEC_PF_DIM = 512\r\n",
        "ENC_DROPOUT = 0.1\r\n",
        "DEC_DROPOUT = 0.1\r\n",
        "\r\n",
        "enc = Encoder(INPUT_DIM, \r\n",
        "              HID_DIM, \r\n",
        "              ENC_LAYERS, \r\n",
        "              ENC_HEADS, \r\n",
        "              ENC_PF_DIM, \r\n",
        "              ENC_DROPOUT, \r\n",
        "              device)\r\n",
        "\r\n",
        "dec = Decoder(OUTPUT_DIM, \r\n",
        "              HID_DIM, \r\n",
        "              DEC_LAYERS, \r\n",
        "              DEC_HEADS, \r\n",
        "              DEC_PF_DIM, \r\n",
        "              DEC_DROPOUT, \r\n",
        "              device)\r\n",
        "\r\n",
        "SRC_PAD_IDX = SRC.vocab.stoi[SRC.pad_token]\r\n",
        "TRG_PAD_IDX = TRG.vocab.stoi[TRG.pad_token]\r\n",
        "\r\n",
        "model = Seq2Seq(enc, dec, SRC_PAD_IDX, TRG_PAD_IDX, device).to(device)"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dx2KhnfvduVq"
      },
      "source": [
        "def initialize_weights(m):\r\n",
        "    if hasattr(m, 'weight') and m.weight.dim() > 1:\r\n",
        "        nn.init.xavier_uniform_(m.weight.data)\r\n",
        "\r\n",
        "model.apply(initialize_weights);"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZjkCCx7Fd1aK",
        "outputId": "6b8362a1-55c4-4c9f-df7d-68db1b35b3df"
      },
      "source": [
        "def count_parameters(model):\r\n",
        "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\r\n",
        "\r\n",
        "print(f'The model has {count_parameters(model):,} trainable parameters')"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The model has 5,743,202 trainable parameters\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8KyVSx1Pd6yy"
      },
      "source": [
        "LEARNING_RATE = 0.0005\r\n",
        "\r\n",
        "optimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)\r\n",
        "\r\n",
        "criterion = nn.CrossEntropyLoss(ignore_index = TRG_PAD_IDX)"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Bh7Byisd_t5"
      },
      "source": [
        "def train(model, iterator, optimizer, criterion, clip):\r\n",
        "    \r\n",
        "    model.train()\r\n",
        "    \r\n",
        "    epoch_loss = 0\r\n",
        "    \r\n",
        "    for i, batch in enumerate(iterator):\r\n",
        "        \r\n",
        "        src = batch.src\r\n",
        "        trg = batch.trg\r\n",
        "        \r\n",
        "        optimizer.zero_grad()\r\n",
        "        \r\n",
        "        output, _ = model(src, trg[:,:-1])\r\n",
        "                \r\n",
        "        #output = [batch size, trg len - 1, output dim]\r\n",
        "        #trg = [batch size, trg len]\r\n",
        "            \r\n",
        "        output_dim = output.shape[-1]\r\n",
        "            \r\n",
        "        output = output.contiguous().view(-1, output_dim)\r\n",
        "        trg = trg[:,1:].contiguous().view(-1)\r\n",
        "                \r\n",
        "        #output = [batch size * trg len - 1, output dim]\r\n",
        "        #trg = [batch size * trg len - 1]\r\n",
        "            \r\n",
        "        loss = criterion(output, trg)\r\n",
        "        \r\n",
        "        loss.backward()\r\n",
        "\r\n",
        "        torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\r\n",
        "        \r\n",
        "        optimizer.step()\r\n",
        "        \r\n",
        "        epoch_loss += loss.item()\r\n",
        "        \r\n",
        "    return epoch_loss / len(iterator)"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yO07uDlleG__"
      },
      "source": [
        "def evaluate(model, iterator, criterion):\r\n",
        "    \r\n",
        "    model.eval()\r\n",
        "    \r\n",
        "    epoch_loss = 0\r\n",
        "    \r\n",
        "    with torch.no_grad():\r\n",
        "    \r\n",
        "        for i, batch in enumerate(iterator):\r\n",
        "\r\n",
        "            src = batch.src\r\n",
        "            trg = batch.trg\r\n",
        "\r\n",
        "            output, _ = model(src, trg[:,:-1])\r\n",
        "            \r\n",
        "            #output = [batch size, trg len - 1, output dim]\r\n",
        "            #trg = [batch size, trg len]\r\n",
        "            \r\n",
        "            output_dim = output.shape[-1]\r\n",
        "            \r\n",
        "            output = output.contiguous().view(-1, output_dim)\r\n",
        "            trg = trg[:,1:].contiguous().view(-1)\r\n",
        "            \r\n",
        "            #output = [batch size * trg len - 1, output dim]\r\n",
        "            #trg = [batch size * trg len - 1]\r\n",
        "            \r\n",
        "            loss = criterion(output, trg)\r\n",
        "\r\n",
        "            epoch_loss += loss.item()\r\n",
        "        \r\n",
        "    return epoch_loss / len(iterator)"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WvRw5x6zeKan"
      },
      "source": [
        "def epoch_time(start_time, end_time):\r\n",
        "    elapsed_time = end_time - start_time\r\n",
        "    elapsed_mins = int(elapsed_time / 60)\r\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\r\n",
        "    return elapsed_mins, elapsed_secs"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pOTVSixJePPf",
        "outputId": "c24d5e69-4dcf-4a7c-cbde-52e7ea511fe2"
      },
      "source": [
        "N_EPOCHS = 20\r\n",
        "CLIP = 1\r\n",
        "\r\n",
        "best_valid_loss = float('inf')\r\n",
        "\r\n",
        "for epoch in range(N_EPOCHS):\r\n",
        "    \r\n",
        "    start_time = time.time()\r\n",
        "    \r\n",
        "    train_loss = train(model, train_iterator, optimizer, criterion, CLIP)\r\n",
        "    valid_loss = evaluate(model, valid_iterator, criterion)\r\n",
        "    \r\n",
        "    end_time = time.time()\r\n",
        "    \r\n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\r\n",
        "    \r\n",
        "    if valid_loss < best_valid_loss:\r\n",
        "        best_valid_loss = valid_loss\r\n",
        "        torch.save(model.state_dict(), 'tut6-model.pt')\r\n",
        "    \r\n",
        "    print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\r\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\r\n",
        "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 01 | Time: 0m 18s\n",
            "\tTrain Loss: 4.739 | Train PPL: 114.281\n",
            "\t Val. Loss: 3.172 |  Val. PPL:  23.867\n",
            "Epoch: 02 | Time: 0m 17s\n",
            "\tTrain Loss: 3.041 | Train PPL:  20.923\n",
            "\t Val. Loss: 2.508 |  Val. PPL:  12.279\n",
            "Epoch: 03 | Time: 0m 17s\n",
            "\tTrain Loss: 2.535 | Train PPL:  12.615\n",
            "\t Val. Loss: 2.183 |  Val. PPL:   8.869\n",
            "Epoch: 04 | Time: 0m 17s\n",
            "\tTrain Loss: 2.242 | Train PPL:   9.412\n",
            "\t Val. Loss: 1.991 |  Val. PPL:   7.321\n",
            "Epoch: 05 | Time: 0m 17s\n",
            "\tTrain Loss: 2.038 | Train PPL:   7.676\n",
            "\t Val. Loss: 1.886 |  Val. PPL:   6.591\n",
            "Epoch: 06 | Time: 0m 17s\n",
            "\tTrain Loss: 1.868 | Train PPL:   6.474\n",
            "\t Val. Loss: 1.795 |  Val. PPL:   6.017\n",
            "Epoch: 07 | Time: 0m 17s\n",
            "\tTrain Loss: 1.720 | Train PPL:   5.582\n",
            "\t Val. Loss: 1.723 |  Val. PPL:   5.601\n",
            "Epoch: 08 | Time: 0m 18s\n",
            "\tTrain Loss: 1.591 | Train PPL:   4.907\n",
            "\t Val. Loss: 1.643 |  Val. PPL:   5.170\n",
            "Epoch: 09 | Time: 0m 18s\n",
            "\tTrain Loss: 1.471 | Train PPL:   4.354\n",
            "\t Val. Loss: 1.592 |  Val. PPL:   4.915\n",
            "Epoch: 10 | Time: 0m 18s\n",
            "\tTrain Loss: 1.367 | Train PPL:   3.922\n",
            "\t Val. Loss: 1.553 |  Val. PPL:   4.725\n",
            "Epoch: 11 | Time: 0m 17s\n",
            "\tTrain Loss: 1.265 | Train PPL:   3.544\n",
            "\t Val. Loss: 1.512 |  Val. PPL:   4.535\n",
            "Epoch: 12 | Time: 0m 18s\n",
            "\tTrain Loss: 1.182 | Train PPL:   3.260\n",
            "\t Val. Loss: 1.513 |  Val. PPL:   4.541\n",
            "Epoch: 13 | Time: 0m 17s\n",
            "\tTrain Loss: 1.101 | Train PPL:   3.008\n",
            "\t Val. Loss: 1.503 |  Val. PPL:   4.495\n",
            "Epoch: 14 | Time: 0m 17s\n",
            "\tTrain Loss: 1.027 | Train PPL:   2.792\n",
            "\t Val. Loss: 1.477 |  Val. PPL:   4.381\n",
            "Epoch: 15 | Time: 0m 17s\n",
            "\tTrain Loss: 0.958 | Train PPL:   2.607\n",
            "\t Val. Loss: 1.478 |  Val. PPL:   4.384\n",
            "Epoch: 16 | Time: 0m 18s\n",
            "\tTrain Loss: 0.898 | Train PPL:   2.456\n",
            "\t Val. Loss: 1.465 |  Val. PPL:   4.328\n",
            "Epoch: 17 | Time: 0m 17s\n",
            "\tTrain Loss: 0.840 | Train PPL:   2.315\n",
            "\t Val. Loss: 1.456 |  Val. PPL:   4.289\n",
            "Epoch: 18 | Time: 0m 18s\n",
            "\tTrain Loss: 0.794 | Train PPL:   2.212\n",
            "\t Val. Loss: 1.471 |  Val. PPL:   4.353\n",
            "Epoch: 19 | Time: 0m 17s\n",
            "\tTrain Loss: 0.745 | Train PPL:   2.107\n",
            "\t Val. Loss: 1.481 |  Val. PPL:   4.398\n",
            "Epoch: 20 | Time: 0m 18s\n",
            "\tTrain Loss: 0.704 | Train PPL:   2.022\n",
            "\t Val. Loss: 1.497 |  Val. PPL:   4.467\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4pDCSWjyhKjn",
        "outputId": "f6782686-0970-4a56-a7e3-41e2a80b556a"
      },
      "source": [
        "\r\n",
        "import pickle\r\n",
        "\r\n",
        "with open(\"SRC_TRG_stio\",\"rb\") as f:\r\n",
        "  stoi = pickle.load(f)\r\n",
        "with open(\"SRC_TRG_itos\",\"rb\") as f:\r\n",
        "  itos = pickle.load(f)\r\n",
        "\r\n",
        "# Load model\r\n",
        "trained_model = 'tut6-model.pt'\r\n",
        "model.load_state_dict(torch.load(trained_model));\r\n",
        "model.eval()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Seq2Seq(\n",
              "  (encoder): Encoder(\n",
              "    (tok_embedding): Embedding(1151, 256)\n",
              "    (pos_embedding): PositionalEncodingComponent(\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (layers): ModuleList(\n",
              "      (0): EncoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadedAttentionComponent(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (feed_forward): FeedForwardComponent(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (fc1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (1): EncoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadedAttentionComponent(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (feed_forward): FeedForwardComponent(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (fc1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (2): EncoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadedAttentionComponent(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (feed_forward): FeedForwardComponent(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (fc1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (decoder): Decoder(\n",
              "    (tok_embedding): Embedding(2914, 256)\n",
              "    (pos_embedding): PositionalEncodingComponent(\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (layers): ModuleList(\n",
              "      (0): DecoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (enc_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadedAttentionComponent(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (encoder_attention): MultiHeadedAttentionComponent(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (feed_forward): FeedForwardComponent(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (fc1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (1): DecoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (enc_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadedAttentionComponent(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (encoder_attention): MultiHeadedAttentionComponent(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (feed_forward): FeedForwardComponent(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (fc1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "      (2): DecoderLayer(\n",
              "        (self_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (enc_attn_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (ff_layer_norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)\n",
              "        (self_attention): MultiHeadedAttentionComponent(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (encoder_attention): MultiHeadedAttentionComponent(\n",
              "          (fc_q): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_k): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_v): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (fc_o): Linear(in_features=256, out_features=256, bias=True)\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "        )\n",
              "        (feed_forward): FeedForwardComponent(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (fc1): Linear(in_features=256, out_features=512, bias=True)\n",
              "          (fc2): Linear(in_features=512, out_features=256, bias=True)\n",
              "        )\n",
              "        (dropout): Dropout(p=0.1, inplace=False)\n",
              "      )\n",
              "    )\n",
              "    (fc_out): Linear(in_features=256, out_features=2914, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o0X6MdGEhhU9"
      },
      "source": [
        "import spacy\r\n",
        "spacy_en = spacy.load('en')\r\n",
        "\r\n",
        "def encode_inputs(input,vocab):\r\n",
        "\r\n",
        "  tokenized_input = [tok.text.lower() for tok in spacy_en.tokenizer(input)]\r\n",
        "  tokenized_input = ['<sos>'] + tokenized_input +['<eos>']\r\n",
        "\r\n",
        "  numericalized_input = [vocab[i] for i in tokenized_input]\r\n",
        "\r\n",
        "  tensor_input = torch.LongTensor([numericalized_input])\r\n",
        "  \r\n",
        "  return tensor_input\r\n",
        "\r\n",
        "def decode_outputs(output,vocab):\r\n",
        "  # output: [1,1,hid_dim]\r\n",
        "  predicted_token = output.argmax(-1)\r\n",
        "  return vocab[predicted_token.item()], predicted_token"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Ng3cU0Yh0Jd",
        "outputId": "21d60dc2-277c-4589-c3ba-c1626eab092a"
      },
      "source": [
        "print(\" Enter q or quit to exit.\")\r\n",
        "\r\n",
        "answer_max_len = 200\r\n",
        "\r\n",
        "while(True):\r\n",
        "\r\n",
        "  input_ = input(\"Enter text:\")\r\n",
        "\r\n",
        "  if input_=='q' or input_=='quit':\r\n",
        "    break\r\n",
        "\r\n",
        "  src = encode_inputs(input_,stoi).to(device)\r\n",
        "  # src_mask = torch.ones([1,1,1,src.shape[-1]]).to(device)\r\n",
        "  src_mask = model.make_src_mask(src)\r\n",
        "\r\n",
        "  with torch.no_grad():\r\n",
        "    enc_src = model.encoder(src,src_mask)\r\n",
        "  \r\n",
        "  trg = '<sos>'\r\n",
        "  trg_indexes = [stoi[trg]]\r\n",
        "  # trg_mask = torch.ones([1,1,1,1]).to(device)\r\n",
        "\r\n",
        "  decoder_outputs = []\r\n",
        "  for i in range(answer_max_len):\r\n",
        "    trg_tensor = torch.LongTensor(trg_indexes).unsqueeze(0).to(device)\r\n",
        "    trg_mask = model.make_trg_mask(trg_tensor)\r\n",
        "    \r\n",
        "    with torch.no_grad():\r\n",
        "      decoder_output,_ = model.decoder(trg_tensor,enc_src,trg_mask,src_mask)\r\n",
        "\r\n",
        "    pred_token = decoder_output.argmax(2)[:,-1].item()\r\n",
        "\r\n",
        "    if pred_token == TRG.vocab.stoi[TRG.eos_token]:\r\n",
        "      break\r\n",
        "    decoder_outputs.append(itos[pred_token])\r\n",
        "    trg_indexes.append(pred_token)\r\n",
        "\r\n",
        "    \r\n",
        "  print(\"\".join(decoder_outputs))"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " Enter q or quit to exit.\n",
            "Enter text:write a program to add two numbers\n",
            "num1='00001'\n",
            "num2='10001'\n",
            "sum=num1+num2\n",
            "print(num1,sum)\n",
            "Enter text:Write a Python function to calculate the geometric sum of n-1\n",
            "def<unk>(n):\n",
            "\treturn(n*(n-1)+1\n",
            "Enter text:Write a Python program to compute the sum of all the multiples of 3 or 5 below 500\n",
            "n=[]\n",
            "foriinrange(0,n+1):\n",
            "\tifi%2==0:\n",
            "\t\ti=i\n",
            "foriinrange(0,n):\n",
            "\tprint(i)\n",
            "Enter text:write a program to substract two numbers\n",
            "num1=int(input(<unk>))\n",
            "num2=int(input(<unk>))\n",
            "print(num1,num2,num1,num2)\n",
            "Enter text:surprise me\n",
            "def<unk>(n):\n",
            "\treturn[<unk>.<unk>(n)foriinrange(n)]\n",
            "Enter text:q\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}