onnx_remove_shared_weights.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "onnx-remove-shared-weights.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7_cdVTFl4RES"
      },
      "source": [
        "Source - [Tianlei Wu](https://github.com/onnx/onnx/issues/3278#issuecomment-781948998)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sA4vvhV6HYdD"
      },
      "source": [
        "### Installing dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pgVNLYuJ4JKz",
        "outputId": "3d75f344-de0d-4e7c-f839-188f800edeab"
      },
      "source": [
        "!pip install --upgrade transformers sentencepiece\r\n",
        "!pip install --upgrade onnxruntime\r\n",
        "!pip install --upgrade onnxruntime-tools"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/98/87/ef312eef26f5cecd8b17ae9654cdd8d1fae1eb6dbd87257d6d73c128a4d0/transformers-4.3.2-py3-none-any.whl (1.8MB)\n",
            "\u001b[K     |████████████████████████████████| 1.8MB 5.3MB/s \n",
            "\u001b[?25hCollecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/67/e42bd1181472c95c8cda79305df848264f2a7f62740995a46945d9797b67/sentencepiece-0.1.95-cp36-cp36m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 17.5MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied, skipping upgrade: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied, skipping upgrade: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.5)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/fd/5b/44baae602e0a30bcc53fbdbc60bd940c15e143d252d658dfdefce736ece5/tokenizers-0.10.1-cp36-cp36m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 28.5MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from transformers) (3.4.0)\n",
            "Requirement already satisfied, skipping upgrade: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 51.2MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied, skipping upgrade: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.0)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied, skipping upgrade: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=ec3903950f4e11457099cd178d121e1a72b642d5d0b5092cfda74055c100127b\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers, sentencepiece\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.95 tokenizers-0.10.1 transformers-4.3.2\n",
            "Collecting onnxruntime\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/b3/a9/f009251fd1b91a2e1ce6f22d4b5be9936fbd0072842c5087a2a49706c509/onnxruntime-1.6.0-cp36-cp36m-manylinux2014_x86_64.whl (4.1MB)\n",
            "\u001b[K     |████████████████████████████████| 4.1MB 4.2MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: protobuf in /usr/local/lib/python3.6/dist-packages (from onnxruntime) (3.12.4)\n",
            "Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in /usr/local/lib/python3.6/dist-packages (from onnxruntime) (1.19.5)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.9 in /usr/local/lib/python3.6/dist-packages (from protobuf->onnxruntime) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->onnxruntime) (53.0.0)\n",
            "Installing collected packages: onnxruntime\n",
            "Successfully installed onnxruntime-1.6.0\n",
            "Collecting onnxruntime-tools\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/47/46/a8ff112aa4f4d6ba7fa0189ad941b0959a4488a31d9610e07ac808ae92db/onnxruntime_tools-1.6.0-py3-none-any.whl (186kB)\n",
            "\u001b[K     |████████████████████████████████| 194kB 4.1MB/s \n",
            "\u001b[?25hCollecting onnx\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f1/db/608877fea324c3a44aaa50dbcb23ff5b7e3d222a7c5511c19d1651db512e/onnx-1.8.1-cp36-cp36m-manylinux2010_x86_64.whl (14.5MB)\n",
            "\u001b[K     |████████████████████████████████| 14.5MB 319kB/s \n",
            "\u001b[?25hCollecting coloredlogs\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/84/a6/837dbf6eac344cb74f0ba86b79e8180855570af3e26bcb1ea5f650cf944c/coloredlogs-15.0-py2.py3-none-any.whl (45kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 5.6MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: packaging in /usr/local/lib/python3.6/dist-packages (from onnxruntime-tools) (20.9)\n",
            "Requirement already satisfied, skipping upgrade: psutil in /usr/local/lib/python3.6/dist-packages (from onnxruntime-tools) (5.4.8)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from onnxruntime-tools) (1.19.5)\n",
            "Collecting py3nvml\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/53/b3/cb30dd8cc1198ae3fdb5a320ca7986d7ca76e23d16415067eafebff8685f/py3nvml-0.2.6-py3-none-any.whl (55kB)\n",
            "\u001b[K     |████████████████████████████████| 61kB 6.8MB/s \n",
            "\u001b[?25hCollecting py-cpuinfo\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f6/f5/8e6e85ce2e9f6e05040cf0d4e26f43a4718bcc4bce988b433276d4b1a5c1/py-cpuinfo-7.0.0.tar.gz (95kB)\n",
            "\u001b[K     |████████████████████████████████| 102kB 9.7MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in /usr/local/lib/python3.6/dist-packages (from onnx->onnxruntime-tools) (3.7.4.3)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from onnx->onnxruntime-tools) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: protobuf in /usr/local/lib/python3.6/dist-packages (from onnx->onnxruntime-tools) (3.12.4)\n",
            "Collecting humanfriendly>=9.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/93/66/363d01a81da2108a5cf446daf619779f06d49a0c4426dd02b40734f10e2f/humanfriendly-9.1-py2.py3-none-any.whl (86kB)\n",
            "\u001b[K     |████████████████████████████████| 92kB 9.1MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->onnxruntime-tools) (2.4.7)\n",
            "Collecting xmltodict\n",
            "  Downloading https://files.pythonhosted.org/packages/28/fd/30d5c1d3ac29ce229f6bdc40bbc20b28f716e8b363140c26eff19122d8a5/xmltodict-0.12.0-py2.py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->onnx->onnxruntime-tools) (53.0.0)\n",
            "Building wheels for collected packages: py-cpuinfo\n",
            "  Building wheel for py-cpuinfo (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for py-cpuinfo: filename=py_cpuinfo-7.0.0-cp36-none-any.whl size=20072 sha256=de2224205b2b03048f1f12a319fce84e0cbeed2548bc98738d0297c53792aa9b\n",
            "  Stored in directory: /root/.cache/pip/wheels/f1/93/7b/127daf0c3a5a49feb2fecd468d508067c733fba5192f726ad1\n",
            "Successfully built py-cpuinfo\n",
            "Installing collected packages: onnx, humanfriendly, coloredlogs, xmltodict, py3nvml, py-cpuinfo, onnxruntime-tools\n",
            "Successfully installed coloredlogs-15.0 humanfriendly-9.1 onnx-1.8.1 onnxruntime-tools-1.6.0 py-cpuinfo-7.0.0 py3nvml-0.2.6 xmltodict-0.12.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IROWa6LEHc3d"
      },
      "source": [
        "### Using model from Transformers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BASpmQPY43g_",
        "outputId": "f3292a90-c193-42a1-a73d-9977d77d5517"
      },
      "source": [
        "from transformers import AutoTokenizer, AutoModelForQuestionAnswering\r\n",
        "import torch\r\n",
        "\r\n",
        "##your model name\r\n",
        "source_name = \"ktrapeznikov/albert-xlarge-v2-squad-v2\"\r\n",
        "\r\n",
        "model_name = source_name.split(\"/\")[-1]\r\n",
        "dir = \"transformers-model\"\r\n",
        "\r\n",
        "!rm -rf {dir}\r\n",
        "!mkdir {dir}\r\n",
        "\r\n",
        "tokenizer = AutoTokenizer.from_pretrained(source_name)\r\n",
        "model = AutoModelForQuestionAnswering.from_pretrained(source_name)\r\n",
        "model.eval()"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "AlbertForQuestionAnswering(\n",
              "  (albert): AlbertModel(\n",
              "    (embeddings): AlbertEmbeddings(\n",
              "      (word_embeddings): Embedding(30000, 128, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 128)\n",
              "      (token_type_embeddings): Embedding(2, 128)\n",
              "      (LayerNorm): LayerNorm((128,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): AlbertTransformer(\n",
              "      (embedding_hidden_mapping_in): Linear(in_features=128, out_features=2048, bias=True)\n",
              "      (albert_layer_groups): ModuleList(\n",
              "        (0): AlbertLayerGroup(\n",
              "          (albert_layers): ModuleList(\n",
              "            (0): AlbertLayer(\n",
              "              (full_layer_layer_norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)\n",
              "              (attention): AlbertAttention(\n",
              "                (query): Linear(in_features=2048, out_features=2048, bias=True)\n",
              "                (key): Linear(in_features=2048, out_features=2048, bias=True)\n",
              "                (value): Linear(in_features=2048, out_features=2048, bias=True)\n",
              "                (attention_dropout): Dropout(p=0.1, inplace=False)\n",
              "                (output_dropout): Dropout(p=0.1, inplace=False)\n",
              "                (dense): Linear(in_features=2048, out_features=2048, bias=True)\n",
              "                (LayerNorm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)\n",
              "              )\n",
              "              (ffn): Linear(in_features=2048, out_features=8192, bias=True)\n",
              "              (ffn_output): Linear(in_features=8192, out_features=2048, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (qa_outputs): Linear(in_features=2048, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kfaYS88bHpO7"
      },
      "source": [
        "#### Exporting model to ONNX"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qXpp63YB5Bco",
        "outputId": "cb05e3a0-89ba-4573-943d-4b1b95a88a71"
      },
      "source": [
        "question = \"what is google specialization\"\r\n",
        "text = \"Google LLC is an American multinational technology company that specializes in Internet-related services and products, which include online advertising technologies, a search engine, cloud computing, software, and hardware.\"\r\n",
        "encoding = tokenizer.encode_plus(question, text)\r\n",
        "input_ids, attention_mask, token_type_ids = encoding[\"input_ids\"],encoding[\"attention_mask\"], encoding[\"token_type_ids\"]\r\n",
        "\r\n",
        "input_ids = torch.tensor([input_ids])\r\n",
        "attention_mask = torch.tensor([attention_mask])\r\n",
        "token_type_ids = torch.tensor([token_type_ids])\r\n",
        "\r\n",
        "torch.onnx.export(\r\n",
        "    model,\r\n",
        "    (input_ids,attention_mask, token_type_ids),\r\n",
        "    f\"{dir}/{model_name}.onnx\",\r\n",
        "    input_names = ['input_ids','attention_mask', 'token_type_ids'], ## Be carefule to write this names\r\n",
        "    output_names = ['qa_outputs'], ## Be carefule to write this names\r\n",
        "    opset_version=12,\r\n",
        "    do_constant_folding=True,\r\n",
        "    use_external_data_format=True,\r\n",
        "    dynamic_axes = {\r\n",
        "        'input_ids' : {0: 'batch', 1: 'sequence'},\r\n",
        "        'attention_mask' : {0: 'batch', 1: 'sequence'}, \r\n",
        "        'token_type_ids' : {0: 'batch', 1: 'sequence'}, \r\n",
        "        'qa_outputs': {0: 'batch'}\r\n",
        "    }\r\n",
        ")"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/transformers/models/albert/modeling_albert.py:231: TracerWarning: Converting a tensor to a Python index might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
            "  position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]\n",
            "/usr/local/lib/python3.6/dist-packages/transformers/modeling_utils.py:1760: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
            "  input_tensor.shape[chunk_dim] == tensor_shape for input_tensor in input_tensors\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "au1-mHUA6Wh5"
      },
      "source": [
        "torch.save(model,\"model.pth\")"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y5e0XkdeHt1F"
      },
      "source": [
        "The model exported to ONNX has shared weights and because of that the size of the model increases in a huge amount"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M9C8bKC65cuE",
        "outputId": "66df7f45-a91c-4200-c284-b7b9c7f0b0c6"
      },
      "source": [
        "!du -sh model.pth\r\n",
        "!du -sh {dir}"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "209M\tmodel.pth\n",
            "4.6G\ttransformers-model\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bbo_9u1OH3iH"
      },
      "source": [
        "### Removing shared weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FXVim4_s5fBy"
      },
      "source": [
        "def has_same_value(val_one,val_two):\r\n",
        "  if val_one.raw_data == val_two.raw_data:\r\n",
        "    return True\r\n",
        "  else:\r\n",
        "    return False"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5odKb4yi5j3i"
      },
      "source": [
        "from onnxruntime.transformers.onnx_model import OnnxModel\r\n",
        "import onnx\r\n",
        "\r\n",
        "path = f\"{dir}/{model_name}.onnx\"\r\n",
        "model=onnx.load(path)\r\n",
        "onnx_model=OnnxModel(model)"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WXAVCaaX5x28"
      },
      "source": [
        "output_path = f\"{model_name}.onnx\"\r\n",
        "\r\n",
        "count = len(model.graph.initializer)\r\n",
        "same = [-1] * count\r\n",
        "for i in range(count - 1):\r\n",
        "  if same[i] >= 0:\r\n",
        "\t    continue\r\n",
        "  for j in range(i+1, count):\r\n",
        "      if has_same_value(model.graph.initializer[i], model.graph.initializer[j]):\r\n",
        "\t      same[j] = i\r\n",
        "\r\n",
        "for i in range(count):\r\n",
        "   if same[i] >= 0:\r\n",
        "        onnx_model.replace_input_of_all_nodes(model.graph.initializer[i].name, model.graph.initializer[same[i]].name)\r\n",
        "\r\n",
        "onnx_model.update_graph()\r\n",
        "\r\n",
        "onnx_model.save_model_to_file(output_path)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WNccRhtg6QCA",
        "outputId": "c267fbd6-758c-4341-cae4-6de80bf64366"
      },
      "source": [
        "!du -sh {output_path}"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "209M\talbert-xlarge-v2-squad-v2.onnx\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QorPJbAvH7s7"
      },
      "source": [
        "### Inference using the converted model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3u3gQKQXOiid"
      },
      "source": [
        "import numpy as np\r\n",
        "from onnxruntime import InferenceSession, SessionOptions, get_all_providers\r\n",
        "\r\n",
        "def sigmoid(x):\r\n",
        "  return 1 / (1 + np.exp(-x))\r\n",
        "\r\n",
        "def create_model_for_provider(model_path: str, provider: str) -> InferenceSession:\r\n",
        "\r\n",
        "  assert provider in get_all_providers(), f\"provider {provider} not found, {get_all_providers()}\"\r\n",
        "\r\n",
        "  # Few properties than might have an impact on performances (provided by MS)\r\n",
        "  options = SessionOptions()\r\n",
        "  options.intra_op_num_threads = 1\r\n",
        "\r\n",
        "  return InferenceSession(model_path, options, providers=[provider])"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9FUoCXgEAWjp"
      },
      "source": [
        "cpu_model = create_model_for_provider(output_path, \"CPUExecutionProvider\")"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iJEy5bBKbdIp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7c97afc3-44c1-4748-efc2-ab827c322718"
      },
      "source": [
        "question = \"what is google's specialization\"\r\n",
        "text = \"Google LLC is an American multinational technology company that specializes in Internet-related services and products, which include online advertising technologies, a search engine, cloud computing, software, and hardware.\"\r\n",
        "inputs = tokenizer.encode_plus(question,text, add_special_tokens=True,return_tensors=\"np\")\r\n",
        "\r\n",
        "answer_start_scores, answer_end_scores = cpu_model.run(None, {\"attention_mask\":inputs['attention_mask'],\"input_ids\":inputs[\"input_ids\"],\"token_type_ids\":inputs[\"token_type_ids\"]})\r\n",
        "\r\n",
        "ans_tokens = inputs[\"input_ids\"][0][np.argmax(answer_start_scores) : np.argmax(answer_end_scores) + 1 ]\r\n",
        "answer_tokens = tokenizer.convert_ids_to_tokens(ans_tokens , skip_special_tokens=True)\r\n",
        "\r\n",
        "answer_tokens_to_string = tokenizer.convert_tokens_to_string(answer_tokens)\r\n",
        "\r\n",
        "print (\"\\nQuestion : \",question)\r\n",
        "print (\"\\nAnswer : \",answer_tokens_to_string)"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "Question :  what is google's specialization\n",
            "\n",
            "Answer :  internet-related services and products\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}