LeNet_brevitas.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mahdi259/RISC-V_files/blob/main/LeNet_brevitas.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XKzfJ2GGhsAs"
      },
      "source": [
        "**Model Initialization**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fWVYzWkuY8Qp",
        "outputId": "53b3451f-2d8d-42d9-c42f-7c2f29a19012"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting brevitas\n",
            "  Downloading brevitas-0.10.2-py3-none-any.whl (626 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m626.6/626.6 kB\u001b[0m \u001b[31m6.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting dependencies==2.0.1 (from brevitas)\n",
            "  Downloading dependencies-2.0.1-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from brevitas) (24.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from brevitas) (67.7.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from brevitas) (1.12)\n",
            "Requirement already satisfied: torch>=1.9.1 in /usr/local/lib/python3.10/dist-packages (from brevitas) (2.2.1+cu121)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4 in /usr/local/lib/python3.10/dist-packages (from brevitas) (4.10.0)\n",
            "Collecting unfoldNd (from brevitas)\n",
            "  Downloading unfoldNd-0.2.1-py3-none-any.whl (16 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.9.1->brevitas) (3.13.3)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.9.1->brevitas) (3.2.1)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.9.1->brevitas) (3.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.9.1->brevitas) (2023.6.0)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m23.7/23.7 MB\u001b[0m \u001b[31m33.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m823.6/823.6 kB\u001b[0m \u001b[31m26.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.1/14.1 MB\u001b[0m \u001b[31m46.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m731.7/731.7 MB\u001b[0m \u001b[31m1.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m410.6/410.6 MB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.6/121.6 MB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.5/56.5 MB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m124.2/124.2 MB\u001b[0m \u001b[31m8.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m196.0/196.0 MB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nccl-cu12==2.19.3 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_nccl_cu12-2.19.3-py3-none-manylinux1_x86_64.whl (166.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m166.0/166.0 MB\u001b[0m \u001b[31m2.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.1/99.1 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.9.1->brevitas) (2.2.0)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.9.1->brevitas)\n",
            "  Downloading nvidia_nvjitlink_cu12-12.4.99-py3-none-manylinux2014_x86_64.whl (21.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m21.1/21.1 MB\u001b[0m \u001b[31m58.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->brevitas) (1.3.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from unfoldNd->brevitas) (1.25.2)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.9.1->brevitas) (2.1.5)\n",
            "Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, dependencies, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, unfoldNd, brevitas\n",
            "Successfully installed brevitas-0.10.2 dependencies-2.0.1 nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.19.3 nvidia-nvjitlink-cu12-12.4.99 nvidia-nvtx-cu12-12.1.105 unfoldNd-0.2.1\n"
          ]
        }
      ],
      "source": [
        "!pip3 install brevitas\n",
        "# !pip3 install torchvision --upgrade"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 852
        },
        "id": "ZMNsgBujhy1Q",
        "outputId": "a0f093dc-6590-456c-91da-8bca8ea06fc6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 9912422/9912422 [00:00<00:00, 259880680.36it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting data/MNIST/raw/train-images-idx3-ubyte.gz to data/MNIST/raw\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 28881/28881 [00:00<00:00, 68632121.15it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting data/MNIST/raw/train-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 1648877/1648877 [00:00<00:00, 149808109.97it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting data/MNIST/raw/t10k-images-idx3-ubyte.gz to data/MNIST/raw\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 4542/4542 [00:00<00:00, 11153705.37it/s]"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Extracting data/MNIST/raw/t10k-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([60000, 400]) torch.Size([60000])\n",
            "torch.Size([60000, 400]) torch.Size([60000])\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 16 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGgCAYAAAA3u2SKAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8p0lEQVR4nO3de3xV1Z3//0+CJICQCCIJgQRiQaxVoYJJKSg4xiK1eKuOdrxQcLR1YCyllpbKRSltFC2NKBWrtmitilRLq+OgndRCO0BoEGorLaIGGggJRMwVSCDZ3z/mZ379fHbIpdn7nL2zX8/HI3+8T87Ze3Hyyclir7XXSnAcxxEAABBZifFuAAAAiC86AwAARBydAQAAIo7OAAAAEUdnAACAiKMzAABAxNEZAAAg4ugMAAAQcXQGAACIODoDAABEnG+dgZUrV8rw4cOlV69ekpubK1u3bvXrVAAAoAsS/NibYM2aNXLrrbfKqlWrJDc3VwoKCmTt2rWya9cuGTRoUJuvbW5ulrKyMunXr58kJCR43bRIcBxHamtrJSMjQxITw3nxhzroGmoA3aEGRKiDrupwHTg+yMnJcWbNmtWSm5qanIyMDCc/P7/d15aWljoiwpcHX6WlpX78eGOCOqAGqAFqwHGog1jVwSniscbGRtm2bZvMnz+/5bHExETJy8uTzZs3u57f0NAgDQ0NLdlhE0XP9OvXL95N6DDqwB/UAMJUAyLUgV/aqwPPrx1VVlZKU1OTpKWlqcfT0tKkvLzc9fz8/HxJTU1t+crKyvK6SZEVpktq1IE/qAGEqQZEqAO/tFcHns8ZKCsrkyFDhsimTZtk/PjxLY/PmzdPNmzYIEVFRer5thdYU1MjmZmZXjYpsqqrqyUlJSXezegQ6sAf1ADCVAMi1IFf2qsDz4cJBg4cKD169JCKigr1eEVFhaSnp7uen5ycLMnJyV43AyFDHYAagAh1EC+eDxMkJSXJ2LFjpbCwsOWx5uZmKSwsVFcKAABAMHh+ZUBEZO7cuTJ9+nQZN26c5OTkSEFBgdTX18uMGTP8OB0AAOgCXzoDN9xwgxw6dEgWLVok5eXlMmbMGFm/fr1rUiEQZj169HA91r9/f5UHDhyocmVlpcofffSRyk1NTR61DgA6zpfOgIjI7NmzZfbs2X4dHgAAeCS8y1IBAABP0BkAACDifBsm6E6SkpJUbm195yFDhqg8dOhQlaurq1U+9dRTVb7gggtULi0tVfmNN95Q+ciRI220GH6wP/dPf/rTruf848qbIiKTJk1SecOGDSrfc889Kv/tb3/rShMRQvYe+nPPPVflN998U+Vjx4753iZ4r0+fPipPmTJFZXvr/SuvvKLyvn37/GnY/4crAwAARBydAQAAIo7OAAAAEReJOQOpqakq2zG6kSNHqnzOOeeobMfw7L3kIiIDBgxQ2e4QVVxcrLIdHxo2bJjKL7zwgsq//e1vXeeEv+w6AnaOQH5+vus1l1xyicr19fUq2zkEl19+ucoHDhxQ2c41gda7d2+V7c9o165dKn/44Ye+t6mzPvvZz6o8ceJElbds2aIycwbCaerUqSo/+uijKtu5ae+8847KzBkAAAC+ojMAAEDE0RkAACDiuuWcgVNO0f+syy67TOWHHnpIZTunwG6f2bNnT5Xr6uraPafdwvm5555T2a5Bb8eGS0pKVK6trXWdE96y9wF/7nOfU/muu+5SecKECa5j2LUIPvjgA5X//Oc/qzxnzhyVbR389Kc/PXmDIaeffrrKt912m8qPPPKIykGYM2A/K8aMGdPm848fP+5ja+AXu5bMpZdeqrKde7Zp0yaVq6qqfGnXyXBlAACAiKMzAABAxNEZAAAg4rrlnAE7Hr97926Vd+zYobIdu7n44otV3rNnj8oPPPCA65x2nsGIESNU/u///u+TthfBcP7556tsx5vt2hB//etfXcew8w7svcN2vYjRo0ernJWV1bHGQkRE+vbtq7J9P+1aEUFg5whcccUVKtt5Iq3NUUKw2N97EZGbb75Z5S9+8Ysq28+PefPmqbxz506PWtcxXBkAACDi6AwAABBxdAYAAIi4bjlnwHEcle0az7NmzVI5LS1NZTtmt3XrVpXtmgEiIkeOHFHZrl2A+LNrAJx55pkqf+9731N54MCBKtvx/u9///uuc9h9Luy4oJ2vcv3116s8fvx4lYcMGaLy/v37XeeMsrPPPltluydIECQkJKjc3l4ntkYQfGeddZbrMft3xs4fevLJJ1X+05/+pPKJEyc8al3HcGUAAICIozMAAEDE0RkAACDiuuWcAcuOvdhxV3sf78aNG1W247gZGRmuc/ztb39TOQhroEOz9/Dfc889Ktu1IZ544gmV7VoRmzdvdp1jw4YNnWpTc3OzymPHjlV53LhxKjNnQJs4caLKdo2RhoaGWDanVXb+0OTJk1W2c5zKysr8bhK6yO6Jccstt7iec95556n88ssvq/ziiy+qHO89KLgyAABAxNEZAAAg4ugMAAAQcZGYM9CempoalR9++GGV7f3o9957r+sYdqx43bp1KpeXl6tsxwnhPXtf7+WXX67ytddeq7JdN+Cxxx5T2Y4/+zHG17NnT5V79+7t+Tm6E7uugJ1Tcfjw4TZfb/cusNnuOWK/b/esFxEZNmyYyldffbXK//qv/6ry888/r7L9rED8nXKK/lN5ySWXqHzjjTe6XvPuu++q/Oijj6p86NAhj1rnDa4MAAAQcXQGAACIuE53BjZu3CjTpk2TjIwMSUhIcF0OdxxHFi1aJIMHD5bevXtLXl6eawthAAAQHJ2eM1BfXy+jR4+WmTNnusZcRUSWLVsmK1askKefflqys7Nl4cKFMmXKFNm5c6f06tXLk0Z7zY7ff/DBByovW7ZM5fnz57uOcd9996ls1x//4Q9/2OY57P3m6LqhQ4eqPH36dJX37t2r8ksvvaSynUsSC3ZNjCDcJx9k9vfG3v89YcIEle04bXp6usqZmZkq270P7DwU+3oRkerqapVra2tVtvNA7HoVdt0TxJ7dx+Qzn/mMyt/61rdUtnMKRESWL1+u8qZNmzxqnT863RmYOnWqTJ06tdXvOY4jBQUFsmDBArnqqqtEROSZZ56RtLQ0WbduXauTLAAAQHx5ejdBSUmJlJeXS15eXstjqampkpubK5s3b261M9DQ0KD+9xOP/40h/qgDUAMQoQ7ixdMJhB/fEmO3BE5LSzvp7TL5+fmSmpra8mUv0yEaqANQAxChDuIl7ncTzJ8/X6qrq1u+SktL490kxAF1AGoAItRBvHg6TPDxZJqKigoZPHhwy+MVFRUyZsyYVl+TnJzsWtgj3uykpN///vcqV1RUuF5jN6qYMWOGynZik90k5/333+90O7sTP+pg1KhRKmdnZ6sci8VeEhISVP7EJz6hsq2LoqIilbdv3+55m4Lqn6kBu3nU6NGjVf7mN7+psp2gaRctKikpUflPf/qTyu+9957Krf3e2snBF110kco33XSTynahJLvZUtQE4W/CaaedpvLs2bNVthuI7dixw3UMuxBd0CcDe3plIDs7W9LT06WwsLDlsZqaGikqKnLt/AcAAIKh01cG6urqVO+4pKREduzYIQMGDJCsrCyZM2eOLF26VEaOHNlya2FGRoZrSU4AABAMne4MFBcXq3WZ586dKyL/dw/36tWrZd68eVJfXy933HGHVFVVycSJE2X9+vWBXWMAAICo63RnYPLkyW1uspOQkCBLliyRJUuWdKlhQWL/vXaMT0TkZz/7mcpf/OIXVZ42bZrKdqzTLoBjxzbReXZTGTtm9/bbb6tcX1/f5XPajYbOOecclRcsWKDy0aNHVbabZNm6gPbaa6+pvG3bNpXtz6M9doEgmxsbGzt1PBH3pja7du1SmZ9x8Fx22WUq259hVVWVyi+++KLrGGFbeTfudxMAAID4ojMAAEDE0RkAACDiPF1noLuwkx3tGgnXXXed6zWTJk1SediwYSofPnxYZXu/MnME/GfH5z/66COV25oL05rU1FTXY3bzrpkzZ6ps72P/8pe/rDILrHSO/b1pbT5PvNnPk3fffVflAwcOxLI56IC7775b5TPOOEPlNWvWqPzjH//YdYywrRfBlQEAACKOzgAAABFHZwAAgIiLxJwBuz78Kafof7ZdL/5LX/qSytdcc43KI0aMcJ3Djg/ZNeafeuopld955502Wgw/DBgwQOWbb75Z5crKSpWPHz+u8uTJk1XOyclxneOTn/ykynYtg4ceekhl5ghET9++fVXu3bu3ynYuC7xn/ybk5eWpbHdKtHNTfvvb36rcHbZZ5soAAAARR2cAAICIozMAAEDEdcs5A/a+XrvH+fnnn6/yjBkzVLbrChw7dkzlLVu2uM750ksvqfyrX/1K5fLycpVZV8B/dXV1Ktuf4xe+8AWVx44dq7Jdd8DucW7XmBcReeaZZ1R+7rnnVA7iffCIrZSUFJXtnAH4r1+/firbeWH9+/dXubCwUOUNGzao3B0+z7kyAABAxNEZAAAg4ugMAAAQcaGcM5CUlKTyhRdeqPLll1+u8vXXX6/y8OHD2zye3V985cqVKv/iF79wtenvf/+7ys3Nza7nILbsPf6PPvqoynfeeafKti727dunsl0r4oUXXnCdc9u2bZ1tJiJm4MCBKtvxa/ivZ8+eKtufiV2H4A9/+IPKdn+J7oArAwAARBydAQAAIo7OAAAAERfKOQN2bW97j+jFF1+scnV1tco/+9nPVLbjP6+88orKe/bsUdner45gOnz4sMp2zsDLL7+scp8+fVQ+cuSIynatiPr6+q42ERFgx5vtHhlHjx6NZXMg7t9du27A0KFDVT548KDvbYo3rgwAABBxdAYAAIg4OgMAAERcgmMXYI+zmpoaSU1NjXczuoXq6mrXOuhhQR14gxpAmGtAhDrwSnt1wJUBAAAijs4AAAARR2cAAICIozMAAEDE0RkAACDi6AwAABBxgesMBOxOx1AL83sZ5rYHSZjfxzC3PUjC/j6Gvf1B0d77GLjOQG1tbbyb0G2E+b0Mc9uDJMzvY5jbHiRhfx/D3v6gaO99DNyiQ83NzVJWViaO40hWVpaUlpYGdsGMmpoayczMDFwbHceR2tpaycjIkMTEwPX3OiQsdUAN+Ica6JruUAMi1EFXdbQOArdrYWJiogwdOlRqampERCQlJSVQb2xrgtjGsK/YFbY6CGL7qIHYCmL7wl4DItSBFzpSB+HtLgIAAE/QGQAAIOIC2xlITk6WxYsXS3JycrybclJhaGPYBf09Dnr7uoOgv8dBb193EfT3Oejta0/gJhACAIDYCuyVAQAAEBt0BgAAiDg6AwAARBydAQAAIo7OAAAAEUdnAACAiKMzAABAxPnWGVi5cqUMHz5cevXqJbm5ubJ161a/TgUAALrAl0WH1qxZI7feequsWrVKcnNzpaCgQNauXSu7du2SQYMGtfnaj3eo6tevnyQkJHjdtEjoDruVUQddQw2gO9SACHXQVR2uA8cHOTk5zqxZs1pyU1OTk5GR4eTn57f72tLSUkdE+PLgq7S01I8fb0xQB9QANUANOA51EKs68HwL48bGRtm2bZvMnz+/5bHExETJy8uTzZs3u57f0NAgDQ0NLdlhdWTP9OvXL95N6DDqwB/UAMJUAyLUgV/aqwPPrx1VVlZKU1OTpKWlqcfT0tKkvLzc9fz8/HxJTU1t+crKyvK6SZEVpktq1IE/qAGEqQZEqAO/tFcHns8ZKCsrkyFDhsimTZtk/PjxLY/PmzdPNmzYIEVFRer5thdYU1MjmZmZXjYpsqqrqyUlJSXezegQ6sAf1ADCVAMi1IFf2qsDz4cJBg4cKD169JCKigr1eEVFhaSnp7uen5ycHNotH+Ed6gDUAESog3jxfJggKSlJxo4dK4WFhS2PNTc3S2FhobpSAAAAgsHzKwMiInPnzpXp06fLuHHjJCcnRwoKCqS+vl5mzJjhx+kAAEAX+NIZuOGGG+TQoUOyaNEiKS8vlzFjxsj69etdkwoBIMj69u2r8tChQ1Wur693vaasrEzlpqYm7xsGeMyXzoCIyOzZs2X27Nl+HR4AAHgkvMtSAQAAT9AZAAAg4nwbJgC6u9bW+ba3RA0bNkzlY8eOqbx3716VWW0tvnr06KHyVVddpfLy5ctVbm0Dtttuu03lgwcPetQ6xIqtg9TUVJUzMjJUnjRpkusYSUlJnTpnSUmJyq+//rrKR48e7dTxOosrAwAARBydAQAAIo7OAAAAEcecgVbYseBTTz1V5d69e7teY8eY6urqVG5ublbZ3ntsx5IRe3Yjj169eql85plnqnzllVe6jnHBBReoPHz4cJX37Nmj8j/u7iki8t5773WkqYiRU07RH5H29/yMM85wvaZPnz6+tgnes0vlX3/99SpfcsklKtvNk0aPHu06pq2V9uzfv1/lJUuWqPz000+r3NjY2Knjt4crAwAARBydAQAAIo7OAAAAEcecARHp2bOnyrm5uSrfdNNNKo8bN851DLuG+f/+7/+qfOTIEZXffvttlZ999lmVmUPgPzs35LzzzlP52muvbTPbcUMRkV27dql86NAhlT//+c+r/Oqrr6ps5xScOHHCdQ74x87tqaysVNnve70RH5/5zGdUnjdvnsp2bohdO8L+3oq41wzp37+/yqeffrrKdu2CG2+8UeVXXnlF5fLyctc5u4IrAwAARBydAQAAIo7OAAAAERfJOQP2fnJ7j+gjjzyickpKisovv/yy65h/+ctfVLbbN9t5Bs8884zKL774osrMGfCfHfP/4Q9/qPLFF1+ssq2b4uJi1zG/9rWvqWzHCadOnarygAEDVG5tvwPEjr033NaI/Sw4fPiw6xiDBg1SubS0VGW7xgjiz/4uFxQUqGzXltm+fbvKVVVV7Z4jJydH5e9973sq2zUt/v73v6tcX1/f7jm6gk8eAAAijs4AAAARR2cAAICIi8ScAbvH/OTJk1VeunSpynZMb86cOSr/9re/dZ3D3qN+2mmnqWzvS/31r3+tst/jQXCzP7O0tDSV7TwQO664atUq1zHfeustlW+++eauNBExZn/37dhwWVmZymeddZbrGLfffrvKu3fvVvmjjz7qShPhg3379qn88MMPq2znC9n1P+yaAiLueQbnn3++ynZ+kF3DYvPmzSrb/W68xpUBAAAijs4AAAARR2cAAICI65ZzBpKSklS2a8rff//9Ktvx/K985Ssq79ixQ2U7FiTiXkfa3mu8YsUKld944w2Vufc49n7zm9+oXFFRofLevXtV/vDDD1XuyL4Bw4cP/+cah7iwY79btmxR+b333lN51KhRrmPYNec7u6894q+93227JsCYMWNcz7nqqqtUvu6661S2f6fs58369etVbm1egpe4MgAAQMTRGQAAIOLoDAAAEHHdcs6A3XvarhNg1/1fuHChyh988IHK9n70WbNmuc75hS98QeX/+Z//UflHP/qRyrW1ta5jILZsHdh1BOzckEsvvVRlu/+4iPveYbu/gb2f+a9//avKHZmHAKBr7LoB6enpKl9++eUq278BPXv2VNmuXSMiMnLkSJXtZ8Mf/vAHlR977DGVKysrXcf0E1cGAACIODoDAABEHJ0BAAAirtNzBjZu3CgPPvigbNu2TQ4cOCC//OUv5eqrr275vuM4snjxYnniiSekqqpKJkyYII899phr/MRPdqzXju1u27ZNZXsv+MyZM1W240VDhw51nbNXr14qr1mzRuXy8vKTNxhx0adPH5UvuugiladMmaLyFVdcoXJr603YPSns/cgvvfSSynYNi+bm5pO2F4A3Tj31VJX/4z/+o81sn2/nHNg1A0Tc6wLYOUpvvvmmyna/miNHjriO6adOXxmor6+X0aNHy8qVK1v9/rJly2TFihWyatUqKSoqklNPPVWmTJnieiMAAEAwdPrKwNSpU2Xq1Kmtfs9xHCkoKJAFCxa0rL70zDPPSFpamqxbt861Sp+ISENDgzQ0NLTkmpqazjYJ3QB1AGoAItRBvHg6Z6CkpETKy8slLy+v5bHU1FTJzc11bcf4sfz8fElNTW35yszM9LJJCAnqANQARKiDePG0M/DxuLjdFz4tLe2kY+bz58+X6urqlq/S0lIvm4SQoA5ADUCEOoiXuC86lJycLMnJyZ4e024oYzd8mDFjhsq5ubkq/+53v1P5F7/4hcpf/OIXXee0C0ps3bpVZb83mQg7P+rATvJJSUlR2daBXUzqj3/8o8p2UZABAwa4zmkXuLL/pokTJ6p8ySWXqPzLX/5S5SjNtfGjBvxma0zEvVFRa5PLcHKxqIPGxkaVN23apLKdVG43MevIRN+srCyV7QTk888/X2U7SbG+vr7dc3jJ0ysDH6/iZN+4iooK1wpPAAAgGDztDGRnZ0t6eroUFha2PFZTUyNFRUUyfvx4L08FAAA80ulhgrq6OrWnd0lJiezYsUMGDBggWVlZMmfOHFm6dKmMHDlSsrOzZeHChZKRkaHWIgAAAMHR6c5AcXGxGuecO3euiIhMnz5dVq9eLfPmzZP6+nq54447pKqqSiZOnCjr1693Lcrjp6qqKpW//e1vq/zwww+rbMePDh06pPKnPvUplW+77TbXOX//+9+rzO0w8WdnIds6sGN4dm7Jd7/7XZXt4lX2+yIi1dXVKv/85z9XecSIESp/61vfUtleQduwYYPKdgjObnYCf9XV1anc1NTkes6FF16o8tixY1UuKyvzvmHoFPuZ/8Ybb6i8ZcsWle3cnY7MAbND4xdccEFnmhhzne4MTJ48uc03IiEhQZYsWSJLlizpUsMAAEBssDcBAAARR2cAAICIi/s6A36wwxh23QGb22PHCe3mMyIif/nLX1RmzkDsnXHGGSovWLBA5X9cGVNEZMWKFSrb8flp06apfMstt6hs56aIiPznf/6nynYegl3r4OKLL1bZrn1g2/Dss8+qzJyB2Nq4caPKdt6JiEjfvn1VthtgvfLKK943DG2yn9n9+/dX2f5N+Oijjzp1/H79+rkes7XR2rokQcKVAQAAIo7OAAAAEUdnAACAiOuWcwa8ZtdIaG09csSfXef/2muvVfno0aMq2/vB7fj8sGHDVLbj89/73vdcbfjHBblERE6cOKGyvV/Z7kVQXFys8qBBg1TevXu365yIHTuvpLU1A0aNGqWy3fvEji/X1tZ61Dp8bPDgwSpPnz5dZbsGQH5+vspHjhxp8/h2/P+aa65xPecb3/iGyr1791bZzlOwnxWxxpUBAAAijs4AAAARR2cAAICIY85AB9j711tbZwDxd/jwYZXt2g92HHHIkCEq2z3NH3jgAZXfeustlQ8ePOhqQ0fWLP9Hdm37PXv2tJkRX/v27VPZ1oSIyFlnnaWynfeRlJTkfcOg2LUd5syZo/Lx48dVtnMA7PetT3ziEyqfd955rufY3+0nnnhC5SeffFLl1tYtiSWuDAAAEHF0BgAAiDg6AwAARByD3x2wc+dOle3YtIh7PfIePXqoHO97SKPgzTffVNne85+RkaGyHf+194zzM4Nl16ooLCx0PceuV4HY++CDD1Tevn27ynZNkptvvrnN49mfu/0bYOcbiYj84he/UHndunUqV1dXt3nOWOPKAAAAEUdnAACAiKMzAABAxDFnoAPKy8tVXrt2res5U6ZMUdmOJf7mN79RubP3o6N93LMPv9l5JHZcWERky5Ytbb4m3veTR8GOHTtUvvPOO1WeOXOmylOnTlX5z3/+s8qvv/56m9+vrKx0tcHOKwj6HCSuDAAAEHF0BgAAiDg6AwAARFyCE7DB65qaGklNTY13M5SEhASV7brUIiKLFy9W2a47YPfTrqmp8ah1J1ddXS0pKSm+n8cPQayDMKIGEOYaEKEOvNJeHXBlAACAiKMzAABAxNEZAAAg4lhnoAPstIr333/f9ZzvfOc7Ko8YMULlxsZG7xsGAIAHuDIAAEDE0RkAACDiAjdMELA7HVvVWhubm5tVtktPxuPfFYb38mTC3PYgCfP7GOa2B0nY38ewtz8o2nsfA9cZqK2tjXcT/in79+9vM8dDbW1taO/PDWsdBA01gDDXgAh14JX26iBwiw41NzdLWVmZOI4jWVlZUlpaGtgFM2pqaiQzMzNwbXQcR2prayUjI0MSE8M5EhSWOqAG/EMNdE13qAER6qCrOloHgbsykJiYKEOHDm1ZoS8lJSVQb2xrgtjGMP9PQCR8dRDE9lEDsRXE9oW9BkSoAy90pA7C210EAACeoDMAAEDEBbYzkJycLIsXL5bk5OR4N+WkwtDGsAv6exz09nUHQX+Pg96+7iLo73PQ29eewE0gBAAAsRXYKwMAACA26AwAABBxdAYAAIg4OgMAAEQcnQEAACKOzgAAABFHZwAAgIijMwAAQMT51hlYuXKlDB8+XHr16iW5ubmydetWv04FAAC6wJcVCNesWSO33nqrrFq1SnJzc6WgoEDWrl0ru3btkkGDBrX52o+3q+zXr58kJCR43bRI6A5bl1IHXUMNoDvUgAh10FUdrgPHBzk5Oc6sWbNaclNTk5ORkeHk5+e3+9rS0lJHRPjy4Ku0tNSPH29MUAfUADVADTgOdRCrOjhFPNbY2Cjbtm2T+fPntzyWmJgoeXl5snnzZtfzGxoapKGhoSU7bJXgmX79+sW7CR1GHfiDGkCYakCEOvBLe3Xg+bWjyspKaWpqkrS0NPV4WlqalJeXu56fn58vqampLV9ZWVleNymywnRJjTrwBzWAMNWACHXgl/bqwPM5A2VlZTJkyBDZtGmTjB8/vuXxefPmyYYNG6SoqEg93/YCa2pqJDMz08smRVZ1dbWkpKTEuxkdQh34gxpAmGpAhDrwS3t14PkwwcCBA6VHjx5SUVGhHq+oqJD09HTX85OTk0O7/zO8Qx2AGoAIdRAvng8TJCUlydixY6WwsLDlsebmZiksLFRXCgAAQDB4fmVARGTu3Lkyffp0GTdunOTk5EhBQYHU19fLjBkz/DgdAADoAl86AzfccIMcOnRIFi1aJOXl5TJmzBhZv369a1IhAK1Pnz4qZ2RkqPz3v/9d5cbGRt/bFCV2kpWdgT1gwACVm5qaXMc4duyYylVVVSofP368Cy0E/OFLZ0BEZPbs2TJ79my/Dg8AADwS3mWpAACAJ+gMAAAQcb4NE4SZHTe047bDhg1zvWbTpk2+tgndU1JSksrTp09X+ZZbblH53/7t31Tes2ePL+2KqrFjx6p8++23qzxp0iSVq6urXcc4ePCgyvfdd5/KxcXFXWki4AuuDAAAEHF0BgAAiDg6AwAARBxzBlphx3HvvfdelVNTU12v2b59u8pHjx71vF3ofq688kqVv/71r6v8xz/+UeWamhrf2xRl9nf7uuuuU7l///4qt7a1i90zvrm5WeW77rpL5b1793a6neh+bN307NlTZbvmRY8ePVS2nw2d/RvElQEAACKOzgAAABFHZwAAgIhjzkAr7NjNOeeco7KdUyAi0rdvX5WZMxB+vXr1Unn48OEqtzbW297P3dbJ1KlTVbbjy88++6zKdp17eOutt95See3atSrfeuutKh85csR1jNNPP13lT3/60yoPHjxYZeYMRMMpp+g/t0OHDlXZrnFx9tlnq2znF9k6W7JkicrPPPNMp9rHlQEAACKOzgAAABFHZwAAgIhjzsA/wY79iLjv+UT4TZw4UeVvf/vbKt99992u1+zYsaNTx5wwYYLKq1evVvl3v/udynZOAbz10UcfqWznbFxwwQUqjxo1qt1j2nkF77///j/ZOoSJnQNwxRVXqPz5z39e5XHjxql8/PhxlYuKilR+7rnnVH733Xf/qXZ+jCsDAABEHJ0BAAAijs4AAAARx5yBDmhqalL5tNNOcz0nIyND5fLycj+bBB/YuSDnnnuuyiNHjlTZrhnQmvbWFbC1ZO9zZ72K+Hr77bdVXrp0qcrLly93vSYlJUXlAQMGqJybm6vyq6++2pUmIg7sZ8W//Mu/uJ5j97Sx69MUFxer/OCDD6q8f/9+lUtKSlT+8MMPVT5x4sTJG9wBXBkAACDi6AwAABBxdAYAAIg45gy0wt7fuW/fPpXt2LGIe1wQ4TNixAiVv/KVr6i8ZcsWlTtyX6+9t/i6665T+fnnn1d58+bN7R4T/rHrxX/5y19WefLkyW0+vzX19fUq27FfBE9CQoLK9ud8ySWXqHz++ee7jvHmm2+q/MMf/lDlysrKrjTRc1wZAAAg4ugMAAAQcXQGAACIOOYMtMLer3ngwAGVW1sf3o4xIXjs/hHDhg1T+Z577lG5d+/eKtv9wQ8fPuw6h11f3O5nYPeuf/zxx1Wura11HROxY9eT//rXv65y//79Ve7I7729v/yMM874J1uHWBk4cKDKdg0Auz7IN77xDdcx7O96XV2dN43zCVcGAACIODoDAABEXKc7Axs3bpRp06ZJRkaGJCQkyLp169T3HceRRYsWyeDBg6V3796Sl5cnu3fv9qq9AADAY52eM1BfXy+jR4+WmTNnyrXXXuv6/rJly2TFihXy9NNPS3Z2tixcuFCmTJkiO3fulF69ennS6HhrbU161hkIHjtHYPz48Sp///vfV3nChAkq273tb7zxRpV79uzpOue0adNUTk1NVfk73/mOynSUg8Wu8/DUU0+pbO8vtzUmIjJ48GCV+/Tpo3JWVlZXmogYGDRokMo33HCDymvWrFG5qqrKdYygzxGwOt0ZmDp1qmuzlY85jiMFBQWyYMECueqqq0Tk/yZdpaWlybp161wfpgAAIP48vZugpKREysvLJS8vr+Wx1NRUyc3Nlc2bN7faGWhoaJCGhoaWXFNT42WTEBLUAagBiFAH8eLpBMKPt+1NS0tTj6elpZ10S9/8/HxJTU1t+crMzPSySQgJ6gDUAESog3iJ+90E8+fPl+rq6pav0tLSeDcJcUAdgBqACHUQL54OE6Snp4uISEVFhZpEU1FRIWPGjGn1NcnJyZKcnOxlMzxnr2qkpKS4npORkRGr5nRLftTBeeedp7KdMPjZz35W5RdeeEFlu1iM3WTo6quvdp3TTi61tWMn0doJaE1NTa5jRkUQPgsOHjyo8tKlS1X+8Y9/rPIpp7g/Qq+88kqV7cJF9sqpnYhqN0qLmljUgV1QzP5eVlRUqPzII4+ofP3116s8Z84c1zkWL16s8pEjRzrbzJjy9MpAdna2pKenS2FhYctjNTU1UlRU5JrJDQAAgqHTVwbq6urkvffea8klJSWyY8cOGTBggGRlZcmcOXNk6dKlMnLkyJZbCzMyMlr9XxQAAIi/TncGiouL1b22c+fOFRGR6dOny+rVq2XevHlSX18vd9xxh1RVVcnEiRNl/fr13WaNAQAAuptOdwYmT54sjuOc9PsJCQmyZMkSWbJkSZcaFiQffPCByq2N6dlFhxgHjL/LLrtM5bKyMpXtAkFbtmxR2W4oYxePufTSS13ntJuTVFdXq2zHQu2mWAgWe1ubzXbsWeT/nzv1MVs3dp5IYmLc53F3e7m5uSrbDcS2bt2qsp0j8IMf/EDliy66qM3ji7jnD0VqzgAAAAgfOgMAAEQcnQEAACLO03UGuis7Dtza8pijRo1S2Y4X2U1v4L9f//rXKj/xxBMq281FEhISVB42bJjKQ4YMUfn3v/+965xf/epXVba1YuugubnZdQx4p1+/fiqPHj1a5Z07d6p8+PDhNo9na2Ts2LGu51xzzTVtHvPVV19V+R+X3oU/7FL49lZ3O5/ozjvvVHno0KEqDxw4UGW7oZWIyIcfftjpdsYTVwYAAIg4OgMAAEQcnQEAACKOOQMd8P7776u8bds213PsGvTHjh3ztU1o365duzr1/NNPP13lu+66S2V7v/jy5ctdx7C1gvi64oorVLb7BKxdu1bl1157TWX7e2z3u7jttttc57Tjy08++aTK/7iCK2LjscceU9muD3HxxRerbFfMraysVNmuQ2D3NREJ3z4jXBkAACDi6AwAABBxdAYAAIg45gx0gB37aW2dAbsGPWvOh4/dq+Dyyy9X2d5L3No6AwiWv/3tbyofPHhQ5VmzZqk8Y8YMlY8ePary8OHDVbbrGIi497h4+OGHVWY+Uezt3r1b5bvvvltlO4fA7h9h1wOxfwO6w1oRXBkAACDi6AwAABBxdAYAAIg45gx0QFJSksp23FDEvea8fc3x48c9bxe8NWHCBJX37Nmj8tNPP61yXV2d301CF9m9B+677z6VL7vsMpUnTZqk8qc+9SmVBwwYoLKdkyAi8u1vf7vd5yC2HMdR2f7u8rvMlQEAACKPzgAAABFHZwAAgIhjzkAH2DUDWtuboLi4WOXGxkZf2wTvfec731HZ7kWwf//+WDYHHrC/h/b31I7n/+QnP1HZriNga6K1NUf27dvX6XYC8caVAQAAIo7OAAAAEUdnAACAiGPOQAdUVVWp/NWvfjU+DYGv7Lr16P7au9+8oqIils0B4oYrAwAARBydAQAAIo7OAAAAEUdnAACAiKMzAABAxNEZAAAg4gLXGbBbTeKfF+b3MsxtD5Iwv49hbnuQhP19DHv7g6K99zFwnYHa2tp4N6HbCPN7Gea2B0mY38cwtz1Iwv4+hr39QdHe+5jgBKzb1dzcLGVlZeI4jmRlZUlpaamkpKTEu1mtqqmpkczMzMC10XEcqa2tlYyMDElMDFx/r0PCUgfUgH+oga7pDjUgQh10VUfrIHArECYmJsrQoUNbdgNLSUkJ1BvbmiC2MTU1Nd5N6JKw1UEQ20cNxFYQ2xf2GhChDrzQkToIb3cRAAB4gs4AAAARF9jOQHJysixevFiSk5Pj3ZSTCkMbwy7o73HQ29cdBP09Dnr7uougv89Bb197AjeBEAAAxFZgrwwAAIDYoDMAAEDE0RkAACDi6AwAABBxdAYAAIg4OgMAAEQcnQEAACLOt87AypUrZfjw4dKrVy/Jzc2VrVu3+nUqAADQBb4sOrRmzRq59dZbZdWqVZKbmysFBQWydu1a2bVrlwwaNKjN1368Q1W/fv0kISHB66ZFQnfYrYw66BpqAN2hBkSog67qcB04PsjJyXFmzZrVkpuampyMjAwnPz+/3deWlpY6IsKXB1+lpaV+/HhjgjqgBqgBasBxqINY1YHnWxg3NjbKtm3bZP78+S2PJSYmSl5enmzevNn1/IaGBmloaGjJDqsje6Zfv37xbkKHUQf+oAYQphoQoQ780l4deH7tqLKyUpqamiQtLU09npaWJuXl5a7n5+fnS2pqastXVlaW102KrDBdUqMO/EENIEw1IEId+KW9OvB8zkBZWZkMGTJENm3aJOPHj295fN68ebJhwwYpKipSz7e9wJqaGsnMzPSySZFVXV0tKSkp8W5Gh3SXOujbt6/Kt9xyi8rvv/++ym+88Yav7aEGEKYaEKEO/NJeHXg+TDBw4EDp0aOHVFRUqMcrKiokPT3d9fzk5OTQbvkI71AHoAYgQh3Ei+fDBElJSTJ27FgpLCxseay5uVkKCwvVlQIAABAMnl8ZEBGZO3euTJ8+XcaNGyc5OTlSUFAg9fX1MmPGDD9OBwAAusCXzsANN9wghw4dkkWLFkl5ebmMGTNG1q9f75pUCIRJUlKSyhMmTHA952tf+5rK5557rsoLFizwvmEA0EW+dAZERGbPni2zZ8/26/AAAMAj4V2WCgAAeILOAAAAEefbMIGf7L3cdlEKey+lvaXRrsR08OBBlZubm1W2SzFUVla62vTuu++qfOTIEddzEC69evVS+YorrlB52bJlrtfYhT1+/vOfq/zyyy971DrEwogRI1SePHmyyuvWrXO9prXPByDouDIAAEDE0RkAACDi6AwAABBxoZwzMH36dJX//d//XeU+ffqobOcI9OzZU+Xq6mqVm5qa2jz/0aNHXY/t3r1b5VdffVVlO7Zoz4n4s+P9dq7JHXfcofKQIUNcx3j22WdVfvzxx1VubGzsShPhM1sDl19+ucrLly9X+aOPPnId46WXXvK+YQi0Hj16uB7r3bu3yvbvjp2TdPz4cZUPHz6ssp3L5jWuDAAAEHF0BgAAiDg6AwAARFwo5wzYdQXq6upU/vDDD1W26wiUlZW1eXy7RsApp+i36bzzznO95sorr1T5/PPPV/mdd95Rubi4uM02IPbsXJPbbrtN5VGjRqn83HPPuY5x//33q9xerSFY7Joi+/fvV7mhoUFluw4Buic7vj9o0CCVx4wZ43rNZZddpvLw4cNVHjt2rMp79+5V+Rvf+IbKmzZt6khT/2lcGQAAIOLoDAAAEHF0BgAAiLhQzhnIz89X2d7Pacf87TjfsWPHOnU+e3+oXddARORzn/ucysnJySonJtLvCrovf/nLKl933XUq2zkCDz74oOsYrd13jvDaunWryvv27VP5s5/9rOs1SUlJKrO2RPDYeWCTJk1S+TOf+YzKZ5xxhsqf/OQnVT7zzDNd57DrlNg9dSx7jttvv11l5gwAAABf0RkAACDi6AwAABBxoZwzUFVV1WbuLDuef/rpp6s8Y8YMle+8807XMew8hQULFqi8ffv2rjQRHrDrzp9zzjkq33XXXSrbPSj+67/+S+Wampp2z9m/f3+V7f3Kdo5BZ+ezwF+1tbUqnzhxQuXc3FzXa8466yyV//KXv3jfMHSK/b2zn+nf/OY3Vbb7jtjP9z179rT5ehGRyspKle28MruOgN0T591333Ud009cGQAAIOLoDAAAEHF0BgAAiLhQzhnw2tlnn63ykiVLVL700ktVTklJcR3D7mP/+uuvq2z3qkbs2XuLc3JyVE5LS1PZ3tdrxwlbu2/4E5/4hMp2bNLuWfHmm2+q/Mgjj6hs99lAsNg1BUTce1wg/uy+AF/5yldUzs7OVvnAgQMqFxQUqPzyyy+rXFJS4jqnnQNw0003qWzrxH4WrF692nVMP3FlAACAiKMzAABAxNEZAAAg4pgzIO77P6dNm6ay3ZugNXZt6/vuu09lO4egqKhI5UOHDqlsx5vQdXaux8UXX6yyHf/dtWuXynYNAPszFxGZM2eOynYNc7t2gV2zwu6j8fDDD6ts73cG0L7LLrtM5czMTJXff/99le0cgZ/85Ccq299DuxeNiMjYsWNVvvDCC1W2+1zYeQix3ueEKwMAAEQcnQEAACKu052BjRs3yrRp0yQjI0MSEhJk3bp16vuO48iiRYtk8ODB0rt3b8nLy5Pdu3d71V4AAOCxTs8ZqK+vl9GjR8vMmTPl2muvdX1/2bJlsmLFCnn66aclOztbFi5cKFOmTJGdO3e61ocOis2bN6ts7+8cMWKEygMGDHAdw+5nfdttt6l84403qvy73/1O5Yceekhlv/eujoIePXqo/KlPfUrlCy64QOXi4mKVX3rpJZU//elPq2znB4iIjBw5UuUnn3xSZbve+N13363y0KFDXcdEcNl9TRBMf/zjH1W2cwDs562d42XnCNifu50fICKyfPlyle3fiPz8fJWfeeYZle38Ib91ujMwdepUmTp1aqvfcxxHCgoKZMGCBXLVVVeJyP/9A9PS0mTdunWuP4gAACD+PL2boKSkRMrLyyUvL6/lsdTUVMnNzZXNmze32hloaGhQPaCO7ASH7oc6ADUAEeogXjy9xlVeXi4i7mVd09LSWr5n5efnS2pqasuXveUD0UAdgBqACHUQL3Ef8Jo/f75UV1e3fJWWlsa7SYgD6gDUAESog3jxdJggPT1dREQqKipk8ODBLY9XVFTImDFjWn1NcnJyqws2xJJdAGj79u0q28VqBg0a5DrGx3MkPjZz5kyV7STEK6+8UmU7udJupNHdfyH8qAN7vCuuuEJlu2nQddddp7JdZMhO+LFXwETcm1zZO2m++93vqmw3Ivr5z3+ucpQWGQrCZ0Fntfbzqauri0NLug8/6mDLli1t5s5KTU1V+Z577nE9x26EZn+3bY73cIinVways7MlPT1dCgsLWx6rqamRoqIiGT9+vJenAgAAHun0lYG6ujp57733WnJJSYns2LFDBgwYIFlZWTJnzhxZunSpjBw5suXWwoyMDLn66qu9bDcAAPBIpzsDxcXFcskll7TkuXPniojI9OnTZfXq1TJv3jypr6+XO+64Q6qqqmTixImyfv36wK4xAABA1HW6MzB58mRxHOek309ISJAlS5a4xk7DpLGxUeXKyso2s8j/XSH5R3YRi5/+9KcqZ2dnqzxq1Kg2v9/d5wz4wS46NHDgQJXt5iTvvPOOytdcc43KdtOhH/3oR65z/vWvf1XZLkxUUVGhsp2HsGPHDtcxEVytbSZz4MCBOLQEfkpISFDZXum2m56JiNTW1qr8s5/9TOWysjJvGueRuN9NAAAA4ovOAAAAEUdnAACAiPN0nYGgsmPHdjzezoGwY8kdcfToUZXtHILDhw+32Ybq6mqVDx482Ok2QGtqalLZ3tN/6qmnqmw3CfrHtTJE3OsWfOlLX3Kdc/bs2So/9dRTKt97770q27ppbm52HRPB1drPi59h+Nk5AnZNEbs/j/2sERFZuXKlynZDvKDhygAAABFHZwAAgIijMwAAQMR1yzkDdo6AvQd00aJFKts1EdqbM3DKKe63zY4p2fFku46AXcvgrbfeUnnPnj1ttgHtO378uMp2DYA+ffqo/IMf/EBluy6B/bmffvrprnO+/fbbKq9evVrl+vr6kzcYgWPnE9mx4db2KbFzT+x8IARfVlaWyt///vdVtsvrL1++3HWMJ554QuWg71nBlQEAACKOzgAAABFHZwAAgIjrlnMG7FjwTTfdpLKdQ3DRRRepbMcFT5w4obJdI0BE5Pbbb1f5ggsuaLNNdj/tH//4xyofO3bMdQ50jp0z8Nprr6nct29flSdNmqTy3r17Vd65c6fK27dvd51z27ZtKtv1JhAuR44cUdnO5Wnts+CMM87ws0nwgV1DxH4W2L0IXn/9dZUff/xx1zHtPiRBx5UBAAAijs4AAAARR2cAAICI65ZzBhoaGlS24/PTpk1TeeHChSp/9atfVdnOIbBjzSIip512msp2L+u1a9eq/Mgjj6hsx5rhPTuGZ38GNgP2d3///v0qp6SkuF6TmZnpa5vgvQsvvFBlu8fIBx98oPIDDzygcnfYS4YrAwAARBydAQAAIo7OAAAAEdct5wzY+8tffPFFlfft26eyXWfArglg97U/dOiQ65xlZWUq23vQt27dqrJdr7y1/bABBMtPf/pTlXv37u16jl2PAsFj53hNmTJFZbvvyLJly1S2P2O7h0UYcWUAAICIozMAAEDE0RkAACDiEpyADXbU1NRIampqvJvRLVRXV7d6H3QYUAfeoAYQ5hoQ8acOhg8frvJTTz2l8u7du1W+++67Va6rq/O0PbHQXh1wZQAAgIijMwAAQMTRGQAAIOK65ToDAACczIEDB1S+5557VC4vL1c5jHMEOosrAwAARBydAQAAIi5wwwQBu9Mx1ML8Xoa57UES5vcxzG0PkrC/j3603x7zxIkTKjc3N3t+znhr730MXGegtrY23k3oNmpra0N7nzZ14A1qAGGuARF/6qCxsVHl4uJiz88RNO3VQeAWHWpubpaysjJxHEeysrKktLQ0sAtm1NTUSGZmZuDa6DiO1NbWSkZGhiQmhnMkKCx1QA34hxromu5QAyLUQVd1tA4Cd2UgMTFRhg4dKjU1NSIikpKSEqg3tjVBbGOY/ycgEr46CGL7qIHYCmL7wl4DItSBFzpSB+HtLgIAAE/QGQAAIOIC2xlITk6WxYsXS3JycrybclJhaGPYBf09Dnr7uoOgv8dBb193EfT3Oejta0/gJhACAIDYCuyVAQAAEBt0BgAAiDg6AwAARBydAQAAIo7OAAAAERfYzsDKlStl+PDh0qtXL8nNzZWtW7fGpR0bN26UadOmSUZGhiQkJMi6devU9x3HkUWLFsngwYOld+/ekpeXJ7t3745LW7uboNSACHUQT0GpA2ogfqgB/wWyM7BmzRqZO3euLF68WN566y0ZPXq0TJkyRQ4ePBjzttTX18vo0aNl5cqVrX5/2bJlsmLFClm1apUUFRXJqaeeKlOmTJFjx47FuKXdS5BqQIQ6iJcg1QE1EB/UQIw4AZSTk+PMmjWrJTc1NTkZGRlOfn5+HFvlOCLi/PKXv2zJzc3NTnp6uvPggw+2PFZVVeUkJyc7zz//fBxa2H0EtQYchzqIpaDWATUQO9RAbATuykBjY6Ns27ZN8vLyWh5LTEyUvLw82bx5cxxb5lZSUiLl5eWqrampqZKbmxu4toZJmGpAhDrwS5jqgBrwBzUQO4HrDFRWVkpTU5OkpaWpx9PS0qS8vDxOrWrdx+0JQ1vDJEw1IEId+CVMdUAN+IMaiJ3AdQYAAEBsBa4zMHDgQOnRo4dUVFSoxysqKiQ9PT1OrWrdx+0JQ1vDJEw1IEId+CVMdUAN+IMaiJ3AdQaSkpJk7NixUlhY2PJYc3OzFBYWyvjx4+PYMrfs7GxJT09Xba2pqZGioqLAtTVMwlQDItSBX8JUB9SAP6iBGIr3DMbWvPDCC05ycrKzevVqZ+fOnc4dd9zhnHbaaU55eXnM21JbW+ts377d2b59uyMizvLly53t27c7e/fudRzHce6//37ntNNOc371q185b7/9tnPVVVc52dnZztGjR2Pe1u4kSDXgONRBvASpDqiB+KAGYiOQnQHHcZxHHnnEycrKcpKSkpycnBxny5YtcWnHm2++6YiI62v69OmO4/zf7SQLFy500tLSnOTkZOfSSy91du3aFZe2djdBqQHHoQ7iKSh1QA3EDzXgvwTHcZzYXYcAAABBE7g5AwAAILboDAAAEHF0BgAAiDg6AwAARBydAQAAIo7OAAAAEUdnAACAiKMzAABAxNEZAAAg4ugMAAAQcXQGAACIuP8H3x+NJ6RvSCQAAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import torchvision\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import brevitas.nn as qnn\n",
        "import torch.optim as optim\n",
        "import torchvision.transforms as F\n",
        "\n",
        "# Load MNIST data\n",
        "\n",
        "\n",
        "train = torchvision.datasets.MNIST('data', train=True, download=True, transform=\n",
        "        F.Compose([F.Resize(20), F.ToTensor(), F.Lambda(lambda x: torch.flatten(x))]))\n",
        "\n",
        "test = torchvision.datasets.MNIST('data', train=True, download=True, transform=\n",
        "        F.Compose([F.Resize(20), F.ToTensor(), F.Lambda(lambda x: torch.flatten(x))]))\n",
        "\n",
        "# each sample becomes a vector of values 0-1\n",
        "X_train = torch.empty((60000, 400), dtype=torch.float32)\n",
        "X_test = torch.empty((60000, 400), dtype=torch.float32)\n",
        "\n",
        "for i in range(60000):\n",
        "  pil1, label1 = train[i]\n",
        "  pil2, label2 = train[i]\n",
        "  X_train[i] = pil1\n",
        "  X_test[i]  = pil2\n",
        "\n",
        "# X_train = train.data.reshape(-1, 784).float() / 255.0\n",
        "y_train = train.targets\n",
        "# X_test = test.data.reshape(-1, 784).float() / 255.0\n",
        "y_test = test.targets\n",
        "\n",
        "print(X_train.shape, y_train.shape)\n",
        "print(X_test.shape, y_test.shape)\n",
        "\n",
        "# Depict some samples of MNIST training data\n",
        "fig, ax = plt.subplots(4, 4, sharex=True, sharey=True)\n",
        "for i in range(4):\n",
        "    for j in range(4):\n",
        "        ax[i][j].imshow(X_train[4*i+j].reshape(20, 20), cmap=\"gray\")\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zeONkDVmhvvn"
      },
      "source": [
        "**Model definition and training**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rJHes8jWZuQl"
      },
      "outputs": [],
      "source": [
        "from brevitas.nn import QuantIdentity, QuantReLU, QuantLinear\n",
        "BW = 2 # ---- bit width\n",
        "# ---- Model\n",
        "input_dim = 400\n",
        "l1_out_dim = 200\n",
        "l2_out_dim = 10\n",
        "class Baseline(nn.Module):\n",
        "    def __init__(self):\n",
        "        super().__init__()\n",
        "        self.quant_vec1 = QuantIdentity(return_quant_tensor=True, bias=False, bit_width=BW)\n",
        "        self.layer1 = qnn.QuantLinear(input_dim, l1_out_dim, return_quant_tensor=True, bias=False, weight_bit_width=BW)\n",
        "        self.act1 = qnn.QuantReLU(return_quant_tensor=True, bias=False, bit_width=BW)\n",
        "        self.layer2 = qnn.QuantLinear(l1_out_dim, l2_out_dim, return_quant_tensor=True, bias=False, weight_bit_width=BW)\n",
        "    def forward(self, x):\n",
        "        x = self.act1(self.layer1(self.quant_vec1(x)))\n",
        "        x = self.layer2(x)\n",
        "        return x"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f2EcVYerrrHS",
        "outputId": "c8372464-2e21-44b8-da6d-67bbf34600a8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/torch/_tensor.py:1394: UserWarning: Named tensors and all their associated APIs are an experimental feature and subject to change. Please do not use them for anything important until they are released as stable. (Triggered internally at ../c10/core/TensorImpl.h:1908.)\n",
            "  return super().rename(names)\n",
            "<ipython-input-4-9c8b6f7b310b>:19: UserWarning: Defining your `__torch_function__` as a plain method is deprecated and will be an error in future, please define it as a classmethod. (Triggered internally at ../torch/csrc/utils/python_arg_parser.cpp:294.)\n",
            "  acc = (torch.argmax(y_pred, 1) == y_test).float().mean()\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 0: model accuracy 28.28%\n",
            "Epoch 1: model accuracy 55.49%\n",
            "Epoch 2: model accuracy 64.19%\n",
            "Epoch 3: model accuracy 75.86%\n",
            "Epoch 4: model accuracy 73.93%\n",
            "Epoch 5: model accuracy 81.77%\n",
            "Epoch 6: model accuracy 75.88%\n",
            "Epoch 7: model accuracy 79.28%\n",
            "Epoch 8: model accuracy 82.96%\n",
            "Epoch 9: model accuracy 83.86%\n"
          ]
        }
      ],
      "source": [
        "model = Baseline()\n",
        "\n",
        "optimizer = optim.SGD(model.parameters(), lr=0.01)\n",
        "loss_fn = torch.nn.CrossEntropyLoss()\n",
        "loader = torch.utils.data.DataLoader(list(zip(X_train, y_train)), shuffle=True, batch_size=100)\n",
        "\n",
        "n_epochs = 10\n",
        "for epoch in range(n_epochs):\n",
        "    model.train()\n",
        "    for X_batch, y_batch in loader:\n",
        "        y_pred = model(X_batch)\n",
        "        loss = loss_fn(y_pred, y_batch)\n",
        "        optimizer.zero_grad()\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "    # Validation\n",
        "    model.eval()\n",
        "    y_pred = model(X_test)\n",
        "    acc = (torch.argmax(y_pred, 1) == y_test).float().mean()\n",
        "    print(\"Epoch %d: model accuracy %.2f%%\" % (epoch, acc*100))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fCfdEOlQeU9d"
      },
      "outputs": [],
      "source": [
        "def ulppack_2D(inp, BW, tiled_inverse):\n",
        "  RN , CN = inp.shape\n",
        "  if(BW == 2):\n",
        "    div_factor = 4\n",
        "  elif(BW == 4):\n",
        "    div_factor = 2\n",
        "  Area_len = int(32 / div_factor)\n",
        "  RN_P = int(inp.shape[0] / div_factor)\n",
        "  outp = np.zeros([RN_P, CN], dtype=int)\n",
        "  if(tiled_inverse == True):\n",
        "    for k in range(CN):\n",
        "      for i in range(RN_P):\n",
        "        for j in range(div_factor):\n",
        "          outp[i, k] = outp[i, k] | (inp[div_factor*i+j, k]<<(Area_len*(div_factor-j-1)))\n",
        "  else:\n",
        "    for k in range(CN):\n",
        "      for i in range(RN_P):\n",
        "        for j in range(div_factor):\n",
        "          outp[i, k] = outp[i, k] | (inp[div_factor*i+j, k]<<(Area_len*j))\n",
        "  # print(outp)\n",
        "  return outp\n",
        "\n",
        "\n",
        "def ulppack_1D(inp, BW, tiled_inverse):\n",
        "  RN = inp.shape\n",
        "  if(BW == 2):\n",
        "    div_factor = 4\n",
        "  elif(BW == 4):\n",
        "    div_factor = 2\n",
        "  Area_len = int(32 / div_factor)\n",
        "  RN_P = int(inp.shape[0] / div_factor)\n",
        "  outp = np.zeros([RN_P,], dtype=int)\n",
        "  if(tiled_inverse == True):\n",
        "    for i in range(RN_P):\n",
        "      for j in range(div_factor):\n",
        "        outp[i] = outp[i] | (inp[div_factor*i+j]<<(Area_len*(div_factor-j-1)))\n",
        "  else:\n",
        "    for i in range(RN_P):\n",
        "      for j in range(div_factor):\n",
        "        outp[i] = outp[i] | (inp[div_factor*i+j]<<(Area_len*j))\n",
        "  # print(outp)\n",
        "  return outp\n",
        "\n",
        "def mat_Rshift_1D(inp, SN):\n",
        "  outp = np.zeros_like(inp)\n",
        "  RN = inp.shape[0]\n",
        "  for i in range(RN):\n",
        "    outp[i] = (inp[i]&0xff000000)>>SN\n",
        "  return outp\n",
        "\n",
        "def mat_Rshift_2D(inp, SN):\n",
        "  outp = np.zeros_like(inp)\n",
        "  RN, CN = inp.shape\n",
        "  for i in range(RN):\n",
        "    for j in range(CN):\n",
        "      outp[i, j] = (inp[i, j]&0xff000000)>>SN\n",
        "  return outp"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0zqMoSiLVKnd",
        "outputId": "9161e5e3-7c83-4487-c06a-b822721b2db1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input scale: 0.6014745\n",
            "FC1 min: -1.0\n",
            "FC1 max: 1.0\n",
            "FC1 scale: 0.20550515\n",
            "ACT1 scale: 0.8378324\n",
            "FC2 min: -1.0\n",
            "FC2 max: 1.0\n",
            "FC2 scale: 0.87515634\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "# ---- Extraction of QNN parameters\n",
        "# ---- QuantIdentity\n",
        "Input = X_train.detach().numpy().T\n",
        "Input_Scale = model.quant_vec1.quant_output_scale().detach().numpy()\n",
        "Input_Uint = np.floor(Input/Input_Scale).astype(int)\n",
        "Input_ULPPACK = ulppack_2D(Input_Uint, BW, False)\n",
        "\n",
        "print('Input scale:', Input_Scale)\n",
        "# ---- FC\n",
        "FC1_Weight = model.layer1.quant_weight()[0].detach().numpy().T\n",
        "FC1_Scale = model.layer1.quant_weight_scale().detach().numpy()\n",
        "FC1_Weight_Int = FC1_Weight/FC1_Scale\n",
        "FC1_Weight_Bias = - FC1_Weight_Int.min()\n",
        "FC1_Weight_UInt = (FC1_Weight_Int + FC1_Weight_Bias).astype(int)\n",
        "FC1_Weight_ULPPACK = ulppack_2D(FC1_Weight_UInt, BW, True)\n",
        "\n",
        "print('FC1 min:', FC1_Weight_Int.min())\n",
        "print('FC1 max:', FC1_Weight_Int.max())\n",
        "print('FC1 scale:', FC1_Scale)\n",
        "# ---- Act\n",
        "ACT1_Scale = model.act1.quant_output_scale().detach().numpy()\n",
        "\n",
        "print('ACT1 scale:', ACT1_Scale)\n",
        "# ---- FC\n",
        "FC2_Weight = model.layer2.quant_weight()[0].detach().numpy().T\n",
        "FC2_Scale = model.layer2.quant_weight_scale().detach().numpy()\n",
        "FC2_Weight_Int = FC2_Weight/FC2_Scale\n",
        "FC2_Weight_Bias = - FC2_Weight_Int.min()\n",
        "FC2_Weight_UInt = (FC2_Weight_Int + FC2_Weight_Bias).astype(int)\n",
        "FC2_Weight_ULPPACK = ulppack_2D(FC2_Weight_UInt, BW, True)\n",
        "\n",
        "print('FC2 min:', FC2_Weight_Int.min())\n",
        "print('FC2 max:', FC2_Weight_Int.max())\n",
        "print('FC2 scale:', FC2_Scale)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2Nj4jTXYLTHd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 499
        },
        "outputId": "0ed9eefe-f60d-4e29-e4d7-d5332925fc4c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[2.73077943e-04 4.62768761e-02 2.46380035e-03 8.69205681e-01\n",
            " 1.61142171e-06 2.22292292e-02 1.31173767e-04 1.06778736e-02\n",
            " 4.62768761e-02 2.46380035e-03]\n",
            "True: 3 Predicted: 3\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAa4AAAGdCAYAAABKG5eZAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAlqUlEQVR4nO3df3BV9Z3/8ddNIDfAJjcokORK+GUFKkLQbMmG6hcsKSHLILAtYsYtQRE7TphVWbrIjhpanU39sdpWGGS3QHTYijCjsCNMXIj8qBCkELICKkPYkMDADYU1NyRIyCaf7x8dbnvLvYFbzk3yuXk+Zj4znnM+5513Pt7kxbn35h6XMcYIAABLxHV1AwAARILgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYpVdXN+CE9vZ2nT17VklJSXK5XF3dDgAgQsYYXbp0SV6vV3FxHV9TxURwnT17VhkZGV3dBgDgFp0+fVqDBw/ucE5MPFWYlJTU1S0AABxwM7/PYyK4eHoQAGLDzfw+j4ngAgD0HAQXAMAqUQuulStXatiwYUpMTFR2drYOHDjQ4fxNmzZp9OjRSkxM1NixY7Vt27ZotQYAsJmJgg0bNpiEhASzdu1ac+zYMbNw4UKTkpJi6uvrQ87fu3eviY+PN6+++qr54osvzPPPP2969+5tjhw5clNfz+/3G0kMBoPBsHz4/f4b/s6PSnBNmDDBFBUVBbbb2tqM1+s1JSUlIec//PDDZvr06UH7srOzzY9//OOb+noEF4PBYMTGuJngcvypwqtXr+rQoUPKzc0N7IuLi1Nubq4qKipCnlNRURE0X5Ly8vLCzm9paVFjY2PQAAD0DI4H14ULF9TW1qbU1NSg/ampqfL5fCHP8fl8Ec0vKSmRx+MJDP74GAB6DivfVbhs2TL5/f7AOH36dFe3BADoJI5/5NOAAQMUHx+v+vr6oP319fVKS0sLeU5aWlpE891ut9xutzMNAwCs4vgVV0JCgrKyslReXh7Y197ervLycuXk5IQ8JycnJ2i+JG3fvj3sfABAD3ZTb9uL0IYNG4zb7TalpaXmiy++ME8++aRJSUkxPp/PGGPMj370I/Pcc88F5u/du9f06tXLvP766+bLL780xcXFvB2ewWAweuDosrfDG2PMW2+9ZYYMGWISEhLMhAkTzP79+wPHJk2aZAoLC4Pmb9y40YwcOdIkJCSYMWPGmK1bt9701yK4GAwGIzbGzQSXyxhjZLnGxkZ5PJ6ubgMAcIv8fr+Sk5M7nGPluwoBAD0XwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsEqvrm4AcIrL5XKsVlJSkiN1brvtNkfqSFJbW5sjda5cueJIHUlqaGhwpE5ra6sjddAzcMUFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsIrjwVVSUqLvfOc7SkpK0qBBgzRr1iwdP368w3NKS0vlcrmCRmJiotOtAQBigOPBtXv3bhUVFWn//v3avn27WltbNXXqVDU3N3d4XnJyss6dOxcYtbW1TrcGAIgBjt9IsqysLGi7tLRUgwYN0qFDh/T//t//C3uey+VSWlqa0+0AAGJM1O+A7Pf7Jd34TrBNTU0aOnSo2tvbdd999+lf/uVfNGbMmJBzW1pa1NLSEthubGx0rmFYKysry7FaCxcudKTOpEmTHKkj/fFn6VadP3/ekTqS9NOf/tSROgcPHnSkDnqGqL45o729Xc8884y++93v6p577gk7b9SoUVq7dq22bNmi9evXq729XRMnTtSZM2dCzi8pKZHH4wmMjIyMaH0LAIBuJqrBVVRUpKNHj2rDhg0dzsvJydG8efM0fvx4TZo0SR988IEGDhyo1atXh5y/bNky+f3+wDh9+nQ02gcAdENRe6pw0aJF+uijj7Rnzx4NHjw4onN79+6te++9V9XV1SGPu91uud1uJ9oEAFjG8SsuY4wWLVqkDz/8UJ988omGDx8ecY22tjYdOXJE6enpTrcHALCc41dcRUVF+s1vfqMtW7YoKSlJPp9PkuTxeNSnTx9J0rx583THHXeopKREkvSzn/1Mf/M3f6Nvfetbamho0Guvvaba2lo98cQTTrcHALCc48G1atUqSdLkyZOD9q9bt07z58+XJNXV1Sku7o8Xe19//bUWLlwon8+n/v37KysrS/v27dPdd9/tdHsAAMs5HlzGmBvO2bVrV9D2m2++qTfffNPpVgAAMYjPKgQAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYJWr34wI6m8fjcazWD3/4Q0fq9O/f35E60s19DujN+NMPuL5V7e3tjtT5h3/4B0fqSFJtba1jtdA9ccUFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsAp3QEbMqKysdKzWpk2bHKkzb948R+pI0uXLlx2pc/vttztSR5LuvfdeR+qkp6c7UkfiDsg9AVdcAACrEFwAAKsQXAAAqxBcAACrEFwAAKs4HlzLly+Xy+UKGqNHj+7wnE2bNmn06NFKTEzU2LFjtW3bNqfbAgDEiKhccY0ZM0bnzp0LjE8//TTs3H379qmgoEALFizQ4cOHNWvWLM2aNUtHjx6NRmsAAMtFJbh69eqltLS0wBgwYEDYub/85S81bdo0/eQnP9G3v/1tvfTSS7rvvvu0YsWKaLQGALBcVILrxIkT8nq9GjFihB599FHV1dWFnVtRUaHc3NygfXl5eaqoqAh7TktLixobG4MGAKBncDy4srOzVVpaqrKyMq1atUo1NTV64IEHdOnSpZDzfT6fUlNTg/alpqbK5/OF/RolJSXyeDyBkZGR4ej3AADovhwPrvz8fM2ZM0fjxo1TXl6etm3bpoaGBm3cuNGxr7Fs2TL5/f7AOH36tGO1AQDdW9Q/qzAlJUUjR45UdXV1yONpaWmqr68P2ldfX6+0tLSwNd1ut9xut6N9AgDsEPW/42pqatLJkyfDfohmTk6OysvLg/Zt375dOTk50W4NAGAhx4NryZIl2r17t06dOqV9+/Zp9uzZio+PV0FBgaQ/fFr2smXLAvOffvpplZWV6V//9V/11Vdfafny5Tp48KAWLVrkdGsAgBjg+FOFZ86cUUFBgS5evKiBAwfq/vvv1/79+zVw4EBJUl1dneLi/piXEydO1G9+8xs9//zz+ud//mfddddd2rx5s+655x6nWwMAxADHg2vDhg0dHt+1a9d1++bMmaM5c+Y43QoAIAbxWYUAAKsQXAAAq0T97fBAZ/n6668dq7V+/XpH6tx3332O1JGkUaNGOVbLKZcvX3akzsmTJx2pg56BKy4AgFUILgCAVQguAIBVCC4AgFUILgCAVQguAIBVCC4AgFUILgCAVQguAIBVCC4AgFUILgCAVQguAIBVCC4AgFUILgCAVQguAIBVCC4AgFUILgCAVbgDMhDC559/7kidl19+2ZE6kvTGG284Uic5OdmROpJ02223OVInOzvbkTqS9NFHHzlWC90TV1wAAKsQXAAAqxBcAACrEFwAAKsQXAAAqxBcAACrEFwAAKsQXAAAqxBcAACrEFwAAKsQXAAAqxBcAACrEFwAAKsQXAAAqzgeXMOGDZPL5bpuFBUVhZxfWlp63dzExESn2wIAxAjH78f1u9/9Tm1tbYHto0eP6vvf/77mzJkT9pzk5GQdP348sO1yuZxuCwAQIxwProEDBwZt//znP9edd96pSZMmhT3H5XIpLS3N6VYAADEoqq9xXb16VevXr9fjjz/e4VVUU1OThg4dqoyMDM2cOVPHjh2LZlsAAIs5fsX1pzZv3qyGhgbNnz8/7JxRo0Zp7dq1GjdunPx+v15//XVNnDhRx44d0+DBg0Oe09LSopaWlsB2Y2Oj063DQuEeL3+Jjh6zkZg8ebIjdSRnvz+nNDc3O1KnpqbGkTroGaJ6xbVmzRrl5+fL6/WGnZOTk6N58+Zp/PjxmjRpkj744AMNHDhQq1evDntOSUmJPB5PYGRkZESjfQBANxS14KqtrdWOHTv0xBNPRHRe7969de+996q6ujrsnGXLlsnv9wfG6dOnb7VdAIAlohZc69at06BBgzR9+vSIzmtra9ORI0eUnp4edo7b7VZycnLQAAD0DFEJrvb2dq1bt06FhYXq1Sv4ZbR58+Zp2bJlge2f/exn+q//+i/9z//8jyorK/X3f//3qq2tjfhKDQDQM0TlzRk7duxQXV2dHn/88euO1dXVKS7uj3n59ddfa+HChfL5fOrfv7+ysrK0b98+3X333dFoDQBguagE19SpU2WMCXls165dQdtvvvmm3nzzzWi0AQCIQXxWIQDAKgQXAMAqBBcAwCoEFwDAKgQXAMAqBBcAwCoEFwDAKgQXAMAqBBcAwCoEFwDAKgQXAMAqUb0DMtCZsrKyHKv17LPPOlKnf//+jtSRJJfL5VgtpyQkJDhSZ+DAgY7UQc/AFRcAwCoEFwDAKgQXAMAqBBcAwCoEFwDAKgQXAMAqBBcAwCoEFwDAKgQXAMAqBBcAwCoEFwDAKgQXAMAqBBcAwCoEFwDAKgQXAMAqBBcAwCoEFwDAKgQXAMAqvbq6AcApFRUVjtVas2aNI3UefPBBR+pIUnx8vCN10tPTHakjSX379nWkzpAhQxypg56BKy4AgFUILgCAVQguAIBVCC4AgFUILgCAVSIOrj179mjGjBnyer1yuVzavHlz0HFjjF588UWlp6erT58+ys3N1YkTJ25Yd+XKlRo2bJgSExOVnZ2tAwcORNoaAKAHiDi4mpublZmZqZUrV4Y8/uqrr+pXv/qV3n77bX322Wfq16+f8vLydOXKlbA133//fS1evFjFxcWqrKxUZmam8vLydP78+UjbAwDEuIiDKz8/Xy+//LJmz5593TFjjH7xi1/o+eef18yZMzVu3Di9++67Onv27HVXZn/qjTfe0MKFC/XYY4/p7rvv1ttvv62+fftq7dq1kbYHAIhxjr7GVVNTI5/Pp9zc3MA+j8ej7OzssH8cevXqVR06dCjonLi4OOXm5oY9p6WlRY2NjUEDANAzOBpcPp9PkpSamhq0PzU1NXDsz124cEFtbW0RnVNSUiKPxxMYGRkZDnQPALCBle8qXLZsmfx+f2CcPn26q1sCAHQSR4MrLS1NklRfXx+0v76+PnDszw0YMEDx8fERneN2u5WcnBw0AAA9g6PBNXz4cKWlpam8vDywr7GxUZ999plycnJCnpOQkKCsrKygc9rb21VeXh72HABAzxXxp8M3NTWpuro6sF1TU6OqqirddtttGjJkiJ555hm9/PLLuuuuuzR8+HC98MIL8nq9mjVrVuCcKVOmaPbs2Vq0aJEkafHixSosLNRf//Vfa8KECfrFL36h5uZmPfbYY7f+HQIAYkrEwXXw4MGgWzUsXrxYklRYWKjS0lL90z/9k5qbm/Xkk0+qoaFB999/v8rKypSYmBg45+TJk7pw4UJge+7cufr973+vF198UT6fT+PHj1dZWdl1b9gAAMBljDFd3cStamxslMfj6eo20MUGDRrkWK0lS5Y4UifW78f1p/8gvRVPP/20I3Uk6d1333WsFjqf3++/4fsWrHxXIQCg5+IOyIgZTn5E2Msvv+xInX/7t39zpI4k9erlzI/rQw895EgdSXr22WcdqePkywK9e/d2pE5ra6sjdeA8rrgAAFYhuAAAViG4AABWIbgAAFYhuAAAViG4AABWIbgAAFYhuAAAViG4AABWIbgAAFYhuAAAViG4AABWIbgAAFYhuAAAViG4AABWIbgAAFYhuAAAViG4AABWceZe4ECMaWxs7FZ1JKlPnz6O1ElLS3OkjiT17dvXkTrx8fGO1JGkuDj+PR7r+D8MALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsArBBQCwCsEFALAKwQUAsErEwbVnzx7NmDFDXq9XLpdLmzdvDhxrbW3V0qVLNXbsWPXr109er1fz5s3T2bNnO6y5fPlyuVyuoDF69OiIvxkAQOyLOLiam5uVmZmplStXXnfs8uXLqqys1AsvvKDKykp98MEHOn78uB566KEb1h0zZozOnTsXGJ9++mmkrQEAeoCIbySZn5+v/Pz8kMc8Ho+2b98etG/FihWaMGGC6urqNGTIkPCN9Orl6A3uAACxKep3QPb7/XK5XEpJSelw3okTJ+T1epWYmKicnByVlJSEDbqWlha1tLQEtp28y2ysS0pKcqROZmamI3Uk6YsvvnCkzv/+7/86UsdJLpfLsVpZWVmO1Jk9e7YjdSTn1vyjjz5ypI6koN8NiE1RfXPGlStXtHTpUhUUFCg5OTnsvOzsbJWWlqqsrEyrVq1STU2NHnjgAV26dCnk/JKSEnk8nsDIyMiI1rcAAOhmohZcra2tevjhh2WM0apVqzqcm5+frzlz5mjcuHHKy8vTtm3b1NDQoI0bN4acv2zZMvn9/sA4ffp0NL4FAEA3FJWnCq+FVm1trT755JMOr7ZCSUlJ0ciRI1VdXR3yuNvtltvtdqJVAIBlHL/iuhZaJ06c0I4dO3T77bdHXKOpqUknT55Uenq60+0BACwXcXA1NTWpqqpKVVVVkqSamhpVVVWprq5Ora2t+uEPf6iDBw/qP/7jP9TW1iafzyefz6erV68GakyZMkUrVqwIbC9ZskS7d+/WqVOntG/fPs2ePVvx8fEqKCi49e8QABBTIn6q8ODBg3rwwQcD24sXL5YkFRYWavny5frP//xPSdL48eODztu5c6cmT54sSTp58qQuXLgQOHbmzBkVFBTo4sWLGjhwoO6//37t379fAwcOjLQ9AECMizi4Jk+eLGNM2OMdHbvm1KlTQdsbNmyItA0AQA/FZxUCAKxCcAEArEJwAQCsQnABAKxCcAEArEJwAQCsQnABAKxCcAEArEJwAQCsQnABAKxCcAEArBKV+3Gh+5o+fbojdZ599llH6kjSpk2bHKmzbds2R+pIf7h7txPGjh3rSB1JWrBggSN1Bg8e7EgdSfr1r3/tSJ1w994DQuGKCwBgFYILAGAVggsAYBWCCwBgFYILAGAVggsAYBWCCwBgFYILAGAVggsAYBWCCwBgFYILAGAVggsAYBWCCwBgFYILAGAVggsAYBWCCwBgFYILAGAV7oDcw3z11VeO1Dl//rwjdSSpqKjIkTqPPfaYI3Uk6ZtvvnGkzrBhwxypI0lJSUmO1Nm/f78jdSTpl7/8pSN1nLrjNHoGrrgAAFYhuAAAViG4AABWIbgAAFYhuAAAVok4uPbs2aMZM2bI6/XK5XJp8+bNQcfnz58vl8sVNKZNm3bDuitXrtSwYcOUmJio7OxsHThwINLWAAA9QMTB1dzcrMzMTK1cuTLsnGnTpuncuXOB8d5773VY8/3339fixYtVXFysyspKZWZmKi8vz9G3XAMAYkPEf8eVn5+v/Pz8Due43W6lpaXddM033nhDCxcuDPwdzttvv62tW7dq7dq1eu655yJtEQAQw6LyGteuXbs0aNAgjRo1Sk899ZQuXrwYdu7Vq1d16NAh5ebm/rGpuDjl5uaqoqIi5DktLS1qbGwMGgCAnsHx4Jo2bZreffddlZeX65VXXtHu3buVn5+vtra2kPMvXLigtrY2paamBu1PTU2Vz+cLeU5JSYk8Hk9gZGRkOP1tAAC6Kcc/8umRRx4J/PfYsWM1btw43Xnnndq1a5emTJniyNdYtmyZFi9eHNhubGwkvACgh4j62+FHjBihAQMGqLq6OuTxAQMGKD4+XvX19UH76+vrw75O5na7lZycHDQAAD1D1IPrzJkzunjxotLT00MeT0hIUFZWlsrLywP72tvbVV5erpycnGi3BwCwTMTB1dTUpKqqKlVVVUmSampqVFVVpbq6OjU1NeknP/mJ9u/fr1OnTqm8vFwzZ87Ut771LeXl5QVqTJkyRStWrAhsL168WP/+7/+ud955R19++aWeeuopNTc3O/pp3wCA2BDxa1wHDx7Ugw8+GNi+9lpTYWGhVq1apc8//1zvvPOOGhoa5PV6NXXqVL300ktyu92Bc06ePKkLFy4EtufOnavf//73evHFF+Xz+TR+/HiVlZVd94YNAAAiDq7JkyfLGBP2+Mcff3zDGqdOnbpu36JFi7Ro0aJI2wEA9DB8ViEAwCoEFwDAKi7T0fN+lmhsbJTH4+nqNqyQkJDgSJ1x48Y5UkeSvv/97ztSZ9KkSY7UkaQxY8Y4UueOO+5wpI4kffXVV47UWbhwoSN1JGnv3r2O1QIkye/33/BPnLjiAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiFOyCjy/3VX/2VI3X69evnSB1JSkpKcqRO3759Hakj/eFx7oQzZ844UkeS/u///s+xWoDEHZABADGI4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWCXi4NqzZ49mzJghr9crl8ulzZs3Bx13uVwhx2uvvRa25vLly6+bP3r06Ii/GQBA7Is4uJqbm5WZmamVK1eGPH7u3LmgsXbtWrlcLv3gBz/osO6YMWOCzvv0008jbQ0A0AP0ivSE/Px85efnhz2elpYWtL1lyxY9+OCDGjFiRMeN9Op13bkAAPy5qL7GVV9fr61bt2rBggU3nHvixAl5vV6NGDFCjz76qOrq6sLObWlpUWNjY9AAAPQQ5hZIMh9++GHY46+88orp37+/+eabbzqss23bNrNx40bz3//936asrMzk5OSYIUOGmMbGxpDzi4uLjSQGg8FgxNjw+/03zp4bzujoZHUcXKNGjTKLFi2KuO7XX39tkpOTza9//euQx69cuWL8fn9gnD59ussXm8FgMBi3Pm4muCJ+jetm/fa3v9Xx48f1/vvvR3xuSkqKRo4cqerq6pDH3W633G73rbYIALBQ1F7jWrNmjbKyspSZmRnxuU1NTTp58qTS09Oj0BkAwGYRB1dTU5OqqqpUVVUlSaqpqVFVVVXQmykaGxu1adMmPfHEEyFrTJkyRStWrAhsL1myRLt379apU6e0b98+zZ49W/Hx8SooKIi0PQBArIv09aedO3eGfF6ysLAwMGf16tWmT58+pqGhIWSNoUOHmuLi4sD23LlzTXp6uklISDB33HGHmTt3rqmurr7pnvx+f5c/L8tgMBiMWx838xqXyxhjZLnGxkZ5PJ6ubgMAcIv8fr+Sk5M7nMNnFQIArEJwAQCsQnABAKxCcAEArEJwAQCsQnABAKxCcAEArEJwAQCsQnABAKxCcAEArEJwAQCsQnABAKxCcAEArEJwAQCsQnABAKxCcAEArEJwAQCsQnABAKxCcAEArEJwAQCsQnABAKxCcAEArEJwAQCsQnABAKxCcAEArEJwAQCsQnABAKxCcAEArEJwAQCsEhPBZYzp6hYAAA64md/nMRFcly5d6uoWAAAOuJnf5y4TA5cr7e3tOnv2rJKSkuRyucLOa2xsVEZGhk6fPq3k5ORO7PDW0HfnsrVvyd7e6btzdce+jTG6dOmSvF6v4uI6vqbq1Uk9RVVcXJwGDx580/OTk5O7zf+sSNB357K1b8ne3um7c3W3vj0ez03Ni4mnCgEAPQfBBQCwSo8KLrfbreLiYrnd7q5uJSL03bls7Vuyt3f67ly29n1NTLw5AwDQc/SoKy4AgP0ILgCAVQguAIBVCC4AgFViLrhWrlypYcOGKTExUdnZ2Tpw4ECH8zdt2qTRo0crMTFRY8eO1bZt2zqp0z8oKSnRd77zHSUlJWnQoEGaNWuWjh8/3uE5paWlcrlcQSMxMbGTOv6D5cuXX9fD6NGjOzynq9dakoYNG3Zd3y6XS0VFRSHnd+Va79mzRzNmzJDX65XL5dLmzZuDjhtj9OKLLyo9PV19+vRRbm6uTpw4ccO6kf6MONl3a2urli5dqrFjx6pfv37yer2aN2+ezp4922HNv+Tx5mTfkjR//vzrepg2bdoN63bleksK+Xh3uVx67bXXwtbsjPW+FTEVXO+//74WL16s4uJiVVZWKjMzU3l5eTp//nzI+fv27VNBQYEWLFigw4cPa9asWZo1a5aOHj3aaT3v3r1bRUVF2r9/v7Zv367W1lZNnTpVzc3NHZ6XnJysc+fOBUZtbW0ndfxHY8aMCerh008/DTu3O6y1JP3ud78L6nn79u2SpDlz5oQ9p6vWurm5WZmZmVq5cmXI46+++qp+9atf6e2339Znn32mfv36KS8vT1euXAlbM9KfEaf7vnz5siorK/XCCy+osrJSH3zwgY4fP66HHnrohnUjebw53fc106ZNC+rhvffe67BmV6+3pKB+z507p7Vr18rlcukHP/hBh3Wjvd63xMSQCRMmmKKiosB2W1ub8Xq9pqSkJOT8hx9+2EyfPj1oX3Z2tvnxj38c1T47cv78eSPJ7N69O+ycdevWGY/H03lNhVBcXGwyMzNven53XGtjjHn66afNnXfeadrb20Me7w5rbYwxksyHH34Y2G5vbzdpaWnmtddeC+xraGgwbrfbvPfee2HrRPoz4nTfoRw4cMBIMrW1tWHnRPp4u1Wh+i4sLDQzZ86MqE53XO+ZM2ea733vex3O6ez1jlTMXHFdvXpVhw4dUm5ubmBfXFyccnNzVVFREfKcioqKoPmSlJeXF3Z+Z/D7/ZKk2267rcN5TU1NGjp0qDIyMjRz5kwdO3asM9oLcuLECXm9Xo0YMUKPPvqo6urqws7tjmt99epVrV+/Xo8//niHH87cHdb6z9XU1Mjn8wWtqcfjUXZ2dtg1/Ut+RjqD3++Xy+VSSkpKh/MiebxFy65duzRo0CCNGjVKTz31lC5evBh2bndc7/r6em3dulULFiy44dzusN7hxExwXbhwQW1tbUpNTQ3an5qaKp/PF/Icn88X0fxoa29v1zPPPKPvfve7uueee8LOGzVqlNauXastW7Zo/fr1am9v18SJE3XmzJlO6zU7O1ulpaUqKyvTqlWrVFNTowceeCDsLQm621pL0ubNm9XQ0KD58+eHndMd1jqUa+sWyZr+JT8j0XblyhUtXbpUBQUFHX7Ya6SPt2iYNm2a3n33XZWXl+uVV17R7t27lZ+fr7a2tpDzu+N6v/POO0pKStLf/d3fdTivO6x3R2Li0+FjRVFRkY4ePXrD55JzcnKUk5MT2J44caK+/e1va/Xq1XrppZei3aYkKT8/P/Df48aNU3Z2toYOHaqNGzfe1L/muoM1a9YoPz9fXq837JzusNaxqrW1VQ8//LCMMVq1alWHc7vD4+2RRx4J/PfYsWM1btw43Xnnndq1a5emTJnSKT3cqrVr1+rRRx+94RuMusN6dyRmrrgGDBig+Ph41dfXB+2vr69XWlpayHPS0tIimh9NixYt0kcffaSdO3dGdIsWSerdu7fuvfdeVVdXR6m7G0tJSdHIkSPD9tCd1lqSamtrtWPHDj3xxBMRndcd1lpSYN0iWdO/5GckWq6FVm1trbZv3x7xrTVu9HjrDCNGjNCAAQPC9tCd1luSfvvb3+r48eMRP+al7rHefypmgishIUFZWVkqLy8P7Gtvb1d5eXnQv5j/VE5OTtB8Sdq+fXvY+dFgjNGiRYv04Ycf6pNPPtHw4cMjrtHW1qYjR44oPT09Ch3enKamJp08eTJsD91hrf/UunXrNGjQIE2fPj2i87rDWkvS8OHDlZaWFrSmjY2N+uyzz8Ku6V/yMxIN10LrxIkT2rFjh26//faIa9zo8dYZzpw5o4sXL4btobus9zVr1qxRVlaWMjMzIz63O6x3kK5+d4iTNmzYYNxutyktLTVffPGFefLJJ01KSorx+XzGGGN+9KMfmeeeey4wf+/evaZXr17m9ddfN19++aUpLi42vXv3NkeOHOm0np966inj8XjMrl27zLlz5wLj8uXLgTl/3vdPf/pT8/HHH5uTJ0+aQ4cOmUceecQkJiaaY8eOdVrf//iP/2h27dplampqzN69e01ubq4ZMGCAOX/+fMieu8NaX9PW1maGDBlili5det2x7rTWly5dMocPHzaHDx82kswbb7xhDh8+HHj33c9//nOTkpJitmzZYj7//HMzc+ZMM3z4cPPNN98Eanzve98zb731VmD7Rj8j0e776tWr5qGHHjKDBw82VVVVQY/5lpaWsH3f6PEW7b4vXbpklixZYioqKkxNTY3ZsWOHue+++8xdd91lrly5Erbvrl7va/x+v+nbt69ZtWpVyBpdsd63IqaCyxhj3nrrLTNkyBCTkJBgJkyYYPbv3x84NmnSJFNYWBg0f+PGjWbkyJEmISHBjBkzxmzdurVT+5UUcqxbty5s388880zge0xNTTV/+7d/ayorKzu177lz55r09HSTkJBg7rjjDjN37lxTXV0dtmdjun6tr/n444+NJHP8+PHrjnWntd65c2fIx8a1/trb280LL7xgUlNTjdvtNlOmTLnuexo6dKgpLi4O2tfRz0i0+66pqQn7mN+5c2fYvm/0eIt235cvXzZTp041AwcONL179zZDhw41CxcuvC6Autt6X7N69WrTp08f09DQELJGV6z3reC2JgAAq8TMa1wAgJ6B4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBYheACAFiF4AIAWIXgAgBY5f8Dk7LMQHTxMM8AAAAASUVORK5CYII=\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "Index = 10\n",
        "# ---- FC1\n",
        "QNN1_const1 = FC1_Weight_Bias * Input_Uint[:,Index].sum()\n",
        "QNN1_const2 = 0 * np.sum(FC1_Weight_UInt, axis=0)\n",
        "QNN1_const3 = l1_out_dim * 0 * FC1_Weight_Bias\n",
        "# QNN1 = Input_Scale * FC1_Scale * (np.matmul(Input_Uint[:,Index], FC1_Weight_UInt) - QNN1_const1 - QNN1_const2 + QNN1_const3)\n",
        "QNN1 = Input_Scale * FC1_Scale * (mat_Rshift_1D(np.matmul(Input_ULPPACK[:,Index], FC1_Weight_ULPPACK), 24) - QNN1_const1 - QNN1_const2 + QNN1_const3)\n",
        "\n",
        "# ---- ACT1\n",
        "QNN2_rectified = np.where(QNN1 < 0, 0, QNN1)\n",
        "QNN2 = np.round(QNN2_rectified / ACT1_Scale).astype(int)\n",
        "QNN2_ULPPACK = ulppack_1D(QNN2, BW, False)\n",
        "\n",
        "# ---- FC2\n",
        "QNN3_const1 = FC2_Weight_Bias * QNN2.sum()\n",
        "QNN3_const2 = 0 * np.sum(FC2_Weight_UInt, axis=0)\n",
        "QNN3_const3 = l2_out_dim * 0 * FC2_Weight_Bias\n",
        "# QNN3 = ACT1_Scale * FC2_Scale * (np.matmul(QNN2, FC2_Weight_UInt) - QNN3_const1 - QNN3_const2 + QNN3_const3)\n",
        "QNN3 = ACT1_Scale * FC2_Scale * (mat_Rshift_1D(np.matmul(QNN2_ULPPACK, FC2_Weight_ULPPACK), 24) - QNN3_const1 - QNN3_const2 + QNN3_const3)\n",
        "\n",
        "prediction = np.exp(QNN3)/np.exp(QNN3).sum()\n",
        "print(prediction)\n",
        "print('True:', y_train[Index].numpy(), 'Predicted:', prediction.argmax())\n",
        "# data = Input_Uint[:,7].reshape(28, 28)\n",
        "fig, ax = plt.subplots(1, 1, sharex=True, sharey=True)\n",
        "ax.imshow(Input[:,Index].reshape(20, 20), cmap=\"gray\")\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vmDaNGTXBrnR"
      },
      "source": [
        "# Write Parameters to file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H8JB0KGf9R3z"
      },
      "outputs": [],
      "source": [
        "# *********************************************************\n",
        "# ---- Constants\n",
        "file1 = open('parameters.h', 'w')\n",
        "\n",
        "file1.write('// ---- Constants\\n')\n",
        "file1.write('int QNN1_const1 = ')\n",
        "file1.write(str(int(QNN1_const1))+';\\n')\n",
        "\n",
        "RN = QNN1_const2.shape[0]\n",
        "\n",
        "file1.write('int QNN1_const2[' + str(RN) + '] = {')\n",
        "for i in range(RN):\n",
        "  file1.write(str(int(QNN1_const2[i])))\n",
        "  if(i<RN-1):\n",
        "    file1.write(',')\n",
        "  if(i%10==0):\n",
        "    file1.write('\\n')\n",
        "file1.write('};\\n')\n",
        "\n",
        "file1.write('int QNN1_const3 = ')\n",
        "file1.write(str(int(QNN1_const3))+';\\n')\n",
        "\n",
        "file1.write('int QNN3_const1 = ')\n",
        "file1.write(str(0)+';// Computing during run time\\n')\n",
        "\n",
        "RN = QNN3_const2.shape[0]\n",
        "\n",
        "file1.write('int QNN3_const2[' + str(RN) + '] = {')\n",
        "for i in range(RN):\n",
        "  file1.write(str(int(QNN3_const2[i])))\n",
        "  if(i<RN-1):\n",
        "    file1.write(',')\n",
        "  if(i%10==0):\n",
        "    file1.write('\\n')\n",
        "file1.write('};\\n')\n",
        "\n",
        "file1.write('int QNN3_const3 = ')\n",
        "file1.write(str(int(QNN3_const3))+';\\n')\n",
        "\n",
        "\n",
        "# *********************************************************\n",
        "# ---- Input\n",
        "RN, CN = Input_ULPPACK.shape\n",
        "\n",
        "file1.write('// ---- Input\\n')\n",
        "file1.write('float Input_Scale = ')\n",
        "file1.write(str(Input_Scale)+';\\n')\n",
        "file1.write('unsigned int Input_ULPPACK[')\n",
        "file1.write(str(RN) + '] = ')\n",
        "file1.write('{')\n",
        "for i in range(RN):\n",
        "  file1.write(str(Input_ULPPACK[i,Index]))\n",
        "  if(i<RN-1):\n",
        "    file1.write(',')\n",
        "  if(i%10==0):\n",
        "    file1.write('\\n')\n",
        "file1.write('};\\n')\n",
        "# *********************************************************\n",
        "# ---- FC1\n",
        "RN, CN = FC1_Weight_ULPPACK.shape\n",
        "\n",
        "file1.write('// ---- FC1\\n')\n",
        "file1.write('float FC1_Scale = ')\n",
        "file1.write(str(FC1_Scale)+';\\n')\n",
        "file1.write('int FC1_Weight_Bias = ')\n",
        "file1.write(str(int(FC1_Weight_Bias))+';\\n')\n",
        "\n",
        "\n",
        "file1.write('unsigned int FC1_ULPPACK[')\n",
        "file1.write(str(RN * CN) + '] = ')\n",
        "file1.write('{')\n",
        "for i in range(CN):\n",
        "  for j in range(RN):\n",
        "    file1.write(str(FC1_Weight_ULPPACK[j,i]))\n",
        "    if(i+j<CN+RN-2):\n",
        "      file1.write(',')\n",
        "    if(j%10==0):\n",
        "      file1.write('\\n')\n",
        "file1.write('};\\n')\n",
        "# *********************************************************\n",
        "# ---- ACT1\n",
        "RN, CN = FC1_Weight_ULPPACK.shape\n",
        "\n",
        "file1.write('// ---- ACT1\\n')\n",
        "file1.write('float ACT1_Scale = ')\n",
        "file1.write(str(ACT1_Scale)+';\\n')\n",
        "# *********************************************************\n",
        "# ---- FC2\n",
        "RN, CN = FC2_Weight_ULPPACK.shape\n",
        "\n",
        "file1.write('// ---- FC2\\n')\n",
        "file1.write('float FC2_Scale = ')\n",
        "file1.write(str(FC2_Scale)+';\\n')\n",
        "file1.write('int FC2_Weight_Bias = ')\n",
        "file1.write(str(int(FC2_Weight_Bias))+';\\n')\n",
        "file1.write('unsigned int FC2_ULPPACK[')\n",
        "file1.write(str(RN * CN) + '] = ')\n",
        "file1.write('{')\n",
        "for i in range(CN):\n",
        "  for j in range(RN):\n",
        "    file1.write(str(FC2_Weight_ULPPACK[j,i]))\n",
        "    if(i+j<CN+RN-2):\n",
        "      file1.write(',')\n",
        "    if(j%10==0):\n",
        "      file1.write('\\n')\n",
        "file1.write('};\\n')\n",
        "\n",
        "file1.close()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F_pUz6METFm1"
      },
      "source": [
        "# Keras2C test"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cQbhN1TMXEK9"
      },
      "outputs": [],
      "source": [
        "!wget https://github.com/f0uriest/keras2c/archive/refs/heads/master.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3HFIXMeBGdKP"
      },
      "outputs": [],
      "source": [
        "%cd /content\n",
        "!git clone https://github.com/f0uriest/keras2c.git"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QHgdzsmjTJtD",
        "outputId": "4d5bfbb6-07ce-4b72-98e5-32f495f5c76f"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/keras2c\n"
          ]
        }
      ],
      "source": [
        "%cd /content/keras2c"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dujWj_jGTQWB"
      },
      "outputs": [],
      "source": [
        "!pip install -r requirements.txt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "92WLEOs_rt2K"
      },
      "source": [
        "**Model parameters extraction**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3-vwrEAlsbH1",
        "outputId": "9824c073-bb76-46a6-ebc3-68d712139a2e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tensor(0.1333, grad_fn=<DivBackward0>)\n",
            "tensor(0.1462, grad_fn=<DivBackward0>)\n",
            "tensor(0.0077, grad_fn=<DivBackward0>)\n",
            "tensor(0.0219, grad_fn=<DivBackward0>)\n",
            "tensor(0.0688, grad_fn=<DivBackward0>)\n"
          ]
        }
      ],
      "source": [
        "print(model.quant_vec1.quant_output_scale())\n",
        "print(model.quant_vec2.quant_output_scale())\n",
        "print(model.layer1.quant_weight_scale())\n",
        "print(model.layer2.quant_weight_scale())\n",
        "print(model.act1.quant_output_scale())\n",
        "#model.layer2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "S6ms8MoGiL3S"
      },
      "outputs": [],
      "source": [
        "#quant_input = QuantIdentity(return_quant_tensor=True, bias=False, bit_width=BW)\n",
        "# quant_layer1 = QuantLinear(784, 784, return_quant_tensor=True, bias=False, input_bit_width = 4, weight_bit_width=BW)\n",
        "# quant_act1 = QuantReLU(return_quant_tensor=True, bias=False, bit_width=BW)\n",
        "# quant_layer2 = QuantLinear(784, 10, return_quant_tensor=True, bias=False, weight_bit_width=BW)\n",
        "\n",
        "# quant_layer1.quant_weight = model.layer1.quant_weight\n",
        "# quant_act1 = model.act1\n",
        "# quant_layer2.quant_weight = model.layer2.quant_weight\n",
        "\n",
        "#input = quant_input(X_train[141])\n",
        "# input = X_train[141]\n",
        "# olayer1 = quant_layer1(input)\n",
        "# oact1 = quant_act1(olayer1)\n",
        "# olayer2 = quant_layer2(oact1)\n",
        "# input\n",
        "\n",
        "#print(f\"Float input:\\n {X_train[0]} \\n\")\n",
        "#print(f\"Quant input:\\n {quant_input} \\n\")\n",
        "# print(f\"Quant output:\\n {quant_output}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-E21iH1rHG6M"
      },
      "outputs": [],
      "source": [
        "# quant_layer1 = QuantLinear(784, 784, return_quant_tensor=True, bias=False, input_bit_width = BW, weight_bit_width=BW)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nB1bzstnbU9H"
      },
      "outputs": [],
      "source": [
        "print(model.layer1.int_weight())\n",
        "print(model.layer1.quant_weight_scale())\n",
        "print(model.layer2.int_weight())\n",
        "print(model.layer2.quant_weight_scale())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kTcbwd-4XYD0",
        "outputId": "eb6acdb9-3637-4897-dc75-f75206c37899"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[ 0.7415085  -0.31655747  0.32580552  0.4280337   0.00816492  0.01103932\n",
            "  0.47023308 -0.02899381  0.15525857  0.12513998]\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "\n",
        "input = X_train[0].detach().numpy().T\n",
        "\n",
        "layer1 = model.layer1.quant_weight()[0].detach().numpy().T\n",
        "res_layer1 = np.matmul(input,layer1)\n",
        "res_layer1 = np.where(res_layer1 > 0, res_layer1, 0.0)\n",
        "layer2 = model.layer2.quant_weight()[0].detach().numpy().T\n",
        "res_layer2 = np.matmul(res_layer1,layer2)\n",
        "\n",
        "\n",
        "qlayer1_scale = model.layer1.quant_weight_scale().detach().numpy()\n",
        "qlayer1 = model.layer1.int_weight().float().detach().numpy().T\n",
        "qres_layer1 = np.matmul(input,qlayer1)*qlayer1_scale\n",
        "qres_layer1 = np.where(qres_layer1 > 0, qres_layer1, 0.0)\n",
        "qlayer2_scale = model.layer2.quant_weight_scale().detach().numpy()\n",
        "qlayer2 = model.layer2.int_weight().float().detach().numpy().T*qlayer2_scale\n",
        "qres_layer2 = np.matmul(qres_layer1,qlayer2)\n",
        "\n",
        "\n",
        "\n",
        "diff2 = qres_layer2\n",
        "print(diff2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7ZuwEOm1l4WV",
        "outputId": "0bd99a13-9b0b-4513-b62c-1ead8a81ac15"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "(784, 784)"
            ]
          },
          "execution_count": 196,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model.layer1.quant_weight()[0].detach().numpy().T.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q0GiLff5gOFF",
        "outputId": "c2e28e9b-65ec-4a5d-ea9f-8fd7204def9f"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "array([[0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       ...,\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.],\n",
              "       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32)"
            ]
          },
          "execution_count": 161,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "x = model.layer1.quant_weight()\n",
        "\n",
        "# dir(x)\n",
        "x[0].detach().numpy() - x[0].detach().numpy().T.T"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kNddi6sb0Bov"
      },
      "source": [
        "# Scratch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kS7n3WXq0Dwl",
        "outputId": "893da2a9-b99d-4dc7-bafc-b71fd0c62030"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Expected FP32 Y:\n",
            "[[242.46051  95.31735 217.99707 574.97864]\n",
            " [-88.28122 172.45425 216.39151 212.0112 ]]\n",
            "Expected FP32 Y Quantized:\n",
            "[[10  4  9 24]\n",
            " [-4  7  9  9]]\n",
            "Expected Quantized Y_q from Quantized Matrix Multiplication:\n",
            "[[10  4  9 25]\n",
            " [-4  7  9  9]]\n",
            "Expected Quantized Y_q from Quantized Matrix Multiplication Dequantized:\n",
            "[[235.29411   94.117645 211.76471  588.2353  ]\n",
            " [-94.117645 164.70589  211.76471  211.76471 ]]\n"
          ]
        }
      ],
      "source": [
        "import numpy as np\n",
        "\n",
        "\n",
        "def quantization(x, s, z, alpha_q, beta_q):\n",
        "\n",
        "    x_q = np.round(1 / s * x + z, decimals=0)\n",
        "    x_q = np.clip(x_q, a_min=alpha_q, a_max=beta_q)\n",
        "\n",
        "    return x_q\n",
        "\n",
        "\n",
        "def quantization_int8(x, s, z):\n",
        "\n",
        "    x_q = quantization(x, s, z, alpha_q=-128, beta_q=127)\n",
        "    x_q = x_q.astype(np.int8)\n",
        "\n",
        "    return x_q\n",
        "\n",
        "\n",
        "def dequantization(x_q, s, z):\n",
        "\n",
        "    # x_q - z might go outside the quantization range.\n",
        "    x_q = x_q.astype(np.int32)\n",
        "    x = s * (x_q - z)\n",
        "    x = x.astype(np.float32)\n",
        "\n",
        "    return x\n",
        "\n",
        "\n",
        "def generate_quantization_constants(alpha, beta, alpha_q, beta_q):\n",
        "\n",
        "    # Affine quantization mapping\n",
        "    s = (beta - alpha) / (beta_q - alpha_q)\n",
        "    z = int((beta * alpha_q - alpha * beta_q) / (beta - alpha))\n",
        "\n",
        "    return s, z\n",
        "\n",
        "\n",
        "def generate_quantization_int8_constants(alpha, beta):\n",
        "\n",
        "    b = 8\n",
        "    alpha_q = -2**(b - 1)\n",
        "    beta_q = 2**(b - 1) - 1\n",
        "\n",
        "    s, z = generate_quantization_constants(alpha=alpha,\n",
        "                                           beta=beta,\n",
        "                                           alpha_q=alpha_q,\n",
        "                                           beta_q=beta_q)\n",
        "\n",
        "    return s, z\n",
        "\n",
        "\n",
        "def quantization_matrix_multiplication_int8(X_q, W_q, b_q, s_X, z_X, s_W, z_W,\n",
        "                                            s_b, z_b, s_Y, z_Y):\n",
        "\n",
        "    p = W_q.shape[0]\n",
        "\n",
        "    # Y_q_simulated is FP32\n",
        "    Y_q_simulated = (z_Y + (s_b / s_Y * (b_q.astype(np.int32) - z_b)) + (\n",
        "        (s_X * s_W / s_Y) *\n",
        "        (np.matmul(X_q.astype(np.int32), W_q.astype(np.int32)) -\n",
        "         z_W * np.sum(X_q.astype(np.int32), axis=1, keepdims=True) - z_X *\n",
        "         np.sum(W_q.astype(np.int32), axis=0, keepdims=True) + p * z_X * z_W)))\n",
        "\n",
        "    Y_q_simulated = np.round(Y_q_simulated, decimals=0)\n",
        "    Y_q_simulated = np.clip(Y_q_simulated, a_min=-128, a_max=127)\n",
        "    Y_q_simulated = Y_q_simulated.astype(np.int8)\n",
        "\n",
        "    return Y_q_simulated\n",
        "\n",
        "\n",
        "def main():\n",
        "\n",
        "    # Set random seed for reproducibility\n",
        "    random_seed = 0\n",
        "    np.random.seed(random_seed)\n",
        "\n",
        "    # Random matrices\n",
        "    m = 2\n",
        "    p = 3\n",
        "    n = 4\n",
        "\n",
        "    # X\n",
        "    alpha_X = -100.0\n",
        "    beta_X = 80.0\n",
        "    s_X, z_X = generate_quantization_int8_constants(alpha=alpha_X, beta=beta_X)\n",
        "    X = np.random.uniform(low=alpha_X, high=beta_X,\n",
        "                          size=(m, p)).astype(np.float32)\n",
        "    X_q = quantization_int8(x=X, s=s_X, z=z_X)\n",
        "    X_q_dq = dequantization(x_q=X_q, s=s_X, z=z_X)\n",
        "\n",
        "    # W\n",
        "    alpha_W = -20.0\n",
        "    beta_W = 10.0\n",
        "    s_W, z_W = generate_quantization_int8_constants(alpha=alpha_W, beta=beta_W)\n",
        "    W = np.random.uniform(low=alpha_W, high=beta_W,\n",
        "                          size=(p, n)).astype(np.float32)\n",
        "    W_q = quantization_int8(x=W, s=s_W, z=z_W)\n",
        "    W_q_dq = dequantization(x_q=W_q, s=s_W, z=z_W)\n",
        "\n",
        "    # b\n",
        "    alpha_b = -500.0\n",
        "    beta_b = 500.0\n",
        "    s_b, z_b = generate_quantization_int8_constants(alpha=alpha_b, beta=beta_b)\n",
        "    b = np.random.uniform(low=alpha_b, high=beta_b,\n",
        "                          size=(1, n)).astype(np.float32)\n",
        "    b_q = quantization_int8(x=b, s=s_b, z=z_b)\n",
        "    b_q_dq = dequantization(x_q=b_q, s=s_b, z=z_b)\n",
        "\n",
        "    # Y\n",
        "    alpha_Y = -3000.0\n",
        "    beta_Y = 3000.0\n",
        "    s_Y, z_Y = generate_quantization_int8_constants(alpha=alpha_Y, beta=beta_Y)\n",
        "    Y_expected = np.matmul(X, W) + b\n",
        "    Y_q_expected = quantization_int8(x=Y_expected, s=s_Y, z=z_Y)\n",
        "\n",
        "    Y_expected_prime = np.matmul(X_q_dq, W_q_dq) + b_q_dq\n",
        "    Y_expected_prime_q = quantization_int8(x=Y_expected_prime, s=s_Y, z=z_Y)\n",
        "    Y_expected_prime_q_dq = dequantization(x_q=Y_expected_prime_q,\n",
        "                                           s=s_Y,\n",
        "                                           z=z_Y)\n",
        "\n",
        "    print(\"Expected FP32 Y:\")\n",
        "    print(Y_expected)\n",
        "    print(\"Expected FP32 Y Quantized:\")\n",
        "    print(Y_q_expected)\n",
        "\n",
        "    Y_q_simulated = quantization_matrix_multiplication_int8(X_q=X_q,\n",
        "                                                            W_q=W_q,\n",
        "                                                            b_q=b_q,\n",
        "                                                            s_X=s_X,\n",
        "                                                            z_X=z_X,\n",
        "                                                            s_W=s_W,\n",
        "                                                            z_W=z_W,\n",
        "                                                            s_b=s_b,\n",
        "                                                            z_b=z_b,\n",
        "                                                            s_Y=s_Y,\n",
        "                                                            z_Y=z_Y)\n",
        "    Y_simulated = dequantization(x_q=Y_q_simulated, s=s_Y, z=z_Y)\n",
        "\n",
        "    print(\"Expected Quantized Y_q from Quantized Matrix Multiplication:\")\n",
        "    print(Y_q_simulated)\n",
        "    print(\n",
        "        \"Expected Quantized Y_q from Quantized Matrix Multiplication Dequantized:\"\n",
        "    )\n",
        "    print(Y_simulated)\n",
        "\n",
        "    # Ensure the algorithm implementation is correct\n",
        "    assert (np.array_equal(Y_simulated, Y_expected_prime_q_dq))\n",
        "    assert (np.array_equal(Y_q_simulated, Y_expected_prime_q))\n",
        "\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "\n",
        "    main()\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyOYj1BDjFVPBS6ottclVt92",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}