diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 00000000..faeb4064 Binary files /dev/null and b/.DS_Store differ diff --git a/docs/.DS_Store b/docs/.DS_Store new file mode 100644 index 00000000..43040c5e Binary files /dev/null and b/docs/.DS_Store differ diff --git a/docs/examples/.DS_Store b/docs/examples/.DS_Store new file mode 100644 index 00000000..5008ddfc Binary files /dev/null and b/docs/examples/.DS_Store differ diff --git a/docs/tutorials/sconce_model_compression_pruning.rst b/docs/tutorials/sconce_model_compression_pruning.rst new file mode 100644 index 00000000..a636392b --- /dev/null +++ b/docs/tutorials/sconce_model_compression_pruning.rst @@ -0,0 +1,407 @@ +Model Pruning +============= + +A Detialed Explanation/Tutorial for Pruning on Deep Learning can be +found here:`Pruning Basics and DL implementation +walkthrough `__ + +snn Model Pruning +================= + +Install Packages +================ + +.. code:: ipython3 + + !pip install snntorch -q + !pip install sconce -q + %pip show sconce + + +.. parsed-literal:: + +  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 109.0/109.0 kB 1.1 MB/s eta 0:00:00 +  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 153.0/153.0 kB 2.0 MB/s eta 0:00:00 + [?25h Installing build dependencies ... [?25l[?25hdone + Getting requirements to build wheel ... [?25l[?25hdone + Installing backend dependencies ... [?25l[?25hdone + Preparing metadata (pyproject.toml) ... [?25l[?25hdone +  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 11.6/11.6 MB 70.2 MB/s eta 0:00:00 +  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 18.2/18.2 MB 69.5 MB/s eta 0:00:00 +  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.6/3.6 MB 114.1 MB/s eta 0:00:00 +  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.8/2.8 MB 113.5 MB/s eta 0:00:00 +  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 53.1/53.1 kB 5.4 MB/s eta 0:00:00 +  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 121.1/121.1 kB 16.0 MB/s eta 0:00:00 +  ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.6/1.6 MB 69.3 MB/s eta 0:00:00 + [?25h Building wheel for lit (pyproject.toml) ... [?25l[?25hdone + ERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts. + lida 0.0.10 requires fastapi, which is not installed. + lida 0.0.10 requires kaleido, which is not installed. + lida 0.0.10 requires python-multipart, which is not installed. + lida 0.0.10 requires uvicorn, which is not installed. + cupy-cuda11x 11.0.0 requires numpy<1.26,>=1.20, but you have numpy 1.26.2 which is incompatible. + imageio 2.31.6 requires pillow<10.1.0,>=8.3.2, but you have pillow 10.1.0 which is incompatible. + tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.8.0 which is incompatible. +  + +Import Libraries +================ + +.. code:: ipython3 + + from collections import defaultdict, OrderedDict + + import numpy as np + import torch + from torch import nn + from torch.optim import * + from torch.optim.lr_scheduler import * + from torch.utils.data import DataLoader + from torchvision.datasets import * + from torchvision.transforms import * + import torch.optim as optim + from sconce import sconce + + assert torch.cuda.is_available(), \ + "The current runtime does not have CUDA support." \ + "Please go to menu bar (Runtime - Change runtime type) and select GPU" + +**Spiking Neural Network Compression** +====================================== + +.. code:: ipython3 + + # Import snntorch libraries + import snntorch as snn + from snntorch import surrogate + from snntorch import backprop + from snntorch import functional as SF + from snntorch import utils + from snntorch import spikeplot as splt + from torch import optim + + import torch + import torch.nn as nn + from torch.utils.data import DataLoader + from torchvision import datasets, transforms + import torch.nn.functional as F + + import matplotlib.pyplot as plt + import numpy as np + import itertools + + + + +.. parsed-literal:: + + :4: DeprecationWarning: The module snntorch.backprop will be deprecated in a future release. Writing out your own training loop will lead to substantially faster performance. + from snntorch import backprop + + +Dataset +======= + +.. code:: ipython3 + + + # Event Drive Data + + # dataloader arguments + batch_size = 128 + data_path = "./data/mnist" + + dtype = torch.float + device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") + + # Define a transform + transform = transforms.Compose( + [ + transforms.Resize((28, 28)), + transforms.Grayscale(), + transforms.ToTensor(), + transforms.Normalize((0,), (1,)), + ] + ) + + mnist_train = datasets.MNIST(data_path, train=True, download=True, transform=transform) + mnist_test = datasets.MNIST(data_path, train=False, download=True, transform=transform) + + # Create DataLoaders + train_loader = DataLoader( + mnist_train, batch_size=batch_size, shuffle=True, drop_last=True + ) + test_loader = DataLoader( + mnist_test, batch_size=batch_size, shuffle=True, drop_last=True + ) + + + +.. parsed-literal:: + + Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz + Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./data/mnist/MNIST/raw/train-images-idx3-ubyte.gz + + +.. parsed-literal:: + + 100%|██████████| 9912422/9912422 [00:00<00:00, 82101508.40it/s] + + +.. parsed-literal:: + + Extracting ./data/mnist/MNIST/raw/train-images-idx3-ubyte.gz to ./data/mnist/MNIST/raw + + Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz + Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./data/mnist/MNIST/raw/train-labels-idx1-ubyte.gz + + +.. parsed-literal:: + + 100%|██████████| 28881/28881 [00:00<00:00, 111748795.04it/s] + + +.. parsed-literal:: + + Extracting ./data/mnist/MNIST/raw/train-labels-idx1-ubyte.gz to ./data/mnist/MNIST/raw + + Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz + Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ./data/mnist/MNIST/raw/t10k-images-idx3-ubyte.gz + + +.. parsed-literal:: + + 100%|██████████| 1648877/1648877 [00:00<00:00, 26490461.97it/s] + + +.. parsed-literal:: + + Extracting ./data/mnist/MNIST/raw/t10k-images-idx3-ubyte.gz to ./data/mnist/MNIST/raw + + Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz + Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ./data/mnist/MNIST/raw/t10k-labels-idx1-ubyte.gz + + +.. parsed-literal:: + + 100%|██████████| 4542/4542 [00:00<00:00, 6970555.71it/s] + + +.. parsed-literal:: + + Extracting ./data/mnist/MNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/mnist/MNIST/raw + + + +Instantiate an Object of sconce +=============================== + +.. code:: ipython3 + + + sconces = sconce() + + +Set you Dataloader +================== + +.. code:: ipython3 + + + dataloader = {} + dataloader["train"] = train_loader + dataloader["test"] = test_loader + sconces.dataloader = dataloader + +#Enable snn in sconce + +.. code:: ipython3 + + + sconces.snn = True + + +Load your snn Model +=================== + +.. code:: ipython3 + + spike_grad = surrogate.fast_sigmoid(slope=25) + beta = 0.5 + snn_model = nn.Sequential( + nn.Conv2d(1, 12, 5), + nn.MaxPool2d(2), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), + nn.Conv2d(12, 64, 5), + nn.MaxPool2d(2), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True), + nn.Flatten(), + nn.Linear(64 * 4 * 4, 10), + snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True, output=True), + ).to('cuda') + + + +Load the pretrained weights +=========================== + +.. code:: ipython3 + + snn_pretrained_model_path = "./snn_model.pth" + snn_model.load_state_dict(torch.load(snn_pretrained_model_path)) # Model Definition + sconces.model = snn_model + +Set the Optimizizer and Type of Pruning Operation to Perform on the model +========================================================================= + +.. code:: ipython3 + + + sconces.optimizer = optim.Adam(sconces.model.parameters(), lr=1e-4) + sconces.scheduler = optim.lr_scheduler.CosineAnnealingLR(sconces.optimizer, T_max=200) + + sconces.criterion = SF.ce_rate_loss() + + sconces.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + sconces.experiment_name = "snn-gmp" # Define your experiment name here + sconces.prune_mode = "GMP" + sconces.num_finetune_epochs = 1 + + +Test the Pre-Trained Model Accuracy +=================================== + +.. code:: ipython3 + + sconces.evaluate() + + +.. parsed-literal:: + + + + + +.. parsed-literal:: + + 97.11538461538461 + + + +Prune the Model +=============== + +The Compression does a series of steps as explained below: + +1. It evaluates the dense model accuracy +2. Given the model, the package finds the best parameters for pruning + such that the accuracy degradation is minimal. +3. The retreived optimal parameters from the above steps are used to + prune the model. +4. At times, certain pruning techniques might require a fine-tuning on + the dataset. For which the pruned model is fine-tuned on the dataset. +5. Pruned Model is saved and Compared for Latency, Paramater, MAC and + model size. + +.. code:: ipython3 + + sconces.compress() + + +.. parsed-literal:: + + + Original Dense Model Size Model=0.11 MiB + + +.. parsed-literal:: + + + +.. parsed-literal:: + + Original Model Validation Accuracy: 97.11538461538461 % + Granular-Magnitude Pruning + + +.. parsed-literal:: + + + +.. parsed-literal:: + + Sensitivity Scan Time(secs): 204.14258646965027 + Sparsity for each Layer: {'0.weight': 0.6500000000000001, '3.weight': 0.5000000000000001, '7.weight': 0.7000000000000002} + Pruning Time Consumed (mins): 2.8362054 + Total Pruning Time Consumed (mins): 3.402399698893229 + + +.. parsed-literal:: + + + +.. parsed-literal:: + + + Pruned Model has size=0.05 MiB(non-zeros) = 43.13% of Original model size + + +.. parsed-literal:: + + + +.. parsed-literal:: + + + Pruned Model has Accuracy=95.94 MiB(non-zeros) = -1.17% of Original model Accuracy + + + ========== Fine-Tuning ========== + + +.. parsed-literal:: + + + +.. parsed-literal:: + + Epoch:1 Train Loss: 0.00000 Validation Accuracy: 95.96354 + + +.. parsed-literal:: + + + +.. parsed-literal:: + + + ................. Comparison Table ................. + Original Pruned Reduction Ratio + Latency (ms) 16.7 15.6 1.1 + MACs (M) 160 160 1.0 + Param (M) 0.01 0.01 1.0 + Accuracies (%) 97.115 95.964 -1.152 + Fine-Tuned Sparse model has size=0.05 MiB = 43.13% of Original model size + Fine-Tuned Pruned Model Validation Accuracy: 95.96354166666667 + + +.. parsed-literal:: + + /usr/local/lib/python3.10/dist-packages/torchprofile/profile.py:22: UserWarning: No handlers found: "prim::pythonop". Skipped. + warnings.warn('No handlers found: "{}". Skipped.'.format( + /usr/local/lib/python3.10/dist-packages/torchprofile/profile.py:22: UserWarning: No handlers found: "prim::pythonop". Skipped. + warnings.warn('No handlers found: "{}". Skipped.'.format( + + +Note: +===== + +- The Latency is reduced and Parameters will be reduced(the numbers are + rounded to .2f hence we cannnot see the parameter pruning here, large + model will be able to showcase this deliberately), +- The MAC is remains the same, sicne the Conv operation used here is + Conv2d and the MAC are calculated Channel Wise and **not Element + Wise**. +- If specialised sparsity aware, Software/Hardware is used then we + reach the ultimate goal of compressing the model and leveraging the + inherent sparsity in the model diff --git a/docs/tutorials/tutorials.rst b/docs/tutorials/tutorials.rst index 9043367b..0a355a30 100644 --- a/docs/tutorials/tutorials.rst +++ b/docs/tutorials/tutorials.rst @@ -54,6 +54,12 @@ The tutorial consists of a series of Google Colab notebooks. Static non-editable :alt: Open In Colab :target: https://colab.research.google.com/github/jeshraghian/snntorch/blob/master/examples/tutorial_7_neuromorphic_datasets.ipynb + * - `Tutorial 8 `_ + - Model Compresion: sconce + snnTorch + - .. image:: https://colab.research.google.com/assets/colab-badge.svg + :alt: Open In Colab + :target: https://colab.research.google.com/github/jeshraghian/snntorch/blob/master/examples/sconce_model_compression_pruning.ipynb + .. list-table:: @@ -78,9 +84,14 @@ The tutorial consists of a series of Google Colab notebooks. Static non-editable :alt: Open In Colab :target: https://colab.research.google.com/github/jeshraghian/snntorch/blob/master/examples/tutorial_regression_2.ipynb + * - `Model Pruning `_ + - .. image:: https://colab.research.google.com/assets/colab-badge.svg + :alt: Open In Colab + :target: https://github.com/satabios/snntorch/blob/master/examples/sconce_model_compression_pruning.ipynb * - `Accelerating snnTorch on IPUs `_ - — -Future tutorials on spiking neurons and training are under development. \ No newline at end of file + +Future tutorials on spiking neurons and training are under development. diff --git a/examples/.DS_Store b/examples/.DS_Store new file mode 100644 index 00000000..0260a239 Binary files /dev/null and b/examples/.DS_Store differ diff --git a/examples/sconce_model_compression_pruning.ipynb b/examples/sconce_model_compression_pruning.ipynb new file mode 100644 index 00000000..3bbc2a69 --- /dev/null +++ b/examples/sconce_model_compression_pruning.ipynb @@ -0,0 +1,655 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Model Pruning\n", + "\n", + "A Detialed Explanation/Tutorial for Pruning on Deep Learning can be found here:[Pruning Basics and DL implementation walkthrough](https://github.com/satabios/sconce/blob/main/tutorials/Pruning.ipynb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# snn Model Pruning" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "31yxU2xZnxM3" + }, + "source": [ + "# Install Packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Uyzh1SIAZWFD", + "outputId": "81df7a95-a322-4f92-facf-d5b43e610e95" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m109.0/109.0 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m153.0/153.0 kB\u001b[0m \u001b[31m2.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", + " Installing backend dependencies ... \u001b[?25l\u001b[?25hdone\n", + " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.6/11.6 MB\u001b[0m \u001b[31m70.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m18.2/18.2 MB\u001b[0m \u001b[31m69.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m114.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.8/2.8 MB\u001b[0m \u001b[31m113.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.1/53.1 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m121.1/121.1 kB\u001b[0m \u001b[31m16.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m69.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", + "\u001b[?25h Building wheel for lit (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "lida 0.0.10 requires fastapi, which is not installed.\n", + "lida 0.0.10 requires kaleido, which is not installed.\n", + "lida 0.0.10 requires python-multipart, which is not installed.\n", + "lida 0.0.10 requires uvicorn, which is not installed.\n", + "cupy-cuda11x 11.0.0 requires numpy<1.26,>=1.20, but you have numpy 1.26.2 which is incompatible.\n", + "imageio 2.31.6 requires pillow<10.1.0,>=8.3.2, but you have pillow 10.1.0 which is incompatible.\n", + "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.8.0 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0m" + ] + } + ], + "source": [ + "!pip install snntorch -q\n", + "!pip install sconce -q\n", + "%pip show sconce" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "l9M2kqJwn0Uc" + }, + "source": [ + "# Import Libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2023-12-01T07:32:53.170470032Z", + "start_time": "2023-12-01T07:32:50.766056935Z" + }, + "id": "xEDfNQnMIW9H" + }, + "outputs": [], + "source": [ + "from collections import defaultdict, OrderedDict\n", + "\n", + "import numpy as np\n", + "import torch\n", + "from torch import nn\n", + "from torch.optim import *\n", + "from torch.optim.lr_scheduler import *\n", + "from torch.utils.data import DataLoader\n", + "from torchvision.datasets import *\n", + "from torchvision.transforms import *\n", + "import torch.optim as optim\n", + "from sconce import sconce\n", + "\n", + "assert torch.cuda.is_available(), \\\n", + "\"The current runtime does not have CUDA support.\" \\\n", + "\"Please go to menu bar (Runtime - Change runtime type) and select GPU\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "oJjAdrRK0aRD" + }, + "source": [ + "# **Spiking Neural Network Compression**\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "V9qFLeGl0y24", + "outputId": "3ced031a-a68f-4d7a-9be2-81fdd17f0654" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + ":4: DeprecationWarning: The module snntorch.backprop will be deprecated in a future release. Writing out your own training loop will lead to substantially faster performance.\n", + " from snntorch import backprop\n" + ] + } + ], + "source": [ + "# Import snntorch libraries\n", + "import snntorch as snn\n", + "from snntorch import surrogate\n", + "from snntorch import backprop\n", + "from snntorch import functional as SF\n", + "from snntorch import utils\n", + "from snntorch import spikeplot as splt\n", + "from torch import optim\n", + "\n", + "import torch\n", + "import torch.nn as nn\n", + "from torch.utils.data import DataLoader\n", + "from torchvision import datasets, transforms\n", + "import torch.nn.functional as F\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import itertools\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "0iqMEfCgnufM" + }, + "source": [ + "# Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "zk0TCGvRKOWh", + "outputId": "78c643ab-1d99-4f2e-d64c-ddc6aedc9940" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n", + "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to ./data/mnist/MNIST/raw/train-images-idx3-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 9912422/9912422 [00:00<00:00, 82101508.40it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ./data/mnist/MNIST/raw/train-images-idx3-ubyte.gz to ./data/mnist/MNIST/raw\n", + "\n", + "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n", + "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to ./data/mnist/MNIST/raw/train-labels-idx1-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 28881/28881 [00:00<00:00, 111748795.04it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ./data/mnist/MNIST/raw/train-labels-idx1-ubyte.gz to ./data/mnist/MNIST/raw\n", + "\n", + "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n", + "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to ./data/mnist/MNIST/raw/t10k-images-idx3-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 1648877/1648877 [00:00<00:00, 26490461.97it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ./data/mnist/MNIST/raw/t10k-images-idx3-ubyte.gz to ./data/mnist/MNIST/raw\n", + "\n", + "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n", + "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to ./data/mnist/MNIST/raw/t10k-labels-idx1-ubyte.gz\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████| 4542/4542 [00:00<00:00, 6970555.71it/s]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Extracting ./data/mnist/MNIST/raw/t10k-labels-idx1-ubyte.gz to ./data/mnist/MNIST/raw\n", + "\n" + ] + } + ], + "source": [ + "\n", + "# Event Drive Data\n", + "\n", + "# dataloader arguments\n", + "batch_size = 128\n", + "data_path = \"./data/mnist\"\n", + "\n", + "dtype = torch.float\n", + "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n", + "\n", + "# Define a transform\n", + "transform = transforms.Compose(\n", + " [\n", + " transforms.Resize((28, 28)),\n", + " transforms.Grayscale(),\n", + " transforms.ToTensor(),\n", + " transforms.Normalize((0,), (1,)),\n", + " ]\n", + ")\n", + "\n", + "mnist_train = datasets.MNIST(data_path, train=True, download=True, transform=transform)\n", + "mnist_test = datasets.MNIST(data_path, train=False, download=True, transform=transform)\n", + "\n", + "# Create DataLoaders\n", + "train_loader = DataLoader(\n", + " mnist_train, batch_size=batch_size, shuffle=True, drop_last=True\n", + ")\n", + "test_loader = DataLoader(\n", + " mnist_test, batch_size=batch_size, shuffle=True, drop_last=True\n", + ")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "khyN02zynLkO" + }, + "source": [ + "# Instantiate an Object of sconce" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "cu7BrvWvnKjm" + }, + "outputs": [], + "source": [ + "\n", + "sconces = sconce()\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "-hTGOBb-nsBo" + }, + "source": [ + "# Set you Dataloader" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9pj-ZzMQU9_Q" + }, + "outputs": [], + "source": [ + "\n", + "dataloader = {}\n", + "dataloader[\"train\"] = train_loader\n", + "dataloader[\"test\"] = test_loader\n", + "sconces.dataloader = dataloader" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AV4QDhgznnHQ" + }, + "source": [ + "#Enable snn in sconce" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YnqX4AxNnlwG" + }, + "outputs": [], + "source": [ + "\n", + "sconces.snn = True\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "F1umQlu1nptt" + }, + "source": [ + "# Load your snn Model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LzEHUCSx0hN6" + }, + "outputs": [], + "source": [ + "spike_grad = surrogate.fast_sigmoid(slope=25)\n", + "beta = 0.5\n", + "snn_model = nn.Sequential(\n", + " nn.Conv2d(1, 12, 5),\n", + " nn.MaxPool2d(2),\n", + " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True),\n", + " nn.Conv2d(12, 64, 5),\n", + " nn.MaxPool2d(2),\n", + " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True),\n", + " nn.Flatten(),\n", + " nn.Linear(64 * 4 * 4, 10),\n", + " snn.Leaky(beta=beta, spike_grad=spike_grad, init_hidden=True, output=True),\n", + ").to('cuda')\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Suw2PXg6nc6i" + }, + "source": [ + "# Load the pretrained weights" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "7Hkv8zdJncZ4" + }, + "outputs": [], + "source": [ + "snn_pretrained_model_path = \"./snn_model.pth\"\n", + "snn_model.load_state_dict(torch.load(snn_pretrained_model_path)) # Model Definition\n", + "sconces.model = snn_model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Set the Optimizizer and Type of Pruning Operation to Perform on the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qZLTwjV0VdJc" + }, + "outputs": [], + "source": [ + "\n", + "sconces.optimizer = optim.Adam(sconces.model.parameters(), lr=1e-4)\n", + "sconces.scheduler = optim.lr_scheduler.CosineAnnealingLR(sconces.optimizer, T_max=200)\n", + "\n", + "sconces.criterion = SF.ce_rate_loss()\n", + "\n", + "sconces.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "sconces.experiment_name = \"snn-gmp\" # Define your experiment name here\n", + "sconces.prune_mode = \"GMP\"\n", + "sconces.num_finetune_epochs = 1\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Test the Pre-Trained Model Accuracy" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "lRlavnVPbKUT", + "outputId": "8becb541-1e43-4d75-9f84-3d45340d65e4" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "data": { + "text/plain": [ + "97.11538461538461" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sconces.evaluate()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Prune the Model\n", + "\n", + "The Compression does a series of steps as explained below:\n", + "\n", + "1. It evaluates the dense model accuracy\n", + "2. Given the model, the package finds the best parameters for pruning such that the accuracy degradation is minimal.\n", + "3. The retreived optimal parameters from the above steps are used to prune the model.\n", + "4. At times, certain pruning techniques might require a fine-tuning on the dataset. For which the pruned model is fine-tuned on the dataset.\n", + "5. Pruned Model is saved and Compared for Latency, Paramater, MAC and model size.\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "K6kWCtSk0kVq", + "outputId": "53bebd3c-898e-4635-cb9e-d6d67603b451" + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Original Dense Model Size Model=0.11 MiB\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Original Model Validation Accuracy: 97.11538461538461 %\n", + "Granular-Magnitude Pruning\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Sensitivity Scan Time(secs): 204.14258646965027\n", + "Sparsity for each Layer: {'0.weight': 0.6500000000000001, '3.weight': 0.5000000000000001, '7.weight': 0.7000000000000002}\n", + "Pruning Time Consumed (mins): 2.8362054\n", + "Total Pruning Time Consumed (mins): 3.402399698893229\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Pruned Model has size=0.05 MiB(non-zeros) = 43.13% of Original model size\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Pruned Model has Accuracy=95.94 MiB(non-zeros) = -1.17% of Original model Accuracy\n", + "\n", + " \n", + "========== Fine-Tuning ==========\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Epoch:1 Train Loss: 0.00000 Validation Accuracy: 95.96354\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + " ................. Comparison Table .................\n", + " Original Pruned Reduction Ratio\n", + "Latency (ms) 16.7 15.6 1.1 \n", + "MACs (M) 160 160 1.0 \n", + "Param (M) 0.01 0.01 1.0 \n", + "Accuracies (%) 97.115 95.964 -1.152 \n", + "Fine-Tuned Sparse model has size=0.05 MiB = 43.13% of Original model size\n", + "Fine-Tuned Pruned Model Validation Accuracy: 95.96354166666667\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.10/dist-packages/torchprofile/profile.py:22: UserWarning: No handlers found: \"prim::pythonop\". Skipped.\n", + " warnings.warn('No handlers found: \"{}\". Skipped.'.format(\n", + "/usr/local/lib/python3.10/dist-packages/torchprofile/profile.py:22: UserWarning: No handlers found: \"prim::pythonop\". Skipped.\n", + " warnings.warn('No handlers found: \"{}\". Skipped.'.format(\n" + ] + } + ], + "source": [ + "sconces.compress()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Note:\n", + "* The Latency is reduced and Parameters will be reduced(the numbers are rounded to .2f hence we cannnot see the parameter pruning here, large model will be able to showcase this deliberately),\n", + "* The MAC is remains the same, sicne the Conv operation used here is Conv2d and the MAC are calculated Channel Wise and **not Element Wise**. \n", + "* If specialised sparsity aware, Software/Hardware is used then we reach the ultimate goal of compressing the model and leveraging the inherent sparsity in the model\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "machine_shape": "hm", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/snn_model.pth b/examples/snn_model.pth new file mode 100644 index 00000000..dc3b018e Binary files /dev/null and b/examples/snn_model.pth differ