Fashion MNIST Convolution Neural Network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model: Convolutional Neural Network\n",
    "### Dataset - MNIST Fashion\n",
    "\n",
    "Models included: \n",
    "\n",
    "| Model A | Model B   | Model C |\n",
    "| ----- | ---- |\n",
    "|  ReLU | ReLU | ReLU |\n",
    "| 2 Convolution Layers | 2 Convolution Layers | 3 Convolution Layers |\n",
    "| 2 Hidden Layers | 1 Hidden Layers | 3 Hidden Layers | \n",
    "| 91.58% | 95.76% | 96.13 |"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 0. Necessary Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Necessary imports\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.transforms as transforms \n",
    "import torchvision.datasets as dsets\n",
    "from torch.autograd import Variable\n",
    "import os\n",
    "import gzip\n",
    "import urllib.request\n",
    "import os.path\n",
    "import torch.utils.data as data\n",
    "import codecs\n",
    "from PIL import Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1. Create and Load Dataset\n",
    "\n",
    "#### 1.1 Dataset Class of MNIST Fashion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FashionMNISTDataset(data.Dataset):\n",
    "    '''Fashion MNIST Dataset'''\n",
    "    \n",
    "    urls = \t[\n",
    "        'http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-images-idx3-ubyte.gz',\n",
    "        'http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/train-labels-idx1-ubyte.gz',\n",
    "        'http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-images-idx3-ubyte.gz',\n",
    "        'http://fashion-mnist.s3-website.eu-central-1.amazonaws.com/t10k-labels-idx1-ubyte.gz'\n",
    "            ]\n",
    "\n",
    "    file_name =\t[\n",
    "                    'train-images-idx3-ubyte',\n",
    "                    'train-labels-idx1-ubyte',\n",
    "                    't10k-images-idx3-ubyte',\n",
    "                    't10k-labels-idx1-ubyte'\n",
    "                ]\n",
    "    \n",
    "    training_file = 'training.pt'\n",
    "    test_file = 'test.pt'\n",
    "    \n",
    "    def __init__(self, source , train=True, transform=None):\n",
    "        \n",
    "        self.source = source\n",
    "        \n",
    "        if os.path.exists(source) == False:\n",
    "            self.download(self.source)\n",
    "        \n",
    "        if self.check_processed(train=train, source=source) is not True:\n",
    "            self.process_files(train=train,source=source);\n",
    "        \n",
    "        if train is True:\n",
    "            self.X, self.Y = torch.load(\n",
    "                os.path.join(source, self.training_file))\n",
    "        else:\n",
    "            self.X, self.Y = torch.load(\n",
    "                os.path.join(source, self.test_file))\n",
    "        \n",
    "        self.transform = transform;\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.X);\n",
    "    \n",
    "    def check_processed(self, train, source):\n",
    "        return os.path.exists(os.path.join(source, self.training_file)) and \\\n",
    "            os.path.exists(os.path.join(source, self.test_file))\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        item = self.X[idx];\n",
    "        label = self.Y[idx];\n",
    "        \n",
    "        item = Image.fromarray(item.numpy(), mode='L')\n",
    "        \n",
    "        if self.transform:\n",
    "            item = self.transform(item);\n",
    "        \n",
    "        return (item, label);\n",
    "    \n",
    "    def download(self, source):\n",
    "        \n",
    "        try:\n",
    "\t\t\t      os.makedirs(self.source)\n",
    "        except OSError as exc:\n",
    "            if exc.errno != errno.EEXIST:\n",
    "              raise\n",
    "        pass\n",
    "        \n",
    "        for file_index in range(len(self.file_name)):\n",
    "            print(\"Downloading:\",self.urls[file_index])\n",
    "            urllib.request.urlretrieve(self.urls[file_index],(self.file_name[file_index]+'.gz'))\n",
    "            f = gzip.open(self.file_name[file_index]+'.gz', 'rb')\n",
    "            with open(source+self.file_name[file_index],'wb+') as w:\n",
    "                for line in f.readlines():\n",
    "                    w.write(line)\n",
    "            f.close()\n",
    "            os.remove(self.file_name[file_index]+\".gz\")\n",
    "    \n",
    "    @classmethod\n",
    "    def process_files(self, train, source):\n",
    "        \n",
    "        if train is True:\n",
    "            imgs='train-images-idx3-ubyte'\n",
    "            lbls='train-labels-idx1-ubyte'\n",
    "            n=60000\n",
    "        else:\n",
    "            imgs='t10k-images-idx3-ubyte'\n",
    "            lbls='t10k-labels-idx1-ubyte'\n",
    "            n=10000\n",
    "        \n",
    "        f = open(str(source+imgs), \"rb\")        \n",
    "        l = open(str(source+lbls), \"rb\")\n",
    "        \n",
    "        to_set = (\n",
    "            read_image_file(f),\n",
    "            read_label_file(l)\n",
    "        )\n",
    "        \n",
    "        if train is True:\n",
    "            fh = open(str(source+self.training_file), 'wb+')\n",
    "        else:\n",
    "            fh = open(str(source+self.test_file), 'wb+')\n",
    "        \n",
    "        torch.save(to_set, fh)\n",
    "        \n",
    "        fh.close()\n",
    "        \n",
    "        f.close()\n",
    "        l.close()\n",
    "\n",
    "        \n",
    "def get_int(b):\n",
    "    return int(codecs.encode(b, 'hex'), 16)\n",
    "\n",
    "\n",
    "def read_label_file(f):\n",
    "    data = f.read()\n",
    "    length = get_int(data[4:8])\n",
    "    parsed = np.frombuffer(data, dtype=np.uint8, offset=8)\n",
    "    return torch.from_numpy(parsed).view(length).long()\n",
    "\n",
    "\n",
    "def read_image_file(f):\n",
    "    data = f.read()\n",
    "    length = get_int(data[4:8])\n",
    "    num_rows = get_int(data[8:12])\n",
    "    num_cols = get_int(data[12:16])\n",
    "    parsed = np.frombuffer(data, dtype=np.uint8, offset=16)\n",
    "    return torch.from_numpy(parsed).view(length, num_rows, num_cols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1.2 Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = FashionMNISTDataset(source='./data/',\n",
    "                                    train=True,\n",
    "                                    transform=transforms.ToTensor())\n",
    "test_dataset = FashionMNISTDataset(source='./data/',\n",
    "                                   train=False,\n",
    "                                   transform=transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Make Dataset iterable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 100\n",
    "no_iters = 12000\n",
    "no_epochs = no_iters / ( len(train_dataset) / batch_size )\n",
    "no_epochs = int(no_epochs)\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model A: 2 Layer CNN along with maxpool downscalling with Batch Normalization + 2 Connected Layers\n",
    "#### Optimizer - SGD + Nesterov Momentum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 2000, Loss: 0.12439420074224472, Train Accuracy: 94.91833333333334, Test Accuracy: 91.15\n",
      "Iteration: 4000, Loss: 0.07574900984764099, Train Accuracy: 98.32333333333334, Test Accuracy: 92.11\n",
      "Iteration: 6000, Loss: 0.019024420529603958, Train Accuracy: 99.54333333333334, Test Accuracy: 92.27\n",
      "Iteration: 8000, Loss: 0.0004733180976472795, Train Accuracy: 99.895, Test Accuracy: 92.5\n",
      "Iteration: 10000, Loss: 0.00043110846308991313, Train Accuracy: 100.0, Test Accuracy: 92.66\n",
      "Iteration: 12000, Loss: 0.0005130863282829523, Train Accuracy: 100.0, Test Accuracy: 92.69\n"
     ]
    }
   ],
   "source": [
    "#### 3. Create Model Class\n",
    "class CNNModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNNModel, self).__init__()\n",
    "        \n",
    "        #Convolution 1\n",
    "        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=32, kernel_size=5, stride=1, padding=2)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.norm1 = nn.BatchNorm2d(32)\n",
    "        \n",
    "        # Max pool 1\n",
    "        self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n",
    "        \n",
    "        # Convolution 2\n",
    "        self.cnn2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5, stride=1, padding=2)\n",
    "        self.relu2 = nn.ReLU()\n",
    "        self.norm2 = nn.BatchNorm2d(64)\n",
    "        \n",
    "        # Max pool 2\n",
    "        self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n",
    "        \n",
    "        #FNN 1\n",
    "        self.fc1 = nn.Linear(64*7*7, 64*7*7)\n",
    "        self.relu3 = nn.ReLU()\n",
    "        \n",
    "        #Linear\n",
    "        self.fc2 = nn.Linear(64*7*7, 10)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        #Convolution 1\n",
    "        out = self.cnn1(x)\n",
    "        out = self.relu1(out)\n",
    "        out = self.norm1(out)\n",
    "        \n",
    "        #Max pool 1\n",
    "        out = self.maxpool1(out)\n",
    "        \n",
    "        #Convolution 2\n",
    "        out = self.cnn2(out)\n",
    "        out = self.relu2(out)\n",
    "        out = self.norm2(out)\n",
    "        \n",
    "        #Max pool 2\n",
    "        out = self.maxpool2(out)\n",
    "        \n",
    "        #Resize\n",
    "        #original size: (100, 64, 7, 7)\n",
    "        #out.size(0): 100\n",
    "        #New out size: (100, 64*7*7)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        \n",
    "        #FNN 1\n",
    "        out = self.fc1(out)\n",
    "        out = self.relu3(out)\n",
    "        \n",
    "        #Linear\n",
    "        out = self.fc2(out)\n",
    "        \n",
    "        return out\n",
    "    \n",
    "#### 4. Instantiate Model Class\n",
    "if torch.cuda.is_available():\n",
    "    model =CNNModel().cuda()\n",
    "else:\n",
    "    model = CNNModel()\n",
    "    \n",
    "#### 5. Instantiate Loss Class\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "#### 6. Instantiate Optimizer Class\n",
    "learning_rate = 0.01\n",
    "moment = 0.9\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum = moment, nesterov = True)\n",
    "\n",
    "#### 7. Train Model\n",
    "iter = 1\n",
    "for epoch in range(no_epochs):\n",
    "    train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
    "    for i,(images,labels) in enumerate(train_loader):\n",
    "        #Variables\n",
    "        if torch.cuda.is_available():\n",
    "            images = images.cuda()\n",
    "            labels = labels.cuda()\n",
    "            \n",
    "        #Clear Gradient\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        #Outpus\n",
    "        outputs = model(images)\n",
    "        \n",
    "        #Calculate loss: softmax -> cross entropy loss\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        #Generate gradients w.r.t. parameters\n",
    "        loss.backward()\n",
    "        \n",
    "        #Update Parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        iter += 1\n",
    "        if iter %2000 ==0:\n",
    "            #Calculate Accuracy\n",
    "            correct=0\n",
    "            total=0\n",
    "            train_accuracy = 0\n",
    "            test_accuracy = 0\n",
    "            \n",
    "            #Train Accuracy\n",
    "            for images,labels in train_loader:\n",
    "                if torch.cuda.is_available():\n",
    "                    images = images.cuda()\n",
    "                #outputs\n",
    "                outputs = model(images)\n",
    "                \n",
    "                #Get predictions from maximum value\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                \n",
    "                #Total number of labels\n",
    "                total += labels.size(0)\n",
    "                \n",
    "                #Total Correct Labels\n",
    "                if torch.cuda.is_available():\n",
    "                    correct += (predicted.cpu() == labels.cpu()).sum()\n",
    "                    \n",
    "            train_accuracy = 100 * float(correct) / total\n",
    "            correct=0\n",
    "            total=0\n",
    "            \n",
    "            #Test Accuracy\n",
    "            for images,labels in test_loader:\n",
    "                if torch.cuda.is_available():\n",
    "                    images = images.cuda()\n",
    "                #outputs\n",
    "                outputs = model(images)\n",
    "                \n",
    "                #Get predictions from maximum value\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                \n",
    "                #Total number of labels\n",
    "                total += labels.size(0)\n",
    "                \n",
    "                #Total Correct Labels\n",
    "                if torch.cuda.is_available():\n",
    "                    correct += (predicted.cpu() == labels.cpu()).sum()\n",
    "                    \n",
    "            test_accuracy = 100 * float(correct) / total\n",
    "            \n",
    "            print('Iteration: {}, Loss: {}, Train Accuracy: {}, Test Accuracy: {}'.format(iter, loss.item(), train_accuracy, test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model B: 2 Layer CNN along with maxpool downscalling with Batch Normalization + 1 Connected Layers\n",
    "#### Optimizer - SGD + Nesterov Momentum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 2000, Loss: 0.12232035398483276, Train Accuracy: 93.24166666666666, Test Accuracy: 90.38\n",
      "Iteration: 4000, Loss: 0.09956430643796921, Train Accuracy: 95.58833333333334, Test Accuracy: 91.21\n",
      "Iteration: 6000, Loss: 0.07204921543598175, Train Accuracy: 96.51333333333334, Test Accuracy: 91.01\n",
      "Iteration: 8000, Loss: 0.0532103031873703, Train Accuracy: 98.50833333333334, Test Accuracy: 91.08\n",
      "Iteration: 10000, Loss: 0.022581472992897034, Train Accuracy: 98.35666666666667, Test Accuracy: 91.11\n",
      "Iteration: 12000, Loss: 0.059082165360450745, Train Accuracy: 99.02666666666667, Test Accuracy: 90.98\n"
     ]
    }
   ],
   "source": [
    "#### 3. Create Model Class\n",
    "class CNNModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNNModel, self).__init__()\n",
    "        \n",
    "        #Convolution 1\n",
    "        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=32, kernel_size=5, stride=1, padding=2)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.norm1 = nn.BatchNorm2d(32)\n",
    "        \n",
    "        # Max pool 1\n",
    "        self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n",
    "        \n",
    "        # Convolution 2\n",
    "        self.cnn2 = nn.Conv2d(in_channels=32, out_channels=64, kernel_size=5, stride=1, padding=2)\n",
    "        self.relu2 = nn.ReLU()\n",
    "        self.norm2 = nn.BatchNorm2d(64)\n",
    "        \n",
    "        # Max pool 2\n",
    "        self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n",
    "        \n",
    "        #Linear\n",
    "        self.fc = nn.Linear(64*7*7, 10)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        #Convolution 1\n",
    "        out = self.cnn1(x)\n",
    "        out = self.relu1(out)\n",
    "        out = self.norm1(out)\n",
    "        \n",
    "        #Max pool 1\n",
    "        out = self.maxpool1(out)\n",
    "        \n",
    "        #Convolution 2\n",
    "        out = self.cnn2(out)\n",
    "        out = self.relu2(out)\n",
    "        out = self.norm2(out)\n",
    "        \n",
    "        #Max pool 2\n",
    "        out = self.maxpool2(out)\n",
    "        \n",
    "        #Resize\n",
    "        #original size: (100, 64, 7, 7)\n",
    "        #out.size(0): 100\n",
    "        #New out size: (100, 64*7*7)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        \n",
    "        #Linear\n",
    "        out = self.fc(out)\n",
    "        \n",
    "        return out\n",
    "    \n",
    "#### 4. Instantiate Model Class\n",
    "if torch.cuda.is_available():\n",
    "    model =CNNModel().cuda()\n",
    "else:\n",
    "    model = CNNModel()\n",
    "    \n",
    "#### 5. Instantiate Loss Class\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "#### 6. Instantiate Optimizer Class\n",
    "learning_rate = 0.01\n",
    "moment = 0.9\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum = moment, nesterov = True)\n",
    "\n",
    "#### 7. Train Model\n",
    "iter = 1\n",
    "for epoch in range(no_epochs):\n",
    "    train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
    "    for i,(images,labels) in enumerate(train_loader):\n",
    "        #Variables\n",
    "        if torch.cuda.is_available():\n",
    "            images = images.cuda()\n",
    "            labels = labels.cuda()\n",
    "            \n",
    "        #Clear Gradient\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        #Outpus\n",
    "        outputs = model(images)\n",
    "        \n",
    "        #Calculate loss: softmax -> cross entropy loss\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        #Generate gradients w.r.t. parameters\n",
    "        loss.backward()\n",
    "        \n",
    "        #Update Parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        iter += 1\n",
    "        if iter %2000 ==0:\n",
    "            #Calculate Accuracy\n",
    "            correct=0\n",
    "            total=0\n",
    "            train_accuracy = 0\n",
    "            test_accuracy = 0\n",
    "            \n",
    "            #Train Accuracy\n",
    "            for images,labels in train_loader:\n",
    "                if torch.cuda.is_available():\n",
    "                    images = images.cuda()\n",
    "                #outputs\n",
    "                outputs = model(images)\n",
    "                \n",
    "                #Get predictions from maximum value\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                \n",
    "                #Total number of labels\n",
    "                total += labels.size(0)\n",
    "                \n",
    "                #Total Correct Labels\n",
    "                if torch.cuda.is_available():\n",
    "                    correct += (predicted.cpu() == labels.cpu()).sum()\n",
    "                    \n",
    "            train_accuracy = 100 * float(correct) / total\n",
    "            correct=0\n",
    "            total=0\n",
    "            \n",
    "            #Test Accuracy\n",
    "            for images,labels in test_loader:\n",
    "                if torch.cuda.is_available():\n",
    "                    images = images.cuda()\n",
    "                #outputs\n",
    "                outputs = model(images)\n",
    "                \n",
    "                #Get predictions from maximum value\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                \n",
    "                #Total number of labels\n",
    "                total += labels.size(0)\n",
    "                \n",
    "                #Total Correct Labels\n",
    "                if torch.cuda.is_available():\n",
    "                    correct += (predicted.cpu() == labels.cpu()).sum()\n",
    "                    \n",
    "            test_accuracy = 100 * float(correct) / total\n",
    "            \n",
    "            print('Iteration: {}, Loss: {}, Train Accuracy: {}, Test Accuracy: {}'.format(iter, loss.item(), train_accuracy, test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model C: 3 Layer CNN along with maxpool downscalling and Batch Normalization + 2 Connected Layers\n",
    "#### Optimizer - SGD + Nesterov Momentum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration: 2000, Loss: 0.13402174413204193, Train Accuracy: 94.91166666666666, Test Accuracy: 91.91\n",
      "Iteration: 4000, Loss: 0.12557491660118103, Train Accuracy: 97.73, Test Accuracy: 92.22\n",
      "Iteration: 6000, Loss: 0.018099628388881683, Train Accuracy: 99.10666666666667, Test Accuracy: 92.31\n",
      "Iteration: 8000, Loss: 0.002088089007884264, Train Accuracy: 99.97666666666667, Test Accuracy: 92.87\n",
      "Iteration: 10000, Loss: 0.0002571010554675013, Train Accuracy: 100.0, Test Accuracy: 93.18\n",
      "Iteration: 12000, Loss: 0.00036275864113122225, Train Accuracy: 100.0, Test Accuracy: 93.1\n"
     ]
    }
   ],
   "source": [
    "#### 3. Create Model Class\n",
    "class CNNModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNNModel, self).__init__()\n",
    "        \n",
    "        #Convolution 1\n",
    "        self.cnn1 = nn.Conv2d(in_channels=1, out_channels=16, kernel_size=3, stride=1, padding=1)\n",
    "        self.relu1 = nn.ReLU()\n",
    "        self.norm1 = nn.BatchNorm2d(16)\n",
    "        \n",
    "        # Max pool 1\n",
    "        self.maxpool1 = nn.MaxPool2d(kernel_size=2,stride=1)\n",
    "        \n",
    "        # Convolution 2\n",
    "        self.cnn2 = nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, stride=1, padding=1)\n",
    "        self.relu2 = nn.ReLU()\n",
    "        self.norm2 = nn.BatchNorm2d(32)\n",
    "        \n",
    "        # Max pool 2\n",
    "        self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n",
    "        \n",
    "        #Convolution 3\n",
    "        self.cnn3 = nn.Conv2d(in_channels=32,out_channels=64, kernel_size=3,stride=1,padding=1)\n",
    "        self.relu3 = nn.ReLU()\n",
    "        self.norm3 = nn.BatchNorm2d(64)\n",
    "        \n",
    "        #Max pool 3\n",
    "        self.maxpool3=nn.MaxPool2d(kernel_size=2)\n",
    "\n",
    "        #FNN 1\n",
    "        self.fc1 = nn.Linear(64*6*6, 64*6*6)\n",
    "        self.relu4 = nn.ReLU()\n",
    "        \n",
    "        #Linear\n",
    "        self.fc2 = nn.Linear(64*6*6, 10)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        #Convolution 1\n",
    "        out = self.cnn1(x)\n",
    "        out = self.relu1(out)\n",
    "        out = self.norm1(out)\n",
    "        \n",
    "        #Max pool 1\n",
    "        out = self.maxpool1(out)\n",
    "        \n",
    "        #Convolution 2\n",
    "        out = self.cnn2(out)\n",
    "        out = self.relu2(out)\n",
    "        out = self.norm2(out)\n",
    "        \n",
    "        #Max pool 2\n",
    "        out = self.maxpool2(out)\n",
    "        \n",
    "        #Convolution 3\n",
    "        out = self.cnn3(out)\n",
    "        out = self.relu3(out)\n",
    "        out = self.norm3(out)\n",
    "        \n",
    "        #Max pool 3\n",
    "        out = self.maxpool3(out)\n",
    "        \n",
    "        #Resize\n",
    "        #original size: (100, 64, 6, 6)\n",
    "        #out.size(0): 100\n",
    "        #New out size: (100, 64*6*6)\n",
    "        out = out.view(out.size(0), -1)\n",
    "        #FNN 1\n",
    "        out = self.fc1(out)\n",
    "        out = self.relu4(out)\n",
    "        \n",
    "        #Linear\n",
    "        out = self.fc2(out)\n",
    "        \n",
    "        return out\n",
    "    \n",
    "#### 4. Instantiate Model Class\n",
    "if torch.cuda.is_available():\n",
    "    model =CNNModel().cuda()\n",
    "else:\n",
    "    model = CNNModel()\n",
    "    \n",
    "#### 5. Instantiate Loss Class\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "#### 6. Instantiate Optimizer Class\n",
    "learning_rate = 0.01\n",
    "moment = 0.9\n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum = moment, nesterov = True)\n",
    "\n",
    "#### 7. Train Model\n",
    "iter = 1\n",
    "for epoch in range(no_epochs):\n",
    "    train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
    "    for i,(images,labels) in enumerate(train_loader):\n",
    "        #Variables\n",
    "        if torch.cuda.is_available():\n",
    "            images = images.cuda()\n",
    "            labels = labels.cuda()\n",
    "            \n",
    "        #Clear Gradient\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        #Outpus\n",
    "        outputs = model(images)\n",
    "        \n",
    "        #Calculate loss: softmax -> cross entropy loss\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        #Generate gradients w.r.t. parameters\n",
    "        loss.backward()\n",
    "        \n",
    "        #Update Parameters\n",
    "        optimizer.step()\n",
    "        \n",
    "        iter += 1\n",
    "        if iter %2000 ==0:\n",
    "            #Calculate Accuracy\n",
    "            correct=0\n",
    "            total=0\n",
    "            train_accuracy = 0\n",
    "            test_accuracy = 0\n",
    "            \n",
    "            #Train Accuracy\n",
    "            for images,labels in train_loader:\n",
    "                if torch.cuda.is_available():\n",
    "                    images = images.cuda()\n",
    "                #outputs\n",
    "                outputs = model(images)\n",
    "                \n",
    "                #Get predictions from maximum value\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                \n",
    "                #Total number of labels\n",
    "                total += labels.size(0)\n",
    "                \n",
    "                #Total Correct Labels\n",
    "                if torch.cuda.is_available():\n",
    "                    correct += (predicted.cpu() == labels.cpu()).sum()\n",
    "                    \n",
    "            train_accuracy = 100 * float(correct) / total\n",
    "            correct=0\n",
    "            total=0\n",
    "            \n",
    "            #Test Accuracy\n",
    "            for images,labels in test_loader:\n",
    "                if torch.cuda.is_available():\n",
    "                    images = images.cuda()\n",
    "                #outputs\n",
    "                outputs = model(images)\n",
    "                \n",
    "                #Get predictions from maximum value\n",
    "                _, predicted = torch.max(outputs.data, 1)\n",
    "                \n",
    "                #Total number of labels\n",
    "                total += labels.size(0)\n",
    "                \n",
    "                #Total Correct Labels\n",
    "                if torch.cuda.is_available():\n",
    "                    correct += (predicted.cpu() == labels.cpu()).sum()\n",
    "                    \n",
    "            test_accuracy = 100 * float(correct) / total\n",
    "            \n",
    "            print('Iteration: {}, Loss: {}, Train Accuracy: {}, Test Accuracy: {}'.format(iter, loss.item(), train_accuracy, test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}