4_neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f8165d69",
   "metadata": {},
   "source": [
    "# Neural network example\n",
    "\n",
    "This notebook is demonstrating a simple neural network, it is based on this excellent medium article by [elvis](https://medium.com/dair-ai/a-simple-neural-network-from-scratch-with-pytorch-and-google-colab-c7f3830618e0) (see the Credits)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b15f06fa",
   "metadata": {},
   "source": [
    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/insop/ML_crash_course/blob/main/4_neural_network.ipynb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "98014308",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "acf7b6bb",
   "metadata": {},
   "source": [
    "## Generate sample data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "10a12f40",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = torch.tensor(([2, 9], [1, 5], [3, 6]), dtype=torch.float) # 3 X 2 tensor\n",
    "y = torch.tensor(([92], [100], [89]), dtype=torch.float) # 3 X 1 tensor\n",
    "xPredicted = torch.tensor(([4, 8]), dtype=torch.float) # 1 X 2 tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "420fd476",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 2])\n",
      "torch.Size([3, 1])\n"
     ]
    }
   ],
   "source": [
    "print(X.size())\n",
    "print(y.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5930ea04",
   "metadata": {},
   "source": [
    "## Scaling dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "4771218c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.5000, 1.0000])\n"
     ]
    }
   ],
   "source": [
    "# scale units\n",
    "X_max, _ = torch.max(X, 0)\n",
    "xPredicted_max, _ = torch.max(xPredicted, 0)\n",
    "\n",
    "X = torch.div(X, X_max)\n",
    "xPredicted = torch.div(xPredicted, xPredicted_max)\n",
    "y = y / 100  # max test score is 100\n",
    "print(xPredicted)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e692746",
   "metadata": {},
   "source": [
    "## Model definition\n",
    "\n",
    "inputSize = 2\n",
    "\n",
    "hiddenSize = 3\n",
    "\n",
    "outputSize = 1\n",
    "\n",
    "        \n",
    "***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8e12d332",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Neural_Network(nn.Module):\n",
    "    def __init__(self, ):\n",
    "        super(Neural_Network, self).__init__()\n",
    "        # parameters\n",
    "        # TODO: parameters can be parameterized instead of declaring them here\n",
    "        self.inputSize = 2\n",
    "        self.outputSize = 1\n",
    "        self.hiddenSize = 3\n",
    "        \n",
    "        # weights\n",
    "        self.W1 = torch.randn(self.inputSize, self.hiddenSize) # 3 X 2 tensor\n",
    "        self.W2 = torch.randn(self.hiddenSize, self.outputSize) # 3 X 1 tensor\n",
    "        \n",
    "    def forward(self, X):\n",
    "        self.z = torch.matmul(X, self.W1) # 3 X 3 \".dot\" does not broadcast in PyTorch\n",
    "        self.z2 = self.sigmoid(self.z) # activation function\n",
    "        self.z3 = torch.matmul(self.z2, self.W2)\n",
    "        o = self.sigmoid(self.z3) # final activation function\n",
    "        return o\n",
    "        \n",
    "    def sigmoid(self, s):\n",
    "        return 1 / (1 + torch.exp(-s))\n",
    "    \n",
    "    def sigmoidPrime(self, s):\n",
    "        # derivative of sigmoid\n",
    "        return s * (1 - s)\n",
    "    \n",
    "    def backward(self, X, y, o):\n",
    "        self.o_error = y - o # error in output\n",
    "        self.o_delta = self.o_error * self.sigmoidPrime(o) # derivative of sig to error\n",
    "        self.z2_error = torch.matmul(self.o_delta, torch.t(self.W2))\n",
    "        self.z2_delta = self.z2_error * self.sigmoidPrime(self.z2)\n",
    "        self.W1 += torch.matmul(torch.t(X), self.z2_delta)\n",
    "        self.W2 += torch.matmul(torch.t(self.z2), self.o_delta)\n",
    "        \n",
    "    def train(self, X, y):\n",
    "        # forward + backward pass for training\n",
    "        o = self.forward(X)\n",
    "        self.backward(X, y, o)\n",
    "        \n",
    "    def saveWeights(self, model):\n",
    "        # we will use the PyTorch internal storage functions\n",
    "        torch.save(model, \"NN\")\n",
    "        # you can reload model with all the weights and so forth with:\n",
    "        # torch.load(\"NN\")\n",
    "        \n",
    "    def predict(self):\n",
    "        print (\"Predicted data based on trained weights: \")\n",
    "        print (\"Input (scaled): \\n\" + str(xPredicted))\n",
    "        print (\"Output: \\n\" + str(self.forward(xPredicted)))\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11001b7e",
   "metadata": {},
   "source": [
    "## Model define & training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "0c628486",
   "metadata": {},
   "outputs": [],
   "source": [
    "NN = Neural_Network()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "86eb61ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#0 Loss: 0.13196535408496857\n",
      "#100 Loss: 0.003118229331448674\n",
      "#200 Loss: 0.002868946408852935\n",
      "#300 Loss: 0.00271593383513391\n",
      "#400 Loss: 0.0025623550172895193\n",
      "#500 Loss: 0.0024063093587756157\n",
      "#600 Loss: 0.002250604098662734\n",
      "#700 Loss: 0.002098219934850931\n",
      "#800 Loss: 0.0019517795881256461\n",
      "#900 Loss: 0.0018133349949494004\n",
      "Predicted data based on trained weights: \n",
      "Input (scaled): \n",
      "tensor([0.5000, 1.0000])\n",
      "Output: \n",
      "tensor([0.9363])\n",
      "Finished training!\n"
     ]
    }
   ],
   "source": [
    "for i in range(1000):  # trains the NN 1,000 times\n",
    "    if (i % 100) == 0:\n",
    "        print (\"#\" + str(i) + \" Loss: \" + str(torch.mean((y - NN(X))**2).detach().item()))  # mean sum squared loss\n",
    "    NN.train(X, y)\n",
    "NN.saveWeights(NN)\n",
    "NN.predict()\n",
    "\n",
    "print(\"Finished training!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8f102f89",
   "metadata": {},
   "source": [
    "## Credits\n",
    "\n",
    "This notebook uses the contents from the followring materials:\n",
    "\n",
    "- [A Simple Neural Network from Scratch with PyTorch and Google Colab](https://medium.com/dair-ai/a-simple-neural-network-from-scratch-with-pytorch-and-google-colab-c7f3830618e0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "849f47e7",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}