HW1_MNIST_NN2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2c2ef604",
   "metadata": {},
   "source": [
    "### 1. Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "50520dbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import warnings\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.datasets import fetch_openml\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "%matplotlib inline\n",
    "mpl.rc('axes', labelsize=14)\n",
    "mpl.rc('xtick', labelsize=12)\n",
    "mpl.rc('ytick', labelsize=12)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56983607",
   "metadata": {},
   "source": [
    "### 2. Import from mlcblab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "26051ee7",
   "metadata": {},
   "outputs": [],
   "source": [
    "from mlcvlab.models.nn1 import NN1\n",
    "from mlcvlab.models.nn2 import NN2\n",
    "from mlcvlab.nn.losses import l2\n",
    "from mlcvlab.optim.sgd import SGD\n",
    "from mlcvlab.optim.adam import Adam"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30d59b83",
   "metadata": {},
   "source": [
    "### 3. Set Seed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c69454d",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02f3f225",
   "metadata": {},
   "source": [
    "### 4. Helper functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2cf1c5ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset():\n",
    "    '''Loads the whole dataset with true labels included.'''\n",
    "    x, y = fetch_openml(\"mnist_784\", version=1, return_X_y=True, as_frame=False)\n",
    "    return x,y\n",
    "\n",
    "def prepare_data(x, y):\n",
    "    '''Converts 10-ary labels in binary labels. If even then label is 1 otherwise 0.'''\n",
    "    y = y.astype(int)\n",
    "    y = y.reshape(len(y),1)\n",
    "    y =  (y+1) % 2\n",
    "    return x, y\n",
    "\n",
    "def split_train_test(x,y):\n",
    "    '''Partitioning the dataset into 10,000 test samles and the remaining 60,000 as training samples. \n",
    "    The shape  of the data will be M x N where M = 784 and N = 60000 for X and N = 10000 for y.'''   \n",
    "    X_train, X_test = x[:60000].T, x[60000:].T\n",
    "    y_train, y_test = y[:60000].T, y[60000:].T\n",
    "    \n",
    "    # adding -1 to the end of every x  as a bias term\n",
    "    bias_train = np.ones((1, np.shape(X_train)[1])) * -1\n",
    "    bias_test = np.ones((1, np.shape(X_test)[1])) * -1\n",
    "    \n",
    "    X_train = np.append(X_train, bias_train, axis = 0)\n",
    "    X_test = np.append(X_test, bias_test, axis = 0)\n",
    "    \n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "def initialize_model(X_train, X_test, y_train, y_test):\n",
    "    '''Setting up the size of the weights/layer vectors. W0 is M x K shape where M = arbitrary number and K = 785 and W1 is M x 1 which is 60000 x 1 '''\n",
    "\n",
    "    # M is a hyper parameter\n",
    "    M = 120\n",
    "    \n",
    "    #Initialization with ones\n",
    "    # W1 = np.ones((M,1))\n",
    "    \n",
    "    # Random initialization\n",
    "    W1 = np.random.rand(M + 1, 1)\n",
    "    \n",
    "    # Xavier initialization\n",
    "    #W1 = np.random.randn(M + 1, 1) * np.sqrt(1/M)\n",
    "    \n",
    "    # add bias term to W1\n",
    "    W1[-1] = -1\n",
    "    \n",
    "    # He initialization\n",
    "    W0 = np.ones((M,np.shape(X_train)[0]))\n",
    "    \n",
    "    W0 = np.random.randn(np.shape(W0)[0], np.shape(W0)[1]) * np.sqrt(2/np.shape(W0)[0])\n",
    "    \n",
    "    # adding -1 to the end of every x  as a bias term\n",
    "    bias_train = np.ones((1, np.shape(W0)[1])) * -1\n",
    "    W0 = np.append(W0, bias_train, axis = 0)\n",
    "    \n",
    "    two_layer_nn  = NN2()\n",
    "    two_layer_nn.W = [W0, W1]\n",
    "    two_layer_nn.layers[0].W = W0\n",
    "    two_layer_nn.layers[1].W = W1\n",
    "\n",
    "    return two_layer_nn\n",
    "\n",
    "def train_model(model, X_train, y_train):\n",
    "    '''Training the model using SGD or Adam optimizer.'''\n",
    "    final_W = SGD(model, X_train, y_train, lr=0.1)\n",
    "    # final_W = Adam(model, X_train, y_train)\n",
    "    \n",
    "    return final_W\n",
    "\n",
    "def test_model(model, X_test, y_test, final_W):\n",
    "    '''Tests the accuracy of the neural network.'''\n",
    "    accuracy = None\n",
    "    model.W = final_W\n",
    "    \n",
    "    # set the weights in the layers\n",
    "    for layer in range(len(model.layers)):\n",
    "        model.layers[layer].W = final_W[layer]\n",
    "    \n",
    "    # get the predictions of the algorithm using testing x as the input\n",
    "    y_hat = model.nn2(X_test)\n",
    "\n",
    "    # get the number of test instances\n",
    "    T = np.shape(y_test)[1]\n",
    "\n",
    "    A = np.absolute(y_test - y_hat)\n",
    "    \n",
    "    # check if the value is greater than 0 and set it 1 if so.\n",
    "    for x in range(np.shape(A)[1]):\n",
    "            if A[0][x] > 0: \n",
    "                A[0][x] = 1\n",
    "\n",
    "    # calculate the accuracy \n",
    "    accuracy = 1/T * np.sum(A)\n",
    "    \n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f4f01d2",
   "metadata": {},
   "source": [
    "### 5. Run the program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e9c60b8",
   "metadata": {},
   "outputs": [],
   "source": [
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "#load data\n",
    "x, y = load_dataset()\n",
    "\n",
    "#prepare data\n",
    "x, y = prepare_data(x,y)\n",
    "\n",
    "# split data set\n",
    "X_train, X_test, y_train, y_test = split_train_test(x,y)\n",
    "\n",
    "#initialize model\n",
    "model = initialize_model(X_train, X_test, y_train, y_test)\n",
    "\n",
    "#training model\n",
    "final_W = train_model(model, X_train, y_train)\n",
    "print(f\"Completed training model - final W : {final_W}\")\n",
    "\n",
    "\n",
    "#testing model\n",
    "accuracy = test_model(model, X_test, y_test, final_W)\n",
    "percentage = 100 * accuracy\n",
    "print(f\"Completed testing model (nn2) - Accuracy : {percentage:2.1f}%\")       "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "527d2373",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}