NVRM_cifar10_LabelNoise40_demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "\n",
    "import torch\n",
    "import torch.backends.cudnn as cudnn\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "from model import *\n",
    "from variable_optim import *\n",
    "from noisy_cifar import CIFAR10, CIFAR100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> Preparing data..\n"
     ]
    }
   ],
   "source": [
    "print('==> Preparing data..')\n",
    "transform_train = transforms.Compose([\n",
    "    transforms.RandomCrop(32, padding=4),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "])\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def optimizers(net, opti_name, lr, weight_decay, vb=0, num_iters=0):\n",
    "    if opti_name == 'SGD':\n",
    "        return optim.SGD(net.parameters(), lr=lr, momentum=0, weight_decay=weight_decay)\n",
    "    elif opti_name == 'Momentum':\n",
    "        return optim.SGD(net.parameters(), lr=lr, momentum=0.9, weight_decay=weight_decay, nesterov=False)\n",
    "    elif opti_name == 'Adam':\n",
    "        return optim.Adam(net.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-08, weight_decay=weight_decay)\n",
    "    elif opti_name == 'VSGD-Gaussian':\n",
    "        return VSGD(net.parameters(), lr=lr, variability=vb, num_iters=num_iters, weight_decay=weight_decay, noise_type='Gaussian')\n",
    "    elif opti_name == 'VAdam-Gaussian':\n",
    "        return VAdam(net.parameters(), lr=lr, variability=vb, num_iters=num_iters, betas=(0.9, 0.999), eps=1e-08, weight_decay=weight_decay, noise_type='Gaussian')\n",
    "    elif opti_name == 'VSGD-Uniform':\n",
    "        return VSGD(net.parameters(), lr=lr, variability=vb, num_iters=num_iters, weight_decay=weight_decay, noise_type='Uniform')\n",
    "    elif opti_name == 'VAdam-Uniform':\n",
    "        return VAdam(net.parameters(), lr=lr, variability=vb, num_iters=num_iters, betas=(0.9, 0.999), eps=1e-08, weight_decay=weight_decay, noise_type='Uniform')\n",
    "    elif opti_name == 'VSGD-Laplace':\n",
    "        return VSGD(net.parameters(), lr=lr, variability=vb, num_iters=num_iters, weight_decay=weight_decay, noise_type='Laplace')\n",
    "    elif opti_name == 'VAdam-Laplce':\n",
    "        return VAdam(net.parameters(), lr=lr, variability=vb, num_iters=num_iters, betas=(0.9, 0.999), eps=1e-08, weight_decay=weight_decay, noise_type='Laplace')\n",
    "    else:\n",
    "        raise('Unspecified optimizer.')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss(reduction='mean')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(net, optimizer, epoch):\n",
    "    print('Epoch: %d' % (epoch+1))\n",
    "    net.train()\n",
    "    train_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets, _) in enumerate(trainloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        train_loss += loss.item() * targets.size(0)\n",
    "        _, predicted = outputs.max(1)\n",
    "        total += targets.size(0)\n",
    "        correct += predicted.eq(targets).sum().item()\n",
    "    print(\"Training Loss: \", train_loss/total)\n",
    "    print(\"Training error:\", 1-correct/total)\n",
    "    return 1 - correct/total, train_loss/total\n",
    "\n",
    "def test(net):\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (inputs, targets, _) in enumerate(testloader):\n",
    "            inputs, targets = inputs.to(device), targets.to(device)\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total += targets.size(0)\n",
    "            correct += predicted.eq(targets).sum().item()\n",
    "    print(\"Test error:\", 1-correct/total)\n",
    "    return 1 - correct/total\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def define_models(model):\n",
    "    if model == 'resnet18':\n",
    "        return ResNet18()\n",
    "    elif model == 'resnet34':\n",
    "        return ResNet34()\n",
    "    elif model == 'vgg16':\n",
    "        return VGG('VGG16')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "9 10\n",
      "50000\n",
      "Actual noise 0.40\n",
      "[[0.6 0.4 0.  0.  0.  0.  0.  0.  0.  0. ]\n",
      " [0.  0.6 0.4 0.  0.  0.  0.  0.  0.  0. ]\n",
      " [0.  0.  0.6 0.4 0.  0.  0.  0.  0.  0. ]\n",
      " [0.  0.  0.  0.6 0.4 0.  0.  0.  0.  0. ]\n",
      " [0.  0.  0.  0.  0.6 0.4 0.  0.  0.  0. ]\n",
      " [0.  0.  0.  0.  0.  0.6 0.4 0.  0.  0. ]\n",
      " [0.  0.  0.  0.  0.  0.  0.6 0.4 0.  0. ]\n",
      " [0.  0.  0.  0.  0.  0.  0.  0.6 0.4 0. ]\n",
      " [0.  0.  0.  0.  0.  0.  0.  0.  0.6 0.4]\n",
      " [0.4 0.  0.  0.  0.  0.  0.  0.  0.  0.6]]\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainset = CIFAR10(root='../data/',\n",
    "                        download=True,\n",
    "                        train=True,\n",
    "                        transform=transform_train,\n",
    "                        noise_type='pairflip',\n",
    "                        noise_rate=0.4\n",
    "                       )\n",
    "\n",
    "testset = CIFAR10(root='../data/',\n",
    "                       download=True,\n",
    "                       train=False,\n",
    "                       transform=transform_test,\n",
    "                       noise_type='pairflip',\n",
    "                       noise_rate=0.4\n",
    "                      )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inp_peformance(model, learning_rate, batch_size, vb, weight_decay, epochs, N, mode):\n",
    "    net = define_models(model)\n",
    "    net = net.to(device)\n",
    "    if device == 'cuda':\n",
    "        net = torch.nn.DataParallel(net)\n",
    "        cudnn.benchmark = True\n",
    "    \n",
    "    train_err = []\n",
    "    train_loss = []\n",
    "    test_err = []\n",
    "    \n",
    "    #num_iters = the num of minibatches, if we evaluate our model every epoch\n",
    "    num_iters = math.ceil(N/ batch_size) \n",
    "    optimizer = optimizers(net, mode, learning_rate, weight_decay,vb=vb,num_iters=num_iters)\n",
    "    \n",
    "    lambda_lr = lambda epoch: 0.1 ** (epoch // 100)\n",
    "    scheduler = optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=lambda_lr)\n",
    "    \n",
    "    start_time = time.time()\n",
    "    print(\"-\"*30+\" Mode \"+ mode+\" starts\" )\n",
    "    for epoch in range(epochs):\n",
    "        train_err_i, train_loss_i = train(net, optimizer, epoch)\n",
    "        train_err.append(train_err_i)\n",
    "        train_loss.append(train_loss_i)\n",
    "        test_err.append(test(net))\n",
    "        scheduler.step()\n",
    "        print (\"--- %s seconds ---\" % (time.time() - start_time))\n",
    "        \n",
    "    save_err({mode:train_loss}, {mode:train_err}, {mode:test_err}, model+'_'+mode, vb, learning_rate, batch_size, weight_decay, epochs, N)\n",
    "    return train_loss, train_err, test_err"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def inp_comparison(model, batch_size, weight_decay, epochs, N):\n",
    "    train_loss, train_err, test_err = {}, {}, {}\n",
    "    mode_list =  ['SGD','VSGD-Gaussian','VSGD-Uniform','VSGD-Laplace']\n",
    "    \n",
    "    #the learning rates for each optimizers in mode_list\n",
    "    lr_list = [1., 1., 1., 1.]\n",
    "    \n",
    "    #the variability scales for each optimizers in mode_list\n",
    "    vb_list = [0., 0.05, 0.05, 0.05]\n",
    "    \n",
    "    for i,mode in enumerate(mode_list):\n",
    "        train_loss[mode], train_err[mode], test_err[mode] = inp_peformance(model, lr_list[i], batch_size, vb_list[i], weight_decay, epochs, N, mode)\n",
    "    return train_loss, train_err, test_err \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_figure(model, train_loss, train_err, test_err, batch_size, weight_decay, epochs, N): \n",
    "    figure_name = 'LabelNoise40_' + model + '_B'+str(batch_size) + '_N'+ str(N) + '_E' + str(epochs)\n",
    "    \n",
    "    plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots\n",
    "    plt.rcParams['image.interpolation'] = 'nearest'\n",
    "    plt.rcParams['image.cmap'] = 'gray'\n",
    "\n",
    "    fig = plt.figure()\n",
    "\n",
    "    axes = plt.gca()\n",
    "    for key in test_err:\n",
    "        break\n",
    "    axes.set_ylim([0., 1.])\n",
    "    axes.set_xlim([0,epochs])\n",
    "    colors = ['red','blue','green','orange','brown','yellow','magenta', 'pink','black', 'cyan']\n",
    "    fix_str1 = 'Test: '\n",
    "    fix_str2 = 'Train: '\n",
    "    for idx, mode in enumerate(test_err):\n",
    "        plt.plot(np.arange(1,epochs+1), test_err[mode], label=fix_str1+mode, ls='solid', linewidth=2, color=colors[idx])\n",
    "        plt.plot(np.arange(1,epochs+1), train_err[mode], label=fix_str2+mode, ls='dashed', linewidth=2, color=colors[idx])\n",
    "\n",
    "    plt.ylabel('Errors')\n",
    "    plt.xlabel('Epochs')\n",
    "    plt.grid()\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "    \n",
    "    fig.savefig('NVRM_'+figure_name + '.png')\n",
    "    fig.savefig('NVRM_'+figure_name+'.pdf', format='pdf',bbox_inches = 'tight')\n",
    "    \n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_err(train_loss, train_err, test_err, model, vb, learning_rate, batch_size, weight_decay, epochs, N):\n",
    "    csvname = 'LabelNoiseA40_' + model  + '_S'+str(vb) + '_LR'+str(learning_rate) + '_B'+str(batch_size) + '_N'+ str(N) + '_E' + str(epochs)\n",
    "    current_name = csvname +'.csv'\n",
    "    \n",
    "    files_present = glob.glob(current_name)\n",
    "    if files_present:\n",
    "        print('WARNING: This file already exists!')\n",
    "    data_dict = {}\n",
    "    for mode in test_err:\n",
    "        data_dict[mode+'_test_err'] = test_err[mode]\n",
    "        data_dict[mode+'_training_err'] = train_err[mode]\n",
    "        data_dict[mode+'_training_loss'] = train_loss[mode]\n",
    "    df = pd.DataFrame(data=data_dict)\n",
    "    if not files_present:\n",
    "        df.to_csv(current_name, sep=',', header=True, index=False)\n",
    "    else:\n",
    "        print('WARNING: This file already exists!')\n",
    "        for i in range(1,30):\n",
    "            files_present = glob.glob(csvname+'_'+str(i)+'.csv')\n",
    "            if not files_present:\n",
    "                df.to_csv(csvname+'_'+str(i)+'.csv', sep=',', header=True, index=False)\n",
    "                return None\n",
    "    return None\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 128\n",
    "weight_decay = 1e-4\n",
    "epochs = 300\n",
    "model = 'resnet34'\n",
    "\n",
    "#Adjust the training data size\n",
    "N = 50000\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "subset_indices = []\n",
    "for i in range(N):\n",
    "    subset_indices.append(int(i*50000/N))\n",
    "    \n",
    "trainset_1 = torch.utils.data.Subset(trainset, subset_indices)\n",
    "trainloader = torch.utils.data.DataLoader(trainset_1, batch_size=batch_size, shuffle=True, num_workers=2)\n",
    "\n",
    "testloader = torch.utils.data.DataLoader(testset, batch_size=batch_size, shuffle=True, num_workers=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loss, train_err, test_err = inp_comparison(model, batch_size, weight_decay, epochs, N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_figure(model, train_loss, train_err, test_err, batch_size, weight_decay, epochs, N)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}