LTC_Implementation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "hVqpE8DSHMi5",
    "outputId": "8691d6ba-aa84-42d6-f9fa-3c3fe974ce82"
   },
   "outputs": [],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive/', force_remount=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "neqosKA9Znby"
   },
   "source": [
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1v57yOjjZnb0"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import torch\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.utils import shuffle\n",
    "from torch.autograd import Variable\n",
    "from sklearn.metrics import r2_score\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "q4iBmlTWaSJb"
   },
   "source": [
    "### Model - LTC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XOScChkVHv4n"
   },
   "outputs": [],
   "source": [
    "class LTC(nn.Module):\n",
    "    \n",
    "    \"\"\"\n",
    "    The LTC network.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, input_size, opt, d, pretrained=False):\n",
    "      \n",
    "        \"\"\"LTC Model\"\"\"\n",
    "        \n",
    "        super(LTC, self).__init__()\n",
    "        self.opt = opt\n",
    "        \n",
    "        self.LTC_conv1 = nn.Conv3d(opt[0], 64, kernel_size=(3, 3, 3), \n",
    "                                   padding=(1, 1, 1))\n",
    "        self.LTC_pool1 = nn.MaxPool3d(kernel_size=(1, 2, 2), stride=(1, 2, 2))\n",
    "        \n",
    "        self.LTC_conv2 = nn.Conv3d(64, 128, kernel_size=(3, 3, 3), \n",
    "                                   padding=(1, 1, 1))\n",
    "        self.LTC_pool2 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))\n",
    "        \n",
    "        self.LTC_conv3 = nn.Conv3d(128, 256, kernel_size=(3, 3, 3), \n",
    "                                   padding=(1, 1, 1))\n",
    "        self.LTC_pool3 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))\n",
    "        \n",
    "        self.LTC_conv4 = nn.Conv3d(256, 256, kernel_size=(3, 3, 3), \n",
    "                                   padding=(1, 1, 1))\n",
    "        self.LTC_pool4 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))\n",
    "        \n",
    "        self.LTC_conv5 = nn.Conv3d(256, 256, kernel_size=(3, 3, 3), \n",
    "                                   padding=(1, 1, 1))\n",
    "        self.LTC_pool5 = nn.MaxPool3d(kernel_size=(2, 2, 2), stride=(2, 2, 2))\n",
    "        \n",
    "        oT = math.floor(opt[1]/16); # 4 times max pooling of 1/2\n",
    "        oH = math.floor(opt[2]/32); # 5 times max pooling of 1/2\n",
    "        oW = math.floor(opt[3]/32); # 5 times max pooling of 1/2\n",
    "\n",
    "        self.LTC_fc1 = nn.Linear(256*oT*oH*oW, 2048)\n",
    "        self.LTC_fc2 = nn.Linear(2048, 2048)\n",
    "        \n",
    "        \n",
    "        \"\"\"Regression Model\"\"\"\n",
    "        \n",
    "        self.Reg_fc1 = nn.Linear(input_size, 512)\n",
    "        self.Reg_fc2 = nn.Linear(512, 1024)\n",
    "        \n",
    "        \n",
    "        \"\"\"Combined LTC + Regression feature maps\"\"\"\n",
    "        \n",
    "        self.Merged_fc1 = nn.Linear(3072, 1024)\n",
    "        self.batchnorm1 = nn.BatchNorm1d(1024)\n",
    "        \n",
    "        self.Merged_fc2 = nn.Linear(1024, 256)\n",
    "        self.batchnorm2 = nn.BatchNorm1d(256)\n",
    "\n",
    "        self.Merged_fc3 = nn.Linear(256, 64)\n",
    "        self.batchnorm3 = nn.BatchNorm1d(64)\n",
    "        \n",
    "        self.Merged_fc4 = nn.Linear(64, 1)\n",
    "        \n",
    "        self.relu = nn.ReLU()\n",
    "      \n",
    "        self.dropout_video = nn.Dropout(0.7)\n",
    "        self.dropout_meta  = nn.Dropout(0.5)\n",
    "        \n",
    "\n",
    "    def forward(self, x, metadata):\n",
    "      \n",
    "        \"\"\" Training - LTC (Video) \"\"\"\n",
    "        \n",
    "        x_conv1 = self.relu(self.LTC_conv1(x))\n",
    "        x_pool1 = self.LTC_pool1(x_conv1)\n",
    "        x_pool1 = self.dropout_video(x_pool1)\n",
    "\n",
    "        x_conv2 = self.relu(self.LTC_conv2(x_pool1))\n",
    "        x_pool2 = self.LTC_pool2(x_conv2)\n",
    "        x_pool2 = self.dropout_video(x_pool2)\n",
    "        \n",
    "        x_conv3 = self.relu(self.LTC_conv3(x_pool2))\n",
    "        x_pool3 = self.LTC_pool3(x_conv3)\n",
    "        x_pool3 = self.dropout_video(x_pool3)\n",
    "        \n",
    "        x_conv4 = self.relu(self.LTC_conv4(x_pool3))\n",
    "        x_pool4 = self.LTC_pool4(x_conv4)\n",
    "        x_pool4 = self.dropout_video(x_pool4)\n",
    "        \n",
    "        x_conv5 = self.relu(self.LTC_conv5(x_pool4))\n",
    "        x_pool5 = self.LTC_pool5(x_conv5)\n",
    "        x_pool5 = self.dropout_video(x_pool5)\n",
    "        \n",
    "        oT = math.floor(self.opt[1]/16); # 4 times max pooling of 1/2\n",
    "        oH = math.floor(self.opt[2]/32); # 5 times max pooling of 1/2\n",
    "        oW = math.floor(self.opt[3]/32); # 5 times max pooling of 1/2\n",
    "\n",
    "        x_fc = x_pool5.view(x_pool5.size()[0],256*oT*oH*oW)\n",
    "        x_fc1 = self.relu(self.LTC_fc1(x_fc))\n",
    "        x_fc2 = self.relu(self.LTC_fc2(x_fc1))\n",
    "\n",
    "        \n",
    "        \"\"\"Training - Regression (Metadata)\"\"\"\n",
    "        \n",
    "        metadata_fc1 = self.relu(self.Reg_fc1(metadata))\n",
    "        metadata_fc1_dropout = self.dropout_meta(metadata_fc1)\n",
    "        metadata_fc2 = self.relu(self.Reg_fc2(metadata_fc1_dropout))\n",
    "        \n",
    "        \n",
    "        \"\"\"Concatenated LTC + Regression feature maps\"\"\"\n",
    "        \n",
    "        flattened_layer = torch.cat((x_fc2, metadata_fc2), 1)\n",
    "        merged_fc1 = self.relu(self.Merged_fc1(flattened_layer))   \n",
    "        merged_fc1_dropout = self.dropout_meta(merged_fc1)\n",
    "       \n",
    "        merged_fc2 = self.relu(self.Merged_fc2(merged_fc1_dropout))   \n",
    "        merged_fc2_dropout = self.dropout_meta(merged_fc2)\n",
    "        \n",
    "        merged_fc3 = self.relu(self.Merged_fc3(merged_fc2_dropout))   \n",
    "        merged_fc3_dropout = self.dropout_meta(merged_fc3)\n",
    "        \n",
    "        result = self.Merged_fc4(merged_fc3_dropout)\n",
    "        \n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gsz8sGrQPOLN"
   },
   "outputs": [],
   "source": [
    "def train(traintensor, label, metadata, batchsize):\n",
    "    model.train()\n",
    "    traintensor = traintensor.cpu().data.numpy()\n",
    "    metadata = metadata.cpu().data.numpy()\n",
    "    nbatch = traintensor.shape[0] / batchsize\n",
    "    split_tensor = np.array_split(traintensor, nbatch)\n",
    "    split_data = np.array_split(metadata, nbatch)\n",
    "    start, end = 0, 0\n",
    "    trainloss_batch = []\n",
    "    prediction = []\n",
    "    for pos in range(0, len(split_tensor)):\n",
    "        optimizer.zero_grad()\n",
    "        output = model.forward(torch.FloatTensor(split_tensor[pos]).cuda(), torch.FloatTensor(split_data[pos]).cuda())\n",
    "        start = end\n",
    "        end = start + split_tensor[pos].shape[0]\n",
    "        L = loss(output, torch.tensor(label[start:end]))\n",
    "        L.backward()\n",
    "        optimizer.step()\n",
    "        trainloss_batch.append(L.item())\n",
    "        prediction.extend([output.tolist(), label[start:end]])\n",
    "    print(\"Train Loss: \", L.item())\n",
    "    return np.average(trainloss_batch), prediction\n",
    "\n",
    "\n",
    "def test(testtensor, label, metadata, batchsize):\n",
    "    model.eval()\n",
    "    testtensor = testtensor.cpu().data.numpy()\n",
    "    metadata = metadata.cpu().data.numpy()\n",
    "    nbatch = testtensor.shape[0] / batchsize\n",
    "    split_tensor = np.array_split(testtensor, nbatch)\n",
    "    split_data = np.array_split(metadata, nbatch)\n",
    "    start, end = 0, 0\n",
    "    testloss_batch = []\n",
    "    prediction = []\n",
    "    for pos in range(0, len(split_tensor)):\n",
    "        output = model.forward(torch.FloatTensor(split_tensor[pos]).cuda(), torch.FloatTensor(split_data[pos]).cuda())\n",
    "        start = end\n",
    "        end = start + split_tensor[pos].shape[0]\n",
    "        L = loss(output, torch.tensor(label[start:end]))\n",
    "        testloss_batch.append(L.item())\n",
    "        prediction.extend([output.tolist(), label[start:end]])\n",
    "    print(\"Test Loss: \", np.average(testloss_batch))\n",
    "    return np.average(testloss_batch), prediction\n",
    "\n",
    "\n",
    "def split_tensor(tensor, channels, timeDepth, xSize, ySize, train_id,\n",
    "                 test_id):\n",
    "    traintensor = torch.FloatTensor(len(train_id), channels,\n",
    "                                    timeDepth, xSize, ySize)\n",
    "    testtensor = torch.FloatTensor(len(test_id), channels,\n",
    "                                   timeDepth, xSize, ySize)\n",
    "\n",
    "    for i in range(len(traintensor)):\n",
    "        traintensor[i] = tensor[train_id[i]]\n",
    "\n",
    "    for i in range(len(testtensor)):\n",
    "        testtensor[i] = tensor[test_id[i]]\n",
    "\n",
    "    return traintensor, testtensor\n",
    "\n",
    "\n",
    "def split_metadata(X, Y, train_id, test_id):\n",
    "    return X[train_id], X[test_id], Y[train_id], Y[test_id]\n",
    "\n",
    "\n",
    "def save_model(model, output_modelpath, e, trainLoss, testLoss):\n",
    "    torch.save(model.state_dict(), newmodelpath + str(e + 1) + \".pt\")\n",
    "    np.savetxt(output_modelpath+'train'+str(e+1)+'.out', np.array(trainLoss), delimiter=',')\n",
    "    np.savetxt(output_modelpath+'test'+str(e+1)+'.out', np.array(testLoss), delimiter=',')\n",
    "    \n",
    "    \n",
    "def plot_lossgraph(loss, loss_=None):\n",
    "    fig, axs = plt.subplots(1, 1, figsize=(10, 10))\n",
    "    fig.suptitle(loss_)\n",
    "    axs.plot(loss)\n",
    "    axs.plot(loss, \"o\")\n",
    "    axs.set_ylabel('Loss')\n",
    "    axs.set_xlabel('Epochs')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "D2d3OzhePc3V",
    "outputId": "507c428c-66ca-4bb1-c4c3-3f35575d740e"
   },
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "\n",
    "    train_metadatapath = \"gdrive/My Drive/Project - Box Office Prediction/Dataset/Metadata/Train_2382.csv\"\n",
    "    test_metadatapath = \"gdrive/My Drive/Project - Box Office Prediction/Dataset/Metadata/Test_298.csv\"\n",
    "    output_modelpath = \"gdrive/My Drive/Project - Box Office Prediction/Mehmood/Model/LTC_0318/\"\n",
    "    # modelpath = output_modelpath+\"2100.pt\"    #Load a pre-trained model to resume the training\n",
    "    timeDepth = 16\n",
    "    channels = 6\n",
    "    xSize, ySize = 58, 58\n",
    "    trainLoss = []\n",
    "    testLoss = []\n",
    "    epochs = 2000\n",
    "    checkpoint_epoch = 1\n",
    "\n",
    "    print(\"Checking runtime configuation...\")\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    print(\"Runtime: \", device)\n",
    "    if device == \"cpu\":\n",
    "        print(\"Change the runtime to GPU\")\n",
    "        exit(0)\n",
    "        \n",
    "    \"\"\"Process Metadata\"\"\"\n",
    "    \n",
    "    print(\"Processing metadata...\")\n",
    "    train_data = pd.read_csv(train_metadatapath)\n",
    "    test_data = pd.read_csv(test_metadatapath)\n",
    "    drop_columns = ['videoname']\n",
    "    \n",
    "    train_data = train_data.drop(columns=drop_columns)\n",
    "    test_data = test_data.drop(columns=drop_columns)\n",
    "    \n",
    "    Y_train = train_data['revenue'].values\n",
    "    train_data = train_data.drop(columns='revenue')\n",
    "    X_train = train_data.values\n",
    "    Y_test = test_data['revenue'].values\n",
    "    test_data = test_data.drop(columns='revenue')\n",
    "    \n",
    "    X_test = test_data.values\n",
    "    tr_datacolumns = train_data.columns\n",
    "    tr_num_cols = []\n",
    "    tst_datacolumns = test_data.columns\n",
    "    tst_num_cols = []\n",
    "    for i in range(0,len(tr_datacolumns)):\n",
    "        if(np.max(train_data[tr_datacolumns[i]]) > 1):\n",
    "            tr_num_cols.append(i)\n",
    "    for i in range(0,len(tst_datacolumns)):\n",
    "        if(np.max(test_data[tst_datacolumns[i]]) > 1):\n",
    "            tst_num_cols.append(i)\n",
    "\n",
    "    scaler_X = MinMaxScaler()\n",
    "    X_train[:, tr_num_cols] = scaler_X.fit_transform(X_train[:, tr_num_cols])\n",
    "    X_test[:, tr_num_cols] = scaler_X.transform(X_test[:, tr_num_cols])\n",
    "    scaler_Y = MinMaxScaler()\n",
    "    Y_train = scaler_Y.fit_transform(Y_train.reshape(-1, 1))\n",
    "    Y_test = scaler_Y.transform(Y_test.reshape(-1, 1))\n",
    "    \n",
    "    print(X_train.shape, X_test.shape)\n",
    "\n",
    "    print(\"Metadata processed successfully...\")\n",
    "\n",
    "    \"\"\"Process Trailers\"\"\"\n",
    "    \n",
    "    print(\"Loading Trailer Tensors... \")\n",
    "    trailertensor1 = torch.load('gdrive/My Drive/LTCTensor/TrailerTensor_1489_augmented_train.pt')\n",
    "    trailertensor2 = torch.load('gdrive/My Drive/LTCTensor/TrailerTensor_1489_augmented_train1.pt')\n",
    "    traintensor = torch.cat((trailertensor1, trailertensor2), 0)\n",
    "    testtensor = torch.load('gdrive/My Drive/LTCTensor/TrailerTensor_1489_augmented_test.pt')\n",
    "    print(\"Trailer Tensors Loaded Successfully...\")\n",
    "\n",
    "    model = LTC(input_size=X_train.shape[1],\n",
    "              opt=[channels, timeDepth, xSize, ySize], d=0.5, pretrained=True)\n",
    "    optimizer = optim.Adam(model.parameters(), lr=1e-7,\n",
    "                          betas=(0.9, 0.999), eps=1e-08)\n",
    "    loss = nn.MSELoss()\n",
    "\n",
    "    if torch.cuda.is_available():\n",
    "        model = torch.nn.DataParallel(model)\n",
    "        print(\"Model: \", type(model))\n",
    "        print(\"Devices: \", model.device_ids)\n",
    "        model = model.cuda()\n",
    "        # model.load_state_dict(torch.load(modelpath))\n",
    "        print('here')\n",
    "        loss = loss.cuda()\n",
    "        traintensor = traintensor.cuda()\n",
    "        testtensor  = testtensor.cuda()\n",
    "        X_train     = torch.FloatTensor(X_train).cuda()\n",
    "        Y_train     = torch.FloatTensor(Y_train).cuda()\n",
    "        X_test      = torch.FloatTensor(X_test).cuda()\n",
    "        Y_test      = torch.FloatTensor(Y_test).cuda()\n",
    "\n",
    "    print(\"Training the model...\")\n",
    "    for e in range(epochs):\n",
    "        print(\"\\nEpoch: \", e + 1, \"/\", epochs)\n",
    "        batchsize = 40\n",
    "        optimizer.zero_grad()\n",
    "        trLoss, pred_train = train(traintensor, Y_train,\n",
    "                      X_train, batchsize)\n",
    "        trainLoss.append(trLoss)\n",
    "        tstLoss, prediction = test(testtensor, Y_test,\n",
    "                X_test, batchsize)\n",
    "        testLoss.append(tstLoss)\n",
    "        pred = []\n",
    "        pred_tr = []\n",
    "        for pr in range(len(prediction)):\n",
    "            if pr%2==0:\n",
    "                pred.extend(prediction[pr])\n",
    "\n",
    "        for pr in range(len(pred_train)):\n",
    "            if pr%2==0:\n",
    "                pred_tr.extend(pred_train[pr])    \n",
    "\n",
    "        Y_t = scaler_Y.inverse_transform(Y_test.cpu().data.numpy())\n",
    "        p_t = scaler_Y.inverse_transform(np.array(pred).reshape(-1,1))\n",
    "        Y_tr = scaler_Y.inverse_transform(Y_train.cpu().data.numpy())\n",
    "        p_tr = scaler_Y.inverse_transform(np.array(pred_tr).reshape(-1,1))\n",
    "\n",
    "        print(\"r2_train\", r2_score(Y_tr, p_tr))\n",
    "        print(\"r2_test\", r2_score(Y_t, p_t))\n",
    "\n",
    "        if (e + 1) % checkpoint_epoch == 0:\n",
    "            # save_model(model, output_modelpath, e, trainLoss, testLoss)\n",
    "            plot_lossgraph(trainLoss, \"Training Loss\")\n",
    "            plot_lossgraph(testLoss, \"Testing Loss\")\n",
    "            \n",
    "    print(\"Training compeleted...\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wjqSnYmp04zA"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "LTC_Implementation.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}