Model_Parsing_Eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from darknet_parser import Darknet\n",
    "import cv2\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "from coco_evaluator import evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.set_device(torch.device('cuda:1')) # set cudea device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# YOLOv4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "cfgfile = '/workspace/GitHub/YOLO/cfg/yolov4.cfg'\n",
    "weights = '/workspace/GitHub/YOLO/weights/yolov4.weights'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model load from cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "convalution havn't activate linear\n",
      "convalution havn't activate linear\n",
      "convalution havn't activate linear\n",
      "layer     filters    size              input                output\n",
      "    0 conv     32  3 x 3 / 1   608 x 608 x   3   ->   608 x 608 x  32\n",
      "    1 conv     64  3 x 3 / 2   608 x 608 x  32   ->   304 x 304 x  64\n",
      "    2 conv     64  1 x 1 / 1   304 x 304 x  64   ->   304 x 304 x  64\n",
      "    3 route  1\n",
      "    4 conv     64  1 x 1 / 1   304 x 304 x  64   ->   304 x 304 x  64\n",
      "    5 conv     32  1 x 1 / 1   304 x 304 x  64   ->   304 x 304 x  32\n",
      "    6 conv     64  3 x 3 / 1   304 x 304 x  32   ->   304 x 304 x  64\n",
      "    7 shortcut 4\n",
      "    8 conv     64  1 x 1 / 1   304 x 304 x  64   ->   304 x 304 x  64\n",
      "    9 route  8 2\n",
      "   10 conv     64  1 x 1 / 1   304 x 304 x 128   ->   304 x 304 x  64\n",
      "   11 conv    128  3 x 3 / 2   304 x 304 x  64   ->   152 x 152 x 128\n",
      "   12 conv     64  1 x 1 / 1   152 x 152 x 128   ->   152 x 152 x  64\n",
      "   13 route  11\n",
      "   14 conv     64  1 x 1 / 1   152 x 152 x 128   ->   152 x 152 x  64\n",
      "   15 conv     64  1 x 1 / 1   152 x 152 x  64   ->   152 x 152 x  64\n",
      "   16 conv     64  3 x 3 / 1   152 x 152 x  64   ->   152 x 152 x  64\n",
      "   17 shortcut 14\n",
      "   18 conv     64  1 x 1 / 1   152 x 152 x  64   ->   152 x 152 x  64\n",
      "   19 conv     64  3 x 3 / 1   152 x 152 x  64   ->   152 x 152 x  64\n",
      "   20 shortcut 17\n",
      "   21 conv     64  1 x 1 / 1   152 x 152 x  64   ->   152 x 152 x  64\n",
      "   22 route  21 12\n",
      "   23 conv    128  1 x 1 / 1   152 x 152 x 128   ->   152 x 152 x 128\n",
      "   24 conv    256  3 x 3 / 2   152 x 152 x 128   ->    76 x  76 x 256\n",
      "   25 conv    128  1 x 1 / 1    76 x  76 x 256   ->    76 x  76 x 128\n",
      "   26 route  24\n",
      "   27 conv    128  1 x 1 / 1    76 x  76 x 256   ->    76 x  76 x 128\n",
      "   28 conv    128  1 x 1 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   29 conv    128  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   30 shortcut 27\n",
      "   31 conv    128  1 x 1 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   32 conv    128  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   33 shortcut 30\n",
      "   34 conv    128  1 x 1 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   35 conv    128  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   36 shortcut 33\n",
      "   37 conv    128  1 x 1 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   38 conv    128  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   39 shortcut 36\n",
      "   40 conv    128  1 x 1 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   41 conv    128  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   42 shortcut 39\n",
      "   43 conv    128  1 x 1 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   44 conv    128  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   45 shortcut 42\n",
      "   46 conv    128  1 x 1 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   47 conv    128  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   48 shortcut 45\n",
      "   49 conv    128  1 x 1 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   50 conv    128  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   51 shortcut 48\n",
      "   52 conv    128  1 x 1 / 1    76 x  76 x 128   ->    76 x  76 x 128\n",
      "   53 route  52 25\n",
      "   54 conv    256  1 x 1 / 1    76 x  76 x 256   ->    76 x  76 x 256\n",
      "   55 conv    512  3 x 3 / 2    76 x  76 x 256   ->    38 x  38 x 512\n",
      "   56 conv    256  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 256\n",
      "   57 route  55\n",
      "   58 conv    256  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 256\n",
      "   59 conv    256  1 x 1 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   60 conv    256  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   61 shortcut 58\n",
      "   62 conv    256  1 x 1 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   63 conv    256  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   64 shortcut 61\n",
      "   65 conv    256  1 x 1 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   66 conv    256  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   67 shortcut 64\n",
      "   68 conv    256  1 x 1 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   69 conv    256  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   70 shortcut 67\n",
      "   71 conv    256  1 x 1 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   72 conv    256  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   73 shortcut 70\n",
      "   74 conv    256  1 x 1 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   75 conv    256  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   76 shortcut 73\n",
      "   77 conv    256  1 x 1 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   78 conv    256  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   79 shortcut 76\n",
      "   80 conv    256  1 x 1 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   81 conv    256  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   82 shortcut 79\n",
      "   83 conv    256  1 x 1 / 1    38 x  38 x 256   ->    38 x  38 x 256\n",
      "   84 route  83 56\n",
      "   85 conv    512  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 512\n",
      "   86 conv   1024  3 x 3 / 2    38 x  38 x 512   ->    19 x  19 x1024\n",
      "   87 conv    512  1 x 1 / 1    19 x  19 x1024   ->    19 x  19 x 512\n",
      "   88 route  86\n",
      "   89 conv    512  1 x 1 / 1    19 x  19 x1024   ->    19 x  19 x 512\n",
      "   90 conv    512  1 x 1 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "   91 conv    512  3 x 3 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "   92 shortcut 89\n",
      "   93 conv    512  1 x 1 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "   94 conv    512  3 x 3 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "   95 shortcut 92\n",
      "   96 conv    512  1 x 1 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "   97 conv    512  3 x 3 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "   98 shortcut 95\n",
      "   99 conv    512  1 x 1 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "  100 conv    512  3 x 3 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "  101 shortcut 98\n",
      "  102 conv    512  1 x 1 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "  103 route  102 87\n",
      "  104 conv   1024  1 x 1 / 1    19 x  19 x1024   ->    19 x  19 x1024\n",
      "  105 conv    512  1 x 1 / 1    19 x  19 x1024   ->    19 x  19 x 512\n",
      "  106 conv   1024  3 x 3 / 1    19 x  19 x 512   ->    19 x  19 x1024\n",
      "  107 conv    512  1 x 1 / 1    19 x  19 x1024   ->    19 x  19 x 512\n",
      "  108 max          5 x 5 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "  109 route  107\n",
      "  110 max          9 x 9 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "  111 route  107\n",
      "  112 max          13 x 13 / 1    19 x  19 x 512   ->    19 x  19 x 512\n",
      "  113 route  112 110 108 107\n",
      "  114 conv    512  1 x 1 / 1    19 x  19 x2048   ->    19 x  19 x 512\n",
      "  115 conv   1024  3 x 3 / 1    19 x  19 x 512   ->    19 x  19 x1024\n",
      "  116 conv    512  1 x 1 / 1    19 x  19 x1024   ->    19 x  19 x 512\n",
      "  117 conv    256  1 x 1 / 1    19 x  19 x 512   ->    19 x  19 x 256\n",
      "  118 upsample           * 2    19 x  19 x 256   ->    38 x  38 x 256\n",
      "  119 route  85\n",
      "  120 conv    256  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 256\n",
      "  121 route  120 118\n",
      "  122 conv    256  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 256\n",
      "  123 conv    512  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 512\n",
      "  124 conv    256  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 256\n",
      "  125 conv    512  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 512\n",
      "  126 conv    256  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 256\n",
      "  127 conv    128  1 x 1 / 1    38 x  38 x 256   ->    38 x  38 x 128\n",
      "  128 upsample           * 2    38 x  38 x 128   ->    76 x  76 x 128\n",
      "  129 route  54\n",
      "  130 conv    128  1 x 1 / 1    76 x  76 x 256   ->    76 x  76 x 128\n",
      "  131 route  130 128\n",
      "  132 conv    128  1 x 1 / 1    76 x  76 x 256   ->    76 x  76 x 128\n",
      "  133 conv    256  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 256\n",
      "  134 conv    128  1 x 1 / 1    76 x  76 x 256   ->    76 x  76 x 128\n",
      "  135 conv    256  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 256\n",
      "  136 conv    128  1 x 1 / 1    76 x  76 x 256   ->    76 x  76 x 128\n",
      "  137 conv    256  3 x 3 / 1    76 x  76 x 128   ->    76 x  76 x 256\n",
      "  138 conv    255  1 x 1 / 1    76 x  76 x 256   ->    76 x  76 x 255\n",
      "  139 detection\n",
      "  140 route  136\n",
      "  141 conv    256  3 x 3 / 2    76 x  76 x 128   ->    38 x  38 x 256\n",
      "  142 route  141 126\n",
      "  143 conv    256  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 256\n",
      "  144 conv    512  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 512\n",
      "  145 conv    256  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 256\n",
      "  146 conv    512  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 512\n",
      "  147 conv    256  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 256\n",
      "  148 conv    512  3 x 3 / 1    38 x  38 x 256   ->    38 x  38 x 512\n",
      "  149 conv    255  1 x 1 / 1    38 x  38 x 512   ->    38 x  38 x 255\n",
      "  150 detection\n",
      "  151 route  147\n",
      "  152 conv    512  3 x 3 / 2    38 x  38 x 256   ->    19 x  19 x 512\n",
      "  153 route  152 116\n",
      "  154 conv    512  1 x 1 / 1    19 x  19 x1024   ->    19 x  19 x 512\n",
      "  155 conv   1024  3 x 3 / 1    19 x  19 x 512   ->    19 x  19 x1024\n",
      "  156 conv    512  1 x 1 / 1    19 x  19 x1024   ->    19 x  19 x 512\n",
      "  157 conv   1024  3 x 3 / 1    19 x  19 x 512   ->    19 x  19 x1024\n",
      "  158 conv    512  1 x 1 / 1    19 x  19 x1024   ->    19 x  19 x 512\n",
      "  159 conv   1024  3 x 3 / 1    19 x  19 x 512   ->    19 x  19 x1024\n",
      "  160 conv    255  1 x 1 / 1    19 x  19 x1024   ->    19 x  19 x 255\n",
      "  161 detection\n"
     ]
    }
   ],
   "source": [
    "m = Darknet(cfgfile)\n",
    "m.load_weights(weights)\n",
    "m.print_network()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class opt:\n",
    "    anno_json = '/workspace/GitHub/YOLO/coco/annotations/instances_val2017.json'\n",
    "    pred_json = './YOLOv4_pred_parserTest.json'\n",
    "    img_path = '/workspace/GitHub/YOLO/coco/images/val2017/'\n",
    "    img_size = 416\n",
    "    batch_size = 4\n",
    "    conf_thresh = 0.001\n",
    "    nms_thresh = 0.6\n",
    "    use_cuda = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=0.38s)\n",
      "creating index...\n",
      "index created!\n",
      "loading annotations into memory...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 1/1250 [00:00<02:28,  8.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done (t=0.49s)\n",
      "creating index...\n",
      "index created!\n",
      "STRAT detection\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1250/1250 [01:56<00:00, 10.69it/s]\n",
      "  1%|          | 57/5000 [00:00<00:08, 561.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Convert results to COCO format\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [00:08<00:00, 560.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading and preparing results...\n",
      "DONE (t=3.20s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=68.40s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=6.12s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.459\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.703\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.495\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.250\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.519\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.629\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.338\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.542\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.583\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.366\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.648\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.763\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<pycocotools.cocoeval.COCOeval at 0x7fbd17170410>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(m, torch.cuda.current_device(), opt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# YOLOv4 - Tiny"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "cfgfile = '/workspace/GitHub/YOLO/cfg/yolov4-tiny.cfg'\n",
    "weights = '/workspace/GitHub/YOLO/weights/yolov4-tiny.weights'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model load from cfg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "convalution havn't activate linear\n",
      "convalution havn't activate linear\n",
      "layer     filters    size              input                output\n",
      "    0 conv     32  3 x 3 / 2   416 x 416 x   3   ->   208 x 208 x  32\n",
      "    1 conv     64  3 x 3 / 2   208 x 208 x  32   ->   104 x 104 x  64\n",
      "    2 conv     64  3 x 3 / 1   104 x 104 x  64   ->   104 x 104 x  64\n",
      "    3 route  2\n",
      "    4 conv     32  3 x 3 / 1   104 x 104 x  64   ->   104 x 104 x  32\n",
      "    5 conv     32  3 x 3 / 1   104 x 104 x  32   ->   104 x 104 x  32\n",
      "    6 route  5 4\n",
      "    7 conv     64  1 x 1 / 1   104 x 104 x  64   ->   104 x 104 x  64\n",
      "    8 route  2 7\n",
      "    9 max          2 x 2 / 2   104 x 104 x 128   ->    52 x  52 x  64\n",
      "   10 conv    128  3 x 3 / 1    52 x  52 x  64   ->    52 x  52 x 128\n",
      "   11 route  10\n",
      "   12 conv     64  3 x 3 / 1    52 x  52 x 128   ->    52 x  52 x  64\n",
      "   13 conv     64  3 x 3 / 1    52 x  52 x  64   ->    52 x  52 x  64\n",
      "   14 route  13 12\n",
      "   15 conv    128  1 x 1 / 1    52 x  52 x 128   ->    52 x  52 x 128\n",
      "   16 route  10 15\n",
      "   17 max          2 x 2 / 2    52 x  52 x 256   ->    26 x  26 x 128\n",
      "   18 conv    256  3 x 3 / 1    26 x  26 x 128   ->    26 x  26 x 256\n",
      "   19 route  18\n",
      "   20 conv    128  3 x 3 / 1    26 x  26 x 256   ->    26 x  26 x 128\n",
      "   21 conv    128  3 x 3 / 1    26 x  26 x 128   ->    26 x  26 x 128\n",
      "   22 route  21 20\n",
      "   23 conv    256  1 x 1 / 1    26 x  26 x 256   ->    26 x  26 x 256\n",
      "   24 route  18 23\n",
      "   25 max          2 x 2 / 2    26 x  26 x 512   ->    13 x  13 x 256\n",
      "   26 conv    512  3 x 3 / 1    13 x  13 x 256   ->    13 x  13 x 512\n",
      "   27 conv    256  1 x 1 / 1    13 x  13 x 512   ->    13 x  13 x 256\n",
      "   28 conv    512  3 x 3 / 1    13 x  13 x 256   ->    13 x  13 x 512\n",
      "   29 conv    255  1 x 1 / 1    13 x  13 x 512   ->    13 x  13 x 255\n",
      "   30 detection\n",
      "   31 route  27\n",
      "   32 conv    128  1 x 1 / 1    13 x  13 x 256   ->    13 x  13 x 128\n",
      "   33 upsample           * 2    13 x  13 x 128   ->    26 x  26 x 128\n",
      "   34 route  33 23\n",
      "   35 conv    256  3 x 3 / 1    26 x  26 x 384   ->    26 x  26 x 256\n",
      "   36 conv    255  1 x 1 / 1    26 x  26 x 256   ->    26 x  26 x 255\n",
      "   37 detection\n"
     ]
    }
   ],
   "source": [
    "m_tiny = Darknet(cfgfile)\n",
    "m_tiny.load_weights(weights)\n",
    "m_tiny.print_network()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading annotations into memory...\n",
      "Done (t=1.59s)\n",
      "creating index...\n",
      "index created!\n",
      "loading annotations into memory...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 2/1250 [00:00<01:22, 15.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done (t=0.29s)\n",
      "creating index...\n",
      "index created!\n",
      "STRAT detection\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1250/1250 [01:05<00:00, 18.95it/s]\n",
      "  2%|▏         | 78/5000 [00:00<00:06, 769.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Convert results to COCO format\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5000/5000 [00:08<00:00, 592.69it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading and preparing results...\n",
      "DONE (t=3.22s)\n",
      "creating index...\n",
      "index created!\n",
      "Running per image evaluation...\n",
      "Evaluate annotation type *bbox*\n",
      "DONE (t=60.70s).\n",
      "Accumulating evaluation results...\n",
      "DONE (t=7.64s).\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.213\n",
      " Average Precision  (AP) @[ IoU=0.50      | area=   all | maxDets=100 ] = 0.404\n",
      " Average Precision  (AP) @[ IoU=0.75      | area=   all | maxDets=100 ] = 0.199\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.081\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.257\n",
      " Average Precision  (AP) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.299\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=  1 ] = 0.199\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets= 10 ] = 0.327\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=   all | maxDets=100 ] = 0.357\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= small | maxDets=100 ] = 0.144\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area=medium | maxDets=100 ] = 0.432\n",
      " Average Recall     (AR) @[ IoU=0.50:0.95 | area= large | maxDets=100 ] = 0.505\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<pycocotools.cocoeval.COCOeval at 0x7fbd48727550>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate(m_tiny, torch.cuda.current_device(), opt)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}