HandRecognizer.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Here are some installations, you dont need them if they are already installed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hGL97-GXjSUw",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#installations not needed but here for reference\n",
    "#pip install PyHamcrest-1.9.0-py2.py3-none-any.whl\n",
    "!pip install -U --pre tensorflow==\"2.*\"\n",
    "!pip install tf_slim\n",
    "!pip install pycocotools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ykA0c-om51s1"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pathlib\n",
    "if \"models\" in pathlib.Path.cwd().parts:\n",
    "  while \"models\" in pathlib.Path.cwd().parts:\n",
    "    os.chdir('..')\n",
    "elif not pathlib.Path('models').exists():\n",
    "  !git clone --depth 1 https://github.com/tensorflow/models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PY41vdYYNlXc"
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "cd models/research/\n",
    "protoc object_detection/protos/*.proto --python_out=."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "s62yJyQUcYbp"
   },
   "outputs": [],
   "source": [
    "%%bash \n",
    "cd models/research\n",
    "pip install ."
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "The following code are for imports that are needed:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hV4P5gyTWKMI",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\louli\\Anaconda\\lib\\site-packages\\object_detection\\utils\\visualization_utils.py:26: UserWarning: \n",
      "This call to matplotlib.use() has no effect because the backend has already\n",
      "been chosen; matplotlib.use() must be called *before* pylab, matplotlib.pyplot,\n",
      "or matplotlib.backends is imported for the first time.\n",
      "\n",
      "The backend was *originally* set to 'module://ipykernel.pylab.backend_inline' by the following code:\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\runpy.py\", line 193, in _run_module_as_main\n",
      "    \"__main__\", mod_spec)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\runpy.py\", line 85, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py\", line 16, in <module>\n",
      "    app.launch_new_instance()\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\traitlets\\config\\application.py\", line 658, in launch_instance\n",
      "    app.start()\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\ipykernel\\kernelapp.py\", line 499, in start\n",
      "    self.io_loop.start()\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 132, in start\n",
      "    self.asyncio_loop.run_forever()\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\asyncio\\base_events.py\", line 523, in run_forever\n",
      "    self._run_once()\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\asyncio\\base_events.py\", line 1758, in _run_once\n",
      "    handle._run()\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\asyncio\\events.py\", line 88, in _run\n",
      "    self._context.run(self._callback, *self._args)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\tornado\\platform\\asyncio.py\", line 122, in _handle_events\n",
      "    handler_func(fileobj, events)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\tornado\\stack_context.py\", line 300, in null_wrapper\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 450, in _handle_events\n",
      "    self._handle_recv()\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 480, in _handle_recv\n",
      "    self._run_callback(callback, msg)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\zmq\\eventloop\\zmqstream.py\", line 432, in _run_callback\n",
      "    callback(*args, **kwargs)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\tornado\\stack_context.py\", line 300, in null_wrapper\n",
      "    return fn(*args, **kwargs)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 283, in dispatcher\n",
      "    return self.dispatch_shell(stream, msg)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 233, in dispatch_shell\n",
      "    handler(stream, idents, msg)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\ipykernel\\kernelbase.py\", line 399, in execute_request\n",
      "    user_expressions, allow_stdin)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\ipykernel\\ipkernel.py\", line 208, in do_execute\n",
      "    res = shell.run_cell(code, store_history=store_history, silent=silent)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\ipykernel\\zmqshell.py\", line 537, in run_cell\n",
      "    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2662, in run_cell\n",
      "    raw_cell, store_history, silent, shell_futures)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2785, in _run_cell\n",
      "    interactivity=interactivity, compiler=compiler, result=result)\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2901, in run_ast_nodes\n",
      "    if self.run_code(code, result):\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\IPython\\core\\interactiveshell.py\", line 2961, in run_code\n",
      "    exec(code_obj, self.user_global_ns, self.user_ns)\n",
      "  File \"<ipython-input-1-1632146cb95d>\", line 12, in <module>\n",
      "    from matplotlib import pyplot as plt\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\matplotlib\\pyplot.py\", line 71, in <module>\n",
      "    from matplotlib.backends import pylab_setup\n",
      "  File \"C:\\Users\\louli\\Anaconda\\lib\\site-packages\\matplotlib\\backends\\__init__.py\", line 16, in <module>\n",
      "    line for line in traceback.format_stack()\n",
      "\n",
      "\n",
      "  import matplotlib; matplotlib.use('Agg')  # pylint: disable=multiple-statements\n"
     ]
    }
   ],
   "source": [
    "#imports needed, some errors may arise but dont worry about them too much\n",
    "import numpy as np\n",
    "import os\n",
    "import six.moves.urllib as urllib\n",
    "import sys\n",
    "import tarfile\n",
    "import tensorflow as tf\n",
    "import zipfile\n",
    "\n",
    "from collections import defaultdict\n",
    "from io import StringIO\n",
    "from matplotlib import pyplot as plt\n",
    "from PIL import Image\n",
    "from IPython.display import display\n",
    "\n",
    "from object_detection.utils import ops as utils_ops\n",
    "from object_detection.utils import label_map_util\n",
    "from object_detection.utils import visualization_utils as vis_util"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mF-YlMl8c_bM"
   },
   "outputs": [],
   "source": [
    "# patching is needed from tensorflow 1.0 to 2.0, here are the patches needed:\n",
    "# patch tf1 into `utils.ops`\n",
    "utils_ops.tf = tf.compat.v1\n",
    "\n",
    "# Patch the location of gfile\n",
    "tf.gfile = tf.io.gfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zm8xp-0eoItE"
   },
   "outputs": [],
   "source": [
    "#load model\n",
    "def load_model(model_name):\n",
    "  base_url = 'http://download.tensorflow.org/models/object_detection/'\n",
    "  model_file = model_name + '.tar.gz'\n",
    "  model_dir = tf.keras.utils.get_file(\n",
    "    fname=model_name, \n",
    "    origin=base_url + model_file,\n",
    "    untar=True)\n",
    "\n",
    "  model_dir = pathlib.Path(model_dir)/\"saved_model\"\n",
    "\n",
    "  model = tf.saved_model.load(str(model_dir))\n",
    "\n",
    "  return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hDbpHkiWWKMX"
   },
   "outputs": [],
   "source": [
    "# List of the labels and path to label_map\n",
    "PATH_TO_LABELS = 'Tensorflow/workspace/annotations/label_map.pbtxt'\n",
    "category_index = label_map_util.create_category_index_from_labelmap(PATH_TO_LABELS, use_display_name=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1XNT0wxybKR6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.4.1\n"
     ]
    }
   ],
   "source": [
    "#model_name = 'ssd_mobilenet_v1_coco_2017_11_17'\n",
    "#detection_model = load_model(model_name)\n",
    "#!conda install tensorflow==2.3.0\n",
    "import tensorflow as tf\n",
    "print(tf.__version__)\n",
    "detection_model = tf.saved_model.load('Tensorflow/workspace/exported-models/my_model/saved_model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#functions used to run the inferences\n",
    "def run_inference_for_single_image(model, image):\n",
    "  image = np.asarray(image)\n",
    "  # The input needs to be a tensor, convert it using `tf.convert_to_tensor`.\n",
    "  input_tensor = tf.convert_to_tensor(image)\n",
    "  # The model expects a batch of images, so add an axis with `tf.newaxis`.\n",
    "  input_tensor = input_tensor[tf.newaxis,...]\n",
    "\n",
    "  # Run inference\n",
    "  model_fn = model.signatures['serving_default']\n",
    "  output_dict = model_fn(input_tensor)\n",
    "#   print(output_dict)\n",
    "  # All outputs are batches tensors.\n",
    "  # Convert to numpy arrays, and take index [0] to remove the batch dimension.\n",
    "  # We're only interested in the first num_detections.\n",
    "  num_detections = int(output_dict.pop('num_detections'))\n",
    "  output_dict = {key:value[0, :num_detections].numpy() \n",
    "                 for key,value in output_dict.items()}\n",
    "  output_dict['num_detections'] = num_detections\n",
    "\n",
    "  # detection_classes should be ints.\n",
    "  output_dict['detection_classes'] = output_dict['detection_classes'].astype(np.int64)\n",
    "#   print(output_dict['detection_classes'])\n",
    "  # Handle models with masks:\n",
    "  if 'detection_masks' in output_dict:\n",
    "    # Reframe the the bbox mask to the image size.\n",
    "    detection_masks_reframed = utils_ops.reframe_box_masks_to_image_masks(\n",
    "              output_dict['detection_masks'], output_dict['detection_boxes'],\n",
    "               image.shape[0], image.shape[1])      \n",
    "    detection_masks_reframed = tf.cast(detection_masks_reframed > 0.8,\n",
    "                                       tf.uint8)\n",
    "    output_dict['detection_masks_reframed'] = detection_masks_reframed.numpy()\n",
    "    \n",
    "  return output_dict\n",
    "\n",
    "\n",
    "def show_inference(model, image_np):\n",
    "  # the array based representation of the image will be used later in order to prepare the\n",
    "  # result image with boxes and labels on it.\n",
    "#   image_np = np.array(Image.open(image_path))\n",
    "  # Actual detection.\n",
    "  output_dict = run_inference_for_single_image(model, image_np)\n",
    "\n",
    "#   print(category_index)\n",
    "  # Visualization of the results of a detection.\n",
    "  final_img =vis_util.visualize_boxes_and_labels_on_image_array(\n",
    "          image_np,\n",
    "          output_dict['detection_boxes'],\n",
    "          output_dict['detection_classes'],\n",
    "          output_dict['detection_scores'],\n",
    "          category_index,\n",
    "          max_boxes_to_draw=1,\n",
    "          min_score_thresh=.6,\n",
    "          instance_masks=output_dict.get('detection_masks_reframed', None),\n",
    "          use_normalized_coordinates=True,\n",
    "          line_thickness=8)\n",
    "  return(final_img)\n",
    "#   display(Image.fromarray(image_np))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#real time aspect. turns on the camera\n",
    "#press q to escape and stop the web cam feed\n",
    "import cv2\n",
    "\n",
    "cap = cv2.VideoCapture(0)\n",
    "\n",
    "while 1:\n",
    "    ret,img = cap.read()\n",
    "    img = cv2.cvtColor(img,cv2.COLOR_BGR2RGB)\n",
    "    final_img = show_inference(detection_model,img)\n",
    "    final_img = cv2.cvtColor(final_img,cv2.COLOR_RGB2BGR)\n",
    "    cv2.imshow('img',final_img)\n",
    "    if cv2.waitKey(1) == ord('q'):\n",
    "        break\n",
    "\n",
    "cap.release()\n",
    "cv2.destroyAllWindows()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "last_runtime": {
    "build_target": "//learning/brain/python/client:colab_notebook",
    "kind": "private"
   },
   "name": "object_detection_tutorial.ipynb",
   "private_outputs": true,
   "provenance": [
    {
     "file_id": "/piper/depot/google3/third_party/tensorflow_models/object_detection/colab_tutorials/object_detection_tutorial.ipynb",
     "timestamp": 1594335690840
    },
    {
     "file_id": "1LNYL6Zsn9Xlil2CVNOTsgDZQSBKeOjCh",
     "timestamp": 1566498233247
    },
    {
     "file_id": "/piper/depot/google3/third_party/tensorflow_models/object_detection/object_detection_tutorial.ipynb?workspaceId=markdaoust:copybara_AFABFE845DCD573AD3D43A6BAFBE77D4_0::citc",
     "timestamp": 1566488313397
    },
    {
     "file_id": "/piper/depot/google3/third_party/py/tensorflow_docs/g3doc/en/r2/tutorials/generative/object_detection_tutorial.ipynb?workspaceId=markdaoust:copybara_AFABFE845DCD573AD3D43A6BAFBE77D4_0::citc",
     "timestamp": 1566145894046
    },
    {
     "file_id": "1nBPoWynOV0auSIy40eQcBIk9C6YRSkI8",
     "timestamp": 1566145841085
    },
    {
     "file_id": "/piper/depot/google3/third_party/tensorflow_models/object_detection/object_detection_tutorial.ipynb?workspaceId=markdaoust:copybara_AFABFE845DCD573AD3D43A6BAFBE77D4_0::citc",
     "timestamp": 1556295408037
    },
    {
     "file_id": "1layerger-51XwWOwYMY_5zHaCavCeQkO",
     "timestamp": 1556214267924
    },
    {
     "file_id": "/piper/depot/google3/third_party/tensorflow_models/object_detection/object_detection_tutorial.ipynb?workspaceId=markdaoust:copybara_AFABFE845DCD573AD3D43A6BAFBE77D4_0::citc",
     "timestamp": 1556207836484
    },
    {
     "file_id": "1w6mqQiNV3liPIX70NOgitOlDF1_4sRMw",
     "timestamp": 1556154824101
    },
    {
     "file_id": "https://github.com/tensorflow/models/blob/master/research/object_detection/object_detection_tutorial.ipynb",
     "timestamp": 1556150293326
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}