06_Data_analysis_with_spaCy.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "06. Data analysis with spaCy.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMCZ7CNnyylGX10C9sa505e",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tanaymukherjee/Natural-Language-Processing/blob/master/06_Data_analysis_with_spaCy.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nlDmHI9gy9kC",
        "colab_type": "text"
      },
      "source": [
        "# Data Analysis with SpaCy"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KFobMjLGzBap",
        "colab_type": "text"
      },
      "source": [
        "## Import libraries and create NLP object"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wOaul2dAxspD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Create an NLP object\n",
        "from spacy.lang.en import English\n",
        "nlp = English()"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d45T_NoEx_kq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Import the Doc class\n",
        "from spacy.tokens import Doc"
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Mla0pgOTy67Q",
        "colab_type": "text"
      },
      "source": [
        "## Strings to hashes"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fcYhs62UyvGK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "d6aef8ed-fc50-4e31-ca3e-18acbbf1cab9"
      },
      "source": [
        "# Look up the hash for the string label \"PERSON\"\n",
        "person_hash = nlp.vocab.strings['PERSON']\n",
        "print(person_hash)\n",
        "\n",
        "# Look up the person_hash to get the string\n",
        "person_string = nlp.vocab.strings[person_hash]\n",
        "print(person_string)"
      ],
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "380\n",
            "PERSON\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JY_lhFyhzIpv",
        "colab_type": "text"
      },
      "source": [
        "## Docs, Spans and Entities"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VsOQJw-uyJR6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "5e7535d0-8d33-4a98-db7a-52fa6847bf03"
      },
      "source": [
        "# Desired text: \"spaCy is cool!\"\n",
        "words = ['spaCy', 'is', 'cool', '!']\n",
        "spaces = [True, True, False, False]\n",
        "\n",
        "# Create a Doc from the words and spaces\n",
        "doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
        "print(doc.text)"
      ],
      "execution_count": 46,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "spaCy is cool!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XFgZDpT8zRYn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "014603df-1fd3-471f-eab1-646d2b4734e3"
      },
      "source": [
        "# Import the Doc and Span classes\n",
        "from spacy.tokens import Doc, Span\n",
        "\n",
        "words = ['I', 'like', 'Rafael', 'Nadal']\n",
        "spaces = [True, True, True, False]\n",
        "\n",
        "# Create a doc from the words and spaces\n",
        "doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
        "print(doc.text)"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I like Rafael Nadal\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vxEtOs3UzXXo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "85e78df1-06e4-4e74-ac6d-f3c676999070"
      },
      "source": [
        "# Import the Doc and Span classes\n",
        "from spacy.tokens import Doc, Span\n",
        "\n",
        "# Create a doc from the words and spaces\n",
        "doc = Doc(nlp.vocab, words=['I', 'like', 'Rafael', 'Nadal'], spaces=[True, True, True, False])\n",
        "\n",
        "# Create a span for \"Rafeal Nadal\" from the doc and assign it the label \"PERSON\"\n",
        "span = Span(doc, 2, 4, label='PERSON')\n",
        "print(span.text, span.label_)"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Rafael Nadal PERSON\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XD-dcc5gzfLP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "b7ea276c-8a5e-438c-e5e3-c81c1952fd5c"
      },
      "source": [
        "# Import the Doc and Span classes\n",
        "from spacy.tokens import Doc, Span\n",
        "\n",
        "# Create a doc from the words and spaces\n",
        "doc = Doc(nlp.vocab, words=['I', 'like', 'Rafael', 'Nadal'], spaces=[True, True, True, False])\n",
        "\n",
        "# Create a span for \"Rafeal Nadal\"  from the doc and assign it the label \"PERSON\"\n",
        "span = Span(doc, 2, 4, label='PERSON')\n",
        "\n",
        "# Add the span to the doc's entities\n",
        "doc.ents = [span]\n",
        "\n",
        "# Print entities' text and labels\n",
        "print([(ent.text, ent.label_) for ent in doc.ents])"
      ],
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('Rafael Nadal', 'PERSON')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XXu_naDX06uT",
        "colab_type": "text"
      },
      "source": [
        "Code in this example is trying to analyze a text and collect all proper nouns. If the token following the proper noun is a verb, it should also be extracted."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Hi7YE51z8mY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "3c84255a-b88c-41d6-8180-93b9e2719da3"
      },
      "source": [
        "# Create a doc from the words and spaces\n",
        "doc = Doc(nlp.vocab, words=['I', 'live', 'in', 'New York City'], spaces=[True, True, True, False])\n",
        "print(doc.text)\n",
        "\n",
        "# Get all tokens and part-of-speech tags\n",
        "pos_tags = [token.pos_ for token in doc]\n",
        "\n",
        "for index, pos in enumerate(pos_tags):\n",
        "    # Check if the current token is a proper noun\n",
        "    if pos == 'PROPN':\n",
        "        # Check if the next token is a verb\n",
        "        if pos_tags[index + 1] == 'VERB':\n",
        "            print('Found a verb after a proper noun!')"
      ],
      "execution_count": 50,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I live in New York City\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "31Zr5bK5_d6N",
        "colab_type": "text"
      },
      "source": [
        "## Word vectors and similarity"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZTnd0DiY-c0Y",
        "colab_type": "text"
      },
      "source": [
        "### Inspecting word vectors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "81AQ1SjA-W2l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy"
      ],
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0WL_8H4L-aiF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        },
        "outputId": "aa5cc099-2005-4dc1-b73f-71634f50ab4e"
      },
      "source": [
        "!python -m spacy download en_core_web_md"
      ],
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: en_core_web_md==2.2.5 from https://github.com/explosion/spacy-models/releases/download/en_core_web_md-2.2.5/en_core_web_md-2.2.5.tar.gz#egg=en_core_web_md==2.2.5 in /usr/local/lib/python3.6/dist-packages (2.2.5)\n",
            "Requirement already satisfied: spacy>=2.2.2 in /usr/local/lib/python3.6/dist-packages (from en_core_web_md==2.2.5) (2.2.4)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (1.18.5)\n",
            "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (1.0.0)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (0.7.0)\n",
            "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (1.0.2)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (2.23.0)\n",
            "Requirement already satisfied: thinc==7.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (7.4.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (4.41.1)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (3.0.2)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (1.0.2)\n",
            "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (1.1.3)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (2.0.3)\n",
            "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (0.4.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from spacy>=2.2.2->en_core_web_md==2.2.5) (49.1.0)\n",
            "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_md==2.2.5) (1.7.0)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_md==2.2.5) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_md==2.2.5) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_md==2.2.5) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests<3.0.0,>=2.13.0->spacy>=2.2.2->en_core_web_md==2.2.5) (3.0.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy>=2.2.2->en_core_web_md==2.2.5) (3.1.0)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the model via spacy.load('en_core_web_md')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UHJfrSmz-6tG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import en_core_web_md\n",
        "nlp = en_core_web_md.load()"
      ],
      "execution_count": 53,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4AqH04z1-Jk6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 897
        },
        "outputId": "d8520abe-94aa-4874-e253-e6a2760ad67d"
      },
      "source": [
        "# Load the en_core_web_md model\n",
        "# nlp = spacy.load('en_core_web_md')\n",
        "\n",
        "# Process a text\n",
        "doc = nlp(\"Two bananas in pyjamas\")\n",
        "\n",
        "# Get the vector for the token \"bananas\"\n",
        "bananas_vector = doc[1].vector\n",
        "print(bananas_vector)"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[-2.2009e-01 -3.0322e-02 -7.9859e-02 -4.6279e-01 -3.8600e-01  3.6962e-01\n",
            " -7.7178e-01 -1.1529e-01  3.3601e-02  5.6573e-01 -2.4001e-01  4.1833e-01\n",
            "  1.5049e-01  3.5621e-01 -2.1508e-01 -4.2743e-01  8.1400e-02  3.3916e-01\n",
            "  2.1637e-01  1.4792e-01  4.5811e-01  2.0966e-01 -3.5706e-01  2.3800e-01\n",
            "  2.7971e-02 -8.4538e-01  4.1917e-01 -3.9181e-01  4.0434e-04 -1.0662e+00\n",
            "  1.4591e-01  1.4643e-03  5.1277e-01  2.6072e-01  8.3785e-02  3.0340e-01\n",
            "  1.8579e-01  5.9999e-02 -4.0270e-01  5.0888e-01 -1.1358e-01 -2.8854e-01\n",
            " -2.7068e-01  1.1017e-02 -2.2217e-01  6.9076e-01  3.6459e-02  3.0394e-01\n",
            "  5.6989e-02  2.2733e-01 -9.9473e-02  1.5165e-01  1.3540e-01 -2.4965e-01\n",
            "  9.8078e-01 -8.0492e-01  1.9326e-01  3.1128e-01  5.5390e-02 -4.2423e-01\n",
            " -1.4082e-02  1.2708e-01  1.8868e-01  5.9777e-02 -2.2215e-01 -8.3950e-01\n",
            "  9.1987e-02  1.0180e-01 -3.1299e-01  5.5083e-01 -3.0717e-01  4.4201e-01\n",
            "  1.2666e-01  3.7643e-01  3.2333e-01  9.5673e-02  2.5083e-01 -6.4049e-02\n",
            "  4.2143e-01 -1.9375e-01  3.8026e-01  7.0883e-03 -2.0371e-01  1.5402e-01\n",
            " -3.7877e-03 -2.9396e-01  9.6518e-01  2.0068e-01 -5.6572e-01 -2.2581e-01\n",
            "  3.2251e-01 -3.4634e-01  2.7064e-01 -2.0687e-01 -4.7229e-01  3.1704e-01\n",
            " -3.4665e-01 -2.5188e-01 -1.1201e-01 -3.3937e-01  3.1518e-01 -3.2221e-01\n",
            " -2.4530e-01 -7.1571e-02 -4.3971e-01 -1.2070e+00  3.3365e-01 -5.8208e-02\n",
            "  8.0899e-01  4.2335e-01  3.8678e-01 -6.0797e-01 -7.3760e-01 -2.0547e-01\n",
            " -1.7499e-01 -3.7842e-03  2.1930e-01 -5.2486e-02  3.4869e-01  4.3852e-01\n",
            " -3.4471e-01  2.8910e-01  7.2554e-02 -4.8625e-01 -3.8390e-01 -4.4760e-01\n",
            "  4.3278e-01 -2.7128e-03 -9.0067e-01 -3.0819e-02 -3.8630e-01 -8.0798e-02\n",
            " -1.6243e-01  2.8830e-01 -2.6349e-01  1.7628e-01  3.5958e-01  5.7672e-01\n",
            " -5.4624e-01  3.8555e-02 -2.0182e+00  3.2916e-01  3.4672e-01  1.5398e-01\n",
            " -4.3446e-01 -4.1428e-02 -6.9588e-02  5.1513e-01 -1.3489e-01 -5.7239e-02\n",
            "  4.9241e-01  1.8643e-01  3.8596e-01 -3.7329e-02 -5.4216e-01 -1.8152e-01\n",
            "  4.3110e-01 -4.6967e-01  6.6801e-02  5.0323e-01 -2.4059e-01  3.6742e-01\n",
            "  2.9300e-01 -8.7883e-02 -4.7940e-01 -4.3431e-02 -2.6137e-01 -6.2658e-01\n",
            "  1.1446e-01  2.7682e-01  3.4800e-01  5.0018e-01  1.4269e-01 -3.3545e-01\n",
            " -3.9712e-01 -3.3121e-01 -3.4434e-01 -4.1627e-01 -3.5707e-03 -6.2350e-01\n",
            "  3.7794e-01 -1.6765e-01 -4.1954e-01 -3.3134e-01  3.1232e-01 -3.9494e-01\n",
            " -4.6921e-03 -4.8884e-01 -2.2059e-02 -2.6174e-01  1.7937e-01  3.6628e-01\n",
            "  5.8971e-02 -3.5991e-01 -4.4393e-01 -1.1890e-01  3.3487e-01  3.6505e-02\n",
            " -3.2788e-01  3.3425e-01 -5.6361e-01 -1.1190e-01  5.3770e-01  2.0311e-01\n",
            "  1.5110e-01  1.0623e-02  3.3401e-01  4.6084e-01  5.6293e-01 -7.5432e-02\n",
            "  5.4813e-01  1.9395e-01 -2.6265e-01 -3.1699e-01 -8.1778e-01  5.8169e-02\n",
            " -5.7866e-02 -1.1781e-01 -5.8742e-02 -1.4092e-01 -9.9394e-01 -9.4532e-02\n",
            "  2.3503e-01 -4.9027e-01  8.5832e-01  1.1540e-01 -1.5049e-01  1.9065e-01\n",
            " -2.6705e-01  2.5326e-01 -6.7579e-01 -1.0633e-02 -5.5158e-02 -3.1004e-01\n",
            " -5.8036e-02 -1.7200e-01  1.3298e-01 -3.2899e-01 -7.5481e-02  2.9425e-02\n",
            " -3.2949e-01 -1.8691e-01 -9.5323e-01 -3.5468e-01 -3.3162e-01  5.6441e-02\n",
            "  2.1790e-02  1.7182e-01 -4.4267e-01  6.9765e-01 -2.6876e-01  1.1659e-01\n",
            " -1.6584e-01  3.8296e-01  2.9109e-01  3.6318e-01  3.6961e-01  1.6305e-01\n",
            "  1.8152e-01  2.2453e-01  3.9866e-02 -3.7607e-02 -3.6089e-01  7.0818e-02\n",
            " -2.1509e-01  3.6551e-01 -5.1603e-01 -5.8102e-03 -4.8320e-01 -2.5068e-01\n",
            " -5.2062e-02 -2.0828e-01  2.9060e-01  2.2084e-02 -6.8123e-01  4.2063e-01\n",
            "  9.5973e-02  8.1720e-01 -1.5241e-01  6.2994e-01  2.6449e-01 -1.3516e-01\n",
            "  3.2450e-01  3.0503e-01  1.2357e-01  1.5107e-01  2.8327e-01 -3.3838e-01\n",
            "  4.6106e-02 -1.2361e-01  1.4516e-01 -2.7947e-02  2.6231e-02 -5.9591e-01\n",
            " -4.4183e-01  7.8440e-01 -3.4375e-02 -1.3928e+00  3.5248e-01  6.5220e-01]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ziT2E_4R_hEE",
        "colab_type": "text"
      },
      "source": [
        "### Comparing similarities"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ytUwvtlq_i7M",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "0c543329-7f23-47d8-c6da-8d21f9874926"
      },
      "source": [
        "doc1 = nlp(\"It's a warm summer day\")\n",
        "doc2 = nlp(\"It's sunny outside\")\n",
        "\n",
        "# Get the similarity of doc1 and doc2\n",
        "similarity = doc1.similarity(doc2)\n",
        "print(similarity)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.8789265574516525\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZbgU2WBX_nPr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "dae784e2-699d-4605-adce-f517e407bd86"
      },
      "source": [
        "doc = nlp(\"TV and books\")\n",
        "token1, token2 = doc[0], doc[2]\n",
        "\n",
        "# Get the similarity of the tokens \"TV\" and \"books\" \n",
        "similarity = token1.similarity(token2)\n",
        "print(similarity)"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.22325331\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ucu84Yoy_sOb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "ca258edd-5a15-4d91-978e-0d1c5843defa"
      },
      "source": [
        "doc = nlp(\"This was a great restaurant. Afterwards, we went to a really nice bar.\")\n",
        "\n",
        "# Create spans for \"great restaurant\" and \"really nice bar\"\n",
        "span1 = doc[3:5]\n",
        "span2 = doc[12:15]\n",
        "\n",
        "# Get the similarity of the spans\n",
        "similarity = span1.similarity(span2)\n",
        "print(similarity)"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.75173926\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nS6lOppIQc0H",
        "colab_type": "text"
      },
      "source": [
        "## Debugging Patterns"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AFV3ZBOrQceL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Process a text\n",
        "doc = nlp(\"Twitch Prime, the perks program for Amazon Prime members offering free loot, games and other benefits, is ditching one of its best features: ad-free viewing. According to an email sent out to Amazon Prime members today, ad-free viewing will no longer be included as a part of Twitch Prime for new members, beginning on September 14. However, members with existing annual subscriptions will be able to continue to enjoy ad-free viewing until their subscription comes up for renewal. Those with monthly subscriptions will have access to ad-free viewing until October 15.\")"
      ],
      "execution_count": 58,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C1TKMsV1Q4mI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from spacy.matcher import Matcher\n",
        "matcher = Matcher(nlp.vocab)"
      ],
      "execution_count": 64,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I4lMyyr2QpZW",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 123
        },
        "outputId": "97853b19-e194-4a4d-ba9d-b29eeb478b28"
      },
      "source": [
        "# Create the match patterns\n",
        "pattern1 = [{'LOWER': 'amazon'}, {'IS_TITLE': True, 'POS': 'PROPN'}]\n",
        "pattern2 = [{'LOWER': 'ad'}, {'TEXT': '-'}, {'LOWER': 'free'}, {'POS': 'NOUN'}]\n",
        "\n",
        "# Initialize the Matcher and add the patterns\n",
        "matcher = Matcher(nlp.vocab)\n",
        "matcher.add('PATTERN1', None, pattern1)\n",
        "matcher.add('PATTERN2', None, pattern2)\n",
        "\n",
        "# Iterate over the matches\n",
        "for match_id, start, end in matcher(doc):\n",
        "    # Print pattern string name and text of matched span\n",
        "    print(doc.vocab.strings[match_id], doc[start:end].text)"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "PATTERN1 Amazon Prime\n",
            "PATTERN2 ad-free viewing\n",
            "PATTERN1 Amazon Prime\n",
            "PATTERN2 ad-free viewing\n",
            "PATTERN2 ad-free viewing\n",
            "PATTERN2 ad-free viewing\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cu_lyWngRRmL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Import the PhraseMatcher and initialize it\n",
        "from spacy.matcher import PhraseMatcher\n",
        "matcher = PhraseMatcher(nlp.vocab)"
      ],
      "execution_count": 68,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E7Sh6oNCRk2f",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "COUNTRIES = ['Afghanistan',\n",
        " 'Åland Islands',\n",
        " 'Albania',\n",
        " 'Algeria',\n",
        " 'American Samoa',\n",
        " 'Andorra',\n",
        " 'Angola',\n",
        " 'Anguilla',\n",
        " 'Antarctica',\n",
        " 'Antigua and Barbuda',\n",
        " 'Argentina',\n",
        " 'Armenia',\n",
        " 'Aruba',\n",
        " 'Australia',\n",
        " 'Austria',\n",
        " 'Azerbaijan',\n",
        " 'Bahamas',\n",
        " 'Bahrain',\n",
        " 'Bangladesh',\n",
        " 'Barbados',\n",
        " 'Belarus',\n",
        " 'Belgium',\n",
        " 'Belize',\n",
        " 'Benin',\n",
        " 'Bermuda',\n",
        " 'Bhutan',\n",
        " 'Bolivia (Plurinational State of)',\n",
        " 'Bonaire, Sint Eustatius and Saba',\n",
        " 'Bosnia and Herzegovina',\n",
        " 'Botswana',\n",
        " 'Bouvet Island',\n",
        " 'Brazil',\n",
        " 'British Indian Ocean Territory',\n",
        " 'United States Minor Outlying Islands',\n",
        " 'Virgin Islands (British)',\n",
        " 'Virgin Islands (U.S.)',\n",
        " 'Brunei Darussalam',\n",
        " 'Bulgaria',\n",
        " 'Burkina Faso',\n",
        " 'Burundi',\n",
        " 'Cambodia',\n",
        " 'Cameroon',\n",
        " 'Canada',\n",
        " 'Cabo Verde',\n",
        " 'Cayman Islands',\n",
        " 'Central African Republic',\n",
        " 'Chad',\n",
        " 'Chile',\n",
        " 'China',\n",
        " 'Christmas Island',\n",
        " 'Cocos (Keeling) Islands',\n",
        " 'Colombia',\n",
        " 'Comoros',\n",
        " 'Congo',\n",
        " 'Congo (Democratic Republic of the)',\n",
        " 'Cook Islands',\n",
        " 'Costa Rica',\n",
        " 'Croatia',\n",
        " 'Cuba',\n",
        " 'Curaçao',\n",
        " 'Cyprus',\n",
        " 'Czech Republic',\n",
        " 'Denmark',\n",
        " 'Djibouti',\n",
        " 'Dominica',\n",
        " 'Dominican Republic',\n",
        " 'Ecuador',\n",
        " 'Egypt',\n",
        " 'El Salvador',\n",
        " 'Equatorial Guinea',\n",
        " 'Eritrea',\n",
        " 'Estonia',\n",
        " 'Ethiopia',\n",
        " 'Falkland Islands (Malvinas)',\n",
        " 'Faroe Islands',\n",
        " 'Fiji',\n",
        " 'Finland',\n",
        " 'France',\n",
        " 'French Guiana',\n",
        " 'French Polynesia',\n",
        " 'French Southern Territories',\n",
        " 'Gabon',\n",
        " 'Gambia',\n",
        " 'Georgia',\n",
        " 'Germany',\n",
        " 'Ghana',\n",
        " 'Gibraltar',\n",
        " 'Greece',\n",
        " 'Greenland',\n",
        " 'Grenada',\n",
        " 'Guadeloupe',\n",
        " 'Guam',\n",
        " 'Guatemala',\n",
        " 'Guernsey',\n",
        " 'Guinea',\n",
        " 'Guinea-Bissau',\n",
        " 'Guyana',\n",
        " 'Haiti',\n",
        " 'Heard Island and McDonald Islands',\n",
        " 'Holy See',\n",
        " 'Honduras',\n",
        " 'Hong Kong',\n",
        " 'Hungary',\n",
        " 'Iceland',\n",
        " 'India',\n",
        " 'Indonesia',\n",
        " \"Côte d'Ivoire\",\n",
        " 'Iran (Islamic Republic of)',\n",
        " 'Iraq',\n",
        " 'Ireland',\n",
        " 'Isle of Man',\n",
        " 'Israel',\n",
        " 'Italy',\n",
        " 'Jamaica',\n",
        " 'Japan',\n",
        " 'Jersey',\n",
        " 'Jordan',\n",
        " 'Kazakhstan',\n",
        " 'Kenya',\n",
        " 'Kiribati',\n",
        " 'Kuwait',\n",
        " 'Kyrgyzstan',\n",
        " \"Lao People's Democratic Republic\",\n",
        " 'Latvia',\n",
        " 'Lebanon',\n",
        " 'Lesotho',\n",
        " 'Liberia',\n",
        " 'Libya',\n",
        " 'Liechtenstein',\n",
        " 'Lithuania',\n",
        " 'Luxembourg',\n",
        " 'Macao',\n",
        " 'Macedonia (the former Yugoslav Republic of)',\n",
        " 'Madagascar',\n",
        " 'Malawi',\n",
        " 'Malaysia',\n",
        " 'Maldives',\n",
        " 'Mali',\n",
        " 'Malta',\n",
        " 'Marshall Islands',\n",
        " 'Martinique',\n",
        " 'Mauritania',\n",
        " 'Mauritius',\n",
        " 'Mayotte',\n",
        " 'Mexico',\n",
        " 'Micronesia (Federated States of)',\n",
        " 'Moldova (Republic of)',\n",
        " 'Monaco',\n",
        " 'Mongolia',\n",
        " 'Montenegro',\n",
        " 'Montserrat',\n",
        " 'Morocco',\n",
        " 'Mozambique',\n",
        " 'Myanmar',\n",
        " 'Namibia',\n",
        " 'Nauru',\n",
        " 'Nepal',\n",
        " 'Netherlands',\n",
        " 'New Caledonia',\n",
        " 'New Zealand',\n",
        " 'Nicaragua',\n",
        " 'Niger',\n",
        " 'Nigeria',\n",
        " 'Niue',\n",
        " 'Norfolk Island',\n",
        " \"Korea (Democratic People's Republic of)\",\n",
        " 'Northern Mariana Islands',\n",
        " 'Norway',\n",
        " 'Oman',\n",
        " 'Pakistan',\n",
        " 'Palau',\n",
        " 'Palestine, State of',\n",
        " 'Panama',\n",
        " 'Papua New Guinea',\n",
        " 'Paraguay',\n",
        " 'Peru',\n",
        " 'Philippines',\n",
        " 'Pitcairn',\n",
        " 'Poland',\n",
        " 'Portugal',\n",
        " 'Puerto Rico',\n",
        " 'Qatar',\n",
        " 'Republic of Kosovo',\n",
        " 'Réunion',\n",
        " 'Romania',\n",
        " 'Russian Federation',\n",
        " 'Rwanda',\n",
        " 'Saint Barthélemy',\n",
        " 'Saint Helena, Ascension and Tristan da Cunha',\n",
        " 'Saint Kitts and Nevis',\n",
        " 'Saint Lucia',\n",
        " 'Saint Martin (French part)',\n",
        " 'Saint Pierre and Miquelon',\n",
        " 'Saint Vincent and the Grenadines',\n",
        " 'Samoa',\n",
        " 'San Marino',\n",
        " 'Sao Tome and Principe',\n",
        " 'Saudi Arabia',\n",
        " 'Senegal',\n",
        " 'Serbia',\n",
        " 'Seychelles',\n",
        " 'Sierra Leone',\n",
        " 'Singapore',\n",
        " 'Sint Maarten (Dutch part)',\n",
        " 'Slovakia',\n",
        " 'Slovenia',\n",
        " 'Solomon Islands',\n",
        " 'Somalia',\n",
        " 'South Africa',\n",
        " 'South Georgia and the South Sandwich Islands',\n",
        " 'Korea (Republic of)',\n",
        " 'South Sudan',\n",
        " 'Spain',\n",
        " 'Sri Lanka',\n",
        " 'Sudan',\n",
        " 'Suriname',\n",
        " 'Svalbard and Jan Mayen',\n",
        " 'Swaziland',\n",
        " 'Sweden',\n",
        " 'Switzerland',\n",
        " 'Syrian Arab Republic',\n",
        " 'Taiwan',\n",
        " 'Tajikistan',\n",
        " 'Tanzania, United Republic of',\n",
        " 'Thailand',\n",
        " 'Timor-Leste',\n",
        " 'Togo',\n",
        " 'Tokelau',\n",
        " 'Tonga',\n",
        " 'Trinidad and Tobago',\n",
        " 'Tunisia',\n",
        " 'Turkey',\n",
        " 'Turkmenistan',\n",
        " 'Turks and Caicos Islands',\n",
        " 'Tuvalu',\n",
        " 'Uganda',\n",
        " 'Ukraine',\n",
        " 'United Arab Emirates',\n",
        " 'United Kingdom of Great Britain and Northern Ireland',\n",
        " 'United States of America',\n",
        " 'Uruguay',\n",
        " 'Uzbekistan',\n",
        " 'Vanuatu',\n",
        " 'Venezuela (Bolivarian Republic of)',\n",
        " 'Viet Nam',\n",
        " 'Wallis and Futuna',\n",
        " 'Western Sahara',\n",
        " 'Yemen',\n",
        " 'Zambia',\n",
        " 'Zimbabwe']"
      ],
      "execution_count": 69,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9oTcelWkSGDg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "370a27e1-6069-456b-acd6-6c3ebc915b03"
      },
      "source": [
        "doc = nlp(\"Czech Republic may help Slovakia protect its airspace\")\n",
        "\n",
        "# Create pattern Doc objects and add them to the matcher\n",
        "# This is the faster version of: [nlp(country) for country in COUNTRIES]\n",
        "patterns = list(nlp.pipe(COUNTRIES))\n",
        "matcher.add('COUNTRY', None, *patterns)\n",
        "\n",
        "# Call the matcher on the test document and print the result\n",
        "matches = matcher(doc)\n",
        "print([doc[start:end] for match_id, start, end in matches])"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[Czech Republic, Slovakia]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LkFp_dZdSxd3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "text = 'After the Cold War, the UN saw a radical expansion in its peacekeeping duties, taking on more missions in ten years than it had in the previous four decades.Between 1988 and 2000, the number of adopted Security Council resolutions more than doubled, and the peacekeeping budget increased more than tenfold. The UN negotiated an end to the Salvadoran Civil War, launched a successful peacekeeping mission in Namibia, and oversaw democratic elections in post-apartheid South Africa and post-Khmer Rouge Cambodia. In 1991, the UN authorized a US-led coalition that repulsed the Iraqi invasion of Kuwait. Brian Urquhart, Under-Secretary-General from 1971 to 1985, later described the hopes raised by these successes as a \"false renaissance\" for the organization, given the more troubled missions that followed. Though the UN Charter had been written primarily to prevent aggression by one nation against another, in the early 1990s the UN faced a number of simultaneous, serious crises within nations such as Somalia, Haiti, Mozambique, and the former Yugoslavia. The UN mission in Somalia was widely viewed as a failure after the US withdrawal following casualties in the Battle of Mogadishu, and the UN mission to Bosnia faced \"worldwide ridicule\" for its indecisive and confused mission in the face of ethnic cleansing. In 1994, the UN Assistance Mission for Rwanda failed to intervene in the Rwandan genocide amid indecision in the Security Council. Beginning in the last decades of the Cold War, American and European critics of the UN condemned the organization for perceived mismanagement and corruption. In 1984, the US President, Ronald Reagan, withdrew his nation\\'s funding from UNESCO (the United Nations Educational, Scientific and Cultural Organization, founded 1946) over allegations of mismanagement, followed by Britain and Singapore. Boutros Boutros-Ghali, Secretary-General from 1992 to 1996, initiated a reform of the Secretariat, reducing the size of the organization somewhat. His successor, Kofi Annan (1997–2006), initiated further management reforms in the face of threats from the United States to withhold its UN dues. In the late 1990s and 2000s, international interventions authorized by the UN took a wider variety of forms. The UN mission in the Sierra Leone Civil War of 1991–2002 was supplemented by British Royal Marines, and the invasion of Afghanistan in 2001 was overseen by NATO. In 2003, the United States invaded Iraq despite failing to pass a UN Security Council resolution for authorization, prompting a new round of questioning of the organization\\'s effectiveness. Under the eighth Secretary-General, Ban Ki-moon, the UN has intervened with peacekeepers in crises including the War in Darfur in Sudan and the Kivu conflict in the Democratic Republic of Congo and sent observers and chemical weapons inspectors to the Syrian Civil War. In 2013, an internal review of UN actions in the final battles of the Sri Lankan Civil War in 2009 concluded that the organization had suffered \"systemic failure\". One hundred and one UN personnel died in the 2010 Haiti earthquake, the worst loss of life in the organization\\'s history. The Millennium Summit was held in 2000 to discuss the UN\\'s role in the 21st century. The three day meeting was the largest gathering of world leaders in history, and culminated in the adoption by all member states of the Millennium Development Goals (MDGs), a commitment to achieve international development in areas such as poverty reduction, gender equality, and public health. Progress towards these goals, which were to be met by 2015, was ultimately uneven. The 2005 World Summit reaffirmed the UN\\'s focus on promoting development, peacekeeping, human rights, and global security. The Sustainable Development Goals were launched in 2015 to succeed the Millennium Development Goals. In addition to addressing global challenges, the UN has sought to improve its accountability and democratic legitimacy by engaging more with civil society and fostering a global constituency. In an effort to enhance transparency, in 2016 the organization held its first public debate between candidates for Secretary-General. On 1 January 2017, Portuguese diplomat António Guterres, who previously served as UN High Commissioner for Refugees, became the ninth Secretary-General. Guterres has highlighted several key goals for his administration, including an emphasis on diplomacy for preventing conflicts, more effective peacekeeping efforts, and streamlining the organization to be more responsive and versatile to global needs.'"
      ],
      "execution_count": 73,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6TadcdpsTiA8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from spacy.lang.en import English\n",
        "nlp = English()"
      ],
      "execution_count": 76,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tiMakfI_S6R_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "5c4683f2-7035-4837-b73f-c2f8d8c434d0"
      },
      "source": [
        "# Create a doc and find matches in it\n",
        "doc = nlp(text)\n",
        "\n",
        "# Iterate over the matches\n",
        "for match_id, start, end in matcher(doc):\n",
        "    # Create a Span with the label for \"GPE\"\n",
        "    span = Span(doc, start, end, label='GPE')\n",
        "\n",
        "    # Overwrite the doc.ents and add the span\n",
        "    doc.ents = list(doc.ents) + [span]\n",
        "    \n",
        "# Print the entities in the document\n",
        "print([(ent.text, ent.label_) for ent in doc.ents if ent.label_ == 'GPE'])"
      ],
      "execution_count": 77,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[('Namibia', 'GPE'), ('South Africa', 'GPE'), ('Cambodia', 'GPE'), ('Kuwait', 'GPE'), ('Somalia', 'GPE'), ('Haiti', 'GPE'), ('Mozambique', 'GPE'), ('Somalia', 'GPE'), ('Rwanda', 'GPE'), ('Singapore', 'GPE'), ('Sierra Leone', 'GPE'), ('Afghanistan', 'GPE'), ('Iraq', 'GPE'), ('Sudan', 'GPE'), ('Congo', 'GPE'), ('Haiti', 'GPE')]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yTV3SB3mT6Ix",
        "colab_type": "text"
      },
      "source": [
        "Update the script and get the matched span's root head token."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9LRmj6RoT4QV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "outputId": "478cbfd4-e86a-4dda-d178-dc1e93445741"
      },
      "source": [
        "# Create a doc and find matches in it\n",
        "doc = nlp(text)\n",
        "\n",
        "# Iterate over the matches\n",
        "for match_id, start, end in matcher(doc):\n",
        "    # Create a Span with the label for \"GPE\" and overwrite the doc.ents\n",
        "    span = Span(doc, start, end, label='GPE')\n",
        "    doc.ents = list(doc.ents) + [span]\n",
        "    \n",
        "    # Get the span's root head token\n",
        "    span_root_head = span.root.head\n",
        "    # Print the text of the span root's head token and the span text\n",
        "    print(span_root_head.text, '-->', span.text)"
      ],
      "execution_count": 78,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Namibia --> Namibia\n",
            "South --> South Africa\n",
            "Cambodia --> Cambodia\n",
            "Kuwait --> Kuwait\n",
            "Somalia --> Somalia\n",
            "Haiti --> Haiti\n",
            "Mozambique --> Mozambique\n",
            "Somalia --> Somalia\n",
            "Rwanda --> Rwanda\n",
            "Singapore --> Singapore\n",
            "Sierra --> Sierra Leone\n",
            "Afghanistan --> Afghanistan\n",
            "Iraq --> Iraq\n",
            "Sudan --> Sudan\n",
            "Congo --> Congo\n",
            "Haiti --> Haiti\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}