Ch2 Large-scale data analysis with spaCy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 2: Large-scale data analysis with spaCy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Strings to Hashes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5439657043933447811\n",
      "cat\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "doc = nlp(\"I have a cat\")\n",
    "\n",
    "# Look up the hash for the word \"cat\"\n",
    "cat_hash = nlp.vocab.strings['cat']\n",
    "print(cat_hash)\n",
    "\n",
    "# Look up the cat_hash to get the string\n",
    "cat_string = nlp.vocab.strings[cat_hash]\n",
    "print(cat_string)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "380\n",
      "PERSON\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "doc = nlp(\"David Bowie is a PERSON\")\n",
    "\n",
    "# Look up the hash for the string label \"PERSON\"\n",
    "person_hash = nlp.vocab.strings['PERSON']\n",
    "print(person_hash)\n",
    "\n",
    "# Look up the person_hash to get the string\n",
    "person_string = nlp.vocab.strings[person_hash]\n",
    "print(person_string)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vocab, hashes and lexemes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2644858412616767388\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "\"[E018] Can't retrieve string for hash '2644858412616767388'. This usually refers to an issue with the `Vocab` or `StringStore`.\"",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [5], line 12\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[39mprint\u001b[39m(bowie_id)\n\u001b[1;32m     11\u001b[0m \u001b[39m# Look up the ID for \"Bowie\" in the vocab\u001b[39;00m\n\u001b[0;32m---> 12\u001b[0m \u001b[39mprint\u001b[39m(nlp_de\u001b[39m.\u001b[39;49mvocab\u001b[39m.\u001b[39;49mstrings[bowie_id])\n",
      "File \u001b[0;32m/usr/local/lib/python3.8/site-packages/spacy/strings.pyx:159\u001b[0m, in \u001b[0;36mspacy.strings.StringStore.__getitem__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: \"[E018] Can't retrieve string for hash '2644858412616767388'. This usually refers to an issue with the `Vocab` or `StringStore`.\""
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "# Create an English and German nlp object\n",
    "nlp = spacy.blank(\"en\")\n",
    "nlp_de = spacy.blank(\"de\")\n",
    "\n",
    "# Get the ID for the string 'Bowie'\n",
    "bowie_id = nlp.vocab.strings[\"Bowie\"]\n",
    "print(bowie_id)\n",
    "\n",
    "# Look up the ID for \"Bowie\" in the vocab\n",
    "print(nlp_de.vocab.strings[bowie_id])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a Doc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spaCy is cool!\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "\n",
    "# Import the Doc class\n",
    "from spacy.tokens import Doc\n",
    "\n",
    "# Desired text: \"spaCy is cool!\"\n",
    "words = [\"spaCy\", \"is\", \"cool\", \"!\"]\n",
    "spaces = [True, True, False, False]\n",
    "\n",
    "# Create a Doc from the words and spaces\n",
    "doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
    "print(doc.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go, get started!\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "\n",
    "# Import the Doc class\n",
    "from spacy.tokens import Doc\n",
    "\n",
    "# Desired text: \"Go, get started!\"\n",
    "words = [\"Go\", \",\", \"get\", \"started\", \"!\"]\n",
    "spaces = [False, True, True, False, False]\n",
    "\n",
    "# Create a Doc from the words and spaces\n",
    "doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
    "print(doc.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Oh, really?!\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "\n",
    "# Import the Doc class\n",
    "from spacy.tokens import Doc\n",
    "\n",
    "# Desired text: \"Oh, really?!\"\n",
    "words = ['Oh', ',', 'really', '?', '!']\n",
    "spaces = [False, True, False, False, False]\n",
    "\n",
    "# Create a Doc from the words and spaces\n",
    "doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
    "print(doc.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Docs, spans and entities from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I like David Bowie\n",
      "David Bowie PERSON\n",
      "[('David Bowie', 'PERSON')]\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "\n",
    "# Import the Doc and Span classes\n",
    "from spacy.tokens import Doc, Span\n",
    "\n",
    "words = [\"I\", \"like\", \"David\", \"Bowie\"]\n",
    "spaces = [True, True, True, False]\n",
    "\n",
    "# Create a doc from the words and spaces\n",
    "doc = Doc(nlp.vocab, words=words, spaces=spaces)\n",
    "print(doc.text)\n",
    "\n",
    "# Create a span for \"David Bowie\" from the doc and assign it the label \"PERSON\"\n",
    "span = Span(doc=doc, start=2, end=4, label='PERSON')\n",
    "print(span.text, span.label_)\n",
    "\n",
    "# Add the span to the doc's entities\n",
    "doc.ents = [span]\n",
    "\n",
    "# Print entities' text and labels\n",
    "print([(ent.text, ent.label_) for ent in doc.ents])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data structures best practices"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found proper noun before a verb: Berlin\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "doc = nlp(\"Berlin looks like a nice city\")\n",
    "\n",
    "# Get all tokens and part-of-speech tags\n",
    "token_texts = [token.text for token in doc]\n",
    "pos_tags = [token.pos_ for token in doc]\n",
    "\n",
    "for index, pos in enumerate(pos_tags):\n",
    "    # Check if the current token is a proper noun\n",
    "    if pos == \"PROPN\":\n",
    "        # Check if the next token is a verb\n",
    "        if pos_tags[index + 1] == \"VERB\":\n",
    "            result = token_texts[index]\n",
    "            print(\"Found proper noun before a verb:\", result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found proper noun before a verb: Berlin\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "doc = nlp(\"Berlin looks like a nice city\")\n",
    "\n",
    "for token in doc:\n",
    "    # Check if the current token is a proper noun\n",
    "    if token.pos_ == \"PROPN\" and doc[token.i+1].pos_ == \"VERB\":\n",
    "        print(\"Found proper noun before a verb:\", token.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inspecting word vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting en-core-web-md==3.4.1\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_md-3.4.1/en_core_web_md-3.4.1-py3-none-any.whl (42.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.8/42.8 MB\u001b[0m \u001b[31m271.3 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:04\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: spacy<3.5.0,>=3.4.0 in /usr/local/lib/python3.8/site-packages (from en-core-web-md==3.4.1) (3.4.2)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (2.0.7)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (3.0.10)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (0.6.2)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (1.23.4)\n",
      "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (8.1.5)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (1.0.3)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (0.4.2)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (2.28.1)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (3.3.0)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (1.10.2)\n",
      "Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (3.1.2)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (1.0.9)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (3.0.8)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (2.4.5)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (0.10.1)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (57.5.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (4.64.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (21.3)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.8/site-packages (from spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (2.0.8)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/site-packages (from packaging>=20.0->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (3.0.9)\n",
      "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in /usr/local/lib/python3.8/site-packages (from pathy>=0.3.5->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (5.2.1)\n",
      "Requirement already satisfied: typing-extensions>=4.1.0 in /usr/local/lib/python3.8/site-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (4.4.0)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /usr/local/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (2.1.1)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (1.26.12)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/site-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (2022.9.24)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.8/site-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (0.7.9)\n",
      "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.8/site-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (0.0.3)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.8/site-packages (from typer<0.5.0,>=0.3.0->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (8.1.3)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.8/site-packages (from jinja2->spacy<3.5.0,>=3.4.0->en-core-web-md==3.4.1) (2.1.1)\n",
      "Installing collected packages: en-core-web-md\n",
      "Successfully installed en-core-web-md-3.4.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('en_core_web_md')\n"
     ]
    }
   ],
   "source": [
    "!python3 -m spacy download en_core_web_md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-2.1689e-01 -2.5989e+00 -1.3144e+00  2.2500e+00 -4.6767e-01 -2.0695e+00\n",
      " -6.3379e-01 -4.0222e-01 -3.4022e+00 -3.6932e-01 -7.9938e-01 -1.0412e+00\n",
      "  9.3756e-01  1.6070e+00  8.8330e-01 -2.8483e+00  1.3349e-01 -3.1656e+00\n",
      "  8.1896e-01 -4.8113e+00  1.5655e+00  1.6665e+00 -4.7081e-01 -1.9475e+00\n",
      " -1.1779e+00 -1.3810e+00 -2.0071e+00 -2.1639e-01  9.0609e-01  1.5279e+00\n",
      "  1.2587e-04 -2.9000e+00  7.6069e-01 -2.2825e+00  1.2495e-02 -1.5653e+00\n",
      "  2.0052e+00 -1.7747e+00  5.9220e-01 -1.1428e+00 -1.3441e+00  3.4784e-01\n",
      "  1.7492e+00  1.9086e+00  1.0600e+00  1.2965e+00  4.1431e-01  7.9416e-01\n",
      " -1.1277e+00 -1.1403e+00  7.5891e-01 -9.4419e-01  1.4413e+00 -2.2554e+00\n",
      "  1.6226e-01  3.8901e-01  1.2299e-01  1.1577e+00  1.5524e+00  1.3853e+00\n",
      "  1.1112e+00  7.5767e-01  3.9431e+00 -2.8506e-01 -2.1645e+00 -1.0862e+00\n",
      " -1.4973e+00 -1.2781e+00  2.4643e+00 -1.5886e+00  2.5679e-01  6.4918e-01\n",
      "  1.6809e-01  5.7693e-01  3.1121e-01 -4.5278e-01 -2.7555e+00 -2.1846e+00\n",
      "  4.4865e+00  2.7107e-01 -5.3831e-01  8.3013e-01  6.7752e-02  1.4234e-01\n",
      "  1.2585e+00 -8.5423e-01  9.2971e-01 -3.9940e-01 -5.8663e-01  6.6604e-01\n",
      "  2.3871e+00  4.9333e-01  2.3922e+00 -3.7396e+00 -3.9524e-01 -6.3799e-01\n",
      "  3.3500e+00 -2.0430e+00 -1.5601e+00 -2.3594e+00 -1.4671e+00 -3.2848e+00\n",
      "  1.5197e+00 -1.1674e+00 -1.2885e+00  3.4890e+00 -4.0526e+00  1.6946e+00\n",
      " -1.5310e+00  2.6790e+00  1.2865e+00 -2.9286e-02  3.1037e+00  1.3635e-01\n",
      " -1.3327e-01  5.4603e-01  1.2937e+00 -2.3662e+00  2.8862e-01  1.6226e+00\n",
      "  6.3531e-01  1.5498e+00 -2.3349e+00  1.6150e+00 -2.0071e+00 -4.8784e-01\n",
      "  1.7768e+00 -2.6920e+00  1.6341e+00  4.0537e-01  3.7324e-01  7.8494e-01\n",
      "  1.3917e+00 -2.9035e-01  2.5224e+00  1.2485e+00 -1.5159e+00 -2.0832e+00\n",
      " -1.8766e-01 -1.3394e+00 -5.3597e-01 -2.4915e+00  1.6341e+00 -3.0336e+00\n",
      "  1.8791e-01 -2.4776e-01  1.6347e+00 -7.0009e-01  2.1221e+00 -2.3470e+00\n",
      " -2.1513e+00  1.2630e+00 -2.8195e-01 -5.6535e-01 -9.0373e-01  1.3455e+00\n",
      "  4.1099e+00  6.9325e-01  4.6835e-01  2.9949e-01 -3.2456e-01 -2.1713e+00\n",
      "  4.6691e-01 -1.3795e-01  6.0910e-01 -1.3374e+00 -8.3586e-01  1.8260e+00\n",
      " -4.5386e-01  1.2555e+00 -6.6705e-01 -3.0835e-01 -1.4692e-01  3.9952e+00\n",
      " -1.1289e+00  1.7926e-01 -5.8095e-01  6.2500e-01  2.6151e+00 -1.3212e+00\n",
      " -1.9355e+00  2.4898e+00 -1.7301e+00  8.6154e-01 -2.5272e+00 -3.0166e+00\n",
      "  5.8867e-01 -1.4207e-01  1.7611e+00 -1.0399e+00  5.7063e-01  7.2554e-01\n",
      "  8.7684e-01  1.6996e+00  1.1084e+00 -2.6877e-01 -2.2970e+00  1.8850e+00\n",
      "  1.2536e-01  1.1671e+00 -1.3038e+00  8.2640e-01  3.8002e-01 -1.0354e+00\n",
      "  2.2547e+00 -2.4526e-01  8.3622e-01 -7.9525e-01  1.4251e+00  4.5795e-02\n",
      "  2.5062e+00  7.2175e-01 -3.8197e-01  1.1387e+00  1.6779e+00 -1.6925e+00\n",
      "  8.2290e-02 -3.3117e-01 -6.1974e-01  8.0306e-01  2.1189e+00 -7.8231e-01\n",
      "  3.3266e-01 -7.1646e-01  1.7721e+00 -1.2573e+00  1.5601e-01  9.6129e-01\n",
      "  4.8986e-01  3.1657e+00 -3.0332e+00 -9.1339e-01 -2.8627e-01 -9.8174e-01\n",
      " -2.7599e+00  2.9160e+00 -1.1540e+00  2.9537e+00 -1.1502e+00 -2.2760e+00\n",
      "  1.1679e+00  2.0314e-01 -2.2882e+00  1.0485e+00 -4.9545e-01  2.4065e+00\n",
      "  9.1433e-01  1.3808e-01 -1.1361e+00 -1.2278e+00 -1.8958e+00 -2.4934e-01\n",
      " -4.1792e+00 -1.3781e+00 -8.9829e-01 -1.4802e+00 -1.2894e+00  2.2336e+00\n",
      " -2.4411e+00  3.0410e-01  2.1452e+00  1.8540e+00 -3.3640e-01  4.2768e-01\n",
      " -4.2577e-01  2.2337e-01  3.3650e+00 -2.1996e-01 -1.7426e+00  1.4554e+00\n",
      "  2.5087e+00 -1.1257e+00  1.4096e+00  1.3704e-01  7.6963e-01 -7.6688e-02\n",
      " -6.1875e-01 -2.0445e-01  4.9965e-01 -6.7246e-01 -4.0058e-01 -1.1536e+00\n",
      " -9.2837e-01 -2.5444e+00 -1.9880e-01 -2.1432e+00 -1.8218e+00 -5.7831e-01\n",
      " -1.6785e+00  2.6610e+00  1.0343e+00 -1.4297e+00  1.4563e+00 -2.0388e+00\n",
      " -1.4377e+00 -9.7985e-01  2.9488e+00 -2.7107e-01 -2.4010e-01 -1.3845e-01]\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "# Load the en_core_web_md pipeline\n",
    "nlp = spacy.load('en_core_web_md')\n",
    "\n",
    "# Process a text\n",
    "doc = nlp(\"Two bananas in pyjamas\")\n",
    "\n",
    "# Get the vector for the token \"bananas\"\n",
    "bananas_vector = doc[1].vector\n",
    "print(bananas_vector)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Comparing similarities"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8220092482601077\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_md\")\n",
    "\n",
    "doc1 = nlp(\"It's a warm summer day\")\n",
    "doc2 = nlp(\"It's sunny outside\")\n",
    "\n",
    "# Get the similarity of doc1 and doc2\n",
    "similarity = doc1.similarity(doc2)\n",
    "print(similarity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.10219938308000565\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_md\")\n",
    "\n",
    "doc = nlp(\"TV and books\")\n",
    "token1, token2 = doc[0], doc[2]\n",
    "\n",
    "# Get the similarity of the tokens \"TV\" and \"books\"\n",
    "similarity = token1.similarity(token2)\n",
    "print(similarity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6348509788513184\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_md\")\n",
    "\n",
    "doc = nlp(\"This was a great restaurant. Afterwards, we went to a really nice bar.\")\n",
    "\n",
    "# Create spans for \"great restaurant\" and \"really nice bar\"\n",
    "span1 = doc[3:5]\n",
    "span2 = doc[-4:-1]\n",
    "\n",
    "# Get the similarity of the spans\n",
    "similarity = span1.similarity(span2)\n",
    "print(similarity)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Debugging patterns (2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PATTERN1 Amazon Prime\n",
      "PATTERN2 ad-free viewing\n",
      "PATTERN1 Amazon Prime\n",
      "PATTERN2 ad-free viewing\n",
      "PATTERN2 ad-free viewing\n",
      "PATTERN2 ad-free viewing\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "from spacy.matcher import Matcher\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "doc = nlp(\n",
    "    \"Twitch Prime, the perks program for Amazon Prime members offering free \"\n",
    "    \"loot, games and other benefits, is ditching one of its best features: \"\n",
    "    \"ad-free viewing. According to an email sent out to Amazon Prime members \"\n",
    "    \"today, ad-free viewing will no longer be included as a part of Twitch \"\n",
    "    \"Prime for new members, beginning on September 14. However, members with \"\n",
    "    \"existing annual subscriptions will be able to continue to enjoy ad-free \"\n",
    "    \"viewing until their subscription comes up for renewal. Those with \"\n",
    "    \"monthly subscriptions will have access to ad-free viewing until October 15.\"\n",
    ")\n",
    "\n",
    "# Create the match patterns\n",
    "pattern1 = [{\"LOWER\": \"amazon\"}, {\"IS_TITLE\": True, \"POS\": \"PROPN\"}]\n",
    "pattern2 = [{\"LOWER\": \"ad\"}, {'TEXT': '-'}, {'LOWER': 'free'}, {'POS': 'NOUN'}]\n",
    "\n",
    "# Initialize the Matcher and add the patterns\n",
    "matcher = Matcher(nlp.vocab)\n",
    "matcher.add(\"PATTERN1\", [pattern1])\n",
    "matcher.add(\"PATTERN2\", [pattern2])\n",
    "\n",
    "# Iterate over the matches\n",
    "for match_id, start, end in matcher(doc):\n",
    "    # Print pattern string name and text of matched span\n",
    "    print(doc.vocab.strings[match_id], doc[start:end].text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Efficient phrase matching"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Czech Republic, Slovakia]\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import spacy\n",
    "\n",
    "with open(\"ws/exercises/en/countries.json\", encoding=\"utf8\") as f:\n",
    "    COUNTRIES = json.loads(f.read())\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "doc = nlp(\"Czech Republic may help Slovakia protect its airspace\")\n",
    "\n",
    "# Import the PhraseMatcher and initialize it\n",
    "from spacy.matcher import PhraseMatcher\n",
    "\n",
    "matcher = PhraseMatcher(nlp.vocab)\n",
    "\n",
    "# Create pattern Doc objects and add them to the matcher\n",
    "# This is the faster version of: [nlp(country) for country in COUNTRIES]\n",
    "patterns = list(nlp.pipe(COUNTRIES))\n",
    "matcher.add(\"COUNTRY\", patterns)\n",
    "\n",
    "# Call the matcher on the test document and print the result\n",
    "matches = matcher(doc)\n",
    "print([doc[start:end] for match_id, start, end in matches])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extracting countries nd relationship"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "in --> Namibia\n",
      "in --> South Africa\n",
      "Africa --> Cambodia\n",
      "of --> Kuwait\n",
      "as --> Somalia\n",
      "Somalia --> Haiti\n",
      "Haiti --> Mozambique\n",
      "in --> Somalia\n",
      "for --> Rwanda\n",
      "Britain --> Singapore\n",
      "War --> Sierra Leone\n",
      "of --> Afghanistan\n",
      "invaded --> Iraq\n",
      "in --> Sudan\n",
      "of --> Congo\n",
      "earthquake --> Haiti\n",
      "[('Namibia', 'GPE'), ('South Africa', 'GPE'), ('Cambodia', 'GPE'), ('Kuwait', 'GPE'), ('Somalia', 'GPE'), ('Haiti', 'GPE'), ('Mozambique', 'GPE'), ('Somalia', 'GPE'), ('Rwanda', 'GPE'), ('Singapore', 'GPE'), ('Sierra Leone', 'GPE'), ('Afghanistan', 'GPE'), ('Iraq', 'GPE'), ('Sudan', 'GPE'), ('Congo', 'GPE'), ('Haiti', 'GPE')]\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "from spacy.matcher import PhraseMatcher\n",
    "from spacy.tokens import Span\n",
    "import json\n",
    "\n",
    "with open(\"ws/exercises/en/countries.json\", encoding=\"utf8\") as f:\n",
    "    COUNTRIES = json.loads(f.read())\n",
    "with open(\"ws/exercises/en/country_text.txt\", encoding=\"utf8\") as f:\n",
    "    TEXT = f.read()\n",
    "\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "matcher = PhraseMatcher(nlp.vocab)\n",
    "patterns = list(nlp.pipe(COUNTRIES))\n",
    "matcher.add(\"COUNTRY\", patterns)\n",
    "\n",
    "# Create a doc and reset existing entities\n",
    "doc = nlp(TEXT)\n",
    "doc.ents = []\n",
    "\n",
    "# Iterate over the matches\n",
    "for match_id, start, end in matcher(doc):\n",
    "    # Create a Span with the label for \"GPE\"\n",
    "    span = Span(doc, start, end, label='GPE')\n",
    "\n",
    "    # Overwrite the doc.ents and add the span\n",
    "    doc.ents = list(doc.ents) + [span]\n",
    "\n",
    "    # Get the span's root head token\n",
    "    span_root_head = span.root.head\n",
    "    # Print the text of the span root's head token and the span text\n",
    "    print(span_root_head.text, \"-->\", span.text)\n",
    "\n",
    "# Print the entities in the document\n",
    "print([(ent.text, ent.label_) for ent in doc.ents if ent.label_ == \"GPE\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}