Ch4 Training a neural network model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 4: Training a neural network model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating training data (1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[iPhone X]\n",
      "[iPhone X]\n",
      "[iPhone X]\n",
      "[iPhone 8]\n",
      "[iPhone 11, iPhone 8]\n",
      "[]\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "import spacy\n",
    "from spacy.matcher import Matcher\n",
    "from spacy.tokens import Span\n",
    "\n",
    "with open(\"/ws/exercises/en/iphone.json\", encoding=\"utf8\") as f:\n",
    "    TEXTS = json.loads(f.read())\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "matcher = Matcher(nlp.vocab)\n",
    "\n",
    "# Two tokens whose lowercase forms match \"iphone\" and \"x\"\n",
    "pattern1 = [{\"LOWER\": \"iphone\"}, {\"LOWER\": \"x\"}]\n",
    "\n",
    "# Token whose lowercase form matches \"iphone\" and a digit\n",
    "pattern2 = [{\"LOWER\": \"iphone\"}, {\"IS_DIGIT\": True}]\n",
    "\n",
    "# Add patterns to the matcher and create docs with matched entities\n",
    "matcher.add(\"GADGET\", [pattern1, pattern2])\n",
    "docs = []\n",
    "for doc in nlp.pipe(TEXTS):\n",
    "    matches = matcher(doc)\n",
    "    spans = [Span(doc, start, end, label=match_id) for match_id, start, end in matches]\n",
    "    print(spans)\n",
    "    doc.ents = spans\n",
    "    docs.append(doc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating training data (2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import spacy\n",
    "from spacy.matcher import Matcher\n",
    "from spacy.tokens import Span, DocBin\n",
    "\n",
    "with open(\"/ws/exercises/en/iphone.json\", encoding=\"utf8\") as f:\n",
    "    TEXTS = json.loads(f.read())\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "matcher = Matcher(nlp.vocab)\n",
    "# Add patterns to the matcher\n",
    "pattern1 = ([{\"LOWER\": \"iphone\"}, {\"LOWER\": \"x\"}])\n",
    "pattern2 = [{\"LOWER\": \"iphone\"}, {\"IS_DIGIT\": True}]\n",
    "matcher.add(\"GADGET\", [pattern1, pattern2])\n",
    "docs = []\n",
    "for doc in nlp.pipe(TEXTS):\n",
    "    matches = matcher(doc)\n",
    "    spans = [Span(doc, start, end, label=match_id) for match_id, start, end in matches]\n",
    "    doc.ents = spans\n",
    "    docs.append(doc)\n",
    "\n",
    "doc_bin = DocBin(docs=docs)\n",
    "doc_bin.to_disk(\"/ws/output/train.spacy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating a config file"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;3m⚠ To generate a more effective transformer-based config (GPU-only),\n",
      "install the spacy-transformers package and re-run this command. The config\n",
      "generated now does not use transformers.\u001b[0m\n",
      "\u001b[38;5;4mℹ Generated config template specific for your use case\u001b[0m\n",
      "- Language: en\n",
      "- Pipeline: ner\n",
      "- Optimize for: efficiency\n",
      "- Hardware: CPU\n",
      "- Transformer: None\n",
      "\u001b[38;5;2m✔ Auto-filled config with all values\u001b[0m\n",
      "\u001b[38;5;2m✔ Saved config\u001b[0m\n",
      "ws/output/config.cfg\n",
      "You can now add your data and train your pipeline:\n",
      "python -m spacy train config.cfg --paths.train ./train.spacy --paths.dev ./dev.spacy\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy init config ws/output/config.cfg --lang en --pipeline ner"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[paths]\n",
      "train = null\n",
      "dev = null\n",
      "vectors = null\n",
      "init_tok2vec = null\n",
      "\n",
      "[system]\n",
      "gpu_allocator = null\n",
      "seed = 0\n",
      "\n",
      "[nlp]\n",
      "lang = \"en\"\n",
      "pipeline = [\"tok2vec\",\"ner\"]\n",
      "batch_size = 1000\n",
      "disabled = []\n",
      "before_creation = null\n",
      "after_creation = null\n",
      "after_pipeline_creation = null\n",
      "tokenizer = {\"@tokenizers\":\"spacy.Tokenizer.v1\"}\n",
      "\n",
      "[components]\n",
      "\n",
      "[components.ner]\n",
      "factory = \"ner\"\n",
      "incorrect_spans_key = null\n",
      "moves = null\n",
      "scorer = {\"@scorers\":\"spacy.ner_scorer.v1\"}\n",
      "update_with_oracle_cut_size = 100\n",
      "\n",
      "[components.ner.model]\n",
      "@architectures = \"spacy.TransitionBasedParser.v2\"\n",
      "state_type = \"ner\"\n",
      "extra_state_tokens = false\n",
      "hidden_width = 64\n",
      "maxout_pieces = 2\n",
      "use_upper = true\n",
      "nO = null\n",
      "\n",
      "[components.ner.model.tok2vec]\n",
      "@architectures = \"spacy.Tok2VecListener.v1\"\n",
      "width = ${components.tok2vec.model.encode.width}\n",
      "upstream = \"*\"\n",
      "\n",
      "[components.tok2vec]\n",
      "factory = \"tok2vec\"\n",
      "\n",
      "[components.tok2vec.model]\n",
      "@architectures = \"spacy.Tok2Vec.v2\"\n",
      "\n",
      "[components.tok2vec.model.embed]\n",
      "@architectures = \"spacy.MultiHashEmbed.v2\"\n",
      "width = ${components.tok2vec.model.encode.width}\n",
      "attrs = [\"NORM\",\"PREFIX\",\"SUFFIX\",\"SHAPE\"]\n",
      "rows = [5000,1000,2500,2500]\n",
      "include_static_vectors = false\n",
      "\n",
      "[components.tok2vec.model.encode]\n",
      "@architectures = \"spacy.MaxoutWindowEncoder.v2\"\n",
      "width = 96\n",
      "depth = 4\n",
      "window_size = 1\n",
      "maxout_pieces = 3\n",
      "\n",
      "[corpora]\n",
      "\n",
      "[corpora.dev]\n",
      "@readers = \"spacy.Corpus.v1\"\n",
      "path = ${paths.dev}\n",
      "max_length = 0\n",
      "gold_preproc = false\n",
      "limit = 0\n",
      "augmenter = null\n",
      "\n",
      "[corpora.train]\n",
      "@readers = \"spacy.Corpus.v1\"\n",
      "path = ${paths.train}\n",
      "max_length = 0\n",
      "gold_preproc = false\n",
      "limit = 0\n",
      "augmenter = null\n",
      "\n",
      "[training]\n",
      "dev_corpus = \"corpora.dev\"\n",
      "train_corpus = \"corpora.train\"\n",
      "seed = ${system.seed}\n",
      "gpu_allocator = ${system.gpu_allocator}\n",
      "dropout = 0.1\n",
      "accumulate_gradient = 1\n",
      "patience = 1600\n",
      "max_epochs = 0\n",
      "max_steps = 20000\n",
      "eval_frequency = 200\n",
      "frozen_components = []\n",
      "annotating_components = []\n",
      "before_to_disk = null\n",
      "\n",
      "[training.batcher]\n",
      "@batchers = \"spacy.batch_by_words.v1\"\n",
      "discard_oversize = false\n",
      "tolerance = 0.2\n",
      "get_length = null\n",
      "\n",
      "[training.batcher.size]\n",
      "@schedules = \"compounding.v1\"\n",
      "start = 100\n",
      "stop = 1000\n",
      "compound = 1.001\n",
      "t = 0.0\n",
      "\n",
      "[training.logger]\n",
      "@loggers = \"spacy.ConsoleLogger.v1\"\n",
      "progress_bar = false\n",
      "\n",
      "[training.optimizer]\n",
      "@optimizers = \"Adam.v1\"\n",
      "beta1 = 0.9\n",
      "beta2 = 0.999\n",
      "L2_is_weight_decay = true\n",
      "L2 = 0.01\n",
      "grad_clip = 1.0\n",
      "use_averages = false\n",
      "eps = 0.00000001\n",
      "learn_rate = 0.001\n",
      "\n",
      "[training.score_weights]\n",
      "ents_f = 1.0\n",
      "ents_p = 0.0\n",
      "ents_r = 0.0\n",
      "ents_per_type = null\n",
      "\n",
      "[pretraining]\n",
      "\n",
      "[initialize]\n",
      "vectors = ${paths.vectors}\n",
      "init_tok2vec = ${paths.init_tok2vec}\n",
      "vocab_data = null\n",
      "lookups = null\n",
      "before_init = null\n",
      "after_init = null\n",
      "\n",
      "[initialize.components]\n",
      "\n",
      "[initialize.tokenizer]"
     ]
    }
   ],
   "source": [
    "!cat ws/output/config.cfg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the training CLI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[38;5;4mℹ Saving to output directory: /ws/output\u001b[0m\n",
      "\u001b[38;5;4mℹ Using CPU\u001b[0m\n",
      "\u001b[1m\n",
      "=========================== Initializing pipeline ===========================\u001b[0m\n",
      "[2022-11-26 12:57:54,402] [INFO] Set up nlp object from config\n",
      "[2022-11-26 12:57:54,408] [INFO] Pipeline: ['tok2vec', 'ner']\n",
      "[2022-11-26 12:57:54,410] [INFO] Created vocabulary\n",
      "[2022-11-26 12:57:54,411] [INFO] Finished initializing nlp object\n",
      "[2022-11-26 12:57:54,888] [INFO] Initialized pipeline components: ['tok2vec', 'ner']\n",
      "\u001b[38;5;2m✔ Initialized pipeline\u001b[0m\n",
      "\u001b[1m\n",
      "============================= Training pipeline =============================\u001b[0m\n",
      "\u001b[38;5;4mℹ Pipeline: ['tok2vec', 'ner']\u001b[0m\n",
      "\u001b[38;5;4mℹ Initial learn rate: 0.001\u001b[0m\n",
      "E    #       LOSS TOK2VEC  LOSS NER  ENTS_F  ENTS_P  ENTS_R  SCORE \n",
      "---  ------  ------------  --------  ------  ------  ------  ------\n",
      "  0       0          0.00     20.33    1.69    1.04    4.44    0.02\n",
      "  1     200         31.47    998.53   75.82   75.00   76.67    0.76\n",
      "  2     400         72.61    235.81   83.15   84.09   82.22    0.83\n",
      "  4     600         60.55    110.24   84.15   82.80   85.56    0.84\n",
      "  6     800         61.79     70.68   85.25   83.87   86.67    0.85\n",
      "  9    1000         72.09     58.52   85.23   87.21   83.33    0.85\n",
      " 12    1200         61.90     37.37   86.36   88.37   84.44    0.86\n",
      " 16    1400         69.13     32.22   82.02   82.95   81.11    0.82\n",
      " 22    1600         42.84     10.44   83.70   81.91   85.56    0.84\n",
      " 28    1800         14.06      3.20   89.01   88.04   90.00    0.89\n",
      " 36    2000         86.64     20.32   83.98   83.52   84.44    0.84\n",
      " 46    2200        250.51     57.67   82.49   83.91   81.11    0.82\n",
      " 58    2400        212.01     61.61   86.52   87.50   85.56    0.87\n",
      " 70    2600         71.07     13.48   83.43   85.88   81.11    0.83\n",
      " 83    2800        142.46     22.57   83.62   85.06   82.22    0.84\n",
      " 95    3000         11.22      1.41   86.52   87.50   85.56    0.87\n",
      "108    3200          0.00      0.00   86.52   87.50   85.56    0.87\n",
      "120    3400        123.55     19.06   84.15   82.80   85.56    0.84\n",
      "\u001b[38;5;2m✔ Saved pipeline to output directory\u001b[0m\n",
      "/ws/output/model-last\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy train /ws/exercises/en/config_gadget.cfg --output /ws/output --paths.train /ws/exercises/en/train_gadget.spacy --paths.dev /ws/exercises/en/dev_gadget.spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Good data vs. bad data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.tokens import Span\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "\n",
    "doc1 = nlp(\"i went to amsterdem last year and the canals were beautiful\")\n",
    "doc1.ents = [Span(doc1, 3, 4, label=\"GPE\")]\n",
    "\n",
    "doc2 = nlp(\"You should visit Paris once, but the Eiffel Tower is kinda boring\")\n",
    "doc2.ents = [Span(doc2, 3, 4, label=\"GPE\")]\n",
    "\n",
    "doc3 = nlp(\"There's also a Paris in Arkansas, lol\")\n",
    "doc3.ents = [Span(doc2, 4, 5, label=\"GPE\"), Span(doc2, 6, 7, label=\"GPE\")]\n",
    "\n",
    "doc4 = nlp(\"Berlin is perfect for summer holiday: great nightlife and cheap beer!\")\n",
    "doc4.ents = [Span(doc4, 0, 1, label=\"GPE\")]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training multiple labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.tokens import Span\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "\n",
    "doc1 = nlp(\"Reddit partners with Patreon to help creators build communities\")\n",
    "doc1.ents = [\n",
    "    Span(doc1, 0, 1, label=\"WEBSITE\"),\n",
    "    Span(doc1, 3, 4, label=\"WEBSITE\"),\n",
    "]\n",
    "\n",
    "doc2 = nlp(\"PewDiePie smashes YouTube record\")\n",
    "doc2.ents = [Span(doc2, 2, 3, label=\"WEBSITE\")]\n",
    "\n",
    "doc3 = nlp(\"Reddit founder Alexis Ohanian gave away two Metallica tickets to fans\")\n",
    "doc3.ents = [Span(doc3, 0, 1, label=\"WEBSITE\")]\n",
    "\n",
    "# And so on..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Part 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.tokens import Span\n",
    "\n",
    "nlp = spacy.blank(\"en\")\n",
    "\n",
    "doc1 = nlp(\"Reddit partners with Patreon to help creators build communities\")\n",
    "doc1.ents = [\n",
    "    Span(doc1, 0, 1, label=\"WEBSITE\"),\n",
    "    Span(doc1, 3, 4, label=\"WEBSITE\"),\n",
    "]\n",
    "\n",
    "doc2 = nlp(\"PewDiePie smashes YouTube record\")\n",
    "doc2.ents = [Span(doc2, 0, 1, label=\"PERSON\"), Span(doc2, 2, 3, label=\"WEBSITE\")]\n",
    "\n",
    "doc3 = nlp(\"Reddit founder Alexis Ohanian gave away two Metallica tickets to fans\")\n",
    "doc3.ents = [Span(doc3, 0, 1, label=\"WEBSITE\"), Span(doc3, 2, 4, label=\"PERSON\")]\n",
    "\n",
    "# And so on..."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}