Day_2_NLP.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "76c0e2a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a130e62f",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "225d97e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/am_i_the_asshole_example.txt', 'r', encoding='UTF-8') as f:\n",
    "    text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bf83aa2b",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8d385be9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "One day DATE\n",
      "this day DATE\n",
      "Italian NORP\n",
      "Canada GPE\n",
      "Italian NORP\n",
      "Franks GPE\n",
      "Sriracha PERSON\n",
      "Calabrian NORP\n",
      "Italian NORP\n",
      "Sriracha PERSON\n",
      "all day DATE\n",
      "yesterday DATE\n",
      "only 3 months DATE\n",
      "34 DATE\n",
      "35 DATE\n",
      "23 CARDINAL\n",
      "22 DATE\n"
     ]
    }
   ],
   "source": [
    "for token in doc.ents:\n",
    "    print(token.text, token.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3d1b633e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Countries, cities, states'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spacy.explain('GPE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "62b771c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: spacytextblob in /home/tinat/anaconda3/lib/python3.9/site-packages (4.0.0)\n",
      "Requirement already satisfied: textblob<0.16.0,>=0.15.3 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacytextblob) (0.15.3)\n",
      "Requirement already satisfied: spacy<4.0,>=3.0 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacytextblob) (3.3.1)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (2.28.1)\n",
      "Requirement already satisfied: jinja2 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (2.11.3)\n",
      "Requirement already satisfied: setuptools in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (63.4.1)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (2.0.6)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (1.0.4)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (3.0.6)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (2.0.8)\n",
      "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (0.4.2)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (1.0.7)\n",
      "Requirement already satisfied: thinc<8.1.0,>=8.0.14 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (8.0.15)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (0.7.7)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (21.3)\n",
      "Requirement already satisfied: pathy>=0.3.5 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (0.10.1)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (1.8.2)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (2.4.3)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (4.64.1)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (1.21.5)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (3.3.0)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (3.0.11)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /home/tinat/anaconda3/lib/python3.9/site-packages (from spacy<4.0,>=3.0->spacytextblob) (0.10.1)\n",
      "Requirement already satisfied: nltk>=3.1 in /home/tinat/anaconda3/lib/python3.9/site-packages (from textblob<0.16.0,>=0.15.3->spacytextblob) (3.7)\n",
      "Requirement already satisfied: click in /home/tinat/anaconda3/lib/python3.9/site-packages (from nltk>=3.1->textblob<0.16.0,>=0.15.3->spacytextblob) (8.0.4)\n",
      "Requirement already satisfied: joblib in /home/tinat/anaconda3/lib/python3.9/site-packages (from nltk>=3.1->textblob<0.16.0,>=0.15.3->spacytextblob) (1.1.0)\n",
      "Requirement already satisfied: regex>=2021.8.3 in /home/tinat/anaconda3/lib/python3.9/site-packages (from nltk>=3.1->textblob<0.16.0,>=0.15.3->spacytextblob) (2022.7.9)\n",
      "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home/tinat/anaconda3/lib/python3.9/site-packages (from packaging>=20.0->spacy<4.0,>=3.0->spacytextblob) (3.0.9)\n",
      "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /home/tinat/anaconda3/lib/python3.9/site-packages (from pathy>=0.3.5->spacy<4.0,>=3.0->spacytextblob) (5.2.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/tinat/anaconda3/lib/python3.9/site-packages (from pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4->spacy<4.0,>=3.0->spacytextblob) (4.3.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/tinat/anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<4.0,>=3.0->spacytextblob) (1.26.11)\n",
      "Requirement already satisfied: charset-normalizer<3,>=2 in /home/tinat/anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<4.0,>=3.0->spacytextblob) (2.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/tinat/anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<4.0,>=3.0->spacytextblob) (2022.9.14)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/tinat/anaconda3/lib/python3.9/site-packages (from requests<3.0.0,>=2.13.0->spacy<4.0,>=3.0->spacytextblob) (3.3)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in /home/tinat/anaconda3/lib/python3.9/site-packages (from jinja2->spacy<4.0,>=3.0->spacytextblob) (2.0.1)\n",
      "[nltk_data] Downloading package brown to /home/tinat/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/brown.zip.\n",
      "[nltk_data] Downloading package punkt to /home/tinat/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
      "[nltk_data] Downloading package wordnet to /home/tinat/nltk_data...\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /home/tinat/nltk_data...\n",
      "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n",
      "[nltk_data] Downloading package conll2000 to /home/tinat/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/conll2000.zip.\n",
      "[nltk_data] Downloading package movie_reviews to\n",
      "[nltk_data]     /home/tinat/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/movie_reviews.zip.\n",
      "Finished.\n"
     ]
    }
   ],
   "source": [
    "#!pip install spacytextblob\n",
    "#!python -m textblob.download_corpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cc25327f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacytextblob.spacytextblob import SpacyTextBlob\n",
    "from textblob import TextBlob\n",
    "from textblob.sentiments import NaiveBayesAnalyzer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2a73dbe0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tok2vec', <spacy.pipeline.tok2vec.Tok2Vec at 0x7fa30d15ed00>),\n",
       " ('tagger', <spacy.pipeline.tagger.Tagger at 0x7fa30d15ed60>),\n",
       " ('parser', <spacy.pipeline.dep_parser.DependencyParser at 0x7fa30d33e270>),\n",
       " ('senter', <spacy.pipeline.senter.SentenceRecognizer at 0x7fa30d15eee0>),\n",
       " ('attribute_ruler',\n",
       "  <spacy.pipeline.attributeruler.AttributeRuler at 0x7fa30d0ded40>),\n",
       " ('lemmatizer',\n",
       "  <spacy.lang.en.lemmatizer.EnglishLemmatizer at 0x7fa30d0d6a40>),\n",
       " ('ner', <spacy.pipeline.ner.EntityRecognizer at 0x7fa30d33e200>)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.components"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "fa71ffe7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tok2vec', <spacy.pipeline.tok2vec.Tok2Vec at 0x7fa30d15ed00>),\n",
       " ('tagger', <spacy.pipeline.tagger.Tagger at 0x7fa30d15ed60>),\n",
       " ('parser', <spacy.pipeline.dep_parser.DependencyParser at 0x7fa30d33e270>),\n",
       " ('attribute_ruler',\n",
       "  <spacy.pipeline.attributeruler.AttributeRuler at 0x7fa30d0ded40>),\n",
       " ('lemmatizer',\n",
       "  <spacy.lang.en.lemmatizer.EnglishLemmatizer at 0x7fa30d0d6a40>),\n",
       " ('ner', <spacy.pipeline.ner.EntityRecognizer at 0x7fa30d33e200>)]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c045bfe1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<spacytextblob.spacytextblob.SpacyTextBlob at 0x7fa3061d0880>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.add_pipe('spacytextblob')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "29bf7a1d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('tok2vec', <spacy.pipeline.tok2vec.Tok2Vec at 0x7fa30d15ed00>),\n",
       " ('tagger', <spacy.pipeline.tagger.Tagger at 0x7fa30d15ed60>),\n",
       " ('parser', <spacy.pipeline.dep_parser.DependencyParser at 0x7fa30d33e270>),\n",
       " ('attribute_ruler',\n",
       "  <spacy.pipeline.attributeruler.AttributeRuler at 0x7fa30d0ded40>),\n",
       " ('lemmatizer',\n",
       "  <spacy.lang.en.lemmatizer.EnglishLemmatizer at 0x7fa30d0d6a40>),\n",
       " ('ner', <spacy.pipeline.ner.EntityRecognizer at 0x7fa30d33e200>),\n",
       " ('spacytextblob',\n",
       "  <spacytextblob.spacytextblob.SpacyTextBlob at 0x7fa3061d0880>)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nlp.pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a8dccaf7",
   "metadata": {},
   "outputs": [],
   "source": [
    "doc = nlp(text.replace('\\n',''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6bae6738",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.06848148148148149"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc._.polarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7acf808f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0b2b375d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Every Sentiment(classification='neg', p_pos=0.4970282317979198, p_neg=0.5029717682020802)\n",
      "\n",
      "Sunday Sentiment(classification='pos', p_pos=0.5434782608695654, p_neg=0.4565217391304346)\n",
      "\n",
      "my Sentiment(classification='pos', p_pos=0.5, p_neg=0.5)\n",
      "\n",
      "family Sentiment(classification='pos', p_pos=0.6222527472527473, p_neg=0.3777472527472528)\n",
      "\n",
      "rotates Sentiment(classification='pos', p_pos=0.75, p_neg=0.2499999999999997)\n",
      "\n",
      "who Sentiment(classification='pos', p_pos=0.5059999999999999, p_neg=0.494)\n",
      "\n",
      "makes Sentiment(classification='pos', p_pos=0.5499306518723993, p_neg=0.4500693481276005)\n",
      "\n",
      "dinner Sentiment(classification='pos', p_pos=0.5299999999999999, p_neg=0.4700000000000002)\n",
      "\n",
      ". Sentiment(classification='pos', p_pos=0.5, p_neg=0.5)\n",
      "\n",
      "One Sentiment(classification='pos', p_pos=0.5061902082160945, p_neg=0.4938097917839054)\n",
      "\n",
      "day Sentiment(classification='pos', p_pos=0.5533472803347281, p_neg=0.4466527196652719)\n",
      "\n",
      "it Sentiment(classification='pos', p_pos=0.5, p_neg=0.5)\n",
      "\n",
      "'s Sentiment(classification='pos', p_pos=0.5, p_neg=0.5)\n",
      "\n",
      "my Sentiment(classification='pos', p_pos=0.5, p_neg=0.5)\n",
      "\n",
      "wife Sentiment(classification='pos', p_pos=0.570528967254408, p_neg=0.429471032745592)\n",
      "\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_424146/1831320447.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mspan\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdoc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mblob\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextBlob\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspan\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0manalyzer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNaiveBayesAnalyzer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspan\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblob\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msentiment\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m    \u001b[0;31m# print(span.text, span._.polarity)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m''\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/textblob/decorators.py\u001b[0m in \u001b[0;36m__get__\u001b[0;34m(self, obj, cls)\u001b[0m\n\u001b[1;32m     22\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__dict__\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/textblob/blob.py\u001b[0m in \u001b[0;36msentiment\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    430\u001b[0m         \u001b[0;34m:\u001b[0m\u001b[0mrtype\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnamedtuple\u001b[0m \u001b[0mof\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mform\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mSentiment\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpolarity\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubjectivity\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    431\u001b[0m         \"\"\"\n\u001b[0;32m--> 432\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0manalyzer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0manalyze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    433\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    434\u001b[0m     \u001b[0;34m@\u001b[0m\u001b[0mcached_property\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/textblob/en/sentiments.py\u001b[0m in \u001b[0;36manalyze\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m     86\u001b[0m         \"\"\"\n\u001b[1;32m     87\u001b[0m         \u001b[0;31m# Lazily train the classifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mNaiveBayesAnalyzer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0manalyze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m         \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mword_tokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minclude_punc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m         \u001b[0mfiltered\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtokens\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>=\u001b[0m \u001b[0;36m3\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/textblob/base.py\u001b[0m in \u001b[0;36manalyze\u001b[0;34m(self, text)\u001b[0m\n\u001b[1;32m     92\u001b[0m         \u001b[0;31m# Lazily train the classifier\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trained\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 94\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     95\u001b[0m         \u001b[0;31m# Analyze text\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/textblob/decorators.py\u001b[0m in \u001b[0;36mdecorated\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     33\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdecorated\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mLookupError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/textblob/en/sentiments.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     79\u001b[0m             nltk.corpus.movie_reviews.words(fileids=[f])), 'pos') for f in pos_ids]\n\u001b[1;32m     80\u001b[0m         \u001b[0mtrain_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mneg_feats\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mpos_feats\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 81\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_classifier\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclassify\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNaiveBayesClassifier\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     82\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     83\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0manalyze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.9/site-packages/nltk/classify/naivebayes.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(cls, labeled_featuresets, estimator)\u001b[0m\n\u001b[1;32m    212\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mfname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfval\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfeatureset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m                 \u001b[0;31m# Increment freq(fval|label, fname)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 214\u001b[0;31m                 \u001b[0mfeature_freqdist\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlabel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfval\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    215\u001b[0m                 \u001b[0;31m# Record that fname can take the value fval.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m                 \u001b[0mfeature_values\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfname\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for span in doc:\n",
    "    blob = TextBlob(span.text, analyzer=NaiveBayesAnalyzer())\n",
    "    print(span, blob.sentiment)\n",
    "   # print(span.text, span._.polarity)\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7660de50",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import gensim.corpora as corpora\n",
    "from gensim.models import CoherenceModel\n",
    "import numpy as np\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "af818ddb",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e3c215bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_folder = 'data/topic_models/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "6d0da747",
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_files = sorted([os.path.join(path_to_folder, f) for f in os.listdir(path_to_folder)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c7a9cbd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(path_to_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "6f72760f",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenized_corpus = []\n",
    "extra_stop = ['mr', 'ms', 'mrs', 'hon']\n",
    "\n",
    "for my_file in path_to_files:\n",
    "    with open(my_file, 'r', encoding='UTF-8') as f:\n",
    "        text = f.read().replace('\\n','')\n",
    "        \n",
    "        doc = nlp(text)\n",
    "        \n",
    "        text_proc = []\n",
    "        \n",
    "        for token in doc:\n",
    "            if not token.is_stop \\\n",
    "            and not token.is_punct and not token.like_num \\\n",
    "            and token.lemma_.lower() not in extra_stop:\n",
    "                text_proc.append(token.lemma_.lower())\n",
    "    tokenized_corpus.append(text_proc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "972974e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(tokenized_corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "7dd724ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "words_id = corpora.Dictionary(tokenized_corpus)\n",
    "corpus = [words_id.doc2bow(txt) for txt in tokenized_corpus]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e20c1e33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 -7.264499187714998 0.3005033290695279\n",
      "6 -7.2705373125776855 0.30891421938885066\n",
      "7 -7.287679808488493 0.2944448799598958\n",
      "8 -7.275602999669263 0.29606435963498823\n",
      "9 -7.29659943033535 0.30126579583002266\n",
      "10 -7.283444353330029 0.31285474198726015\n",
      "11 -7.284507971359415 0.3260300910879996\n",
      "12 -7.284488173368267 0.3233559262392099\n",
      "13 -7.291600443374959 0.3437767195153501\n",
      "14 -7.299302758694808 0.3648875326489233\n",
      "15 -7.308257399703309 0.34227402435872684\n"
     ]
    }
   ],
   "source": [
    "k_init = 5\n",
    "k_final =15\n",
    "\n",
    "for k in range(k_init, k_final +1):\n",
    "    lda_model = gensim.models.ldamodel.LdaModel(corpus = corpus,\n",
    "                                              id2word = words_id,\n",
    "                                              num_topics = k,\n",
    "                                              random_state = 50,\n",
    "                                              passes = 20,\n",
    "                                              per_word_topics = True)\n",
    "\n",
    "    per_lda = lda_model.log_perplexity(corpus)\n",
    "    coherence_model_lda = CoherenceModel(model = lda_model, texts = tokenized_corpus, dictionary=words_id,\n",
    "                                        coherence='c_v')\n",
    "    coherence_lda = coherence_model_lda.get_coherence()\n",
    "    print(k, per_lda, coherence_lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "4f95a530",
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model = gensim.models.ldamodel.LdaModel(corpus = corpus,\n",
    "                                             id2word = words_id,\n",
    "                                             num_topics = 14,\n",
    "                                             random_state = 50,\n",
    "                                             passes = 20,\n",
    "                                             per_word_topics = True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "347d4429",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.021*\"new\" + 0.016*\"zealand\" + 0.014*\"trade\" + 0.013*\"government\" + 0.012*\"think\" + 0.012*\"agreement\" + 0.009*\"impact\" + 0.008*\"exporter\" + 0.008*\"change\" + 0.007*\"national\" + 0.007*\"brand\" + 0.006*\"know\" + 0.006*\"want\" + 0.006*\"economic\" + 0.006*\"labour\" + 0.006*\"issue\" + 0.006*\"time\" + 0.006*\"policy\" + 0.006*\"free\" + 0.006*\"ardern\"'),\n",
       " (1,\n",
       "  '0.026*\"programme\" + 0.021*\"youth\" + 0.018*\"court\" + 0.014*\"young\" + 0.011*\"judge\" + 0.010*\"offender\" + 0.010*\"family\" + 0.010*\"child\" + 0.009*\"government\" + 0.009*\"minister\" + 0.009*\"bill\" + 0.007*\"justice\" + 0.007*\"deal\" + 0.007*\"camp\" + 0.007*\"think\" + 0.006*\"acknowledge\" + 0.006*\"system\" + 0.006*\"say\" + 0.006*\"social\" + 0.006*\"boot\"'),\n",
       " (2,\n",
       "  '0.017*\"bill\" + 0.014*\"legal\" + 0.014*\"justice\" + 0.013*\"minister\" + 0.012*\"think\" + 0.010*\"committee\" + 0.009*\"system\" + 0.008*\"issue\" + 0.008*\"important\" + 0.008*\"legislation\" + 0.007*\"law\" + 0.007*\"point\" + 0.006*\"time\" + 0.006*\"government\" + 0.006*\"power\" + 0.006*\"work\" + 0.006*\"victim\" + 0.006*\"clause\" + 0.005*\"case\" + 0.005*\"give\"'),\n",
       " (3,\n",
       "  '0.026*\"order\" + 0.021*\"protection\" + 0.019*\"child\" + 0.018*\"legislation\" + 0.016*\"bill\" + 0.015*\"local\" + 0.012*\"government\" + 0.011*\"board\" + 0.010*\"young\" + 0.009*\"violence\" + 0.009*\"family\" + 0.009*\"submission\" + 0.009*\"way\" + 0.009*\"domestic\" + 0.009*\"committee\" + 0.008*\"process\" + 0.008*\"auckland\" + 0.007*\"age\" + 0.007*\"police\" + 0.007*\"provision\"'),\n",
       " (4,\n",
       "  '0.012*\"bill\" + 0.010*\"think\" + 0.009*\"point\" + 0.009*\"issue\" + 0.008*\"government\" + 0.008*\"committee\" + 0.007*\"offender\" + 0.006*\"order\" + 0.006*\"labour\" + 0.006*\"house\" + 0.006*\"time\" + 0.006*\"member\" + 0.006*\"come\" + 0.005*\"victim\" + 0.005*\"levy\" + 0.005*\"fine\" + 0.005*\"new\" + 0.005*\"select\" + 0.005*\"look\" + 0.005*\"pay\"'),\n",
       " (5,\n",
       "  '0.017*\"bill\" + 0.010*\"member\" + 0.010*\"think\" + 0.010*\"government\" + 0.008*\"committee\" + 0.008*\"court\" + 0.007*\"new\" + 0.007*\"justice\" + 0.007*\"fact\" + 0.006*\"zealand\" + 0.006*\"system\" + 0.006*\"labour\" + 0.005*\"legislation\" + 0.005*\"area\" + 0.005*\"give\" + 0.005*\"people\" + 0.005*\"point\" + 0.005*\"way\" + 0.005*\"criminal\" + 0.005*\"link\"'),\n",
       " (6,\n",
       "  '0.018*\"question\" + 0.016*\"bill\" + 0.011*\"answer\" + 0.010*\"member\" + 0.010*\"think\" + 0.009*\"speaker\" + 0.009*\"ardern\" + 0.008*\"minister\" + 0.008*\"issue\" + 0.008*\"want\" + 0.008*\"bennett\" + 0.008*\"house\" + 0.007*\"māori\" + 0.007*\"order\" + 0.006*\"see\" + 0.006*\"jacinda\" + 0.006*\"new\" + 0.006*\"david\" + 0.006*\"way\" + 0.006*\"need\"'),\n",
       " (7,\n",
       "  '0.019*\"bill\" + 0.016*\"committee\" + 0.012*\"think\" + 0.009*\"act\" + 0.009*\"point\" + 0.008*\"select\" + 0.008*\"law\" + 0.008*\"set\" + 0.008*\"member\" + 0.007*\"amendment\" + 0.007*\"year\" + 0.007*\"house\" + 0.007*\"period\" + 0.007*\"give\" + 0.006*\"time\" + 0.006*\"labour\" + 0.006*\"issue\" + 0.006*\"claim\" + 0.005*\"come\" + 0.005*\"defence\"'),\n",
       " (8,\n",
       "  '0.010*\"bill\" + 0.010*\"government\" + 0.010*\"think\" + 0.009*\"committee\" + 0.009*\"come\" + 0.009*\"point\" + 0.008*\"member\" + 0.008*\"issue\" + 0.007*\"legislation\" + 0.007*\"new\" + 0.007*\"order\" + 0.007*\"labour\" + 0.006*\"change\" + 0.006*\"ardern\" + 0.006*\"people\" + 0.006*\"person\" + 0.006*\"right\" + 0.005*\"want\" + 0.005*\"minister\" + 0.005*\"child\"'),\n",
       " (9,\n",
       "  '0.025*\"people\" + 0.025*\"government\" + 0.021*\"young\" + 0.013*\"work\" + 0.011*\"job\" + 0.010*\"youth\" + 0.010*\"bill\" + 0.009*\"member\" + 0.008*\"new\" + 0.008*\"issue\" + 0.007*\"labour\" + 0.007*\"percent\" + 0.007*\"education\" + 0.007*\"minister\" + 0.007*\"unemployment\" + 0.007*\"see\" + 0.007*\"hear\" + 0.006*\"benefit\" + 0.006*\"time\" + 0.006*\"fact\"'),\n",
       " (10,\n",
       "  '0.049*\"student\" + 0.023*\"bill\" + 0.012*\"university\" + 0.011*\"people\" + 0.010*\"service\" + 0.010*\"association\" + 0.009*\"member\" + 0.009*\"choice\" + 0.009*\"young\" + 0.007*\"government\" + 0.006*\"house\" + 0.006*\"study\" + 0.006*\"point\" + 0.005*\"give\" + 0.005*\"ask\" + 0.005*\"institution\" + 0.005*\"voice\" + 0.005*\"education\" + 0.005*\"able\" + 0.005*\"time\"'),\n",
       " (11,\n",
       "  '0.017*\"proportion\" + 0.016*\"speaker\" + 0.014*\"paula\" + 0.014*\"bennett\" + 0.012*\"package\" + 0.012*\"service\" + 0.011*\"small\" + 0.010*\"$\" + 0.010*\"benefit\" + 0.009*\"ardern\" + 0.008*\"minister\" + 0.007*\"jacinda\" + 0.007*\"card\" + 0.007*\"number\" + 0.005*\"school\" + 0.005*\"transitions\" + 0.005*\"connect\" + 0.005*\"administration\" + 0.005*\"report\" + 0.005*\"information\"'),\n",
       " (12,\n",
       "  '0.033*\"council\" + 0.020*\"government\" + 0.019*\"auckland\" + 0.015*\"organisation\" + 0.013*\"way\" + 0.013*\"waterfront\" + 0.013*\"control\" + 0.009*\"development\" + 0.009*\"public\" + 0.009*\"central\" + 0.007*\"local\" + 0.007*\"say\" + 0.006*\"operate\" + 0.006*\"concern\" + 0.006*\"aucklander\" + 0.006*\"submitter\" + 0.005*\"recession\" + 0.005*\"leadership\" + 0.005*\"strong\" + 0.005*\"change\"'),\n",
       " (13,\n",
       "  '0.020*\"bill\" + 0.014*\"member\" + 0.010*\"think\" + 0.009*\"point\" + 0.008*\"government\" + 0.008*\"legislation\" + 0.008*\"child\" + 0.008*\"committee\" + 0.008*\"new\" + 0.007*\"honorary\" + 0.007*\"provision\" + 0.006*\"justice\" + 0.006*\"young\" + 0.006*\"minister\" + 0.006*\"prison\" + 0.005*\"clause\" + 0.005*\"student\" + 0.005*\"zealand\" + 0.005*\"amendment\" + 0.005*\"like\"')]"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda_model.show_topics(num_words=20, num_topics=14)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05941db8",
   "metadata": {},
   "source": [
    "Load the data from r/AmItheAsshole \n",
    "Create a dictionary with the id as key and the text as value of each submission (the first 500)\n",
    "Get the entities of each text in a dictionary with the submission id as key and the entities in a list as value\n",
    "Create a dictionary id as key and sentiment analysis (using NaiveBaiesAnalyzer) as value (first 50)\n",
    "Apply gensim's LDA topic model\n",
    "Check the TF-IDF value of a few words in a few texts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "04d7d0bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "9b46b622",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open ('data/amitheasshole_corpus.json') as f:\n",
    "    data = json.load(f)\n",
    "data = data[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "68a1cdc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "big_dic = dict()\n",
    "for dic in data:\n",
    "    key = dic['id']\n",
    "    big_dic[key] = dic['selftext']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "0dba0cc6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'dict_keys'>\n"
     ]
    }
   ],
   "source": [
    "ids = big_dic.keys()\n",
    "ent_dic = dict()\n",
    "for key in ids:\n",
    "    doc = nlp(big_dic[key])\n",
    "    ents = []\n",
    "    for token in doc.ents:\n",
    "        ents.append(token.label_)\n",
    "    ent_dic[key] = ents\n",
    "print(type(ids))\n",
    "#    for token in doc.ents:\n",
    "#        print(token.text, token.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "f4252a55",
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = list(ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "445bf6e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "ids = ids[:50]\n",
    "bay_dic = dict()\n",
    "for key in ids:\n",
    "    doc = nlp(big_dic[key])\n",
    "    blob = TextBlob(doc.text, analyzer=NaiveBayesAnalyzer())\n",
    "    bay_dic[key]=blob.sentiment\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "a2425c54",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'b7umhd': Sentiment(classification='neg', p_pos=0.07053810604213762, p_neg=0.9294618939578514), 'b7uqbc': Sentiment(classification='neg', p_pos=0.010081616066492573, p_neg=0.9899183839335151), 'b7uql4': Sentiment(classification='neg', p_pos=0.0016224541974929375, p_neg=0.9983775458025028), 'b7ur4w': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7urfq': Sentiment(classification='pos', p_pos=0.9999998732873295, p_neg=1.2671267871025073e-07), 'b7usq5': Sentiment(classification='pos', p_pos=0.7640500503732923, p_neg=0.2359499496267003), 'b7usr7': Sentiment(classification='neg', p_pos=0.43115969473902876, p_neg=0.5688403052609715), 'b7ustg': Sentiment(classification='pos', p_pos=0.9955498634918254, p_neg=0.004450136508138082), 'b7uugl': Sentiment(classification='pos', p_pos=0.9998580979176017, p_neg=0.00014190208240617738), 'b7uuqj': Sentiment(classification='pos', p_pos=0.9999998667318948, p_neg=1.3326813946807605e-07), 'b7uux6': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7uwjg': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7uxny': Sentiment(classification='pos', p_pos=0.9992016203549462, p_neg=0.0007983796450503507), 'b7uyy5': Sentiment(classification='pos', p_pos=0.999998845353316, p_neg=1.1546466767310947e-06), 'b7uz0r': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7uz2k': Sentiment(classification='pos', p_pos=0.8277741271300787, p_neg=0.17222587286992824), 'b7uzly': Sentiment(classification='pos', p_pos=0.9999932828852793, p_neg=6.717114730632535e-06), 'b7v0uq': Sentiment(classification='pos', p_pos=0.9911017648049595, p_neg=0.008898235195037733), 'b7v0vn': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7v3b2': Sentiment(classification='pos', p_pos=0.999999996661802, p_neg=3.3382138292875192e-09), 'b7v3go': Sentiment(classification='pos', p_pos=0.9999384416386015, p_neg=6.155836141755273e-05), 'b7v4c0': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7v5b2': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7v6ni': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7v70v': Sentiment(classification='pos', p_pos=0.9692085971002756, p_neg=0.030791402899718632), 'b7v7ix': Sentiment(classification='pos', p_pos=0.9991552429957531, p_neg=0.0008447570042337648), 'b7v8a8': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7v8r9': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7v932': Sentiment(classification='neg', p_pos=0.03906394848385614, p_neg=0.9609360515161587), 'b7vaa1': Sentiment(classification='neg', p_pos=0.1784716429356814, p_neg=0.8215283570642801), 'b7vdza': Sentiment(classification='neg', p_pos=0.45362116477796094, p_neg=0.5463788352220317), 'b7ve6p': Sentiment(classification='pos', p_pos=0.9999999961104263, p_neg=3.889571216580716e-09), 'b7veaa': Sentiment(classification='pos', p_pos=0.9868850050100003, p_neg=0.01311499498999397), 'b7vf3k': Sentiment(classification='pos', p_pos=0.985518885142547, p_neg=0.014481114857453849), 'b7vf7y': Sentiment(classification='pos', p_pos=0.8539027014800324, p_neg=0.14609729851996509), 'b7vfx7': Sentiment(classification='pos', p_pos=0.8024862508555998, p_neg=0.1975137491444062), 'b7vg0f': Sentiment(classification='neg', p_pos=0.4622077361443207, p_neg=0.537792263855688), 'b7vgxu': Sentiment(classification='pos', p_pos=0.9941626319444288, p_neg=0.005837368055570446), 'b7vi6v': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7vkow': Sentiment(classification='neg', p_pos=0.02631166510218357, p_neg=0.9736883348978024), 'b7vmb3': Sentiment(classification='neg', p_pos=0.0749497660920781, p_neg=0.9250502339079252), 'b7vmob': Sentiment(classification='pos', p_pos=0.9778268449795784, p_neg=0.02217315502042899), 'b7vn6e': Sentiment(classification='pos', p_pos=0.9406328955557208, p_neg=0.05936710444427403), 'b7vnw2': Sentiment(classification='pos', p_pos=0.9999999999998423, p_neg=1.9127675964388584e-13), 'b7vo4y': Sentiment(classification='pos', p_pos=0.5, p_neg=0.5), 'b7vow9': Sentiment(classification='pos', p_pos=0.9908814536354952, p_neg=0.00911854636449087), 'b7voxc': Sentiment(classification='neg', p_pos=0.1764243121713492, p_neg=0.8235756878286465), 'b7vp1q': Sentiment(classification='pos', p_pos=0.9957285454843275, p_neg=0.004271454515680308), 'b7vpd3': Sentiment(classification='pos', p_pos=0.9999999982274344, p_neg=1.7725752063125753e-09), 'b7vq3z': Sentiment(classification='pos', p_pos=0.999974803569071, p_neg=2.5196430933518342e-05)}\n"
     ]
    }
   ],
   "source": [
    "print(bay_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "4bb11a7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = list(big_dic.values())    \n",
    "tokenized_corpus = []\n",
    "for text in texts:\n",
    "    text = text.replace('\\n','')\n",
    "        \n",
    "    doc = nlp(text)\n",
    "        \n",
    "    text_proc = []\n",
    "        \n",
    "    for token in doc:\n",
    "        if not token.is_stop \\\n",
    "        and not token.is_punct and not token.like_num \\\n",
    "        and token.lemma_.lower():\n",
    "            text_proc.append(token.lemma_.lower())\n",
    "    tokenized_corpus.append(text_proc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "9426b585",
   "metadata": {},
   "outputs": [],
   "source": [
    "words_id = corpora.Dictionary(tokenized_corpus)\n",
    "corpus = [words_id.doc2bow(txt) for txt in tokenized_corpus]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "e02ea1c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 -7.38165957638928 0.2796241965391861\n",
      "6 -7.389926153975857 0.29005080540071576\n",
      "7 -7.39326923988637 0.28244279196534927\n",
      "8 -7.412276935220026 0.2911501772109961\n",
      "9 -7.431213394051309 0.2914648938296913\n",
      "10 -7.432607603328135 0.29057710331337827\n",
      "11 -7.443105276696423 0.29064768894927717\n",
      "12 -7.449090640405304 0.28695463469366184\n",
      "13 -7.465852793751797 0.2890726978574362\n",
      "14 -7.467677282568409 0.30715440264471433\n",
      "15 -7.467095063869201 0.2947829989363893\n"
     ]
    }
   ],
   "source": [
    "k_init = 5\n",
    "k_final =15\n",
    "\n",
    "for k in range(k_init, k_final +1):\n",
    "    lda_model = gensim.models.ldamodel.LdaModel(corpus = corpus,\n",
    "                                              id2word = words_id,\n",
    "                                              num_topics = k,\n",
    "                                              random_state = 50,\n",
    "                                              passes = 20,\n",
    "                                              per_word_topics = True)\n",
    "\n",
    "    per_lda = lda_model.log_perplexity(corpus)\n",
    "    coherence_model_lda = CoherenceModel(model = lda_model, texts = tokenized_corpus, dictionary=words_id,\n",
    "                                        coherence='c_v')\n",
    "    coherence_lda = coherence_model_lda.get_coherence()\n",
    "    print(k, per_lda, coherence_lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "59a22eb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "lda_model = gensim.models.ldamodel.LdaModel(corpus = corpus,\n",
    "                                             id2word = words_id,\n",
    "                                             num_topics = 14,\n",
    "                                             random_state = 50,\n",
    "                                             passes = 20,\n",
    "                                             per_word_topics = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "cd4149b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0,\n",
       "  '0.018*\"say\" + 0.015*\"like\" + 0.014*\"friend\" + 0.013*\"tell\" + 0.011*\"feel\" + 0.011*\"want\" + 0.011*\"time\" + 0.010*\"go\" + 0.010*\"know\" + 0.008*\"year\" + 0.008*\"try\" + 0.008*\"get\" + 0.007*\"start\" + 0.006*\"good\" + 0.006*\"relationship\" + 0.006*\"think\" + 0.006*\"thing\" + 0.005*\"talk\" + 0.005*\"stop\" + 0.005*\"pretty\"'),\n",
       " (1,\n",
       "  '0.010*\"time\" + 0.010*\"work\" + 0.009*\"tell\" + 0.008*\"ask\" + 0.008*\"customer\" + 0.008*\"try\" + 0.008*\"like\" + 0.007*\"say\" + 0.007*\"feel\" + 0.007*\"thing\" + 0.006*\"want\" + 0.006*\"care\" + 0.006*\"people\" + 0.005*\"day\" + 0.005*\"help\" + 0.005*\"nancy\" + 0.005*\"property\" + 0.005*\"get\" + 0.005*\"wife\" + 0.004*\"talk\"'),\n",
       " (2,\n",
       "  '0.023*\"not\" + 0.022*\"tell\" + 0.017*\"go\" + 0.015*\"see\" + 0.014*\"time\" + 0.012*\"feel\" + 0.012*\"like\" + 0.012*\"d\" + 0.011*\"say\" + 0.011*\"break\" + 0.010*\"want\" + 0.010*\"come\" + 0.009*\"day\" + 0.008*\"bad\" + 0.008*\"work\" + 0.008*\"ask\" + 0.007*\"m\" + 0.007*\"call\" + 0.006*\"think\" + 0.006*\"people\"'),\n",
       " (3,\n",
       "  '0.015*\"say\" + 0.012*\"get\" + 0.012*\"time\" + 0.011*\"tell\" + 0.009*\"ask\" + 0.009*\"go\" + 0.008*\"want\" + 0.008*\"come\" + 0.007*\"like\" + 0.007*\"thing\" + 0.006*\"start\" + 0.006*\"people\" + 0.006*\"talk\" + 0.006*\"day\" + 0.006*\"know\" + 0.005*\"try\" + 0.005*\"friend\" + 0.005*\"think\" + 0.005*\"good\" + 0.005*\"call\"'),\n",
       " (4,\n",
       "  '0.018*\"friend\" + 0.017*\"tell\" + 0.016*\"go\" + 0.013*\"want\" + 0.010*\"ask\" + 0.010*\"know\" + 0.010*\"say\" + 0.009*\"year\" + 0.009*\"time\" + 0.009*\"like\" + 0.008*\"boyfriend\" + 0.007*\"get\" + 0.006*\"month\" + 0.006*\"work\" + 0.006*\"talk\" + 0.005*\"think\" + 0.005*\"wife\" + 0.005*\"feel\" + 0.005*\"close\" + 0.005*\"car\"'),\n",
       " (5,\n",
       "  '0.010*\"know\" + 0.010*\"room\" + 0.009*\"tell\" + 0.009*\"feel\" + 0.008*\"go\" + 0.008*\"like\" + 0.008*\"say\" + 0.007*\"bike\" + 0.007*\"ask\" + 0.007*\"year\" + 0.007*\"time\" + 0.006*\"find\" + 0.006*\"get\" + 0.006*\"think\" + 0.006*\"friend\" + 0.005*\"work\" + 0.005*\"relationship\" + 0.005*\"thing\" + 0.005*\"guy\" + 0.004*\"let\"'),\n",
       " (6,\n",
       "  '0.046*\" \" + 0.014*\"sister\" + 0.013*\"say\" + 0.012*\"time\" + 0.011*\"want\" + 0.011*\"like\" + 0.010*\"go\" + 0.009*\"feel\" + 0.009*\"people\" + 0.007*\"nephew\" + 0.007*\"tell\" + 0.007*\"month\" + 0.006*\"try\" + 0.006*\"know\" + 0.006*\"give\" + 0.006*\"leave\" + 0.005*\"gf\" + 0.005*\"thing\" + 0.005*\"get\" + 0.005*\"away\"'),\n",
       " (7,\n",
       "  '0.017*\"say\" + 0.016*\"want\" + 0.013*\"tell\" + 0.012*\"come\" + 0.012*\"like\" + 0.011*\"get\" + 0.011*\"ask\" + 0.010*\"feel\" + 0.009*\"go\" + 0.008*\"think\" + 0.008*\"know\" + 0.007*\"time\" + 0.007*\"mom\" + 0.007*\"asshole\" + 0.006*\"room\" + 0.006*\"start\" + 0.006*\"work\" + 0.006*\"guy\" + 0.006*\"kid\" + 0.006*\"try\"'),\n",
       " (8,\n",
       "  '0.013*\"time\" + 0.013*\"year\" + 0.013*\"job\" + 0.011*\"pay\" + 0.010*\"tell\" + 0.010*\"want\" + 0.009*\"like\" + 0.008*\"know\" + 0.008*\"think\" + 0.008*\"come\" + 0.007*\"home\" + 0.007*\"travel\" + 0.007*\"live\" + 0.007*\"house\" + 0.007*\"take\" + 0.007*\"go\" + 0.007*\"month\" + 0.006*\"away\" + 0.006*\"work\" + 0.006*\"family\"'),\n",
       " (9,\n",
       "  '0.010*\"want\" + 0.010*\"play\" + 0.010*\"go\" + 0.009*\"like\" + 0.009*\"look\" + 0.008*\"tell\" + 0.007*\"get\" + 0.007*\"kid\" + 0.007*\"people\" + 0.006*\"mom\" + 0.006*\"game\" + 0.006*\"think\" + 0.006*\"say\" + 0.006*\" \" + 0.006*\"time\" + 0.006*\"ask\" + 0.006*\"end\" + 0.005*\"sibling\" + 0.004*\"sock\" + 0.004*\"deadbeat\"'),\n",
       " (10,\n",
       "  '0.013*\"friend\" + 0.013*\"like\" + 0.011*\"feel\" + 0.009*\"tell\" + 0.008*\"day\" + 0.008*\"week\" + 0.008*\"time\" + 0.007*\"party\" + 0.007*\"family\" + 0.007*\"work\" + 0.007*\"know\" + 0.006*\"ask\" + 0.006*\"say\" + 0.006*\"get\" + 0.006*\"spend\" + 0.006*\"go\" + 0.006*\"thing\" + 0.006*\"want\" + 0.006*\"asshole\" + 0.006*\"kid\"'),\n",
       " (11,\n",
       "  '0.024*\"friend\" + 0.018*\"tell\" + 0.018*\" \" + 0.016*\"say\" + 0.016*\"like\" + 0.013*\"want\" + 0.013*\"time\" + 0.009*\"ask\" + 0.009*\"think\" + 0.009*\"go\" + 0.008*\"year\" + 0.008*\"know\" + 0.008*\"day\" + 0.008*\"talk\" + 0.007*\"people\" + 0.007*\"thing\" + 0.007*\"feel\" + 0.006*\"dad\" + 0.006*\"come\" + 0.006*\"leave\"'),\n",
       " (12,\n",
       "  '0.098*\"remove\" + 0.010*\"feel\" + 0.010*\"blood\" + 0.009*\"work\" + 0.008*\"f\" + 0.007*\"like\" + 0.007*\"way\" + 0.007*\"father\" + 0.007*\"want\" + 0.006*\"seat\" + 0.006*\"people\" + 0.005*\"know\" + 0.005*\"month\" + 0.005*\"able\" + 0.005*\"body\" + 0.005*\"count\" + 0.005*\"ab+\" + 0.005*\"fiancée\" + 0.005*\"talk\" + 0.005*\"ask\"'),\n",
       " (13,\n",
       "  '0.010*\"light\" + 0.009*\"student\" + 0.008*\"get\" + 0.008*\"go\" + 0.008*\"dress\" + 0.008*\"lamp\" + 0.007*\"people\" + 0.007*\"bra\" + 0.006*\"room\" + 0.006*\"work\" + 0.006*\"ask\" + 0.006*\"race\" + 0.005*\"wedding\" + 0.005*\"sport\" + 0.005*\"card\" + 0.005*\"mensa\" + 0.005*\"discount\" + 0.005*\"space\" + 0.005*\" \" + 0.005*\"turn\"')]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lda_model.show_topics(num_words=20, num_topics=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec09f299",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}