GensimStreamingTweetCorpus-Rob.ipynb

{
 "metadata": {
  "name": "GensimStreamingTweetCorpus-Rob"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "This code analyzes corpus of tweets using Gensim"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%pylab inline\n",
      "plt.rc('figure', figsize=(8, 5))\n",
      "sys.path.append('/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/')\n",
      "sys.path.append('/Users/doug/SW_Dev/NLTK_Experiments')\n",
      "import nltk\n",
      "import pprint\n",
      "import json\n",
      "import os\n",
      "import gzip\n",
      "import json\n",
      "import collections\n",
      "import operator\n",
      "from BeautifulSoup import BeautifulSoup, NavigableString\n",
      "from ttp import ttp\n",
      "import string\n",
      "import logging\n",
      "from gensim import corpora, models, similarities\n",
      "import logging, gensim, bz2\n"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Welcome to pylab, a matplotlib-based Python environment [backend: module://IPython.zmq.pylab.backend_inline].\n",
        "For more information, type 'help(pylab)'.\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dir_path_root = '/Users/doug/SW_Dev/NLTK_Experiments/MyCorpus/'\n",
      "dir_path = dir_path_root+'RIT/'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#generator function to return all tweets in a zipped firectory\n",
      "def all_tweets(directory, extract=lambda x:json.loads(x)):\n",
      "    #print directory\n",
      "    for fn in os.listdir(directory):\n",
      "        print fn\n",
      "        if fn.endswith('.json'):\n",
      "            f = open(directory+fn, 'rb')\n",
      "            for line in f:\n",
      "                yield extract(line)\n",
      "            f.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def myfilter(x):\n",
      "    x = json.loads(x)\n",
      "    \n",
      "    #pprint.pprint(x)\n",
      "    #raise Exception\n",
      "    result={}\n",
      "    if 'delete' not in x:\n",
      "        result['id_str'] = x['id_str']\n",
      "        #result['text'] = x['text']\n",
      "        result['screen_name'] = x['user']['screen_name']\n",
      "        #result['name'] = x['user']['name']\n",
      "        result['user_mentions'] = [men['screen_name'].lower() for men in x['entities']['user_mentions']]\n",
      "        result['hashtags'] = [entry['text'].lower() for entry in x['entities']['hashtags']]\n",
      "        result['lang'] = x['lang']\n",
      "        return result\n",
      "    else:\n",
      "        False"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#test routine for all_tweets\n",
      "total_tweets = 0\n",
      "user_dict = corpora.Dictionary([])\n",
      "ment_dict = corpora.Dictionary([])\n",
      "hash_dict = corpora.Dictionary([])\n",
      "def foo():\n",
      "    for x in all_tweets(dir_path, extract=myfilter):\n",
      "        if x and x['lang']=='en':\n",
      "            global total_tweets\n",
      "            total_tweets += 1\n",
      "            user_dict.doc2bow([x['screen_name']], allow_update=True)\n",
      "            hash_dict.doc2bow(x['hashtags'], allow_update=True)\n",
      "            ment_dict.doc2bow(x['user_mentions'], allow_update=True)\n",
      "\n",
      "    print 'total english tweets: ', total_tweets\n",
      "\n",
      "foo()\n",
      "user_dict.filter_extremes(no_below=5, no_above=0.2, keep_n=1000)\n",
      "#ment_dict.filter_extremes(no_below=5, no_above=0.2, keep_n=50)\n",
      "#hash_dict.filter_extremes(no_below=5, no_above=0.2, keep_n=50)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        ".DS_Store\n",
        "rithashtagdump.json\n",
        "total english tweets: "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        " 1419\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Attempt at creating my own corpus"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def parse_tweet(text, include_text=True, include_users=False, include_tags=False, include_reply=False, include_url=False):\n",
      "    \"Simple Tweet Parser\"\n",
      "    result = []\n",
      "    tweet = ttp.Parser().parse(text)\n",
      "    \n",
      "    if include_text:\n",
      "        soup = BeautifulSoup(tweet.html)\n",
      "        result =  [x.split() for x in soup.contents if type(x) is NavigableString]\n",
      "        result = [item for sublist in result for item in sublist]  #flatten list\n",
      "        result = [''.join(c for c in s if c not in string.punctuation) for s in result] # remove punctuation\n",
      "        result = [s for s in result if s] # empty empty strings\n",
      "        \n",
      "    if include_reply:\n",
      "        result.append(\"@{}\".format(tweet.reply))\n",
      "        \n",
      "    if include_users:\n",
      "        result = result + [\"@{}\".format(tag) for tag in tweet.users]\n",
      "        \n",
      "    if include_tags:\n",
      "        result = result + [\"#{}\".format(tag) for tag in tweet.tags]\n",
      "        \n",
      "    if include_url:\n",
      "        result = result + tweet.urls\n",
      "    \n",
      "    return result\n",
      "\n",
      "#parse_tweet(\"@ianozsvald, you now support #IvoWertzel's tweet parser! https://github.com/ianozsvald/\", include_reply=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#logging.basicConfig(level=logging.WARNING)\n",
      "\n",
      "def default_extract(line):\n",
      "        tweet = json.loads(line)\n",
      "        #pprint.pprint(tweet)\n",
      "        if ('delete' not in tweet) and ('disconnect' not in tweet):\n",
      "            return parse_tweet(tweet['text'])\n",
      "        else:\n",
      "            False\n",
      "\n",
      "class MyCorpus(corpora.TextCorpus):\n",
      "    stoplist = set('for a of the and to in on'.split())\n",
      "    \n",
      "    def __init__(self, corpus_prefix=None, corpus_path_root=None, count=None, sample=None, extract=None):\n",
      "        self.input = input\n",
      "        self.dict = corpora.Dictionary()\n",
      "        self.stop_words = nltk.corpus.stopwords.words('english')\n",
      "        if corpus_path_root:\n",
      "            self.corpus_path = corpus_path_root\n",
      "        else:\n",
      "            self.corpus_path = '/Users/doug/SW_Dev/NLTK_Experiments/MyCorpus/'\n",
      "            \n",
      "        if corpus_prefix:\n",
      "            self.corpus_path = self.corpus_path + corpus_prefix + '/'\n",
      "        else:\n",
      "            self.corpus_path = self.corpus_path + 'RIT/'\n",
      "            \n",
      "        if extract:\n",
      "            self.extract = extract\n",
      "        else:\n",
      "            self.extract = default_extract\n",
      "\n",
      "\n",
      "    def get_texts(self, limit=False):\n",
      "        count = 0\n",
      "        \n",
      "        for fn in os.listdir(self.corpus_path):\n",
      "            if fn.endswith('.json'):\n",
      "                f = open(self.corpus_path+fn, 'rb')\n",
      "                try:\n",
      "                    for line in f:\n",
      "                        if limit and count >= limit:\n",
      "                            raise Exception('limit')\n",
      "                        tokens = set(self.extract(line))\n",
      "                        tokens = tokens.difference(self.stop_words)\n",
      "                        if tokens:\n",
      "                            yield tokens\n",
      "                            count += 1\n",
      "                except IOError:\n",
      "                    pass\n",
      "                except Exception:\n",
      "                    pass\n",
      "                f.close()\n",
      "\n",
      "    def __iter__(self, limit=False, allow_update=True):\n",
      "        for tokens in self.get_texts(limit=limit):\n",
      "            #d = self.dict\n",
      "            yield(self.dict.doc2bow(tokens, allow_update=allow_update))\n",
      "\n",
      "    def get_dict(self):\n",
      "        return self.dict\n",
      "\n",
      "    def __len__(self):\n",
      "        \"\"\"Define this so we can use `len(corpus)`\"\"\"\n",
      "        if 'length' not in self.__dict__:\n",
      "            #logger.info(\"caching corpus size (calculating number of documents)\")\n",
      "            #self.length = sum(1 for doc in self.get_texts())\n",
      "            self.length = sum(1 for doc in self)\n",
      "        return self.length"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "m = MyCorpus()\n",
      "print 'Corpus size: ', len(m)\n",
      "print m.get_dict()\n",
      "id2word = m.get_dict()\n",
      "\n",
      "# extract 100 LDA topics, using 1 pass and updating once every 1 chunk (10,000 documents)\n",
      "#lda = gensim.models.ldamodel.LdaModel(corpus=m, id2word=id2word, num_topics=100, update_every=1, chunksize=10000, passes=20)\n",
      "lda = gensim.models.ldamodel.LdaModel(corpus=m, id2word=id2word, num_topics=100, update_every=0, passes=20)\n",
      "lda.print_topics(20)\n",
      "print 'done'\n",
      "lda.show_topics()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Corpus size:  "
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1394\n",
        "Dictionary(2759 unique tokens)\n",
        "done"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n"
       ]
      },
      {
       "output_type": "pyout",
       "prompt_number": 9,
       "text": [
        "[u'0.104*I + 0.083*tweet + 0.046*DAY + 0.039*Its + 0.037*RT + 0.037*4pm + 0.037*GAME + 0.037*Stadium + 0.037*vs + 0.037*Tiger',\n",
        " u'0.117*Best + 0.083*treasury + 0.065*word + 0.041*tre + 0.019*everyone + 0.019*ended + 0.019*hours + 0.019*ago + 0.019*shooting + 0.019*totally',\n",
        " u'0.038*volleyball + 0.029*RECAP + 0.028*Classic + 0.028*NYU + 0.028*Violet + 0.019*Korra + 0.019*drops + 0.019*matches + 0.019*DeSales + 0.010*RT',\n",
        " u'0.042*The + 0.042*comes + 0.028*RT + 0.028*\\u2026 + 0.028*Entrepreneurial + 0.028*Leadership + 0.028*afternoon + 0.028*Spending + 0.028*Summit + 0.028*right',\n",
        " u'0.039*RT + 0.018*event + 0.018*planning + 0.018*check + 0.018*Hey + 0.018*I + 0.018*In + 0.009*\\u2026 + 0.009*campus + 0.009*class',\n",
        " u'0.044*vs + 0.031*forces + 0.027*set + 0.024*2 + 0.021*week + 0.021*suit + 0.021*fifth + 0.021*2517 + 0.021*22 + 0.021*Tied',\n",
        " u'0.033*Invitational + 0.033*country + 0.033*fourth + 0.033*teams + 0.033*places + 0.033*13 + 0.033*Yellowjacket + 0.033*womens + 0.033*cross + 0.017*come',\n",
        " u'0.065*RT + 0.036*This + 0.029*Stephanie + 0.029*Congrats + 0.029*Advisory + 0.029*Board + 0.029*Dr + 0.029*Ludi + 0.029*Principles + 0.029*appointed',\n",
        " u'0.056*finds + 0.037*Great + 0.020*30 + 0.019*sets + 0.019*drops + 0.019*2826 + 0.019*volleyball + 0.019*2512 + 0.019*76 + 0.019*won',\n",
        " u'0.064*2 + 0.040*MSOCC + 0.038*assists + 0.029*us + 0.029*afternoon + 0.029*scores + 0.028*30 + 0.021*3 + 0.019*Picnic + 0.019*Saunders']"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lda.print_topics(200)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "m = MyCorpus()\n",
      "print 'Corpus size: ', len(m)\n",
      "print m.get_dict()\n",
      "id2word = m.get_dict()\n",
      "\n",
      "# extract 100 LDA topics, using 1 pass and updating once every 1 chunk (10,000 documents)\n",
      "#lda = gensim.models.ldamodel.LdaModel(corpus=m, id2word=id2word, num_topics=100, update_every=1, chunksize=10000, passes=20)\n",
      "lsi = gensim.models.lsimodel.LsiModel(corpus=m, id2word=id2word, num_topics=100)\n",
      "lsi.print_topics(20)\n",
      "print 'done'\n",
      "lsi.show_topics()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Corpus size:  "
       ]
      },
      {
       "ename": "TypeError",
       "evalue": "__init__() got an unexpected keyword argument 'passes'",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-14-4bc36fb00678>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# extract 100 LDA topics, using 1 pass and updating once every 1 chunk (10,000 documents)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m#lda = gensim.models.ldamodel.LdaModel(corpus=m, id2word=id2word, num_topics=100, update_every=1, chunksize=10000, passes=20)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mlsi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgensim\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlsimodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLsiModel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcorpus\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mid2word\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mid2word\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnum_topics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m100\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpasses\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0mlsi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprint_topics\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;32mprint\u001b[0m \u001b[0;34m'done'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'passes'"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "1394\n",
        "Dictionary(2759 unique tokens)\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\n",
      "#import my_feature_ex as fx\n",
      "import my_word_cloud as wcloud"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def sorted_counts(wc):\n",
      "    sorted_wc = sorted(wc.iteritems(), key=operator.itemgetter(1), reverse=True)\n",
      "    sorted_wc = [val for val in sorted_wc if val[1]>1]   #prune singleton values\n",
      "    #print len(sorted_wc)\n",
      "    words = [word for word, count in sorted_wc]\n",
      "    counts = [count for word, count in sorted_wc]\n",
      "    return words, counts\n",
      "    \n",
      "def display_cloud(wc):\n",
      "    words, counts = sorted_counts(wc)\n",
      "    print words[:20]\n",
      "    words2 =  np.array(words[:200], np.dtype('string'))\n",
      "    counts2 =  np.array(counts[:200], np.int32)\n",
      "    wcloud.display_wordcloud(words2,counts2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#display_cloud(users)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#display_cloud(mentions)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#display_cloud(hashtags)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    }
   ],
   "metadata": {}
  }
 ]
}