Gensim Tutorials.ipynb

{
 "metadata": {
  "name": "Gensim Tutorials"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "To Do:\n",
      "integrate gensim"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sys\n",
      "import cPickle as pickle\n",
      "import pprint\n",
      "import collections\n",
      "#import nltk\n",
      "#from nltk.stem import WordNetLemmatizer\n",
      "sys.path.append('/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/')\n",
      "sys.path.append('/Users/doug/SW_Dev/NLTK_Experiments/')\n",
      "\n",
      "import math\n",
      "import numpy as np\n",
      "\n",
      "#import cmu_tweet_word_clusters\n",
      "import my_feature_ex as fx"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "skipped:  111010100010\t\t212\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import logging\n",
      "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
      "from gensim import corpora, models, similarities"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Corpora and Vector Spaces Tutorial"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "documents = [\"Human machine interface for lab abc computer applications\",\n",
      "             \"A survey of user opinion of computer system response time\",\n",
      "             \"The EPS user interface management system\",\n",
      "             \"System and human system engineering testing of EPS\",\n",
      "             \"Relation of user perceived response time to error measurement\",\n",
      "             \"The generation of random binary unordered trees\",\n",
      "             \"The intersection graph of paths in trees\",\n",
      "             \"Graph minors IV Widths of trees and well quasi ordering\",\n",
      "             \"Graph minors A survey\"]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# remove common words and tokenize\n",
      "stoplist = set('for a of the and to in'.split())\n",
      "texts = [[word for word in document.lower().split() if word not in stoplist]\n",
      "         for document in documents]\n",
      "\n",
      "# remove words that appear only once\n",
      "all_tokens = sum(texts, [])\n",
      "tokens_once = set(word for word in set(all_tokens) if all_tokens.count(word) == 1)\n",
      "texts = [[word for word in text if word not in tokens_once]\n",
      "         for text in texts]\n",
      "\n",
      "pprint.pprint(texts)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[['human', 'interface', 'computer'],\n",
        " ['survey', 'user', 'computer', 'system', 'response', 'time'],\n",
        " ['eps', 'user', 'interface', 'system'],\n",
        " ['system', 'human', 'system', 'eps'],\n",
        " ['user', 'response', 'time'],\n",
        " ['trees'],\n",
        " ['graph', 'trees'],\n",
        " ['graph', 'minors', 'trees'],\n",
        " ['graph', 'minors', 'survey']]\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dictionary = corpora.Dictionary(texts)\n",
      "dictionary.save('/tmp/deerwester.dict') # store the dictionary, for future reference\n",
      "print dictionary"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Dictionary(12 unique tokens)\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print dictionary.token2id"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'minors': 11, 'graph': 10, 'system': 5, 'trees': 9, 'eps': 8, 'computer': 0, 'survey': 4, 'user': 7, 'human': 1, 'time': 6, 'interface': 2, 'response': 3}\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "new_doc = \"Human computer interaction\"\n",
      "new_vec = dictionary.doc2bow(new_doc.lower().split())\n",
      "print new_vec # the word \"interaction\" does not appear in the dictionary and is ignored"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(0, 1), (1, 1)]\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corpus = [dictionary.doc2bow(text) for text in texts]\n",
      "corpora.MmCorpus.serialize('/tmp/deerwester.mm', corpus) # store to disk, for later use\n",
      "pprint.pprint(corpus)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[(0, 1), (1, 1), (2, 1)],\n",
        " [(0, 1), (3, 1), (4, 1), (5, 1), (6, 1), (7, 1)],\n",
        " [(2, 1), (5, 1), (7, 1), (8, 1)],\n",
        " [(1, 1), (5, 2), (8, 1)],\n",
        " [(3, 1), (6, 1), (7, 1)],\n",
        " [(9, 1)],\n",
        " [(9, 1), (10, 1)],\n",
        " [(9, 1), (10, 1), (11, 1)],\n",
        " [(4, 1), (10, 1), (11, 1)]]\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "class MyCorpus(object):\n",
      "    def __iter__(self):\n",
      "        for line in open('/Users/doug/mycorpus.txt'):\n",
      "            # assume there's one document per line, tokens separated by whitespace\n",
      "            yield dictionary.doc2bow(line.lower().split())"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corpus_memory_friendly = MyCorpus() # doesn't load the corpus into memory!\n",
      "print corpus_memory_friendly"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "<__main__.MyCorpus object at 0x115cb4b10>\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for vector in corpus_memory_friendly: # load one vector into memory at a time\n",
      "    print vector"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(0, 1), (1, 1), (2, 1)]\n",
        "[(0, 1), (3, 1), (4, 1), (5, 1), (6, 1), (7, 1)]\n",
        "[(2, 1), (5, 1), (7, 1), (8, 1)]\n",
        "[(1, 1), (5, 2), (8, 1)]\n",
        "[(3, 1), (6, 1), (7, 1)]\n",
        "[(9, 1)]\n",
        "[(9, 1), (10, 1)]\n",
        "[(9, 1), (10, 1), (11, 1)]\n",
        "[(4, 1), (10, 1), (11, 1)]\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# collect statistics about all tokens\n",
      "dictionary = corpora.Dictionary(line.lower().split() for line in open('mycorpus.txt'))\n",
      "# remove stop words and words that appear only once\n",
      "stop_ids = [dictionary.token2id[stopword] for stopword in stoplist\n",
      "            if stopword in dictionary.token2id]\n",
      "once_ids = [tokenid for tokenid, docfreq in dictionary.dfs.iteritems() if docfreq == 1]\n",
      "dictionary.filter_tokens(stop_ids + once_ids) # remove stop words and words that appear only once\n",
      "dictionary.compactify() # remove gaps in id sequence after words that were removed\n",
      "print dictionary"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Dictionary(12 unique tokens)\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from gensim import corpora\n",
      "# create a toy corpus of 2 documents, as a plain Python list\n",
      "corpus = [[(1, 0.5)], []]  # make one document empty, for the heck of it\n",
      "\n",
      "corpora.MmCorpus.serialize('/tmp/corpus.mm', corpus)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corpora.SvmLightCorpus.serialize('/tmp/corpus.svmlight', corpus)\n",
      "corpora.BleiCorpus.serialize('/tmp/corpus.lda-c', corpus)\n",
      "corpora.LowCorpus.serialize('/tmp/corpus.low', corpus)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stderr",
       "text": [
        "WARNING:gensim.corpora.lowcorpus:List-of-words format can only save vectors with integer elements; 1 float entries were truncated to integer value\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corpus = corpora.MmCorpus('/tmp/corpus.mm')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print corpus"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "MmCorpus(2 documents, 2 features, 1 non-zero entries)\n"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# one way of printing a corpus: load it entirely into memory\n",
      "print list(corpus) # calling list() will convert any sequence to a plain Python list"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[(1, 0.5)], []]\n"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# another way of doing it: print one document at a time, making use of the streaming interface\n",
      "for doc in corpus:\n",
      "    print doc"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(1, 0.5)]\n",
        "[]\n"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corpora.BleiCorpus.serialize('/tmp/corpus.lda-c', corpus)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 19
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Converting between gensim, numpy and scipy"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#import gensim\n",
      "corpus = gensim.matutils.Dense2Corpus(numpy_matrix)\n",
      "numpy_matrix = gensim.matutils.corpus2dense(corpus)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 38
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "corpus = gensim.matutils.Sparse2Corpus(scipy_sparse_matrix)\n",
      "scipy_csc_matrix = gensim.matutils.corpus2csc(corpus)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Topics and Transformations Tutorial"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dictionary = corpora.Dictionary.load('/tmp/deerwester.dict')\n",
      "corpus = corpora.MmCorpus('/tmp/deerwester.mm')\n",
      "print corpus"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "MmCorpus(9 documents, 12 features, 28 non-zero entries)\n"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tfidf = models.TfidfModel(corpus) # step 1 -- initialize a model"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "doc_bow = [(0, 1), (1, 1)]\n",
      "print tfidf[doc_bow] # step 2 -- use the model to transform vectors"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(0, 0.7071067811865476), (1, 0.7071067811865476)]\n"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# apply a transformation to a whole corpus:\n",
      "corpus_tfidf = tfidf[corpus]\n",
      "for doc in corpus_tfidf:\n",
      "    print doc"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(0, 0.5773502691896257), (1, 0.5773502691896257), (2, 0.5773502691896257)]\n",
        "[(0, 0.44424552527467476), (3, 0.44424552527467476), (4, 0.44424552527467476), (5, 0.3244870206138555), (6, 0.44424552527467476), (7, 0.3244870206138555)]\n",
        "[(2, 0.5710059809418182), (5, 0.4170757362022777), (7, 0.4170757362022777), (8, 0.5710059809418182)]\n",
        "[(1, 0.49182558987264147), (5, 0.7184811607083769), (8, 0.49182558987264147)]\n",
        "[(3, 0.6282580468670046), (6, 0.6282580468670046), (7, 0.45889394536615247)]\n",
        "[(9, 1.0)]\n",
        "[(9, 0.7071067811865475), (10, 0.7071067811865475)]\n",
        "[(9, 0.5080429008916749), (10, 0.5080429008916749), (11, 0.695546419520037)]\n",
        "[(4, 0.6282580468670046), (10, 0.45889394536615247), (11, 0.6282580468670046)]\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Transformations can also be serialized, one on top of another, in a sort of chain:\n",
      "lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=2) # initialize an LSI transformation\n",
      "corpus_lsi = lsi[corpus_tfidf] # create a double wrapper over the original corpus: bow->tfidf->fold-in-lsi\n",
      "lsi.print_topics(2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 24,
       "text": [
        "['0.703*\"trees\" + 0.538*\"graph\" + 0.402*\"minors\" + 0.187*\"survey\" + 0.061*\"system\" + 0.060*\"response\" + 0.060*\"time\" + 0.058*\"user\" + 0.049*\"computer\" + 0.035*\"interface\"',\n",
        " '-0.460*\"system\" + -0.373*\"user\" + -0.332*\"eps\" + -0.328*\"interface\" + -0.320*\"time\" + -0.320*\"response\" + -0.293*\"computer\" + -0.280*\"human\" + -0.171*\"survey\" + 0.161*\"trees\"']"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for doc in corpus_lsi: # both bow->tfidf and tfidf->lsi transformations are actually executed here, on the fly\n",
      "    print doc"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(0, 0.066007833960902221), (1, -0.52007033063618546)]\n",
        "[(0, 0.1966759285914238), (1, -0.76095631677000497)]\n",
        "[(0, 0.089926399724462952), (1, -0.7241860626752511)]\n",
        "[(0, 0.075858476521780432), (1, -0.63205515860034311)]\n",
        "[(0, 0.10150299184980073), (1, -0.57373084830029542)]\n",
        "[(0, 0.70321089393783154), (1, 0.16115180214025637)]\n",
        "[(0, 0.87747876731198349), (1, 0.16758906864659232)]\n",
        "[(0, 0.90986246868185794), (1, 0.14086553628718845)]\n",
        "[(0, 0.61658253505692784), (1, -0.053929075663894654)]\n"
       ]
      }
     ],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lsi.save('/tmp/model.lsi') # same for tfidf, lda, ...\n",
      "lsi = models.LsiModel.load('/tmp/model.lsi')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 26
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Available Transformations"
     ]
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "TF-IDF"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = tfidfmodel.TfidfModel(bow_corpus, normalize=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 28
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "LSI"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = lsimodel.LsiModel(tfidf_corpus, id2word=dictionary, num_topics=300)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model.add_documents(another_tfidf_corpus) # now LSI has been trained on tfidf_corpus + another_tfidf_corpus\n",
      "lsi_vec = model[tfidf_vec] # convert some new document into the LSI space, without affecting the model\n",
      ">>> ...\n",
      "model.add_documents(more_documents) # tfidf_corpus + another_tfidf_corpus + more_documents\n",
      "lsi_vec = model[tfidf_vec]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Random Projections"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = rpmodel.RpModel(tfidf_corpus, num_topics=500)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "LDA"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = ldamodel.LdaModel(bow_corpus, id2word=dictionary, num_topics=100)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "HDP"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "model = hdpmodel.HdpModel(bow_corpus, id2word=dictionary)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Similarity Tutorial"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dictionary = corpora.Dictionary.load('/tmp/deerwester.dict')\n",
      "corpus = corpora.MmCorpus('/tmp/deerwester.mm') # comes from the first tutorial, \"From strings to vectors\"\n",
      "print corpus"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "MmCorpus(9 documents, 12 features, 28 non-zero entries)\n"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "lsi = models.LsiModel(corpus, id2word=dictionary, num_topics=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 30
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "doc = \"Human computer interaction\"\n",
      "vec_bow = dictionary.doc2bow(doc.lower().split())\n",
      "vec_lsi = lsi[vec_bow] # convert the query to LSI space\n",
      "print vec_lsi"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(0, -0.46182100453271474), (1, -0.070027665279000048)]\n"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Initializing Query structures\n",
      "index = similarities.MatrixSimilarity(lsi[corpus]) # transform corpus to LSI space and index it"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "index.save('/tmp/deerwester.index')\n",
      "index = similarities.MatrixSimilarity.load('/tmp/deerwester.index')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sims = index[vec_lsi] # perform a similarity query against the corpus\n",
      "print list(enumerate(sims)) # print (document_number, document_similarity) 2-tuples"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(0, 0.99809301), (1, 0.93748635), (2, 0.99844527), (3, 0.9865886), (4, 0.90755945), (5, -0.12416792), (6, -0.10639259), (7, -0.098794639), (8, 0.050041765)]\n"
       ]
      }
     ],
     "prompt_number": 34
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Wikipedia Analysis"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#LSA\n",
      "import logging, gensim, bz2\n",
      "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
      "\n",
      "# load id->word mapping (the dictionary), one of the results of step 2 above\n",
      "id2word = gensim.corpora.Dictionary.load_from_text('wiki_en_wordids.txt')\n",
      "# load corpus iterator\n",
      "mm = gensim.corpora.MmCorpus('wiki_en_tfidf.mm')\n",
      "# mm = gensim.corpora.MmCorpus(bz2.BZ2File('wiki_en_tfidf.mm.bz2')) # use this if you compressed the TFIDF output (recommended)\n",
      "\n",
      "print mm"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# extract 400 LSI topics; use the default one-pass algorithm\n",
      "lsi = gensim.models.lsimodel.LsiModel(corpus=mm, id2word=id2word, num_topics=400)\n",
      "\n",
      "# print the most contributing words (both positively and negatively) for each of the first ten topics\n",
      "lsi.print_topics(10)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# LDA\n",
      "import logging, gensim, bz2\n",
      "logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)\n",
      "\n",
      "# load corpus iterator\n",
      "mm = gensim.corpora.MmCorpus('wiki_en_tfidf.mm')\n",
      "# mm = gensim.corpora.MmCorpus(bz2.BZ2File('wiki_en_tfidf.mm.bz2')) # use this if you compressed the TFIDF output\n",
      "\n",
      "print mm"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# extract 100 LDA topics, using 1 pass and updating once every 1 chunk (10,000 documents)\n",
      "lda = gensim.models.ldamodel.LdaModel(corpus=mm, id2word=id2word, num_topics=100, update_every=1, chunksize=10000, passes=1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# print the most contributing words for 20 randomly selected topics\n",
      "lda.print_topics(20)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# extract 100 LDA topics, using 20 full passes, no online updates\n",
      "lda = gensim.models.ldamodel.LdaModel(corpus=mm, id2word=id2word, num_topics=100, update_every=0, passes=20)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "doc_lda = lda[doc_bow]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}