Parse Tweets using CMU-ROB.ipynb

{
 "metadata": {
  "name": "Parse Tweets using CMU-ROB"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Example of use of CMU Tweet Parser.\n",
      "\n",
      "Processing of UCLA Tweets, including word frequency distribution"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%pylab inline\n",
      "plt.rc('figure', figsize=(8, 5))\n",
      "import nltk\n",
      "import pprint\n",
      "import json"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Welcome to pylab, a matplotlib-based Python environment [backend: module://IPython.zmq.pylab.backend_inline].\n",
        "For more information, type 'help(pylab)'.\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Use CMU's Tweet Tagger. Python wrapper calls Java tagger\n",
      "    Tagger: http://www.ark.cs.cmu.edu/TweetNLP/\n",
      "    Wrapper: https://github.com/ianozsvald/ark-tweet-nlp-python"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sys.path.append('/Users/doug/SW_Dev/ark-tweet-nlp-0.3.2')\n",
      "import CMUTweetTagger\n",
      "print CMUTweetTagger.runtagger_parse(['example tweet 1', 'example tweet 2'])\n",
      "\n",
      "RUN_TAGGER_CMD = \"java -XX:ParallelGCThreads=2 -Xmx500m -jar /Users/doug/SW_Dev/ark-tweet-nlp-0.3.2/ark-tweet-nlp-0.3.2.jar\"\n",
      "\n",
      "def annotate_pos(tweets):\n",
      "    ids = []\n",
      "    texts = []\n",
      "    for key, value in tweets.items():\n",
      "        ids.append(key)\n",
      "        texts.append(json.dumps({'text':value['text']}))\n",
      "    pos = CMUTweetTagger.runtagger_parse(texts, run_tagger_cmd=RUN_TAGGER_CMD)\n",
      "    \n",
      "    if len(ids) != len(pos):\n",
      "        raise Exception(\"Error: Tweet Tagger returned incorrect results\") \n",
      "\n",
      "    for i in range(0, len(ids)):\n",
      "        tweets[ids[i]]['pos'] = pos[i]\n",
      "        tweets[ids[i]]['tokens'] = [tag[0] for tag in pos[i]]\n",
      "\n",
      "    print pos[0]\n",
      "    print [tag[0] for tag in pos[0]]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[[('example', 'N', 0.979), ('tweet', 'V', 0.7763), ('1', '$', 0.9916)], [('example', 'N', 0.979), ('tweet', 'V', 0.7713), ('2', '$', 0.5832)]]\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def load_tweet_corpus(fname):\n",
      "    tweets = dict()\n",
      "    fd = open(fname)\n",
      "    for json_text in fd:\n",
      "        tweet = json.loads(json_text)['tweet']\n",
      "        tweets[tweet['id_str']] = tweet\n",
      "    print 'loaded'\n",
      "    return tweets\n",
      "\n",
      "def load_UCLA_tweet_corpus():\n",
      "    fname = '/Users/doug/Desktop/sentiment/UCLA.json'\n",
      "    return load_tweet_corpus(fname)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Read in Tweet Corpus and parse"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "UCLA_tweets = load_UCLA_tweet_corpus() \n",
      "\n",
      "annotate_pos(UCLA_tweets)\n",
      "#subset_tweets = [t['text'] for t in UCLA_tweets.values()[:10]]\n",
      "#pprint.pprint(CMUTweetTagger.runtagger_parse(subset_tweets))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "loaded\n",
        "[('Should', 'V', 0.9952), ('I', 'O', 0.9994), ('worry', 'V', 0.9988), (',', ',', 0.9982), ('or', '&', 0.9936), ('take', 'V', 0.9366), ('comfort', 'N', 0.9568), ('?', ',', 0.9972), ('@UCLAnewsroom10m', '@', 0.9761), ('Eating', 'V', 0.9401), ('probiotics', 'N', 0.9535), ('affects', 'V', 0.9974), ('brain', 'N', 0.9788), ('function', 'N', 0.9814), (',', ',', 0.9967), ('UCLA', '^', 0.9757), ('study', 'N', 0.8871), ('shows', 'V', 0.7623), ('.', ',', 0.998), ('http://t.co/eskRkWb4Bl', 'U', 0.9984)]"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "['Should', 'I', 'worry', ',', 'or', 'take', 'comfort', '?', '@UCLAnewsroom10m', 'Eating', 'probiotics', 'affects', 'brain', 'function', ',', 'UCLA', 'study', 'shows', '.', 'http://t.co/eskRkWb4Bl']\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import cPickle as pickle\n",
      "\n",
      "poutput = open('ucla_tweets.pkl', 'wb')\n",
      "pickle.dump(UCLA_tweets, poutput, -1)\n",
      "poutput.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    }
   ],
   "metadata": {}
  }
 ]
}