Tweet Anatomy for UCLA Corpus.ipynb

{
 "metadata": {
  "name": "Tweet Anatomy for UCLA Corpus"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Tweet analys with focus on RT handling, also other @mentions"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sys\n",
      "import cPickle as pickle\n",
      "import pprint\n",
      "import collections\n",
      "#import nltk\n",
      "#from nltk.stem import WordNetLemmatizer\n",
      "sys.path.append('/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/')\n",
      "sys.path.append('/Users/doug/SW_Dev/NLTK_Experiments/')\n",
      "\n",
      "from sklearn.feature_extraction import DictVectorizer\n",
      "from sklearn.feature_extraction.text import TfidfTransformer\n",
      "from sklearn.cluster import KMeans, MiniBatchKMeans\n",
      "from sklearn import metrics\n",
      "from sklearn.metrics import pairwise_distances\n",
      "import math\n",
      "\n",
      "#import cmu_tweet_word_clusters\n",
      "import my_feature_ex as fx\n",
      "import pprint as pprint"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "skipped:  111010100010\t\t212\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Analysis of Retweets\n",
      "\n",
      "Returns:\n",
      "    is_retweet\n",
      "    is_retweet_thanks\n",
      "    author\n",
      "    comment_pos\n",
      "    comment_tokens\n",
      "    orig_tokens\n",
      "    orig_pos"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# covers use cases:\n",
      "# <comment> [RT|RT:] @<author> [|:]<orig_tweet>\n",
      "unexpected_pos = {}\n",
      "def parse_retweet(tweet):\n",
      "    #print tweet['text']\n",
      "    \"\"\"returns False if not retweet\"\"\"\n",
      "    pos = [p for t,p,c in tweet['pos']]\n",
      "    tokens = tweet['tokens']\n",
      "    length = len(tweet['pos'])\n",
      "    #print pos\n",
      "    #print tokens\n",
      "    \n",
      "    for i, p in enumerate(pos):\n",
      "        result = {}\n",
      "        t = tokens[i]\n",
      "        if not t.lower() in ['rt', 'rtrt']:\n",
      "            continue\n",
      "\n",
      "        result['comment_tokens'] = tokens[0:i]\n",
      "        result['comment_pos'] = tweet['pos'][0:i]\n",
      "        ptr = i + 1\n",
      "        if ptr >= length:\n",
      "            continue\n",
      "      \n",
      "        if pos[ptr] == ',':  #skip punctuation, if present\n",
      "           ptr = ptr + 1\n",
      "           if ptr >= length:\n",
      "               continue \n",
      "        #print tweet['text']                  \n",
      "        if pos[ptr] != '@':  #identify source\n",
      "            continue\n",
      "        result['author'] = tokens[ptr]    \n",
      "        ptr = ptr + 1\n",
      "        if ptr >= length:\n",
      "           continue \n",
      "        #print tweet['text']                  \n",
      "        if tokens[ptr] == ':':  #skip punctuation\n",
      "           ptr = ptr + 1\n",
      "           if ptr >= length:\n",
      "               continue \n",
      "                \n",
      "        result['orig_tokens'] = tokens[ptr:]\n",
      "        result['orig_pos'] = tweet['pos'][ptr:]\n",
      "        result['is_retweet'] = True\n",
      "        \n",
      "        if 'thank' in tokens[0].lower():\n",
      "            result['is_retweet_thanks'] = True\n",
      "        else:\n",
      "            result['is_retweet_thanks'] = False\n",
      "            \n",
      "        return result\n",
      "\n",
      "def annotate_retweet(tweets):       \n",
      "    for i, tweet in enumerate(UCLA_tweets.values()): \n",
      "        result = parse_retweet(tweet)\n",
      "        if result:\n",
      "            tweet.update(result)\n",
      "        else:\n",
      "            tweet['is_retweet'] = False"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Various other parts of speech identified as '~' by CMU tweet parser.  Note:  text converted to lower() before analysis\n",
      "\n",
      "'rt', 'rtrt', 'cont', '+1', ':', '||', '//', \"\",'///', '<', '<<', '<<<', '<-', '<--',\n",
      "'<~', '<~~', '<=', '>', '>>', '>>>', '->','\u2026','...', ',...', '....','---', '\u2014']"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#CMU Parse discourse parts of speech\n",
      "known_discourse = ['rt', 'rtrt', 'cont', '+1', ':', '||', '//', \"\",\n",
      "                   '///', '<', '<<', '<<<', '<-', '<--',\n",
      "                   '<~', '<~~', '<=', '>', '>>', '>>>', '->',\n",
      "                   '\u2026','...', ',...', '....','---', '\u2014']\n",
      "\n",
      "def is_discourse(p, t):\n",
      "    return (p == '~' and t.lower() in known_discourse)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# load previously tokenized/clasified  tweet Corpus\n",
      "pinput = open('ucla_tweets.pkl', 'rb')\n",
      "UCLA_tweets = pickle.load(pinput)\n",
      "pinput.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#select subset of tweets for experimentation\n",
      "\n",
      "MAX_TWEETS = 100000   #subset Corpus for now to improve Clustering run time\n",
      "tweet_set = [{'text':t['text'], 'pos':t['pos'], 'raw_tokens':t['tokens']} for t in UCLA_tweets.values()[0:MAX_TWEETS]]\n",
      "\n",
      "#sample tweet\n",
      "#pprint.pprint(tweet_set[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "annotate_retweet(UCLA_tweets.values())\n",
      "for tweet in UCLA_tweets.values(): \n",
      "    if tweet['is_retweet'] and tweet['is_retweet_thanks']:\n",
      "        print tweet['text']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Thanks for spreading the word. Appreciate the help. RT @USMale7 Message Board Forum for @Bruin247: http://t.co/O71eHS1DVP via @247Sports\n"
       ]
      }
     ],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "rt = {'cnt':0, 'example':False}\n",
      "fav = {'cnt':0, 'example':False}\n",
      "irt = {'cnt':0, 'example':False}\n",
      "irsid = {'cnt':0, 'example':False}\n",
      "irss = {'cnt':0, 'example':False}\n",
      "irsuid = {'cnt':0, 'example':False}\n",
      "irsus = {'cnt':0, 'example':False}\n",
      "irsn = {'cnt':0, 'example':False}\n",
      "place = {'cnt':0, 'example':False}\n",
      "men = {'cnt':0, 'example':False}\n",
      "combined = {'cnt':0, 'example':False}\n",
      "combined1 = {'cnt':0, 'example':False}\n",
      "combined2 = {'cnt':0, 'example':False}\n",
      "combined3 = {'cnt':0, 'example':False}\n",
      "combined4 = {'cnt':0, 'example':False}\n",
      "\n",
      "for i, t in enumerate(UCLA_tweets.values()):\n",
      "    if t['favorited']:\n",
      "        fav['cnt'] += 1\n",
      "        fav['example'] = i\n",
      "        \n",
      "    if t['retweeted']:\n",
      "        rt['cnt'] += 1\n",
      "        ft['example'] = i\n",
      "            \n",
      "    if t[\"in_reply_to_status_id\"]:\n",
      "        irsid['cnt'] += 1\n",
      "        irsid['example'] = i\n",
      "        \n",
      "    if t[\"in_reply_to_status_id_str\"]:\n",
      "        irss['cnt'] += 1\n",
      "        irss['example'] = i\n",
      "        \n",
      "    if t[\"in_reply_to_user_id\"]:\n",
      "        irsuid['cnt'] += 1\n",
      "        irsuid['example'] = i\n",
      "        \n",
      "    if t[\"in_reply_to_user_id_str\"]: \n",
      "        irsus['cnt'] += 1\n",
      "        irsus['example'] = i\n",
      "        \n",
      "    if t[\"in_reply_to_screen_name\"]:\n",
      "        irsn['cnt'] += 1\n",
      "        irsn['example'] = i\n",
      "        \n",
      "    if not t[\"in_reply_to_status_id\"] and t[\"in_reply_to_screen_name\"]:\n",
      "        combined['cnt'] += 1\n",
      "        combined['example'] = i\n",
      "        \n",
      "    if t[\"in_reply_to_status_id\"] and not t[\"in_reply_to_screen_name\"]:\n",
      "        combined1['cnt'] += 1\n",
      "        combined1['example'] = i\n",
      "        \n",
      "    if t['place']:\n",
      "        place['cnt'] += 1\n",
      "        place['example'] = i\n",
      "        \n",
      "    if t['entities'][\"user_mentions\"]:\n",
      "        men['cnt'] += 1\n",
      "        men['example'] = i\n",
      "        \n",
      "    if t['entities'][\"user_mentions\"] and t[\"in_reply_to_screen_name\"]:\n",
      "        combined2['cnt'] += 1\n",
      "        combined2['example'] = i\n",
      "        \n",
      "    if t['entities'][\"user_mentions\"] and not t[\"in_reply_to_screen_name\"]:\n",
      "        combined3['cnt'] += 1\n",
      "        combined3['example'] = i\n",
      "        \n",
      "    if not t['entities'][\"user_mentions\"] and t[\"in_reply_to_screen_name\"]:\n",
      "        combined4['cnt'] += 1\n",
      "        combined4['example'] = i\n",
      "        \n",
      "print \"rt: \", rt['cnt']\n",
      "print \"fav: \", fav['cnt']\n",
      "print \"irsid: \", irsid['cnt']\n",
      "print \"irss: \", irss['cnt']\n",
      "print \"irsuid: \", irsuid['cnt']\n",
      "print \"irsus: \", irsus['cnt']\n",
      "print \"irsn: \", irsn['cnt']\n",
      "print \"combined: \", combined['cnt']\n",
      "print \"combined: \", combined1['cnt']\n",
      "print \"place: \", place['cnt']\n",
      "print \"mentions: \", men['cnt']\n",
      "print \"combined2: \", combined2['cnt']\n",
      "print \"combined3: \", combined3['cnt']\n",
      "print \"combined4: \", combined4['cnt']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "rt:  0\n",
        "fav:  0\n",
        "irsid:  3357\n",
        "irss:  3357\n",
        "irsuid:  4146\n",
        "irsus:  4146\n",
        "irsn:  4146\n",
        "combined:  789\n",
        "combined:  0\n",
        "place:  781\n",
        "mentions:  7101\n",
        "combined2:  4141\n",
        "combined3:  2960\n",
        "combined4:  5\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Two categories of tweets based on in_reply:\n",
      "    'in_reply_to_status_*'\n",
      "        - \"This method has not been officially added into the documentation ... don't recommend using\n",
      "           it in a production application ... \"\n",
      "        - \"about 'related_results' to a tweet -- not necessarily replies\"\n",
      "        - BASED ON MY ANALYSIS, THIS IS A PROPER SUBSET OF in_reply_to_user_*\n",
      "    'in_reply_to_user_*'\n",
      "    \n",
      "Other useful fields:\n",
      "    'place'\n",
      "    'user_mentions'\n",
      "        - Mentions is almost always a proper superset of in_reply_to_user\n",
      "            except when user forgets @name in reply\n",
      "        - 1/3 of Mentions don't include in_reply_to_user"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pprint.pprint(UCLA_tweets.values()[irsid['example']]['text'])\n",
      "pprint.pprint(UCLA_tweets.values()[irsid['example']]['in_reply_to_status_id'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u'@carsonhawk81 @theherd631 Maybe you can teach me English. Because, you see, we only speak Spanish at UCLA.'\n",
        "3.420633516538839e+17\n"
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pprint.pprint(UCLA_tweets.values()[irsuid['example']]['text'])\n",
      "pprint.pprint(UCLA_tweets.values()[irsuid['example']]['in_reply_to_user_id'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u'@carsonhawk81 @theherd631 Maybe you can teach me English. Because, you see, we only speak Spanish at UCLA.'\n",
        "1002878462\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pprint.pprint(UCLA_tweets.values()[combined['example']]['text'])\n",
      "pprint.pprint(UCLA_tweets.values()[combined['example']][\"in_reply_to_status_id\"])\n",
      "pprint.pprint(UCLA_tweets.values()[combined['example']]['in_reply_to_screen_name'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u'@berkeleyside A message from David N. Myers, chair of UCLA History Dept, about cutting Office of the President http://t.co/3ae2jIx9ug'\n",
        "None\n",
        "u'berkeleyside'\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pprint.pprint(UCLA_tweets.values()[combined1['example']]['text'])\n",
      "pprint.pprint(UCLA_tweets.values()[combined1['example']][\"in_reply_to_status_id\"])\n",
      "pprint.pprint(UCLA_tweets.values()[combined1['example']]['in_reply_to_screen_name'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u'Should I worry, or take comfort? @UCLAnewsroom10m Eating probiotics affects brain function, UCLA study shows. http://t.co/eskRkWb4Bl'\n",
        "None\n",
        "None\n"
       ]
      }
     ],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pprint.pprint(UCLA_tweets.values()[place['example']]['text'])\n",
      "pprint.pprint(UCLA_tweets.values()[place['example']]['place'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u'My Dream College Is UCLA'\n",
        "{u'attributes': {},\n",
        " u'bounding_box': {u'coordinates': [[[-83.694237, 41.580266],\n",
        "                                     [-83.694237, 41.732844],\n",
        "                                     [-83.454229, 41.732844],\n",
        "                                     [-83.454229, 41.580266]]],\n",
        "                   u'type': u'Polygon'},\n",
        " u'country': u'United States',\n",
        " u'country_code': u'US',\n",
        " u'full_name': u'Toledo, OH',\n",
        " u'id': u'7068dd9474ab6973',\n",
        " u'name': u'Toledo',\n",
        " u'place_type': u'city',\n",
        " u'url': u'http://api.twitter.com/1/geo/id/7068dd9474ab6973.json'}\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pprint.pprint(UCLA_tweets.values()[men['example']]['text'])\n",
      "pprint.pprint(UCLA_tweets.values()[men['example']]['entities'][\"user_mentions\"])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u'@carsonhawk81 @theherd631 Maybe you can teach me English. Because, you see, we only speak Spanish at UCLA.'\n",
        "[{u'id': 1002878462,\n",
        "  u'id_str': u'1002878462',\n",
        "  u'indices': [0, 13],\n",
        "  u'name': u'Jeff Carson',\n",
        "  u'screen_name': u'carsonhawk81'},\n",
        " {u'id': 25707550,\n",
        "  u'id_str': u'25707550',\n",
        "  u'indices': [14, 25],\n",
        "  u'name': u'chris hansen',\n",
        "  u'screen_name': u'theherd631'}]\n"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#both in reply to user and mention\n",
      "pprint.pprint(UCLA_tweets.values()[combined2['example']]['text'])\n",
      "pprint.pprint(UCLA_tweets.values()[combined2['example']]['in_reply_to_screen_name'])\n",
      "pprint.pprint(UCLA_tweets.values()[combined2['example']]['entities'][\"user_mentions\"])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u'@carsonhawk81 @theherd631 Maybe you can teach me English. Because, you see, we only speak Spanish at UCLA.'\n",
        "u'carsonhawk81'\n",
        "[{u'id': 1002878462,\n",
        "  u'id_str': u'1002878462',\n",
        "  u'indices': [0, 13],\n",
        "  u'name': u'Jeff Carson',\n",
        "  u'screen_name': u'carsonhawk81'},\n",
        " {u'id': 25707550,\n",
        "  u'id_str': u'25707550',\n",
        "  u'indices': [14, 25],\n",
        "  u'name': u'chris hansen',\n",
        "  u'screen_name': u'theherd631'}]\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# mention and no in reply to user\n",
      "pprint.pprint(UCLA_tweets.values()[combined3['example']]['text'])\n",
      "pprint.pprint(UCLA_tweets.values()[combined2['example']]['entities'][\"user_mentions\"])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u'Prob not sthg to watch over snacks... @AudioVision UCLA Broadcasts Brain Surgery Live to Twitter, Vine, &amp; Instagram http://t.co/zDRmgv0RHn'\n",
        "[{u'id': 1002878462,\n",
        "  u'id_str': u'1002878462',\n",
        "  u'indices': [0, 13],\n",
        "  u'name': u'Jeff Carson',\n",
        "  u'screen_name': u'carsonhawk81'},\n",
        " {u'id': 25707550,\n",
        "  u'id_str': u'25707550',\n",
        "  u'indices': [14, 25],\n",
        "  u'name': u'chris hansen',\n",
        "  u'screen_name': u'theherd631'}]\n"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#sample in_reply_to_user and no mention\n",
      "pprint.pprint(UCLA_tweets.values()[combined4['example']]['text'])\n",
      "pprint.pprint(UCLA_tweets.values()[combined4['example']]['in_reply_to_screen_name'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u'Important part of that UCLA Health Patient Passport: \"Attention Triage Nurse\" - pre-approved orders, literally or figuratively. #pfcc2013'\n",
        "u'ePatientDave'\n"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#basic Retweet\n",
      "rt = {'cnt':0, 'example':False}\n",
      "for i, t in enumerate(UCLA_tweets.values()):\n",
      "    if 'RT' in t['text']:\n",
      "        rt['cnt'] += 1\n",
      "        rt['example'] = i\n",
      "        \n",
      "print \"rt: \", rt['cnt']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "rt:  879\n"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pprint.pprint(UCLA_tweets.values()[rt['example']]['text'])\n",
      "pprint.pprint(UCLA_tweets.values()[rt['example']]['pos'])\n",
      "pprint.pprint(UCLA_tweets.values()[rt['example']]['tokens'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u\"#UCLA UCLAAthletics: RT @UclaVarsityClub: 2day is the 3 year anniversary of Coach Wooden's... http://t.co/Hgv5SPCOI8 #SportsRoadhouse\"\n",
        "[('#UCLA', '#', 0.8555),\n",
        " ('UCLAAthletics', '^', 0.5797),\n",
        " (':', ',', 0.4923),\n",
        " ('RT', '~', 0.976),\n",
        " ('@UclaVarsityClub', '@', 0.9989),\n",
        " (':', '~', 0.9358),\n",
        " ('2day', 'N', 0.9369),\n",
        " ('is', 'V', 0.9985),\n",
        " ('the', 'D', 0.9981),\n",
        " ('3', '$', 0.9865),\n",
        " ('year', 'N', 0.9095),\n",
        " ('anniversary', 'N', 0.9979),\n",
        " ('of', 'P', 0.9971),\n",
        " ('Coach', 'N', 0.6432),\n",
        " (\"Wooden's\", 'Z', 0.3925),\n",
        " ('...', ',', 0.7104),\n",
        " ('http://t.co/Hgv5SPCOI8', 'U', 0.9983),\n",
        " ('#SportsRoadhouse', '#', 0.9729)]\n",
        "['#UCLA',\n",
        " 'UCLAAthletics',\n",
        " ':',\n",
        " 'RT',\n",
        " '@UclaVarsityClub',\n",
        " ':',\n",
        " '2day',\n",
        " 'is',\n",
        " 'the',\n",
        " '3',\n",
        " 'year',\n",
        " 'anniversary',\n",
        " 'of',\n",
        " 'Coach',\n",
        " \"Wooden's\",\n",
        " '...',\n",
        " 'http://t.co/Hgv5SPCOI8',\n",
        " '#SportsRoadhouse']\n"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pprint.pprint(UCLA_tweets.values()[rt['example']]['text'])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u\"#UCLA UCLAAthletics: RT @UclaVarsityClub: 2day is the 3 year anniversary of Coach Wooden's... http://t.co/Hgv5SPCOI8 #SportsRoadhouse\"\n"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#sample Tweet\n",
      "pprint.pprint(UCLA_tweets.values()[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{u'contributors': None,\n",
        " u'coordinates': None,\n",
        " u'created_at': u'Tue May 28 21:52:51 +0000 2013',\n",
        " u'entities': {u'hashtags': [],\n",
        "               u'symbols': [],\n",
        "               u'urls': [{u'display_url': u'ucla.in/13gzsdQ',\n",
        "                          u'expanded_url': u'http://ucla.in/13gzsdQ',\n",
        "                          u'indices': [110, 132],\n",
        "                          u'url': u'http://t.co/eskRkWb4Bl'}],\n",
        "               u'user_mentions': []},\n",
        " u'favorite_count': 0,\n",
        " u'favorited': False,\n",
        " u'filter_level': u'low',\n",
        " u'geo': None,\n",
        " u'id': 3.394995339753062e+17,\n",
        " u'id_str': u'339499533975306240',\n",
        " u'in_reply_to_screen_name': None,\n",
        " u'in_reply_to_status_id': None,\n",
        " u'in_reply_to_status_id_str': None,\n",
        " u'in_reply_to_user_id': None,\n",
        " u'in_reply_to_user_id_str': None,\n",
        " 'is_retweet': False,\n",
        " u'lang': u'en',\n",
        " u'place': None,\n",
        " 'pos': [('Should', 'V', 0.9952),\n",
        "         ('I', 'O', 0.9994),\n",
        "         ('worry', 'V', 0.9988),\n",
        "         (',', ',', 0.9982),\n",
        "         ('or', '&', 0.9936),\n",
        "         ('take', 'V', 0.9366),\n",
        "         ('comfort', 'N', 0.9568),\n",
        "         ('?', ',', 0.9972),\n",
        "         ('@UCLAnewsroom10m', '@', 0.9761),\n",
        "         ('Eating', 'V', 0.9401),\n",
        "         ('probiotics', 'N', 0.9535),\n",
        "         ('affects', 'V', 0.9974),\n",
        "         ('brain', 'N', 0.9788),\n",
        "         ('function', 'N', 0.9814),\n",
        "         (',', ',', 0.9967),\n",
        "         ('UCLA', '^', 0.9757),\n",
        "         ('study', 'N', 0.8871),\n",
        "         ('shows', 'V', 0.7623),\n",
        "         ('.', ',', 0.998),\n",
        "         ('http://t.co/eskRkWb4Bl', 'U', 0.9984)],\n",
        " u'possibly_sensitive': False,\n",
        " u'retweet_count': 0,\n",
        " u'retweeted': False,\n",
        " u'source': u'web',\n",
        " u'text': u'Should I worry, or take comfort? @UCLAnewsroom10m Eating probiotics affects brain function, UCLA study shows. http://t.co/eskRkWb4Bl',\n",
        " 'tokens': ['Should',\n",
        "            'I',\n",
        "            'worry',\n",
        "            ',',\n",
        "            'or',\n",
        "            'take',\n",
        "            'comfort',\n",
        "            '?',\n",
        "            '@UCLAnewsroom10m',\n",
        "            'Eating',\n",
        "            'probiotics',\n",
        "            'affects',\n",
        "            'brain',\n",
        "            'function',\n",
        "            ',',\n",
        "            'UCLA',\n",
        "            'study',\n",
        "            'shows',\n",
        "            '.',\n",
        "            'http://t.co/eskRkWb4Bl'],\n",
        " u'truncated': False,\n",
        " u'user': {u'contributors_enabled': False,\n",
        "           u'created_at': u'Fri Jan 09 03:35:17 +0000 2009',\n",
        "           u'default_profile': False,\n",
        "           u'default_profile_image': False,\n",
        "           u'description': u'Ascencia Executive Director, Unitarian & Rotarian; entirely a work in progress.',\n",
        "           u'favourites_count': 47,\n",
        "           u'follow_request_sent': None,\n",
        "           u'followers_count': 1242,\n",
        "           u'following': None,\n",
        "           u'friends_count': 2000,\n",
        "           u'geo_enabled': True,\n",
        "           u'id': 18790772,\n",
        "           u'id_str': u'18790772',\n",
        "           u'is_translator': False,\n",
        "           u'lang': u'en',\n",
        "           u'listed_count': 58,\n",
        "           u'location': u'\\xdcT: 34.12864,-118.261412',\n",
        "           u'name': u'NatalieProfantKomuro',\n",
        "           u'notifications': None,\n",
        "           u'profile_background_color': u'EDEED2',\n",
        "           u'profile_background_image_url': u'http://a0.twimg.com/profile_background_images/7873751/anacapablog.jpg',\n",
        "           u'profile_background_image_url_https': u'https://si0.twimg.com/profile_background_images/7873751/anacapablog.jpg',\n",
        "           u'profile_background_tile': True,\n",
        "           u'profile_banner_url': u'https://pbs.twimg.com/profile_banners/18790772/1359531216',\n",
        "           u'profile_image_url': u'http://a0.twimg.com/profile_images/1180113084/b41d4047-25bc-43da-818a-bb2db63aeb1f_normal.png',\n",
        "           u'profile_image_url_https': u'https://si0.twimg.com/profile_images/1180113084/b41d4047-25bc-43da-818a-bb2db63aeb1f_normal.png',\n",
        "           u'profile_link_color': u'3B0EAF',\n",
        "           u'profile_sidebar_border_color': u'C4D1BD',\n",
        "           u'profile_sidebar_fill_color': u'91BDF7',\n",
        "           u'profile_text_color': u'333333',\n",
        "           u'profile_use_background_image': True,\n",
        "           u'protected': False,\n",
        "           u'screen_name': u'Anacapa',\n",
        "           u'statuses_count': 8240,\n",
        "           u'time_zone': u'Alaska',\n",
        "           u'url': u'http://www.ascenciaCA.org',\n",
        "           u'utc_offset': -32400,\n",
        "           u'verified': False}}\n"
       ]
      }
     ],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 20
    }
   ],
   "metadata": {}
  }
 ]
}