Pattern NLP.ipynb

{
 "metadata": {
  "name": "Pattern NLP"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Hacked version, returns full raw status structure as result.raw.   Currently implemented for for twitter search and streams.  Note,  raw data requires parsing by ast.literal_eval()"
     ]
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Documentation: http://www.clips.ua.ac.be/pages/pattern\n",
      "\n",
      "see also MBSP: http://www.clips.ua.ac.be/pages/MBSP"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sys\n",
      "import cPickle as pickle\n",
      "import pprint\n",
      "import collections\n",
      "import nltk\n",
      "#from nltk.stem import WordNetLemmatizer\n",
      "sys.path.append('/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/')\n",
      "sys.path.append('/Users/doug/SW_Dev/NLTK_Experiments/')\n",
      "\n",
      "import math\n",
      "import numpy as np\n",
      "import time\n",
      "import ast   #used to import json-like structures directly\n",
      "\n",
      "import pattern\n",
      "from pattern.web import Twitter, plaintext"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%pylab inline\n",
      "plt.rc('figure', figsize=(8, 8))\n",
      "\n",
      "import pprint\n",
      "import collections\n",
      "import operator\n",
      "\n",
      "#import my_feature_ex as fx\n",
      "#import my_word_cloud as wcloud\n",
      "\n",
      "import json\n",
      "#sys.path.append('/Users/doug/SW_Dev/ark-tweet-nlp-0.3.2')\n",
      "#import CMUTweetTagger\n",
      "\n",
      "from pattern.en import parse, pprint\n",
      "import pattern.en\n",
      "from pattern.en import parsetree"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "Welcome to pylab, a matplotlib-based Python environment [backend: module://IPython.zmq.pylab.backend_inline].\n",
        "For more information, type 'help(pylab)'.\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Replace Twitter license keys"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import my_twitter_license as my_license    #my twitter oauth tokens stored in user-specific file\n",
      "\n",
      "# save default license and substitute user-specific license\n",
      "old_twitter_license = pattern.web.TWITTER_LICENSE\n",
      "pattern.web.TWITTER_LICENSE = my_license.get_license()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#help(pattern.web)\n",
      "#help(Twitter)\n",
      "help(Twitter.__init__)\n",
      "help(Twitter.search)\n",
      "help(Twitter.stream)\n",
      "help(Twitter.trends)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Help on method __init__ in module pattern.web:\n",
        "\n",
        "__init__(self, license=None, throttle=0.5, language=None) unbound pattern.web.Twitter method\n",
        "\n",
        "Help on method search in module pattern.web:\n",
        "\n",
        "search(self, query, type='search', start=1, count=10, sort='relevancy', size=None, cached=False, **kwargs) unbound pattern.web.Twitter method\n",
        "    Returns a list of results from Twitter for the given query.\n",
        "    - type : SEARCH,\n",
        "    - start: Result.id or int,\n",
        "    - count: maximum 100.\n",
        "    There is a limit of 150+ queries per 15 minutes.\n",
        "\n",
        "Help on method stream in module pattern.web:\n",
        "\n",
        "stream(self, query, **kwargs) unbound pattern.web.Twitter method\n",
        "    Returns a live stream of Result objects for the given query.\n",
        "\n",
        "Help on method trends in module pattern.web:\n",
        "\n",
        "trends(self, **kwargs) unbound pattern.web.Twitter method\n",
        "    Returns a list with 10 trending topics on Twitter.\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sample = False\n",
      "for tweet in Twitter().search('\"more important than\"', cached=False, count=20, language='en'):\n",
      "    print plaintext(tweet.text)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "RT @Epic_Women: The expression a woman wears on her face is far more important than the clothes she wears on her back -Dale Carnegie\n",
        "RT @nolimit_nolan29: Schools more important than having stupid ass drama in my life anyways.\n",
        "IRS offers FREE resources to assist with staffing for your many insurance jobs. Find out why COST doesn\u2019t matter. http://t.co/0Ixz0OjI4k\n",
        "Curriculum Is More Important Than Learning Technology http://t.co/VjX7vzfIE8 via @TeachThought\n",
        "RT @Epic_Women: The expression a woman wears on her face is far more important than the clothes she wears on her back -Dale Carnegie\n",
        "RT @HeaIthyTips: What you think of yourself is much more important than what people think of you.\n",
        "What you think of yourself is much more important than what others think of you\n",
        "RT @JessKristenson: Integrity is more important than popularity\n",
        "RT @Merlin_23_TW: 'The love that binds us is more important than the power we wield' #Merlin #Mordred <3\n",
        "How people perceive what you say is much more important than what you actually say\n",
        "Politico: Surveillance panel shut down http://t.co/LwgX1oPvKg.\"How could this be more important than kids cancer trials at NIH?\u201d\n",
        "RT @aprillaureles1: Your relationship with God is more important than anything because you know for sure that\u2019s a relationship that will la\u2026\n",
        "Parents such more important than your lover k remember that\n",
        "'The love that binds us is more important than the power we wield' #Merlin #Mordred <3\n",
        "Leave it to Mariah to find her nap more important than her health. \ud83d\ude12\n",
        "@IrisjeXx @LindaKwakernaat Strange mum. Reading books is much more important than being clad well. She should know! Must be a blonde!\n",
        "RT @princessg4y: IF U CONSTANTLY REFER TO UR BF AS \"\"\"\"\"MY BOYFRIEND\"\"\"\"\" SLAP URSELF FORREAL HES MORE IMPORTANT THAN A TITLE\n",
        "RT @neiltyson: \"Imagination is more important than knowledge\" -Albert Einstein . . . Except on Jeopardy.\n",
        "RT @GavinBoyleCRH: Nurses choosing to lead has never been more important than it is today - for patients & for the service. #WeNurses\n",
        "and remember nobody's feelings are more important than your own :/ so take time to love yourself :D\n"
       ]
      }
     ],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sample = False\n",
      "for tweet in Twitter().search('\"more important than\"', cached=False, count=1, language='en'):\n",
      "    print plaintext(tweet.text)\n",
      "    pprint.pprint(ast.literal_eval(tweet.raw))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "ename": "AttributeError",
       "evalue": "'function' object has no attribute 'pprint'",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-7-58456447769a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtweet\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mTwitter\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'\"more important than\"'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcached\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlanguage\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'en'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mprint\u001b[0m \u001b[0mplaintext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtweet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mpprint\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mast\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mliteral_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtweet\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;31mAttributeError\u001b[0m: 'function' object has no attribute 'pprint'"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "So like American Horror Story is returning this Wednesday....its more important than what my life consists of.\n"
       ]
      }
     ],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "\"\"\"\n",
      "print type(sample.json)\n",
      "sample_string = sample.json\n",
      "tweet = ast.literal_eval(sample_string)\n",
      "tweet['text']\n",
      "\"\"\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 8,
       "text": [
        "\"\\nprint type(sample.json)\\nsample_string = sample.json\\ntweet = ast.literal_eval(sample_string)\\ntweet['text']\\n\""
       ]
      }
     ],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for tweet in Twitter().search('\"#RIT\"', cached=False, count=5, language='en'):\n",
      "   print plaintext(tweet.text)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Olha a galera que participou do Prog. @noiteadoradores de Deus Hoje \u00e1s 22:00h na #RIT #TV . ( Banda Megafone,... http://t.co/p4KWPSsNv0\n",
        "Had very tiring yet interesting day. #IITTechfest #ReformNation #RIT\n",
        "RT @RITsports: RECAP - #RIT women's crew excels at 25th annual Head of the Genesee Regatta. #RITNews #TigerPride http://t.co/yjKRDfBhzR\n",
        "RT @TrendyCave: SUPER! by Kate Bertin http://t.co/hzLOGBwDst #IAmGoldRetweet #SuperRT#LPRO #smartbizuk #FF #RIT\n",
        "smoky-rosy by LanaD http://t.co/6XqRfVqlVk #FF #RIT #handmadebot\n"
       ]
      }
     ],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Search\n",
      "t = Twitter()\n",
      "i = None\n",
      "for j in range(3):\n",
      "    for tweet in t.search('#openstack', start=i, count=2, language='en'):\n",
      "        print tweet.text\n",
      "        print\n",
      "        i = tweet.id"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "RT @openQRMgmbh: Check how @openQRM featurewise beats any other Cloud Computing solution - http://t.co/oZoD1RihMR #iaas #openstack #eucalyp\u2026\n",
        "\n",
        "RT @hpcloud: Register for the #OpenStack Canadian Chapter Event Oct 8 in Montreal http://t.co/lVNJ5LxZWa  #HPCloud\n",
        "\n",
        "RT @hpcloud: Register for the #OpenStack Canadian Chapter Event Oct 8 in Montreal http://t.co/lVNJ5LxZWa  #HPCloud"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n",
        "#OpenStack includes a voting process for project leaders. That's so cool.\n",
        "\n",
        "#OpenStack includes a voting process for project leaders. That's so cool."
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n",
        "[Video] Is #OpenStack a #cloud? @JeffreyLush answers questions on OpenStack | http://t.co/eAYRbJnELA via @DellSolves\"\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# streams\n",
      "sample = False\n",
      "s = Twitter().stream('#NFL')\n",
      "for i in range(10):\n",
      "    time.sleep(1) \n",
      "    s.update(bytes=1024)\n",
      "    #print s[-1].text if s else ''\n",
      "    if s:\n",
      "         pprint(ast.literal_eval(s[-1].raw))\n",
      "        \n",
      "s.clear()\n",
      "\n",
      "#print type(sample)\n",
      "#pprint.pprint(sample)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# trends\n",
      "print Twitter().trends(cached=False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[u'#danandphilthevamps', u'#BranquelosETitiaNoMelhorDoBrasil', u'#TagSomeoneYouNeverWantToLose', u'#\\xd6zelG\\xfcc\\xfcmOlsa', u'#OverloadSecretSong', u'Dion Jordan', u'Reshad Jones', u'Chris Crocker', u'Brent Celek', u'LiderHALA FENERBAH\\xc7E Ulan']\n"
       ]
      }
     ],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pattern.en import parse, pprint\n",
      "s = 'The mobile web is more important than mobile apps.'\n",
      "s = parse(s, relations=True, lemmata=True)\n",
      "pprint(s)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "          WORD   TAG    CHUNK    ROLE   ID     PNP    LEMMA       \n",
        "                                                                  \n",
        "           The   DT     NP       SBJ    1      -      the         \n",
        "        mobile   JJ     NP ^     SBJ    1      -      mobile      \n",
        "           web   NN     NP ^     SBJ    1      -      web         \n",
        "            is   VBZ    VP       -      1      -      be          \n",
        "          more   RBR    ADJP     -      -      -      more        \n",
        "     important   JJ     ADJP ^   -      -      -      important   \n",
        "          than   IN     PP       -      -      PNP    than        \n",
        "        mobile   JJ     NP       -      -      PNP    mobile      \n",
        "          apps   NNS    NP ^     -      -      PNP    app         \n",
        "             .   .      -        -      -      -      .           \n"
       ]
      }
     ],
     "prompt_number": 13
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "s = '@RITWSoccer: GAME DAY! RIT vs @RedDragonSports! Kick-off at 1pm at Hartwick. #RIT #GoTigers'\n",
      "s = parse(s, relations=True, lemmata=True)\n",
      "pprint(s)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA        \n",
        "                                                                  \n",
        "             @   IN     PP      -      -      PNP    @            \n",
        "    RITWSoccer   NNP    NP      -      -      PNP    ritwsoccer   \n",
        "             :   :      -       -      -      -      :            \n",
        "          GAME   NNP    NP      -      -      -      game         \n",
        "           DAY   NNP    NP ^    -      -      -      day          \n",
        "             !   .      -       -      -      -      !            \n",
        "\n",
        "           WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA             \n",
        "                                                                        \n",
        "            RIT   NNP    NP      -      -      -      rit               \n",
        "             vs   NNP    NP ^    -      -      -      vs                \n",
        "              @   IN     PP      -      -      PNP    @                 \n",
        "RedDragonSports   NNP    NP      -      -      PNP    reddragonsports   \n",
        "              !   .      -       -      -      -      !                 \n",
        "\n",
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA      \n",
        "                                                                \n",
        "      Kick-off   NN     NP      -      -      -      kick-off   \n",
        "            at   IN     PP      -      -      -      at         \n",
        "           1pm   CD     -       -      -      -      1pm        \n",
        "            at   IN     PP      -      -      PNP    at         \n",
        "      Hartwick   NNP    NP      -      -      PNP    hartwick   \n",
        "             .   .      -       -      -      -      .          \n",
        "\n",
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA      \n",
        "                                                                \n",
        "             #   #      -       -      -      -      #          \n",
        "           RIT   NNP    NP      -      -      -      rit        \n",
        "             #   #      -       -      -      -      #          \n",
        "      GoTigers   NNP    NP      -      -      -      gotigers   \n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Transformations:  Remove Prefix.  Remove hash tags.  Replace 'vs' with .versus'.  Parse mixed-case expressions. "
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "s = 'GAME DAY! RIT versus Red Dragon Sports! Kick-off at 1pm at Hartwick.'\n",
      "s = parse(s, relations=True, lemmata=True)\n",
      "pprint(s)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA   \n",
        "                                                             \n",
        "          GAME   NNP    NP      -      -      -      game    \n",
        "           DAY   NNP    NP ^    -      -      -      day     \n",
        "             !   .      -       -      -      -      !       \n",
        "\n",
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA    \n",
        "                                                              \n",
        "           RIT   NNS    NP      -      -      -      rit      \n",
        "        versus   IN     PP      -      -      PNP    versus   \n",
        "           Red   NNP    NP      -      -      PNP    red      \n",
        "        Dragon   NNP    NP ^    -      -      PNP    dragon   \n",
        "        Sports   NNPS   NP ^    -      -      PNP    sports   \n",
        "             !   .      -       -      -      -      !        \n",
        "\n",
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA      \n",
        "                                                                \n",
        "      Kick-off   NN     NP      -      -      -      kick-off   \n",
        "            at   IN     PP      -      -      -      at         \n",
        "           1pm   CD     -       -      -      -      1pm        \n",
        "            at   IN     PP      -      -      PNP    at         \n",
        "      Hartwick   NNP    NP      -      -      PNP    hartwick   \n",
        "             .   .      -       -      -      -      .          \n"
       ]
      }
     ],
     "prompt_number": 15
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# spell out RIT, parse 1pm, substitute starts for kick-off\n",
      "s = 'GAME DAY! Rochester Institute of Technology versus Red Dragon Sports! Starts at 1 pm at Hartwick.'\n",
      "s = parse(s, relations=True, lemmata=True)\n",
      "pprint(s)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA   \n",
        "                                                             \n",
        "          GAME   NNP    NP      -      -      -      game    \n",
        "           DAY   NNP    NP ^    -      -      -      day     \n",
        "             !   .      -       -      -      -      !       \n",
        "\n",
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA        \n",
        "                                                                  \n",
        "     Rochester   NNP    NP      -      -      -      rochester    \n",
        "     Institute   NNP    NP ^    -      -      -      institute    \n",
        "            of   IN     PP      -      -      PNP    of           \n",
        "    Technology   NNP    NP      -      -      PNP    technology   \n",
        "        versus   IN     PP      -      -      PNP    versus       \n",
        "           Red   NNP    NP      -      -      PNP    red          \n",
        "        Dragon   NNP    NP ^    -      -      PNP    dragon       \n",
        "        Sports   NNPS   NP ^    -      -      PNP    sports       \n",
        "             !   .      -       -      -      -      !            \n",
        "\n",
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA      \n",
        "                                                                \n",
        "        Starts   VBZ    VP      -      -      -      start      \n",
        "            at   IN     PP      -      -      PNP    at         \n",
        "             1   CD     NP      -      -      PNP    1          \n",
        "            pm   NN     NP ^    -      -      PNP    pm         \n",
        "            at   IN     PP      -      -      PNP    at         \n",
        "      Hartwick   NNP    NP      -      -      PNP    hartwick   \n",
        "             .   .      -       -      -      -      .          \n"
       ]
      }
     ],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pattern.en import parse, pprint\n",
      "s = 'The mobile web is more important than mobile apps.'\n",
      "s = parse(s, relations=True, lemmata=True)\n",
      "pprint(s)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "          WORD   TAG    CHUNK    ROLE   ID     PNP    LEMMA       \n",
        "                                                                  \n",
        "           The   DT     NP       SBJ    1      -      the         \n",
        "        mobile   JJ     NP ^     SBJ    1      -      mobile      \n",
        "           web   NN     NP ^     SBJ    1      -      web         \n",
        "            is   VBZ    VP       -      1      -      be          \n",
        "          more   RBR    ADJP     -      -      -      more        \n",
        "     important   JJ     ADJP ^   -      -      -      important   \n",
        "          than   IN     PP       -      -      PNP    than        \n",
        "        mobile   JJ     NP       -      -      PNP    mobile      \n",
        "          apps   NNS    NP ^     -      -      PNP    app         \n",
        "             .   .      -        -      -      -      .           \n"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pattern.en import parsetree\n",
      "s = 'The mobile web is more important than mobile apps.'\n",
      "t = parsetree(s)\n",
      "for sentence in t: \n",
      "    for chunk in sentence.chunks:\n",
      "        for word in chunk.words:\n",
      "            print word,\n",
      "        print"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Word(u'The/DT') Word(u'mobile/JJ') Word(u'web/NN')\n",
        "Word(u'is/VBZ')\n",
        "Word(u'more/RBR') Word(u'important/JJ')\n",
        "Word(u'than/IN')\n",
        "Word(u'mobile/JJ') Word(u'apps/NNS')\n"
       ]
      }
     ],
     "prompt_number": 18
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "s = 'GAME DAY! Rochester Institute of Technology versus Red Dragon Sports! Kick-off at 1 pm at Hartwick.'\n",
      "t = parsetree(s)\n",
      "for sentence in t: \n",
      "    print \"Sentence:\"\n",
      "    for chunk in sentence.chunks:\n",
      "        print \"Chunk:\"\n",
      "        for word in chunk.words:\n",
      "            print word,\n",
      "        print"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Sentence:\n",
        "Chunk:\n",
        "Word(u'GAME/NNP') Word(u'DAY/NNP')\n",
        "Sentence:\n",
        "Chunk:\n",
        "Word(u'Rochester/NNP') Word(u'Institute/NNP')\n",
        "Chunk:\n",
        "Word(u'of/IN')\n",
        "Chunk:\n",
        "Word(u'Technology/NNP')\n",
        "Chunk:\n",
        "Word(u'versus/IN')\n",
        "Chunk:\n",
        "Word(u'Red/NNP') Word(u'Dragon/NNP') Word(u'Sports/NNPS')\n",
        "Sentence:\n",
        "Chunk:\n",
        "Word(u'Kick-off/NN')\n",
        "Chunk:\n",
        "Word(u'at/IN')\n",
        "Chunk:\n",
        "Word(u'1/CD') Word(u'pm/NN')\n",
        "Chunk:\n",
        "Word(u'at/IN')\n",
        "Chunk:\n",
        "Word(u'Hartwick/NNP')\n"
       ]
      }
     ],
     "prompt_number": 19
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "pattern.en"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pattern.en import referenced, article\n",
      "from pattern.en import pluralize, singularize\n",
      "from pattern.en import comparative, superlative\n",
      "from pattern.en import conjugate, lemma, lexeme, tenses\n",
      "from pattern.en import spelling"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# DETERMINER\n",
      "# The article is the most common determiner (DT) in English. It defines \n",
      "# whether the noun following it is definite (the cat) or indefinite (a cat). \n",
      "# The definite article is always the. The indefinite article can be either \n",
      "# a or an \u2013 depending on how the successive noun is pronounced.\n",
      "\n",
      "#determines indefinite article for a word.  The is always the definite article\n",
      "\n",
      "print referenced('university')\n",
      "print referenced('university', article='definite')\n",
      "print article('university')\n",
      "print article('university', function='definite')\n",
      "print referenced('hour')\n",
      "#help(article)\n",
      "#help(referenced)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "a university\n",
        "the university\n",
        "a\n",
        "the\n",
        "an hour\n"
       ]
      }
     ],
     "prompt_number": 21
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# NOUN\n",
      "# The pluralize() function returns the singular form of a plural noun. It \n",
      "# handles 96% of exceptions correctly. The singularize() function returns \n",
      "# the plural form of a singular noun. The pos parameter (part-of-speech)\n",
      "# can be set to NOUN or ADJECTIVE, but only a small number of possessive\n",
      "# adjectives inflect (e.g. my \u2192 our). The custom dictionary is for\n",
      "# user-defined replacements.\n",
      "\n",
      "print pluralize('child')\n",
      "print singularize('wolves')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "children\n",
        "wolf\n"
       ]
      }
     ],
     "prompt_number": 22
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# ADJECTIVE\n",
      "# function give the comparative or superlative form of an adjective. Words\n",
      "# with three or more syllables are simply preceded by more or most.\n",
      "print comparative('bad')\n",
      "print superlative('bad')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "worse\n",
        "worst\n"
       ]
      }
     ],
     "prompt_number": 23
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# VERB conjugation\n",
      "verb = 'ran'\n",
      "print conjugate(verb, \n",
      "    tense = 'present',        # INFINITIVE, PRESENT, PAST, FUTURE\n",
      "   person = 3,              # 1, 2, 3 or None\n",
      "   number = 'sg',       # SG, PL\n",
      "     mood = 'indicative',     # INDICATIVE, IMPERATIVE, CONDITIONAL, SUBJUNCTIVE\n",
      "   aspect = 'imperfective',   # IMPERFECTIVE, PERFECTIVE, PROGRESSIVE \n",
      "  negated = False)\n",
      "print lemma(verb)\n",
      "print lexeme(verb)\n",
      "print tenses(verb)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "runs\n",
        "run\n",
        "[u'run', u'runs', u'running', u'ran']\n",
        "[('past', None, None, 'indicative', 'imperfective'), ('past', None, 'plural', 'indicative', 'imperfective'), ('past', 1, 'plural', 'indicative', 'imperfective'), ('past', 1, 'singular', 'indicative', 'imperfective'), ('past', 2, 'plural', 'indicative', 'imperfective'), ('past', 2, 'singular', 'indicative', 'imperfective'), ('past', 3, 'plural', 'indicative', 'imperfective'), ('past', 3, 'singular', 'indicative', 'imperfective')]\n"
       ]
      }
     ],
     "prompt_number": 24
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#Spelling\n",
      "#help(pattern.en.spelling)\n",
      "pattern.en.spelling.suggest(\"adress\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 25,
       "text": [
        "[('dress', 0.6448598130841121), ('address', 0.35514018691588783)]"
       ]
      }
     ],
     "prompt_number": 25
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print pattern.en.ngrams(\"I am eating a pizza. But I am almost done.\", n=3)\n",
      "print\n",
      "print pattern.en.ngrams(\"I am eating a pizza. But I am almost done.\", n=3, continuous=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[('I', 'am', 'eating'), ('am', 'eating', 'a'), ('eating', 'a', 'pizza'), ('But', 'I', 'am'), ('I', 'am', 'almost'), ('am', 'almost', 'done')]\n",
        "\n",
        "[('I', 'am', 'eating'), ('am', 'eating', 'a'), ('eating', 'a', 'pizza'), ('a', 'pizza', 'But'), ('pizza', 'But', 'I'), ('But', 'I', 'am'), ('I', 'am', 'almost'), ('am', 'almost', 'done')]\n"
       ]
      }
     ],
     "prompt_number": 26
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pattern.en import parse\n",
      "print parse('I eat pizza with a fork.')\n",
      "print\n",
      "print parse('I eat pizza with a fork.', relations=True)\n",
      "print\n",
      "print parse('I ate pizza with a fork.', lemmata=True)\n",
      "\n",
      "# POS:   (NN, JJ, ...)\n",
      "# chunk: (NP, VP, PNP, ...)\n",
      "# PNP  prepositional noun phrase\n",
      "# Relat: (SBJ, OBJ, ...)\n",
      "\n",
      "# Word   /PoS  /chunk   /PNP   Relations (\n",
      "#\n",
      "# I     /PRP   /B-NP    /O     NP-SBJ-1\n",
      "# eat   /VBP   /B-VP    /O     VP-1\n",
      "# pizza /VBG   /I-VP    /O     VP-1\n",
      "#\n",
      "# with  /IN    /B-PP   /B-PNP  O \n",
      "#\n",
      "# a     /DT    /B-NP   /I-PNP  O\n",
      "# fork  /NN    /I-NP   /I-PN   O \n",
      "#\n",
      "# .     /.     /O      /O      O"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "I/PRP/B-NP/O eat/VBP/B-VP/O pizza/VBG/I-VP/O with/IN/B-PP/B-PNP a/DT/B-NP/I-PNP fork/NN/I-NP/I-PNP ././O/O\n",
        "\n",
        "I/PRP/B-NP/O/NP-SBJ-1 eat/VBP/B-VP/O/VP-1 pizza/VBG/I-VP/O/VP-1 with/IN/B-PP/B-PNP/O a/DT/B-NP/I-PNP/O fork/NN/I-NP/I-PNP/O ././O/O/O\n",
        "\n",
        "I/PRP/B-NP/O/i ate/VBD/B-VP/O/eat pizza/NN/B-NP/O/pizza with/IN/B-PP/B-PNP/with a/DT/B-NP/I-PNP/a fork/NN/I-NP/I-PNP/fork ././O/O/.\n"
       ]
      }
     ],
     "prompt_number": 27
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for tweet in Twitter().search('\"#RIT\"', cached=False, count=2, language='en'):\n",
      "   print plaintext(tweet.text)\n",
      "   pprint(parse(plaintext(tweet.text), relations=True, lemmata=True))\n",
      "   print"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Olha a galera que participou do Prog. @noiteadoradores de Deus Hoje \u00e1s 22:00h na #RIT #TV . ( Banda Megafone,... http://t.co/p4KWPSsNv0\n",
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA        \n",
        "                                                                  \n",
        "          Olha   NNP    NP      -      -      -      olha         \n",
        "             a   DT     NP ^    -      -      -      a            \n",
        "        galera   NN     NP ^    -      -      -      galera       \n",
        "           que   FW     -       -      -      -      que          \n",
        "    participou   NN     NP      SBJ    1      -      participou   \n",
        "            do   VBP    VP      -      1      -      do           \n",
        "          Prog   NNP    NP      OBJ    1      -      prog         \n",
        "             .   .      -       -      -      -      .            \n",
        "\n",
        "           WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA            \n",
        "                                                                       \n",
        "              @   IN     PP      -      -      PNP    @                \n",
        "noiteadoradores   NNS    NP      -      -      PNP    noiteadoradore   \n",
        "             de   FW     -       -      -      -      de               \n",
        "           Deus   FW     -       -      -      -      deus             \n",
        "           Hoje   NNP    NP      -      -      -      hoje             \n",
        "             \u00e1s   NNS    NP ^    -      -      -      \u00e1                \n",
        "         22:00h   CD     -       -      -      -      22:00h           \n",
        "             na   TO     PP      -      -      -      na               \n",
        "              #   #      -       -      -      -      #                \n",
        "            RIT   NNP    NP      -      -      -      rit              \n",
        "              #   #      -       -      -      -      #                \n",
        "             TV   NN     NP      -      -      -      tv               \n",
        "              .   .      -       -      -      -      .                \n",
        "\n",
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA      \n",
        "                                                                \n",
        "             (   (      -       -      -      -      (          \n",
        "         Banda   NNP    NP      -      -      -      banda      \n",
        "      Megafone   NNP    NP ^    -      -      -      megafone   \n",
        "             ,   ,      -       -      -      -      ,          \n",
        "           ...   :      -       -      -      -      ...        \n",
        "\n",
        "                  WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA   \n",
        "                                                                     \n",
        "http://t.co/p4KWPSsNv0   NNP    NP      -      -      -      http:   \n",
        "\n",
        "Had very tiring yet interesting day. #IITTechfest #ReformNation #RIT\n",
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA         \n",
        "                                                                   \n",
        "           Had   VBD    VP      -      1      -      have          \n",
        "          very   RB     VP ^    -      1      -      very          \n",
        "        tiring   VBG    VP ^    -      1      -      tire          \n",
        "           yet   RB     NP      OBJ    1      -      yet           \n",
        "   interesting   JJ     NP ^    OBJ    1      -      interesting   \n",
        "           day   NN     NP ^    OBJ    1      -      day           \n",
        "             .   .      -       -      -      -      .             \n",
        "\n",
        "          WORD   TAG    CHUNK   ROLE   ID     PNP    LEMMA          \n",
        "                                                                    \n",
        "             #   #      -       -      -      -      #              \n",
        "   IITTechfest   NNP    NP      -      -      -      iittechfest    \n",
        "             #   #      -       -      -      -      #              \n",
        "  ReformNation   NNP    NP      -      -      -      reformnation   \n",
        "             #   #      -       -      -      -      #              \n",
        "           RIT   NNP    NP      -      -      -      rit            \n",
        "\n"
       ]
      }
     ],
     "prompt_number": 28
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pattern.en import sentiment, polarity, subjectivity, positive\n",
      "s = \"The movie attempts to be surreal by incorporating various time paradoxes, but it's presented in such a ridiculous way it's seriously boring.\"\n",
      "print sentiment(s)\n",
      "print polarity(s)\n",
      "print subjectivity(s)\n",
      "print positive(s)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "(-0.4166666666666667, 0.2)\n",
        "-0.416666666667\n",
        "0.2\n",
        "False\n"
       ]
      }
     ],
     "prompt_number": 29
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pattern.en import Sentence, parse, modality, mood, negated\n",
      "s = \"Some amino acids tend to be acidic while others may be basic.\" # weaseling\n",
      "s = Sentence(parse(s, chunks=False, lemmata=True))\n",
      "print modality(s)   # >0.5 is factural\n",
      "print mood(s)\n",
      "print negated(s)\n",
      "print\n",
      "s = \"The movie attempts to be surreal by incorporating various time paradoxes, but it's presented in such a ridiculous way it's seriously boring.\"\n",
      "s = Sentence(parse(s, chunks=False, lemmata=True))\n",
      "print modality(s)   # >0.5 is factural\n",
      "print mood(s)\n",
      "print negated(s)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "0.111111111111\n",
        "indicative\n",
        "False\n",
        "\n",
        "-1.0\n",
        "indicative\n",
        "False\n"
       ]
      }
     ],
     "prompt_number": 30
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Additional code to parse test using patternNLP and MBSP\n",
      "\n",
      "Try PatternNLP http://www.clips.ua.ac.be/pages/pattern-en#parser\n",
      "Tag Format: http://www.clips.ua.ac.be/pages/MBSP-tags"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import nltk\n",
      "\n",
      "sent = nltk.pos_tag(nltk.word_tokenize(\"The quick brown fox jumped over the lazy dog.\"))\n",
      "print sent\n",
      "print\n",
      "sent = nltk.corpus.treebank.tagged_sents()[22]\n",
      "chunk = nltk.ne_chunk(sent)\n",
      "print chunk\n",
      "print\n",
      "#sent = \"The U.S. is one of the few industrialized nations that *T*-7 doesn't have a higher standard of regulation for the smooth, needle-like fibers such as crocidolite that *T*-1 are classified *-5 as amphobiles, according to Brooke T. Mossman, a professor of pathlogy at the University of Vermont College of Medicine.\"\n",
      "#sent = nltk.pos_tag(nltk.word_tokenize(sent))\n",
      "#print sent\n",
      "#print\n",
      "#print nltk.ne_chunk(sent, binary=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[('The', 'DT'), ('quick', 'NN'), ('brown', 'NN'), ('fox', 'NN'), ('jumped', 'VBD'), ('over', 'IN'), ('the', 'DT'), ('lazy', 'NN'), ('dog', 'NN'), ('.', '.')]\n",
        "\n",
        "(S\n",
        "  The/DT\n",
        "  (GPE U.S./NNP)\n",
        "  is/VBZ\n",
        "  one/CD\n",
        "  of/IN\n",
        "  the/DT\n",
        "  few/JJ\n",
        "  industrialized/VBN\n",
        "  nations/NNS\n",
        "  that/WDT\n",
        "  *T*-7/-NONE-\n",
        "  does/VBZ\n",
        "  n't/RB\n",
        "  have/VB\n",
        "  a/DT\n",
        "  higher/JJR\n",
        "  standard/NN\n",
        "  of/IN\n",
        "  regulation/NN\n",
        "  for/IN\n",
        "  the/DT\n",
        "  smooth/JJ\n",
        "  ,/,\n",
        "  needle-like/JJ\n",
        "  fibers/NNS\n",
        "  such/JJ\n",
        "  as/IN\n",
        "  crocidolite/NN\n",
        "  that/WDT\n",
        "  *T*-1/-NONE-\n",
        "  are/VBP\n",
        "  classified/VBN\n",
        "  *-5/-NONE-\n",
        "  as/IN\n",
        "  amphobiles/NNS\n",
        "  ,/,\n",
        "  according/VBG\n",
        "  to/TO\n",
        "  (PERSON Brooke/NNP T./NNP Mossman/NNP)\n",
        "  ,/,\n",
        "  a/DT\n",
        "  professor/NN\n",
        "  of/IN\n",
        "  pathlogy/NN\n",
        "  at/IN\n",
        "  the/DT\n",
        "  (ORGANIZATION University/NNP)\n",
        "  of/IN\n",
        "  (PERSON Vermont/NNP College/NNP)\n",
        "  of/IN\n",
        "  (GPE Medicine/NNP)\n",
        "  ./.)"
       ]
      },
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 31
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#import nltk.chunk.util\n",
      "nltk.chunk.util.tree2conllstr(chunk)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "pyout",
       "prompt_number": 32,
       "text": [
        "\"The DT O\\nU.S. NNP B-GPE\\nis VBZ O\\none CD O\\nof IN O\\nthe DT O\\nfew JJ O\\nindustrialized VBN O\\nnations NNS O\\nthat WDT O\\n*T*-7 -NONE- O\\ndoes VBZ O\\nn't RB O\\nhave VB O\\na DT O\\nhigher JJR O\\nstandard NN O\\nof IN O\\nregulation NN O\\nfor IN O\\nthe DT O\\nsmooth JJ O\\n, , O\\nneedle-like JJ O\\nfibers NNS O\\nsuch JJ O\\nas IN O\\ncrocidolite NN O\\nthat WDT O\\n*T*-1 -NONE- O\\nare VBP O\\nclassified VBN O\\n*-5 -NONE- O\\nas IN O\\namphobiles NNS O\\n, , O\\naccording VBG O\\nto TO O\\nBrooke NNP B-PERSON\\nT. NNP I-PERSON\\nMossman NNP I-PERSON\\n, , O\\na DT O\\nprofessor NN O\\nof IN O\\npathlogy NN O\\nat IN O\\nthe DT O\\nUniversity NNP B-ORGANIZATION\\nof IN O\\nVermont NNP B-PERSON\\nCollege NNP I-PERSON\\nof IN O\\nMedicine NNP B-GPE\\n. . O\""
       ]
      }
     ],
     "prompt_number": 32
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pattern.en import parse, pprint\n",
      "import pattern.en\n",
      "import pprint\n",
      "s = 'The mobile web is more important than mobile apps.'\n",
      "s = parse(s, relations=True, lemmata=True)\n",
      "pprint.pprint(s)\n",
      "print\n",
      "pattern.en.pprint(s)\n",
      "print\n",
      "print s\n",
      "#print\n",
      "#for a in s.split()[0]: print a"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "u'The/DT/B-NP/O/NP-SBJ-1/the mobile/JJ/I-NP/O/NP-SBJ-1/mobile web/NN/I-NP/O/NP-SBJ-1/web is/VBZ/B-VP/O/VP-1/be more/RBR/B-ADJP/O/O/more important/JJ/I-ADJP/O/O/important than/IN/B-PP/B-PNP/O/than mobile/JJ/B-NP/I-PNP/O/mobile apps/NNS/I-NP/I-PNP/O/app ././O/O/O/.'\n",
        "\n",
        "          WORD   TAG    CHUNK    ROLE   ID     PNP    LEMMA       \n",
        "                                                                  \n",
        "           The   DT     NP       SBJ    1      -      the         \n",
        "        mobile   JJ     NP ^     SBJ    1      -      mobile      \n",
        "           web   NN     NP ^     SBJ    1      -      web         \n",
        "            is   VBZ    VP       -      1      -      be          \n",
        "          more   RBR    ADJP     -      -      -      more        \n",
        "     important   JJ     ADJP ^   -      -      -      important   \n",
        "          than   IN     PP       -      -      PNP    than        \n",
        "        mobile   JJ     NP       -      -      PNP    mobile      \n",
        "          apps   NNS    NP ^     -      -      PNP    app         \n",
        "             .   .      -        -      -      -      .           \n",
        "\n",
        "The/DT/B-NP/O/NP-SBJ-1/the mobile/JJ/I-NP/O/NP-SBJ-1/mobile web/NN/I-NP/O/NP-SBJ-1/web is/VBZ/B-VP/O/VP-1/be more/RBR/B-ADJP/O/O/more important/JJ/I-ADJP/O/O/important than/IN/B-PP/B-PNP/O/than mobile/JJ/B-NP/I-PNP/O/mobile apps/NNS/I-NP/I-PNP/O/app ././O/O/O/.\n"
       ]
      }
     ],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "s = 'The mobile web is more important than mobile apps.'\n",
      "s = parse(s, relations=True, lemmata=True)\n",
      "#pattern.en.pprint(s)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 34
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pattern.en import parsetree\n",
      "s = 'The mobile web is more important than mobile apps.'\n",
      "s = parsetree(s, relations=True, lemmata=True)\n",
      "pattern.en.pprint(s)\n",
      "print repr(s)\n",
      "print\n",
      "pprint.pprint(s)\n",
      "#print\n",
      "#print\n",
      "#s = 'The mobile web is more important than mobile apps.'\n",
      "#s = parsetree(s)\n",
      "#pattern.en.pprint(s)\n",
      "#print repr(s)\n",
      "#print\n",
      "#pprint.pprint(s)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "          WORD   TAG    CHUNK    ROLE   ID     PNP    LEMMA       \n",
        "                                                                  \n",
        "           The   DT     NP       SBJ    1      -      the         \n",
        "        mobile   JJ     NP ^     SBJ    1      -      mobile      \n",
        "           web   NN     NP ^     SBJ    1      -      web         \n",
        "            is   VBZ    VP       -      1      -      be          \n",
        "          more   RBR    ADJP     -      -      -      more        \n",
        "     important   JJ     ADJP ^   -      -      -      important   \n",
        "          than   IN     PP       -      -      PNP    than        \n",
        "        mobile   JJ     NP       -      -      PNP    mobile      \n",
        "          apps   NNS    NP ^     -      -      PNP    app         \n",
        "             .   .      -        -      -      -      .           \n",
        "[Sentence('The/DT/B-NP/O/NP-SBJ-1/the mobile/JJ/I-NP/O/NP-SBJ-1/mobile web/NN/I-NP/O/NP-SBJ-1/web is/VBZ/B-VP/O/VP-1/be more/RBR/B-ADJP/O/O/more important/JJ/I-ADJP/O/O/important than/IN/B-PP/B-PNP/O/than mobile/JJ/B-NP/I-PNP/O/mobile apps/NNS/I-NP/I-PNP/O/app ././O/O/O/.')]\n",
        "\n",
        "[Sentence('The/DT/B-NP/O/NP-SBJ-1/the mobile/JJ/I-NP/O/NP-SBJ-1/mobile web/NN/I-NP/O/NP-SBJ-1/web is/VBZ/B-VP/O/VP-1/be more/RBR/B-ADJP/O/O/more important/JJ/I-ADJP/O/O/important than/IN/B-PP/B-PNP/O/than mobile/JJ/B-NP/I-PNP/O/mobile apps/NNS/I-NP/I-PNP/O/app ././O/O/O/.')]\n"
       ]
      }
     ],
     "prompt_number": 35
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "s = 'The quick brown fox jumped over the lazy dog.'\n",
      "s = parsetree(s)\n",
      "print s[0]\n",
      "print \"chunks:\"\n",
      "print s[0].chunks\n",
      "print \"subjects:\"\n",
      "print s[0].subjects\n",
      "print \"objects:\"\n",
      "print s[0].objects\n",
      "print \"verbs:\"\n",
      "print s[0].verbs\n",
      "print \"relations:\"\n",
      "print s[0].relations\n",
      "print \"pnp:\"\n",
      "print s[0].pnp\n",
      "print \"constituents:\"\n",
      "print s[0].constituents()\n",
      "print\n",
      "print s[0].constituents(pnp=True)\n",
      "print\n",
      "for c in s[0].chunks:\n",
      "    print c\n",
      "    print c.string\n",
      "    print c.words\n",
      "    print c.type\n",
      "    print c.role\n",
      "    print c.relation\n",
      "    print c.relations\n",
      "    print c.related\n",
      "    print c.subject\n",
      "    print c.object\n",
      "    print"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Sentence('The/DT/B-NP/O quick/JJ/I-NP/O brown/JJ/I-NP/O fox/NN/I-NP/O jumped/VBD/B-VP/O over/IN/B-PP/B-PNP the/DT/B-NP/I-PNP lazy/JJ/I-NP/I-PNP dog/NN/I-NP/I-PNP ././O/O')\n",
        "chunks:\n",
        "[Chunk('The quick brown fox/NP'), Chunk('jumped/VP'), Chunk('over/PP'), Chunk('the lazy dog/NP')]\n",
        "subjects:\n",
        "[]\n",
        "objects:\n",
        "[]\n",
        "verbs:\n",
        "[Chunk('jumped/VP')]\n",
        "relations:\n",
        "{'SBJ': {}, 'VP': {None: Chunk('jumped/VP')}, 'OBJ': {}}\n",
        "pnp:\n",
        "[Chunk('over the lazy dog/PNP')]\n",
        "constituents:\n",
        "[Chunk('The quick brown fox/NP'), Chunk('jumped/VP'), Chunk('over/PP'), Chunk('the lazy dog/NP'), Word(u'./.')]\n",
        "\n",
        "[Chunk('The quick brown fox/NP'), Chunk('jumped/VP'), Chunk('over the lazy dog/PNP'), Word(u'./.')]\n",
        "\n",
        "Chunk('The quick brown fox/NP')\n",
        "The quick brown fox\n",
        "[Word(u'The/DT'), Word(u'quick/JJ'), Word(u'brown/JJ'), Word(u'fox/NN')]\n",
        "NP\n",
        "None\n",
        "None\n",
        "[]\n",
        "[]\n",
        "None\n",
        "None\n",
        "\n",
        "Chunk('jumped/VP')\n",
        "jumped\n",
        "[Word(u'jumped/VBD')]\n",
        "VP\n",
        "None\n",
        "None\n",
        "[]\n",
        "[]\n",
        "None\n",
        "None\n",
        "\n",
        "Chunk('over/PP')\n",
        "over\n",
        "[Word(u'over/IN')]\n",
        "PP\n",
        "None\n",
        "None\n",
        "[]\n",
        "[]\n",
        "None\n",
        "None\n",
        "\n",
        "Chunk('the lazy dog/NP')\n",
        "the lazy dog\n",
        "[Word(u'the/DT'), Word(u'lazy/JJ'), Word(u'dog/NN')]\n",
        "NP\n",
        "None\n",
        "None\n",
        "[]\n",
        "[]\n",
        "None\n",
        "None\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#s = 'The quick brown fox jumped over the lazy dog.'\n",
      "s = \"You aren't nice\"\n",
      "s = \"I dislike you\"\n",
      "#s = \"I think you are nice.\"\n",
      "#s = \"I know you are nice.\"\n",
      "s = parsetree(s)\n",
      "print \"sentiment: \",\n",
      "#print pattern.en.sentiment(\"the quick brown fox jumped over the lazy dog\")\n",
      "print pattern.en.sentiment(s[0])\n",
      "print \"polarity: \",\n",
      "print pattern.en.polarity(s[0])\n",
      "print \"subjectivity: \",\n",
      "print pattern.en.subjectivity(s[0])\n",
      "print \"positive: \",\n",
      "print pattern.en.positive(s[0])\n",
      "print \"mood: \",\n",
      "print pattern.en.mood(s[0])\n",
      "print \"mood-p: \",\n",
      "print pattern.en.mood(s[0], predictive=False), ' ',\n",
      "print pattern.en.mood(s[0], predictive=True)\n",
      "print \"modality: \",\n",
      "print pattern.en.modality(s[0])\n",
      "print \"negated: \",\n",
      "print pattern.en.negated(s[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "sentiment:  (0.0, 0.0)\n",
        "polarity:  0.0\n",
        "subjectivity:  0.0\n",
        "positive:  False\n",
        "mood:  indicative\n",
        "mood-p:  indicative   indicative\n",
        "modality:  1.0\n",
        "negated:  False\n"
       ]
      }
     ],
     "prompt_number": 37
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Use MBSP to Parse"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sys\n",
      "sys.path.append('/Users/doug/SW_Dev/MBSP/')\n",
      "import mbsp"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 38
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def mbsp_parsetree(s, *args, **kwargs):\n",
      "    \"\"\" Returns a parsed Text from the given string.\n",
      "    \"\"\"\n",
      "    return pattern.text.Text(mbsp.parse(s, *args, **kwargs))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 44
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#sent =\"The quick brown fox jumped over the lazy dog.  All quiet on the western front.\"\n",
      "#sent =\"The quick brown fox jumped over the lazy dog.\"\n",
      "sent = \"The U.S. is one of the few industrialized nations that *T*-7 doesn't have a higher standard of regulation for the smooth, needle-like fibers such as crocidolite that *T*-1 are classified *-5 as amphobiles, according to Brooke T. Mossman, a professor of pathlogy at the University of Vermont College of Medicine.\"\n",
      "s = mbsp.parse(sent)\n",
      "print\n",
      "print type(s)\n",
      "print\n",
      "print s.tags\n",
      "print\n",
      "#print s.tokens\n",
      "pprint.pprint(s)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\n",
        "<class 'mbsp.TokenString'>\n",
        "\n",
        "['word', 'part-of-speech', 'chunk', 'preposition', 'relation', 'anchor', 'lemma']\n",
        "\n",
        "u\"The/DT/I-NP/O/NP-SBJ-1/O/the U.S./NNP/I-NP/O/NP-SBJ-1/O/U.S. is/VBZ/I-VP/O/VP-1/O/be one/CD/I-NP/O/NP-PRD-1/A1/one of/IN/I-PP/B-PNP/O/P1/of the/DT/I-NP/I-PNP/O/P1/the few/JJ/I-NP/I-PNP/O/P1/few industrialized/VBN/I-NP/I-PNP/O/P1/industrialize nations/NNS/I-NP/I-PNP/O/P1/nation that/IN/B-NP/O/O/O/that */SYM/O/O/O/O/* T*-7/NN/I-NP/O/NP-SBJ-2/O/T*-7 does/VBZ/I-VP/O/VP-2/O/do n't/RB/I-VP/O/VP-2/O/n't have/VB/I-VP/O/VP-2/O/have a/DT/I-NP/O/NP-OBJ-2/A6-A7/a higher/JJR/I-NP/O/NP-OBJ-2/A6-A7/high standard/NN/I-NP/O/NP-OBJ-2/A6-A7/standard of/IN/I-PP/B-PNP/O/P6/of regulation/NN/I-NP/I-PNP/O/P6/regulation for/IN/I-PP/B-PNP/O/P7/for the/DT/I-NP/I-PNP/O/P7-A10/the smooth/JJ/I-NP/I-PNP/O/P7-A10/smooth ,/,/I-NP/I-PNP/O/P7-A10/, needle-like/JJ/I-NP/I-PNP/O/P7-A10/needle-like fibers/NNS/I-NP/I-PNP/O/P7-A10/fiber such/JJ/I-PP/B-PNP/O/P10/such as/IN/I-PP/I-PNP/O/P10/as crocidolite/NN/I-NP/I-PNP/O/P10/crocidolite that/IN/I-SBAR/O/O/O/that */SYM/O/O/O/O/* T*-1/NN/I-NP/O/NP-OBJ;NP-SBJ-3/O/T*-1 are/VBP/I-VP/O/VP-3/A2-A3/be classified/VBN/I-VP/O/VP-3/A2-A3/classify */SYM/O/O/O/O/* -/:/O/O/O/O/- 5/CD/I-NP/O/O/O/5 as/IN/I-PP/B-PNP/O/P2/as amphobiles/NNS/I-NP/I-PNP/O/P2/amphobile ,/,/O/O/O/O/, according/VBG/I-PP/B-PNP/O/P3/accord to/TO/B-PP/I-PNP/O/P3/to Brooke/NNP/I-NP/I-PNP/O/P3/Brooke T./NNP/I-NP/I-PNP/O/P3/T. Mossman/NNP/I-NP/I-PNP/O/P3/Mossman ,/,/O/O/O/O/, a/DT/I-NP/O/O/A4-A5/a professor/NN/I-NP/O/O/A4-A5/professor of/IN/I-PP/B-PNP/O/P5/of pathlogy/NN/I-NP/I-PNP/O/P5/pathlogy at/IN/I-PP/B-PNP/O/P4/at the/DT/I-NP/I-PNP/O/P4-A8/the University/NNP/I-NP/I-PNP/O/P4-A8/University of/IN/I-PP/B-PNP/O/P8/of Vermont/NNP/I-NP/I-PNP/O/P8-A9/Vermont College/NNP/I-NP/I-PNP/O/P8-A9/College of/IN/I-PP/B-PNP/O/P9/of Medicine/NNP/I-NP/I-PNP/O/P9/Medicine ././O/O/O/O/.\"\n"
       ]
      }
     ],
     "prompt_number": 40
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#print type(s.tokens[0])\n",
      "print [(i, v) for i, v in enumerate(s.tags)]\n",
      "print \n",
      "print [(s.tags[i], v) for i, v in enumerate(s.tags)]\n",
      "print\n",
      "pprint.pprint( [[ dict([(s.tags[i], v) for i, v in enumerate(t)]) for t in sent] for sent in s.tokens] )\n",
      "print\n",
      "#print [ [(s.tags[i], v) for i, v in enumerate(t)] for t in s.tokens]\n",
      "#[s.tags(i) for i, v in enumerate(s.tokens[0])]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "[(0, 'word'), (1, 'part-of-speech'), (2, 'chunk'), (3, 'preposition'), (4, 'relation'), (5, 'anchor'), (6, 'lemma')]\n",
        "\n",
        "[('word', 'word'), ('part-of-speech', 'part-of-speech'), ('chunk', 'chunk'), ('preposition', 'preposition'), ('relation', 'relation'), ('anchor', 'anchor'), ('lemma', 'lemma')]\n",
        "\n",
        "[[{'anchor': u'O',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'the',\n",
        "   'part-of-speech': u'DT',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'NP-SBJ-1',\n",
        "   'word': u'The'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'U.S.',\n",
        "   'part-of-speech': u'NNP',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'NP-SBJ-1',\n",
        "   'word': u'U.S.'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'I-VP',\n",
        "   'lemma': u'be',\n",
        "   'part-of-speech': u'VBZ',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'VP-1',\n",
        "   'word': u'is'},\n",
        "  {'anchor': u'A1',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'one',\n",
        "   'part-of-speech': u'CD',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'NP-PRD-1',\n",
        "   'word': u'one'},\n",
        "  {'anchor': u'P1',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'of',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'B-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'of'},\n",
        "  {'anchor': u'P1',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'the',\n",
        "   'part-of-speech': u'DT',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'the'},\n",
        "  {'anchor': u'P1',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'few',\n",
        "   'part-of-speech': u'JJ',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'few'},\n",
        "  {'anchor': u'P1',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'industrialize',\n",
        "   'part-of-speech': u'VBN',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'industrialized'},\n",
        "  {'anchor': u'P1',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'nation',\n",
        "   'part-of-speech': u'NNS',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'nations'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'B-NP',\n",
        "   'lemma': u'that',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u'that'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'O',\n",
        "   'lemma': u'*',\n",
        "   'part-of-speech': u'SYM',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u'*'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'T*-7',\n",
        "   'part-of-speech': u'NN',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'NP-SBJ-2',\n",
        "   'word': u'T*-7'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'I-VP',\n",
        "   'lemma': u'do',\n",
        "   'part-of-speech': u'VBZ',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'VP-2',\n",
        "   'word': u'does'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'I-VP',\n",
        "   'lemma': u\"n't\",\n",
        "   'part-of-speech': u'RB',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'VP-2',\n",
        "   'word': u\"n't\"},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'I-VP',\n",
        "   'lemma': u'have',\n",
        "   'part-of-speech': u'VB',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'VP-2',\n",
        "   'word': u'have'},\n",
        "  {'anchor': u'A6-A7',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'a',\n",
        "   'part-of-speech': u'DT',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'NP-OBJ-2',\n",
        "   'word': u'a'},\n",
        "  {'anchor': u'A6-A7',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'high',\n",
        "   'part-of-speech': u'JJR',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'NP-OBJ-2',\n",
        "   'word': u'higher'},\n",
        "  {'anchor': u'A6-A7',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'standard',\n",
        "   'part-of-speech': u'NN',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'NP-OBJ-2',\n",
        "   'word': u'standard'},\n",
        "  {'anchor': u'P6',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'of',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'B-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'of'},\n",
        "  {'anchor': u'P6',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'regulation',\n",
        "   'part-of-speech': u'NN',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'regulation'},\n",
        "  {'anchor': u'P7',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'for',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'B-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'for'},\n",
        "  {'anchor': u'P7-A10',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'the',\n",
        "   'part-of-speech': u'DT',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'the'},\n",
        "  {'anchor': u'P7-A10',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'smooth',\n",
        "   'part-of-speech': u'JJ',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'smooth'},\n",
        "  {'anchor': u'P7-A10',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u',',\n",
        "   'part-of-speech': u',',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u','},\n",
        "  {'anchor': u'P7-A10',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'needle-like',\n",
        "   'part-of-speech': u'JJ',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'needle-like'},\n",
        "  {'anchor': u'P7-A10',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'fiber',\n",
        "   'part-of-speech': u'NNS',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'fibers'},\n",
        "  {'anchor': u'P10',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'such',\n",
        "   'part-of-speech': u'JJ',\n",
        "   'preposition': u'B-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'such'},\n",
        "  {'anchor': u'P10',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'as',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'as'},\n",
        "  {'anchor': u'P10',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'crocidolite',\n",
        "   'part-of-speech': u'NN',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'crocidolite'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'I-SBAR',\n",
        "   'lemma': u'that',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u'that'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'O',\n",
        "   'lemma': u'*',\n",
        "   'part-of-speech': u'SYM',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u'*'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'T*-1',\n",
        "   'part-of-speech': u'NN',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'NP-OBJ;NP-SBJ-3',\n",
        "   'word': u'T*-1'},\n",
        "  {'anchor': u'A2-A3',\n",
        "   'chunk': u'I-VP',\n",
        "   'lemma': u'be',\n",
        "   'part-of-speech': u'VBP',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'VP-3',\n",
        "   'word': u'are'},\n",
        "  {'anchor': u'A2-A3',\n",
        "   'chunk': u'I-VP',\n",
        "   'lemma': u'classify',\n",
        "   'part-of-speech': u'VBN',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'VP-3',\n",
        "   'word': u'classified'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'O',\n",
        "   'lemma': u'*',\n",
        "   'part-of-speech': u'SYM',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u'*'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'O',\n",
        "   'lemma': u'-',\n",
        "   'part-of-speech': u':',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u'-'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'5',\n",
        "   'part-of-speech': u'CD',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u'5'},\n",
        "  {'anchor': u'P2',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'as',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'B-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'as'},\n",
        "  {'anchor': u'P2',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'amphobile',\n",
        "   'part-of-speech': u'NNS',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'amphobiles'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'O',\n",
        "   'lemma': u',',\n",
        "   'part-of-speech': u',',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u','},\n",
        "  {'anchor': u'P3',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'accord',\n",
        "   'part-of-speech': u'VBG',\n",
        "   'preposition': u'B-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'according'},\n",
        "  {'anchor': u'P3',\n",
        "   'chunk': u'B-PP',\n",
        "   'lemma': u'to',\n",
        "   'part-of-speech': u'TO',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'to'},\n",
        "  {'anchor': u'P3',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'Brooke',\n",
        "   'part-of-speech': u'NNP',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'Brooke'},\n",
        "  {'anchor': u'P3',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'T.',\n",
        "   'part-of-speech': u'NNP',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'T.'},\n",
        "  {'anchor': u'P3',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'Mossman',\n",
        "   'part-of-speech': u'NNP',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'Mossman'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'O',\n",
        "   'lemma': u',',\n",
        "   'part-of-speech': u',',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u','},\n",
        "  {'anchor': u'A4-A5',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'a',\n",
        "   'part-of-speech': u'DT',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u'a'},\n",
        "  {'anchor': u'A4-A5',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'professor',\n",
        "   'part-of-speech': u'NN',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u'professor'},\n",
        "  {'anchor': u'P5',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'of',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'B-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'of'},\n",
        "  {'anchor': u'P5',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'pathlogy',\n",
        "   'part-of-speech': u'NN',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'pathlogy'},\n",
        "  {'anchor': u'P4',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'at',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'B-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'at'},\n",
        "  {'anchor': u'P4-A8',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'the',\n",
        "   'part-of-speech': u'DT',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'the'},\n",
        "  {'anchor': u'P4-A8',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'University',\n",
        "   'part-of-speech': u'NNP',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'University'},\n",
        "  {'anchor': u'P8',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'of',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'B-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'of'},\n",
        "  {'anchor': u'P8-A9',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'Vermont',\n",
        "   'part-of-speech': u'NNP',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'Vermont'},\n",
        "  {'anchor': u'P8-A9',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'College',\n",
        "   'part-of-speech': u'NNP',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'College'},\n",
        "  {'anchor': u'P9',\n",
        "   'chunk': u'I-PP',\n",
        "   'lemma': u'of',\n",
        "   'part-of-speech': u'IN',\n",
        "   'preposition': u'B-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'of'},\n",
        "  {'anchor': u'P9',\n",
        "   'chunk': u'I-NP',\n",
        "   'lemma': u'Medicine',\n",
        "   'part-of-speech': u'NNP',\n",
        "   'preposition': u'I-PNP',\n",
        "   'relation': u'O',\n",
        "   'word': u'Medicine'},\n",
        "  {'anchor': u'O',\n",
        "   'chunk': u'O',\n",
        "   'lemma': u'.',\n",
        "   'part-of-speech': u'.',\n",
        "   'preposition': u'O',\n",
        "   'relation': u'O',\n",
        "   'word': u'.'}]]\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 41
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sent =\"The quick brown fox jumped over the lazy dog.  All quiet on the western front.\"\n",
      "#sent =\"The quick brown fox jumped over the lazy dog.\"\n",
      "#sent = \"The U.S. is one of the few industrialized nations that *T*-7 doesn't have a higher standard of regulation for the smooth, needle-like fibers such as crocidolite that *T*-1 are classified *-5 as amphobiles, according to Brooke T. Mossman, a professor of pathlogy at the University of Vermont College of Medicine.\"\n",
      "s = mbsp_parsetree(sent)\n",
      "print type(s)\n",
      "print\n",
      "print s\n",
      "print\n",
      "print len(s)\n",
      "print\n",
      "print s[0]\n",
      "print\n",
      "print s[1]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "<class 'pattern.text.tree.Text'>\n",
        "\n",
        "[Sentence('The/DT/B-NP/O/NP-SBJ-1/O/the quick/JJ/I-NP/O/NP-SBJ-1/O/quick brown/JJ/I-NP/O/NP-SBJ-1/O/brown fox/NN/I-NP/O/NP-SBJ-1/O/fox jumped/VBD/B-VP/O/VP-1/A1/jump over/IN/B-PP/B-PNP/PP-DIR-1/P1/over the/DT/B-NP/I-PNP/NP-DIR-1/P1/the lazy/JJ/I-NP/I-PNP/NP-DIR-1/P1/lazy dog/NN/I-NP/I-PNP/NP-DIR-1/P1/dog ././O/O/O/O/.'), Sentence('All/DT/B-NP/O/O/A1/all quiet/NN/I-NP/O/O/A1/quiet on/IN/B-PP/B-PNP/O/P1/on the/DT/B-NP/I-PNP/O/P1/the western/JJ/I-NP/I-PNP/O/P1/western front/NN/I-NP/I-PNP/O/P1/front ././O/O/O/O/.')]\n",
        "\n",
        "2\n",
        "\n",
        "Sentence('The/DT/B-NP/O/NP-SBJ-1/O/the quick/JJ/I-NP/O/NP-SBJ-1/O/quick brown/JJ/I-NP/O/NP-SBJ-1/O/brown fox/NN/I-NP/O/NP-SBJ-1/O/fox jumped/VBD/B-VP/O/VP-1/A1/jump over/IN/B-PP/B-PNP/PP-DIR-1/P1/over the/DT/B-NP/I-PNP/NP-DIR-1/P1/the lazy/JJ/I-NP/I-PNP/NP-DIR-1/P1/lazy dog/NN/I-NP/I-PNP/NP-DIR-1/P1/dog ././O/O/O/O/.')\n",
        "\n",
        "Sentence('All/DT/B-NP/O/O/A1/all quiet/NN/I-NP/O/O/A1/quiet on/IN/B-PP/B-PNP/O/P1/on the/DT/B-NP/I-PNP/O/P1/the western/JJ/I-NP/I-PNP/O/P1/western front/NN/I-NP/I-PNP/O/P1/front ././O/O/O/O/.')\n"
       ]
      }
     ],
     "prompt_number": 45
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sent = 'The quick brown fox jumped over the lazy dog.'\n",
      "s = mbsp_parsetree(sent)\n",
      "print s[0]\n",
      "print\n",
      "print \"chunks: \"\n",
      "print \"    \",s[0].chunks\n",
      "print\n",
      "print \"subjects: \", s[0].subjects\n",
      "print\n",
      "print \"objects: \", s[0].objects\n",
      "print\n",
      "print \"verbs: \", s[0].verbs\n",
      "print\n",
      "print \"relations: \", s[0].relations\n",
      "print\n",
      "print \"pnp: \", \n",
      "print s[0].pnp\n",
      "print\n",
      "print \"constituents: \"\n",
      "print \"   pnp=False: \", s[0].constituents(pnp=False)\n",
      "print\n",
      "print \"   pnp=True:  \", s[0].constituents(pnp=True)\n",
      "print \"-------------------------------------\"\n",
      "for c in s[0].chunks:\n",
      "    print \"chunk: \", c\n",
      "    print \"string: \", c.string\n",
      "    print \"words: \", c.words\n",
      "    print \"type: \", c.type\n",
      "    print \"role: \", c.role\n",
      "    print \"relation: \", c.relation\n",
      "    print \"relations: \", c.relations\n",
      "    print \"related: \", c.related\n",
      "    print \"subject: \", c.subject\n",
      "    print \"object: \", c.object\n",
      "    print \"---\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Sentence('The/DT/B-NP/O/NP-SBJ-1/O/the quick/JJ/I-NP/O/NP-SBJ-1/O/quick brown/JJ/I-NP/O/NP-SBJ-1/O/brown fox/NN/I-NP/O/NP-SBJ-1/O/fox jumped/VBD/B-VP/O/VP-1/A1/jump over/IN/B-PP/B-PNP/PP-DIR-1/P1/over the/DT/B-NP/I-PNP/NP-DIR-1/P1/the lazy/JJ/I-NP/I-PNP/NP-DIR-1/P1/lazy dog/NN/I-NP/I-PNP/NP-DIR-1/P1/dog ././O/O/O/O/.')\n",
        "\n",
        "chunks: \n",
        "     [Chunk('The quick brown fox/NP-SBJ-1'), Chunk('jumped/VP-1'), Chunk('over/PP-DIR-1'), Chunk('the lazy dog/NP-DIR-1')]\n",
        "\n",
        "subjects:  [Chunk('The quick brown fox/NP-SBJ-1')]\n",
        "\n",
        "objects:  []\n",
        "\n",
        "verbs:  [Chunk('jumped/VP-1')]\n",
        "\n",
        "relations:  {'SBJ': {1: Chunk('The quick brown fox/NP-SBJ-1')}, 'VP': {1: Chunk('jumped/VP-1')}, 'OBJ': {}}\n",
        "\n",
        "pnp:  [Chunk('over the lazy dog/PNP')]\n",
        "\n",
        "constituents: \n",
        "   pnp=False:  [Chunk('The quick brown fox/NP-SBJ-1'), Chunk('jumped/VP-1'), Chunk('over/PP-DIR-1'), Chunk('the lazy dog/NP-DIR-1'), Word(u'./.')]\n",
        "\n",
        "   pnp=True:   [Chunk('The quick brown fox/NP-SBJ-1'), Chunk('jumped/VP-1'), Chunk('over the lazy dog/PNP'), Word(u'./.')]\n",
        "-------------------------------------\n",
        "chunk:  Chunk('The quick brown fox/NP-SBJ-1')\n",
        "string:  The quick brown fox\n",
        "words:  [Word(u'The/DT'), Word(u'quick/JJ'), Word(u'brown/JJ'), Word(u'fox/NN')]\n",
        "type:  NP\n",
        "role:  SBJ\n",
        "relation:  1\n",
        "relations:  [(1, u'SBJ')]\n",
        "related:  [Chunk('jumped/VP-1'), Chunk('over/PP-DIR-1'), Chunk('the lazy dog/NP-DIR-1')]\n",
        "subject:  None\n",
        "object:  None\n",
        "---\n",
        "chunk:  Chunk('jumped/VP-1')\n",
        "string:  jumped\n",
        "words:  [Word(u'jumped/VBD')]\n",
        "type:  VP\n",
        "role:  None\n",
        "relation:  1\n",
        "relations:  [(1, None)]\n",
        "related:  [Chunk('The quick brown fox/NP-SBJ-1'), Chunk('over/PP-DIR-1'), Chunk('the lazy dog/NP-DIR-1')]\n",
        "subject:  Chunk('The quick brown fox/NP-SBJ-1')\n",
        "object:  None\n",
        "---\n",
        "chunk:  Chunk('over/PP-DIR-1')\n",
        "string:  over\n",
        "words:  [Word(u'over/IN')]\n",
        "type:  PP\n",
        "role:  DIR\n",
        "relation:  1\n",
        "relations:  [(1, u'DIR')]\n",
        "related:  [Chunk('The quick brown fox/NP-SBJ-1'), Chunk('jumped/VP-1'), Chunk('the lazy dog/NP-DIR-1')]\n",
        "subject:  Chunk('The quick brown fox/NP-SBJ-1')\n",
        "object:  None\n",
        "---\n",
        "chunk:  Chunk('the lazy dog/NP-DIR-1')\n",
        "string:  the lazy dog\n",
        "words:  [Word(u'the/DT'), Word(u'lazy/JJ'), Word(u'dog/NN')]\n",
        "type:  NP\n",
        "role:  DIR\n",
        "relation:  1\n",
        "relations:  [(1, u'DIR')]\n",
        "related:  [Chunk('The quick brown fox/NP-SBJ-1'), Chunk('jumped/VP-1'), Chunk('over/PP-DIR-1')]\n",
        "subject:  Chunk('The quick brown fox/NP-SBJ-1')\n",
        "object:  None\n",
        "---\n"
       ]
      }
     ],
     "prompt_number": 46
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#s = 'The quick brown fox jumped over the lazy dog.'\n",
      "s = \"You aren't nice\"\n",
      "s = \"I dislike you\"\n",
      "#s = \"I think you are nice.\"\n",
      "#s = \"I know you are nice.\"\n",
      "s = mbsp_parsetree(s)\n",
      "print \"sentiment: \",\n",
      "#print pattern.en.sentiment(\"the quick brown fox jumped over the lazy dog\")\n",
      "print pattern.en.sentiment(s[0])\n",
      "print \"polarity: \",\n",
      "print pattern.en.polarity(s[0])\n",
      "print \"subjectivity: \",\n",
      "print pattern.en.subjectivity(s[0])\n",
      "print \"positive: \",\n",
      "print pattern.en.positive(s[0])\n",
      "print \"mood: \",\n",
      "print pattern.en.mood(s[0])\n",
      "print \"mood-p: \",\n",
      "print pattern.en.mood(s[0], predictive=False), ' ',\n",
      "print pattern.en.mood(s[0], predictive=True)\n",
      "print \"modality: \",\n",
      "print pattern.en.modality(s[0])\n",
      "print \"negated: \",\n",
      "print pattern.en.negated(s[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "sentiment:  (0.0, 0.0)\n",
        "polarity:  0.0\n",
        "subjectivity:  0.0\n",
        "positive:  False\n",
        "mood:  indicative\n",
        "mood-p:  indicative   indicative\n",
        "modality:  1.0\n",
        "negated:  False\n"
       ]
      }
     ],
     "prompt_number": 47
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}