Tweet Clustering scikit-learn-Experimental-DBSCAN.ipynb

{
 "metadata": {
  "name": "Tweet Clustering scikit-learn-Experimental-DBSCAN"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "To Do:\n",
      "\n",
      "Support family of solvers\n",
      "    DBScan\n",
      "    Hierarchical\n",
      "    OPTICS?\n",
      "    \n",
      "    K-MEANS - DONE\n",
      "    Mini-batch K-MEANS - DONE\n",
      "    \n",
      "Feature Extractions and Feature Selection\n",
      "    TF-IDF\n",
      "\n",
      "Evaluation:\n",
      "    Silhouette - DONE\n",
      "\n",
      "Reporting\n",
      "    Include cluster centroid for each group-- Done\n",
      "    \n",
      "other:  Can we use centroid values as basis to remove certain features with low centroid value\n",
      "-- perhaps in a training phase.\n",
      "-- may be able to accomplish this by zeroing out value in matrix, then re-running test."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import sys\n",
      "import cPickle as pickle\n",
      "import pprint\n",
      "import collections\n",
      "#import nltk\n",
      "#from nltk.stem import WordNetLemmatizer\n",
      "sys.path.append('/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/')\n",
      "sys.path.append('/Users/doug/SW_Dev/NLTK_Experiments/')\n",
      "\n",
      "from sklearn.feature_extraction import DictVectorizer\n",
      "from sklearn.feature_extraction.text import TfidfTransformer\n",
      "from sklearn.cluster import KMeans, MiniBatchKMeans, DBSCAN, Ward\n",
      "from sklearn import metrics\n",
      "from sklearn.metrics import pairwise_distances\n",
      "import math\n",
      "import numpy as np\n",
      "\n",
      "#import cmu_tweet_word_clusters\n",
      "import my_feature_ex as fx"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "skipped:  111010100010\t\t212\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 1
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Tweet Clustering using scikit-learn library"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#help(DBSCAN)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def tokens2dict (tokens, exclude=[]):\n",
      "    \"\"\"used to convert features to DictVectorizer compatible format\"\"\"\n",
      "    result = {}\n",
      "    for t in tokens:\n",
      "        if t not in exclude:\n",
      "            result[t]=True\n",
      "    return result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 3
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def gen_feature_matrix(tweets,tfidf=False, exclude=[]):\n",
      "    \"\"\"Converts set of tweets with tokenized values into feature matrix\"\"\"\n",
      "    \n",
      "    vectorizer = DictVectorizer()\n",
      "    feat_matrix = vectorizer.fit_transform([tokens2dict(tweet['tokens'], exclude=exclude) for tweet in tweets])\n",
      "    #print feat_matrix.toarray()\n",
      "    #print vectorizer.get_feature_names()\n",
      "    \n",
      "    if tfidf:\n",
      "        feat_matrix = TfidfTransformer().fit_transform(feat_matrix)\n",
      "        \n",
      "    return feat_matrix, vectorizer.get_feature_names()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 4
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def clusterize_matrix(feat_matrix,solver='kmeans', batch=False,k=False):\n",
      "    \"\"\"Supported woldervs are kmeans and DBSCAN\"\"\"\n",
      "    \n",
      "    if solver == 'kmeans':\n",
      "        if not k:\n",
      "            # setting k based on rule of thumb:\n",
      "            # source: http://en.wikipedia.org/wiki/Determining_the_number_of_clusters_in_a_data_set#Rule_of_thumb\n",
      "            k = int(math.sqrt(feat_matrix.shape[0]/2))\n",
      "        if batch:\n",
      "            sol = MiniBatchKMeans(n_clusters=k, init='k-means++', init_size=3*k,verbose=False)\n",
      "        else:\n",
      "            sol = KMeans(n_clusters=k, init='k-means++', max_iter=100, n_init=1,\n",
      "                         verbose=False)\n",
      "    elif solver == 'DBSCAN':\n",
      "        sol = DBSCAN(eps=0.1)\n",
      "    elif solver == \"ward\":\n",
      "        sol = Ward(n_clusters=k)\n",
      "        \n",
      "    sol.fit(feat_matrix.toarray())\n",
      "    groupings = sol.labels_\n",
      "    if solver== 'kmeans':\n",
      "        cluster_centers = sol.cluster_centers_\n",
      "        sil_score = metrics.silhouette_score(feat_matrix, groupings, metric='euclidean')\n",
      "    else:\n",
      "        cluster_centers = False\n",
      "        sil_score = False\n",
      "    \n",
      "    #print 'score: {}'.format(sil_score)\n",
      "    return groupings, sil_score, cluster_centers"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 5
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def analyze_features(tweets,feat_matrix, groupings,cluster_centers,feature_names,threshold=0):\n",
      "\n",
      "    groups = collections.defaultdict(list)\n",
      "    for tweet_idx, cluster in enumerate(groupings):\n",
      "        groups[cluster].append(tweet_idx)\n",
      "    \n",
      "    samples = metrics.silhouette_samples(feat_matrix, groupings)\n",
      "    #feature_ratings = collections.defaultdict(lambda: {'min_cid':-1, 'min_coef':0, 'min_sil':-1, 'max_cid':-1, 'max_coef':0, 'max_sil':-1,})\n",
      "    feature_ratings = collections.defaultdict(lambda: {'cid':-1, 'coef':0, 'sil':-1})\n",
      "    \n",
      "    #feature_ratings = {}\n",
      "        \n",
      "    #print 'samples len: ', len(samples)\n",
      "   \n",
      "    for cid, cluster in groups.items():\n",
      "        if len(cluster) > 1:    #ignore single entry clusters\n",
      "            cluster_sil = np.mean([samples[i] for i in cluster])\n",
      "            features = [(feature_names[fnum], coef) for fnum, coef in enumerate(cluster_centers[cid])]\n",
      "            for fname, coef in features:\n",
      "                if coef > feature_ratings[fname]['coef']:\n",
      "                    feature_ratings[fname]['cid'] = cid\n",
      "                    feature_ratings[fname]['coef'] = coef\n",
      "                    feature_ratings[fname]['sil'] = cluster_sil\n",
      "        elif len(cluster) == 1:\n",
      "            features = [(feature_names[fnum], -1) for fnum, coef in enumerate(cluster_centers[cid])]\n",
      "            for fname, coef in features:\n",
      "                if coef > feature_ratings[fname]['coef']:\n",
      "                    feature_ratings[fname]['cid'] = cid\n",
      "                    feature_ratings[fname]['coef'] = coef\n",
      "\n",
      "    #result = sorted([(fname, d['clu'], d['coef'], d['sil']) for fname, d in feature_ratings.items()],\n",
      "    #                    key=lambda tup: tup[2], reverse=True)            \n",
      "    \n",
      "    return feature_ratings"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 6
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def display_clusters(tweets,feat_matrix, groupings,cluster_centers,feature_names,report=True,annotate_prefix=\"\",quality=False,threshold=0):\n",
      "\n",
      "    groups = collections.defaultdict(list)\n",
      "    for tweet_idx, cluster in enumerate(groupings):\n",
      "        groups[cluster].append(tweet_idx)\n",
      "    \n",
      "    #print 'groupings max: ', max(groupings)\n",
      "    \n",
      "    if quality:\n",
      "        samples = metrics.silhouette_samples(feat_matrix, groupings)\n",
      "        \n",
      "    #print 'samples len: ', len(samples)\n",
      "   \n",
      "    skipped = 0\n",
      "    clust_cnt = 0\n",
      "    for cid, cluster in groups.items():\n",
      "        if len(cluster) == 1:\n",
      "            skipped += 1\n",
      "        else:\n",
      "            clust_cnt += 1\n",
      "            if annotate_prefix:\n",
      "                for i in cluster:\n",
      "                    tweet_set[i]['composite'].append('{}_{}'.format(annotate_prefix, clust_cnt))\n",
      "            elif report:\n",
      "                print \"=========\"\n",
      "                \n",
      "                if quality:\n",
      "                    cluster_sil = np.mean([samples[i] for i in cluster])\n",
      "                    if cluster_sil <= threshold:\n",
      "                        skipped += 1   # correct cluster counts\n",
      "                        clust_cnt -= 1\n",
      "                        continue\n",
      "                    print 'Cluster silhouette mean:{:3.2f}'.format(cluster_sil) \n",
      "                    f = sorted([(feature, coef) for feature, coef in enumerate(cluster_centers[cid]) if coef > 0.015],\n",
      "                                 key=lambda pair: pair[1], reverse=True)\n",
      "                    print 'Features:', ' '.join(['{}:{:3.2f} '.format(feature_names[feature],coef) for feature, coef in f])\n",
      "                    print\n",
      "                    \n",
      "                for i in cluster:\n",
      "                    if quality:\n",
      "                        if samples[i] > threshold:\n",
      "                            print \"[{}] {:3.2f} -> \".format(i, samples[i]),tweet_set[i][\"text\"]\n",
      "                    else:\n",
      "                        print \"[{}]-> \".format(i),tweet_set[i][\"text\"]\n",
      "            \n",
      "    print \"clusters: {} skipped {} ({}%) tweets our of {} total tweets\".format(clust_cnt, skipped, 100.0*skipped/len(tweets),len(tweets), )\n",
      "    return {'clusters':clust_cnt, 'skipped':skipped, 'total':len(tweets)}"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 7
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def scan_k(feat_matrix,batch, incr, kmin, kmax):\n",
      "        \n",
      "    kmax = min(kmax, feat_matrix.shape[0])\n",
      "    kmin = max(kmin, 8)\n",
      "   \n",
      "    k = kmax\n",
      "    max_score = -1\n",
      "    max_score_k = 0\n",
      "    #coarse scan\n",
      "    while k > kmin :\n",
      "        #print k\n",
      "        print('.'),\n",
      "        groupings, sil_score, ignored = clusterize_matrix(feat_matrix,k=k,batch=batch)\n",
      "        #print sil_score\n",
      "        if sil_score > max_score:\n",
      "                max_score = sil_score\n",
      "                max_score_k = k\n",
      "        k = int(k/incr)\n",
      "    print    \n",
      "    return max_score_k"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def find_best_k(feat_matrix,batch=False, kmin=False, kmax=False, incr=False, refine=False):\n",
      "    sqrt2 = math.sqrt(2.)\n",
      "\n",
      "    if not incr:\n",
      "        incr = math.sqrt(2.)\n",
      "    if not kmax:\n",
      "        kmax = feat_matrix.shape[0]\n",
      "    if not kmin:\n",
      "        kmin = 8\n",
      "        \n",
      "    print(\"Initial Scan for k \"),\n",
      "    max_score_k = scan_k(feat_matrix,batch, incr, kmin, kmax)\n",
      " \n",
      "    if refine:\n",
      "        print(\"Refinement phase \"),\n",
      "    \n",
      "        incr2 =  math.sqrt(math.sqrt(incr))\n",
      "        min_k = int(max_score_k / incr)\n",
      "        max_k = int(max_score_k * incr)\n",
      "\n",
      "        max_score_k = scan_k(feat_matrix,batch, incr2, min_k, max_k)\n",
      "     \n",
      "    print \"k=\",max_score_k\n",
      "    return max_score_k"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 9
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# use k=\"auto\" for automatic discover of optimal k value\n",
      "\n",
      "def perform_trial_clustering(tweets,tfidf=False,batch=False,k=False,refine=True,report=True,annotate_prefix=\"\",quality=True,threshold=0):\n",
      "\n",
      "    feat_matrix, fnames = gen_feature_matrix(tweets,tfidf=tfidf)\n",
      "    if k == \"auto\":\n",
      "        k = find_best_k(feat_matrix, refine=refine, batch=batch)\n",
      "    groupings, sil_score, cluster_centers = clusterize_matrix(feat_matrix,k=k,batch=batch)\n",
      "    result = analyze_features(tweets,feat_matrix, groupings,cluster_centers, fnames)\n",
      "    return result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 10
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# use k=\"auto\" for automatic discover of optimal k value\n",
      "\n",
      "def perform_tweet_clustering(tweets,tfidf=False,solver='kmeans', batch=False,k=False,refine=True,report=True,annotate_prefix=\"\",quality=True,threshold=0, exclude=[]):\n",
      "\n",
      "    feat_matrix, fnames = gen_feature_matrix(tweets,tfidf=tfidf,exclude=exclude)\n",
      "    if k == \"auto\":\n",
      "        k = find_best_k(feat_matrix, refine=refine, batch=batch)\n",
      "    groupings, sil_score, cluster_centers = clusterize_matrix(feat_matrix,solver=solver,k=k,batch=batch)\n",
      "    result = display_clusters(tweets,feat_matrix, groupings,cluster_centers, fnames, report=True,annotate_prefix=\"\",quality=quality,threshold=threshold)\n",
      "    result['sil_score'] = sil_score\n",
      "    return result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 11
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# load previously tokenized/clasified  tweet Corpus\n",
      "pinput = open('ucla_tweets.pkl', 'rb')\n",
      "UCLA_tweets = pickle.load(pinput)\n",
      "pinput.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 12
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#select subset of tweets for experimentation\n",
      "\n",
      "MAX_TWEETS = 4000   #subset Corpus for now to improve Clustering run time\n",
      "tweet_set = [{'text':t['text'], 'pos':t['pos'], 'raw_tokens':t['tokens']} for t in UCLA_tweets.values()[0:MAX_TWEETS]]\n",
      "\n",
      "#sample tweet\n",
      "#pprint.pprint(tweet_set[0])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 16
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#test code, delete later\n",
      "fx.reset_features(tweet_set)\n",
      "fx.extract_lemmatize_tokens(tweet_set, exclude=['ucla', '#ucla', 'rt'])\n",
      "print \"token extracton done\"\n",
      "#terms = perform_trial_clustering(tweet_set,solver='DBSCAN', quality=False,batch=False, tfidf=False, threshold=0.0, k=500)\n",
      "#result = perform_tweet_clustering(tweet_set,solver='DBSCAN',quality=False,batch=False, tfidf=True, threshold=0.0, k=300)\n",
      "result = perform_tweet_clustering(tweet_set,solver='ward',quality=False,batch=False, tfidf=False, threshold=0.0, k=300)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#sorted_terms = sorted([(fname, d['cid'], d['coef'], d['sil']) for fname, d in terms.items()],\n",
      "#                        key=lambda tup: tup[2], reverse=True) \n",
      "#\n",
      "#exclude = [d[0] for d in sorted_terms if d[2] < 0.015]\n",
      "#result = perform_tweet_clustering(tweet_set,quality=True,batch=False, tfidf=True, threshold=0.0, k=500, exclude=exclude)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 19
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#result = perform_tweet_clustering(tweet_set,quality=True,batch=False, tfidf=True, threshold=0.0, k=500, exclude=exclude)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 20
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "{'total': 2000, 'clusters': 323, 'sil_score': 0.17622666868761888, 'skipped': 177}\n"
       ]
      }
     ],
     "prompt_number": 17
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Now try using bigrams"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#test code, delete later\n",
      "fx.reset_features(tweet_set)\n",
      "fx.extract_lemmatize_tokens(tweet_set, exclude=['ucla', '#ucla', 'rt'])\n",
      "fx.extract_lemmatize_bigrams(tweet_set, exclude=['ucla', '#ucla', 'rt'])\n",
      "print \"token extracton done\"\n",
      "terms = perform_trial_clustering(tweet_set,quality=True,batch=False, tfidf=True, threshold=0.0, k=500)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sorted_terms = sorted([(fname, d['cid'], d['coef'], d['sil']) for fname, d in terms.items()],\n",
      "                        key=lambda tup: tup[2], reverse=True) \n",
      "\n",
      "exclude = [d[0] for d in sorted_terms if d[2] < 0.015]\n",
      "result = perform_tweet_clustering(tweet_set,quality=True,batch=False, tfidf=True, threshold=0.0, k=500, exclude=exclude)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print result"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Feature extraction routines:\n",
      "- extract_tokens()            -- simple set of tokens, ignoring stop words and non-word tokens\n",
      "- extract_lemmatize_tokens()  -- extends above using lemmatized tokens in place of actual tokens\n",
      "- extract_bigrams()           -- bigrams of simple tokens using above filtering rules\n",
      "- extract_lemmatize_bigrams() -- extends above using lemmatized tokens in place of actual tokens"
     ]
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "Compare clustering results for various feature extraction approaches"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "fx.reset_features(tweet_set)\n",
      "fx.extract_tokens(tweet_set)\n",
      "#perform_tweet_clustering(tweet_set)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 8
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#fx.reset_features(tweet_set)\n",
      "#fx.extract_lemmatize_tokens(tweet_set)\n",
      "#perform_tweet_clustering(tweet_set, report=True, batch=False, tfidf=True, quality=True)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "fx.reset_features(tweet_set)\n",
      "fx.extract_bigrams(tweet_set)\n",
      "#perform_tweet_clustering(tweet_set)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "fx.reset_features(tweet_set)\n",
      "fx.extract_lemmatize_bigrams(tweet_set)\n",
      "fx.extract_lemmatize_tokens(tweet_set, filter='nominal', exclude=['ucla', '#ucla'])\n",
      "fx.extract_tokens(tweet_set, filter='hashtag', exclude=['ucla', '#ucla'])\n",
      "#perform_tweet_clustering(tweet_set, threshold=0.1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 36
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#try combining multiple types of features\n",
      "fx.reset_features(tweet_set)\n",
      "fx.extract_lemmatize_tokens(tweet_set)\n",
      "fx.extract_lemmatize_bigrams(tweet_set)\n",
      "#perform_tweet_clustering(tweet_set)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#try combining multiple types of features\n",
      "fx.reset_features(tweet_set)\n",
      "fx.extract_group_tokens(tweet_set)\n",
      "fx.extract_group_bigrams(tweet_set)\n",
      "#perform_tweet_clustering(tweet_set)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "fx.reset_features(tweet_set)\n",
      "#fx.extract_tokens(tweet_set, filter='hashtag', exclude=['ucla', '#ucla'])\n",
      "fx.extract_tokens(tweet_set, filter='nominal', exclude=['ucla', '#ucla'])\n",
      "fx.extract_bigrams(tweet_set)\n",
      "fx.extract_lemmatize_bigrams(tweet_set)\n",
      "#perform_tweet_clustering(tweet_set, k='200', tfidf=False)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 21
    },
    {
     "cell_type": "raw",
     "metadata": {},
     "source": [
      "This section explores composite clustering, where final clustering based on a set of other clustered results.  Use groupings from other clusterings as features for aggregate clustering."
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# My wrapper for above clustering routine\n",
      "def reset_composite(tweets):\n",
      "    \"resets the composite property\"\n",
      "    for tweet in tweets:\n",
      "        tweet['composite'] = [] \n",
      "    \n",
      "def promote_composite(tweets):\n",
      "    \"moves composite property to tokens property\"\n",
      "    for tweet in tweets:\n",
      "        if tweet['composite']:\n",
      "            tweet['tokens'] = tweet['composite']\n",
      "        else:\n",
      "            tweet['tokens'] = ['**empty**']"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "reset_composite(tweet_set)\n",
      "\n",
      "fx.reset_features(tweet_set)\n",
      "fx.extract_tokens(tweet_set, filter='hashtag', exclude=['ucla', '#ucla'])\n",
      "#annotate_clustering_results(tweet_set, 'hash')\n",
      "\n",
      "fx.reset_features(tweet_set)\n",
      "fx.extract_lemmatize_tokens(tweet_set, filter='nominal', exclude=['ucla', '#ucla'])\n",
      "#annotate_clustering_results(tweet_set, 'nom')\n",
      "\n",
      "#fx.reset_features(tweet_set)\n",
      "#fx.extract_lemmatize_tokens(tweet_set, filter='noun', exclude=['ucla', '#ucla'])\n",
      "#annotate_clustering_results(tweet_set, 'noun')\n",
      "\n",
      "#fx.reset_features(tweet_set)\n",
      "#fx.extract_bigrams(tweet_set)\n",
      "#annotate_clustering_results(tweet_set, 'bi')\n",
      "\n",
      "fx.reset_features(tweet_set)\n",
      "fx.extract_lemmatize_bigrams(tweet_set)\n",
      "#annotate_clustering_results(tweet_set, 'bil')\n",
      "\n",
      "fx.reset_features(tweet_set)\n",
      "promote_composite(tweet_set)\n",
      "#perform_tweet_clustering(tweet_set)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": "*"
    }
   ],
   "metadata": {}
  }
 ]
}