doc_clustering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "=======================================\n",
    "Clustering text documents using k-means\n",
    "=======================================\n",
    "\n",
    "This is an example showing how the scikit-learn can be used to cluster\n",
    "documents by topics using a bag-of-words approach. This example uses\n",
    "a scipy.sparse matrix to store the features instead of standard numpy arrays.\n",
    "\n",
    "Two feature extraction methods can be used in this example:\n",
    "\n",
    "  - TfidfVectorizer uses a in-memory vocabulary (a python dict) to map the most\n",
    "    frequent words to features indices and hence compute a word occurrence\n",
    "    frequency (sparse) matrix. The word frequencies are then reweighted using\n",
    "    the Inverse Document Frequency (IDF) vector collected feature-wise over\n",
    "    the corpus.\n",
    "\n",
    "  - HashingVectorizer hashes word occurrences to a fixed dimensional space,\n",
    "    possibly with collisions. The word count vectors are then normalized to\n",
    "    each have l2-norm equal to one (projected to the euclidean unit-ball) which\n",
    "    seems to be important for k-means to work in high dimensional space.\n",
    "\n",
    "    HashingVectorizer does not provide IDF weighting as this is a stateless\n",
    "    model (the fit method does nothing). When IDF weighting is needed it can\n",
    "    be added by pipelining its output to a TfidfTransformer instance.\n",
    "\n",
    "Two algorithms are demoed: ordinary k-means and its more scalable cousin\n",
    "minibatch k-means.\n",
    "\n",
    "It can be noted that k-means (and minibatch k-means) are very sensitive to\n",
    "feature scaling and that in this case the IDF weighting helps improve the\n",
    "quality of the clustering by quite a lot as measured against the \"ground truth\"\n",
    "provided by the class label assignments of the 20 newsgroups dataset.\n",
    "\n",
    "This improvement is not visible in the Silhouette Coefficient which is small\n",
    "for both as this measure seem to suffer from the phenomenon called\n",
    "\"Concentration of Measure\" or \"Curse of Dimensionality\" for high dimensional\n",
    "datasets such as text data. Other measures such as V-measure and Adjusted Rand\n",
    "Index are information theoretic based evaluation scores: as they are only based\n",
    "on cluster assignments rather than distances, hence not affected by the curse\n",
    "of dimensionality.\n",
    "\n",
    "Note: as k-means is optimizing a non-convex objective function, it will likely\n",
    "end up in a local optimum. Several runs with independent random init might be\n",
    "necessary to get a good convergence.\n",
    "\n",
    "\"\"\"\n",
    "\n",
    "# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>\n",
    "#         Lars Buitinck <L.J.Buitinck@uva.nl>\n",
    "# License: BSD 3 clause\n",
    "\n",
    "from __future__ import print_function\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import Normalizer\n",
    "from sklearn import metrics\n",
    "\n",
    "from sklearn.cluster import KMeans, MiniBatchKMeans\n",
    "\n",
    "import logging\n",
    "from optparse import OptionParser\n",
    "import sys\n",
    "from time import time\n",
    "import math\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=======================================\n",
      "Clustering text documents using k-means\n",
      "=======================================\n",
      "\n",
      "This is an example showing how the scikit-learn can be used to cluster\n",
      "documents by topics using a bag-of-words approach. This example uses\n",
      "a scipy.sparse matrix to store the features instead of standard numpy arrays.\n",
      "\n",
      "Two feature extraction methods can be used in this example:\n",
      "\n",
      "  - TfidfVectorizer uses a in-memory vocabulary (a python dict) to map the most\n",
      "    frequent words to features indices and hence compute a word occurrence\n",
      "    frequency (sparse) matrix. The word frequencies are then reweighted using\n",
      "    the Inverse Document Frequency (IDF) vector collected feature-wise over\n",
      "    the corpus.\n",
      "\n",
      "  - HashingVectorizer hashes word occurrences to a fixed dimensional space,\n",
      "    possibly with collisions. The word count vectors are then normalized to\n",
      "    each have l2-norm equal to one (projected to the euclidean unit-ball) which\n",
      "    seems to be important for k-means to work in high dimensional space.\n",
      "\n",
      "    HashingVectorizer does not provide IDF weighting as this is a stateless\n",
      "    model (the fit method does nothing). When IDF weighting is needed it can\n",
      "    be added by pipelining its output to a TfidfTransformer instance.\n",
      "\n",
      "Two algorithms are demoed: ordinary k-means and its more scalable cousin\n",
      "minibatch k-means.\n",
      "\n",
      "It can be noted that k-means (and minibatch k-means) are very sensitive to\n",
      "feature scaling and that in this case the IDF weighting helps improve the\n",
      "quality of the clustering by quite a lot as measured against the \"ground truth\"\n",
      "provided by the class label assignments of the 20 newsgroups dataset.\n",
      "\n",
      "This improvement is not visible in the Silhouette Coefficient which is small\n",
      "for both as this measure seem to suffer from the phenomenon called\n",
      "\"Concentration of Measure\" or \"Curse of Dimensionality\" for high dimensional\n",
      "datasets such as text data. Other measures such as V-measure and Adjusted Rand\n",
      "Index are information theoretic based evaluation scores: as they are only based\n",
      "on cluster assignments rather than distances, hence not affected by the curse\n",
      "of dimensionality.\n",
      "\n",
      "Note: as k-means is optimizing a non-convex objective function, it will likely\n",
      "end up in a local optimum. Several runs with independent random init might be\n",
      "necessary to get a good convergence.\n",
      "\n",
      "\n",
      "Usage: __main__.py [options]\n",
      "\n",
      "Options:\n",
      "  -h, --help            show this help message and exit\n",
      "  --lsa=N_COMPONENTS    Preprocess documents with latent semantic analysis.\n",
      "  --no-minibatch        Use ordinary k-means algorithm (in batch mode).\n",
      "  --no-idf              Disable Inverse Document Frequency feature weighting.\n",
      "  --use-hashing         Use a hashing feature vectorizer\n",
      "  --n-features=N_FEATURES\n",
      "                        Maximum number of features (dimensions) to extract\n",
      "                        from text.\n",
      "  --verbose             Print progress reports inside k-means algorithm.\n"
     ]
    }
   ],
   "source": [
    "# Display progress logs on stdout\n",
    "logging.basicConfig(level=logging.INFO,\n",
    "                    format='%(asctime)s %(levelname)s %(message)s')\n",
    "\n",
    "# parse commandline arguments\n",
    "op = OptionParser()\n",
    "op.add_option(\"--lsa\",\n",
    "              dest=\"n_components\", type=\"int\",\n",
    "              help=\"Preprocess documents with latent semantic analysis.\")\n",
    "op.add_option(\"--no-minibatch\",\n",
    "              action=\"store_false\", dest=\"minibatch\", default=True,\n",
    "              help=\"Use ordinary k-means algorithm (in batch mode).\")\n",
    "op.add_option(\"--no-idf\",\n",
    "              action=\"store_false\", dest=\"use_idf\", default=True,\n",
    "              help=\"Disable Inverse Document Frequency feature weighting.\")\n",
    "op.add_option(\"--use-hashing\",\n",
    "              action=\"store_true\", default=False,\n",
    "              help=\"Use a hashing feature vectorizer\")\n",
    "op.add_option(\"--n-features\", type=int, default=10000,\n",
    "              help=\"Maximum number of features (dimensions)\"\n",
    "                   \" to extract from text.\")\n",
    "op.add_option(\"--verbose\",\n",
    "              action=\"store_true\", dest=\"verbose\", default=False,\n",
    "              help=\"Print progress reports inside k-means algorithm.\")\n",
    "\n",
    "print(__doc__)\n",
    "op.print_help()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n"
     ]
    }
   ],
   "source": [
    "(opts, args) = op.parse_args([\"\"])\n",
    "print (len(args))\n",
    "if len(args) > 1:\n",
    "    op.error(\"this script takes no arguments.\")\n",
    "    sys.exit(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Values at 0x7f97641a0908: {'minibatch': True, 'use_idf': True, 'n_features': 10000, 'verbose': False, 'n_components': None, 'use_hashing': False}>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "opts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading 20 newsgroups dataset for categories:\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "###############################################################################\n",
    "# Load some categories from the training set\n",
    "categories = [\n",
    "    'alt.atheism',\n",
    "    'talk.religion.misc',\n",
    "    'comp.graphics',\n",
    "    'sci.space',\n",
    "]\n",
    "# Uncomment the following to do the analysis on all the categories\n",
    "categories = None\n",
    "\n",
    "print(\"Loading 20 newsgroups dataset for categories:\")\n",
    "print(categories)\n",
    "\n",
    "dataset = fetch_20newsgroups(subset='all', categories=categories,\n",
    "                             shuffle=True, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18846 documents\n",
      "20 categories\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"%d documents\" % len(dataset.data))\n",
    "print(\"%d categories\" % len(dataset.target_names))\n",
    "print()\n",
    "\n",
    "labels = dataset.target\n",
    "true_k = np.unique(labels).shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Subject: PHIGS User Group Conference\\nFrom: hamlin@ug.eds.com (Griff Hamlin)\\nReply-To: hamlin@ug.eds.com (Griff Hamlin)\\nDistribution: world\\nOrganization: EDS Unigraphics, Cypress CA\\nNntp-Posting-Host: 134.244.15.158\\nLines: 173\\n\\n\\n\\n                FIRST ANNUAL PHIGS USER GROUP CONFERENCE\\n\\n          The First Annual PHIGS User Group Conference was held March 21-24\\n          in Orlando, Florida.  The conference was organized by the Rensse-\\n          laer Design Research Center in co-operation with  IEEE  and  SIG-\\n          GRAPH.   Attendees  came  from five countries spanning three con-\\n          tinents.   A  good  cross-section  of  the  PHIGS  community  was\\n          represented  at this conference with participants including PHIGS\\n          users, workstation vendors, third-party PHIGS implementors, stan-\\n          dards  committee  members,  and  researchers  from  industry  and\\n          academia.  The opening speaker, Dr. Richard Puk, challenged PHIGS\\n          users  to  \"take  charge of your PHIGS\" by participating in PHIGS\\n          standardization activities and communicating their needs to PHIGS\\n          implementors.    The   closing  speaker,  Dr.  Andries  Van  Dam,\\n          described his vision of the future of graphics standards  \"beyond\\n          PHIGS\".\\n\\n          Technical paper sessions in the conference covered the  following\\n          topics:  PHIGS  and  X, Application Toolkits, Application Issues,\\n          Texture Mapping, NURBS,  PHIGS  Extensions,  and  Object-Oriented\\n          Libraries and Frameworks.  Panel sessions on PHIGS and PEX, PHIGS\\n          Non-Retained Data, Real-World CAD Applications Using  PHIGS,  and\\n          Portability  Issues generated enthusiastic discussions and formed\\n          a good forum for exchange of ideas, needs, and experiences.   The\\n          conference  also included a day full of tutorials on topics rang-\\n          ing from mathematics for 3D  graphics  to  object-oriented  tools\\n          based on PHIGS.\\n\\n          Next year\\'s conference is planned for March, 1994.\\n\\n          PHIGS EVERYWHERE\\n\\n          At the conference, PHIGS  vendors   described   and  demonstrated\\n          PHIGS  products  that  run on all types of computers, from PCs to\\n          mainframes.\\n\\n          Megatek Corporation demonstrated their PHIGS extensions including\\n          conditional traversal, composite logical input devices, texturing\\n          and translucency.\\n\\n          Template Graphics  Software  launched  FIGARO+  PRO,  the  Photo-\\n          Realistic  Option  for  PHIGS+.  FIGARO+  PRO  is designed to add\\n          advanced rendering to the existing PHIGS+ API, with features like\\n          ray   tracing,  materials,  anti-aliasing  and  texture  mapping.\\n          Radiosity support is also planned.\\n\\n          FIGARO+ is an example of how TGS continues to add newly  emerging\\n          graphics  features to their products.  FIGARO+ supports immediate\\n          mode extensions to PHIGS and also supports SUN XGL,  HP  Starbase\\n          and SGI GL/OpenGL. FIGARO+ for NT will be released this summer.\\n\\n          TGS also demonstrated the latest versions of FIGraph, a  powerful\\n          \"2-call\"  charting  system  based on PHIGS+, and FIGt, an object-\\n          oriented utility library for PHIGS/PEX developers.\\n\\n          G5G and Gallium Software demonstrated a new version of GPHIGS  on\\n          Silicon  Graphics  workstations. Scheduled for summer, 1993, Ver-\\n          sion 3.0 of GPHIGS, the company\\'s  PHIGS+  library  for  worksta-\\n          tions,  will include an advanced PHIGS debugger that allows PHIGS\\n          developers to display and browse PHIGS structures and other PHIGS\\n          internal  state.  G5G  also  described  their Non-Duplicated Data\\n          Store that stores pointers to application data in the GPHIGS  CSS\\n          for  more  efficient  use  of  memory. In addition, G5G described\\n          their application GSE that allows application callback  functions\\n          during  GPHIGS traversal.  GPHIGS and PHIGURE, G5G\\'s data visual-\\n          izer and application development toolkit, are currently available\\n          on  all  major  workstations  that support GL, X Windows, PEX, or\\n          Starbase.\\n\\n          Wise Software presented a slide show of  Z-PHIGS  for  MS-Windows\\n          and ARENA, a PHIGS based modeller/render. Z-PHIGS implements most\\n          of the PHIGS+ primitives.  In addition Z-PHIGS has built in  many\\n          advanced  rendering features like texture mapping, shadow genera-\\n          tion, area quick updates and ray tracing. A demo disk of  Z-PHIGS\\n          or ARENA is available on request.\\n\\n          ATC exhibited GRAFPAK-PHIGS, their full-featured PHIGS  implemen-\\n          tation  based  on  DEC  PHIGS. GRAFPAK-PHIGS is available on most\\n          workstation platforms with C, FORTRAN and Ada bindings and incor-\\n          porates PEX support.\\n\\n          Within the booth sponsored by Advanced Technology Center, Digital\\n          Equipment  Corporation demonstrated DEC PHIGS V2.4 running on the\\n          DEC 3000/400 AXP PXG. ATCs\\' GRAFPAK-PHIGS is a port of DEC PHIGS.\\n          DEC  PHIGS  V2.4 contains most PHIGS and PHIGS PLUS features with\\n          support for PEX V5.1  protocol.  DEC  PHIGS  also  contains  most\\n          GM/EDS   PHIGS  extensions  including  post-to-view  as  well  as\\n          proprietary extensions to support immediate  mode  rendering  and\\n          the use of PHIGS in an X11 environment.\\n\\n          AXP, DEC, and DEC PHIGS are trademarks of Digital Equipment  Cor-\\n          poration.  GRAFPAK-PHIGS and ATC are trademarks of Advanced Tech-\\n          nology Center. PEX and X11 are trademarks of Massachusetts Insti-\\n          tute of Technology.\\n\\n          The IBM exhibit featured a GTO accelerator attached to an IBM 340\\n          workstation running graPHIGS and PEX.\\n\\n          Hewlett Packard and SHOgraphics demonstrated at the conference. A\\n          Hewlett  Packard  machine was coupled to display on a SHOgraphics\\n          PEX terminal. HP showcased their latest  PHIGS  product  enhance-\\n          ments.\\n\\n\\n          PHIGS USER GROUP\\n\\n          The PHIGS Users Group was formed to aid the development of  PHIGS\\n          applications  and provide user feedback to PHIGS implementors and\\n          PHIGS standards bodies.  For more  information  about  the  PHIGS\\n          Users Group, send e-mail to:\\n\\n                    phigsug@cadrt10.me.vt.edu\\n\\n          or write to:\\n\\n                    Sankar Jayaram\\n                    Virginia Polytechnic Institute\\n                    114 Randolph Hall\\n                    Blacksburg, Va. 24061-0238\\n                    FAX: 703-231-9100\\n\\n\\n          VENDOR CONTACTS\\n\\n          Megatek Corporation\\n          TEL (619) 455-5590\\n          FAX (619) 453-7603\\n\\n          Template Graphics Software\\n          TEL (800) 544-4847\\n          FAX (619) 452-2547\\n\\n          WISE software GmbH\\n          TEL +49-451-3909-413\\n          FAX +49-451-3909-499\\n\\n          G5G - North American Sales\\n          TEL (800) 267-2626\\n          FAX (613) 592-1278\\n\\n          Advanced Technology Center\\n          TEL (800) 999-5711\\n          FAX (714) 583-9213\\n\\n          Digital Equipment Corporation\\n          TEL (603) 884-5111\\n\\n          International Business Machines Corporation\\n          TEL (800) 426-3333\\n\\n          Hewlett Packard Company\\n          TEL (303) 229-3800\\n\\n          COPIES OF THE CONFERENCE PROCEEDINGS\\n\\n          Copies of the conference proceedings may be obtained by  contact-\\n          ing Mary Johnson at:\\n\\n                    Johnson, Mary\\n                    Design and Manufacturing Institute\\n                    Rensselaer Polytechnic Institute\\n                    110 Eighth Street\\n                    Building CII, Room 7015\\n                    Troy, NY  12180-3590\\n                    Tel:  (518)276-6754\\n                    Fax:  (518)276-2702\\n                    Email:  mjohnson@rdrc.rpi.edu\\n\\n\\n          The cost is $75.00 per binder.\\n\\n'"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.data[2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting features from the training dataset using a sparse vectorizer\n"
     ]
    }
   ],
   "source": [
    "print(\"Extracting features from the training dataset using a sparse vectorizer\")\n",
    "t0 = time()\n",
    "if opts.use_hashing:\n",
    "    if opts.use_idf:\n",
    "        # Perform an IDF normalization on the output of HashingVectorizer\n",
    "        hasher = HashingVectorizer(n_features=opts.n_features,\n",
    "                                   stop_words='english', non_negative=True,\n",
    "                                   norm=None, binary=False)\n",
    "        vectorizer = make_pipeline(hasher, TfidfTransformer())\n",
    "    else:\n",
    "        vectorizer = HashingVectorizer(n_features=opts.n_features,\n",
    "                                       stop_words='english',\n",
    "                                       non_negative=False, norm='l2',\n",
    "                                       binary=False)\n",
    "else:\n",
    "    vectorizer = TfidfVectorizer(max_df=0.5, max_features=opts.n_features,\n",
    "                                 min_df=2, stop_words='english',\n",
    "                                 use_idf=opts.use_idf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "size of data: 20399\n",
      "future-title docs = 12554\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "#get the title and future work sections and pass that as the data\n",
    "dataFile = 'PickleCreation/AllDataPickle_e1.pk'\n",
    "data = pickle.load(open(dataFile,'rb'))\n",
    "print (\"size of data: \" + str(len(data)) )\n",
    "myData = []\n",
    "for key in data:\n",
    "    future_work_section = data[key][-1]\n",
    "    title_section = data[key][0]\n",
    "    if not (future_work_section == \"\"):\n",
    "        myData.append(title_section + \" \" + future_work_section)\n",
    "    \n",
    "numDocs = len(myData)\n",
    "print (\"future-title docs = \" + str(numDocs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "myX = vectorizer.fit_transform(myData)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 91.630911s\n",
      "n_samples: 18846, n_features: 10000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "X = vectorizer.fit_transform(dataset.data)\n",
    "\n",
    "print(\"done in %fs\" % (time() - t0))\n",
    "print(\"n_samples: %d, n_features: %d\" % X.shape)\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if opts.n_components:\n",
    "    print(\"Performing dimensionality reduction using LSA\")\n",
    "    t0 = time()\n",
    "    # Vectorizer results are normalized, which makes KMeans behave as\n",
    "    # spherical k-means for better results. Since LSA/SVD results are\n",
    "    # not normalized, we have to redo the normalization.\n",
    "    svd = TruncatedSVD(opts.n_components)\n",
    "    lsa = make_pipeline(svd, Normalizer(copy=False))\n",
    "\n",
    "    X = lsa.fit_transform(X)\n",
    "\n",
    "    print(\"done in %fs\" % (time() - t0))\n",
    "\n",
    "    explained_variance = svd.explained_variance_ratio_.sum()\n",
    "    print(\"Explained variance of the SVD step: {}%\".format(\n",
    "        int(explained_variance * 100)))\n",
    "\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Clustering sparse data with KMeans(copy_x=True, init='k-means++', max_iter=100, n_clusters=6, n_init=1,\n",
      "    n_jobs=1, precompute_distances=True, random_state=None, tol=0.0001,\n",
      "    verbose=False)\n",
      "done in 1.066s\n",
      "\n"
     ]
    }
   ],
   "source": [
    "###############################################################################\n",
    "# Do the actual clustering\n",
    "opts.minibatch = False\n",
    "true_k = 6\n",
    "if opts.minibatch:\n",
    "    #km = MiniBatchKMeans(n_clusters=true_k, init='k-means++', n_init=1,\n",
    "    #                     init_size=1000, batch_size=1000, verbose=opts.verbose)\n",
    "    km = MiniBatchKMeans(n_clusters=true_k, init='k-means++', n_init=1,\n",
    "                         init_size=1000, batch_size=1000, verbose=opts.verbose)\n",
    "else:\n",
    "    km = KMeans(n_clusters=true_k, init='k-means++', max_iter=100, n_init=1,\n",
    "                verbose=opts.verbose)\n",
    "\n",
    "print(\"Clustering sparse data with %s\" % km)\n",
    "t0 = time()\n",
    "km.fit(X)\n",
    "print(\"done in %0.3fs\" % (time() - t0))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n",
      "[  3.33086716e-03   3.58877027e-03   9.65320965e-04 ...,   4.52109627e-05\n",
      "   7.23900031e-04   2.14955888e-04]\n",
      "6\n"
     ]
    }
   ],
   "source": [
    "#Debug\n",
    "print (km.cluster_centers_[6,:])\n",
    "print (km.labels_[0])\n",
    "#km.score(X[0,:])\n",
    "temp = X[0,:]\n",
    "dist = np.linalg.norm(km.cluster_centers_[6,:]-temp)\n",
    "print (dist)\n",
    "print (km.score(X[0,:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6\n",
      "575653\n"
     ]
    }
   ],
   "source": [
    "#Clustering the title-futurework pairs\n",
    "km.fit(myX)\n",
    "print (len(km.cluster_centers_))\n",
    "print (myX.size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.1353352832366127"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def chaufunc(x, mu, std, n):\n",
    "    #z = n*(x - mu)/std\n",
    "    var = std**2\n",
    "    return (1/math.sqrt(2*math.pi*var)*math.e**(-((x-mu)**2)/var))*n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# contains a dictionary of lists where the keys are the cluster centers\n",
    "norm_squared_dic = {}\n",
    "for i in range(numDocs):\n",
    "    key = km.labels_[i]\n",
    "    if key in norm_squared_dic:\n",
    "        norm_squared_dic[key].append((i, -km.score(myX[i,:])))\n",
    "    else:\n",
    "        norm_squared_dic[key] = [(i, -km.score(myX[i,:]))]\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#min max mean\n",
    "outliers = []\n",
    "for key in norm_squared_dic:\n",
    "    dist_list = [ item for (doc_key, item) in norm_squared_dic[key] ]\n",
    "    mu = np.mean(dist_list)\n",
    "    std = np.std(dist_list)\n",
    "    n = len(dist_list)\n",
    "    for (doc_key,x) in norm_squared_dic[key]:\n",
    "        chau_score = chaufunc(x, mu, std,n )\n",
    "        if ( chau_score < 0.5 ):\n",
    "            outliers.append((doc_key, x))\n",
    "    #print (key, min(dist_list), max(dist_list), np.mean(dist_list), np.var(dist_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 182,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(outliers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(10266, 0.9451476100810541), (2027, 0.9437631497758151), (4371, 0.8954141741020347), (2673, 0.8928992655618991), (5661, 0.8890237206205602), (11457, 0.8861112673050836), (6855, 0.8841526920558072), (6790, 0.8820534766715277), (12003, 0.881825102768241), (10132, 0.8791298685321067), (649, 0.8759813527764054), (1187, 0.8758981572944069), (11035, 0.8668234541795165), (2510, 0.798229389345677), (115, 0.7950567124027754), (9922, 0.7947232864617625), (1147, 0.788277242998087), (997, 0.7810672615961858), (6650, 0.7710330077663579), (11740, 0.7646085182461546), (3291, 0.7518380533624386), (6616, 0.7455541825760528), (5261, 0.738866672975184), (10522, 0.7375696475885258)]\n"
     ]
    }
   ],
   "source": [
    "#print (outliers)\n",
    "outliers_sorted = sorted(outliers,key=lambda k_v: k_v[1],reverse=True)\n",
    "print (outliers_sorted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Headline Generation Based On Statistical Translation this paper has presented an alternative to extractive summarization: an approach that makes it possible to generate coherent summaries that are shorter than a single sentence and that attempt to conform to a particular style.our approach applies statistical models of the term selection and term ordering processes to produce short summaries, shorter than those reported previously.given good training corpora, this approach can also be used to generate headlines from a variety of formats: in one case, we experimented with corpora that contained japanese documents and english headlines.this resulted in a working system that could simultaneously translate and summarize japanese documents.8 the performance of the system could be improved by improving either content selection or linearization.recently, we have extended the model to generate multi-sentential summaries as well: for instance, given an initial sentence such as ?clinton to meet visit mideast.?in the example sentence, this generated the subsequent sentence ?us urges israel plan.?we expect to improve both the quality and scope of the summaries produced in future work.'"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myData[6949]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Homogeneity: 0.203\n",
      "Completeness: 0.435\n",
      "V-measure: 0.277\n",
      "Adjusted Rand-Index: 0.072\n",
      "Silhouette Coefficient: 0.003\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Homogeneity: %0.3f\" % metrics.homogeneity_score(labels, km.labels_))\n",
    "print(\"Completeness: %0.3f\" % metrics.completeness_score(labels, km.labels_))\n",
    "print(\"V-measure: %0.3f\" % metrics.v_measure_score(labels, km.labels_))\n",
    "print(\"Adjusted Rand-Index: %.3f\"\n",
    "      % metrics.adjusted_rand_score(labels, km.labels_))\n",
    "print(\"Silhouette Coefficient: %0.3f\"\n",
    "      % metrics.silhouette_score(X, km.labels_, sample_size=1000))\n",
    "\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if not (opts.n_components or opts.use_hashing):\n",
    "    print(\"Top terms per cluster:\")\n",
    "    order_centroids = km.cluster_centers_.argsort()[:, ::-1]\n",
    "    terms = vectorizer.get_feature_names()\n",
    "    for i in range(true_k):\n",
    "        print(\"Cluster %d:\" % i, end='')\n",
    "        for ind in order_centroids[i, :10]:\n",
    "            print(' %s' % terms[ind], end='')\n",
    "        print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}