EMF_RBM_Energy_Densities_Ominglot-250.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#  EMF RBM Energy Densities 250 Epochs\n",
    "\n",
    "Look at different ways of computing the\n",
    "\n",
    "- energy density\n",
    "- free energy density\n",
    "- entropy density\n",
    "\n",
    "We can\n",
    "\n",
    "- use the TAP S and MF Energy *U_naive   \n",
    "    - which takes an equlibration step first  \n",
    "    - S(mv, mh), U_naive(mv, mh)\n",
    "- use a simple h = means_hiddens() step\n",
    "    - S(v, h), E_MF(v, h)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-11-08T23:33:50.367876",
     "start_time": "2016-11-08T23:33:50.361534"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import h5py\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline \n",
    "\n",
    "from sklearn import linear_model, datasets, metrics, preprocessing \n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "import psutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-11-08T23:33:50.512461",
     "start_time": "2016-11-08T23:33:50.507597"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'divide': 'warn', 'invalid': 'warn', 'over': 'warn', 'under': 'ignore'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import fetch_mldata\n",
    "from sklearn.utils.validation import assert_all_finite\n",
    "from scipy.sparse import csc_matrix, csr_matrix, lil_matrix\n",
    "from sklearn.preprocessing import Binarizer\n",
    "np.seterr(all='warn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import rbm_datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_x, train_t, train_char, test_x, test_t, test_char = rbm_datasets.load_omniglot_iwae()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import binarize\n",
    "X = binarize(train_x, threshold=0.001, copy=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.io import loadmat\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib as mpl\n",
    "\n",
    "def show_image(image): \n",
    "    fig = plt.figure()\n",
    "    ax = fig.add_subplot(1,1,1)\n",
    "    imgplot = ax.imshow(image, cmap=mpl.cm.Greys)\n",
    "    imgplot.set_interpolation('nearest')\n",
    "    ax.xaxis.set_ticks_position('top')\n",
    "    ax.yaxis.set_ticks_position('left')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFdCAYAAADSR9wBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAEUBJREFUeJzt3X+sZGV9x/H3pxWDruESsdlNikF0JcE/NL1rpRRX12KC\n0gT9CzNqCPUPQtDG3KTRmJhC4Q8jxF5azTamSVGDmpBYijXAag3+IBYwIEYES6SLKD9WkAQSfijC\n0z9mts5edu/uzJ2z35lz36/kJMyZh3u+z32GD+c+c55z0lpDklTjj6oLkKTNzBCWpEKGsCQVMoQl\nqZAhLEmFDGFJKmQIS1IhQ1iSChnCklTIEJakQnMZwkk+nGRvkmeS3JLkz6trmoUkFyd5Yc12d3Vd\n00iyM8nXkzw46sc5B2lzaZKHkjyd5FtJtlfUOo3D9S/JVQcZy+ur6j1SST6R5LYkTybZl+TaJKcc\npN1Cjt2R9G/exm7uQjjJ+4DPABcDfwb8GNiT5FWlhc3OXcBWYNtoe2ttOVPbAtwJXAS86AYkST4O\nfAS4AHgL8BTDcXzp0SxyA9bt38gNHDiWg6NT2obsBD4LnAa8EzgG+GaSl+1vsOBjd9j+jczP2LXW\n5moDbgH+aex1gF8BH6uubQZ9uxi4o7qODvr1AnDOmn0PAStjr48DngHOra53Rv27Cvj36tpm0LdX\njfr31p6O3cH6N1djN1dnwkmOAXYA396/rw1/a/8FnF5V14y9fvQn7n1Jrk7y6uqCZi3JyQzPLsbH\n8UngVvozjgC7Rn/y/izJ7iSvrC5oCsczPNN/HHo5dgf0b8zcjN1chTDD/2v9MbBvzf59DD8Yi+4W\n4HzgLOBC4GTge0m2VBbVgW0MP/h9HUcY/jl7HvBXwMeAtwPXJ0lpVRMY1XolcHNrbf93E70Zu0P0\nD+Zs7F5ScdDNqrW2Z+zlXUluA34BnMvwTyQtiNbaNWMvf5rkJ8B9wC7gppKiJrcbeANwRnUhHTlo\n/+Zt7ObtTPgx4HmGE+bjtgKPHP1yutVaewK4F1iIb54n8AjDufxNMY4ArbW9DD+/CzGWST4HnA3s\naq09PPZWL8Zunf69SPXYzVUIt9aeA24Hzty/b/QnwpnAD6rq6kqSVzAc+HU/JItm9KF+hAPH8TiG\n31j3bhwBkpwInMACjOUooN4DvKO19sD4e30Yu/X6d4j2pWM3j9MR/wh8IcntwG3ACvBy4AuVRc1C\nkiuA/2Q4BfGnwD8AzwFfraxrGqN57O0Mz5oAXpvkTcDjrbVfMpyL+2SSnwP3A5cxvMrluoJyJ7Ze\n/0bbxcDXGAbWduDTDP+q2fPinzY/kuxmeDnWOcBTSfaf8T7RWnt29M8LO3aH699oXOdr7KovzzjE\nZSUXMRz8Z4D/Bt5cXdOM+vVVhh/mZ4AHgK8AJ1fXNWVf3s7w0p/n12z/NtbmEoaXOz3N8AO+vbru\nWfQPOBa4keF/xM8C/wv8C/An1XUfQb8O1qfngfPWtFvIsTtc/+Zx7DIqTJJUYK7mhCVpszGEJamQ\nISxJhQxhSSpkCEtSIUNYkgqVL9ZIcgLDG9rcz/C6PUladMcCrwH2tNZ+s17DzkI4yYeBv2N456Uf\nA3/bWvvhQZqeBXy5qzokqdAHGC7KOqROQnjs6RgX8Ielx3uSnNJae2xN8/sBrr76ak499dQD3lhZ\nWWF1dbWLEsv1uW/Q7/7Zt8V1tPp3zz338MEPfhBG+baers6EV4DPt9a+BJDkQuCvgQ8Bl69p+yzA\nqaeeyvLy8gFvLC0tvWhfX/S5b9Dv/tm3xVXQv8NOsc78i7lN8nQMSZqJLq6O6PvTMSRpZrxETZIK\ndTEnPNXTMVZWVlhaWjpg30knnTTz4ubFYLAIT0efXp/7Z98W1zz2r5NbWSa5Bbi1tfbR0eswvH/u\nP7fWrljTdhm4/fbbb+/1FwKSNo877riDHTt2AOxord2xXtuuro7o7dMxJGmWOgnh1to1SV4FXMpw\nGuJO4KzW2qNdHE+SFlVnK+Zaa7sZPnJaknQIXh0hSYUMYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAk\nFTKEJamQISxJhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEKGsCQVMoQlqZAhLEmFDGFJKmQIS1IhQ1iS\nChnCklTIEJakQoawJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFTKEJamQISxJ\nhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEKGsCQVMoQlqZAhLEmFDGFJKmQIS1IhQ1iSCs08hJNcnOSF\nNdvdsz6OJPXBSzr6uXcBZwIZvf59R8eRpIXWVQj/vrX2aEc/W5J6o6s54dcneTDJfUmuTvLqjo4j\nSQutixC+BTgfOAu4EDgZ+F6SLR0cS5IW2synI1pre8Ze3pXkNuAXwLnAVbM+niQtsq7mhP9fa+2J\nJPcC29drt7KywtLS0gH7BoMBg8Ggy/IkqVTnIZzkFQwD+EvrtVtdXWV5ebnrciRprnRxnfAVSd6W\n5KQkfwlcCzwHfHXWx5KkRdfFmfCJwFeAE4BHgZuBv2it/aaDY0nSQuviizkncSXpCHnvCEkqZAhL\nUiFDWJIKGcKSVMgQlqRChrAkFTKEJamQISxJhQxhSSpkCEtSIUNYkgp1fitLaZElOXyjBdVaqy5B\neCYsSaUMYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFTKEJamQISxJhQxhSSrksmUdNX1eAryIJhkP\nlzh3xzNhSSpkCEtSIUNYkgoZwpJUyBCWpEKGsCQVMoQlqZAhLEmFDGFJKmQIS1Ihly3rRVxerLVc\n4twdz4QlqZAhLEmFDGFJKmQIS1IhQ1iSChnCklTIEJakQoawJBUyhCWpkCEsSYVctrzAXF7cvUVb\ngutnYvFMfCacZGeSryd5MMkLSc45SJtLkzyU5Okk30qyfTblSlK/TDMdsQW4E7gIeNFpQpKPAx8B\nLgDeAjwF7Eny0g3UKUm9NPF0RGvtRuBGgBz8b5+PApe11r4xanMesA94L3DN9KVKUv/M9Iu5JCcD\n24Bv79/XWnsSuBU4fZbHkqQ+mPXVEdsYTlHsW7N/3+g9SdKYubk6YmVlhaWlpQP2DQYDBoNBUUWS\n1L1Zh/AjQICtHHg2vBX40Xr/4urqKsvLyzMuR5Lm20ynI1prexkG8Zn79yU5DjgN+MEsjyVJfTDx\nmXCSLcB2hme8AK9N8ibg8dbaL4ErgU8m+TlwP3AZ8CvguplULEk9Ms10xJuBmxh+AdeAz4z2fxH4\nUGvt8iQvBz4PHA98H3h3a+13M6hXknplmuuEv8thpjFaa5cAl0xX0ua2aMtOF21Zb99NMh5dfdZ8\nMvNkvIGPJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFTKEJanQ3NxPuM/mYSny\noi0PnYffGSze720Si7bEGfo5Hp4JS1IhQ1iSChnCklTIEJakQoawJBUyhCWpkCEsSYUMYUkqZAhL\nUiFDWJIKuWx5CvOypFbS4vNMWJIKGcKSVMgQlqRChrAkFTKEJamQISxJhQxhSSpkCEtSIUNYkgoZ\nwpJUyBCWpELeO2IKkz52u6t7TfTx8d/zZpKxczw0Dc+EJamQISxJhQxhSSpkCEtSIUNYkgoZwpJU\nyBCWpEKGsCQVMoQlqZAhLEmFJg7hJDuTfD3Jg0leSHLOmvevGu0f366fXcnS0dNaO+Jt0SQ54q0r\nk/x+F/F3fCSmORPeAtwJXAQc6rdyA7AV2DbaBlNVJ0k9N/ENfFprNwI3AuTQ/4v8bWvt0Y0UJkmb\nQVdzwruS7EvysyS7k7yyo+NI0kLr4laWNwBfA/YCrwM+BVyf5PTW10kdSZrSzEO4tXbN2MufJvkJ\ncB+wC7hp1seTpEXW+U3dW2t7kzwGbGedEF5ZWWFpaemAfYPBgMHA7/Qk9VfnIZzkROAE4OH12q2u\nrrK8vNx1OZI0VyYO4SRbGJ7V7r8y4rVJ3gQ8PtouZjgn/Mio3aeBe4E9syhYkvpkmjPhNzOcVmij\n7TOj/V9keO3wG4HzgOOBhxiG79+31p7bcLWS1DPTXCf8Xda/tO1d05cjSZuLT1teYH1+ivM81DCp\neXgyc5dLjI/UIo5dJW/gI0mFDGFJKmQIS1IhQ1iSChnCklTIEJakQoawJBUyhCWpkCEsSYUMYUkq\n5LLlo2CSZZzzsOx0HpbfLqJFG2fNB8+EJamQISxJhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEKGsCQV\nMoQlqZAhLEmFXLY8ZxZt6es81KDpzMMTn1327pmwJJUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKS\nVMgQlqRChrAkFTKEJamQy5YX2KItcVb35mEZ8DzUsEg8E5akQoawJBUyhCWpkCEsSYUMYUkqZAhL\nUiFDWJIKGcKSVMgQlqRChrAkFXLZ8iYxD0tJXTr9Bz7pWPtNdCac5BNJbkvyZJJ9Sa5NcspB2l2a\n5KEkTyf5VpLtsytZkvpj0umIncBngdOAdwLHAN9M8rL9DZJ8HPgIcAHwFuApYE+Sl86kYknqkYmm\nI1prZ4+/TnI+8GtgB3DzaPdHgctaa98YtTkP2Ae8F7hmg/VKUq9s9Iu544EGPA6Q5GRgG/Dt/Q1a\na08CtwKnb/BYktQ7U4dwht8AXAnc3Fq7e7R7G8NQ3rem+b7Re5KkMRu5OmI38AbgjFkUsrKywtLS\n0gH7BoMBg8FgFj9ekubSVCGc5HPA2cDO1trDY289AgTYyoFnw1uBH633M1dXV1leXp6mHElaWBNP\nR4wC+D3AO1prD4y/11rbyzCIzxxrfxzDqyl+sLFSJal/JjoTTrIbGADnAE8l2Tp664nW2rOjf74S\n+GSSnwP3A5cBvwKum0nFktQjk05HXMjwi7fvrNn/N8CXAFprlyd5OfB5hldPfB94d2vtdxsrVZL6\nZ9LrhI9o+qK1dglwyRT1qMdcJts9f8eLxxv4SFIhQ1iSChnCklTIEJakQoawJBUyhCWpkCEsSYUM\nYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFTKEJamQISxJhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEKG\nsCQVMoQlqZAhLEmFDGFJKmQIS1IhQ1iSChnCklTIEJakQoawJBUyhCWpkCEsSYUMYUkqZAhLUiFD\nWJIKGcKSVMgQlqRChrAkFTKEJamQISxJhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEIThXCSTyS5LcmT\nSfYluTbJKWvaXJXkhTXb9bMtW5L6YdIz4Z3AZ4HTgHcCxwDfTPKyNe1uALYC20bbYIN1SlIvvWSS\nxq21s8dfJzkf+DWwA7h57K3fttYe3XB1ktRzG50TPh5owONr9u8aTVf8LMnuJK/c4HEkqZcmOhMe\nlyTAlcDNrbW7x966AfgasBd4HfAp4Pokp7fW2kaKlaS+mTqEgd3AG4Azxne21q4Ze/nTJD8B7gN2\nATcd6oetrKywtLR0wL7BYMBg4HSypP6aKoSTfA44G9jZWnt4vbattb1JHgO2s04Ir66usry8PE05\nkrSwJg7hUQC/B3h7a+2BI2h/InACsG5YS9JmNOl1wruBDwDvB55KsnW0HTt6f0uSy5OcluSkJGcC\n/wHcC+yZdfGStOgmvTriQuA44DvAQ2PbuaP3nwfeCFwH/A/wr8APgbe11p6bQb2S1CuTXie8bmi3\n1p4F3rWhiiRpE/HeEZJUyBCWpEKGsCQVMoQlqZAhLEmFDGFJKmQIS1IhQ1iSChnCklTIEJakQoaw\nJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFTKEJamQISxJhSZ62nJHjgW45557\nquuQpJkYy7NjD9c2rbVuqzlcAcn7gS+XFiFJ3fhAa+0r6zWYhxA+ATgLuB94trQYSZqNY4HXAHta\na79Zr2F5CEvSZuYXc5JUyBCWpEKGsCQVMoQlqZAhLEmFDGFJKmQIS1Kh/wMMMzPnhb48HAAAAABJ\nRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2f87520dd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFdCAYAAADSR9wBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAEPZJREFUeJzt3W+sZHV9x/H3pwUDruESodlNikF0JcEHkt61UqqrazFB\naYI+wowaQn1ACNKYmzQaElIoPDBC6KWVbEOaFDWoCYmlWAOs1uAfYgEDYvhbAl1E+bOCJJDwRxF+\nfTCz6dzL7t2duXP2OzP3/Uom4Z457HwP5+6bc889ZyatNSRJNf6oegBJ2siMsCQVMsKSVMgIS1Ih\nIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFZrKCCf5XJLdSV5OcnuSP6+eaRKSXJzk9VWPB6rnGkeS\n7Um+k+SJwXacuY91Lk3yZJKXknw/ydaKWcdxoO1Lcu0+9uVNVfMerCQXJrkzyQtJ9iS5IcmJ+1hv\nJvfdwWzftO27qYtwkk8CVwIXA38G/ALYleTY0sEm5z5gM7Bl8PhA7Thj2wTcA5wPvOENSJJ8EbgA\nOBd4H/Ai/f34pkM55DqsuX0DN7NyX/YOzWjrsh34CnAK8BHgcOB7SY7cu8KM77sDbt/A9Oy71tpU\nPYDbgX8a+jrAr4EvVM82gW27GLi7eo4Otut14MxVy54Eloa+Pgp4GTiret4Jbd+1wL9XzzaBbTt2\nsH0fmNN9t6/tm6p9N1VHwkkOB7YBP9i7rPX/q/0XcGrVXBP2rsGPuI8muS7J26oHmrQkJ9A/uhje\njy8AdzA/+xFgx+BH3oeS7Ezy1uqBxnA0/SP952Au992K7RsyNftuqiJM//9afwzsWbV8D/1vjFl3\nO3AOcDpwHnAC8OMkmyqH6sAW+t/487ofof/j7NnAXwFfAD4E3JQkpVONYDDrVcBtrbW9v5uYm323\nn+2DKdt3h1W86EbVWts19OV9Se4EfgmcRf9HJM2I1tr1Q1/en+Re4FFgB3BryVCj2wm8G3h/9SAd\n2ef2Tdu+m7Yj4WeB1+ifMB+2GXj60I/Trdba88DDwEz85nkET9M/l78h9iNAa203/e/fmdiXSa4G\nzgB2tNaeGnpqLvbdGtv3BtX7bqoi3Fp7FbgLOG3vssGPCKcBP62aqytJ3kJ/x6/5TTJrBt/UT7Ny\nPx5F/zfWc7cfAZIcBxzDDOzLQaA+Dny4tfb48HPzsO/W2r79rF+676bxdMQ/Al9NchdwJ7AEvBn4\nauVQk5DkCuA/6Z+C+FPgH4BXgW9VzjWOwXnsrfSPmgDekeRk4LnW2q/on4u7KMkjwGPAZfSvcrmx\nYNyRrbV9g8fFwLfpB2sr8GX6P9XseuOfNj2S7KR/OdaZwItJ9h7xPt9ae2XwzzO77w60fYP9Ol37\nrvryjP1cVnI+/Z3/MvDfwHurZ5rQdn2L/jfzy8DjwDeBE6rnGnNbPkT/0p/XVj3+bWidS+hf7vQS\n/W/wrdVzT2L7gCOAW+j/JX4F+F/gX4A/qZ77ILZrX9v0GnD2qvVmct8daPumcd9lMJgkqcBUnROW\npI3GCEtSISMsSYWMsCQVMsKSVMgIS1Kh8ps1khxD/w1tHqN/3Z4kzbojgLcDu1prv11rxc4inORz\nwN/Rf+elXwB/21r72T5WPR34RldzSFKhT9O/KWu/Oonw0KdjnMv/33q8K8mJrbVnV63+GMB1113H\nSSedtOKJpaUllpeXuxix3DxvG8z39rlts+tQbd+DDz7IZz7zGRj0bS1dHQkvAde01r4OkOQ84K+B\nzwKXr1r3FYCTTjqJxcXFFU8sLCy8Ydm8mOdtg/nePrdtdhVs3wFPsU78F3Mb5NMxJGkiurg6Yt4/\nHUOSJsZL1CSpUBfnhMf6dIylpSUWFhZWLDv++OMnPty06PVm4dPRxzfP2+e2za5p3L5O3soyye3A\nHa21zw++Dv33z/3n1toVq9ZdBO6666675voXApI2jrvvvptt27YBbGut3b3Wul1dHTG3n44hSZPU\nSYRba9cnORa4lP5piHuA01trz3TxepI0qzq7Y661tpP+R05LkvbDqyMkqZARlqRCRliSChlhSSpk\nhCWpkBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGW\npEJGWJIKGWFJKmSEJalQZ5+2rOmSpHoEjam1Vj2COuSRsCQVMsKSVMgIS1IhIyxJhYywJBUywpJU\nyAhLUiEjLEmFjLAkFTLCklTI25alKdfVLefeDj0dPBKWpEJGWJIKGWFJKmSEJamQEZakQkZYkgoZ\nYUkqZIQlqZARlqRCRliSChlhSSo08QgnuTjJ66seD0z6dSStT5KDfqg7Xb2Bz33AacDevfeHjl5H\nkmZaVxH+Q2vtmY7+bEmaG12dE35XkieSPJrkuiRv6+h1JGmmdRHh24FzgNOB84ATgB8n2dTBa0nS\nTJv46YjW2q6hL+9LcifwS+As4NpJv54kzbLOP1mjtfZ8koeBrWutt7S0xMLCwoplvV6PXq/X5XiS\nVKrzCCd5C/0Af32t9ZaXl1lcXOx6HEmaKl1cJ3xFkg8mOT7JXwI3AK8C35r0a0nSrOviSPg44JvA\nMcAzwG3AX7TWftvBa0nSTOviF3OexJWkg+RH3s+wabid1I9N79407Gd1xzfwkaRCRliSChlhSSpk\nhCWpkBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgp527IOmWm4/XYWb7MeZeau/ht39efO4v6YNI+E\nJamQEZakQkZYkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpELetjxlvLVX6zENtziPossZ\nZuX72CNhSSpkhCWpkBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgoZYUkqZIQlqZC3LQ+McvvkqLdD\nTsPtobNyC6cOnVm7xXlUXf6dniSPhCWpkBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgoZYUkqZIQl\nqZARlqRC3rY84C2c3fPW6e5Nw37WaEY+Ek6yPcl3kjyR5PUkZ+5jnUuTPJnkpSTfT7J1MuNK0nwZ\n53TEJuAe4HzgDYc2Sb4IXACcC7wPeBHYleRN65hTkubSyKcjWmu3ALcAZN8/+3weuKy19t3BOmcD\ne4BPANePP6okzZ+J/mIuyQnAFuAHe5e11l4A7gBOneRrSdI8mPTVEVvon6LYs2r5nsFzkqQhU3N1\nxNLSEgsLCyuW9Xo9er1e0USS1L1JR/hpIMBmVh4NbwZ+vta/uLy8zOLi4oTHkaTpNtHTEa213fRD\nfNreZUmOAk4BfjrJ15KkeTDykXCSTcBW+ke8AO9IcjLwXGvtV8BVwEVJHgEeAy4Dfg3cOJGJJWmO\njHM64r3ArfR/AdeAKwfLvwZ8trV2eZI3A9cARwM/AT7WWvv9BOaVpLkyznXCP+IApzFaa5cAl4w3\nkuaVt9RqtWm5lb3yk5l9Ax9JKmSEJamQEZakQkZYkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWp\n0NS8n/AsmcVbLaVDZVr+foyicmaPhCWpkBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgoZYUkqZIQl\nqZARlqRC3rY8w2bx9lBJK3kkLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYyw\nJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUaOcJJ\ntif5TpInkrye5MxVz187WD78uGlyI0vS/BjnSHgTcA9wPtD2s87NwGZgy+DRG2s6SZpzh436L7TW\nbgFuAUiS/az2u9baM+sZTJI2gq7OCe9IsifJQ0l2JnlrR68jSTNt5CPhg3Az8G1gN/BO4EvATUlO\nba3t7/SFJG1IE49wa+36oS/vT3Iv8CiwA7h10q8nSbOsiyPhFVpru5M8C2xljQgvLS2xsLCwYlmv\n16PX83d6kuZX5xFOchxwDPDUWustLy+zuLjY9TiSNFVGjnCSTfSPavdeGfGOJCcDzw0eF9M/J/z0\nYL0vAw8DuyYxsCTNk3GOhN9L/7RCGzyuHCz/Gv1rh98DnA0cDTxJP75/31p7dd3TStKcGec64R+x\n9qVtHx1/HEnaWHzvCEkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKmSEJamQEZak\nQkZYkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgoZ\nYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgoZYUkqZIQl\nqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKjRShJNcmOTOJC8k2ZPkhiQn7mO9S5M8meSl\nJN9PsnVyI0vS/Bj1SHg78BXgFOAjwOHA95IcuXeFJF8ELgDOBd4HvAjsSvKmiUwsSXPksFFWbq2d\nMfx1knOA3wDbgNsGiz8PXNZa++5gnbOBPcAngOvXOa8kzZX1nhM+GmjAcwBJTgC2AD/Yu0Jr7QXg\nDuDUdb6WJM2dsSOcJMBVwG2ttQcGi7fQj/KeVavvGTwnSRoy0umIVXYC7wbeP4lBlpaWWFhYWLGs\n1+vR6/Um8cdL0lQaK8JJrgbOALa31p4aeuppIMBmVh4NbwZ+vtafuby8zOLi4jjjSNLMGvl0xCDA\nHwc+3Fp7fPi51tpu+iE+bWj9o+hfTfHT9Y0qSfNnpCPhJDuBHnAm8GKSzYOnnm+tvTL456uAi5I8\nAjwGXAb8GrhxIhNL0hwZ9XTEefR/8fbDVcv/Bvg6QGvt8iRvBq6hf/XET4CPtdZ+v75RJWn+jHqd\n8EGdvmitXQJcMsY8krSh+N4RklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyAhL\nUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmF\njLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLC\nklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUaKcJJLkxyZ5IXkuxJckOSE1etc22S11c9\nbprs2JI0H0Y9Et4OfAU4BfgIcDjwvSRHrlrvZmAzsGXw6K1zTkmaS4eNsnJr7Yzhr5OcA/wG2Abc\nNvTU71prz6x7Okmac+s9J3w00IDnVi3fMThd8VCSnUneus7XkaS5NNKR8LAkAa4CbmutPTD01M3A\nt4HdwDuBLwE3JTm1tdbWM6wkzZuxIwzsBN4NvH94YWvt+qEv709yL/AosAO4dX9/2NLSEgsLCyuW\n9Xo9ej1PJ0uaX2NFOMnVwBnA9tbaU2ut21rbneRZYCtrRHh5eZnFxcVxxpGkmTVyhAcB/jjwodba\n4wex/nHAMcCasZakjWjU64R3Ap8GPgW8mGTz4HHE4PlNSS5PckqS45OcBvwH8DCwa9LDS9KsG/Xq\niPOAo4AfAk8OPc4aPP8a8B7gRuB/gH8FfgZ8sLX26gTmlaS5Mup1wmtGu7X2CvDRdU0kSRuI7x0h\nSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQV\nMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFRvq05Y4cAfDggw9WzyFJEzHUsyMOtG5aa91O\nc6ABkk8B3ygdQpK68enW2jfXWmEaInwMcDrwGPBK6TCSNBlHAG8HdrXWfrvWiuURlqSNzF/MSVIh\nIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFfo/H3nOHf02K54AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2f8549a090>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFdCAYAAADSR9wBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAEPhJREFUeJzt3W+sZHV9x/H3pwWDruESsdlNikF0JcEHmt61UqqrazFB\naYI+wowaQn1ACNqYmzQaE1MoPDBi7KXVbGOaFDWoCYmlWAMs1uAfYgEDYkSwRLqI8mcFSSDhjyL8\n+mBm69zL7l1m7hy+M+e+X8kkzJnDnd/vnuHNuWfOmUlrDUlSjT+qHoAkbWVGWJIKGWFJKmSEJamQ\nEZakQkZYkgoZYUkqZIQlqZARlqRCRliSCs1lhJN8OMn+JE8luSnJn1ePaRaSXJjkuXW3O6vHNY0k\nu5N8I8n9o3mcdYh1Lk7yQJInk3wryc6KsU7jSPNLcvkhtuU1VeN9oZJ8IsktSR5PciDJVUlOPsR6\nC7ntXsj85m3bzV2Ek7wP+CxwIfBnwI+BfUleWTqw2bkD2A7sGN3eWjucqW0DbgcuAJ73ASRJPg58\nBDgPeDPwBMPt+JIXc5CbsOH8Rq5l7bYcvDhD25TdwOeAU4F3AkcD1yd56cEVFnzbHXF+I/Oz7Vpr\nc3UDbgL+aex+gF8BH6se2wzmdiFwW/U4OpjXc8BZ65Y9AKyM3T8WeAo4u3q8M5rf5cC/V49tBnN7\n5Wh+b+3ptjvU/OZq283VnnCSo4FdwLcPLmvD39p/AadVjWvGXjf6E/eeJFckeVX1gGYtyUkM9y7G\nt+PjwM30ZzsC7Bn9yfuzJHuTvKJ6QFM4juGe/qPQy223Zn5j5mbbzVWEGf5f64+BA+uWH2D4wlh0\nNwHnAmcA5wMnAd9Lsq1yUB3YwfCF39ftCMM/Z88B/gr4GPB24JokKR3VBEZjvQy4sbV28L2J3my7\nw8wP5mzbHVXxpFtVa23f2N07ktwC/AI4m+GfSFoQrbUrx+7+NMlPgHuAPcANJYOa3F7g9cBbqgfS\nkUPOb9623bztCT8CPMvwgPm47cBDL/5wutVaewy4G1iId54n8BDDY/lbYjsCtNb2M3z9LsS2TPJ5\n4ExgT2vtwbGHerHtNpjf81Rvu7mKcGvtGeBW4PSDy0Z/IpwO/KBqXF1J8nKGG37DF8miGb2oH2Lt\ndjyW4TvWvduOAElOAI5nAbblKFDvAd7RWrtv/LE+bLuN5neY9Uu33TwejvhH4ItJbgVuAVaAlwFf\nrBzULCT5DPCfDA9B/CnwD8AzwNcqxzWN0XHsnQz3mgBek+SNwKOttV8yPBb3ySQ/B+4FLmF4lsvV\nBcOd2EbzG90uBL7OMFg7gU8z/Ktm3/N/2vxIspfh6VhnAU8kObjH+1hr7enRPy/stjvS/Ebbdb62\nXfXpGYc5reQChhv/KeC/gTdVj2lG8/oawxfzU8B9wFeBk6rHNeVc3s7w1J9n193+bWydixie7vQk\nwxf4zupxz2J+wDHAdQz/I34a+F/gX4A/qR73C5jXoeb0LHDOuvUWctsdaX7zuO0yGpgkqcBcHROW\npK3GCEtSISMsSYWMsCQVMsKSVMgIS1Kh8os1khzP8ANt7mV43p4kLbpjgFcD+1prv9loxc4inOTD\nwN8x/OSlHwN/21r74SFWPQP4SlfjkKRCH2B4UdZhdRLhsW/HOI8/XHq8L8nJrbVH1q1+L8AVV1zB\nKaecsuaBlZUVVldXuxhiuT7PDfo9P+e2uF6s+d1111188IMfhFHfNtLVnvAK8IXW2pcBkpwP/DXw\nIeDSdes+DXDKKaewvLy85oGlpaXnLeuLPs8N+j0/57a4CuZ3xEOsM39jbot8O4YkzUQXZ0f0/dsx\nJGlmPEVNkgp1cUx4qm/HWFlZYWlpac2yE088ceaDmxeDwSJ8O/r0+jw/57a45nF+nXyUZZKbgJtb\nax8d3Q/Dz8/959baZ9atuwzceuutt/b6DQFJW8dtt93Grl27AHa11m7baN2uzo7o7bdjSNIsdRLh\n1tqVSV4JXMzwMMTtwBmttYe7eD5JWlSdXTHXWtvL8CunJUmH4dkRklTICEtSISMsSYWMsCQVMsKS\nVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1Ih\nIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYyw\nJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBWaeYST\nXJjkuXW3O2f9PJLUB0d19HPvAE4HMrr/+46eR5IWWlcR/n1r7eGOfrYk9UZXx4Rfl+T+JPckuSLJ\nqzp6HklaaF1E+CbgXOAM4HzgJOB7SbZ18FyStNBmfjiitbZv7O4dSW4BfgGcDVw+6+eTpEXW1THh\n/9daeyzJ3cDOjdZbWVlhaWlpzbLBYMBgMOhyeJJUqvMIJ3k5wwB/eaP1VldXWV5e7no4kjRXujhP\n+DNJ3pbkxCR/CVwFPAN8bdbPJUmLros94ROArwLHAw8DNwJ/0Vr7TQfPJUkLrYs35jyIK0kvUOfH\nhKVpJDnySgustVY9BM0JP8BHkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJ\nKuRly9qUvl9e3JV5+L11den0JHPz8m33hCWplBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgoZYUkq\nZIQlqZARlqRCXra8RczDZbKaL129JrwUeTLuCUtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUy\nwpJUyAhLUiEjLEmFvGx5znh58ZCXvv7Bor0muhxvH18X7glLUiEjLEmFjLAkFTLCklTICEtSISMs\nSYWMsCQVMsKSVMgIS1IhIyxJhbxs+UWwaJedzsOloX4T8B90NeZFe1321cR7wkl2J/lGkvuTPJfk\nrEOsc3GSB5I8meRbSXbOZriS1C/THI7YBtwOXAA873/RST4OfAQ4D3gz8ASwL8lLNjFOSeqliQ9H\ntNauA64DyKH/nvkocElr7Zujdc4BDgDvBa6cfqiS1D8zfWMuyUnADuDbB5e11h4HbgZOm+VzSVIf\nzPrsiB0MD1EcWLf8wOgxSdKYuTk7YmVlhaWlpTXLBoMBg8GgaESS1L1ZR/ghIMB21u4Nbwd+tNG/\nuLq6yvLy8oyHI0nzbaaHI1pr+xmG+PSDy5IcC5wK/GCWzyVJfTDxnnCSbcBOhnu8AK9J8kbg0dba\nL4HLgE8m+TlwL3AJ8Cvg6pmMWJJ6ZJrDEW8CbmD4BlwDPjta/iXgQ621S5O8DPgCcBzwfeDdrbXf\nzWC8ktQr05wn/F2OcBijtXYRcNF0Q1K1SS5nXcTLgDU/fP34AT6SVMoIS1IhIyxJhYywJBUywpJU\nyAhLUiEjLEmFjLAkFTLCklTICEtSobn5POE+6/O35XqJ83yZh9eE23ky7glLUiEjLEmFjLAkFTLC\nklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhbxseYFNcnnoPFzOuohjmIdLcOfh9zYPv4e+ck9Y\nkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKmSEJamQnx2xRSza50zMC38X\n6pp7wpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1KhiSOcZHeSbyS5P8lz\nSc5a9/jlo+Xjt2tmN2R1rbVWftN03B6LZ5o94W3A7cAFwOG2zrXAdmDH6DaYanSS1HMTf4BPa+06\n4DqAHP7TTX7bWnt4MwOTpK2gq2PCe5IcSPKzJHuTvKKj55GkhdbFR1leC3wd2A+8FvgUcE2S05oH\nlyRpjZlHuLV25djdnyb5CXAPsAe4YdbPJ0mLrPMPdW+t7U/yCLCTDSK8srLC0tLSmmWDwYDBwPf0\nJPVX5xFOcgJwPPDgRuutrq6yvLzc9XAkaa5MHOEk2xju1R48M+I1Sd4IPDq6XcjwmPBDo/U+DdwN\n7JvFgCWpT6bZE34Tw8MKbXT77Gj5lxieO/wG4BzgOOABhvH9+9baM5serST1zDTnCX+XjU9te9f0\nw5GkrcVvW9Zc8mxGbRV+gI8kFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJU\nyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEj\nLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAk\nFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhSaKcJJPJLklyeNJDiS5KsnJh1jv4iQPJHky\nybeS7JzdkCWpPybdE94NfA44FXgncDRwfZKXHlwhyceBjwDnAW8GngD2JXnJTEYsST1y1CQrt9bO\nHL+f5Fzg18Au4MbR4o8Cl7TWvjla5xzgAPBe4MpNjleSemWzx4SPAxrwKECSk4AdwLcPrtBaexy4\nGThtk88lSb0zdYSTBLgMuLG1dudo8Q6GUT6wbvUDo8ckSWMmOhyxzl7g9cBbZjGQlZUVlpaW1iwb\nDAYMBoNZ/HhJmktTRTjJ54Ezgd2ttQfHHnoICLCdtXvD24EfbfQzV1dXWV5enmY4krSwJj4cMQrw\ne4B3tNbuG3+stbafYYhPH1v/WIZnU/xgc0OVpP6ZaE84yV5gAJwFPJFk++ihx1prT4/++TLgk0l+\nDtwLXAL8Crh6JiOWpB6Z9HDE+QzfePvOuuV/A3wZoLV2aZKXAV9gePbE94F3t9Z+t7mhSlL/THqe\n8As6fNFauwi4aIrxSNKW4mdHSFIhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMs\nSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQV\nMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgI\nS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklRooggn+USSW5I8nuRAkquSnLxuncuTPLfu\nds1shy1J/TDpnvBu4HPAqcA7gaOB65O8dN161wLbgR2j22CT45SkXjpqkpVba2eO309yLvBrYBdw\n49hDv22tPbzp0UlSz232mPBxQAMeXbd8z+hwxc+S7E3yik0+jyT10kR7wuOSBLgMuLG1dufYQ9cC\nXwf2A68FPgVck+S01lrbzGAlqW+mjjCwF3g98Jbxha21K8fu/jTJT4B7gD3ADYf7YSsrKywtLa1Z\nNhgMGAw8nCypv6aKcJLPA2cCu1trD260bmttf5JHgJ1sEOHV1VWWl5enGY4kLayJIzwK8HuAt7fW\n7nsB658AHA9sGGtJ2oomPU94L/AB4P3AE0m2j27HjB7fluTSJKcmOTHJ6cB/AHcD+2Y9eEladJOe\nHXE+cCzwHeCBsdvZo8efBd4AXA38D/CvwA+Bt7XWnpnBeCWpVyY9T3jDaLfWngbetakRSdIW4mdH\nSFIhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJ\nhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSoYm+bbkjxwDcdddd1eOQpJkY69kxR1o3rbVu\nR3OkASTvB75SOghJ6sYHWmtf3WiFeYjw8cAZwL3A06WDkaTZOAZ4NbCvtfabjVYsj7AkbWW+MSdJ\nhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTo/wAe+Gxykwh3+gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2f85375bd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFdCAYAAADSR9wBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAEVNJREFUeJzt3X+sZGV9x/H3pxWDruESsdlNikF0JcE/NL1rpVRX12KC\n0gT9i2bUEOofhKCNuUmjMTGFwh9GjL20mm1Mk6IGNSGxFGuAxRr8QSxgQIwIlkgXUX6sIAkk/FCE\np3/MbDv37u7dnblz9jtz9v1KTrJz5mHO97nP8LnnPnOeM2mtIUmq8QfVBUjSscwQlqRChrAkFTKE\nJamQISxJhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEJzGcJJPpxkb5Jnk9ya5E+ra5qFJJckeXHddk91\nXdNIsjPJN5I8NOrHuQdpc1mSh5M8k+RbSbZX1DqNw/UvyVUHGcvrq+o9Ukk+keT2JE8l2Zfk2iSn\nHaTdQo7dkfRv3sZu7kI4yV8BnwUuAf4E+DGwJ8mrSgubnbuBrcC20fa22nKmtgW4C7gYOOAGJEk+\nDnwEuBB4C/A0w3F86dEschM27N/IDawdy8HRKW1TdgKfA84A3gUcB9yU5GX7Gyz42B22fyPzM3at\ntbnagFuBfxx7HOBXwMeqa5tB3y4B7qyuo4N+vQicu27fw8DK2OMTgGeB86rrnVH/rgL+rbq2GfTt\nVaP+va2nY3ew/s3V2M3VmXCS44AdwLf372vDn9p/AmdW1TVjrx/9iXt/kquTvLq6oFlLcirDs4vx\ncXwKuI3+jCPArtGfvD9LsjvJK6sLmsKJDM/0n4Bejt2a/o2Zm7GbqxBm+FvrD4F96/bvY/jGWHS3\nAhcAZwMXAacC30uypbKoDmxj+Mbv6zjC8M/Z84G/AD4GvAO4PklKq5rAqNYrgVtaa/s/m+jN2B2i\nfzBnY/eSioMeq1pre8Ye3p3kduAXwHkM/0TSgmitXTP28KdJfgLcD+wCbi4panK7gTcAb60upCMH\n7d+8jd28nQk/DrzAcMJ83Fbg0aNfTrdaa08C9wEL8cnzBB5lOJd/TIwjQGttL8P370KMZZLPA+cA\nu1prj4w91Yux26B/B6geu7kK4dba88AdwFn7943+RDgL+EFVXV1J8gqGA7/hm2TRjN7Uj7J2HE9g\n+Il178YRIMnJwEkswFiOAuq9wDtbaw+OP9eHsduof4doXzp28zgd8Q/AF5PcAdwOrAAvB75YWdQs\nJPkM8B8MpyD+GPh74Hnga5V1TWM0j72d4VkTwGuTvAl4orX2S4ZzcZ9M8nPgAeByhle5XFdQ7sQ2\n6t9ouwT4OsPA2g58muFfNXsOfLX5kWQ3w8uxzgWeTrL/jPfJ1tpzo38v7Ngdrn+jcZ2vsau+POMQ\nl5VczHDwnwX+C3hzdU0z6tfXGL6ZnwUeBL4KnFpd15R9eQfDS39eWLf961ibSxle7vQMwzf49uq6\nZ9E/4HjgRob/Ez8H/A/wz8AfVdd9BP06WJ9eAM5f124hx+5w/ZvHscuoMElSgbmaE5akY40hLEmF\nDGFJKmQIS1IhQ1iSChnCklSofLFGkpMY3tDmAYbX7UnSojseeA2wp7X2m40adhbCST4M/C3DOy/9\nGPib1toPD9L0bOArXdUhSYU+wHBR1iF1EsJj345xIf+/9HhPktNaa4+va/4AwNVXX83pp5++5omV\nlRVWV1e7KLFcn/sG/e6ffVtcR6t/9957Lx/84AdhlG8b6epMeAX4QmvtywBJLgL+EvgQcMW6ts8B\nnH766SwvL695Ymlp6YB9fdHnvkG/+2ffFldB/w47xTrzD+aOkW/HkKSZ6OLqiL5/O4YkzYyXqElS\noS7mhKf6doyVlRWWlpbW7DvllFNmXty8GAwW4dvRp9fn/tm3xTWP/evkVpZJbgVua619dPQ4DO+f\n+0+ttc+sa7sM3HHHHXf0+gMBSceOO++8kx07dgDsaK3duVHbrq6O6O23Y0jSLHUSwq21a5K8CriM\n4TTEXcDZrbXHujieJC2qzlbMtdZ2M/zKaUnSIXh1hCQVMoQlqZAhLEmFDGFJKmQIS1IhQ1iSChnC\nklTIEJakQoawJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFTKEJamQISxJhQxh\nSSpkCEtSIUNYkgoZwpJUyBCWpEKGsCQVMoQlqZAhLEmFDGFJKmQIS1IhQ1iSChnCklTIEJakQoaw\nJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFTKEJamQISxJhQxhSSo08xBOckmS\nF9dt98z6OJLUBy/p6HXvBs4CMnr8+46OI0kLrasQ/n1r7bGOXluSeqOrOeHXJ3koyf1Jrk7y6o6O\nI0kLrYsQvhW4ADgbuAg4Ffheki0dHEuSFtrMpyNaa3vGHt6d5HbgF8B5wFWzPp4kLbKu5oT/T2vt\nyST3Ads3areyssLS0tKafYPBgMFg0GV5klSq8xBO8gqGAfzljdqtrq6yvLzcdTmSNFe6uE74M0ne\nnuSUJH8OXAs8D3xt1seSpEXXxZnwycBXgZOAx4BbgD9rrf2mg2NJ0kLr4oM5J3El6Qh57whJKmQI\nS1IhQ1iSChnCklTIEJakQoawJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKdX4rS/VbksM30gFaa9Ul\naE54JixJhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEKGsCQVMoQlqZAhLEmFDGFJKuSyZR3Apcjdm+Rn\n3NUS53moQZ4JS1IpQ1iSChnCklTIEJakQoawJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKuWx5zrhk\neHJdLqmdh/HoqgaXIs8Hz4QlqZAhLEmFDGFJKmQIS1IhQ1iSChnCklTIEJakQoawJBUyhCWpkCEs\nSYVctnwUzMPS1670fenrPHzTcVf8tuX5MPGZcJKdSb6R5KEkLyY59yBtLkvycJJnknwryfbZlCtJ\n/TLNdMQW4C7gYuCAX49JPg58BLgQeAvwNLAnyUs3Uack9dLE0xGttRuBGwFy8L9nPgpc3lr75qjN\n+cA+4H3ANdOXKkn9M9MP5pKcCmwDvr1/X2vtKeA24MxZHkuS+mDWV0dsYzhFsW/d/n2j5yRJY+bm\n6oiVlRWWlpbW7BsMBgwGg6KKJKl7sw7hR4EAW1l7NrwV+NFG/+Hq6irLy8szLkeS5ttMpyNaa3sZ\nBvFZ+/clOQE4A/jBLI8lSX0w8Zlwki3AdoZnvACvTfIm4InW2i+BK4FPJvk58ABwOfAr4LqZVCxJ\nPTLNdMSbgZsZfgDXgM+O9n8J+FBr7YokLwe+AJwIfB94T2vtdzOoV5J6ZZrrhL/LYaYxWmuXApdO\nV9L8m4clp5Ny2el8mWQ85uH95jc+d8cb+EhSIUNYkgoZwpJUyBCWpEKGsCQVMoQlqZAhLEmFDGFJ\nKmQIS1IhQ1iSCs3N/YSrzcPS0Ekt2pJPl75OZ9GWOE9i0nr7ONaeCUtSIUNYkgoZwpJUyBCWpEKG\nsCQVMoQlqZAhLEmFDGFJKmQIS1IhQ1iSCvV62fKiLeGE7pao9nG5pw7U5yXO0M/3vGfCklTIEJak\nQoawJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFer1vSPmhfeD0GYs4j0edOQ8\nE5akQoawJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFZo4hJPsTPKNJA8leTHJ\nueuev2q0f3y7fnYlL54kR7y11o54WzST9G0Rfw6TjPMk26LpapwX9X1xONOcCW8B7gIuBg7V0xuA\nrcC20TaYqjpJ6rmJb+DTWrsRuBEgh/41/dvW2mObKUySjgVdzQnvSrIvyc+S7E7yyo6OI0kLrYtb\nWd4AfB3YC7wO+BRwfZIz2yJN1EjSUTDzEG6tXTP28KdJfgLcD+wCbp718SRpkXV+U/fW2t4kjwPb\n2SCEV1ZWWFpaWrNvMBgwGPiZnqT+6jyEk5wMnAQ8slG71dVVlpeXuy5HkubKxCGcZAvDs9r9V0a8\nNsmbgCdG2yUM54QfHbX7NHAfsGcWBUtSn0xzJvxmhtMKbbR9drT/SwyvHX4jcD5wIvAww/D9u9ba\n85uuVpJ6ZprrhL/Lxpe2vXv6ciTp2LJw37bc1bcRz8vyUL9teWhexmPR9Pk90VfewEeSChnCklTI\nEJakQoawJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKSVGjhli13tRS570ucNV9cXqz9PBOWpEKG\nsCQVMoQlqZAhLEmFDGFJKmQIS1IhQ1iSChnCklTIEJakQoawJBVauGXLk5iHJc6Tvrbmi8uL1TXP\nhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFTKEJamQISxJhXq9bHkSXS5PXbSlr/Ow\nzHrRfmbStDwTlqRChrAkFTKEJamQISxJhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEKGsCQVctmyDuCS\nYenomehMOMknktye5Kkk+5Jcm+S0g7S7LMnDSZ5J8q0k22dXsiT1x6TTETuBzwFnAO8CjgNuSvKy\n/Q2SfBz4CHAh8BbgaWBPkpfOpGJJ6pGJpiNaa+eMP05yAfBrYAdwy2j3R4HLW2vfHLU5H9gHvA+4\nZpP1SlKvbPaDuROBBjwBkORUYBvw7f0NWmtPAbcBZ27yWJLUO1OHcIY3nb0SuKW1ds9o9zaGobxv\nXfN9o+ckSWM2c3XEbuANwFtnUcjKygpLS0tr9g0GAwaDwSxeXpLm0lQhnOTzwDnAztbaI2NPPQoE\n2Mras+GtwI82es3V1VWWl5enKUeSFtbE0xGjAH4v8M7W2oPjz7XW9jIM4rPG2p/A8GqKH2yuVEnq\nn4nOhJPsBgbAucDTSbaOnnqytfbc6N9XAp9M8nPgAeBy4FfAdTOpWJJ6ZNLpiIsYfvD2nXX7/xr4\nMkBr7YokLwe+wPDqie8D72mt/W5zpUpS/0x6nfARTV+01i4FLp2iHkk6pngDH0kqZAhLUiFDWJIK\nGcKSVMgQlqRChrAkFTKEJamQISxJhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEKGsCQVMoQlqZAhLEmF\nDGFJKmQIS1IhQ1iSChnCklTIEJakQoawJBUyhCWpkCEsSYUMYUkqZAhLUiFDWJIKGcKSVMgQlqRC\nhrAkFTKEJamQISxJhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEKGsCQVMoQlqZAhLEmFDGFJKmQIS1Ih\nQ1iSChnCklRoohBO8okktyd5Ksm+JNcmOW1dm6uSvLhuu362ZUtSP0x6JrwT+BxwBvAu4DjgpiQv\nW9fuBmArsG20DTZZpyT10ksmadxaO2f8cZILgF8DO4Bbxp76bWvtsU1XJ0k9t9k54ROBBjyxbv+u\n0XTFz5LsTvLKTR5HknppojPhcUkCXAnc0lq7Z+ypG4CvA3uB1wGfAq5PcmZrrW2mWEnqm6lDGNgN\nvAF46/jO1to1Yw9/muQnwP3ALuDmQ73YysoKS0tLa/YNBgMGA6eTJfXXVCGc5PPAOcDO1tojG7Vt\nre1N8jiwnQ1CeHV1leXl5WnKkaSFNXEIjwL4vcA7WmsPHkH7k4GTgA3DWpKORZNeJ7wb+ADwfuDp\nJFtH2/Gj57ckuSLJGUlOSXIW8O/AfcCeWRcvSYtu0qsjLgJOAL4DPDy2nTd6/gXgjcB1wH8D/wL8\nEHh7a+35GdQrSb0y6XXCG4Z2a+054N2bqkiSjiHeO0KSChnCklTIEJakQoawJBUyhCWpkCEsSYUM\nYUkqZAhLUiFDWJIKGcKSVMgQlqRChrAkFTKEJamQISxJhQxhSSpkCEtSIUNYkgoZwpJUyBCWpEKG\nsCQVMoQlqdBE37bckeMB7r333uo6JGkmxvLs+MO1TWut22oOV0DyfuArpUVIUjc+0Fr76kYN5iGE\nTwLOBh4AnistRpJm43jgNcCe1tpvNmpYHsKSdCzzgzlJKmQIS1IhQ1iSChnCklTIEJakQoawJBUy\nhCWp0P8CwZe9q9rpNkYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2f8529b150>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFdCAYAAADSR9wBAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAERxJREFUeJzt3W+sZHV9x/H3pwWDruESsdlNikF0JcEHmt61UqqrazFB\naYI+ohk1hPqAELQxN2k0JqZQeGDE2Eur2cY0KWpQExJLsQZYrcE/xAIGxIhgiXQR5c8KkkDCH0X4\n9cHMtnMvu3d35s7hO3Pu+5VMwpw53vmd+xvfe+6Zc2bSWkOSVOMPqgcgSVuZEZakQkZYkgoZYUkq\nZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJzGeEkH0qyP8nTSW5O8qfVY5qFJBcneX7d7a7qcU0j\nye4kX0/ywGg7zjnEOpcmeTDJU0m+lWRnxVincaTtS3LlIebyuqrxHq0kH09ya5InkhxIck2SUw+x\n3kLO3dFs37zN3dxFOMlfAZ8BLgb+BPgxsC/JK0sHNjt3AtuBHaPbW2uHM7VtwB3ARcALPoAkyceA\nDwMXAG8GnmQ4jy95MQe5CRtu38j1rJ3LwYsztE3ZDXwWOB14J3As8M0kLz24woLP3RG3b2R+5q61\nNlc34GbgH8fuB/gV8NHqsc1g2y4Gbq8eRwfb9TxwzrplDwIrY/ePB54Gzq0e74y270rg36rHNoNt\ne+Vo+97a07k71PbN1dzN1Z5wkmOBXcC3Dy5rw9/afwJnVI1rxl43+hP33iRXJXlV9YBmLckpDPcu\nxufxCeAW+jOPAHtGf/L+LMneJK+oHtAUTmC4p/8Y9HLu1mzfmLmZu7mKMMN/tf4QOLBu+QGGL4xF\ndzNwPnAWcCFwCvC9JNsqB9WBHQxf+H2dRxj+OXse8BfAR4G3A9clSemoJjAa6xXATa21g+9N9Gbu\nDrN9MGdzd0zFk25VrbV9Y3fvTHIr8AvgXIZ/ImlBtNauHrv70yQ/Ae4F9gA3lgxqcnuB1wNvqR5I\nRw65ffM2d/O2J/wo8BzDA+bjtgMPv/jD6VZr7XHgHmAh3nmewMMMj+VviXkEaK3tZ/j6XYi5TPI5\n4GxgT2vtobGHejF3G2zfC1TP3VxFuLX2LHAbcObBZaM/Ec4EflA1rq4keTnDid/wRbJoRi/qh1k7\nj8czfMe6d/MIkOQk4EQWYC5HgXoP8I7W2v3jj/Vh7jbavsOsXzp383g44h+ALyS5DbgVWAFeBnyh\nclCzkOTTwH8wPATxx8DfA88CX60c1zRGx7F3MtxrAnhNkjcCj7XWfsnwWNwnkvwcuA+4jOFZLtcW\nDHdiG23f6HYx8DWGwdoJfIrhXzX7XvjT5keSvQxPxzoHeDLJwT3ex1trz4z+e2Hn7kjbN5rX+Zq7\n6tMzDnNayUUMJ/9p4L+AN1WPaUbb9VWGL+angfuBrwCnVI9rym15O8NTf55bd/vXsXUuYXi601MM\nX+A7q8c9i+0DjgNuYPh/4meA/wH+Gfij6nEfxXYdapueA85bt95Czt2Rtm8e5y6jgUmSCszVMWFJ\n2mqMsCQVMsKSVMgIS1IhIyxJhYywJBUqv1gjyYkMP9DmPobn7UnSojsOeDWwr7X2m41W7CzCST4E\n/C3DT176MfA3rbUfHmLVs4AvdzUOSSr0foYXZR1WJxEe+3aMC/j/S4/3JTm1tfboutXvA7jqqqs4\n7bTT1jywsrLC6upqF0Ms1+dtg35vn9u2uF6s7bv77rv5wAc+AKO+baSrPeEV4POttS8BJLkQ+Evg\ng8Dl69Z9BuC0005jeXl5zQNLS0svWNYXfd426Pf2uW2Lq2D7jniIdeZvzG2Rb8eQpJno4uyIvn87\nhiTNjKeoSVKhLo4JT/XtGCsrKywtLa1ZdvLJJ898cPNiMFiEb0efXp+3z21bXPO4fZ18lGWSm4Fb\nWmsfGd0Pw8/P/afW2qfXrbsM3Hbbbbf1+g0BSVvH7bffzq5duwB2tdZu32jdrs6O6O23Y0jSLHUS\n4dba1UleCVzK8DDEHcBZrbVHung+SVpUnV0x11rby/ArpyVJh+HZEZJUyAhLUiEjLEmFjLAkFTLC\nklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtS\nISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWM\nsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYWMsCQVmnmE\nk1yc5Pl1t7tm/TyS1AfHdPRz7wTOBDK6//uOnkeSFlpXEf59a+2Rjn62JPVGV8eEX5fkgST3Jrkq\nyas6eh5JWmhdRPhm4HzgLOBC4BTge0m2dfBckrTQZn44orW2b+zunUluBX4BnAtcOevnk6RF1tUx\n4f/TWns8yT3Azo3WW1lZYWlpac2ywWDAYDDocniSVKrzCCd5OcMAf2mj9VZXV1leXu56OJI0V7o4\nT/jTSd6W5OQkfw5cAzwLfHXWzyVJi66LPeGTgK8AJwKPADcBf9Za+00HzyVJC62LN+Y8iCtJR8nP\njpCkQkZYkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKtT5R1lq8SQ58kp6\ngdZa9RC0gNwTlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgp52fIW4aXI\n3evqd+zl0P3mnrAkFTLCklTICEtSISMsSYWMsCQVMsKSVMgIS1IhIyxJhYywJBUywpJUyMuWXwST\nXM7qJapaz8uh+809YUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKmSEJamQly2P\ndHlp8STrL9q3Invp63QWbZ4n4WX6k5l4TzjJ7iRfT/JAkueTnHOIdS5N8mCSp5J8K8nO2QxXkvpl\nmsMR24A7gIuAF/wzluRjwIeBC4A3A08C+5K8ZBPjlKRemvhwRGvtBuAGgBz6746PAJe11r4xWuc8\n4ADwXuDq6YcqSf0z0zfmkpwC7AC+fXBZa+0J4BbgjFk+lyT1wazPjtjB8BDFgXXLD4wekySNmZuz\nI1ZWVlhaWlqzbDAYMBgMikYkSd2bdYQfBgJsZ+3e8HbgRxv9D1dXV1leXp7xcCRpvs30cERrbT/D\nEJ95cFmS44HTgR/M8rkkqQ8m3hNOsg3YyXCPF+A1Sd4IPNZa+yVwBfCJJD8H7gMuA34FXDuTEUtS\nj0xzOOJNwI0M34BrwGdGy78IfLC1dnmSlwGfB04Avg+8u7X2uxmMV5J6ZZrzhL/LEQ5jtNYuAS6Z\nbkg1+nxp8aT6fCnppHPX1e9iHl5v8/AtzvMyH5X8AB9JKmSEJamQEZakQkZYkgoZYUkqZIQlqZAR\nlqRCRliSChlhSSpkhCWp0Nx8nrC6NQ+Xe87D5d7z8HuY1Dxc4jwJv215Mu4JS1IhIyxJhYywJBUy\nwpJUyAhLUiEjLEmFjLAkFTLCklTICEtSISMsSYW8bHmBdXU5q5eSLq4+X+IM/XxtuicsSYWMsCQV\nMsKSVMgIS1IhIyxJhYywJBUywpJUyAhLUiEjLEmFjLAkFTLCklTIz45YYIv2eRCL9rkGXer79h2t\neXhdVnNPWJIKGWFJKmSEJamQEZakQkZYkgoZYUkqZIQlqZARlqRCRliSChlhSSo0cYST7E7y9SQP\nJHk+yTnrHr9ytHz8dt3shlyvtTbRTZOb9Hfc1Vwk6eS2aOZlPvpomj3hbcAdwEXA4X6L1wPbgR2j\n22Cq0UlSz038AT6ttRuAGwBy+H/Sf9tae2QzA5OkraCrY8J7khxI8rMke5O8oqPnkaSF1sVHWV4P\nfA3YD7wW+CRwXZIzmgeBJGmNmUe4tXb12N2fJvkJcC+wB7hx1s8nSYus8w91b63tT/IosJMNIryy\nssLS0tKaZYPBgMHA9/Qk9VfnEU5yEnAi8NBG662urrK8vNz1cCRprkwc4STbGO7VHjwz4jVJ3gg8\nNrpdzPCY8MOj9T4F3APsm8WAJalPptkTfhPDwwptdPvMaPkXGZ47/AbgPOAE4EGG8f271tqzmx6t\nJPXMNOcJf5eNT2171/TDkaStxW9b3iIW8VJZdcszRueDH+AjSYWMsCQVMsKSVMgIS1IhIyxJhYyw\nJBUywpJUyAhLUiEjLEmFjLAkFfKy5RfBJJeHennx1uAlwzrIPWFJKmSEJamQEZakQkZYkgoZYUkq\nZIQlqZARlqRCRliSChlhSSpkhCWpkJctzxkvZ5W2FveEJamQEZakQkZYkgoZYUkqZIQlqZARlqRC\nRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgoZYUkqZIQlqZARlqRCRliSChlh\nSSpkhCWpkBGWpEITRTjJx5PcmuSJJAeSXJPk1EOsd2mSB5M8leRbSXbObsiS1B+T7gnvBj4LnA68\nEzgW+GaSlx5cIcnHgA8DFwBvBp4E9iV5yUxGLEk9cswkK7fWzh6/n+R84NfALuCm0eKPAJe11r4x\nWuc84ADwXuDqTY5Xknpls8eETwAa8BhAklOAHcC3D67QWnsCuAU4Y5PPJUm9M3WEkwS4AriptXbX\naPEOhlE+sG71A6PHJEljJjocsc5e4PXAW2YxkJWVFZaWltYsGwwGDAaDWfx4SZpLU0U4yeeAs4Hd\nrbWHxh56GAiwnbV7w9uBH230M1dXV1leXp5mOJK0sCY+HDEK8HuAd7TW7h9/rLW2n2GIzxxb/3iG\nZ1P8YHNDlaT+mWhPOMleYACcAzyZZPvoocdba8+M/vsK4BNJfg7cB1wG/Aq4diYjlqQemfRwxIUM\n33j7zrrlfw18CaC1dnmSlwGfZ3j2xPeBd7fWfre5oUpS/0x6nvBRHb5orV0CXDLFeCRpS/GzIySp\nkBGWpEJGWJIKGWFJKmSEJamQEZakQkZYkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJG\nWJIKGWFJKmSEJamQEZakQkZYkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJ\nKmSEJamQEZakQkZYkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKmSEJamQ\nEZakQkZYkgoZYUkqNFGEk3w8ya1JnkhyIMk1SU5dt86VSZ5fd7tutsOWpH6YdE94N/BZ4HTgncCx\nwDeTvHTdetcD24Edo9tgk+OUpF46ZpKVW2tnj99Pcj7wa2AXcNPYQ79trT2y6dFJUs9t9pjwCUAD\nHlu3fM/ocMXPkuxN8opNPo8k9dJEe8LjkgS4AriptXbX2EPXA18D9gOvBT4JXJfkjNZa28xgJalv\npo4wsBd4PfCW8YWttavH7v40yU+Ae4E9wI2H+2ErKyssLS2tWTYYDBgMPJwsqb+minCSzwFnA7tb\naw9ttG5rbX+SR4GdbBDh1dVVlpeXpxmOJC2siSM8CvB7gLe31u4/ivVPAk4ENoy1JG1Fk54nvBd4\nP/A+4Mkk20e340aPb0tyeZLTk5yc5Ezg34F7gH2zHrwkLbpJz464EDge+A7w4Njt3NHjzwFvAK4F\n/hv4F+CHwNtaa8/OYLyS1CuTnie8YbRba88A79rUiCRpC/GzIySpkBGWpEJGWJIKGWFJKmSEJamQ\nEZakQkZYkgoZYUkqZIQlqZARlqRCRliSChlhSSpkhCWpkBGWpEJGWJIKGWFJKmSEJamQEZakQkZY\nkgoZYUkqZIQlqdBE37bckeMA7r777upxSNJMjPXsuCOtm9Zat6M50gCS9wFfLh2EJHXj/a21r2y0\nwjxE+ETgLOA+4JnSwUjSbBwHvBrY11r7zUYrlkdYkrYy35iTpEJGWJIKGWFJKmSEJamQEZakQkZY\nkgoZYUkq9L/QOo2epN7O6wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2f851e5c10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    show_image(X[i+100].reshape(28,28))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24345, 784)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Helper methods for simple case"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-11-08T22:38:39.742812",
     "start_time": "2016-11-08T22:38:39.737075"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.utils.fixes import expit    \n",
    "from sklearn.utils.extmath import safe_sparse_dot\n",
    "\n",
    "def sig_means(x, b, W):\n",
    "    a = safe_sparse_dot(x, W.T) + b\n",
    "    return expit(a, out=a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-10-20T17:56:14.156132",
     "start_time": "2016-10-20T17:56:14.153682"
    }
   },
   "source": [
    "### Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import emf_rbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-11-08T22:42:32.963306",
     "start_time": "2016-11-08T22:42:32.959982"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sswap(total=0, used=0, free=0, percent=0.0, sin=0, sout=0)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "psutil.swap_memory()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### mean Entropy, Energy, and Free Energy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-11-08T22:55:48.509468",
     "start_time": "2016-11-08T22:45:04.953105"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EMF_RBM(batch_size=100, decay=0.01, learning_rate=0.005, momentum=0.5,\n",
      "    monitor=True, n_components=256, n_iter=250, neq_steps=3,\n",
      "    random_state=<mtrand.RandomState object at 0x7f2fcc151a00>,\n",
      "    sigma=0.001, thresh=1e-08, verbose=True, weight_decay='L1')\n",
      "[EMF_RBM] Iteration 1, pseudo-likelihood = -255.91, time = 9.98s\n",
      "computing TAP Free Energies\n",
      "monitor:  -322.222299406 425.604337703 103.390800505\n",
      "[EMF_RBM] Iteration 2, pseudo-likelihood = -258.78, time = 23.52s\n",
      "computing TAP Free Energies\n",
      "monitor:  -319.559459707 423.026288556 103.529105537\n",
      "[EMF_RBM] Iteration 3, pseudo-likelihood = -233.09, time = 23.69s\n",
      "computing TAP Free Energies\n",
      "monitor:  -308.619515429 397.903697185 90.0274840389\n",
      "[EMF_RBM] Iteration 4, pseudo-likelihood = -220.91, time = 23.76s\n",
      "computing TAP Free Energies\n",
      "monitor:  -298.736464115 369.611713082 72.113449785\n",
      "[EMF_RBM] Iteration 5, pseudo-likelihood = -206.00, time = 23.84s\n",
      "computing TAP Free Energies\n",
      "monitor:  -294.062098858 349.581976044 57.4187073258\n",
      "[EMF_RBM] Iteration 6, pseudo-likelihood = -203.28, time = 23.70s\n",
      "computing TAP Free Energies\n",
      "monitor:  -292.440167967 331.699275047 41.5986528135\n",
      "[EMF_RBM] Iteration 7, pseudo-likelihood = -195.59, time = 23.71s\n",
      "computing TAP Free Energies\n",
      "monitor:  -290.121624841 319.909199133 32.5068666122\n",
      "[EMF_RBM] Iteration 8, pseudo-likelihood = -191.99, time = 23.71s\n",
      "computing TAP Free Energies\n",
      "monitor:  -289.037419451 310.836373962 24.9723505677\n",
      "[EMF_RBM] Iteration 9, pseudo-likelihood = -188.62, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -289.967255489 302.409441082 16.1264423647\n",
      "[EMF_RBM] Iteration 10, pseudo-likelihood = -176.51, time = 23.66s\n",
      "computing TAP Free Energies\n",
      "monitor:  -292.067230836 294.403273871 6.50334876647\n",
      "[EMF_RBM] Iteration 11, pseudo-likelihood = -169.02, time = 23.67s\n",
      "computing TAP Free Energies\n",
      "monitor:  -294.411214747 287.228930279 -2.59363069933\n",
      "[EMF_RBM] Iteration 12, pseudo-likelihood = -167.41, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -296.518747568 281.230164272 -10.3176843817\n",
      "[EMF_RBM] Iteration 13, pseudo-likelihood = -159.32, time = 23.68s\n",
      "computing TAP Free Energies\n",
      "monitor:  -298.513384378 276.201145151 -16.9644547496\n",
      "[EMF_RBM] Iteration 14, pseudo-likelihood = -163.42, time = 23.68s\n",
      "computing TAP Free Energies\n",
      "monitor:  -300.684470731 271.729812309 -23.2315065152\n",
      "[EMF_RBM] Iteration 15, pseudo-likelihood = -163.98, time = 23.67s\n",
      "computing TAP Free Energies\n",
      "monitor:  -303.030426979 267.63222257 -29.3063042798\n",
      "[EMF_RBM] Iteration 16, pseudo-likelihood = -154.20, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -305.503009039 263.764117272 -35.2937209968\n",
      "[EMF_RBM] Iteration 17, pseudo-likelihood = -151.19, time = 23.66s\n",
      "computing TAP Free Energies\n",
      "monitor:  -308.050897565 260.047200596 -41.2298138422\n",
      "[EMF_RBM] Iteration 18, pseudo-likelihood = -152.48, time = 23.78s\n",
      "computing TAP Free Energies\n",
      "monitor:  -310.577069559 256.497966614 -47.0011240109\n",
      "[EMF_RBM] Iteration 19, pseudo-likelihood = -149.85, time = 23.68s\n",
      "computing TAP Free Energies\n",
      "monitor:  -313.023624045 253.129250526 -52.5328785555\n",
      "[EMF_RBM] Iteration 20, pseudo-likelihood = -144.63, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -315.384178937 249.934385061 -57.8225336728\n",
      "[EMF_RBM] Iteration 21, pseudo-likelihood = -144.24, time = 23.67s\n",
      "computing TAP Free Energies\n",
      "monitor:  -317.668970127 246.906990069 -62.8849204057\n",
      "[EMF_RBM] Iteration 22, pseudo-likelihood = -143.91, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -319.893830032 244.043972841 -67.7375152479\n",
      "[EMF_RBM] Iteration 23, pseudo-likelihood = -138.18, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -322.073044277 241.340816642 -72.398076021\n",
      "[EMF_RBM] Iteration 24, pseudo-likelihood = -138.99, time = 23.66s\n",
      "computing TAP Free Energies\n",
      "monitor:  -324.213917018 238.79076522 -76.8800900054\n",
      "[EMF_RBM] Iteration 25, pseudo-likelihood = -137.72, time = 23.68s\n",
      "computing TAP Free Energies\n",
      "monitor:  -326.312406871 236.387764892 -81.1851353478\n",
      "[EMF_RBM] Iteration 26, pseudo-likelihood = -135.66, time = 23.65s\n",
      "computing TAP Free Energies\n",
      "monitor:  -328.355117437 234.128052728 -85.3027530507\n",
      "[EMF_RBM] Iteration 27, pseudo-likelihood = -135.55, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -330.328585299 232.008319246 -89.2213429772\n",
      "[EMF_RBM] Iteration 28, pseudo-likelihood = -133.37, time = 23.73s\n",
      "computing TAP Free Energies\n",
      "monitor:  -332.224585825 230.024575648 -92.934763665\n",
      "[EMF_RBM] Iteration 29, pseudo-likelihood = -134.95, time = 23.65s\n",
      "computing TAP Free Energies\n",
      "monitor:  -334.043777746 228.170491579 -96.4478910737\n",
      "[EMF_RBM] Iteration 30, pseudo-likelihood = -130.63, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -335.795306427 226.4372074 -99.7767354668\n",
      "[EMF_RBM] Iteration 31, pseudo-likelihood = -132.52, time = 23.73s\n",
      "computing TAP Free Energies\n",
      "monitor:  -337.494412635 224.813797797 -102.94579029\n",
      "[EMF_RBM] Iteration 32, pseudo-likelihood = -130.49, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -339.158754888 223.287979293 -105.983788092\n",
      "[EMF_RBM] Iteration 33, pseudo-likelihood = -130.67, time = 23.67s\n",
      "computing TAP Free Energies\n",
      "monitor:  -340.80492125 221.8463057 -108.920133465\n",
      "[EMF_RBM] Iteration 34, pseudo-likelihood = -128.87, time = 23.69s\n",
      "computing TAP Free Energies\n",
      "monitor:  -342.448810452 220.476095119 -111.783255488\n",
      "[EMF_RBM] Iteration 35, pseudo-likelihood = -122.60, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -344.101666373 219.16575402 -114.596263968\n",
      "[EMF_RBM] Iteration 36, pseudo-likelihood = -125.22, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -345.769172608 217.905350501 -117.375316323\n",
      "[EMF_RBM] Iteration 37, pseudo-likelihood = -124.65, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -347.451993931 216.687381554 -120.129207687\n",
      "[EMF_RBM] Iteration 38, pseudo-likelihood = -118.27, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -349.147969785 215.506318415 -122.861940666\n",
      "[EMF_RBM] Iteration 39, pseudo-likelihood = -126.00, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -350.852333081 214.357772214 -125.573696551\n",
      "[EMF_RBM] Iteration 40, pseudo-likelihood = -120.36, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -352.560120716 213.23901262 -128.262717486\n",
      "[EMF_RBM] Iteration 41, pseudo-likelihood = -120.28, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -354.266815192 212.148159381 -130.92673337\n",
      "[EMF_RBM] Iteration 42, pseudo-likelihood = -120.14, time = 23.63s\n",
      "computing TAP Free Energies\n",
      "monitor:  -355.967313893 211.084000303 -133.562106896\n",
      "[EMF_RBM] Iteration 43, pseudo-likelihood = -117.18, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -357.656201568 210.046106105 -136.163996755\n",
      "[EMF_RBM] Iteration 44, pseudo-likelihood = -118.66, time = 23.66s\n",
      "computing TAP Free Energies\n",
      "monitor:  -359.327839177 209.034760947 -138.726495555\n",
      "[EMF_RBM] Iteration 45, pseudo-likelihood = -121.57, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -360.978082104 208.050260051 -141.245081822\n",
      "[EMF_RBM] Iteration 46, pseudo-likelihood = -117.09, time = 23.67s\n",
      "computing TAP Free Energies\n",
      "monitor:  -362.603828872 207.093014713 -143.716047176\n",
      "[EMF_RBM] Iteration 47, pseudo-likelihood = -115.76, time = 23.66s\n",
      "computing TAP Free Energies\n",
      "monitor:  -364.202543413 206.163269604 -146.136367403\n",
      "[EMF_RBM] Iteration 48, pseudo-likelihood = -110.99, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -365.772319483 205.261106673 -148.503779189\n",
      "[EMF_RBM] Iteration 49, pseudo-likelihood = -115.27, time = 23.63s\n",
      "computing TAP Free Energies\n",
      "monitor:  -367.313006598 204.386286064 -150.818085519\n",
      "[EMF_RBM] Iteration 50, pseudo-likelihood = -116.12, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -368.8244812 203.538463079 -153.079214227\n",
      "[EMF_RBM] Iteration 51, pseudo-likelihood = -111.40, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -370.307377155 202.717033598 -155.288120351\n",
      "[EMF_RBM] Iteration 52, pseudo-likelihood = -112.68, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -371.762987725 201.92118882 -157.44666451\n",
      "[EMF_RBM] Iteration 53, pseudo-likelihood = -114.57, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -373.192699674 201.14993913 -159.557007488\n",
      "[EMF_RBM] Iteration 54, pseudo-likelihood = -112.12, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -374.597875998 200.402178447 -161.621429676\n",
      "[EMF_RBM] Iteration 55, pseudo-likelihood = -111.65, time = 23.70s\n",
      "computing TAP Free Energies\n",
      "monitor:  -375.980065672 199.676751813 -163.642494675\n",
      "[EMF_RBM] Iteration 56, pseudo-likelihood = -111.00, time = 23.65s\n",
      "computing TAP Free Energies\n",
      "monitor:  -377.340531788 198.972734772 -165.622250024\n",
      "[EMF_RBM] Iteration 57, pseudo-likelihood = -112.42, time = 23.70s\n",
      "computing TAP Free Energies\n",
      "monitor:  -378.680104404 198.289226244 -167.562291654\n",
      "[EMF_RBM] Iteration 58, pseudo-likelihood = -109.19, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -380.000436038 197.62540113 -169.464985636\n",
      "[EMF_RBM] Iteration 59, pseudo-likelihood = -110.57, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -381.302157388 196.980110735 -171.332004857\n",
      "[EMF_RBM] Iteration 60, pseudo-likelihood = -112.11, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -382.586074512 196.352806212 -173.164603688\n",
      "[EMF_RBM] Iteration 61, pseudo-likelihood = -108.40, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -383.853563668 195.742792418 -174.964748195\n",
      "[EMF_RBM] Iteration 62, pseudo-likelihood = -110.32, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -385.10581675 195.149082225 -176.73455028\n",
      "[EMF_RBM] Iteration 63, pseudo-likelihood = -109.35, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -386.343383423 194.571221745 -178.474925505\n",
      "[EMF_RBM] Iteration 64, pseudo-likelihood = -104.51, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -387.56638248 194.008566167 -180.186583109\n",
      "[EMF_RBM] Iteration 65, pseudo-likelihood = -105.25, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -388.775812735 193.460491718 -181.871085186\n",
      "[EMF_RBM] Iteration 66, pseudo-likelihood = -109.11, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -389.972492917 192.926408172 -183.529765126\n",
      "[EMF_RBM] Iteration 67, pseudo-likelihood = -107.59, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -391.156798379 192.405837457 -185.16342527\n",
      "[EMF_RBM] Iteration 68, pseudo-likelihood = -105.61, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -392.329690538 191.898073028 -186.773706047\n",
      "[EMF_RBM] Iteration 69, pseudo-likelihood = -105.86, time = 23.56s\n",
      "computing TAP Free Energies\n",
      "monitor:  -393.491534769 191.40259067 -188.361460212\n",
      "[EMF_RBM] Iteration 70, pseudo-likelihood = -106.27, time = 23.69s\n",
      "computing TAP Free Energies\n",
      "monitor:  -394.642877069 190.918936134 -189.92765694\n",
      "[EMF_RBM] Iteration 71, pseudo-likelihood = -107.16, time = 23.69s\n",
      "computing TAP Free Energies\n",
      "monitor:  -395.783638564 190.446725012 -191.472556789\n",
      "[EMF_RBM] Iteration 72, pseudo-likelihood = -103.57, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -396.914457912 189.985334394 -192.997416441\n",
      "[EMF_RBM] Iteration 73, pseudo-likelihood = -104.84, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -398.035885796 189.534333674 -194.503194028\n",
      "[EMF_RBM] Iteration 74, pseudo-likelihood = -104.14, time = 23.63s\n",
      "computing TAP Free Energies\n",
      "monitor:  -399.148117532 189.093261623 -195.990530847\n",
      "[EMF_RBM] Iteration 75, pseudo-likelihood = -102.47, time = 23.65s\n",
      "computing TAP Free Energies\n",
      "monitor:  -400.250991939 188.661831287 -197.459553703\n",
      "[EMF_RBM] Iteration 76, pseudo-likelihood = -101.21, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -401.345140444 188.239528425 -198.91140989\n",
      "[EMF_RBM] Iteration 77, pseudo-likelihood = -102.99, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -402.431020399 187.825839772 -200.347080011\n",
      "[EMF_RBM] Iteration 78, pseudo-likelihood = -99.39, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -403.50837682 187.420520772 -201.76654275\n",
      "[EMF_RBM] Iteration 79, pseudo-likelihood = -104.45, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -404.577799441 187.023199397 -203.170764908\n",
      "[EMF_RBM] Iteration 80, pseudo-likelihood = -101.69, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -405.639139601 186.633649922 -204.559824288\n",
      "[EMF_RBM] Iteration 81, pseudo-likelihood = -103.89, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -406.69237455 186.251486831 -205.934092146\n",
      "[EMF_RBM] Iteration 82, pseudo-likelihood = -104.96, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -407.738241473 185.876367322 -207.29465526\n",
      "[EMF_RBM] Iteration 83, pseudo-likelihood = -101.36, time = 23.66s\n",
      "computing TAP Free Energies\n",
      "monitor:  -408.776182683 185.508084634 -208.641150035\n",
      "[EMF_RBM] Iteration 84, pseudo-likelihood = -100.89, time = 23.69s\n",
      "computing TAP Free Energies\n",
      "monitor:  -409.806406152 185.146531084 -209.973890393\n",
      "[EMF_RBM] Iteration 85, pseudo-likelihood = -99.58, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -410.829337263 184.791289811 -211.293739616\n",
      "[EMF_RBM] Iteration 86, pseudo-likelihood = -100.04, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -411.845134905 184.44208437 -212.601131953\n",
      "[EMF_RBM] Iteration 87, pseudo-likelihood = -100.93, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -412.853494813 184.098777876 -213.89589829\n",
      "[EMF_RBM] Iteration 88, pseudo-likelihood = -99.60, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -413.854525032 183.761189338 -215.178334461\n",
      "[EMF_RBM] Iteration 89, pseudo-likelihood = -99.42, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -414.847911338 183.429210458 -216.448221485\n",
      "[EMF_RBM] Iteration 90, pseudo-likelihood = -97.97, time = 23.68s\n",
      "computing TAP Free Energies\n",
      "monitor:  -415.833876258 183.102654791 -217.70596976\n",
      "[EMF_RBM] Iteration 91, pseudo-likelihood = -102.37, time = 23.56s\n",
      "computing TAP Free Energies\n",
      "monitor:  -416.812911608 182.781366129 -218.952223483\n",
      "[EMF_RBM] Iteration 92, pseudo-likelihood = -102.08, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -417.785034228 182.465106491 -220.187239649\n",
      "[EMF_RBM] Iteration 93, pseudo-likelihood = -98.70, time = 23.65s\n",
      "computing TAP Free Energies\n",
      "monitor:  -418.750034993 182.153760379 -221.410914472\n",
      "[EMF_RBM] Iteration 94, pseudo-likelihood = -98.58, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -419.708142874 181.847188733 -222.623603459\n",
      "[EMF_RBM] Iteration 95, pseudo-likelihood = -99.23, time = 23.66s\n",
      "computing TAP Free Energies\n",
      "monitor:  -420.659456138 181.545354115 -223.825430496\n",
      "[EMF_RBM] Iteration 96, pseudo-likelihood = -98.67, time = 23.66s\n",
      "computing TAP Free Energies\n",
      "monitor:  -421.604184038 181.248138346 -225.016715879\n",
      "[EMF_RBM] Iteration 97, pseudo-likelihood = -96.34, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -422.542166472 180.955333341 -226.197502969\n",
      "[EMF_RBM] Iteration 98, pseudo-likelihood = -97.80, time = 23.56s\n",
      "computing TAP Free Energies\n",
      "monitor:  -423.473457647 180.666862262 -227.367907046\n",
      "[EMF_RBM] Iteration 99, pseudo-likelihood = -100.86, time = 23.65s\n",
      "computing TAP Free Energies\n",
      "monitor:  -424.398135993 180.382633701 -228.528097509\n",
      "[EMF_RBM] Iteration 100, pseudo-likelihood = -97.00, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -425.316116546 180.102546681 -229.678078445\n",
      "[EMF_RBM] Iteration 101, pseudo-likelihood = -95.39, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -426.227632048 179.826601411 -230.818061165\n",
      "[EMF_RBM] Iteration 102, pseudo-likelihood = -97.00, time = 23.67s\n",
      "computing TAP Free Energies\n",
      "monitor:  -427.133148091 179.554620288 -231.948684105\n",
      "[EMF_RBM] Iteration 103, pseudo-likelihood = -93.69, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -428.032657862 179.286510382 -233.070028027\n",
      "[EMF_RBM] Iteration 104, pseudo-likelihood = -94.27, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -428.926228655 179.022133475 -234.182292435\n",
      "[EMF_RBM] Iteration 105, pseudo-likelihood = -92.95, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -429.814144395 178.761407054 -235.285835792\n",
      "[EMF_RBM] Iteration 106, pseudo-likelihood = -98.67, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -430.695913583 178.504258719 -236.380230663\n",
      "[EMF_RBM] Iteration 107, pseudo-likelihood = -94.61, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -431.571906796 178.250652768 -237.465868778\n",
      "[EMF_RBM] Iteration 108, pseudo-likelihood = -96.11, time = 23.55s\n",
      "computing TAP Free Energies\n",
      "monitor:  -432.441989279 178.00052138 -238.542668354\n",
      "[EMF_RBM] Iteration 109, pseudo-likelihood = -97.60, time = 23.67s\n",
      "computing TAP Free Energies\n",
      "monitor:  -433.306029071 177.753832689 -239.610511609\n",
      "[EMF_RBM] Iteration 110, pseudo-likelihood = -96.15, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -434.164592909 177.510370534 -240.670194233\n",
      "[EMF_RBM] Iteration 111, pseudo-likelihood = -97.38, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -435.017136014 177.270160305 -241.721130433\n",
      "[EMF_RBM] Iteration 112, pseudo-likelihood = -94.78, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -435.864031241 177.033148705 -242.763735334\n",
      "[EMF_RBM] Iteration 113, pseudo-likelihood = -93.30, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -436.705262184 176.799230009 -243.798088437\n",
      "[EMF_RBM] Iteration 114, pseudo-likelihood = -96.52, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -437.541175509 176.568303128 -244.824630349\n",
      "[EMF_RBM] Iteration 115, pseudo-likelihood = -93.07, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -438.371943786 176.34036222 -245.843532574\n",
      "[EMF_RBM] Iteration 116, pseudo-likelihood = -96.70, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -439.197634856 176.115310103 -246.854951039\n",
      "[EMF_RBM] Iteration 117, pseudo-likelihood = -92.40, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -440.018021105 175.893196284 -247.858593078\n",
      "[EMF_RBM] Iteration 118, pseudo-likelihood = -96.29, time = 23.56s\n",
      "computing TAP Free Energies\n",
      "monitor:  -440.833254088 175.673940372 -248.854683267\n",
      "[EMF_RBM] Iteration 119, pseudo-likelihood = -92.61, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -441.643465165 175.457393464 -249.843502126\n",
      "[EMF_RBM] Iteration 120, pseudo-likelihood = -92.89, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -442.448666903 175.243576181 -250.825024931\n",
      "[EMF_RBM] Iteration 121, pseudo-likelihood = -92.32, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -443.249049028 175.032450254 -251.799476023\n",
      "[EMF_RBM] Iteration 122, pseudo-likelihood = -93.77, time = 23.71s\n",
      "computing TAP Free Energies\n",
      "monitor:  -444.044616948 174.823930808 -252.766935469\n",
      "[EMF_RBM] Iteration 123, pseudo-likelihood = -95.04, time = 23.65s\n",
      "computing TAP Free Energies\n",
      "monitor:  -444.835421661 174.617951622 -253.727511594\n",
      "[EMF_RBM] Iteration 124, pseudo-likelihood = -94.38, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -445.621330616 174.414577675 -254.680992846\n",
      "[EMF_RBM] Iteration 125, pseudo-likelihood = -94.54, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -446.402591927 174.213665941 -255.627767331\n",
      "[EMF_RBM] Iteration 126, pseudo-likelihood = -92.74, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -447.179454891 174.015137061 -256.568161069\n",
      "[EMF_RBM] Iteration 127, pseudo-likelihood = -92.69, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -447.951673166 173.818991293 -257.501915257\n",
      "[EMF_RBM] Iteration 128, pseudo-likelihood = -92.43, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -448.719388048 173.625191963 -258.429208472\n",
      "[EMF_RBM] Iteration 129, pseudo-likelihood = -92.07, time = 23.55s\n",
      "computing TAP Free Energies\n",
      "monitor:  -449.482653064 173.433704099 -259.3501174\n",
      "[EMF_RBM] Iteration 130, pseudo-likelihood = -92.03, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -450.241332883 173.244529289 -260.264495974\n",
      "[EMF_RBM] Iteration 131, pseudo-likelihood = -94.81, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -450.995787215 173.057563461 -261.172798372\n",
      "[EMF_RBM] Iteration 132, pseudo-likelihood = -93.06, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -451.746120225 172.872829373 -262.075097568\n",
      "[EMF_RBM] Iteration 133, pseudo-likelihood = -94.04, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -452.492226931 172.690282824 -262.971323075\n",
      "[EMF_RBM] Iteration 134, pseudo-likelihood = -90.16, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -453.234548992 172.509881348 -263.861951126\n",
      "[EMF_RBM] Iteration 135, pseudo-likelihood = -91.53, time = 23.65s\n",
      "computing TAP Free Energies\n",
      "monitor:  -453.973087869 172.33151973 -264.747092256\n",
      "[EMF_RBM] Iteration 136, pseudo-likelihood = -91.27, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -454.707764736 172.155233173 -265.626622403\n",
      "[EMF_RBM] Iteration 137, pseudo-likelihood = -91.29, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -455.438229467 171.981017797 -266.500180313\n",
      "[EMF_RBM] Iteration 138, pseudo-likelihood = -90.72, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -456.164877225 171.808851955 -267.368173044\n",
      "[EMF_RBM] Iteration 139, pseudo-likelihood = -90.15, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -456.887646633 171.638647163 -268.230628575\n",
      "[EMF_RBM] Iteration 140, pseudo-likelihood = -91.35, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -457.606511114 171.470476423 -269.087433031\n",
      "[EMF_RBM] Iteration 141, pseudo-likelihood = -89.55, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -458.321804268 171.304245015 -269.939011757\n",
      "[EMF_RBM] Iteration 142, pseudo-likelihood = -91.63, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -459.033469596 171.139874772 -270.785390072\n",
      "[EMF_RBM] Iteration 143, pseudo-likelihood = -92.17, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -459.741496503 170.977385727 -271.626520903\n",
      "[EMF_RBM] Iteration 144, pseudo-likelihood = -91.32, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -460.445836217 170.816696184 -272.462440011\n",
      "[EMF_RBM] Iteration 145, pseudo-likelihood = -90.52, time = 23.66s\n",
      "computing TAP Free Energies\n",
      "monitor:  -461.146785073 170.657764282 -273.293482913\n",
      "[EMF_RBM] Iteration 146, pseudo-likelihood = -93.39, time = 23.55s\n",
      "computing TAP Free Energies\n",
      "monitor:  -461.844061677 170.500558661 -274.119395186\n",
      "[EMF_RBM] Iteration 147, pseudo-likelihood = -91.47, time = 23.55s\n",
      "computing TAP Free Energies\n",
      "monitor:  -462.537747435 170.345143081 -274.940182877\n",
      "[EMF_RBM] Iteration 148, pseudo-likelihood = -91.96, time = 23.71s\n",
      "computing TAP Free Energies\n",
      "monitor:  -463.228047272 170.191444908 -275.756118782\n",
      "[EMF_RBM] Iteration 149, pseudo-likelihood = -90.97, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -463.915071891 170.039402836 -276.567369119\n",
      "[EMF_RBM] Iteration 150, pseudo-likelihood = -90.07, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -464.598781338 169.889015013 -277.373892019\n",
      "[EMF_RBM] Iteration 151, pseudo-likelihood = -90.69, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -465.279531814 169.740270373 -278.176048899\n",
      "[EMF_RBM] Iteration 152, pseudo-likelihood = -87.96, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -465.957119837 169.593087682 -278.973715194\n",
      "[EMF_RBM] Iteration 153, pseudo-likelihood = -88.86, time = 23.56s\n",
      "computing TAP Free Energies\n",
      "monitor:  -466.63165131 169.447445648 -279.767018273\n",
      "[EMF_RBM] Iteration 154, pseudo-likelihood = -91.41, time = 23.54s\n",
      "computing TAP Free Energies\n",
      "monitor:  -467.303087545 169.30341562 -280.555830874\n",
      "[EMF_RBM] Iteration 155, pseudo-likelihood = -90.66, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -467.971405809 169.160970689 -281.340153166\n",
      "[EMF_RBM] Iteration 156, pseudo-likelihood = -90.02, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -468.636983893 169.020034772 -282.120431516\n",
      "[EMF_RBM] Iteration 157, pseudo-likelihood = -93.57, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -469.299616666 168.880563963 -282.89650614\n",
      "[EMF_RBM] Iteration 158, pseudo-likelihood = -87.79, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -469.959182302 168.742559824 -283.668248574\n",
      "[EMF_RBM] Iteration 159, pseudo-likelihood = -88.95, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -470.615881991 168.605982432 -284.435895779\n",
      "[EMF_RBM] Iteration 160, pseudo-likelihood = -88.70, time = 23.63s\n",
      "computing TAP Free Energies\n",
      "monitor:  -471.270045809 168.470829967 -285.199778663\n",
      "[EMF_RBM] Iteration 161, pseudo-likelihood = -90.25, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -471.921521432 168.337082074 -285.959765992\n",
      "[EMF_RBM] Iteration 162, pseudo-likelihood = -90.05, time = 23.64s\n",
      "computing TAP Free Energies\n",
      "monitor:  -472.570178792 168.204711596 -286.715748441\n",
      "[EMF_RBM] Iteration 163, pseudo-likelihood = -87.40, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -473.215792134 168.07366695 -287.467556003\n",
      "[EMF_RBM] Iteration 164, pseudo-likelihood = -87.07, time = 23.53s\n",
      "computing TAP Free Energies\n",
      "monitor:  -473.859035078 167.943955883 -288.215837336\n",
      "[EMF_RBM] Iteration 165, pseudo-likelihood = -86.43, time = 23.56s\n",
      "computing TAP Free Energies\n",
      "monitor:  -474.499903467 167.815551123 -288.960613825\n",
      "[EMF_RBM] Iteration 166, pseudo-likelihood = -86.89, time = 23.71s\n",
      "computing TAP Free Energies\n",
      "monitor:  -475.137994443 167.68847542 -289.70145503\n",
      "[EMF_RBM] Iteration 167, pseudo-likelihood = -89.18, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -475.773401184 167.56272603 -290.438450327\n",
      "[EMF_RBM] Iteration 168, pseudo-likelihood = -88.82, time = 23.56s\n",
      "computing TAP Free Energies\n",
      "monitor:  -476.406139631 167.438295435 -291.17162015\n",
      "[EMF_RBM] Iteration 169, pseudo-likelihood = -84.88, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -477.03632927 167.31510115 -291.901171424\n",
      "[EMF_RBM] Iteration 170, pseudo-likelihood = -89.01, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -477.663764572 167.193115147 -292.626917231\n",
      "[EMF_RBM] Iteration 171, pseudo-likelihood = -91.76, time = 23.54s\n",
      "computing TAP Free Energies\n",
      "monitor:  -478.288848793 167.072300689 -293.349299909\n",
      "[EMF_RBM] Iteration 172, pseudo-likelihood = -89.32, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -478.911507752 166.952692871 -294.068206221\n",
      "[EMF_RBM] Iteration 173, pseudo-likelihood = -92.52, time = 23.68s\n",
      "computing TAP Free Energies\n",
      "monitor:  -479.531628986 166.834191029 -294.783625567\n",
      "[EMF_RBM] Iteration 174, pseudo-likelihood = -89.42, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -480.14915525 166.716824417 -295.495471061\n",
      "[EMF_RBM] Iteration 175, pseudo-likelihood = -85.97, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -480.764353697 166.600513193 -296.204095915\n",
      "[EMF_RBM] Iteration 176, pseudo-likelihood = -90.49, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -481.377231338 166.485335791 -296.909413607\n",
      "[EMF_RBM] Iteration 177, pseudo-likelihood = -89.00, time = 23.56s\n",
      "computing TAP Free Energies\n",
      "monitor:  -481.987637958 166.371275925 -297.611282065\n",
      "[EMF_RBM] Iteration 178, pseudo-likelihood = -86.60, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -482.596009584 166.258317606 -298.310152188\n",
      "[EMF_RBM] Iteration 179, pseudo-likelihood = -86.80, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -483.202139325 166.146464967 -299.005804525\n",
      "[EMF_RBM] Iteration 180, pseudo-likelihood = -83.95, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -483.805852193 166.03572319 -299.698056876\n",
      "[EMF_RBM] Iteration 181, pseudo-likelihood = -85.77, time = 23.75s\n",
      "computing TAP Free Energies\n",
      "monitor:  -484.407267831 165.925993787 -300.387140544\n",
      "[EMF_RBM] Iteration 182, pseudo-likelihood = -89.66, time = 23.53s\n",
      "computing TAP Free Energies\n",
      "monitor:  -485.006469046 165.817249557 -301.073167166\n",
      "[EMF_RBM] Iteration 183, pseudo-likelihood = -88.49, time = 23.62s\n",
      "computing TAP Free Energies\n",
      "monitor:  -485.603650307 165.709469203 -301.756340842\n",
      "[EMF_RBM] Iteration 184, pseudo-likelihood = -87.76, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -486.198794398 165.602681323 -302.43660633\n",
      "[EMF_RBM] Iteration 185, pseudo-likelihood = -84.63, time = 23.56s\n",
      "computing TAP Free Energies\n",
      "monitor:  -486.79178727 165.496869207 -303.113871449\n",
      "[EMF_RBM] Iteration 186, pseudo-likelihood = -89.72, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -487.382750348 165.392033036 -303.788249546\n",
      "[EMF_RBM] Iteration 187, pseudo-likelihood = -86.37, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -487.971625182 165.288164877 -304.459688774\n",
      "[EMF_RBM] Iteration 188, pseudo-likelihood = -85.25, time = 23.63s\n",
      "computing TAP Free Energies\n",
      "monitor:  -488.558226068 165.185224979 -305.128044474\n",
      "[EMF_RBM] Iteration 189, pseudo-likelihood = -85.19, time = 23.57s\n",
      "computing TAP Free Energies\n",
      "monitor:  -489.142715049 165.083234082 -305.793457246\n",
      "[EMF_RBM] Iteration 190, pseudo-likelihood = -87.70, time = 23.61s\n",
      "computing TAP Free Energies\n",
      "monitor:  -489.725218169 164.98215056 -306.456094484\n",
      "[EMF_RBM] Iteration 191, pseudo-likelihood = -86.00, time = 23.56s\n",
      "computing TAP Free Energies\n",
      "monitor:  -490.305676698 164.881918865 -307.115954549\n",
      "[EMF_RBM] Iteration 192, pseudo-likelihood = -87.27, time = 23.53s\n",
      "computing TAP Free Energies\n",
      "monitor:  -490.884002898 164.782548501 -307.77293415\n",
      "[EMF_RBM] Iteration 193, pseudo-likelihood = -89.50, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -491.460180839 164.68405355 -308.427000887\n",
      "[EMF_RBM] Iteration 194, pseudo-likelihood = -82.78, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -492.034387729 164.58636363 -309.078406724\n",
      "[EMF_RBM] Iteration 195, pseudo-likelihood = -84.45, time = 23.60s\n",
      "computing TAP Free Energies\n",
      "monitor:  -492.606674755 164.4895107 -309.727169712\n",
      "[EMF_RBM] Iteration 196, pseudo-likelihood = -86.93, time = 23.59s\n",
      "computing TAP Free Energies\n",
      "monitor:  -493.176879305 164.39353308 -310.373079959\n",
      "[EMF_RBM] Iteration 197, pseudo-likelihood = -86.49, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -493.745055162 164.298370769 -311.016255975\n",
      "[EMF_RBM] Iteration 198, pseudo-likelihood = -85.74, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -494.31125919 164.204009364 -311.656768009\n",
      "[EMF_RBM] Iteration 199, pseudo-likelihood = -84.67, time = 23.58s\n",
      "computing TAP Free Energies\n",
      "monitor:  -494.875255891 164.110502161 -312.294321425\n",
      "[EMF_RBM] Iteration 200, pseudo-likelihood = -85.93, time = 23.62s\n",
      "computing TAP Free Energies\n"
     ]
    }
   ],
   "source": [
    "rbm = emf_rbm.EMF_RBM(verbose=True, monitor=True, n_iter=250)\n",
    "print rbm\n",
    "rbm= rbm.fit(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-11-08T22:57:27.868434",
     "start_time": "2016-11-08T22:57:27.697130"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.plot(rbm.entropies-rbm.entropies[0], color='red')\n",
    "plt.plot(rbm.mean_field_energies-rbm.mean_field_energies[0], color='blue')\n",
    "plt.plot(rbm.free_energies-rbm.free_energies[0], color='green')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}