test_plot.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import os\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "from sklearn.metrics import log_loss\n",
    "from decentralized_SGD_logistic import DecentralizedSGDLogistic\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from scipy.special import expit as sigmoid\n",
    "\n",
    "from helpers import plot_losses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RCV dataset from ChocoSGD Paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_logistic(A, y, param,):\n",
    "    m = DecentralizedSGDLogistic(param)\n",
    "    res = m.fit(A, y)\n",
    "    print('{} - score: {1:.4f}'.format(param, m.score(A, y)))\n",
    "    return res, m"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataset...\n"
     ]
    }
   ],
   "source": [
    "dataset_path = os.path.expanduser('../ChocoSGD/data/rcv1_test.binary.bz2')\n",
    "print('Loading dataset...')\n",
    "A, y = load_svmlight_file(dataset_path)\n",
    "A_p = A[:100000]\n",
    "y_p = y[:100000]\n",
    "y_p = 1*(y_p > 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute the baseline with SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/adirlou/.local/lib/python3.6/site-packages/sklearn/linear_model/stochastic_gradient.py:166: FutureWarning: max_iter and tol parameters have been added in SGDClassifier in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.97885\n"
     ]
    }
   ],
   "source": [
    "clf = SGDClassifier(loss='log', penalty='l2',alpha =1/A_p.shape[0])\n",
    "clf.fit(A_p, y_p)\n",
    "print(clf.score(A_p, y_p))\n",
    "x_predict = clf.predict(A_p)\n",
    "exact_optimum = log_loss(x_predict, y_p) + (1 / A_p.shape[0]) * np.sum(x_predict**2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute with Decentralized SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg = 1 / A_p.shape[0]\n",
    "n_features = A_p.shape[1]\n",
    "\n",
    "params_disconnected = Parameters(num_epoch=5, lr_type='bottou',\n",
    "                           initial_lr=10, tau=n_features, regularizer=reg,\n",
    "                           quantization_method='full', n_machines=128,\n",
    "                           communication_method='plain',topology='complete',\n",
    "                           data_distribution_random_seed=2,\n",
    "                           data_distribution_strategy='naive')\n",
    "\n",
    "res_disconnected, dec_log = run_logistic(A_p, y_p, params_disconnected)\n",
    "\n",
    "\"\"\"params_ring = dict(name=\"chocosgd-centralized\", num_epoch=5, lr_type='bottou',\n",
    "                           initial_lr=0.2, tau=n_features, regularizer=reg,\n",
    "                           quantization='full', n_cores=10,\n",
    "                           method='plain', topology='centralized', estimate='final',\n",
    "                           split_data_random_seed=2, distribute_data=True,\n",
    "                           split_data_strategy='naive')\n",
    "\n",
    "res_centralized = run_logistic(A_p, y_p, params_ring)\"\"\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Higgs Boson Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_logistic(A, y, param, logging=False):\n",
    "    m = DecentralizedSGDLogistic(**param)\n",
    "    list_losses = m.fit(A, y, logging=logging)\n",
    "    if logging:\n",
    "        print()\n",
    "        print('{0} - score: {1:.4f}'.format(param, m.score(A, y)))\n",
    "    return list_losses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y, A = helpers.load_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compute the baseline with SGD classifier "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.741736\n",
      "Final loss: 0.5129295355244131\n"
     ]
    }
   ],
   "source": [
    "# Fit a SGD\n",
    "clf = SGDClassifier(loss='log', penalty='l2',alpha =1/A.shape[0], tol=1e-4, n_jobs=1)\n",
    "clf.fit(A, y)\n",
    "\n",
    "accuracy = clf.score(A, y)\n",
    "optimum_x = clf.predict_proba(A)[:, 1]\n",
    "\n",
    "# Optimal loss, useful for plots\n",
    "optimum_loss = (-(y.T.dot(np.log(optimum_x)) + (1 - y).T.dot(np.log(1 - optimum_x))) / A.shape[0])\n",
    "\n",
    "print('Accuracy:', accuracy)\n",
    "print('Final loss:', optimum_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Computing with Decentralized SGD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "reg = 1 / A.shape[0]\n",
    "n_features = A.shape[1]\n",
    "compute_loss_every = 30\n",
    "params_disconnected = dict(num_epoch=3, lr_type='bottou', tol=1e-10,\n",
    "                           initial_lr=0.05, regularizer=reg,\n",
    "                           quantization_method='full', n_machines=512,\n",
    "                           communication_method='plain', topology='disconnected',\n",
    "                           data_distribution_random_seed=2, random_seed=2,\n",
    "                           data_distribution_strategy='naive', \n",
    "                           compute_loss_every=compute_loss_every)\n",
    "\n",
    "losses_disconnected = run_logistic(A, y, params_disconnected, logging=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}