Some-basics-about-learning-process.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 4:  Some basics about learning process\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Activation functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "x = np.arange(-6, 6, 0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Linear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXYAAAD8CAYAAABjAo9vAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xd8lfX5//HXJyFhJBBkhRX2nkoCiFZLFBUVB45vtW6rVFs7/crWah3gaK21WqtWW6ttWhmCiDgwuFFAJYuwwt4zZJB5rt8fod+ftUjgnJPc59x5Px+PPB6c5L4/93V5zDt37tznOs7MEBER/4jxugAREQkvBbuIiM8o2EVEfEbBLiLiMwp2ERGfUbCLiPiMgl1ExGcU7CIiPqNgFxHxmUZeHLRNmzbWrVu3oPYtKSkhISEhvAV5RL1ELj/1o14iUzC9rFixYq+Zta1tO0+CvVu3bixfvjyofZcsWcLo0aPDW5BH1Evk8lM/6iUyBdOLc27T8WynSzEiIj6jYBcR8RkFu4iIz4Ql2J1zLZ1zs5xz+c65Vc65UeFYV0RETly4/nj6BLDIzK5wzsUDzcK0roiInKCQg905lwScCdwIYGYVQEWo64qISHDCcSmmO7AHeNE596Vz7nnnnD9uNBURiUIu1LfGc86lAUuB083sM+fcE8AhM7v7G9tNACYAJCcnp2ZkZAR1vOLiYhITE0OqOVKol8jlp37US2QorjDmra9gfK94msW5oHpJT09fYWZptW5oZiF9AO2BjV97fAbwxrH2SU1NtWBlZmYGvW+kUS+Ry0/9qBdvBQIBW7Byu6Xe/7b1nPKGvZO708yC6wVYbseRyyFfYzeznc65Lc65vma2GjgbyAt1XRGRaLfrUBl3v5bD23m7GNwpib/9YCT9O7So8+OG666YnwCvHLkjpgC4KUzriohEHTPjX8u38MAbq6ioCjDl/H784DvdaRRbPy8dCkuwm9lXQO3XfUREfG7zvlKmzM3i43X7GNG9FQ9fPoTuber3fhJPhoCJiPhNdcD4yycbeeyt1cTGOB64dBDfH9GFmBhX77Uo2EVEQrRmVxETZ2Xx1ZaDnNWvHQ9cOoiOLZt6Vo+CXUQkSBVVAZ55fz1PvreWxMaNeOKqk7l4aEecq/+z9K9TsIuIBCFr60Emzsoif2cRFw3tyK8uGkCbxMZelwUo2EVETsjhimp+9+4anvuwgLbNG/Pc9WmcMyDZ67L+g4JdROQ4fbp+H1PmZLFxXylXj0hh8vn9SWoa53VZ/0XBLiJSi0Nllcx8M5+/f7aZLq2a8fdbRnJarzZel/WtFOwiIseweNUups3NYXdRGbee0Z1fntOXpvGxXpd1TAp2EZGj2Fdczn2v5zF/5Xb6JjfnmetSOTmlpddlHRcFu4jI15gZ81du577X8ygqq+TnY3rzo9G9iG8UPe8kqmAXETliZ2EZ01/L4d1Vuxia0pJHLh9C3/bNvS7rhCnYRaTBMzMylm3hoTdWURkIMP3C/tx0endiPRgHEA4KdhFp0DbtK2Hy7Gw+LdjHqB6tmXn5YLq2ju43gVOwi0iDVB0wXvhoA795ZzVxMTHMuGwwVw1P8XwcQDgo2EWkwVm9s4iJs1aycmshY/q344FLB9M+qYnXZYWNgl1EGoyKqgBPZa7j6SXraNEkjievPoVxQzr44iz96xTsItIgfLn5AJNmZ7FmVzGXntyRey4aSKuEeK/LqhMKdhHxtcMV1fz2ndX8+aMNJLdowgs3pnFWv8ga2hVuCnYR8a1P1u1l8pxsNu8v5fsjuzDl/H40bxJ5Q7vCTcEuIr5TeLiSGQtXkbFsC91aNyNjwqmc2qO112XVGwW7iPjKO3m7mP5aNnuKyvnhmT34+Zg+ET+0K9wU7CLiC3uLy7l3fi4LsnbQr31znrs+jSGdo2NoV7gp2EUkqpkZ877azn2v51JSXs2d5/ThttE9iYuNnqFd4aZgF5Gotf3gYabNzSZz9R5O6VIztKt3cvQN7Qq3sAW7cy4WWA5sM7Nx4VpXROSbAgHjlc838/Cb+VQHjHvGDeCG07pF7dCucAvnGfvPgFVAizCuKSLyHzbsLWHS7Cw+37Cf03u1Zsb4IXRp3czrsiJKWILdOdcZuBB4EPhlONYUEfm6quoACwsqmPfuB8Q3iuGRy4dwZVpn340DCIdwnbH/DpgI6OKWiIRd3vZDTJqdRfa2Ss4dkMz9lw4iuYV/hnaFmzOz0BZwbhxwgZn9yDk3Gvjfo11jd85NACYAJCcnp2ZkZAR1vOLiYhITE0OoOHKol8jlp36iuZfKgDF/fSULCypJiIMrehhndE3wxVl6MM9Lenr6CjNLq3VDMwvpA5gBbAU2AjuBUuDlY+2TmppqwcrMzAx630ijXiKXn/qJ1l6Wb9xvZ/9miXWdtMB+kfGl7S8uj9pejiaYXoDldhy5HPKlGDObAkwB+NoZ+7WhrisiDVNpRRWPvrWav3yykY5JTfnLTcMZ3bed12VFFd3HLiIR46O1e5k8J4utBw5z/aiuTBzbj8TGiqkTFdb/Yma2BFgSzjVFxP8KSyt5cGEe/1q+lR5tEvjXD0cxonsrr8uKWvpRKCKeWpSzk7vn5bC/pILbvtuTn4/pTZO4hjW0K9wU7CLiiT1FNUO73sjewYAOLXjxxuEM6pTkdVm+oGAXkXplZsz5Yhu/XpDH4cpq7jqvLxPO7NGgh3aFm4JdROrN1gOlTJ2bwwdr9pDa9SQevnwIvdpF5z32kUzBLiJ1LhAw/rZ0Ew8vygfg3osGcP2obsRoaFedULCLSJ1av6eYSbOyWL7pAGf0bsND4weT0kpDu+qSgl1E6kRldYBnPyjgicVraRoXy2NXDuXyYZ18MQ4g0inYRSTscrcXMnFWFrnbD3H+oPbcd8lA2jXX0K76omAXkbApq6zmyffW8sz7BZzULJ5nrh3G2EEdvC6rwVGwi0hYLN+4n4mzsyjYU8KVqZ2ZfuEAkprFeV1Wg6RgF5GQFJdX8eiifF5auomOSU156eYRnNmnrddlNWgKdhEJ2vtr9jB1TjbbCw9zw6hu3HVeXxI0tMtzegZE5IQdLK3g/gWrmP3FVnq2TeDVH44irZuGdkUKBbuInJA3s3dw97xcDpRWcEd6L+44q5eGdkUYBbuIHJfdh8q4Z14ui3J3MqhTC/5683AGdtTQrkikYBeRYzIzXl2xlQcW5FFWFWDS2H7cekZ3GmloV8RSsIvIt9qyv5Spc7P5cO1eRnRrxczLB9OjrYZ2RToFu4j8l+qA8dKnG3lk0WpiHNx/6SCuGdFFQ7uihIJdRP7Dut1FTJyVxRebDzK6b1seHD+YTi2bel2WnAAFu4gANUO7/vT+en6/eB3NGsfy+PeGcunJGtoVjRTsIkL21kLumrWS/J1FXDikA/deNJC2zRt7XZYEScEu0oCVVVbzu3fX8tyHBbROiOdP16Vy3sD2XpclIVKwizRQnxXsY/KcbDbsLeF7aSlMvbA/SU01tMsPFOwiDUxRWSUPL8rn5aWbSWnVlFduGcnpvdp4XZaEUcjB7pxLAV4CkgEDnjWzJ0JdV0TCLzN/N9PmZrPjUBk/+E537jy3D83idX7nN+F4RquAO83sC+dcc2CFc+4dM8sLw9oiEgYHSir49YI85n65jd7tEpl9+2kM63KS12VJHQk52M1sB7DjyL+LnHOrgE6Agl3EY2bG5zuquPO371N4uJKfntWLH5/Vi8aNNLTLz8L6O5hzrhtwCvBZONcVkRO361AZ01/L4Z28coZ0TuLlW0bSv0MLr8uSeuDMLDwLOZcIvA88aGZzjvL1CcAEgOTk5NSMjIygjlNcXExioj9mVaiXyBXN/ZgZH2yrIiO/gqoAXNjFuKhPArE+GAcQzc/LNwXTS3p6+gozS6t1QzML+QOIA94Cfnk826emplqwMjMzg9430qiXyBWt/WzaW2Lff+5T6zppgf3PM5/Yhj3FUdvL0TT0XoDldhwZG467YhzwZ2CVmf021PVE5MRVB4wXP97AY2+vplFMDA+OH8TVw2uGdm30ujipd+G4xn46cB2Q7Zz76sjnpprZwjCsLSK1WLOrZmjXV1sOcla/djw4fhAdkjS0qyELx10xHwHRf/FOJMpUVAX445L1/CFzLc2bxPHEVSdz8dCOGtoleuWpSDRaueUgk2Znkb+ziIuGduTeiwbQOlFDu6SGgl0kihyuqObxd9fw/IcFtG3emOeuT+OcAclelyURRsEuEiU+Xb+PKXOy2LivlKtHpDDlgv60aKKhXfLfFOwiEe5QWSUzFubzj88307V1M/5+60hO66mhXfLtFOwiEWzxql1Mm5vD7qIyJpzZg1+M6UPTeI0DkGNTsItEoH3F5dz3eh7zV26nb3JznrkulZNTWnpdlkQJBbtIBDEz5q/czn2v51FUVsnPx/TmR6N7Ed8oxuvSJIoo2EUixM7CMqbNzWZx/m6GprTkkcuH0Ld9c6/LkiikYBfxWCBgZCzbwoyFq6gMBJh+YX9uOr27L4Z2iTcU7CIe2ri3hMlzslhasJ/TerZmxmWD6do6weuyJMop2EU8UFUd4IWPN/Cbt9cQHxvDzMsG873hKRoHIGGhYBepZ/k7DzFpVhYrtxYypn8yD1w6iPZJTbwuS3xEwS5ST8qrqnkqcz1PZ64jqWkcT159CuOGdNBZuoSdgl2kHny5+QCTZmexZlcxl57ckXsuGkirhHivyxKfUrCL1KHSiip++/YaXvh4A8ktmvDCjWmc1U9Du6RuKdhF6sgn6/YyeU42m/eXcs3ILkw+vx/NNbRL6oGCXSTMCg9XMmPhKjKWbaF7mwQyJpzKqT1ae12WNCAKdpEwejt3J9Nfy2FvcTk/PLMHvzinD03iNLRL6peCXSQM9haXc+/8XBZk7aBf++Y8f0MaQzpraJd4Q8EuEgIzY95X27nv9VxKyqu585w+3Da6J3GxGtol3lGwiwRp+8HDTJubTebqPZzSpWZoV+9kDe0S7ynYRU5QIGC88vlmZi5cRcDgnnEDuOG0bhraJRFDwS5yAjbsLWHS7Cw+37Cf7/Rqw4zLBpPSqpnXZYn8BwW7yHGoqg7w/EcbePydNTRuFMMjVwzhytTOGgcgEUnBLlKLvO2HmDQ7i+xthZw3MJn7LxlEuxYa2iWRKyzB7pwbCzwBxALPm9nMcKwr4qXyqmr+8N46/rhkPS2bxfH0NcO4YHAHr8sSqVXIwe6ciwWeAs4BtgLLnHPzzSwv1LVFvLLuQDUP/P4j1u0u5rJhnbhn3ABaNtPQLokO4ThjHwGsM7MCAOdcBnAJoGCXqFNSXsVjb6/mL5+V0bFlU/5y03BG923ndVkiJ8SZWWgLOHcFMNbMbjny+DpgpJnd8Y3tJgATAJKTk1MzMjKCOl5xcTGJiYkh1Rwp1EtkydlbzV9yy9l72Dijg/H9gQk0bRT9fxz1w3Pzbw29l/T09BVmllbbdvX2x1MzexZ4FiAtLc1Gjx4d1DpLliwh2H0jjXqJDIWllTzwRh6vrthKjzYJPH39EEo3ZUVtP98Uzc/NN6mX4xOOYN8GpHztcecjnxOJeItydnL3vBz2l1Rw++ie/Ozs3jSJi2XJJq8rEwleOIJ9GdDbOdedmkC/Cvh+GNYVqTO7i8r41bxc3szZyYAOLXjxxuEM6pTkdVkiYRFysJtZlXPuDuAtam53fMHMckOuTKQOmBlzvtjGrxfkcbiymrvO68uEM3toaJf4SliusZvZQmBhONYSqStbD5QydW4OH6zZQ1rXk5h5+RB6tfPHH+JEvk6vPBXfCwSMlz/bxMNv5mPAfRcP5LpTuxKjoV3iUwp28bX1e4qZNCuL5ZsOcEbvmqFdnU/S0C7xNwW7+FJldYBnPyjgicVraRoXy2NXDuXyYZ00tEsaBAW7+E7OtkImzc4id/shLhjcnnsvHki75hraJQ2Hgl18o6yymt8vXsufPiigVUI8z1w7jLGDNLRLGh4Fu/jCso37mTQ7i4I9JVyZ2pnpFw4gqVmc12WJeELBLlGtuLyKRxfl89LSTXRq2ZS//WAEZ/Ru63VZIp5SsEvUen/NHqbOyWZ74WFuGNWNu87rS0Jj/S8tou8CiToHSyu4f8EqZn+xlZ5tE5h12yhSu7byuiyRiKFgl6iyMHsH98zL4WBpJXek9+KOs3rRJC7W67JEIoqCXaLC7kNl3DMvl0W5OxnUqQV/vXkEAztqaJfI0SjYJaKZGa+u2MoDC/IorwowaWw/bj2jO400tEvkWynYJWJt2V/K1LnZfLh2LyO6tWLm5YPp0VZDu0Rqo2CXiFMdMF76dCOPLFpNjIP7Lx3ENSO6aGiXyHFSsEtEWbe7iImzsvhi80FG923Lg+MH06llU6/LEokqCnaJCJXVAf70/np+v3gdCY1jefx7Q7n0ZA3tEgmGgl08l721kLtmrSR/ZxHjhnTg3osH0iaxsddliUQtBbt4pqyymsffXcNzHxTQJrExz16XyrkD23tdlkjUU7CLJz4r2MfkOdls2FvCVcNTmHJBf5KaamiXSDgo2KVeFZVV8vCifF5eupmUVk155ZaRnN6rjddlifiKgl3qTWb+bqbOzWbnoTJ+8J3u3HluH5rF639BkXDTd5XUuf0lFfz69Vxe+2o7vdslMvv20xjW5SSvyxLxLQW71BkzY0HWDu6dn0vh4Up+elYvfnxWLxo30tAukbqkYJc6setQGdNfy+GdvF0M6ZzEK7eOpF/7Fl6XJdIghBTszrlHgYuACmA9cJOZHQxHYRKdzIx/LtvCgwtXUVEVYOoF/bj5dA3tEqlPoZ6xvwNMMbMq59zDwBRgUuhlSTTavK+UyXOy+GT9PkZ2b8XDlw+hW5sEr8sSaXBCCnYze/trD5cCV4RWjkSj6oDx1sZK5i5+n0YxMTw0fjBXDU/R0C4RjzgzC89Czr0O/NPMXv6Wr08AJgAkJyenZmRkBHWc4uJiEhP9MbrVD71sLQrwQk45BYUBhraN5YaB8bRqEv2XXfzw3PybeolMwfSSnp6+wszSatuu1mB3zr0LHO113tPMbN6RbaYBacBldhw/KdLS0mz58uW1bXZUS5YsYfTo0UHtG2miuZeKqgBPL1nHU5nraN4kjv/p6Zh09dm+GdoVzc/NN6mXyBRML8654wr2Wi/FmNmYWg50IzAOOPt4Ql2i38otB5k4K4vVu4q4eGhHfnXRALKXf+qbUBeJdqHeFTMWmAh818xKw1OSRKrDFTVDu57/sIB2zZvw/PVpjBmQ7HVZIvINod4V8wegMfDOkbO1pWZ2W8hVScT5dP0+Js/JYtO+Uq4e0YUpF/SjRRMN7RKJRKHeFdMrXIVIZDpUVsmMhfn84/PNdG3djL/fOpLTempol0gk0ytP5VstXrWLaXNz2F1UxoQze/CLMX1oGq9xACKRTsEu/2VfcTn3vZ7H/JXb6de+OX+6LpWhKS29LktEjpOCXf6PmTF/5XbunZ9LcXkVvxjTh9tH9yS+UfTfly7SkCjYBYAdhYeZPjeHxfm7OTmlJY9cMYQ+yc29LktEgqBgb+ACAeMfyzYzY2E+VYEA0y/sz02ndydW4wBEopaCvQHbuLeEyXOyWFqwn9N6tmbmZUPo0rqZ12WJSIgU7A1QVXWAFz7ewG/eXkN8bAwzLqsZ2qVXjor4g4K9gcnfeYhJs7JYubWQMf2TeeDSQbRPauJ1WSISRgr2BqK8qpqnMtfzdOY6kprG8eTVpzBuSAedpYv4kIK9Afhi8wEmzcpi7e5ixp/SiXvGDeCkhHivyxKROqJg97HSiip+8/YaXvh4A+1bNOHFG4eT3q+d12WJSB1TsPvUx+v2MnlOFlv2H+baU7swaWw/mmtol0iDoGD3mcLDlcxYuIqMZVvo3iaBf044lZE9WntdlojUIwW7j7ydu5Ppr+Wwr6SC277bk5+P6U2TOA3tEmloFOw+sKeonHtfz+WNrB3079CCP98wnMGdk7wuS0Q8omCPYmbGa19t477X8ygtr+Z/z+3DD7/bk7hYDe0SacgU7FFq28HDTJubzZLVexjWpWZoV692GtolIgr2qBMIGK98tomZb+YTMPjVRQO4flQ3De0Skf+jYI8iBXuKmTw7m8837ueM3m14aPxgUlppaJeI/CcFexSoqg7w/EcbePydNTRuFMOjVwzhitTOGgcgIkelYI9wedsPMXH2SnK2HeK8gcncf8kg2rXQ0C4R+XYK9ghVVlnNH95bxzPvr6dls3j+eM0wzh/cweuyRCQKKNgj0IpN+5k4K4v1e0q4bFjN0K6WzTS0S0SOj4I9gpSUV/HoW6v566cb6ZjUlL/ePILv9mnrdVkiEmXCEuzOuTuBx4C2ZrY3HGs2NB+u3cOUOdlsPXCY60d1ZeLYfiQ21s9dETlxISeHcy4FOBfYHHo5DU9haSUPvJHHqyu20qNtAq/eNorh3Vp5XZaIRLFwnBI+DkwE5oVhrQZlxa4q7nr8ffaXVPCj0T356dka2iUioQsp2J1zlwDbzGyl7qk+fruLyrh3fi4Ls8sZ2LEFL944nEGdNLRLRMLDmdmxN3DuXaD9Ub40DZgKnGtmhc65jUDat11jd85NACYAJCcnp2ZkZARVcHFxMYmJiUHt6zUz4+PtVfwjv4Lyajg/xbikbwKNfDAOIJqfl6PxUz/qJTIF00t6evoKM0urdUMzC+oDGAzsBjYe+aii5jp7+9r2TU1NtWBlZmYGva+XNu8rsWufX2pdJy2wy5/+2NbuKoraXo7GT72Y+asf9RKZgukFWG7Hkc9BX4oxs2zg/95As7Yz9oYqEDBe+nQjj7y1Ggf8+pKBXDuyKzExjq15XlcnIn6k++nq0LrdxUyencXyTQc4s09bHho/iM4naWiXiNStsAW7mXUL11rRrrI6wLMfFPDE4rU0jYvlN1cO5bJhnTS0S0Tqhc7YwyxnWyGTZmeRu/0QFwxuz70XD6Rdcw3tEpH6o2APk7LKan6/eC1/+qCAVgnxPHPtMMYO0tAuEal/CvYwWLZxP5NmZVGwt4QrUzsz/cIBJDWL87osEWmgFOwhKC6v4pFF+bz06SY6tWzK334wgjN6a2iXiHhLwR6kzNW7mTYnmx2HyrjxtG7cdV5fEjS0S0QigJLoBB0oqeD+BXnM+XIbPdsmMOu2UaR21dAuEYkcCvbjZGYszN7Jr+bncLC0kjvSe/GTs3vRuJGGdolIZFGwH4fdh8q4e14Ob+XuYnCnJF66eSQDOrbwuiwRkaNSsB+DmfHqiq08sCCP8qoAk8b249YzutMoNsbr0kREvpWC/Vts2V/KlDnZfLRuLyO6t2LmZYPp0dYfU+VExN8U7N9QHTD++slGHn1rNbExjvsvHcQ1I7oQ44PRuiLSMCjYv2btriImzs7iy80HSe/blgfHD6Zjy6ZelyUickIU7NQM7frjkvX84b11JDSO5XffO5lLTu6ooV0iEpUafLBnby3krlkryd9ZxLghHbj34oG0SWzsdVkiIkFrsMFeVlnN4++u4fkPN9AmMZ7nrk/jnAHJXpclIhKyBhnsnxXsY/KcbDbsLeHqESlMPr8/SU01tEtE/KFBBXtRWSUz38znlc8206VVM/5+y0hO69XG67JERMKqwQT7e/m7mDY3h12HyrjlO9355bl9aBbfYNoXkQbE98m2v6SCX7+ey2tfbad3u0Sevv00TulyktdliYjUGd8Gu5nxetYO7pufS+HhSn52dm9+lN5TQ7tExPd8Gew7C8uY/lo2767azdDOSbxy60j6tdfQLhFpGHwV7GZGxrItPPTGKiqqA0y9oB83n66hXSLSsPgm2DftK2Hy7Gw+LdjHqT1aMfOyIXRrk+B1WSIi9S7qg706YLz48QYee3s1cTExPDR+MFcNT9HQLhFpsEIOdufcT4AfA9XAG2Y2MeSqjtPqnTVDu1ZuOcjZ/drxwPhBdEjS0C4RadhCCnbnXDpwCTDUzMqdc+3CU9axVVQFeHrJOp7KXEfzJnE8cdXJXDxUQ7tERCD0M/bbgZlmVg5gZrtDL+nYCg5WM+PJj1i9q4hLTu7Iry4aSKuE+Lo+rIhI1Ag12PsAZzjnHgTKgP81s2Whl3V0Ty5ey2+XlpHcAv58Qxpn99fQLhGRb3JmduwNnHsXaH+UL00DHgQygZ8Cw4F/Aj3sKIs65yYAEwCSk5NTMzIyTrjYpduryN5dxjUDE2gWF/2XXYqLi0lM9Mfb7fmpF/BXP+olMgXTS3p6+gozS6t1QzML+gNYBKR/7fF6oG1t+6WmplqwMjMzg9430qiXyOWnftRLZAqmF2C5HUc2h/rKndeAdADnXB8gHtgb4poiIhKCUK+xvwC84JzLASqAG478VBEREY+EFOxmVgFcG6ZaREQkDDRERUTEZxTsIiI+o2AXEfEZBbuIiM8o2EVEfKbWV57WyUGd2wNsCnL3NvjnXnn1Ern81I96iUzB9NLVzNrWtpEnwR4K59xyO56X1EYB9RK5/NSPeolMddmLLsWIiPiMgl1ExGeiMdif9bqAMFIvkctP/aiXyFRnvUTdNXYRETm2aDxjFxGRY4jaYHfO/cQ5l++cy3XOPeJ1PaFyzt3pnDPnXBuvawmWc+7RI89JlnNurnOupdc1nSjn3Fjn3Grn3Drn3GSv6wmWcy7FOZfpnMs78j3yM69rCpVzLtY596VzboHXtYTKOdfSOTfryPfLKufcqHCuH5XB/o030R4IPOZxSSFxzqUA5wKbva4lRO8Ag8xsCLAGmOJxPSfEORcLPAWcDwwArnbODfC2qqBVAXea2QDgVODHUdzLv/0MWOV1EWHyBLDIzPoBQwlzX1EZ7HjwJtp17HFgIhDVf/Aws7fNrOrIw6VAZy/rCcIIYJ2ZFRwZSZ1BzQlE1DGzHWb2xZF/F1ETHJ28rSp4zrnOwIXA817XEirnXBJwJvBnqBl/bmYHw3mMaA32f7+J9mfOufedc8O9LihYzrlLgG1mttLrWsLsZuBNr4s4QZ2ALV97vJUoDsN/c851A04BPvO2kpD8jpqTn4DXhYRBd2AP8OKRS0vPO+cSwnmAUN9Bqc7U8ibajYBW1PyKORz4l3PuqG+iHQlq6WUqNZdhosKxejGzeUe2mUZMPxM0AAABn0lEQVTNpYBX6rM2+W/OuURgNvBzMzvkdT3BcM6NA3ab2Qrn3Giv6wmDRsAw4Cdm9plz7glgMnB3OA8QkcxszLd9zTl3OzDnSJB/7pwLUDN3YU991Xcivq0X59xgan56r3TOQc2liy+ccyPMbGc9lnjcjvW8ADjnbgTGAWdH6g/aY9gGpHztcecjn4tKzrk4akL9FTOb43U9ITgduNg5dwHQBGjhnHvZzKL13du2AlvN7N+/Qc2iJtjDJlovxfjiTbTNLNvM2plZNzPrRs0TPixSQ702zrmx1Py6fLGZlXpdTxCWAb2dc92dc/HAVcB8j2sKiqs5U/gzsMrMfut1PaEwsylm1vnI98hVwHtRHOoc+f7e4pzre+RTZwN54TxGxJ6x10Jvoh2Z/gA0Bt458hvIUjO7zduSjp+ZVTnn7gDeAmKBF8ws1+OygnU6cB2Q7Zz76sjnpprZQg9rkv/vJ8ArR04gCoCbwrm4XnkqIuIz0XopRkREvoWCXUTEZxTsIiI+o2AXEfEZBbuIiM8o2EVEfEbBLiLiMwp2ERGf+X/fa9sk+bHEEQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1087dc5f8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def linear(x):\n",
    "    a = []\n",
    "    for item in x:\n",
    "        a.append(item)\n",
    "    return a\n",
    "\n",
    "y = linear(x)\n",
    "\n",
    "plt.plot(x,y)\n",
    "plt.grid()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sigmoid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VfWd//HXJztJCFsgbFF2EBDZxL2CC+JSnaptta3VqsPUqf7sqK3aVp2xM61tnbG2ta1r1akjWrVClYpawdYqqwgYNsMihC2sIfvNvfczfyT6yyCQkNxw7vJ+Ph73kdx7v/fm/SXknZPvPfccc3dERCS5pAUdQEREYk/lLiKShFTuIiJJSOUuIpKEVO4iIklI5S4ikoRU7iIiSUjlLiKShFTuIiJJKCOoL1xYWOgDBgxo02Orq6vJy8uLbaCAJNNcILnmo7nEp1Sfy5IlS3a5e8+WxgVW7gMGDGDx4sVteuy8efOYPHlybAMFJJnmAsk1H80lPqX6XMzs49aM07KMiEgSUrmLiCQhlbuISBJSuYuIJKEWy93MnjCzcjP78BD3m5n9wsxKzWy5mY2PfUwRETkSrdlyfxKYdpj7zweGNl2mA79pfywREWmPFsvd3f8K7DnMkEuAp73RfKCrmfWJVUARETlysdjPvR+wudn1sqbbtsXguUVE4oa7Ux+OUhuKUBeOUBuKUB+ONl4amn0ejhAKR2mIRAmFo4QiTkMkSkPTbWcfV8QJxV07NOtRfROTmU2ncemGoqIi5s2b16bnqaqqavNj400yzQWSaz6aS3xq61yi7lQ1QFXIqW5wqhqcmganpgFqwk5N2KkNQ12zj3WRxo/1EQhFGj/G4qzTFds3sfeYzA79vsSi3LcAxc2u92+67TPc/RHgEYCJEyd6W99llurvUItnyTQfzSU+HTiXmlCYrfvq2F5Rx/b9dezYX8fOyvrGS1U9u6vq2V0doqK2AT9MM+dmpZOfnUF+TuOlZ3YGedkZ5GWlk5udQW5mOrlZ6eRkpZOTkU6nrHRyMtPIyUgnJzOd7Iw0sjPTyEpPJysjjeyMNDIz0shKTyMrI43MdCMzPY2MNMPMDjqXWIpFuc8CbjSzGcBJQIW7a0lGRGIiGnW27a9j465qNuyq5p3VIZ4rW8KmPTVs3VfL3pqGzzymc3YGPTtnU5ifzbCizvTIz6J7XjbdczPplpdF19wsunbKpEunTAo6ZdI5J4PM9OTaM7zFcjezZ4HJQKGZlQH3AJkA7v5bYDZwAVAK1ADf6KiwIpK83J3yynpWbt3P6u2VrN1RyUfllawrr6a2IfLpuIw0OLZHJcXdcxlb3JW+XTvRt2sOfbp0ondBDkUFOXTKSg9wJvGhxXJ39ytbuN+Bb8UskYikhL3VIZZu3ssHmytYtnkfJVsr2FUV+vT+Pl1yGNIrnysmdWdwz3wGFeYxoDCP1Uvnc9aUycEFTxCBHRVSRFJL+f463lu/m/nrd7No415Ky6sASDMY2qszk4f3YlTfAkb2KWBE7wK65GYe9HnWNq1Xy+Gp3EWkQ9SHIyzcsIe31+zk7bU7+aipzDvnZDDx2G58YVw/JhzbjeP7dSEvW1UUa/oXFZGYqaoP85dVO3i9ZAdvr91JVX2YrPQ0Jg3szuUT+nPq4EJG9i0gPU1b3x1N5S4i7VIfjjB39U5mfrCFt1aXUx+OUpifzedP6MPZI4o4dUgPcrNUNUeb/sVFpE1Wb9/Psws2MXPZVvbVNFCYn80VJxZz4Zi+TDy2G2naOg+Uyl1EWq0hEmX2im089e5G3t+0j6z0NKaOKuKyCf05Y0ghGUm2r3giU7mLSIv21zXwzPxNPPnuBnbsr2dgYR4/uPA4Lhvfn255WUHHk4NQuYvIIe2tDvH4Oxt46r2NVNaFOX1IIfddOoYzh/XUskucU7mLyGdU1jXw+DsbeOxvG6gOhZk2qjffmjKE0f26BB1NWknlLiKfCkeiPLtwEw+8+RF7qkNMG9WbW6YOY1hR56CjyRFSuYsIAO98tIt/+1MJH5VXcfKg7tx5/nEdfsxx6Tgqd5EUV15Zx7+/sopZy7ZyTPdcHr5qAlNHFn16WFpJTCp3kRTl7rywpIx7X1lJfUOUm88eyg2TB5OTqSMqJgOVu0gK2rG/jjtfWsFbq8uZNKA7P77seAb3zA86lsSQyl0kxbxesp3vvricuoYId180kmtOHaDdGpOQyl0kRdQ1RPjx7FU89d7HjOpbwC+uHKet9SSmchdJAbtro3zp4fdYXlbB9acP5DvThpOdobX1ZKZyF0lyfy/dxT3v1mJpDTxy1QSmjuoddCQ5ClTuIkns9/M/5p5ZJRTlGr//p9MYpGWYlKFyF0lCkajzH6+u4om/b+CsEb34Yv8qFXuK0fE5RZJMXUOEbz3zPk/8fQPXnjaQR78+kU4Z2hsm1WjLXSSJ7K9rYPrTi5m/fg93XzSSa08fGHQkCYjKXSRJ7KkOcdXjC1izvZIHrxjLJWP7BR1JAqRyF0kCu6rq+eqjC9i4u5pHr57IlOG9go4kAVO5iyS48so6vvLoAsr21vDENSdy2pDCoCNJHFC5iySwPdUhvvroArbuq+XJb0zi5EE9go4kcUJ7y4gkqP11DXz9iQVs2lPD41efqGKX/0PlLpKAakMRrv3dItZsr+S3X5vAKYNV7PJ/aVlGJMGEI1FuenYpSzbt5aGvjGfKCL14Kp+lLXeRBOLu3DWzhDdX7eDfLh7FBcf3CTqSxCmVu0gCeWhuKc8u3MQ/Tx7M108ZEHQciWOtKnczm2Zma8ys1MzuOMj9x5jZXDNbambLzeyC2EcVSW2zV2zj/tfX8g9j+/Kd84YHHUfiXIvlbmbpwEPA+cBI4EozG3nAsB8Az7v7OOAK4NexDiqSylaUVXDL8x8w/piu3HfZGJ28WlrUmi33SUCpu6939xAwA7jkgDEOFDR93gXYGruIIqmtfH8d1z+9iB552Tx81USdwFpaxdz98APMLgemufv1TdevAk5y9xubjekDvA50A/KAc9x9yUGeazowHaCoqGjCjBkz2hS6qqqK/PzkOHxpMs0Fkms+8TCXcNT5ycI6Pq6MctfJnSju3LaXyeJhLrGS6nOZMmXKEnef2OJAdz/sBbgceKzZ9auAXx0w5hbg1qbPTwFWAmmHe94JEyZ4W82dO7fNj403yTQX9+SaTzzM5Z6ZH/qxt7/iMz/Y0q7niYe5xEqqzwVY7C30tru3allmC1Dc7Hr/ptuauw54vumXxXtADqADXIi0wx+XlvHkuxu5/vSBXHxC36DjSIJpTbkvAoaa2UAzy6LxBdNZB4zZBJwNYGbH0VjuO2MZVCSVlJZX8r2XPuSkgd254/wRQceRBNRiubt7GLgRmAOsonGvmBIzu9fMLm4adivwj2a2DHgWuKbpzwcROUKNZ1JaSm5WOr+4chwZ6Xo7ihy5Vh1+wN1nA7MPuO3uZp+vBE6LbTSR1PRvfyphzY5Knrp2EkUFOUHHkQSlTQKROPLq8m08u3AzN0wezJnDegYdRxKYyl0kTmyrqOV7f1zB2OKu3HLusKDjSIJTuYvEgWjUufX5ZTREovz8y2PJ1Dq7tJP+B4nEgcff2cC763Zzz+dHMqAwL+g4kgRU7iIB+2hHJT+bs4apI4v40sTilh8g0goqd5EAhSNRbv3DMvJzMvjRpcfrgGASMzoTk0iAHv7repaXVfDQV8ZTmJ8ddBxJItpyFwnImu2V/PzNtVw4pg8XjtEZlSS2VO4iAYhEne++uJzOOZnce/GooONIElK5iwTgqXc3smzzPu75/Eh6aDlGOoDKXeQoK9tbw/2vr2Hy8J462qN0GJW7yFHk7vzg5Q8B+Pd/GK29Y6TDqNxFjqJXV2xj3pqd3Dp1OP275QYdR5KYyl3kKKmsa+DeP61kVN8Crj7l2KDjSJLTfu4iR8kDb3zEzqp6Hvn6RB2jXTqc/oeJHAUlWyt48t0NfGXSMYwt7hp0HEkBKneRDubu3D2zhG65WXz3PJ0yT44OlbtIB/vj0i0s+Xgvt58/gi65mUHHkRShchfpQJV1Dfxo9mrGFnfl8vH9g44jKUQvqIp0oF/85SN2V9fz+NUTSUvTPu1y9GjLXaSDlJZX8bu/b+TLE4s5QS+iylGmchfpID+avYpOmencdt7woKNIClK5i3SAt9fu5K3V5dx09hAdp10CoXIXibFwJMoPX1nJsT1yufrUAUHHkRSlcheJsf9ZuInS8iq+d8FxZGekBx1HUpTKXSSG9tc18MAbazl5UHemjiwKOo6kMJW7SAz9eu469tY08IMLR+pwvhIolbtIjJTtreGJv2/g0nH9GN2vS9BxJMWp3EVi5P45azDQro8SF1TuIjGwoqyClz/YyvVnDKRv105BxxFRuYu0l7vz4z+vonteFt88c3DQcUSAVpa7mU0zszVmVmpmdxxizJfMbKWZlZjZ/8Q2pkj8+utHu3h33W5uOmsInXN01EeJDy0eOMzM0oGHgHOBMmCRmc1y95XNxgwF7gROc/e9ZtarowKLxJNo1Lnvz6sp7t6Jr56kU+dJ/GjNlvskoNTd17t7CJgBXHLAmH8EHnL3vQDuXh7bmCLxaeayLazatp/bpg4nK0OrnBI/zN0PP8DscmCau1/fdP0q4CR3v7HZmJeBtcBpQDrwr+7+2kGeazowHaCoqGjCjBkz2hS6qqqK/Pz8Nj023iTTXCC55tPSXBqizp1/qyUv07jnlBzS4ni/9lT6viSStsxlypQpS9x9YosD3f2wF+By4LFm168CfnXAmFeAPwKZwEBgM9D1cM87YcIEb6u5c+e2+bHxJpnm4p5c82lpLr97Z70fe/sr/te15UcnUDuk0vclkbRlLsBib6G33b1VyzJbgOJm1/s33dZcGTDL3RvcfQONW/FDW/HcIgmpqj7ML98q5dTBPTh9SGHQcUQ+ozXlvggYamYDzSwLuAKYdcCYl4HJAGZWCAwD1scwp0hceeKdDeyuDvGd84brMAMSl1osd3cPAzcCc4BVwPPuXmJm95rZxU3D5gC7zWwlMBf4jrvv7qjQIkHaUx3ikb+u57xRRYw7plvQcUQOqlXnUHX32cDsA267u9nnDtzSdBFJar+ZV0pNKMxtU3WYAYlf2ndL5Ahsq6jlqfc+5gvj+jO0qHPQcUQOSeUucgR++VYp7s63z9H+AhLfVO4irbRxVzXPL9rMlZOOobh7btBxRA5L5S7SSj9/cy0Z6caNU4YEHUWkRSp3kVZYs72Smcu2cvWpA+hVkBN0HJEWqdxFWuGBN9aSn5XBNz+nQ/pKYlC5i7RgRVkFr5Vs57ozBtItLyvoOCKtonIXacH9r6+ha24m150+MOgoIq2mchc5jEUb9/D22p1888zBOhGHJBSVu8ghuDv3z1lDYX42V58yIOg4IkdE5S5yCKv2RFmwYQ83ThlMp6z0oOOIHBGVu8hBuDsvrg3Rt0sOV550TNBxRI6Yyl3kIOauKWddRZQbzxpKdoa22iXxqNxFDhCNOv/5+lp6djK+OLF/0HFE2kTlLnKAOSXbKdm6n0uGZJKZrh8RSUytOp67SKqIRJ0H3lzL4J55nNr38CePF4ln2iwRaeaV5VtZu6OKfzl3GGk6fZ4kMJW7SJNwJMoDb6xlRO/OXDC6T9BxRNpF5S7S5KX3t7Bxdw23Th1OWpq22iWxqdxFgPpwhAf/8hEn9O/COcf1CjqOSLup3EWA5xZtZsu+Wm6dOhzTWrskAZW7pLzaUIRfvlXKpAHdOWNoYdBxRGJC5S4p7+n3NrKzsp7bztNWuyQPlbuktMq6Bn779jo+N6wnkwZ2DzqOSMyo3CWlPf7OBvbWNHDrucOCjiISUyp3SVl7q0M89rcNTBvVmxOKuwYdRySmVO6Ssn779jqqQ2Fumaqtdkk+KndJSTv21/Hkuxv5wth+DCvqHHQckZhTuUtK+sVfPiISdb59jrbaJTmp3CXlbNxVzXOLNnPFpGKO6ZEbdByRDtGqcjezaWa2xsxKzeyOw4y7zMzczCbGLqJIbP3XG2vJTE/j/501NOgoIh2mxXI3s3TgIeB8YCRwpZmNPMi4zsDNwIJYhxSJlZKtFcxatpVrTx9Ar4KcoOOIdJjWbLlPAkrdfb27h4AZwCUHGfdD4CdAXQzzicTUz+asoUunTKZ/bnDQUUQ6VGvKvR+wudn1sqbbPmVm44Fid381htlEYmr++t3MW7OTf548mC6dMoOOI9Kh2n2aPTNLA/4LuKYVY6cD0wGKioqYN29em75mVVVVmx8bb5JpLhC/83F3fji/jm7ZxsDwJubN29ziY+J1Lm2hucSnDp2Lux/2ApwCzGl2/U7gzmbXuwC7gI1NlzpgKzDxcM87YcIEb6u5c+e2+bHxJpnm4h6/83l1+VY/9vZX/LmFm1r9mHidS1toLvGpLXMBFnsLve3urVqWWQQMNbOBZpYFXAHMavbLocLdC919gLsPAOYDF7v74lj88hFpr4ZIlJ/NWcOwonwum9A/6DgiR0WL5e7uYeBGYA6wCnje3UvM7F4zu7ijA4q014xFm9mwq5rvnjeCdJ0+T1JEq9bc3X02MPuA2+4+xNjJ7Y8lEhtV9WEefHMtkwZ052ydPk9SiN6hKknt4bfXsasqxJ0XjNCJOCSlqNwlaW2rqOXRv63n8yf0Zdwx3YKOI3JUqdwlaf3n62uJRuG75w0POorIUadyl6RUsrWCF98v4xunDaC4uw4OJqlH5S5Jx925908r6dopk3+eMiToOCKBULlL0plTsoMFG/Zwy9ThOsyApCyVuySV+nCEH81exbCifK48sTjoOCKBUblLUnny7xvZtKeGuy4aSUa6/ntL6tL/fkka5ZV1/PKtUs4a0YszhvYMOo5IoFTukjTu+/NqQuEod130mXPJiKQclbskhSUf7+Gl97dw/RkDGViYF3QckcCp3CXhRaLO3TNL6F2Qw7e066MIoHKXJPDswk2UbN3P9y88jrzsdp9/RiQpqNwloe2srOenr63mlEE9uGhMn6DjiMQNlbsktB/PXkVtQ4Qf/sNoHfVRpBmVuySs99bt5qWlW5j+uUEM6ZUfdByRuKJyl4RUH45w18wP6d+tEzdOGRp0HJG4o1efJCH9Zt46Ssur+N01J9IpKz3oOCJxR1vuknA+2lHJQ3NLuWRsX6aM0KnzRA5G5S4JJRp1bn9xOXnZGXonqshhqNwlofz3/I95f9M+7rpwJIX52UHHEYlbKndJGJt213Dfn1dzxtBCLh3fL+g4InFN5S4JIRp1bnthGRlpxk8uG6N92kVaoHKXhPD0extZuGEPd100kr5dOwUdRyTuqdwl7q3fWcV9r61m8vCefHFi/6DjiCQElbvEtYZIlH957gNyMtO571Itx4i0lt7EJHHtF3/5iGVlFfz6q+Pp3SUn6DgiCUNb7hK3Fm/cw0NzS7l8Qn8uOF5HfBQ5Eip3iUsVNQ3cPOMD+nfL5V8vHhV0HJGEo2UZiTvuzndeWEZ5ZR0vfPNU8nUCDpEjpi13iTtPvbuR11fu4PZpIzihuGvQcUQSUqvK3cymmdkaMys1szsOcv8tZrbSzJab2V/M7NjYR5VUsGzzPn40ezVnj+jFdacPDDqOSMJqsdzNLB14CDgfGAlcaWYHHrFpKTDR3ccALwA/jXVQSX67q+q54fdL6Nk5m/u/eIJ2exRph9ZsuU8CSt19vbuHgBnAJc0HuPtcd69pujof0DtN5IiEI1FuenYpu6tDPHzVBLrlZQUdSSShtabc+wGbm10va7rtUK4D/tyeUJJ6fjpnDe+u281/fOF4RvfrEnQckYRn7n74AWaXA9Pc/fqm61cBJ7n7jQcZ+zXgRuBMd68/yP3TgekARUVFE2bMmNGm0FVVVeTnJ8c5M5NpLtC2+byzpYHHVoQ465gMvj4yfg7jm0zfG80lPrVlLlOmTFni7hNbHOjuh70ApwBzml2/E7jzIOPOAVYBvVp6TndnwoQJ3lZz585t82PjTTLNxf3I57Nww24f8r1X/SuPvuehcKRjQrVRMn1vNJf41Ja5AIu9FR3bmmWZRcBQMxtoZlnAFcCs5gPMbBzwMHCxu5e39jeQpLbNe2r4p/9eQnG3XH79lQlkpmvPXJFYafGnyd3DNC61zKFxy/x5dy8xs3vN7OKmYT8D8oE/mNkHZjbrEE8nAsCe6hBXP7GQSNR57OqJdMnNDDqSSFJp1Vv/3H02MPuA2+5u9vk5Mc4lSaw2FOG6pxaxZV8tz1x/EoN6Jsf6qUg80fu65ahqiES56dn3+WDzPn7z1QlMHNA96EgiSUmLnHLURKLOrc8v481V5dx7yWimje4ddCSRpKVyl6MiGnW+99IKZi3byu3TRnDVyTpChUhHUrlLh4tGnXtmlfDc4s3cdNYQbpg8OOhIIklPa+7SoaJR5/svr+DZhZuZ/rlB3HLusKAjiaQElbt0mEjUuePF5fxhSRnfmjKY26YO18HARI4Slbt0iLqGCDfPWMqckh3cfPZQvn3OUBW7yFGkcpeYq6xr4B+fXsz89Xu45/Mj+cZpOi67yNGmcpeY2lUb5fLfvMe6nVU8eMVYLhl7uAOIikhHUblLzCzdtJd736vD09J58huTOH1oYdCRRFKWyl1i4oUlZXz/jysoyIRnbziNIb10SAGRIKncpV3qwxHu/dNKnlmwiVMH9+DKY2tV7CJxQG9ikjb7eHc1X/rtezyzYBP/dOYgnr52Ep2ztEeMSDzQlru0yctLt/CDlz8kzeC3XxvPtNF9go4kIs2o3OWI7KkOcdfMD3l1+TZOHNCNn18xjn5dOwUdS0QOoHKXVnF3XvtwO3fN/JCK2gZumzqMb545mAydPUkkLqncpUWb99Rwz6wS3lpdzqi+Bfz++pMY0bsg6Fgichgqdzmk2lCER/66nt+8XUqaGT+48DiuOXWAttZFEoDKXT4jGnVmLtvCT19bw7aKOs4f3Zu7LhpJX62tiyQMlbt8yt15c1U5//n6GlZvr2R0vwJ+/uWxnDSoR9DRROQIqdyFaNSZU7Kdh+aV8uGW/QwszOOXV47jwuP7kJam/dZFEpHKPYXVhiK8tLSMx9/ZwPqd1QwszOOnl4/h0nH9tK4ukuBU7ilo465qnl24iecWb2ZfTQOj+xXwyyvHccHxfUjXlrpIUlC5p4iaUJg5Jdv5w+Iy3l23m/Q049zjirj29IGcOKCbTqQhkmRU7kmsPhzhnY928erybbxWsp2aUIT+3Tpx67nD+NKJxRQV5AQdUUQ6iMo9yeyva+DtNTt5Y+UO5q4pp7IuTJdOmXx+TF8um9Cficd204ukIilA5Z7gGiJRlpdV8G7pLt5eu5Olm/cRiTo98rI4f3Rvzj++D6cNLiQrQy+QiqQSlXuCqa4Ps7ysgiUf72HRxr0s3riH6lAEgDH9u3DDmYOZMqInY4u76cVRkRSmco9jVfVh1mzfT8nW/Xy4pYLlZRWs3VFJ1BvvH1aUz6Xj+3PK4B6cPKgH3fOygg0sInFD5R4wd2d3VT0bd1ezfmc1pTurWFdexertlZTtrf10XLfcTMb078rUUb0ZW9yF8cd0o2uuylxEDk7l3sGiUWd3dYgd++vYVlHH1n21bNlXS9neGjbtqWHdjhpq57z56fjMdGNgYR5ji7vy5YnFjOhTwKi+BfTpkqPdFUWk1VpV7mY2DXgQSAcec/f7Drg/G3gamADsBr7s7htjGzU+RKJOZV0DFbUN7KtpYG9NiH01DeyuDrGnup7dVSF2VdWzs7Ke8srGj+FP1lGaZGWk0b9bJ4q75dKrTwanjhnKwMI8BhbmcUz3XL07VETarcVyN7N04CHgXKAMWGRms9x9ZbNh1wF73X2ImV0B/AT4ckcEbom70xBxQpEooXDjpT4coT4cpb4hSl04Ql1DhNpQhLpwlNpQmJpQhJpQ423VoTA19RGqQmGq68NU1YWpqg9TWRdmf10DVfVh3A/+tdPTjO55WfTIy6Jn52yG9OpMr4JsehfkUFSQQ+8uOfTr2onC/KxPt8LnzZvH5DMGHcV/IRFJBa3Zcp8ElLr7egAzmwFcAjQv90uAf236/AXgV2Zm7oeqwbZ7btEmHvhbDVkL5xKORGmIOg2RKOGIN5Z5JNrm504zyM3KIC87nbysDPJzMsjPzqA4L5eCnEw652RQ0CmTLk2XbrmZdM3NpGtuY6EX5GRqH3IRiQutKfd+wOZm18uAkw41xt3DZlYB9AB2NR9kZtOB6QBFRUXMmzfviANvLQ/Tt1OU7Mx60tOMDIP0NEg3yEhLb7pAZpo1ffzkYmSmN37MSoesNMhKN7LT///HzDQOWNcON10OEAYqGy9NH/7PP9CRqKqqatO/Q7xKpvloLvFJc2mdo/qCqrs/AjwCMHHiRJ88efIRP8dkYNy8ebTlsfFoXhLNBZJrPppLfNJcWqc1r9xtAYqbXe/fdNtBx5hZBtCFxhdWRUQkAK0p90XAUDMbaGZZwBXArAPGzAKubvr8cuCtjlhvFxGR1mlxWaZpDf1GYA6Nu0I+4e4lZnYvsNjdZwGPA/9tZqXAHhp/AYiISEBatebu7rOB2Qfcdnezz+uAL8Y2moiItJXeLSMikoRU7iIiSUjlLiKShFTuIiJJyILaY9HMdgIft/HhhRzw7tcElkxzgeSaj+YSn1J9Lse6e8+WBgVW7u1hZovdfWLQOWIhmeYCyTUfzSU+aS6to2UZEZEkpHIXEUlCiVrujwQdIIaSaS6QXPPRXOKT5tIKCbnmLiIih5eoW+4iInIYCV3uZnaTma02sxIz+2nQedrLzG41MzezwqCztJWZ/azpe7LczP5oZl2DznSkzGyama0xs1IzuyPoPG1lZsVmNtfMVjb9jNwcdKb2MrN0M1tqZq8EnaW9zKyrmb3Q9POyysxOieXzJ2y5m9kUGk/vd4K7jwLuDzhSu5hZMTAV2BR0lnZ6Axjt7mOAtcCdAec5Is3OGXw+MBK40sxGBpuqzcLAre4+EjgZ+FYCz+UTNwOrgg4RIw8Cr7n7COAEYjyvhC134AbgPnevB3D38oDztNcDwHeBhH4RxN1fd/dPzk04n8aTuySST88Z7O4h4JNzBiccd9/m7u83fV4uZupKAAACN0lEQVRJY3n0CzZV25lZf+BC4LGgs7SXmXUBPkfj4dJx95C774vl10jkch8GnGFmC8zsbTM7MehAbWVmlwBb3H1Z0Fli7Frgz0GHOEIHO2dwwhbiJ8xsADAOWBBsknb5OY0bQNGgg8TAQGAn8LumZabHzCwvll/gqJ5D9UiZ2ZtA74Pc9X0as3en8c/NE4HnzWxQvJ4BqoW5fI/GJZmEcLi5uPvMpjHfp3FZ4JmjmU0+y8zygReBb7v7/qDztIWZXQSUu/sSM5scdJ4YyADGAze5+wIzexC4A7grll8gbrn7OYe6z8xuAF5qKvOFZhal8TgNO49WviNxqLmY2fE0/hZfZmbQuIzxvplNcvftRzFiqx3u+wJgZtcAFwFnx+sv28NozTmDE4aZZdJY7M+4+0tB52mH04CLzewCIAcoMLPfu/vXAs7VVmVAmbt/8pfUCzSWe8wk8rLMy8AUADMbBmSRgAcTcvcV7t7L3Qe4+wAav+nj47XYW2Jm02j80/lid68JOk8btOacwQnBGrcWHgdWuft/BZ2nPdz9Tnfv3/QzcgWN52lO1GKn6ed7s5kNb7rpbGBlLL9GXG+5t+AJ4Akz+xAIAVcn4FZiMvoVkA280fSXyHx3/2awkVrvUOcMDjhWW50GXAWsMLMPmm77XtNpMyV4NwHPNG1ErAe+Ecsn1ztURUSSUCIvy4iIyCGo3EVEkpDKXUQkCancRUSSkMpdRCQJqdxFRJKQyl1EJAmp3EVEktD/AkTul+B4zjIcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10880a2e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def sigmoid(x):\n",
    "    a = []\n",
    "    for item in x:\n",
    "        a.append(1/(1+math.exp(-item)))\n",
    "    return a\n",
    "\n",
    "y = sigmoid(x)\n",
    "\n",
    "plt.plot(x,y)\n",
    "plt.grid()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tahn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD8CAYAAABzTgP2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XmYlfV99/H3d1aWYVgcGBCQRVDEqOAgRkEriko28Wk00Vy1GONDm8Y0TfK00WataVpNetWmjW1NlWgTk4mxGqnBEJcZjRgMSJB9GVBg2GYYYGAYZj3f549zk5zB2TjnzNxn+byu63ju5fc75/vzMPOZezn3be6OiIjIKTlhFyAiIqlFwSAiIh0oGEREpAMFg4iIdKBgEBGRDhQMIiLSgYJBREQ6UDCIiEgHCgYREekgL+wC4lFSUuITJ06Mq++JEycYPHhwcgsKicaSujJpPBpLaopnLG+99dYhdx/ZU7u0DIaJEyeyevXquPpWVlZyzTXXJLegkGgsqSuTxqOxpKZ4xmJmu3rTTruSRESkAwWDiIh0oGAQEZEOFAwiItKBgkFERDpISjCY2RIzqzGzDV2sNzP7VzOrMrN1ZnZpzLpFZrY9eCxKRj0iIhK/ZG0xPA4s6Gb9B4CpwWMx8B8AZjYC+DpwOTAb+LqZDU9STSIiEoekfI/B3V8zs4ndNFkI/LdH7yO60syGmdkY4BrgRXc/DGBmLxINmJ8koy4RSX3uTnNbhObWCM1t7dHp4Lmt3Wltj9DSHp1ui0Sf2yNOuwfPESfiEAmWRTw67+5EIo7D7+e3v9PKtpwduIND8Oycfodjd/99mz8sC5557+2Qe7pDclw3UO7iRRddOZGzigrjecVe668vuI0F9sTMVwfLulr+Hma2mOjWBqWlpVRWVsZVSENDQ9x9U43GkroyaTy9HYu7c7wFDjdFqG9x6pudYy1OQwucaHUa25zGVqexDZranJNt0NLuNLfH+YszXlu39Oe7JcQ6WTa6ZS9nF+X06b+xtPnms7t/H/g+wKxZszzeby9m+zcfU1UmjQUyazynj+VoYwtbDhxn+8Hj7Kg9wbt1J9hd10j10SZa2iLv6V+Yl8PwQQUMG5RPcXE+YwbkMbgwj6LCXAYV5DGoIJcB+aceORTm5VKQl0NBrlGQl0NeTg75uTnk5xp5uTnk5Ri5sQ+LPpvx+3kMcs0wMwzICZa9seJ1rrrqKgyILor2I2b+lKALZjHLYtadzjpb2If68t9YfwXDXmB8zPy4YNleoruTYpdX9lNNItKDSMTZdaydx15/hzW7jrBu71H2HD75+/VFhXlMOGsQ08YMYf70UsYMHcCYoQMYVTyAkUWFnFVUwKCC1Pn7c2CeUVSYOvWkqv76P7QUuMfMyokeaK539/1mthz4h5gDzjcA9/VTTSLSieNNrVRureXlzQep3FbL0cZWYBNjhw1kxvhhfGL2BKafXcx5pUWMLh7Q738pS99LSjCY2U+I/uVfYmbVRM80ygdw9/8ElgEfBKqARuCTwbrDZvZNYFXwUvefOhAtIv0nEnFe3V7LM2v28quNB2huizB8UD7XThtFSdsh7vzgHM4eNjDsMqWfJOuspNt7WO/AZ7pYtwRYkow6ROTMnGhu42er9/D4G+/ybl0jwwblc+uscSycMZZLzxlObo5RWVmpUMgy2tkmkoWaWtv50cpdPFxRxZHGVmaeM4wv3nA+N144moI8XRAh2ykYRLLMLzcc4JvPb2Lv0ZPMnVLC568/j7IJ+l6p/IGCQSRLHDzWxNee28DyjQe5YEwx377lYuZMKQm7LElBCgaRLFCxpYbPP7WWky3tfGnBNO6+ahL5udplJJ1TMIhksLb2CA+9tI2HK3YwfUwx3/vETCaPLAq7LElxCgaRDNXU2s49P17DS5truO2y8XzjpgsZkJ8bdlmSBhQMIhmo/mQrdz+xitW7jvB3N13Ioisnhl2SpBEFg0iGOdrYwu3/9SZVNcf5t9tn8uGLzw67JEkzCgaRDNLY0sZdj69iR00Djy26jKvPGxl2SZKGdFqCSIZoaYvw6R+tYe2eo/zr7TMUChI3bTGIZAB35ys/X8+r22p58KMXseB9Y8IuSdKYthhEMkD5qj08tbqaz147hY9fdk7Y5UiaUzCIpLm39xzl689t5OrzRvJX888LuxzJAAoGkTRW39jKXzy5hpFDCvnux2eQm6N7I0jidIxBJI3d//wmDhxr4plPX8nwwQVhlyMZIilbDGa2wMy2mlmVmd3byfqHzGxt8NhmZkdj1rXHrFuajHpEssErWw7yP2uq+YtrzuWS8cPCLkcySMJbDGaWCzwMXA9UA6vMbKm7bzrVxt0/H9P+s8DMmJc46e4zEq1DJJvUN7Zy3zPrOb90CPdcOyXsciTDJGOLYTZQ5e473b0FKAcWdtP+duAnSXhfkaz1jy9s5lBDC/906yUU5un6R5JcyQiGscCemPnqYNl7mNkEYBLwSsziAWa22sxWmtnNSahHJKNt2FvPT1fv4ZNXTuSicUPDLkcykEVvx5zAC5jdAixw97uD+TuAy939nk7afgkY5+6fjVk21t33mtlkooFxnbvv6KTvYmAxQGlpaVl5eXlc9TY0NFBUlBmXHdZYUldfjcfdeeC3Tew7EeHBqwYxKL/vz0LKpM8m28cyb968t9x9Vo8N3T2hB3AFsDxm/j7gvi7a/g64spvXehy4paf3LCsr83hVVFTE3TfVaCypq6/G84t1+3zCl573H618t09evzOZ9Nlk+1iA1d6L3+vJ2JW0CphqZpPMrAC4DXjP2UVmNg0YDvwmZtlwMysMpkuAOcCm0/uKSPT+Cv+wbDPTRg/h47PGh12OZLCEz0py9zYzuwdYDuQCS9x9o5ndTzSdToXEbUB5kFqnXAA8YmYRosc7HvCYs5lE5A/Kf7ub6iMn+eGnZpOn23JKH0rKF9zcfRmw7LRlXztt/hud9HsDuCgZNYhksqbWdv7j1R3MnjSCq6bqqqnSt/Rnh0ga+NnqPRw81sznrpsadimSBRQMIimuua2df6/cwawJw7ny3LPCLkeygIJBJMX9bHU1++ub+Nz8qZjpInnS9xQMIimsPeI88toOZp4zjLlTSsIuR7KEgkEkhb20+SB7Dp9k8VWTtbUg/UbBIJLCfrDiHcYOG8j100vDLkWyiIJBJEVt3n+MlTsP86dXTND3FqRf6V+bSIr6wYp3GJify226h7P0MwWDSAqqa2jm52v38ceXjmXooPywy5Eso2AQSUFPv1VNS1uEO6+cGHYpkoUUDCIpxt15avUeZk0YztTSIWGXI1lIwSCSYtbsPsKO2hN8TFdQlZAoGERSzFOrqhlUkMuHLh4TdimSpRQMIinkRHMbz6/bx4cvHsPgwqRc/FjkjCkYRFLIL9bv50RLu3YjSagUDCIp5Ger9zB55GDKJgwPuxTJYkkJBjNbYGZbzazKzO7tZP2dZlZrZmuDx90x6xaZ2fbgsSgZ9Yikoz2HG1n17hE+euk4XRdJQpXwTkwzywUeBq4HqoFVZra0k1t0/tTd7zmt7wjg68AswIG3gr5HEq1LJN08v24/ADddcnbIlUi2S8YWw2ygyt13unsLUA4s7GXfG4EX3f1wEAYvAguSUJNI2ln69j5mnjOM8SMGhV2KZLlkBMNYYE/MfHWw7HQfNbN1Zva0mZ06stbbviIZrarmOJv3H+MjF2trQcLXX+fD/S/wE3dvNrM/A54Arj2TFzCzxcBigNLSUiorK+MqpKGhIe6+qUZjSV1nOp5nt7dgwPCGd6is3NVndcUjkz4bjaWX3D2hB3AFsDxm/j7gvm7a5wL1wfTtwCMx6x4Bbu/pPcvKyjxeFRUVcfdNNRpL6jqT8UQiEZ/3nQq/7ZHf9F1BCcikzybbxwKs9l78Xk/GrqRVwFQzm2RmBcBtwNLYBmYW+xXOm4DNwfRy4AYzG25mw4EbgmUiWWPjvmPsPHSCj+igs6SIhHcluXubmd1D9Bd6LrDE3Tea2f1E02kp8JdmdhPQBhwG7gz6HjazbxINF4D73f1wojWJpJNfrN9Pbo7xgfeNDrsUESBJxxjcfRmw7LRlX4uZvo/oLqbO+i4BliSjDpF0tHzjAd4/eQTDBxeEXYoIoG8+i4SqquY4O2tPcOOF2lqQ1KFgEAnR8o0HAbh+emnIlYj8gYJBJES/2niAS8YNZczQgWGXIvJ7CgaRkOyvP8nb1fXcoN1IkmIUDCIheXFTdDfSjRdqN5KkFgWDSEiWbzzA5JGDmTJK93WW1KJgEAlB/clW3tx5mBumazeSpB4Fg0gIfr29lraIc90Fo8IuReQ9FAwiIajYUsvQgfnMHD8s7FJE3kPBINLPIhHn1W01/NF5I8nL1Y+gpB79qxTpZ+v21nOooYVrp2k3kqQmBYNIP6vYUoMZ/NF5I8MuRaRTCgaRflaxtYaZ44fponmSshQMIv2o5ngT66rrtRtJUpqCQaQfvbq1FoB5CgZJYQoGkX5Uua2WUUMKmT6mOOxSRLqUlGAwswVmttXMqszs3k7Wf8HMNpnZOjN72cwmxKxrN7O1wWPp6X1FMkV7xHl9+yGuPm8kZhZ2OSJdSvgObmaWCzwMXA9UA6vMbKm7b4pp9jtglrs3mtmngW8DHw/WnXT3GYnWIZLq1u+tp/5kK1frbCRJccnYYpgNVLn7TndvAcqBhbEN3L3C3RuD2ZXAuCS8r0haeW1bLWYwd0pJ2KWIdMvcPbEXMLsFWODudwfzdwCXu/s9XbT/HnDA3f8+mG8D1gJtwAPu/vMu+i0GFgOUlpaWlZeXx1VvQ0MDRUVFcfVNNRpL6upsPN9aeZK2CHz9yvS6KU8mfTbZPpZ58+a95e6zemzo7gk9gFuAR2Pm7wC+10XbPyG6xVAYs2xs8DwZeBc4t6f3LCsr83hVVFTE3TfVaCyp6/Tx1J9s8cn3/cK//cvN4RSUgEz6bLJ9LMBq78Xv9WTsStoLjI+ZHxcs68DM5gNfBm5y9+aYYNobPO8EKoGZSahJJKW8UVVHe8S5eqqOL0jqS0YwrAKmmtkkMysAbgM6nF1kZjOBR4iGQk3M8uFmVhhMlwBzgNiD1iIZ4dfbaxlckMvMc4aHXYpIjxI+K8nd28zsHmA5kAsscfeNZnY/0c2WpcB3gCLgZ8Fpervd/SbgAuARM4sQDakHvOPZTCJpz915bXstV5xbQkGevjokqS/hYABw92XAstOWfS1men4X/d4ALkpGDSKpalddI3sOn+T/XjU57FJEekV/voj0sderDgE6TVXSh4JBpI+tqDrE2UMHMKlkcNiliPSKgkGkD7VHnN/srGPOlBJdBkPShoJBpA9t2neMo42tzJ2q3UiSPhQMIn3o1PGFK89VMEj6UDCI9KEVVYeYNnoII4cUhl2KSK8pGET6SFNrO6vePaytBUk7CgaRPrJm1xGa2yLMnXpW2KWInBEFg0gfeb3qEHk5xuxJCgZJLwoGkT6yYkcdM8YPo6gwKRcYEOk3CgaRPnCi1VlffZQr9W1nSUMKBpE+sO1IOxGHOedqN5KkHwWDSB/YVNfOgPwcZpwzLOxSRM6YgkGkD2yua+eyiSMozMsNuxSRM6ZgEEmy2uPNVDe4vr8gaSspwWBmC8xsq5lVmdm9nawvNLOfBuvfNLOJMevuC5ZvNbMbk1GPSJh+s7MOgCt1fEHSVMLBYGa5wMPAB4DpwO1mNv20Zp8Cjrj7FOAh4MGg73SitwK9EFgA/HvweiJp642qQwzMg/eNHRp2KSJxScYWw2ygyt13unsLUA4sPK3NQuCJYPpp4DqLXoN4IVDu7s3u/g5QFbyeSNp6Y0cd00bkkpujy2xLekpGMIwF9sTMVwfLOm3j7m1APXBWL/uKpI09hxvZfbiR6SO04SvpK22+kmlmi4HFAKWlpVRWVsb1Og0NDXH3TTUaS+p5rboVgAkDmzNiPJA5nw1oLL2VjGDYC4yPmR8XLOusTbWZ5QFDgbpe9gXA3b8PfB9g1qxZfs0118RVbGVlJfH2TTUaS+p5tvx3lBTVMXVUbkaMBzLnswGNpbeSsStpFTDVzCaZWQHRg8lLT2uzFFgUTN8CvOLuHiy/LThraRIwFfhtEmoS6Xfuzhs76rjy3LN0G09JawlvMbh7m5ndAywHcoEl7r7RzO4HVrv7UuAx4IdmVgUcJhoeBO2eAjYBbcBn3L090ZpEwlBV00Dt8eboaaqN9WGXIxK3pBxjcPdlwLLTln0tZroJuLWLvt8CvpWMOkTC9MaO6PcX5kwpYce6nSFXIxI/ffNZJElWVB1i3PCBjB8xKOxSRBKiYBBJgvaIs3JnHXN0GQzJAAoGkSTYuK+eY01tXDlFl8GQ9KdgEEmCU8cXrpisYJD0p2AQSYIVVYeYOqqIUcUDwi5FJGEKBpEENbe1s+rdw7qaqmQMBYNIgtbsOkpTa4S5U0eGXYpIUigYRBK0ouoQuTnG5ZNHhF2KSFIoGEQStGLHIS4eN5TiAflhlyKSFAoGkQQca2rl7T1HmTtF31+QzKFgEEnAyh11RDx6GQyRTKFgEEnAiqpDDMzPZeY5w8IuRSRpFAwiCVixo47Zk0ZQmKc7tknmUDCIxOlAfRNVNQ3M0WUwJMMoGETi9HrVIQCu1IXzJMMoGETi9OvttZQUFTB9THHYpYgkVULBYGYjzOxFM9sePA/vpM0MM/uNmW00s3Vm9vGYdY+b2TtmtjZ4zEikHpH+Eok4r28/xNwpJeTk6DaeklkS3WK4F3jZ3acCLwfzp2sE/tTdLwQWAP9iZrGncPy1u88IHmsTrEekX2zaf4y6Ey1cfZ4ugyGZJ9FgWAg8EUw/Adx8egN33+bu24PpfUANoJ8mSWuvba8FYO5UHV+QzJNoMJS6+/5g+gBQ2l1jM5sNFAA7YhZ/K9jF9JCZFSZYj0i/eG1bLReMKWbUEF1mWzKPuXv3DcxeAkZ3surLwBPuPiym7RF3f89xhmDdGKASWOTuK2OWHSAaFt8Hdrj7/V30XwwsBigtLS0rLy/vfmRdaGhooKioKK6+qUZjCUdTm/OZlxu5cWI+Hzu/oNM26TSenmgsqSmescybN+8td5/VY0N3j/sBbAXGBNNjgK1dtCsG1gC3dPNa1wDP9+Z9y8rKPF4VFRVx9001Gks4Xtp0wCd86Xl/fXttl23SaTw90VhSUzxjAVZ7L37HJroraSmwKJheBDx3egMzKwCeBf7b3Z8+bd2Y4NmIHp/YkGA9In3u19sPMSA/h1kTO904Fkl7iQbDA8D1ZrYdmB/MY2azzOzRoM3HgKuBOzs5LfVJM1sPrAdKgL9PsB6RPvfatlreP/ksXQZDMlZeIp3dvQ64rpPlq4G7g+kfAT/qov+1iby/SH/bVXeCnYdOcMcVE8IuRaTP6JvPImfglS01AFw7bVTIlYj0HQWDyBl4ZUsNk0cOZsJZg8MuRaTPKBhEeqmxpY03dx7m2vO1tSCZTcEg0ksrqupoaY8wT7uRJMMpGER66ZUtNRQV5nHZxBFhlyLSpxQMIr3g7lRurWHulBIK8vRjI5lN/8JFemHz/uPsr29i3jRd/1Eyn4JBpBcqtkZPU71GB54lCygYRHrhVxsPcMm4oZQW62qqkvkUDCI92F9/krer67nhws4uMiySeRQMIj14cdNBAG5UMEiWUDCI9GD5xgOcO3IwU0ZlxnX8RXqiYBDpxtHGFlbuPKzdSJJVFAwi3XhlSw3tEdduJMkqCgaRbizfeIDRxQO4eOzQsEsR6TcKBpEunGxp57Vth7h+eik5ORZ2OSL9JqFgMLMRZvaimW0Pnju916GZtcfcvW1pzPJJZvammVWZ2U+D24CKpISXtxzkZGs7H7p4TNiliPSrRLcY7gVedvepwMvBfGdOuvuM4HFTzPIHgYfcfQpwBPhUgvWIJM3StfsoLS7URfMk6yQaDAuBJ4LpJ4Cbe9vRzAy4Fng6nv4ifelYUyuVW2v50EVnk6vdSJJlzN3j72x21N2HBdMGHDk1f1q7NmAt0AY84O4/N7MSYGWwtYCZjQdecPf3dfFei4HFAKWlpWXl5eVx1dzQ0EBRUWacj66x9J1fV7fy2IYWvvr+AZw7LPeM+6faeBKhsaSmeMYyb968t9x9Vo8N3b3bB/ASsKGTx0Lg6Gltj3TxGmOD58nAu8C5QAlQFdNmPLChp3rcnbKyMo9XRUVF3H1TjcbSd+547E2f++DLHolE4uqfauNJhMaSmuIZC7Dae/E7Nq8XwTG/q3VmdtDMxrj7fjMbA9R08Rp7g+edZlYJzAT+BxhmZnnu3gaMA/b2VI9IX6traGZF1SH+7OrJRDeERbJLoscYlgKLgulFwHOnNzCz4WZWGEyXAHOATUF6VQC3dNdfpL8t23CA9ojzkUvODrsUkVAkGgwPANeb2XZgfjCPmc0ys0eDNhcAq83sbaJB8IC7bwrWfQn4gplVAWcBjyVYj0jCnllTzXmlRUwbPSTsUkRC0eOupO64ex1wXSfLVwN3B9NvABd10X8nMDuRGkSSafvB4/xu91G+8qELtBtJspa++SwS46nVe8jLMW6eOTbsUkRCo2AQCbS0RXhmzV7mX1BKSVFh2OWIhEbBIBJ4ZUsNdSda+Nhl48IuRSRUCgaRwM9W76G0uJCrp44MuxSRUCkYRIAD9U1UbK3ho5eOIy9XPxaS3fQTIAL8cOW7ANw++5xwCxFJAQoGyXpNre38+M3dzL+glPEjBoVdjkjoFAyS9Z5bu5cjja18cs6ksEsRSQkKBslq7s4PVrzLtNFDeP9k3XdBBBQMkuV+s7OOLQeOc9ecSfqms0hAwSBZbcnr7zB8UD43zdAF80ROUTBI1tqwt56XNtfwyTmTGJB/5jfjEclUCgbJWv/68naGDMjjzjkTwy5FJKUoGCQrbdp3jF9tOshdcyZRPCA/7HJEUoqCQbLSv72ynSGFedylU1RF3kPBIFln075jvLDhAJ+cM5Ghg7S1IHK6hILBzEaY2Ytmtj14Ht5Jm3lmtjbm0WRmNwfrHjezd2LWzUikHpGeuDvfWraJYYPy+dTcyWGXI5KSEt1iuBd42d2nAi8H8x24e4W7z3D3GcC1QCPwq5gmf31qvbuvTbAekW69uOkgK6rq+Pz887S1INKFRINhIfBEMP0EcHMP7W8BXnD3xgTfV+SMNbe1861lm5kyqohPXK6L5Yl0xdw9/s5mR919WDBtwJFT8120fwX4Z3d/Pph/HLgCaCbY4nD35i76LgYWA5SWlpaVl5fHVXNDQwNFRUVx9U01GsuZeeGdVn66tYUvlhVy0ciEbnfeI302qSnbxzJv3ry33H1Wjw3dvdsH8BKwoZPHQuDoaW2PdPM6Y4BaIP+0ZQYUEt3i+FpP9bg7ZWVlHq+Kioq4+6YajaX3dted8OlffcHvXPJmn77PKfpsUlO2jwVY7b34Hdvjn03uPr+rdWZ20MzGuPt+MxsD1HTzUh8DnnX31pjX3h9MNpvZD4D/11M9ImfK3bn3mXWYGd+8+X1hlyOS8hI9xrAUWBRMLwKe66bt7cBPYhcEYXJqN9TNRLdERJLqx7/dzYqqOv72gxcwbrjutyDSk0SD4QHgejPbDswP5jGzWWb26KlGZjYRGA+8elr/J81sPbAeKAH+PsF6RDrYc7iRf/jFZuZOKeH22ePDLkckLSR0BM7d64DrOlm+Grg7Zv5dYGwn7a5N5P1FutPU2s5nfryGHDMe+OhFuqy2SC/17akZIiH6u//dxLrqer5/R5l2IYmcAV0SQzLSU6v28JPf7uYz887lhgtHh12OSFpRMEjGeX37Ib7y8w3MnVLCF64/P+xyRNKOgkEyyto9R1n8w9VMHjmYhz9xKbk5Oq4gcqYUDJIxth88zid/8FvOKirgibtm61pIInFSMEhG+N3uI9z6yG/Iy83hh3ddTmnxgLBLEklbCgZJe69uq+UT//UmxQPyefrPr2BiyeCwSxJJazpdVdKWu/Por9/hwV9u4bzSITx+12WMGqItBZFEKRgkLdWfbOVvnn6b5RsPcuOFpXzn1kt072aRJFEwSNp5cdNBvvrzDRxqaOYrH7qAT82dpG81iySRgkHSxp7DjfzjC5tZtv4A00YP4T/vKGPG+C5v/yEicVIwSMqrOd7Ev1fs4Mk3d5Fjxl/feD6Lr55Mfq7OnRDpCwoGSVmb9x9jyevv8NzafbS787FZ4/ncdVMZPVQHmEX6koJBUsqhhmaef3sfz/5uL29X1zMwP5ePXzaeT82dpNNQRfqJgkFC1dYeYdP+Yzy/s4V/2/wGa3YfwR2mjynmKx+6gFvKxjFsUEHYZYpklYSCwcxuBb4BXADMDu7D0Fm7BcB3gVzgUXc/dUOfSUA5cBbwFnCHu7ckUpOkLndn79GTbN5/nPXVR3m7up41u45wvLkNgPeNbecvr53KBy4azbTRxSFXK5K9Et1i2AD8MfBIVw3MLBd4GLgeqAZWmdlSd98EPAg85O7lZvafwKeA/0iwJgmJu3OsqY3a403sO9rE/vqTVB85ya66RnbVnaCqpoETLe0A5BhMHTWEj8w4m8snjcAPbOXmBVeFPAIRgcTv4LYZ6Okc8tlAlbvvDNqWAwvNbDNwLfCJoN0TRLc+FAx9JBJx2iJOe8Rpi0Roa3daTz23R2htj9DcFqGlLfrc1NpOU2v0ubGlncaWNhqa22hsaed4UyvHTrZRf7KVI40tHDnRwqETLbS0RTq8Z47B2OEDmTBiMLfOGs/U0iLOLx3C9LOLGVTwh39+lZXb+/t/h4h0oT+OMYwF9sTMVwOXE919dNTd22KWv+f2n8n05WfXU7GxkcFrXsUTeB337nt7lzMdF8W+jgPu4DEd3KOPUyLuv2/jDs0tLeS//iLuwTqiv/zdod2jARA7nSwD8nMoHpBP8cB8hgzIY3TxAKaNLuasogJGDSlk5JBCRhcP4OxhAyktHkBBnk4rFUknPQaDmb0EdHYLrC+7+3PJL6nLOhYDiwFKS0uprKw849doPtxCaWGEXDsZvGYC9STYtrP3tuA/FtMjttmpWwuc6tvWGqGgIIIFyyxon2PRrbgcyyEnWJcT88g1yDEjzyA3JzqflwO5OUZ+DuTnQF6OUZCXLMM7AAAEvUlEQVQD+blQmGsU5kJBrjEgl5h7HDjQGjxORBe1A0fh5FHYsRt29PL/UUNDQ1yfaarKpPFoLKmpT8fi7gk/gEpgVhfrrgCWx8zfFzwMOATkddauu0dZWZnHq6KiIu6+qUZjSV2ZNB6NJTXFMxZgtffid2x/bOOvAqaa2SQzKwBuA5YGRVYAtwTtFgH9tgUiIiKdSygYzOz/mFk10b/2f2Fmy4PlZ5vZMgCPHkO4B1gObAaecveNwUt8CfiCmVURPebwWCL1iIhI4hI9K+lZ4NlOlu8DPhgzvwxY1km7nUTPWhIRkRSh00VERKQDBYOIiHSgYBARkQ4UDCIi0oGCQUREOjDv4fIOqcjMaoFdcXYvIfrFukygsaSuTBqPxpKa4hnLBHcf2VOjtAyGRJjZanefFXYdyaCxpK5MGo/Gkpr6cizalSQiIh0oGEREpINsDIbvh11AEmksqSuTxqOxpKY+G0vWHWMQEZHuZeMWg4iIdCNrg8HMPmtmW8xso5l9O+x6EmVmXzQzN7OSsGuJl5l9J/hM1pnZs2Y2LOyazpSZLTCzrWZWZWb3hl1PvMxsvJlVmNmm4Gfkc2HXlCgzyzWz35nZ82HXkigzG2ZmTwc/L5vN7Ipkvn5WBoOZzQMWApe4+4XAP4VcUkLMbDxwA7A77FoS9CLwPne/GNhG9IZOacPMcoGHgQ8A04HbzWx6uFXFrQ34ortPB94PfCaNx3LK54he+j8TfBf4pbtPAy4hyePKymAAPg084O7NAO5eE3I9iXoI+Bs6vcN0+nD3X/kf7gG+EhgXZj1xmA1UuftOd28Byon+AZJ23H2/u68Jpo8T/cXTp/dk70tmNg74EPBo2LUkysyGAlcT3L/G3Vvc/Wgy3yNbg+E84Coze9PMXjWzy8IuKF5mthDY6+5vh11Lkt0FvBB2EWdoLLAnZr6aNP5leoqZTQRmAm+GW0lC/oXoH0+RsAtJgklALfCDYNfYo2Y2OJlvkNCNelKZmb0EjO5k1ZeJjnsE0U3ky4CnzGyyp+gpWj2M5W+J7kZKC92Nxd2fC9p8meiujCf7szZ5LzMrAv4H+Ct3PxZ2PfEwsw8DNe7+lpldE3Y9SZAHXAp81t3fNLPvAvcCX03mG2Qkd5/f1Toz+zTwTBAEvzWzCNHrjtT2V31noquxmNlFRP96eNvMILrrZY2ZzXb3A/1YYq9197kAmNmdwIeB61I1qLuxFxgfMz8uWJaWzCyfaCg86e7PhF1PAuYAN5nZB4EBQLGZ/cjd/yTkuuJVDVS7+6ktuKeJBkPSZOuupJ8D8wDM7DyggDS8sJa7r3f3Ue4+0d0nEv0Hc2mqhkJPzGwB0c39m9y9Mex64rAKmGpmk8ysALgNWBpyTXGx6F8ajwGb3f2fw64nEe5+n7uPC35GbgNeSeNQIPj53mNm5weLrgM2JfM9MnaLoQdLgCVmtgFoARal4V+nmeh7QCHwYrAFtNLd/zzcknrP3dvM7B5gOZALLHH3jSGXFa85wB3AejNbGyz72+D+7RK+zwJPBn+A7AQ+mcwX1zefRUSkg2zdlSQiIl1QMIiISAcKBhER6UDBICIiHSgYRESkAwWDiIh0oGAQEZEOFAwiItLB/wfxnTAjadG1PwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x106f61eb8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def tanh(x, derivative=False):\n",
    "    if (derivative == True):\n",
    "        return (1 - (x ** 2))\n",
    "    return np.tanh(x)\n",
    "\n",
    "\n",
    "y = tanh(x)\n",
    "\n",
    "plt.plot(x,y)\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAHK9JREFUeJzt3Xl4lOW9xvHvQwgBEiRgILJJQHaRJUEEXA7Rat1t6wZIW2tPaUEU64prazfbWqscUbupbQ9LWMRdcQ3uG0nY932HsCdAyPY7fxB6eVogk0kmzzsz9+e65pLAO8z9c8g9b555k8eZGSIiEj0a+A4gIiI1o+IWEYkyKm4RkSij4hYRiTIqbhGRKKPiFhGJMipuEZEoo+IWEYkyKm4RkSjTMBJ/aVpammVkZIR13wMHDpCcnFy3gTzRLMGkWYIrluap6Sx5eXk7zaxVKMdGpLgzMjKYO3duWPedM2cOQ4cOrdtAnmiWYNIswRVL89R0Fufc+lCP1VKJiEiUCam4nXOpzrmZzrllzrmlzrnBkQ4mIiLHFupSyQRgtpld45xrBDSNYCYRETmBaovbOdccOA+4EcDMSoHSyMYSEZHjCWWppBNQCDzvnCtwzv3NORcbb/uKiEQhV91GCs65AcDnwNlm9oVzbgKw38we/LfjRgGjANLT07NycnLCClRcXExKSkpY9w0azRJMmiW4Ymmems6SnZ2dZ2YDQjrYzE54A04B1n3t43OB1090n6ysLAtXbm5u2PcNGs0STJoluGJpnprOAsy1avr46K3apRIz2wZsdM51r/qtC4AlIb+MiIjEgS/W7OLZj9cePcGNqFCvKrkFmFx1Rcka4AeRiyQiEl0Kiw5zy9QCkpMaMnxgB5o2isj3Nv5LSH+7mc0DQlt7ERGJIxWVxq1TC9h3qIx/3DQw4qUNEfqWdxGRePH4Oyv4bM0uHr2mDz3bnFQvj6lveRcRCVPu8h1MzF3FdQPac+2ADvX2uCpuEZEwbN57iJ9Om0ePU5rxi6t61+tjq7hFRGqotLySmyfnU15hPDMyi8aJCfX6+FrjFhGpod+8sZR5G/fyzA2ZdEqr/28k1xm3iEgNvL5gK3//dB03nd2JS85o4yWDiltEJERrCou554UF9D81lfGX9PCWQ8UtIhKCQ6UVjJmcT2KC46kRmTRq6K8+tcYtIhKCB19exPLtRTx/45m0TW3iNYvOuEVEqjH9q43MzNvE2OwuDO3e2nccFbeIyIks2bKfB19exNldTua2b3TzHQdQcYuIHNf+kjLGTM4jtWkiE4b1J6GB8x0J0Bq3iMgxmRl3z1jAxj2HyBk1iLSUJN+R/kVn3CIix/DcJ+uYvXgb91zcnTMzWvqO8/+ouEVE/k3e+j088sZSLuqVzo/O7ew7zn9QcYuIfM3uA6WMnZJPm9TGPHptX5wLxrr212mNW0SkSkWlMS6ngF0HSpk1egjNmyT6jnRMOuMWEaky8f1VfLRyJz+/4nR6t2vuO85xqbhFRICPV+7kifdW8O3+7Rg+sP42RQiHiltE4t62fSWMyymgS6sUfv3t3oFc1/46FbeIxLWyikrGTsnnUFkFz4zMrJfNfmsr+AlFRCLo0beWM3f9Hv5neH+6tG7mO05IdMYtInHrrcXb+MuHa/juoI5c2bet7zghU3GLSFxav+sAd86YT5/2zXng8p6+49SIiltE4k5J2ZFNERq4I5siJDWs381+ayukNW7n3DqgCKgAys1sQCRDiYhE0sOvLmHxlv08+/0BdGjZ1HecGqvJm5PZZrYzYklEROrBiwWbmPrlBkYPPY0Leqb7jhMWLZWISNxYsb2I+2Yt4qxOLbnjwmBsihAOZ2bVH+TcWmAPYMCfzewvxzhmFDAKID09PSsnJyesQMXFxaSkpIR136DRLMGkWYIrkvOUlBsPf3aIA2XwiyGNSW0c2fPWms6SnZ2dF/IytJlVewPaVf23NTAfOO9Ex2dlZVm4cnNzw75v0GiWYNIswRWpeSorK23slHzrNP41+2RVYUQe49/VdBZgroXQx2YW2lKJmW2u+u8O4EVgYMgvIyIink36fD2vzt/CHRd1Z8hpab7j1Fq1xe2cS3bONTv6a+AiYFGkg4mI1IX5G/fyy9eWkt29FaP/6zTfcepEKFeVpAMvVv3QlYbAFDObHdFUIiJ1YO/BUsZMzqdVsyT+eF0/GgRks9/aqra4zWwN0LcesoiI1JnKSuOO6fPZUVTCjJ8MoUVyI9+R6owuBxSRmPSnD1fz3rIdPHBZL/p1SPUdp06puEUk5ny+Zhd/eGs5l/Vpw/cGd/Qdp86puEUkpuwoKuGWqQVkpCXzu6v7BH5ThHDo53GLSMwor6jk1qkFFJWUMemHZ5GSFJsVF5tTiUhcevzdFXy+ZjePXduX7qdEx6YI4dBSiYjEhPeXbeep3NUMO7MDV2e19x0nolTcIhL1Nu05yE+nzadXm5P4+ZWn+44TcSpuEYlqh8sruHlKAZWVxtM3ZNI4Mbo2RQiH1rhFJKr95vWlzN+4lz+NzCQjLdl3nHqhM24RiVqvzt/CPz5bz3+f04mLe7fxHafeqLhFJCqtLixm/AsLyOrYgnsu6eE7Tr1ScYtI1DlYWs7oSXkkJSYwcUR/EhPiq8q0xi0iUcXMeOClRazcUcw/bxpIm+ZNfEeqd/H1MiUiUW/aVxuZlb+ZcRd05dyurXzH8ULFLSJRY9HmfTz0ymLO7ZrGLed39R3HGxW3iESFfYfKGDM5n5ZNG/HE9f1IiJFNEcKhNW4RCTwz464Z89my9xDTfjyIk1OSfEfySmfcIhJ4z368lreXbGf8JT3I6tjSdxzvVNwiEmhz1+3mkTeXcVGvdH54TiffcQJBxS0igbWr+DBjpxTQvkUTHr22b0xuihAOrXGLSCBVVBq3TZvH7oOlvDhmCM2bJPqOFBg64xaRQHry/ZV8tHInD195Oqe3be47TqCouEUkcD5cUciE91byncx2DDuzg+84gaPiFpFA2brvELdNm0e31s341bd6a137GEIubudcgnOuwDn3WiQDiUj8Kq80bp6cz+GyCp4emUnTRnob7lhqcsY9DlgaqSAiIjOWl5K/YS+/vboPp7VK8R0nsEIqbudce+Ay4G+RjSMi8Wr2oq28tb6cG4dkcEXftr7jBFqoZ9xPAHcDlRHMIiJxat3OA9w1YwGdmzfgvkt7+o4TeM7MTnyAc5cDl5rZGOfcUOBOM7v8GMeNAkYBpKenZ+Xk5IQVqLi4mJSU2PgSSbMEk2YJltIK45efl7C7pJJ7+hqnpkX3PEfV9LnJzs7OM7MBIR1sZie8AY8Am4B1wDbgIDDpRPfJysqycOXm5oZ936DRLMGkWYLlnpnzreM9r9n7S7fHxDxH1XQWYK5V08dHb9UulZjZvWbW3swygGHA+2Y2MuSXERGR45iZt4mcrzZyc/ZpZPdo7TtO1NB13CLixbJt+3ngpYUM7nwyP/1GN99xokqNLpI0sznAnIgkEZG4UVRSxphJ+TRrnMiE4f1oGGeb/daW/m+JSL0yM8bPWsi6XQd4cnh/Wjdr7DtS1FFxi0i9+udn63l9wVbu/GZ3BnU+2XecqKTiFpF6U7BhD796fQkX9GjNT847zXecqKXiFpF6sedAKWOnFNC6WWMeu64vDeJ4s9/a0k9wEZGIq6w0bp8+j8Kiw8wcPZjUpo18R4pqOuMWkYh75oPV5C4v5MHLe9KnfarvOFFPxS0iEfXp6p089vZyruzblpGDOvqOExNU3CISMTv2l3Dr1Hl0Skvmke+coU0R6ojWuEUkIsorKhk7tYADh8uZ8qOzSE5S3dQV/Z8UkYj4w9sr+HLtbh6/vi/d0pv5jhNTtFQiInXu3SXb+dMHqxlx1ql8u39733FijopbROrUxt0HuX36PHq3O4mHLu/lO05MUnGLSJ05XF7BmMn5GPDUiEwaJyb4jhSTtMYtInXml68tYeHmffzlu1l0PDnZd5yYpTNuEakTL8/bzKTPNzDqvM5cdPopvuPENBW3iNTaqh1F3DtrIWdmtOCub3b3HSfmqbhFpFYOlpYzelI+TRITeHJ4JonaFCHitMYtImEzM+5/cRGrCov535vO4pTm2hShPuilUUTCNvXLjbxYsJnbLujGOV3TfMeJGypuEQnLos37+PkrizmvWytuOb+L7zhxRcUtIjW272AZoyfncXJKI564vp82RahnWuMWkRoxM+6cOZ+te0uY9uPBtEzWpgj1TWfcIlIjf/1oDe8s2c69l/Ykq2ML33HikopbREL25drd/G72ci7pfQo3nZ3hO07cUnGLSEgKiw4zdko+p7Zsyu+v6aNNETyqtridc42dc1865+Y75xY75x6uj2AiEhwVlca4nAL2HSrj6RsyadY40XekuBbKm5OHgfPNrNg5lwh87Jx708w+j3A2EQmIJ95dwaerd/H7a/rQs81JvuPEvWqL28wMKK76MLHqZpEMJSLBMWf5Dp58fxXXZrXnugEdfMcRwB3p5WoOci4ByAO6AE+Z2T3HOGYUMAogPT09KycnJ6xAxcXFpKSkhHXfoNEswaRZQrfrUCU/+/QQqUmOBwc3ISkhsuva8fzcZGdn55nZgJAONrOQb0AqkAv0PtFxWVlZFq7c3Nyw7xs0miWYNEtoDpdV2FUTP7bTH5ptq3cURexxvi6enxtgroXYxTW6qsTM9lYV98U1uZ+IRJ9H3lzKvI17+d3VfejcKjbOgmNFKFeVtHLOpVb9uglwIbAs0sFExJ83F27l+U/WceOQDC7r08Z3HPk3oVxV0gb4R9U6dwNgupm9FtlYIuLL2p0HuGvmAvp1SOW+S3v6jiPHEMpVJQuA/vWQRUQ8KymrYPSkPBomOJ66IZNGDfU9ekGkHzIlIv/y0MuLWL69iOdvPJN2qU18x5Hj0MupiAAwfe5Gps/dxNjsLgzt3tp3HDkBFbeIsGTLfh58aRFDTjuZ277RzXccqYaKWyTOFZWUcfOUfJo3SWTCsP4kaFOEwNMat0gcMzPueWEBG3YfZOqPBtGqWZLvSBICnXGLxLHnP1nHGwu3cfc3uzOwU0vfcSREKm6ROJW/YQ+/eWMpF/ZKZ9R5nX3HkRpQcYvEod0HShk7OZ82qY35w7V9tSlClNEat0icqaw0bps2j53FpbwwegjNm2hThGijM26ROPNU7io+XFHIQ1f04oz2zX3HkTCouEXiyKerdvL4uyv4Vr+23HDWqb7jSJhU3CJxYtu+Em7NKaBzqxR+/e0ztK4dxbTGLRIHyioquWVqPgdLK8gZlUlykj71o5mePZE48Ie3lvPVuj1MGNaPLq2b+Y4jtaSlEpEY9/bibfz5wzWMHHQqV/Vr5zuO1AEVt0gM27DrIHfMmM8Z7Zrz4OW9fMeROqLiFolRJWUVjJmShwOeviGTpIYJviNJHdEat0iM+sVrS1i0eT9//d4AOrRs6juO1CGdcYvEoBcLNjHliw38+L86c2GvdN9xpI6puEVizIrtRdw3axEDO7Xkrou6+44jEaDiFokhBw6XM2ZyPslJDZk4vD8NE/QpHov0rIrECDPj3lkLWVNYzP8M60frkxr7jiQRouIWiRGTvtjAK/O3cPuF3RjSJc13HIkgFbdIDFiwaS+/fHUJQ7u3YszQLr7jSIRVW9zOuQ7OuVzn3BLn3GLn3Lj6CCYiodl3sIwxk/NJS2nE49f1o4E2+415oVzHXQ7cYWb5zrlmQJ5z7h0zWxLhbCJSjUoz7pgxj+37S5j+48G0SG7kO5LUg2rPuM1sq5nlV/26CFgK6AceiATA7LVlvLt0B/df2pP+p7bwHUfqSY3WuJ1zGUB/4ItIhBGR0H2xZhczV5ZxWZ82fH9Ihu84Uo+cmYV2oHMpwAfAr81s1jH+fBQwCiA9PT0rJycnrEDFxcWkpKSEdd+g0SzBFAuz7D1cyc8+LSGpQSUPn51Mk4axsa4dC8/NUTWdJTs7O8/MBoR0sJlVewMSgbeA20M5Pisry8KVm5sb9n2DRrMEU7TPUl5RacP+/Jl1f+AN++cr7/mOU6ei/bn5uprOAsy1EPrVzEK6qsQBzwJLzeyPIb98iEhEPP7OCj5bs4tffesMOjTTFb3xKJRn/Wzgu8D5zrl5VbdLI5xLRI4hd9kOJuau4voBHbgmq73vOOJJtZcDmtnHQGwsoIlEsU17DvLT6fPo2eYkHr7qdN9xxCN9nSUSBUrLK7l5SgEVFcYzN2TSOFGbIsQzbaQgEgV+88ZS5m/cy59GZpKRluw7jnimM26RgHttwRb+/uk6fnhOJy7u3cZ3HAkAFbdIgK0uLOaemQvIPDWV8Zf08B1HAkLFLRJQh0orGDMpn6TEBCaOyCRRmyJIFa1xiwSQmfHAS4tYsaOIf/xgIG1Tm/iOJAGil3CRAJo+dyMv5G/i1vO7cl63Vr7jSMCouEUCZvGWfTz08mLO6ZLGrRd09R1HAkjFLRIg+0uObIqQ2jSRJ4b1I0GbIsgxaI1bJCDMjLtnLGDTnkNMGzWItJQk35EkoHTGLRIQz368ltmLtzH+4h4MyGjpO44EmIpbJADy1u/mt28u46Je6fz3uZ18x5GAU3GLeLar+DA3Ty6gbWoTHr22L0d+krLI8WmNW8Sjikrjtmnz2H2wlFmjh9C8SaLvSBIFdMYt4tGT76/ko5U7+cWVp9O7XXPfcSRKqLhFPPloZSET3lvJdzLbcf2ZHXzHkSii4hbxYOu+Q4zLmUfX1in86lu9ta4tNaLiFqlnZRWVjJ1SwOGyCp4ZmUXTRnqrSWpG/2JE6tnvZy8jb/0enhzen9NapfiOI1FIZ9wi9Wj2om389aO1fG9wR67o29Z3HIlSKm6RerJ+1wHumjGfvu2bc/9lPX3HkSim4hapByVlFYyelE+DBo6JIzJJaqjNfiV8WuMWqQcPv7qYJVv389yNA+jQsqnvOBLldMYtEmEv5G1i6pcbGTP0NM7vke47jsQAFbdIBC3fVsT9Ly3krE4tuf3Cbr7jSIyotridc88553Y45xbVRyCRWFF8uJzRk/NISUrkyRH9aajNfqWOhPIv6e/AxRHOIRJTzIx7XljAup0HmDiiP62bNfYdSWJItcVtZh8Cu+shi0jM+Odn63l9wVbu/GZ3BnU+2XcciTHOzKo/yLkM4DUz632CY0YBowDS09OzcnJywgpUXFxMSkpsfDeZZgmmSM+yZm8Fv/6ihN5pCYzLTKJBBH8OSSw9LxBb89R0luzs7DwzGxDSwWZW7Q3IABaFcqyZkZWVZeHKzc0N+75Bo1mCKZKz7C4+bEMeec+GPPKe7TlwOGKPc1QsPS9msTVPTWcB5lqIHavruEXqSGWlcfv0eRQWHWbm6MGkNm3kO5LEKL3NLVJHnvlgNbnLC3nw8p70aZ/qO47EsFAuB5wKfAZ0d85tcs79MPKxRKLLp6t38tjby7mib1tGDuroO47EuGqXSsxseH0EEYlWO/aXcOvUeXRKS+aR75yhTREk4rTGLVIL5RWVjJ1awIHD5Uz50VmkJOlTSiJP/8pEauGxd1bw5drdPHZtX7qlN/MdR+KE3pwUCdN7S7fzzJzVDB/Ygauz2vuOI3FExS0Sho27D3L79Pn0anMSP7vidN9xJM6ouEVq6HB5BTdPyafSjGdGZtI4UZsiSP3SGrdIDf369aUs2LSPP43MouPJyb7jSBzSGbdIDbwyfwv//Gw9Pzq3Exf3PsV3HIlTKm6REK3aUcT4FxYwoGML7r64h+84EsdU3CIhOFhazuhJ+TRJTGDiiEwStSmCeKQ1bpFqmBn3v7iIVYXF/O9NZ3FKc22KIH7ptEGkGlO/3MiLBZu57YJunNM1zXccERW3yIks2ryPn7+6mHO7pnHL+V18xxEBVNwix7XvUBmjJ+dxcnIjJgzrT4MG+uFREgxa4xY5BjPjzhnz2bq3hGk/HkzLZG2KIMGhM26RY/jbR2t5Z8l2xl/Sg6yOLXzHEfl/VNwi/+ardbv57exlXNL7FH54TiffcUT+g4pb5Gt2Fh9m7JR8OrRowu+u6aNNESSQVNwiVSoqjXE5Bew9WMbTN2RxUuNE35FEjklvTopUmfDeSj5ZtYvfXX0Gvdqe5DuOyHHpjFsE+GBFIU++v5Jrstpz3YAOvuOInJCKW+Lelr2HuC2ngO7pzfjlVb21ri2Bp+KWuFZaXsnNU/IpqzCeviGTJo20KYIEn9a4Ja799s1lFGzYy1MjMuncKsV3HJGQ6Ixb4tYbC7fy3CdruXFIBpf1aeM7jkjIQipu59zFzrnlzrlVzrnxkQ4lEmlrdx7g7pkL6Nchlfsu7ek7jkiNVFvczrkE4CngEqAXMNw51yvSwUQipbTCGD0pj4YJjokj+tOoob7wlOgSyr/YgcAqM1tjZqVADnBVZGOJRM6kpaUs21bE49f3o32Lpr7jiNRYKG9OtgM2fu3jTcBZkQhzxZMfs2vfQZLzP4jEX1/vDhzULEFTYcaawnLGZnchu3tr33FEwlJnV5U450YBowDS09OZM2dOjf+OlMoSGiZVkuAO1VUsr1I0S/A46NLeyGy0hTlztvpOU2vFxcVhfa4FVSzNE9FZzOyEN2Aw8NbXPr4XuPdE98nKyrJw5ebmhn3foNEswaRZgiuW5qnpLMBcq6aPj95CWeP+CujqnOvknGsEDANeiczLiIiIVKfapRIzK3fOjQXeAhKA58xsccSTiYjIMYW0xm1mbwBvRDiLiIiEQBewiohEGRW3iEiUUXGLiEQZFbeISJRRcYuIRBl35LrvOv5LnSsE1od59zRgZx3G8UmzBJNmCa5Ymqems3Q0s1ahHBiR4q4N59xcMxvgO0dd0CzBpFmCK5bmieQsWioREYkyKm4RkSgTxOL+i+8AdUizBJNmCa5YmidiswRujVtERE4siGfcIiJyAoEtbufcLc65Zc65xc653/vOU1vOuTucc+acS/OdJVzOuUernpMFzrkXnXOpvjPVVKxsfO2c6+Ccy3XOLan6HBnnO1NtOecSnHMFzrnXfGepDedcqnNuZtXnylLn3OC6foxAFrdzLpsj+1r2NbPTgT94jlQrzrkOwEXABt9ZaukdoLeZ9QFWcGRTjagRYxtflwN3mFkvYBBwcxTPctQ4YKnvEHVgAjDbzHoAfYnATIEsbmA08FszOwxgZjs856mtx4G7gah+Q8HM3jaz8qoPPwfa+8wThpjZ+NrMtppZftWvizhSDu38pgqfc649cBnwN99ZasM51xw4D3gWwMxKzWxvXT9OUIu7G3Cuc+4L59wHzrkzfQcKl3PuKmCzmc33naWO3QS86TtEDR1r4+uoLbujnHMZQH/gC79JauUJjpzcVPoOUkudgELg+apln78555Lr+kHqbLPgmnLOvQuccow/up8juVpy5EvAM4HpzrnOFtBLYKqZ5T6OLJNEhRPNYmYvVx1zP0e+VJ9cn9nkPznnUoAXgNvMbL/vPOFwzl0O7DCzPOfcUN95aqkhkAncYmZfOOcmAOOBB+v6Qbwws28c78+cc6OBWVVF/aVzrpIj3/dfWF/5auJ4szjnzuDIK/B85xwcWVrId84NNLNt9RgxZCd6XgCcczcClwMXBPWF9AQ2Ax2+9nH7qt+LSs65RI6U9mQzm+U7Ty2cDVzpnLsUaAyc5JybZGYjPecKxyZgk5kd/epnJkeKu04FdankJSAbwDnXDWhEFP7gGTNbaGatzSzDzDI48qRmBrW0q+Ocu5gjX85eaWYHfecJQ8xsfO2OnAk8Cyw1sz/6zlMbZnavmbWv+hwZBrwfpaVN1ef2Rudc96rfugBYUteP4+2MuxrPAc855xYBpcD3o/DsLhZNBJKAd6q+gvjczH7iN1LoYmzj67OB7wILnXPzqn7vvqr9YcWvW4DJVScHa4Af1PUD6DsnRUSiTFCXSkRE5DhU3CIiUUbFLSISZVTcIiJRRsUtIhJlVNwiIlFGxS0iEmVU3CIiUeb/AHQ91pRElghyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x105e835f8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def relu(x):\n",
    "    a = []\n",
    "    for item in x:\n",
    "        if item > 0:\n",
    "            a.append(item)\n",
    "        else:\n",
    "            a.append(0)\n",
    "    return a\n",
    "\n",
    "\n",
    "y = relu(x)\n",
    "\n",
    "plt.plot(x,y)\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Base model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'2.1.3'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import keras\n",
    "keras.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.datasets import mnist\n",
    "\n",
    "(train_images, train_labels), (test_images, test_labels) = mnist.load_data()\n",
    "# the data, shuffled and split between train and test sets\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = train_images.reshape(60000, 784)\n",
    "x_test = x_test.reshape(10000, 784)\n",
    "\n",
    "x_train = x_train.astype('float32')\n",
    "x_test = x_test.astype('float32')\n",
    "\n",
    "x_train /= 255\n",
    "x_test /= 255\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes=10)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 10)                7850      \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 10)                110       \n",
      "=================================================================\n",
      "Total params: 7,960\n",
      "Trainable params: 7,960\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "10000/10000 [==============================] - 0s 17us/step\n",
      "Test loss: 1.3112017107009888\n",
      "Test accuracy: 0.7167\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.optimizers import sgd\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Dense(10, activation='sigmoid', input_shape=(784,)))\n",
    "model.add(Dense(10, activation='softmax'))\n",
    "\n",
    "model.summary()\n",
    "\n",
    "batch_size = 100\n",
    "num_classes = 10\n",
    "epochs=5\n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='sgd',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=0\n",
    "          )\n",
    "\n",
    "test_loss, test_acc = model.evaluate(x_test, y_test)\n",
    "\n",
    "print('Test loss:', test_loss)\n",
    "print('Test accuracy:', test_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "ReLu activation function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_3 (Dense)              (None, 10)                7850      \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 10)                110       \n",
      "=================================================================\n",
      "Total params: 7,960\n",
      "Trainable params: 7,960\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "10000/10000 [==============================] - 0s 19us/step\n",
      "Model2 - Test loss: 0.36345726056098937\n",
      "Model2 - Test accuracy: 0.8984\n"
     ]
    }
   ],
   "source": [
    "batch_size = 100\n",
    "num_classes = 10\n",
    "epochs=5\n",
    "\n",
    "model2 = Sequential()\n",
    "model2.add(Dense(10, activation='relu', input_shape=(784,)))\n",
    "model2.add(Dense(10, activation='softmax'))\n",
    "\n",
    "\n",
    "model2.summary()\n",
    "\n",
    "\n",
    "model2.compile(loss='categorical_crossentropy',\n",
    "              optimizer='sgd',\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "model2.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=0\n",
    "          )\n",
    "\n",
    "test_loss, test_acc = model2.evaluate(x_test, y_test)\n",
    "\n",
    "print('Model2 - Test loss:', test_loss)\n",
    "print('Model2 - Test accuracy:', test_acc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "512 nodes in Dense layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_5 (Dense)              (None, 512)               401920    \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 10)                5130      \n",
      "=================================================================\n",
      "Total params: 407,050\n",
      "Trainable params: 407,050\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "10000/10000 [==============================] - 0s 41us/step\n",
      "Model3 - Test loss: 0.24305746971666814\n",
      "Model3 - Test accuracy: 0.9323\n"
     ]
    }
   ],
   "source": [
    "model3 = Sequential()\n",
    "model3.add(Dense(512, activation='relu', input_shape=(784,)))\n",
    "model3.add(Dense(10, activation='softmax'))\n",
    "\n",
    "model3.summary()\n",
    "\n",
    "model3.compile(loss='categorical_crossentropy',\n",
    "              optimizer='sgd',\n",
    "              metrics=['accuracy'])\n",
    " \n",
    "epochs = 10\n",
    "model3.fit(x_train, y_train,\n",
    "          batch_size=batch_size,\n",
    "          epochs=epochs,\n",
    "          verbose=0\n",
    "          )\n",
    "\n",
    "test_loss, test_acc = model3.evaluate(x_test, y_test)\n",
    "\n",
    "print('Model3 - Test loss:', test_loss)\n",
    "print('Model3 - Test accuracy:', test_acc)\n",
    "\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}