code/preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook we will be preprocessing the following raw data:\n",
    "- Yellow taxi data (2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.types import *\n",
    "import pyspark.sql.functions as F\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "spark = SparkSession.builder.getOrCreate()\n",
    "\n",
    "spark.conf.set('spark.sql.repl.eagerEval.enabled', True)    # make it looks good\n",
    "spark.conf.set('spark.sql.execution.arrow.pyspark.enabled', True)   # enable spark arrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download dataset\n",
    "from os.path import getsize\n",
    "from urllib.request import urlretrieve\n",
    "\n",
    "# you will need to create the \"large\" folder if it does not exist\n",
    "# you can also change this to change where you want the data to be downloaded\n",
    "output_dir = \"../raw_data\"\n",
    "\n",
    "def download_data(fname_template, start, end):\n",
    "    # template for the type of taxi / year you wish to download\n",
    "\n",
    "    # change range(x, y) to be the months you want\n",
    "    for m in range(start, end):\n",
    "        month = str(m).zfill(2)\n",
    "        out = f'{fname_template}-{month}.csv'\n",
    "        url = f\"https://s3.amazonaws.com/nyc-tlc/trip+data/{out}\"\n",
    "        urlretrieve(url, f\"{output_dir}/{out}\")\n",
    "\n",
    "        print(f\"Done downloading {out} to {output_dir} with size {getsize(f'{output_dir}/{out}') / 1073741824:.2f}GB\")\n",
    "\n",
    "        \n",
    "download_data(\"yellow_tripdata_2018\" 1, 3)\n",
    "download_data(\"yellow_tripdata_2018\" 6, 9)\n",
    "download_data(\"yellow_tripdata_2018\" 12, 13)\n",
    "download_data(\"yellow_tripdata_2018\" 2, 3)\n",
    "download_data(\"yellow_tripdata_2018\" 7, 8)\n",
    "download_data(\"yellow_tripdata_2018\" 12, 13)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>VendorID</th><th>tpep_pickup_datetime</th><th>tpep_dropoff_datetime</th><th>passenger_count</th><th>trip_distance</th><th>RatecodeID</th><th>store_and_fwd_flag</th><th>PULocationID</th><th>DOLocationID</th><th>payment_type</th><th>fare_amount</th><th>extra</th><th>mta_tax</th><th>tip_amount</th><th>tolls_amount</th><th>improvement_surcharge</th><th>total_amount</th></tr>\n",
       "<tr><td>1</td><td>2018-01-01 00:21:05</td><td>2018-01-01 00:24:23</td><td>1</td><td>.50</td><td>1</td><td>N</td><td>41</td><td>24</td><td>2</td><td>4.5</td><td>0.5</td><td>0.5</td><td>0</td><td>0</td><td>0.3</td><td>5.8</td></tr>\n",
       "<tr><td>1</td><td>2018-01-01 00:44:55</td><td>2018-01-01 01:03:05</td><td>1</td><td>2.70</td><td>1</td><td>N</td><td>239</td><td>140</td><td>2</td><td>14</td><td>0.5</td><td>0.5</td><td>0</td><td>0</td><td>0.3</td><td>15.3</td></tr>\n",
       "<tr><td>1</td><td>2018-01-01 00:08:26</td><td>2018-01-01 00:14:21</td><td>2</td><td>.80</td><td>1</td><td>N</td><td>262</td><td>141</td><td>1</td><td>6</td><td>0.5</td><td>0.5</td><td>1</td><td>0</td><td>0.3</td><td>8.3</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+--------+--------------------+---------------------+---------------+-------------+----------+------------------+------------+------------+------------+-----------+-----+-------+----------+------------+---------------------+------------+\n",
       "|VendorID|tpep_pickup_datetime|tpep_dropoff_datetime|passenger_count|trip_distance|RatecodeID|store_and_fwd_flag|PULocationID|DOLocationID|payment_type|fare_amount|extra|mta_tax|tip_amount|tolls_amount|improvement_surcharge|total_amount|\n",
       "+--------+--------------------+---------------------+---------------+-------------+----------+------------------+------------+------------+------------+-----------+-----+-------+----------+------------+---------------------+------------+\n",
       "|       1| 2018-01-01 00:21:05|  2018-01-01 00:24:23|              1|          .50|         1|                 N|          41|          24|           2|        4.5|  0.5|    0.5|         0|           0|                  0.3|         5.8|\n",
       "|       1| 2018-01-01 00:44:55|  2018-01-01 01:03:05|              1|         2.70|         1|                 N|         239|         140|           2|         14|  0.5|    0.5|         0|           0|                  0.3|        15.3|\n",
       "|       1| 2018-01-01 00:08:26|  2018-01-01 00:14:21|              2|          .80|         1|                 N|         262|         141|           1|          6|  0.5|    0.5|         1|           0|                  0.3|         8.3|\n",
       "+--------+--------------------+---------------------+---------------+-------------+----------+------------------+------------+------------+------------+-----------+-----+-------+----------+------------+---------------------+------------+"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load dataset\n",
    "## yellow taxi 2018 (1, 2, 6, 7, 8, 12 month)\n",
    "\n",
    "yellow_1801_sdf = spark.read.csv('../raw_data/yellow_tripdata_2018-01.csv', header=True)\n",
    "yellow_1802_sdf = spark.read.csv('../raw_data/yellow_tripdata_2018-02.csv', header=True)\n",
    "yellow_1806_sdf = spark.read.csv('../raw_data/yellow_tripdata_2018-06.csv', header=True)\n",
    "yellow_1807_sdf = spark.read.csv('../raw_data/yellow_tripdata_2018-07.csv', header=True)\n",
    "yellow_1808_sdf = spark.read.csv('../raw_data/yellow_tripdata_2018-08.csv', header=True)\n",
    "yellow_1812_sdf = spark.read.csv('../raw_data/yellow_tripdata_2018-12.csv', header=True)\n",
    "\n",
    "# merge datasets\n",
    "sdf = yellow_1801_sdf.unionAll(yellow_1802_sdf).unionAll(yellow_1806_sdf).unionAll(yellow_1807_sdf).unionAll(yellow_1808_sdf).unionAll(yellow_1812_sdf)\n",
    "\n",
    "sdf.limit(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49837894"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# raw data row count\n",
    "sdf.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Preprocessing\n",
    "1.  Fix the dtype issues (there are all string type by default)\n",
    "2.  Extract some simple but useful features\n",
    "3. Remove unwanted columns\n",
    "4.  Remove outlier (mean +- 3 * stdv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change column dtypes\n",
    "\n",
    "ints = ('VendorID', 'passenger_count', 'RateCodeID', 'RatecodeID', 'payment_type', 'trip_type', 'DOlocationID', 'DOLocationID', 'PUlocationID', 'PULocationID')\n",
    "doubles = ('trip_distance', 'pickup_longitude', 'pickup_latitude', 'dropoff_longitude', 'dropoff_latitude',\n",
    "           'fare_amount', 'extra', 'mta_tax', 'tip_amount', 'tolls_amount', 'improvement_surcharge', 'total_amount')\n",
    "strings = ('store_and_fwd_flag', 'ehail_fee', 'SR_Flag', 'Dispatching_base_num', 'Dispatching_base_number')\n",
    "dtimes = ('lpep_pickup_datetime', 'lpep_dropoff_datetime', 'Pickup_DateTime', 'DropOff_datetime', 'tpep_pickup_datetime', 'tpep_dropoff_datetime')\n",
    "\n",
    "dtypes = {column: IntegerType() for column in ints}\n",
    "dtypes.update({column: DoubleType() for column in doubles})\n",
    "dtypes.update({column: StringType() for column in strings})\n",
    "dtypes.update({column: TimestampType() for column in dtimes})\n",
    "\n",
    "for column in sdf.columns:\n",
    "    sdf = sdf.withColumn(column, sdf[column].cast(dtypes[column]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "# extract some useful features\n",
    "# trip duration in minutes\n",
    "sdf = sdf.withColumn('trip_duration_min', (F.col('tpep_dropoff_datetime').cast('long') - F.col('tpep_pickup_datetime').cast('long'))/60.)\n",
    "\n",
    "# pickup day of week\n",
    "sdf = sdf.withColumn('pickup_day', F.date_format('tpep_pickup_datetime', 'E'))\n",
    "\n",
    "# pickup hour\n",
    "sdf = sdf.withColumn('pickup_hour', F.date_format('tpep_pickup_datetime', 'H'))\n",
    "sdf = sdf.withColumn('pickup_hour', sdf['pickup_hour'].cast(IntegerType())) # convert to int type\n",
    "\n",
    "# total_surcharge\n",
    "sdf = sdf.withColumn('total_surcharge', (F.col('extra') + F.col('mta_tax') + F.col('tolls_amount') + F.col('improvement_surcharge')))\n",
    "sdf = sdf.withColumn('total_surcharge', sdf['total_surcharge'].cast(DoubleType()))  # convert to double type\n",
    "\n",
    "# remove unwanted columns\n",
    "drop_cols = ['VendorID', 'RatecodeID', 'store_and_fwd_flag', 'extra', 'mta_tax', 'tolls_amount', 'improvement_surcharge']\n",
    "sdf = sdf.drop(*drop_cols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEICAYAAABCnX+uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAYhklEQVR4nO3de7wcdX3/8dc7ORBCAkhIqNzMIaZYw0XbRKsVS0hAQhCwWlv9FZIAKSoaFFovSFpAxVYUrMU+yo/GChEvQH/QEsLFQIE+fljFBEmFIuUYEgkglxMDJFxP8ukf891kz8k5u7Mne5kh7+fjsY+zM7Mz3/dONp/97ndmZxURmJlZeY3odAAzM9s+LuRmZiXnQm5mVnIu5GZmJedCbmZWci7kZmYl50JubSMpJE1uQzuS9G1Jv5F0z3Zu62ZJc5uVrRNeC8/BanMh3wFJWi3pRUkbUrFbKumATueqkDRP0v/fjk0cDhwN7B8Rbx+w7c+n571B0kuSNlVNPzBwQxFxbERcOZwQVfv5eUnrJf1I0kcltez/naTzJV1VPW97noOVgwv5juv4iBgL7AM8CVza4TzNNBFYHREbBy6IiC9HxNj03D8K/GdlOiIOrjwu9eqb8f/j+IjYLWX6W+CzwLeGsyFJXU3IY69BLuQ7uIh4CfgXYEplnqQ9JC2W9LSkNZIWShohaZyktZKOT48bK6lH0pw0fYWkyyQtS73QuyRNHKzdGm28GbgMeGfqJa8fYv19Jd0gaV3K8Odp/mnAoqr1L8i7LyTdKelCSXcDLwCT0rz5afk8SXdLulTSs5J+IWlmnm1HxLMRcQPwp8BcSYdUtTm/KkO/TyNpOOrjkh4GHk7zviHpUUnPSVoh6d1p/izg88Cfpue+cmAbaR8vTPv8qfRvsEda1p3amyvpV5KekXRu3v1nneNCvoOTtCtZcflx1exLgT2AScARwBzglIhYB5wK/JOkvYGvA/dFxOKqdf8M+CIwHrgP+O4QTQ/VxoP07ym/boj1vw+sBfYF/hj4sqSZEfGtAeufl3dfJCcDpwO7AWsGWf77wKr0/M4DrpM0Lu/GI+KelPvdDWR6X2q38mb7U+CtwDjge8C1knaJiFuALwNXp+f+lkG2NS/djiTb92OBbw54zOHAm4CZwF+nN1crMBfyHde/pt7uc2TjyV8FkDSSrLCfExHPR8Rq4GKyAkdE/BC4FrgdOA74yIDtLo2I/4iIl4FzyXrG/cbf67VRT9re4cBnI+KliLiPrBeea/06roiIByKiLyJeHWT5U8DfRcSrEXE18BDZfmjE42RFOK+/iYh1EfEiQERcFRG9KePFwCiywpvHnwGXRMSqiNgAnAN8aMCwzQUR8WJErARWAoO9IViBuJDvuN6XerujgE8Ad0l6PVlPc2f690bXAPtVTV8OHAJ8OyJ6B2z30cqdVCjWkfWaq+Vpo5Z9gXUR8fww16/l0TrLH4v+V5pbw7bPr579yPbLsDJJ+gtJD6bhnfVkn2zG59zWvmy737uA36qa9+uq+y+Q9dqtwFzId3ARsSkirgM2kfVynwFeJTs4V/EG4DHY0pv+v8Bi4GPa9nTCLb1vSWPJep6PD3hMzTaAepfkfBwYJ2m3IdbfHvXa3k+SBrQ78PkNSdLbyAp5ZRx8I7Br1UNeXytTGg//LPAnwJ7pzfhZQAMfO4TH2Xa/95Ed8LaSciHfwaWzM04E9gQejIhNwDXAhZJ2SwcrzwYqp7R9Pv09FfgasDgV94rZkg6XtDPZWPlPIqJfjzJHG08C+6dtbCNt70fA30jaRdJhwGkMPR7fTHsDZ0raSdIHgTcDN9VbSdLukt4L/AC4KiJ+nhbdB7xf0q7pTfG0OpvajazwPg10SfprYPeq5U8C3Rr6jJvvA2dJOjC90VbG1PvqPQcrLhfyHdcSSRvIxsgvBOZGROU86gVkPcVVZD3H7wH/LGkqWcGdk4rxV8h6gJ+r2u73yA4CrgOmko3JDmbQNtKyfwceAH4t6Zkh1v8w0E3Ww7weOC8iluV98tvhJ8Bvk32quBD440GGl6otkfQ82fDIucAlwClVy78OvEJWgK+k/pvRrcDNwP+QDYu8RP+hl2vT315J9w6y/j8D3wH+A3gkrb+gTptWcPIPS1izSLoCWBsRCzudpRUkzQPmR8Thnc5iVs09cjOzknMhNzMrOQ+tmJmVnHvkZmYl1/aL8IwfPz66u7vb3ayZWamtWLHimYiYMNiythfy7u5uli9f3u5mzcxKTdJg1/4BPLRiZlZ6LuRmZiXnQm5mVnIu5GZmJedCbmZWci7kZmYl50JuZlZyLuRmZiXnQm5mVnIu5GZmJedCbmZWci7kZmYl50JuZlZyLuRmZiXnQm5mVnIu5GZmJedCbmZWci7kZmYl50JuZlZybf/NTjOzMpg/fz7r16/niCOOYMGCBZ2OU5MLuZnZIJ544gk2btxIT09Pp6PU5aEVM7OScyE3Mys5F3Izs5JzITczKzkXcjOzknMhNzMrORdyM7OScyE3Mys5F3Izs5JzITczKzkXcjOzknMhNzMrORdyM7OScyE3Mys5F3Izs5JzITczKzkXcjOzknMhNzMrORdyM7OScyE3Mys5F3Izs5JzITczKzkXcjOzknMhNzMrORdyM7OScyE3Mys5F3Izs5JzITczKzkXcjOzknMhNzMrORdyM7OS66r3AEnTgHcD+wIvAvcDt0XEuhZnMzOzHIbskUuaJ+le4BxgNPAQ8BRwOLBM0pWS3tCemGZmNpRaPfIxwLsi4sXBFkp6K/DbwK9akMvMzHIaspBHxD/UWjEi7mt6GjMza1jdg52SLpK0u6SdJN0u6RlJJ7UjnJmZ1ZfnrJX3RMRzwHuBtcBBwKdbmsrMzHLLU8h3Sn9nA9/32SpmZsVS9/RDYImkX5CdeniGpAnAS62NZWZmedXtkUfE54B3AtMi4lXgBeDEVgczM7N88hzs3BX4OPCPada+wLRWhjIzs/zyjJF/G3gF+IM0vRb4UssSmZlZQ/IU8jdGxEXAqwDpC0JqaSozsw57+eWXAXjsscc6nKS+PIX8FUmjgQCQ9Ebg5ZamMjPrsM2bNwPw4ouDfrm9UPKctXI+cAtwgKTvAu8CTmllKDMzy69uIY+IH0paAbyDbEjlkxHxTMuTmZlZLnnOWrk9InojYmlE3BgRz0i6vR3hzMysviF75JJ2AXYFxkvak60HOHcnOwXRzMwKoNbQykeAT5EV7Xur5j8H1LwyopmZtU+ty9h+A/iGpAURcWkbM5mZWQPynLXyrKQ5A2dGxOIW5DEzswblKeRvq7q/CzCTbKjFhdzMrADynH64oHpa0h7Ad1qWyMzMGpLnm50DvUD2W51mZlYAdXvkkpaQvp5PVvinANe0MpSZmeWXZ4z8a1X3+4A1EbG2RXnMzKxBecbI72pHEDMzG548X9F/v6SHJT0r6TlJz0t6rh3hzMysvjxDKxcBx0fEg60OY2Zmjctz1sqTLuJmZsWVp0e+XNLVwL9S9YMSEXFdq0KZmVl+eQr57mTnjr+nal4ALuRmZgWQ56wV/xqQmVmB1boe+Wci4iJJl7L1C0FbRMSZLU1mZma51OqRVw5wLm9HEDMzG55a1yNfkv5e2b44ZmbWqDzXWpkGnAtMrH58RBzWwlxmZpZTnrNWvgt8Gvg5sLm1cczMrFF5CvnTEXFDy5OYmdmw5Cnk50laBNyOvxBkZlY4eQr5KcDvADuxdWjFXwgyMyuIPIX8LRFxaMuTmJnZsOS5aNaPJU1peRIzMxuWPD3yw4G5kh4hGyMXED790MysGPIU8lktT2FmZsNW61orYyNiQ0SsqfeY1kQzM7M8ao2R/5ukiyX9oaQxlZmSJkk6TdKtuLduZtZxta61MlPSbOAjwLsk7Qn0AQ8BS4G5EfHr9sQ0M7Oh1Bwjj4ibgJvalMXMzIYhz+mHZmZWYC7kZmYl50JuZlZyuQq5pJGS9pX0hsqt1cEG6u3t5cwzz6S3t7fdTe/Qir7fi57Pmqunp4fjjjuOnp6ehtetfq1Ub6cyv6enhzPOOIP58+dz6qmnsnlzdmmpjRs3Mn369H63I488kmOOOYaTTz6Zo48+munTp/PFL36R6dOns2TJki1Zjz32WE4//XR6e3tZvnw5M2bMYMWKFU3dJwCK2ObnOPs/QFoAnAc8SdVFs4b7zc5p06bF8uWN/3rcJZdcwpIlSzjhhBM466yzhtO0DUPR93vR81lzzZs3j9WrV9Pd3c0VV1zR0LrVr5WVK1du2c5hhx3GkiVLmDhxIqtXr97ujJK44447tmQFOPHEE7n99tvZsGEDY8eO5cYbbxzOdldExLTBluXpkX8SeFNEHBwRh6ZbW7+e39vbyy233EJEcMstt7j31SZF3+9Fz2fN1dPTs6Uwrl69uqFeefVr5aabbuq3nZtvvpmIaEoRB4gIFi1a1G97S5cuZcOG7LuTGzZsaHqvPE8hfxR4tqmtNujKK6/c8jFn06ZNLF68uJNxdhhF3+9Fz2fN9aUvfanmdC3Vr5VXX32137KB081w1VVX9Zvu6+vrN33eeec1tb08hXwVcKekcySdXbk10oik0yUtl7T86aefbjjkbbfdtmVH9PX1sWzZsoa3YY0r+n4vej5rroE95kZ60NWvlSKo9M6bJU8h/xWwDNgZ2K3qlltEXB4R0yJi2oQJExoOedRRR9HVlX13qauri6OPPrrhbVjjir7fi57Pmqu7u7vmdC3Vr5UiGDt2bFO3V7eQR8QFEXEBcAlwcdV028ydO5cRI7KoI0eOZM6cOe1sfodV9P1e9HzWXAsXLqw5XUv1a2WnnXbqt2zgdDOcdNJJ/aYHvolccEFzS2jdQi7pEEk/A+4HHpC0QtLBTU1Rx1577cWsWbOQxKxZs9hrr73a2fwOq+j7vej5rLkmT568pRfe3d3N5MmTc69b/VqZPXt2v+0ce+yxSGqoh1+LJObPn99ve8cdd9yWXvjYsWOZOnVqU9qqyDO0cjlwdkRMjIiJwF8A/9TUFDnMnTuXQw891L2uNiv6fi96PmuuhQsXMmbMmIZ64xXVr5Xq7VTmL1y4kClTpjB58mQmTZpUc1uSGDVqFAcccMCWHv3MmTMBOPvss7dkHT16NAcddBBz5szh/PPPZ8SIEU3vjUO+88hXRsRb6s3La7jnkZuZtdOMGTPYvHkzY8aMYenSpZ2OU/M88jyj/6sk/RXwnTR9EvBIs8KZmdn2yTO0ciowAbgOuD7dP6WVoczMLL+6PfKI+A1wZhuymJnZMNT6zc6/i4hPSVoCbDOQHhEntDSZmZnlUqtHXhkT/1o7gpiZ2fDU+s3OylVd3hoR36heJumTwF2tDGZmZvnkOdg5d5B585qcw8zMhqnWGPmHgf8DHCjphqpFuwG+XqiZWUHUGiP/EfAEMB64uGr+88B/tTKUmZnlV2uMfA2wBnhn++KYmVmj8lw06x2Sfippg6RXJG2S9Fw7wpmZWX15DnZ+E/gw8DAwGpgPXNrKUGZmll+uK61HRI+kkRGxCfi2pB+1OJeZmeWUp5C/IGln4D5JF5EdAB3T2lhmZpZXnqGVk9PjPgFsBA4APtDKUGZmll/NHrmkkcCFEXES8BLQ1p94MzOz+mr2yNOY+IQ0tGJmZgWUZ4x8NXB3+nbnxsrMiLikVaHMzCy/PIX88XQbQfb1fDMzK5A8PyzhcXEzswKrW8gl3cHgPywxoyWJzMysIXmGVv6y6v4uZKce9rUmjpmZNSrP0MqKAbPuluQflTAzK4g8QyvjqiZHAFOB17cskZmZNSTP0MoKsjFykQ2pPAKc1spQZmaWX56hlQPbEcTMzIan1k+9vb/WihFxXfPjmJlZo2r1yI9Pf/cG/gD49zR9JHAn4EJuZlYAtX7q7RQASTcCUyLiiTS9D/AP7YlnZmb15LmMbXeliCdPAge1KI+ZmTUoz1krd0q6Ffg+2dkrHwLuaGkqMzPLLc9ZK5+Q9EfAH6ZZl0fE9a2NZWZmeeX9zc7rARdvM7MCyjNGbmZmBeZCbmZWcrmGVtJPvf0O2cHOhyLilZamMjOz3PJcNOs44DLgl2TXWzlQ0kci4uZWhzMzs/ry9MgvBo6MiB4ASW8ElgIu5GZmBZBnjPypShFPVgFPtSiPmVkhjBiRlcfRo0d3OEl9eXrkD0i6CbiGbIz8g8BPKxfV8sWzzOy1aNSoUfT19bHffvt1OkpdeQr5LmRfyz8iTT8NjCO7qFbgi2eZmXVUnm92ntKOIGZmNjy1rkf+mYi4SNKlZD3vfiLizJYmMzOzXGr1yB9Mf5e3I4iZmQ1PreuRL5E0EjgkIj7dxkxmZtaAIU8/lNQVEZuAqW3MY2ZmDao1tHIP8HvAzyTdAFwLbKws9GmHZmbFkOf0w3FALzCD7KCn8GmHZmaFUauQ7y3pbOB+thbwim3OYjEzs86oVchHAmPpX8ArXMjNzAqiViF/IiK+0LYkZmY2LLUumjVYT9zMzAqmViGf2bYUZmY2bEMW8ohY184gZmY2PP7NTjOzknMhNzMrORdyM7OScyE3Mys5F3Izs5JzITczKzkXcjOzknMhNzMrORdyM7OScyE3Mys5F3Izs5JzITczKzkXcjOzknMhNzMrORdyM7OScyE3Mys5F3Izs5JzITczKzkXcjOzknMhNzMrORdyM7OScyE3Mys5F3Izs5JzITczKzkXcjOzknMhNzMrORdyM7OS6+p0ADOzItpnn31Yv349kydP7nSUulzIzcwGsWjRok5HyM1DK2ZmJedCbmZWci7kZmYl50JuZlZyLuRmZiXnQm5mVnIu5GZmJedCbmZWci7kZmYl50JuZlZyLuRmZiXnQm5mVnIu5GZmJedCbmZWci7kZmYl50JuZlZyLuRmZiXnQm5mVnIu5GZmJedCbmZWcoqI9jYoPQ2saWujwzMeeKbTIXJy1tZw1tZw1uGZGBETBlvQ9kJeFpKWR8S0TufIw1lbw1lbw1mbz0MrZmYl50JuZlZyLuRDu7zTARrgrK3hrK3hrE3mMXIzs5Jzj9zMrORcyM3MSu41XcglfVDSA5I2S5pWNf9oSSsk/Tz9nVG1bGqa3yPp7yUpzR8l6eo0/yeSuqvWmSvp4XSbWzX/wPTYh9O6OzeaNS07J7X7kKRjOp11QLa3SvqxpPskLZf09nbmbpSkBSnPA5IuKnLWtK2/lBSSxhc1q6SvSvqFpP+SdL2k1xU1a4PPa1bK3SPpc61ub7tExGv2BrwZeBNwJzCtav7vAvum+4cAj1Utuwd4JyDgZuDYNP8M4LJ0/0PA1en+OGBV+rtnur9nWnYN8KF0/zLgY8PIOgVYCYwCDgR+CYzsZNYBuX9Y1e5s4M525m7w9XAkcBswKk3vXdSsaVsHALeSfYFufFGzAu8ButL9rwBfKWrWBp7TyJR3ErBzeh5TWtXe9t5e0z3yiHgwIh4aZP7PIuLxNPkAsEvqCewD7B4R/xnZv+Zi4H3pcScCV6b7/wLMTL2IY4BlEbEuIn4DLANmpWUz0mNJ61a2lTtravcHEfFyRDwC9ABv72TWgdGB3dP9PYDKfm157pz5qn0M+NuIeBkgIp4qcFaArwOfIdvHFYXLGhE/jIi+NPljYP+iZm3A24GeiFgVEa8AP0jZCuk1Xchz+gDws/Sfez9gbdWytWke6e+jAOlF+yywV/X8AevsBayveoFXb6sRQ22/KFk/BXxV0qPA14Bz2pi7UQcB704f2e+S9LaiZpV0AtknxZUDFhUu6wCnkvWwy5C1lna3t126Oh1ge0m6DXj9IIvOjYh/q7PuwWQfBd9TmTXIw6LOskbmd0u6v8GsjbY7nHVqbavmPgZmAmdFxP+T9CfAt4Cj2pR7G3WydpF9LH8H8DbgGkmTCpr182x9XfZbbRjttjRr5bUr6VygD/huJ7M2Sbvb2y6lL+QRcdRw1pO0P3A9MCcifplmr2Xrx0LS/cerlh0ArJXURTaMsC7Nnz5gnTvJLrTzOkldqWexP3B3RBxDYyrtDszU6qyVbdXcx5IWA59Mk9cCi9qYext1sn4MuC59nL9H0mayiyIVKqukQ8nGlFemY4D7A/cqO5BcqKxVmecC7wVmpv1b3W5bszbJUNmLqdOD9O24se0BxNeRHbz4wCCP/SlZj61yEGZ2mv9x+h+EuSbdHwc8QtbT2zPdH5eWXUv/A4hnDCPrwfQ/YLSKrQeMOpo1PfZBYHq6PxNY0c7cDb4OPgp8Id0/iOyjs4qYdUDu1Ww92Fm4rGRj1f8NTBgwv3BZG3hOXSnvgWw92Hlwq9rb7rydDtDSJwd/RPbO+jLwJHBrmr8Q2AjcV3WrnMEwDbif7Ij1N9n67dddyIpdD9kR90lV7Zya5vcAp1TNn5Qe25PWHdVo1rTs3JTnIdLR/U5mHZD7cGBFeqH/BJjaztwNvh52Bq5Kbd8LzChq1gG5V5MKeRGzpnUfZev/pcuKmrXB5zUb+J+U8dxWt7c9N39F38ys5HzWiplZybmQm5mVnAu5mVnJuZCbmZWcC7mZWcm5kJuZlZwLuZlZyf0vEinKeZuczJQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# We now look at the distribution of the raw data\n",
    "\n",
    "# before preprocessing\n",
    "# boxplot of trip duration\n",
    "trip_duration_df = sdf.select('trip_duration_min').toPandas()\n",
    "sns.boxplot(x = trip_duration_df['trip_duration_min'])\n",
    "plt.title('Boxplot of Trip Duration')\n",
    "plt.ylabel('Trip duration (minutes)')\n",
    "plt.xlabel('')\n",
    "plt.savefig('../plots/trip_duration_raw_boxplot.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAEICAYAAABCnX+uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAVP0lEQVR4nO3dfbRddX3n8feXRAghICSghAdzQdZ0BozlIa0yxk6qIAkdtF22a2HLIqCiq61gsWtGrFhg1HZ07CxsXA4oU0iKVhlEZLDGAhUtKrZJ7RQYRCIJhWIwBBBiUAl854/9u9yTO/dh34dzz/3lvF9r7ZX9vL/7d+Bz9/ntfc6JzESSVK+9el2AJGlqDHJJqpxBLkmVM8glqXIGuSRVziCXpMoZ5JpREZERccwMHCci4uqIeCIi/r7bx5N6ySDvUxGxJSKeiYgdJey+HBFH9rquQRFxTkTcMYVdLAdOBY7IzF8eZf/PlfMfHD4xheO1FhFHRcTzEfHJmTjeZETE7RHx9l7XoXYM8v52RmYuABYDjwJrelzPdFoCbMnMn4yxzrczc0HH8K6JHCAi5k6ytrOBJ4AzI2KfSe5DeoFBLjLzp8D1wLGD8yLixRGxLiK2RcSDEXFxROwVEQsj4uGIOKOstyAiNkXE2WX6moi4IiJuiYinI+LrEbFkpOOOcYx/B1wBnFyulJ8cZfvDIuKmiHi81HBemf824KqO7S9r2xYRcW5E3FtqfyAi3tmxbEU59/dGxFbg6lLvRRHxg4jYHhHXRcTCcQ5zNnAx8CxwxrDjZ0T8XkTcX2r4YES8PCK+HRFPlf3v3bH+eeXcHy9tcViZP1D2Nbdj3Reusgff8UTEx8o7ss0Rsaos+zDwWuATM/lORVOQmQ59OABbgFPK+HxgLbCuY/k64EvA/sAA8H3gbWXZG4CtwEuATwPXd2x3DfA08CvAPsDHgTs6lidwTItjnNO53Sjn8HXgk8A84HhgG/D6NtuPthz4NeDlQAD/AdgJnFiWrQB2AR8p57Yv8AfAncARZd6VwF+NcdzXAj8DDqJ5B3TTsOUJ3AQcABxX1r0NOBp4MfB/gdVl3dcBjwEnlmOvAb5Rlg2Ufc3t2PftwNs7zv9Z4DxgDvC7wCNADF/XYfYPPS/AoUcvfBPkO4AnSzg9Aiwty+aUADm2Y/13Ard3TK8B7irbLeqYfw3wuY7pBcBzwJFlOoFjxjtGiyA+sux3/455fwpc03L7c8p5P9kxvHqE9W4E3l3GVwA/B+Z1LL+X8sejTC8uATl3lONeBdxYxk8u676kY3kCr+mY3gi8t2P6z4DLy/j/BD46rK2fLSHeJsg3dSybX9Y/dPi6DrN/sGulv/16Zh5IczX3LuDrEXEocDCwN/Bgx7oPAod3TH8KeAVwdWZuH7bfhwZHMnMH8Dhw2LB12hxjLIcBj2fm05PcHuDOzDywY7gzIlZFxJ2lq+JJ4PRS66Bt2XRFDVoCfDEinizr30vzB+alww8WEfsCvwV8BiAzvw38C/Dbw1Z9tGP8mRGmF5Txw+hov9LW22nfBls7tt1ZRheMsq5mMYNcZOZzmXkDTQAtp3m7/ixNSA16GfCvABExh6YLYR3wuyM8TvjC0y8RsQBYSHPl3mnMY9BcHY7lEWBhROw/yvYTVm48fgH4GPDS8kfur2m6WQYNr+shYNWwPwjzMnOkOn6DpsvkkxGxtfSzH07TZz4Zj9DRfhGxH7CIpg0Gb/LO71j/0Ans269FrYhBrsFnrt9E0297b2Y+B1wHfDgi9i83K98DXFs2+aPy71tpQm9dCfdBp0fE8nJT7oPAdzLzoY7ltDjGo8ARnTf2hm3/EPAt4E8jYl5EvBJ4G+Vqd5L2pnl3sg3YVW7+vWGcba4o57AEICIOKW05ktXAXwBLafr0jwdeAxwfEUsnUe9ngXMj4vjyR+hPaNp6S2Zuown0syJiTkS8labvv61HafrlVQGDvL/974jYATwFfJjmJto9Zdn5NFd1DwB30ITGX0TESTSBe3YJ44/QXL1d1LHfzwKX0HSpnAT8zijHH/EYZdnfAvcAWyPisVG2fwtNX/AjwBeBSzLzlrYnP1zpprmA5g/MEzRdHjeNs9nHyzp/ExFP09z4fNXwlSLicOD1NP3bWzuGjcB6mpCfaL23AR+geRfxQ5qgPrNjlfOA/0TT3XIczR++tj4O/GZ5ouXPJ1qbZtbgHWppWkTENcDDmXlxr2uR+oVX5JJUOYNckipn14okVc4rckmq3GS/9GfSDj744BwYGJjpw0pS1TZu3PhYZh4y0rIZD/KBgQE2bNgw04eVpKpFxIOjLbNrRZIqZ5BLUuUMckmqnEEuSZUzyCWpcga5JFXOIJekyhnkklQ5g1ySKmeQS1LlDHJJqpxBLkmVM8glqXIGuSRVziCXpMoZ5JJUOYNckipnkEtS5QxySapcVUG+Zs0a1qxZ0+syJGlWqSrI169fz/r163tdhiTNKlUFuSTp/2eQS1LlDHJJqpxBLkmVM8glqXIGuSRVziCXpMoZ5JJUOYNckipnkEtS5QxySaqcQS5JlTPIJalyBrkkVc4gl6TKGeSSVDmDXJIqZ5BLUuUMckmqnEEuSZUzyCWpcga5JFXOIJekyhnkklQ5g1ySKmeQS1LlDHJJqpxBLkmVM8glqXIGuSRVziCXpMoZ5JJUOYNckipnkEtS5QxySaqcQS5JlTPIJalyBrkkVc4gl6TKGeSSVDmDXJIqZ5BLUuXm9rqAidi5c2evS5CkWaeqIM/MXpcgSbOOXSuSVDmDXJIqZ5BLUuUMckmq3Lg3OyPiCOBM4LXAYcAzwN3Al4GvZObzXa1QkjSmMYM8Iq4GDgduBj4C/AiYB/wbYCXw/oi4KDO/0e1CJUkjG++K/M8y8+4R5t8N3BARewMvm/6yJEltjRnko4R45/KfA5umtSJJ0oSMe7MzIlaUfnIiYklE3BoRd0bEr3S/PEnSeNo8tfJfgR+X8T8BrgfeDVzepZokSRMw3s3OS2j6wC+MiABOAx4AXgocHBF/DNzuzU5J6p3x+sgvi4hfA+4AFgPfyswPAETEGzLzv8xAjZKkMbT50qwLgf8O/Ax4B0BEHAf8U/fKkiS1NW6QZ+Y3gVcNm3cP8PvdKkqS1N6YNzsjYvk4yw+IiFdMb0mSpIkY74r8zRHxUWA9sBHYRvPJzmOAXwWWAH/Y1QolSWMa72bnhRFxEPCbwG/R3PB8BrgXuDIz7+h+iZKksbTpI38C+HQZJEmzjF9jK0mVM8glqXKtgjwi9mkzT5I089pekX+75TxJ0gwb77tWDqX5YYl9I+IEIMqiA4D5Xa5NktTCeE+tnAacAxxB8zH9QU8Df9SlmiRJEzDec+RrgbUR8ebM/MIM1SRJmoA2X5oFcHNE/DYw0LmN334oSb3XNsi/RPPjEhtpvgVRkjRLtA3yIzJzZVcrkSRNStvHD78VEUu7WokkaVLaXpEvB86JiM00XSsBZGa+smuVSZJaaRvkq7pahSRp0toGeXa1CknSpLUN8i/ThHnQ/LDEUcB9wHFdqkuS1FKrIM/M3W50RsSJwDu7UpEkaUIm9TW2mfmPwC9Ncy2SpElodUUeEe/pmNwLOJHm9zslST3Wto98/47xXTR95n73iiTNAm37yC8DiIj9m8nc0dWqJEmttf2FoFdExHeBu4F7ImJjRLyiu6XtbsWKFbuNDw5XXXUVp512GitWrODMM8/k9NNPZ+PGjVxwwQVs2LCBlStXsmrVKjZt2jQtdWzfvp0LLriA7du3T8v+Zvtxp6rbdU9k/7W24Z6sn16Tbp5r25udnwLek5lLMnMJ8IdlXs9de+21/Oxnzfd4bd26lZ07d3LJJZdw1113cemll/LTn/6UZ555hg996EPTcry1a9dy1113sW7dumnZ32w/7lR1u+6J7L/WNtyT9dNr0s1zbRvk+2Xm1wYnMvN2YL9pr2YUnVfjbezYsYPMZMeOoR6gLVu2TPmqfPv27axfv57MZP369TN2FdGr405Vt+ueyP5rbcM9WT+9Jt0+17ZB/kBEfCAiBspwMbB5WiuZAVO9Kl+7di3PP/88AM8999yMXUX06rhT1e26J7L/WttwT9ZPr0m3z7VtkL8VOAS4AfhiGT+37UEi4h0RsSEiNmzb1runFrds2TKl7W+99VZ27doFwK5du7jlllumoarZe9yp6nbdE9l/rW24J+un16Tb59oqyDPzicy8IDNPzMwTMvPdmflE24Nk5qcyc1lmLjvkkEMmX+0UDQwMTGn7U045hblzmwd95s6dy6mnnjoNVc3e405Vt+ueyP5rbcM9WT+9Jt0+17ZPrSyLiBsi4h8j4p8Hh2mtZAZcfPHFU9p+9erV7LVX02Rz5szh7LPPno6yZu1xp6rbdU9k/7W24Z6sn16Tbp9r266VzwDXAG8GzugYZsTtt98+ofUXLFhARLBgwYIX5g0MDHDMMcdMqY5FixaxcuVKIoKVK1eyaNGiKe1vth93qrpd90T2X2sb7sn66TXp9rm2DfJtmXlTZm7OzAcHh2mtZJLOOuss9tlnHwAOPfRQ5s+fz2WXXcbSpUu59NJLmTdvHvvuu++Ur8YHrV69mqVLl8741UOvjjtV3a57IvuvtQ33ZP30mnTzXCNz/K8aj4jXA28BbqPjx5cz84aJHnDZsmW5YcOGiW4GDD2GONErdEmqXURszMxlIy1r+10r5wL/FngR8HyZlzRPsUiSeqhtkP/i8O8klyTNDm37yO+MiGO7WokkaVLaXpEvB1ZHxGaaPvKg+RbEV3atMklSK22DfGVXq5AkTVrbT3Z2PnL4GM0V+ie7WpkkqZW2n+zcOyJ+PSKuA34InAJc0dXKJEmtjNm1EhGn0jw/fhrwNeAvgV/OzNZfmCVJ6q7x+si/CvwdsDwzNwNExMe7XpUkqbXxgvwk4Ezg1oh4APgcMKfrVUmSWhuzjzwzv5uZ783MlwOXAicAe0fEVyLiHTNRoCRpbG0/EERmfjMz3wUcDlwOnNytoiRJ7bV9jvwFmfk8Td/5V6e/HEnSRLW+IpckzU4GuSRVrnWQR8TyiDi3jB8SEUd1ryxJUlttP9l5CfBe4H1l1ouAa7tVlCSpvbZX5L8BvBH4CUBmPgLs362iJEnttQ3yn2fzm3AJEBH7da8kSdJEtA3y6yLiSuDAiDgPuBX4dPfKkiS1Ne5z5BERwOdpfrPzKeAXgD/OzFu6XJskqYVxgzwzMyJuzMyTAMNbkmaZifxm5y91tRJJ0qS0/Yj+rwLvjIgHaZ5c8Tc7JWmWaBvkq7pahSRp0loFefmtTiLiJcC8rlYkSZqQtp/sfGNE3A9sBr4ObAG+0sW6JEkttb3Z+UHg1cD3M/Mo4PXAN7tWlSSptbZB/mxmbgf2ioi9MvNrwPHdK0uS1Fbbm51PRsQC4BvAZyLiR8Cu7pUlSWprzCvyiHhZGX0TsBO4EFgP/AA4o7ulSZLaGO+K/EbgxMz8SUR8ITPfDKztflmSpLbG6yOPjvGju1mIJGlyxgvyHGVckjRLjNe18osR8RTNlfm+ZRyGPqJ/QFerkySNa8wgz8w5M1WIJGlyWv/4siRpdjLIJalybT8QNCs0P1YkSepUVZDPnz+/1yVI0qxj14okVc4gl6TKGeSSVDmDXJIqZ5BLUuUMckmqnEEuSZUzyCWpcga5JFXOIJekyhnkklQ5g1ySKmeQS1LlDHJJqpxBLkmVM8glqXIGuSRVziCXpMoZ5JJUOYNckipnkEtS5QxySaqcQS5JlTPIJalyBrkkVc4gl6TKGeSSVDmDXJIqZ5BLUuUMckmqnEEuSZUzyCWpcga5JFXOIJekyhnkklQ5g1ySKmeQS1LlDHJJqpxBLkmVM8glqXIGuSRVbm6vC5iIlStX9roESZp1qgry888/v9clSNKsY9eKJFXOIJekyhnkklQ5g1ySKmeQS1LlDHJJqpxBLkmVM8glqXIGuSRVziCXpMoZ5JJUOYNckipnkEtS5QxySaqcQS5JlTPIJalyBrkkVc4gl6TKGeSSVDmDXJIqF5k5sweM2AY8OIVdHAw8Nk3l1M62GGJbDLEthuxJbbEkMw8ZacGMB/lURcSGzFzW6zpmA9tiiG0xxLYY0i9tYdeKJFXOIJekytUY5J/qdQGziG0xxLYYYlsM6Yu2qK6PXJK0uxqvyCVJHQxySapcNUEeESsj4r6I2BQRF/W6nsmKiCMj4msRcW9E3BMR7y7zF0bELRFxf/n3oI5t3lfO+76IOK1j/kkRcVdZ9ucREWX+PhHx+TL/OxEx0LHN6nKM+yNi9Qye+qgiYk5EfDcibi7TfdkWEXFgRFwfEd8r/32c3MdtcWH5/+PuiPiriJjXr23RSmbO+gGYA/wAOBrYG/g/wLG9rmuS57IYOLGM7w98HzgW+ChwUZl/EfCRMn5sOd99gKNKO8wpy/4eOBkI4CvAqjL/94AryviZwOfL+ELggfLvQWX8oFnQJu8BPgvcXKb7si2AtcDby/jewIH92BbA4cBmYN8yfR1wTj+2Res263UBLV/Yk4Gvdky/D3hfr+uapnP7EnAqcB+wuMxbDNw30rkCXy3tsRj4Xsf8twBXdq5TxufSfLItOtcpy64E3tLj8z8CuA14HUNB3ndtARxQwiuGze/HtjgceKiE6VzgZuAN/dgWbYdaulYGX9hBD5d5VStv504AvgO8NDN/CFD+fUlZbbRzP7yMD5+/2zaZuQv4MbBojH310uXAfwae75jXj21xNLANuLp0M10VEfvRh22Rmf8KfAz4F+CHwI8z82/ow7Zoq5YgjxHmVf3cZEQsAL4A/EFmPjXWqiPMyzHmT3abGRcR/xH4UWZubLvJCPP2iLaguSo8EfgfmXkC8BOa7oPR7LFtUfq+30TTTXIYsF9EnDXWJiPM2yPaoq1agvxh4MiO6SOAR3pUy5RFxItoQvwzmXlDmf1oRCwuyxcDPyrzRzv3h8v48Pm7bRMRc4EXA4+Psa9eeQ3wxojYAnwOeF1EXEt/tsXDwMOZ+Z0yfT1NsPdjW5wCbM7MbZn5LHAD8O/pz7Zop9d9Oy37zObS3HQ4iqGbncf1uq5JnksA64DLh83/b+x+I+ejZfw4dr+R8wBDN3L+AXg1QzdyTi/zf5/db+RcV8YX0vTDHlSGzcDCXrdJqW0FQ33kfdkWwN8Bv1DGLy3t0HdtAbwKuAeYX85hLXB+P7ZF6zbrdQETeHFPp3nC4wfA+3tdzxTOYznNW7V/Bv6pDKfT9M/dBtxf/l3Ysc37y3nfR7nrXuYvA+4uyz7B0Cd15wH/C9hEc9f+6I5t3lrmbwLO7XV7dNS1gqEg78u2AI4HNpT/Nm4sQdKvbXEZ8L1yHn9JE9J92RZtBj+iL0mVq6WPXJI0CoNckipnkEtS5QxySaqcQS5JlTPIJalyBrkkVe7/AbgjzGvQtU3cAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# before preprocessing\n",
    "# boxplot of fare amount\n",
    "fare_amount_df = sdf.select('fare_amount').toPandas()\n",
    "sns.boxplot(x = fare_amount_df['fare_amount'])\n",
    "plt.title('Boxplot of Fare Amount')\n",
    "plt.ylabel('Fare Amount ($)')\n",
    "plt.xlabel('')\n",
    "plt.savefig('../plots/fare_amount_raw_boxplot.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list numeric columns to be preprocessed\n",
    "yellow_num_cols = ['passenger_count', 'trip_distance', 'fare_amount', 'tip_amount', 'total_amount', 'trip_duration_min', 'total_surcharge']\n",
    "yellow_mean_stdv = sdf.agg( *[F.mean(colName).alias('mean_{}'.format(colName)) for colName in yellow_num_cols],\n",
    "                           *[F.stddev(colName).alias('stddev_{}'.format(colName)) for colName in yellow_num_cols])\n",
    "\n",
    "yellow_mean_std_min_max = yellow_mean_stdv\n",
    "\n",
    "# flag mean +- 3 * stdv as outliers\n",
    "for colName in yellow_num_cols:\n",
    "    meanCol = 'mean_{}'.format(colName)\n",
    "    stddevCol = 'stddev_{}'.format(colName)\n",
    "    minCol = 'min_{}'.format(colName)\n",
    "    maxCol = 'max_{}'.format(colName)\n",
    "    yellow_mean_std_min_max = yellow_mean_std_min_max.withColumn(minCol, F.col(meanCol) - 3 * F.col(stddevCol))\n",
    "    yellow_mean_std_min_max = yellow_mean_std_min_max.withColumn(maxCol, F.col(meanCol) + 3 * F.col(stddevCol))\n",
    "\n",
    "yellow_outliers = sdf.join(yellow_mean_std_min_max)\n",
    "for colName in yellow_num_cols:\n",
    "    isOutlierCol = 'is_outlier_{}'.format(colName)\n",
    "    minCol = 'min_{}'.format(colName)\n",
    "    maxCol = 'max_{}'.format(colName)\n",
    "    meanCol = 'mean_{}'.format(colName)\n",
    "    stddevCol = 'stddev_{}'.format(colName)\n",
    "    yellow_outliers = yellow_outliers.withColumn(isOutlierCol, F.when((F.col(colName) > F.col(maxCol)) | (F.col(colName) < F.col(minCol)), 1).otherwise(0))    \n",
    "    yellow_outliers = yellow_outliers.drop(minCol,maxCol, meanCol, stddevCol)\n",
    "\n",
    "# filter outliers\n",
    "yellow_clean_sdf = yellow_outliers.filter((yellow_outliers.is_outlier_passenger_count == 0) & \n",
    "                                            (yellow_outliers.is_outlier_trip_distance == 0) & \n",
    "                                            (yellow_outliers.is_outlier_fare_amount == 0) & \n",
    "                                            (yellow_outliers.is_outlier_tip_amount == 0) &\n",
    "                                            (yellow_outliers.is_outlier_total_surcharge == 0) &\n",
    "                                            (yellow_outliers.is_outlier_total_amount == 0) & \n",
    "                                            (yellow_outliers.is_outlier_trip_duration_min == 0))\n",
    "\n",
    "\n",
    "# remove outlier flag columns\n",
    "drop_cols = ['is_outlier_passenger_count', 'is_outlier_trip_distance', 'is_outlier_fare_amount', 'is_outlier_tip_amount', 'is_outlier_total_surcharge', 'is_outlier_total_amount', 'is_outlier_trip_duration_min']\n",
    "clean_sdf = yellow_clean_sdf.drop(*drop_cols)\n",
    "\n",
    "# do some futher simple outlier detection\n",
    "clean_sdf = clean_sdf.filter((sdf.payment_type == '1') | (sdf.payment_type == '2')) # select only payment type 1 and 2\n",
    "clean_sdf = clean_sdf.filter(sdf.trip_duration_min > 0) # 0 minute trip duration seems ridicilous\n",
    "clean_sdf = clean_sdf.filter(sdf.trip_distance > 0) # 0 mile seems ridicolous too!\n",
    "clean_sdf = clean_sdf.filter(sdf.passenger_count > 0)\n",
    "clean_sdf = clean_sdf.filter(sdf.fare_amount > 0)\n",
    "clean_sdf = clean_sdf.filter(sdf.tip_amount >= 0)\n",
    "clean_sdf = clean_sdf.filter(sdf.total_surcharge >= 0)\n",
    "clean_sdf = clean_sdf.filter(sdf.total_amount >= 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45908550"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# clean data rows count\n",
    "clean_sdf.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we plot the distribution of the preprocessed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAn/ElEQVR4nO3deZzddX3v8dd79swkmSwTSEgiCYhoFFmMGEUruAJaqY8ugiLKbaVU0LrdFq22eB/2Xm5rbcvVQhEp4gKKuETEIoq4IhC2AAlISAIZErJAkskyM5nlc//4/U5yOPlN5szk/ObMnLyfj8c8Mue3fs9vJuc93+X3/SkiMDMzK1VX7QKYmdn45IAwM7NMDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IGxVJV0r6TIWO9QJJOyXVp6/vkPQXlTh2erwfS3pfpY43gvN+TtIWSc8c5HEqdq2rpRbew6FIvg/CSklaCxwO9AMDwArgOuCqiBgcxbH+IiJ+OoJ97gC+HhFXj+Rc6b6XAi+MiHNHum8lSZoP/B44MiI2lax7D/Cf6ct6oBnYXVgfEZMrWI47gCVAHxDA48CNwL9GRG+lzlNyzveT/Mxfm8fxbey4BmFD+cOImAIcCVwG/C3wlUqfRFJDpY85ThwJPFsaDgAR8Y2ImJwGwRnA+sLr0nAo1KoO0sXpz3IO8HHgbOAWSRrpgWr452UZHBB2QBGxPSKWAu8C3ifpZQCSrpX0ufT7Dkk3S9om6TlJv5JUJ+lrwAuAH6ZNSH8jaYGkkPTnkp4Cbi9aVvzhc7SkuyVtl/QDSTPSc50qqbO4jJLWSnqTpNOBTwHvSs/3YLp+b5NVWq5PS3pS0iZJ10lqT9cVyvE+SU+lzUN/N9S1kdSe7r85Pd6n0+O/CbgNOCItx7XlXu/0ul4h6RZJu4DTSq71qZI6JX0qLd/atEYyrIjYFRF3AO8AXg28reicnysqw/OucXqOv5W0HNglqUHSJZKekLRD0gpJ70y3fQlwJfDq9L1vG+IcH5C0Kv19WSrpiKJ1IelCSY9L2irpS6MJMzt4DggrS0TcDXQCr8tY/fF03SySpqlPJbvEe4GnSGojkyPin4r2eT3wEuCtQ5zyPOB/AEeQNHVdXkYZ/xv438C30vMdn7HZ+9Ov04CjgMnAF0u2eS1wLPBG4O/TD70s/w9oT4/z+rTM56fNacU1g/cPV/YS7wb+EZgC/Dpj/WygA5gLvA+4StKx5R48Ip4ClpH9sxzKOSSBMi0i+oEn0v3bgc8CX5c0JyJWAhcCd6bvfVrpgSS9Afg/wJ+R1GqeBG4o2eztwCuB49Pthvo9sRw5IGwk1gMzMpb3kfxHPzIi+iLiVzF859al6V+03UOs/1pEPBwRu4DPAH9WoeaW9wBfiIjVEbET+CRwdknt5bMR0R0RDwIPknxIPU9alncBn4yIHRGxFvgX4L0VKOMPIuI3ETEYET1DbPOZiOiNiF8APyL5EB2JoX6WQ7k8ItYVfl4RcWNErE/L+C2Svo2TyzzWe4BrIuK+tB/kkyQ1jgVF21wWEdvSMPs5cMIIymoV4oCwkZgLPJex/J+BVcBPJK2WdEkZx1o3gvVPAo0kfzUfrCPS4xUfu4Gk5lNQPOpoN0kto1QH0JRxrLkVKONw12ZrGpzF5z1iqI2HMNTPsqwySTpP0gNps+I24GWU//N53s8gDepnef61K+dnYDlzQFhZJL2S5D/wfk0e6V/QH4+Io4A/BD4m6Y2F1UMccrgaxvyi719AUkvZAuwCWovKVU/StFXucdeTdCAXH7sf2DjMfqW2pGUqPdbTIzxOluHew3RJbSXnXV/uwZWMsHoF8Kt00fOuKUkT1pBlknQk8GXgYmBm2oz0MKDSbYfwvJ9B+l5mUplrZxXkgLADkjRV0ttJ2oi/HhEPZWzzdkkvTDsSu0iGxg6kqzeStNGP1LmSFklqBf4X8J2IGCAZOtoi6W2SGoFPkwwTLdgILJA01O/29cBHJS2UNJl9fRb9IylcWpZvA/8oaUr6ofkx4OsjOc5B+KykJkmvI2mvv3G4HSS1Sno98APgbuCWdNUDwJmSZkiaDXxkmEO1kYTA5vS455PUIAo2AvMkNQ2x/zeB8yWdIKmZ5GdwV9pMZ+OIA8KG8kNJO0iaFv4O+AJw/hDbHgP8FNgJ3An8RzpaBpLOyE+nTRGfGMH5vwZcS9LU0AJ8GJJRVcAHgatJ/uLcRdJBXlD4oHxW0n0Zx70mPfYvgTVAD/ChEZSr2IfS868mqVl9Mz1+3p4BtpL8Jf4N4MKIePQA238x/VluBP4NuAk4veielq+R9LWsBX4CfOtAJ4+IFST9LXemxzwO+E3RJrcDjwDPSNqSsf/PSPqVbgI2AEeTDL21ccY3yplNIJJOJanJzatyUewQ4BqEmZllckCYmVmmXANC0umSHkvvmNxv6KOkF0u6U1Jvafv0cPuaHYoi4g43L9lYya0PIh1++HvgzSSdiPcA56QdXIVtDiMZ7vZHJGO7P1/uvmZmlq88J946GVgVEasBJN0AnEUyMygA6URmmyS9baT7Zuno6IgFCxZU7A2YmdW6e++9d0tEzMpal2dAzOX5d192Aq+q9L6SLgAuAHjBC17AsmXLRl5SM7NDlKQnh1qXZx9E1uyL5bZnlb1vRFwVEYsjYvGsWZkhaGZmo5BnQHTy/OkS5lH+dAAHs6+ZmVVAngFxD3BMOqVBE8mdkkvHYF8zM6uA3PogIqJf0sXArSSPVbwmIh6RdGG6/sp03pdlwFRgUNJHgEUR0ZW1b15lNTOz/dXUVBuLFy8Od1KbmZVP0r0RsThrne+kNjOzTA4IMzPL5IAwM7NMDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8vkgDAzs0wOCDMzy+SAMDOzTA4IMzPL5IAwM7NMDggzM8uUa0BIOl3SY5JWSbokY70kXZ6uXy7ppKJ1H5X0iKSHJV0vqSXPspqZ2fPlFhCS6oEvAWcAi4BzJC0q2ewM4Jj06wLginTfucCHgcUR8TKgHjg7r7Kamdn+8qxBnAysiojVEbEHuAE4q2Sbs4DrIvE7YJqkOem6BmCSpAagFVifY1nNzKxEngExF1hX9LozXTbsNhHxNPB54ClgA7A9In6SY1nNzKxEngGhjGVRzjaSppPULhYCRwBtks7NPIl0gaRlkpZt3rz5oApsZmb75BkQncD8otfz2L+ZaKht3gSsiYjNEdEHfBd4TdZJIuKqiFgcEYtnzZpVscKbmR3q8gyIe4BjJC2U1ETSyby0ZJulwHnpaKYlJE1JG0ialpZIapUk4I3AyhzLamZmJRryOnBE9Eu6GLiVZBTSNRHxiKQL0/VXArcAZwKrgN3A+em6uyR9B7gP6AfuB67Kq6xmZrY/RZR2C0xcixcvjmXLllW7GGZmE4akeyNicdY630ltZmaZHBBmZpbJAWFmZpkcEGZmlskBYWZmmRwQZmaWyQFhZmaZHBBmZpbJAWFmZpkcEGZmlskBYWZmmRwQZmaWyQFhZmaZHBBmZpbJAWFmZpkcEGZmlskBYWZmmRwQZmaWyQFhZmaZHBBmZpbJAWFmZpkcEGZmlskBYWZmmRwQZmaWyQFhZmaZHBBmZpbJAWFmZpkcEGW67MeP8lDn9moXw8xszDggyhARXH/3U/z5V++he89AtYtjZjYmHBBlWN65ne3dfWza0cu1v11b7eKYmY0JB0QZbn3kGeoEC2a2ccUdq9i+u6/aRTIzy50Dogw/WbGRBR1tnP7Sw+nq6ecXj2+udpHMzHLngBjGE5t3smrTThbNmcrc6a1Maqznvie3VrtYZma5c0AM45H1XQAc1TGZ+jpx/Px27nvKAWFmtc8BMYytu/YA0NZcD0BzQz0r1nd5NJOZ1bxcA0LS6ZIek7RK0iUZ6yXp8nT9ckknFa2bJuk7kh6VtFLSq/Ms61C27k4CorWpAYAjZ7TSPxgs79xWjeKYmY2Z3AJCUj3wJeAMYBFwjqRFJZudARyTfl0AXFG07t+B/46IFwPHAyvzKuuBbNvdR0tjHfV1AmD+jFYA7nUzk5nVuDxrECcDqyJidUTsAW4AzirZ5izgukj8DpgmaY6kqcAfAF8BiIg9EbEtx7IO6blde/bWHgDamhvomNzEfU9WpThmZmMmz4CYC6wret2ZLitnm6OAzcB/Sbpf0tWS2rJOIukCScskLdu8ufLDT7fu3kNrU/3zCz1tEis3dFX8XGZm40meAaGMZVHmNg3AScAVEXEisAvYrw8DICKuiojFEbF41qxZB1PeTNt29+0XEIdNbeHpbd3s7O2v+PnMzMaLPAOiE5hf9HoesL7MbTqBzoi4K13+HZLAGHPP7dpDW1ETE8DhU5oBWLVpZzWKZGY2JvIMiHuAYyQtlNQEnA0sLdlmKXBeOpppCbA9IjZExDPAOknHptu9EViRY1mHtHln7/41iCktADy+cUc1imRmNiYaht9kdCKiX9LFwK1APXBNRDwi6cJ0/ZXALcCZwCpgN3B+0SE+BHwjDZfVJevGRG//AHv6B2ltfv5lmt7WREOdeNw1CDOrYWUFhKSbgGuAH0fEYLkHj4hbSEKgeNmVRd8HcNEQ+z4ALC73XHnYlk7KV1qDqK8THZOb+b1rEGZWw8ptYroCeDfwuKTLJL04xzKNG6U3yRU7bGozj290DcLMaldZARERP42I95B0FK8FbpP0W0nnS2rMs4DVtHVXdg0Ckn6Ip7d1s8sjmcysRpXdSS1pJvB+4C+A+0nudD4JuC2Xko0D+2oQ+wfE4VM9ksnMaltZASHpu8CvgFbgDyPiHRHxrYj4EDA5zwJWUyEgSoe5AsxKh7q6o9rMalW5o5iuTjuc95LUHBG9EVHVjuQ8FWZyzapBzGhrok6wdsuusS6WmdmYKLeJ6XMZy+6sZEHGo627+2iqr6Ohfv/L1FBXx/TWJtY4IMysRh2wBiFpNsncSJMknci+qTGmkjQ31bStu/fQ2rx/7aGgY3Izqx0QZlajhmtieitJx/Q84AtFy3cAn8qpTOPG1l37T9RXrGNyE/c+tZXBwaCuLmtaKTOzieuAARERXwW+KumPI+KmMSrTuLF1d19mB3VBx5RmevoGeaarhyOmTRrDkpmZ5W+4JqZzI+LrwAJJHytdHxFfyNitZnR199HWfICAmJyMZFqzZZcDwsxqznCd1IVnMEwGpmR81bTuvgGaGoa+RIWAcD+EmdWi4ZqY/jP997NjU5zxpbtvgMaMEUwFU1oamNRYz+rNvhfCzGpPuTfK/ZOkqZIaJf1M0hZJ5+ZduGrbvWeApvqhO5/rJBZ2tHmoq5nVpHLvg3hLRHQBbyd5mM+LgP+ZW6nGgYHBYE//II0HaGKCZGZXB4SZ1aJyA6IwId+ZwPUR8VxO5Rk3evoGAGg6QBMTJENd1z23mz39Zc+CbmY2IZQbED+U9CjJ8xl+JmkW0JNfsaqvOw2IA/VBQNJRPRjw1HO7x6JYZmZjptzpvi8BXg0sjog+YBdwVp4Fq7buPeUHBOCOajOrOSN55OhLSO6HKN7nugqXZ9zYV4M48B3SxfdCmJnVknIfOfo14GjgAWAgXRzUckCkNYgD3QcBMKmpnramegeEmdWccmsQi4FF6TOkDwnl9kGAJ+0zs9pUbif1w8DsPAsy3nSXOYoJkjmZVm92QJhZbSm3BtEBrJB0N9BbWBgR78ilVONAT5md1JDUIO59citdPX1MbanZR3Sb2SGm3IC4NM9CjEe795TXSQ3JvRCQPF3u5fOm5VksM7MxU+4w118Aa4HG9Pt7gPtyLFfV7e2DGKaTGjySycxqU7lzMX0A+A7wn+miucD3cyrTuFDundQAM9uakHA/hJnVlHI7qS8CTgG6ACLiceCwvAo1HpR7oxxAQ30d86ZP8kgmM6sp5QZEb0TsKbxIb5ar6SGv3X0D1EvUl/ko0YUdk1mzxXdTm1ntKDcgfiHpU8AkSW8GbgR+mF+xqm/3ngEaG8p/znRf/yBrNu/iELpVxMxqXLkBcQmwGXgI+EvgFuDTeRVqPOgZ5mFBpTqmNLNrzwCbd/QOv7GZ2QRQ1jDXiBiU9H3g+xGxOd8ijQ/dfQNldVAXFIa6PrF5F4dNbcmrWGZmY+aAn4BKXCppC/Ao8JikzZL+fmyKVz3de0ZYg/BQVzOrMcN9An6EZPTSKyNiZkTMAF4FnCLpo3kXrpqS51GX3wfRPqmRpoY6d1SbWc0YLiDOA86JiDWFBRGxGjg3XVezuvcMlHWTXEGdxPTWRtcgzKxmDPcJ2BgRW0oXpv0QNT3p0Ej7ICCd1dU3y5lZjRjuE3DPKNcBIOl0SY9JWiXpkoz1knR5un65pJNK1tdLul/SzcOdq9K6RziKCZKAeOq53fQN+PnUZjbxDfcJeLykroyvHcBxB9pRUj3wJeAMYBFwjqRFJZudARyTfl0AXFGy/q+BlWW+l4rq2TO6GkT/YNC5tTunUpmZjZ0DfgJGRH1ETM34mhIRwzUxnQysiojV6V3YN7D/c6zPAq6LxO+AaZLmAEiaB7wNuHpU7+wgdfeN7EY52DfU1R3VZlYLRvYn8sjMBdYVve5Ml5W7zb8BfwMcsL1G0gWSlklatnlz5W7R2D3CYa4As9Khru6HMLNakGdAZP35XToPReY2kt4ObIqIe4c7SURcFRGLI2LxrFmzRlPO/QwOBr39gyMOiNbmBqa1NnrSPjOrCXkGRCcwv+j1PGB9mducArxD0lqSpqk3SPp6fkV9vp7+8qf6LjWluYE1rkGYWQ3IMyDuAY6RtFBSE3A2sLRkm6XAeelopiXA9ojYEBGfjIh5EbEg3e/2iDg3x7I+z96pvkdwH0RBx+Rm3wthZjWh3EeOjlhE9Eu6GLgVqAeuiYhHJF2Yrr+SZNK/M4FVwG7g/LzKMxLdex8WNLJOaoBZU5q5f902dvX209ac2+U1M8tdrp9gEXELSQgUL7uy6PsgeRjRgY5xB3BHDsUb0kgeFlRqZtGcTC+b217RcpmZjaU8m5gmrL3Pox5FQOwb6upmJjOb2BwQGQ6mBtExuZk6weMbd1S6WGZmY8oBkWFvH8QoOqkb6+uY2dbMymccEGY2sTkgMvTsbWIaeSc1wOz2FlZu6KpkkczMxpwDIsPug2hiApjT3kLn1m66evoqWSwzszHlgMjQ25/M7jHagJjdnjxy9DE3M5nZBOaAyLC3ialudE1Mc9onAbiZycwmNAdEhkINomGUNYipLQ1Maqx3QJjZhOaAyNDbVwiI0dUgJDG7vYUVG9zEZGYTlwMiQ2//APUSdRpdQEDSUf37Z3YwMFg6ga2Z2cTggMjQ0zc46tpDwZz2Frr7BnjyWd9RbWYTkwMiQ2//AA2j7KAumL23o9rNTGY2MTkgMozmYUGlDpvSTH2d3FFtZhOWAyJDT9/AQTcxJVNuNDkgzGzCckBk6O0fpKHu4C/N7PYWHvXNcmY2QTkgMvT2H3wnNSQ3zD29rZvtuz3lhplNPA6IDD19AwfdBwHJSCaAlc+4mcnMJh4HRIakiengaxCzp6YBUdIP8c27njroY5uZ5c0BkaG3b2DU02wUm9LSwKwpzTzUub0CpTIzG1sOiAx7+gdH/SyIYpLoaGvigc5tB18oM7Mx5oDI0NM3UJFRTADzZrSyevMutne7o9rMJhYHRIZKjWICmD+9FYDlrkWY2QTjgMjQ2z846mdBlJo7LZly48F12ypyPDOzseKAyNBToU5qgElN9XRMbuaBde6oNrOJxQFRon9gkP7BqFgTE8D86ZN4YN02Ijz1t5lNHA6IEnsG0udRV6iTGmD+jFa27Oxl3XPdFTummVneHBAlDvZpclkWzGwD4O61z1XsmGZmeXNAlOjpHwAqW4M4bGoz7ZMauWeNA8LMJg4HRIk8ahB1EnPaW7jnSQeEmU0cDogSvf2FgKjspTlyZhurN+9iy87eih7XzCwvDogSPX2FJqbK1SAAFsxMbphb5n4IM5sgHBAl8qpBzJ0+iYY6cZf7IcxsgnBAlOhNO6krMd13sYa6Oo6c2cpvVz1b0eOameXFAVGi0EldiQcGlTp61mQe27iDHT2euM/Mxr9cA0LS6ZIek7RK0iUZ6yXp8nT9ckknpcvnS/q5pJWSHpH013mWs1hhmGslRzEVHD1rMgCrt+yq+LHNzCott4CQVA98CTgDWAScI2lRyWZnAMekXxcAV6TL+4GPR8RLgCXARRn75mLvMNcKNzFB0g/R0ljHE5t2VvzYZmaVlmcN4mRgVUSsjog9wA3AWSXbnAVcF4nfAdMkzYmIDRFxH0BE7ABWAnNzLOteeXVSQ3I/xMKOyTyx2QFhZuNfngExF1hX9LqT/T/kh91G0gLgROCurJNIukDSMknLNm/efLBl3jfMNYcmJoAXzmpj6+4+1rqZyczGuTwDIusTtnQ60wNuI2kycBPwkYjoyjpJRFwVEYsjYvGsWbNGXdiCvTWICk61UezY2VMBuP3RTbkc38ysUvIMiE5gftHrecD6creR1EgSDt+IiO/mWM7n6c2xkxpgRlsTsyY3OyDMbNzLMyDuAY6RtFBSE3A2sLRkm6XAeelopiXA9ojYIEnAV4CVEfGFHMu4n97+QerrRJ3yCQiAF8+ewl1rnmVnb39u5zAzO1i5BURE9AMXA7eSdDJ/OyIekXShpAvTzW4BVgOrgC8DH0yXnwK8F3iDpAfSrzPzKmuxnr6BXEYwFTt29hT6BoJfP37wfSZmZnlpyPPgEXELSQgUL7uy6PsALsrY79dk90/krrd/MJcRTMWOnNlGS2Mdt63YxOkvm5PruczMRst3Upfo7Rus+ER9perrxEtmT+W2Fc+wJ+0UNzMbbxwQJXr6B3KvQQAcN7edrp5+frNqS+7nMjMbDQdEid6+wdzugSj2wsMm09JYx83LN+R+LjOz0XBAlOjtz7+TGpI7tRfNmcpPVjyzd2itmdl44oAoMRad1AUvnzeNHT39/Nz3RJjZOOSAKNHbNzAmTUyQNDNNbWngxmWdY3I+M7ORcECU6O0fzG2ajVJ1EifMn84dv9/Mph09Y3JOM7NyOSBKJE1MY3cLxklHTmNgMPj+/U+P2TnNzMrhgCjR0zdA4xjVIAAOm9LCC2a0cv3d6xgcLJ3L0MysehwQJXb19tPYMLaXZclRM1izZRe/9j0RZjaOOCCKDA4GO3r7mdRYP6bnfdkR7bQ11XPdnU+O6XnNzA7EAVFk555+ImBS49helob6Ol65cAY/e3Qj657bPabnNjMbigOiSFd3HwAtY1yDAFiycCZ1Elf9cvWYn9vMLIsDokhXd/J8hmoExNRJjZw4fxrfWrbOQ17NbFxwQBTp6klqEJOaxj4gAP7gRbPo6x/kK79eU5Xzm5kVc0AUqWYTE0DH5GZePq+dr/52Leu3dVelDGZmBQ6IIl09SRPTWI9iKvaWl85mMOCfb32samUwMwMHxPPsq0FU77JMb23i1UfN5Hv3P80D67ZVrRxmZg6IItur3MRUcOqLZjG5uYHP3byC5KmsZmZjzwFRpKunjynNDdSpKo/D3qu5sZ43LzqcZU9u5UcP+YFCZlYdDogiXd391I/Bw4LK8YojpzOnvYXP3bxyb83GzGwsOSCKdPX0Vb15qaBO4p0nzmXTjh4+u/SRahfHzA5BDogiXd3jJyAA5k1v5dRjD+O79z/Nd+/zQ4XMbGw5IIp09fSP+TxMwznt2MNY2NHGJTc9xD1rn6t2cczsEDK+Pg2rbLzVIADq68R7XvUCprQ0cMF1y3jy2V3VLpKZHSIcEEW6evqqNs3GgbQ2NfC+1yygp2+Q86+9h22791S7SGZ2CHBApAYHg529/eOuBlHQMbmZc5ccyZPP7uacL9/Flp291S6SmdU4B0RqR2/hWRDjMyAAFna0cd6SI1m1aQd/duWdrN68s9pFMrMa5oBIVXuivnIdc/gUzn/NQp7p6uGPvvQbbluxsdpFMrMa5YBI7Z3qe5yNYsqyoKOND576QtqaG/jAdcv4xI0P7i3/N+96qsqlM7Na0VDtAowX42UepnLNaGvir049mtsf3cRN93by21Vb+Lu3LWLQczeZWYU4IFLVfJrcaDXU1fGWRbN5yeyp3HRfJxd98z5mT21hZlsTb33pbOrGybQhZjYxOSBS+5qYJk5AFMyf0cqH33gMyzu3cfujm/irb9zH3GmTeMcJR3DWCUfw4tlTq11EM5uAHBCpJzbvpKFOtDVPzEtSJ3HC/OkcN3caD6/fzgNPbeOqX67mijue4NjDpzB/RisfPO1ojpvbTmP9+O9nMbPqm5ifhhUWEfxo+QaOmtVGU8PE/vCsrxPHz5vG8fOmsbO3n4ef3s6Dndv46cqN/HTlRlqb6nnFkdNZNGcqL54zhWMPn8rRh7XR3DDxak5mlq9cA0LS6cC/A/XA1RFxWcl6pevPBHYD74+I+8rZt5IeWLeNzq3d/MlJ8/I6RVVMbm5gyVEzWXLUTHb29rNmyy7WbNnJqk07+e0TzzIwmHRoN9SJedMnMad9EnPaW5jd3pL+u+/19NamcTMVupmNjdwCQlI98CXgzUAncI+kpRGxomizM4Bj0q9XAVcArypz34r54YMbaKqvY9ERtdtWP7m5gePmtnPc3HYABgaDLTt72djVwzPbe3h21x7Wb+tm5TNddHX3MZgxGGpycwPtkxqZ0tLA1EmNtE9qpK2pnklN9bQ01tPaVM+kxuT75oY6GuvraKivo7FeNNYXXoum+joa6kRjQx2NdXU0NoiGOlGnoq+6pNmsvk5I6fcl6+qUrCsofC+037JkeWGZMpYVH8dBaAb51iBOBlZFxGoASTcAZwHFH/JnAddF8lzN30maJmkOsKCMfSticDD40UPref2xsybUCKaDVV8nDp/awuFTW3h5ScVpMIJdvf1s7+6jq7uP7d197N4zQE/fAN19A3T3DbKpq5ennt3NnoFB+voHk38HBjODpRbsC5/iZcpYVthu/2Qabjvnko1Wx+Rmfvk3p1X8uHkGxFxgXdHrTpJawnDbzC1zXwAkXQBckL7cKemx0RT2buBq6AC2jGb/GuPr4GtQ4OuQGPfXQX876l2PHGpFngGR9fdQ6d+XQ21Tzr7JwoirgKtGVrRskpZFxOJKHGsi83XwNSjwdUgcqtchz4DoBOYXvZ4HrC9zm6Yy9jUzsxzlOabzHuAYSQslNQFnA0tLtlkKnKfEEmB7RGwoc18zM8tRbjWIiOiXdDFwK8lQ1Wsi4hFJF6brrwRuIRniuopkmOv5B9o3r7IWqUhTVQ3wdfA1KPB1SByS10Hhyd3MzCzDxL5t2MzMcuOAMDOzTA4Ikmk9JD0maZWkS6pdnrEkaa2khyQ9IGlZumyGpNskPZ7+O73a5aw0SddI2iTp4aJlQ75vSZ9Mfz8ek/TW6pS6soa4BpdKejr9fXhA0plF62ruGgBImi/p55JWSnpE0l+nyw+p34csh3xAFE3rcQawCDhH0qLqlmrMnRYRJxSN874E+FlEHAP8LH1da64FTi9Zlvm+09+Hs4GXpvv8R/p7M9Fdy/7XAOBf09+HEyLiFqjpawDQD3w8Il4CLAEuSt/vofb7sJ9DPiAomhIkIvYAhWk9DmVnAV9Nv/8q8EfVK0o+IuKXwHMli4d632cBN0REb0SsIRl1d/JYlDNPQ1yDodTkNQCIiA2FSUIjYgewkmQ2h0Pq9yGLA2Lo6T4OFQH8RNK96bQlAIen96OQ/ntY1Uo3toZ634fa78jFkpanTVCFZpVD4hpIWgCcCNyFfx8cEIxgWo8adUpEnETSxHaRpD+odoHGoUPpd+QK4GjgBGAD8C/p8pq/BpImAzcBH4mIrgNtmrGspq5FgQOivClBalZErE//3QR8j6SqvDGdVZf0303VK+GYGup9HzK/IxGxMSIGImIQ+DL7mk5q+hpIaiQJh29ExHfTxYf874MD4hCe1kNSm6Qphe+BtwAPk7z/96WbvQ/4QXVKOOaGet9LgbMlNUtaSPL8krurUL7cFT4QU+8k+X2AGr4G6YPLvgKsjIgvFK065H8fDvlHjlZxWo/x4HDge+lzDRqAb0bEf0u6B/i2pD8HngL+tIplzIWk64FTgQ5JncA/AJeR8b7TKWK+TfI8kn7googYqErBK2iIa3CqpBNImkzWAn8JtXsNUqcA7wUekvRAuuxTHGK/D1k81YaZmWVyE5OZmWVyQJiZWSYHhJmZZXJAmJlZJgeEmZllckBY1UmaWTR76DMls4k2pdu842Bm2pV0raQ/qVB5F0h6d9HrxZIur9CxJel2SVNHsM/BXpuPSGod5b4XSzp/tOe28c3DXG1ckXQpsDMiPl+0rCEi+g/yuNcCN0fEd8rcfshzSjoV+EREvP1gyjTEsd8GvCkiPlrpYx/gnGuBxRGxZRT7tgK/iYgTK14wqzrXIGxcSv/i/4KknwP/V9L7JX2xaN2Vkn4l6feS9vugTv8S/6KkFZJ+RNGEg0qegdGRfr9Y0h3p95dKukrST4Dr0prCryTdl369Jj3EZcDr0hrORyWdKunm9BgzJH0/nezud5JeXnTsayTdIWm1pA8P8dbfQ3rHbnr+RyVdLelhSd+Q9CZJv1HyjIKT0+1Kr83lkn6bnudP0uV7y5i+/mK634eBI4Cfp9caSW+RdGf6nm9UMkcRki5Lr+dySZ8HiIjdwNpCWay2OCBsPHsRyV/TH89YtwB4PfA24EpJLSXr3wkcCxwHfAB4DeV5BXBWRLybZO6dN6eTGb4LKDQjXQL8Kn1ewr+W7P9Z4P6IeDnJ3bjXFa17MfBWkvmN/kHJ/D+lTgHuLXr9QuDfgZen+78beC3wifT4Weak27ydJMyGFBGXk8wjdFpEnJYG56dJrvtJwDLgY5JmkFzTl6bv7XNFh1kGvO5A57GJ6ZCfasPGtRsPMIXBt9MJ5R6XtJrkw/OBovV/AFyf7r9e0u1lnnNpRHSn3zcCX0ynnhggCazhvBb4Y4CIuD3tX2lP1/0oInqBXkmbSKY66SzZf0b6TIKCNRHxEICkR0geYBOSHiIJySzfT6/NCkmHl1HmYktIHpz1GyVTsDQBdwJdQA9wdVoju7lon00k199qjAPCxrNdB1hX2nmW1Zk2VAdbP/tqz6U1j+JzfhTYCByfbt9zgPIUHGgq6N6iZQNk///rl1SXfsCX7jNY9HpwiP1L9ymUp/g9w/7vu3j72yLinP1WJM1IbySZ0PJi4A1Fx+ou3d4mPjcx2UT1p5LqJB0NHAU8VrL+lyQzbtYrmaH0tKJ1a0makiD9a38I7cCG9MP6vSSTOQLsAKYMsc8vSfoRCp3ZW4Z5tkCpx0jeT6U9CSxSMgNpO8kHfUHx+/kdcIqkF0LSCS3pRWk/RHv6CNKPkDwvouBF7Jv11WqIaxA2UT0G/IKkmebCiCj96/57JH/hPgT8Pt224LPAVyR9iuTJYUP5D+AmSX8K/Jx9tYvlJH/pP0jyXOf7i/a5FPgvScuB3eybLrpcPyKZYXXVCPc7oIhYp2QG0uXA4zy/zFcBP5a0Ie2HeD9wvaTmdP2nSULkB2lfj0hqVwWnkFxTqzEe5moTzkiHrE4kaW3nuoh4c7XLUg5JJwIfi4j3VrssVnluYjIbR9JnH39ZI7hRrso6gM9UuxCWD9cgzMwsk2sQZmaWyQFhZmaZHBBmZpbJAWFmZpkcEGZmlun/A5u8XOqXjI+1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Trip duration distribution\n",
    "\n",
    "trip_duration_clean_df = clean_sdf.select('trip_duration_min').toPandas()\n",
    "sns.distplot(trip_duration_clean_df['trip_duration_min'], bins = int(np.sqrt(len(trip_duration_clean_df))))\n",
    "plt.title('Distribution of Trip Duration ')\n",
    "plt.xlabel('Trip duration (minutes)')\n",
    "plt.ylabel('Density')\n",
    "plt.savefig('../plots/trip_duration_clean_distplot.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAmMElEQVR4nO3deZxcZZ3v8c+3lyyQBIQ0WxIISBSjVwEzwIijuAMq8V69IyiizBIZRWXE8YI6Ds7m6Di81LmMGUa5ioCM4hY1I+KCyyAxYTEQQzRCTGIC6RghIUt3V9fv/nGe6hyqT3dXL6c7dH3fr1e9us7+nArUt57nOec5igjMzMzqtUx0AczM7MDkgDAzs0IOCDMzK+SAMDOzQg4IMzMr5IAwM7NCDggbNUlLJf31GO3rWEmPS2pN07dL+rOx2Hfa339JevNY7W8Yx/17SdslPTzexzYbKQeEDUrSBkl7Je2S9KikOyRdIqnvv52IuCQi/q7Bfb10sHUiYmNEzIiI3jEo+1WSbqjb/zkR8bnR7nuY5ZgHXA4sjIijCpafJamagrH2+sY4le3gdLzl43G8kZD0WUl/P9HlaEZtE10Ae1J4dUR8V9IhwAuBTwCnAxeP5UEktUVEZSz3eYA4DvhdRGwbZJ0tETF3pAcYxWf3OqALeLmkoyNi60jLYJOPaxDWsIh4LCKWAa8H3izpWfDEX3iSZkv6Zqpt7JD0Y0ktkj4PHAt8I/1ifa+k+ZJC0p9K2gh8Pzcv/+PlqZJ+JukxSV+XdFg61lmSNufLWKulSDobeB/w+nS8n6flfU1WqVwfkPQbSdskXZ9CkFw53ixpY2oeev9An42kQ9L2nWl/H0j7fylwG3BMKsdnG/28Jb1S0j2SdkraJOmq3LJ+n12a/yeS1kr6vaRbJR03xGHeDCwFVgNvLPgs/0rSakm7JX1G0pGpmW6XpO9Kekpu/fMkrUn/9rdLekZuWUg6MTed/2/mLEmbJV2e/h22Sro4LVuSyvXe8axZWcYBYcMWET8DNgN/VLD48rSsAziS7Es6IuJNwEay2siMiPhobpsXAs8AXjHAIS8C/gQ4BqgAn2ygjN8G/hH4z3S85xSs9pb0ehFwAjAD+L916zwfeDrwEuCD+S+9Ov8KHJL288JU5osj4rvAOWQ1hBkR8Zahyp6zO+3nUOCVwF9Iek3dOn2fXVr2PuB/kX3+Pwa+MNDOJR0LnAXcmF4XFaz2WuBlwNOAVwP/lY4xm+z7451pX09Lx7osHXs52Y+BKQ2e61Fkn98c4E+BayQ9JSKuTWX7aPr8Xt3g/mwMOCBspLYAhxXM7wGOBo6LiJ6I+HEMPeDXVRGxOyL2DrD88xFxf0TsBv4a+GOlTuxReiNwdUQ8GBGPA1cC59fVXj4UEXsj4ufAz4F+QZPK8nrgyojYFREbgH8B3jSMshyTfnnXXn8cEbdHxH0RUY2I1WRfwC+s2y7/2b0V+HBErE3NTf8InDxILeIiYHVE/CLt+5mSTqlb518j4pGI+C1Z4KyIiHsiogv4KlBb//XAtyLitojoAT4GTAee1+D59wB/m/6bWQ48ThbMNoEcEDZSc4AdBfP/GVgPfEfSg5KuaGBfm4ax/DdAO9kv2NE6Ju0vv+82sppPTf6qoz1ktYx6s4EpBfuaM4yybImIQ3OvL0o6XdIPUrPVY8Al9D/v/GdzHPCJWsiQ/ftokHJcRPbrnIjYAvyQrMkp75Hc+70F07XP4wmfZURUU9ka/Qx+V9eHMtBnbePIAWHDJukPyP7H/0n9svQL+vKIOIGsSeLdkl5SWzzALoeqYczLvT+W7NfmdrImmINy5Wola95odL9byL5U8/uu8MQvwUZsT2Wq39dvh7mfejcBy4B5EXEIWV+B6tbJn+Mm4K11QTM9Iu6o37Gk5wELgCslPazs8tvTgQvqalCNesJnKUlk/261z2APuX8rsialRnnI6QnigLCGSZol6VXAzcANEXFfwTqvknRi+oLYCfSmF2RfvCeM4NAXSloo6SDgb4Fb0mWwvwSmpc7cduADwNTcdo8A85W7JLfOF4C/lHS8pBns77MY1tVAqSxfBP5B0szUpPNu4IbBtxzSTGBHROyTdBrwhiHWX0r2hf9M6Os4/98DrPtmss7zhcDJ6fUssi/xc0ZQ1i8Cr5T0kvRvcTnZ1VG1cLoXeIOk1nQBQX1T2WBG+t+NjZIDwhrxDUm7yH6hvh+4moEvcV0AfJesDfmnwL9FxO1p2YeBD6QmkPcM4/ifBz5L1twzjdQxGhGPAW8DPk32S3U3WQd5zZfS399Jurtgv9elff8IeAjYB7xjGOXKe0c6/oNkNaub0v5H423A36bP/oNkX8IDioivAh8Bbpa0E7ifgi97SdOAPybrX3g493qI7PMY9o2EEbEOuJCss347We3x1RHRnVZ5V5r3KFnfz9eGsfvPAAvTfzfD2c5GSX5gkJmZFXENwszMCjkgzMyskAPCzMwKOSDMzKzQpBqsb/bs2TF//vyJLoaZ2ZPGXXfdtT0iOoqWTaqAmD9/PqtWrZroYpiZPWlI+s1Ay9zEZGZmhRwQZmZWyAFhZmaFHBBmZlbIAWFmZoUcEGZmVsgBYWZmhUoNCElnS1onaX3Rk8UknSTpp5K6ioZ/TmPH3yPpm2WW08zM+istINLTva4hG49+IdmTqhbWrbaDbGz/jw2wm3cBa8sqo5mZDazMGsRpwPr0QPhusqeQLc6vEBHbImIl2eMan0DSXOCVZA+DGVc3rdjITSs2jvdhzcwOKGUGxBye+ED1zQzvIe4fB94LVAdbSdISSaskrers7Bx2Ic3MrFiZAVH/cHVo8OHj6bnH2yLirqHWjYhrI2JRRCzq6Cgcb8rMzEagzIDYDMzLTc8FtjS47ZnAeZI2kDVNvVjSaB8Ab2Zmw1BmQKwEFkg6XtIU4HxgWSMbRsSVETE3Iuan7b4fEReWV1QzM6tX2nDfEVGRdClwK9AKXBcRayRdkpYvlXQUsAqYBVQlXQYsjIidZZXLzMwaU+rzICJiObC8bt7S3PuHyZqeBtvH7cDtJRTPzMwG4TupzcyskAPCzMwKOSDMzKyQA8LMzAo5IMzMrJADwszMCjkgzMyskAPCzMwKOSDMzKyQA8LMzAo5IMzMrJADwszMCjkgzMyskAPCzMwKOSDMzKyQA8LMzAo5IMzMrJADwszMCjkgzMyskAPCzMwKlRoQks6WtE7SeklXFCw/SdJPJXVJek9u/jxJP5C0VtIaSe8qs5xmZtZfW1k7ltQKXAO8DNgMrJS0LCJ+kVttB/BO4DV1m1eAyyPibkkzgbsk3Va3rZmZlajMGsRpwPqIeDAiuoGbgcX5FSJiW0SsBHrq5m+NiLvT+13AWmBOiWU1M7M6ZQbEHGBTbnozI/iSlzQfOAVYMcDyJZJWSVrV2dk5knKamVmBMgNCBfNiWDuQZgBfBi6LiJ1F60TEtRGxKCIWdXR0jKCYZmZWpMyA2AzMy03PBbY0urGkdrJwuDEivjLGZTMzsyGUGRArgQWSjpc0BTgfWNbIhpIEfAZYGxFXl1jGQd20YuNEHdrMbMKVdhVTRFQkXQrcCrQC10XEGkmXpOVLJR0FrAJmAVVJlwELgWcDbwLuk3Rv2uX7ImJ5WeU1M7MnKi0gANIX+vK6eUtz7x8ma3qq9xOK+zDMzGyc+E5qMzMr5IAwM7NCDggzMyvkgDAzs0IOCDMzK+SAMDOzQg4IMzMr5IAwM7NCDggzMyvkgDAzs0IOCDMzK+SAMDOzQg4IMzMr5IAwM7NCDggzMyvkgDAzs0IOCDMzK+SAMDOzQg4IMzMr5IAwM7NCDggzMytUakBIOlvSOknrJV1RsPwkST+V1CXpPcPZ1szMylVaQEhqBa4BzgEWAhdIWli32g7gncDHRrBtafZ293LHr7cTEeN1SDOzA06ZNYjTgPUR8WBEdAM3A4vzK0TEtohYCfQMd9sy/WLrTr65eis7dneP1yHNzA44ZQbEHGBTbnpzmjem20paImmVpFWdnZ0jKmi9nt4qAN3pr5lZMyozIFQwr9E2m4a3jYhrI2JRRCzq6OhouHCDqVSzQ3VXHBBm1rzKDIjNwLzc9FxgyzhsO2qVVHPo6XUfhJk1rzIDYiWwQNLxkqYA5wPLxmHbUasFQ4+bmMysibWVteOIqEi6FLgVaAWui4g1ki5Jy5dKOgpYBcwCqpIuAxZGxM6ibcsqa71K1X0QZmalBQRARCwHltfNW5p7/zBZ81FD246XSq0G4T4IM2tivpO6gGsQZmYOiEJ9NQh3UptZE3NAFOipupPazMwBUaB2mavvgzCzZuaAKFBxDcLMzAFRZP+Ncg4IM2teDogCfUNtuJPazJqYA6KA74MwM3NAFPJormZmDohC7qQ2M3NAFHIntZmZA6KQnwdhZuaAKOShNszMHBD99FaD3nAfhJmZA6JOV6UXgFaJnt4qEa5FmFlzckDU6erJag3Tp7RSDV/qambNq6GAkPRlSa+UNOkDpauyPyAA9nU7IMysOTX6hf8p4A3AryT9k6STSizThKo1MR3UngXE3p7eiSyOmdmEaSggIuK7EfFG4FRgA3CbpDskXSypvcwCjrdaDeKgKQ4IM2tuDTcZSToceAvwZ8A9wCfIAuO2Uko2Qfb3QWSP697b7YAws+bUaB/EV4AfAwcBr46I8yLiPyPiHcCMQbY7W9I6SeslXVGwXJI+mZavlnRqbtlfSloj6X5JX5A0bfinN3x9TUx9NYjKeBzWzOyA02gN4tMRsTAiPhwRWwEkTQWIiEVFG0hqBa4BzgEWAhdIWli32jnAgvRaQtbXgaQ5wDuBRRHxLKAVOH84JzZS/ZqY3EltZk2q0YD4+4J5Px1im9OA9RHxYER0AzcDi+vWWQxcH5k7gUMlHZ2WtQHTJbWR1Vy2NFjWUanVIKa7D8LMmlzbYAslHQXMIfuiPgVQWjSL7Et7MHOATbnpzcDpDawzJyJWSfoYsBHYC3wnIr4zQBmXkNU+OPbYY4co0tBqfRAH1fogHBBm1qQGDQjgFWQd03OBq3PzdwHvG2JbFcyrvy25cB1JTyGrXRwPPAp8SdKFEXFDv5UjrgWuBVi0aNGob3vuuw+idplrt/sgzKw5DRoQEfE54HOSXhsRXx7mvjcD83LTc+nfTDTQOi8FHoqITujrJH8e0C8gxlq/TmpfxWRmTWqoJqbar/b5kt5dvzwiri7YrGYlsEDS8cBvyTqZ31C3zjLgUkk3kzU/PRYRWyVtBM6QdBBZE9NLgFWNntRo1N9JvbfHndRm1pyGamI6OP0d8FLWgURERdKlwK1kVyFdFxFrJF2Sli8FlgPnAuuBPcDFadkKSbcAdwMVsvsurh1uGUai7z4I30ltZk1uqCamf09/PzSSnUfEcrIQyM9bmnsfwNsH2PZvgL8ZyXFHY18KhPbWFtpb5T4IM2tajd4o91FJsyS1S/qepO2SLiy7cBOhq1JFQIuykHANwsyaVaP3Qbw8InYCryLrWH4a8FellWoCdVV6aWsVkpjS2uIb5cysaTUaELUB+c4FvhARO0oqz4TrqlRpa8k+lva2lr4mJzOzZjNUJ3XNNyQ9QHZF0dskdQD7yivWxOnqqdLemt2eMaW1hT3ugzCzJtXocN9XAH9INjZSD7Cb/sNmTApZE1OqQbTKfRBm1rQarUEAPIPsfoj8NtePcXkmXNbElNUgsk5q90GYWXNqKCAkfR54KnAvUPtJHUzSgGhPNYgpbS3s853UZtakGq1BLAIWpvsWJrWuSu8TahA793VNcInMzCZGo1cx3Q8cVWZBDhRdPVXaWnNNTL7M1cyaVKM1iNnALyT9DOj7SR0R55VSqgmUv8x1Sqt8mauZNa1GA+KqMgtxIOmq9Pb1QdTupI4IpKKRyc3MJq+GAiIifijpOGBBRHw3jbLaWm7RJkZXpdr3sKApbS30VoPu3ipT2ybl6ZqZDajRsZj+HLgF+Pc0aw7wtZLKNKG6eqq05jqpAfa5H8LMmlCjndRvB84EdgJExK+AI8oq1ESqVPsHhG+WM7Nm1GhAdEVEd20i3Sw3KS95rVSDFtUCIvvrgDCzZtRoQPxQ0vuA6ZJeBnwJ+EZ5xZo4vb1BygWmtGUfj8djMrNm1GhAXAF0AvcBbyV7CNAHyirURHpiDSL1QbgGYWZNqNGrmKqSvgZ8LSI6yy3SxOqtBi11fRDfWv0wzz3usIkslpnZuBu0BqHMVZK2Aw8A6yR1Svrg+BRv/FWqVVpqTUwpIHp6fRWTmTWfoZqYLiO7eukPIuLwiDgMOB04U9Jfll248VatBtVgfxNTW/a32wFhZk1oqIC4CLggIh6qzYiIB4EL07JBSTpb0jpJ6yVdUbBckj6Zlq+WdGpu2aGSbpH0gKS1kv6w8dMamd40FmGtiamvBlFxQJhZ8xkqINojYnv9zNQP0V6wfh9JrcA1wDnAQuACSQvrVjsHWJBeS4BP5ZZ9Avh2RJwEPAdYO0RZR623mgKirpPaNQgza0ZDBUT3CJcBnAasj4gH0z0UN9P/KXSLgesjcydwqKSjJc0CXgB8BiAiuiPi0SGON2qVvoDIptv7+iAm5S0fZmaDGuoqpudI2lkwX8C0IbadA2zKTW8m678Yap05QIXsstr/J+k5wF3AuyJi9xDHHJXe3voaRPbXndRm1owGrUFERGtEzCp4zYyIQZuYyEKk3y4bXKcNOBX4VEScQvYM7H59GACSlkhaJWlVZ+forsCtVLMgqPVBSGJ6eyvd7oMwsybU6I1yI7EZmJebngtsaXCdzcDmiFiR5t9CFhj9RMS1EbEoIhZ1dHSMqsC9dU1MANOntLoGYWZNqcyAWAkskHS8pCnA+cCyunWWARelq5nOAB6LiK0R8TCwSdLT03ovAX5RYlmB/X0QrblnP0xvd0CYWXNq9IFBwxYRFUmXAreSPTviuohYI+mStHwp2ZAd5wLrgT3AxbldvAO4MYXLg3XLSlF/FRPAtPYWut1JbWZNqLSAAIiI5WQhkJ+3NPc+yIYSL9r2XmBRmeWrV6tB5B8ed9CUNvdBmFlTKrOJ6Umnt66TGrImJt8HYWbNyAGRUylqYnIntZk1KQdETqW31km9f9709hYHhJk1JQdETlEntfsgzKxZOSBy+pqYWvJXMbV6qA0za0oOiJyiGoTvgzCzZuWAyCm+kzrrg4hwLcLMmosDImegPohqeMhvM2s+Doic+sH6IOuDANjX7YAws+bigMgpbGJKAbG3p3ciimRmNmEcEDlFN8pNn5J9RA4IM2s2Doic3oLLXFdt+D0Ae7orE1ImM7OJ4oDIqX/kKMDUtqyJ6fF9Dggzay4OiJzaYH2tdcN9Azze5YAws+bigMip9Pbvg6jVIHa5BmFmTcYBkVPUB1GrQeza1zMhZTIzmygOiJyiPojafRC73MRkZk3GAZFTdCd1W4tokZuYzKz5OCByiu6DkMS09lY3MZlZ03FA5PQ9clRPnD+tvdWXuZpZ03FA5BQ9DwJgaluLm5jMrOmUGhCSzpa0TtJ6SVcULJekT6blqyWdWre8VdI9kr5ZZjlregsuc4WsBuFOajNrNqUFhKRW4BrgHGAhcIGkhXWrnQMsSK8lwKfqlr8LWFtWGevVahB1+eAahJk1pTJrEKcB6yPiwYjoBm4GFtetsxi4PjJ3AodKOhpA0lzglcCnSyzjE/RWAzFADcKd1GbWZMoMiDnAptz05jSv0XU+DrwXGPRBDJKWSFolaVVnZ+eoClypRr/+B8hulvNQG2bWbMoMiP7ftFD/3M7CdSS9CtgWEXcNdZCIuDYiFkXEoo6OjpGUs09vtdrvCibIhtvYta/ix46aWVMpMyA2A/Ny03OBLQ2ucyZwnqQNZE1TL5Z0Q3lFzVSq0a95CbImpt5qsK/HT5Uzs+ZRZkCsBBZIOl7SFOB8YFndOsuAi9LVTGcAj0XE1oi4MiLmRsT8tN33I+LCEssKZH0QxQHh8ZjMrPm0lbXjiKhIuhS4FWgFrouINZIuScuXAsuBc4H1wB7g4rLK04iB+iBqI7ru3FfhiFnjXSozs4lRWkAARMRyshDIz1uaex/A24fYx+3A7SUUr5/e3qC1oA/Cz4Qws2bkO6lzBuqD2P9MCDcxmVnzcEDkVGPgy1zBI7qaWXNxQORkNYj+82vPhPCAfWbWTBwQOdl9EAU1iL5OajcxmVnzcEDkVHoH6INwJ7WZNSEHRE5vNWgp+ERaJA6e0uo+CDNrKg6InIGuYgKYMa3NVzGZWVNxQOQMdCc1wMxp7W5iMrOm4oDIqQzQSQ0wa1obj+5xDcLMmocDImegPgiAjplT6dzVNb4FMjObQA6InMH6II6YOY1tDggzayIOiJzeAW6UAzhy1lQe29vDvp7e8S2UmdkEcUDkDHQfBGQ1CMDNTGbWNBwQOYNdxdQxayoA23btG88imZlNGAdETqVaLRysD+DIVIPYttM1CDNrDg6InMH6IH6yfjsAj+x0DcLMmoMDIqdSDVoHaGI6aEorbS3ylUxm1jQcEDmD9UG0SHTMnOqAMLOm4YDIqQxyoxzAETOnuonJzJqGAyJnsBoEQMfMab7M1cyaRqkBIelsSeskrZd0RcFySfpkWr5a0qlp/jxJP5C0VtIaSe8qs5w1ld6Bx2KC7GY5NzGZWbMoLSAktQLXAOcAC4ELJC2sW+0cYEF6LQE+leZXgMsj4hnAGcDbC7Ydc4NdxQTZzXI7dnfTXamWXRQzswlXZg3iNGB9RDwYEd3AzcDiunUWA9dH5k7gUElHR8TWiLgbICJ2AWuBOSWWFaj1QQycEEekm+U6H3ctwswmvzIDYg6wKTe9mf5f8kOuI2k+cAqwouggkpZIWiVpVWdn56gKPFQfxJEpINxRbWbNoMyAKPqmjeGsI2kG8GXgsojYWXSQiLg2IhZFxKKOjo4RFxYGH80VYPWmxwDYsH33qI5jZvZkUGZAbAbm5abnAlsaXUdSO1k43BgRXymxnABUq1kuDXaZ6+EzptLeKn75yONlF8fMbMKVGRArgQWSjpc0BTgfWFa3zjLgonQ10xnAYxGxVZKAzwBrI+LqEsvYp5ICYqA7qQFaW8Txsw9m/bZd41EkM7MJ1VbWjiOiIulS4FagFbguItZIuiQtXwosB84F1gN7gIvT5mcCbwLuk3Rvmve+iFheVnl7azWIQQICYMGRM7lv82NlFcPM7IBRWkAApC/05XXzlubeB/D2gu1+QnH/RGkq1ezS1cEucwVYcMQMlt+3lb3dvUyf0joOJTMzmxi+kzrpq0EMkRBPO3ImEfDrTvdDmNnk5oBIan0QGqKJ6WlHzgDgV+6HMLNJzgGR7O+DGHy94w4/2FcymVlTcEAkjVzFBPClVZs5fvbB/OoR1yDMbHJzQCS9vY1dxQTwrGMO4Z6Nj/bdO2FmNhk5IJK+q5ga+ESed+Jsfre7m3WuRZjZJOaASBq9DwLgzBMPB+C/03OqR+uXj+xiy6N7x2RfZmZjxQGRVIYREEcfMp0TOg4es4B424138w/L147JvszMxooDIhlODeKmFRs586mzWfHQjjF5NsSWR/ey1TUIMzvAOCCSSgOD9eWdeeJs9nT3ctdvfj+q4z7eVWFPd6+fMWFmBxwHRNLbN9RGYyN8/NGC2cyY2sYtd23mphUbR3zc2jOuO3d1kY08YmZ2YHBAJJVhXOYK8PV7t/CaU47hm6u3sKe7MuLjbksPH9rXU+XxrpHvx8xsrDkgkt5hNjEBvOG04+iqVLln46MjPm6+aWnbLjczmdmBwwGRNHondd7CY2Zx6rGHcsevt4+4FrFt5/5Q6HRAmNkBxAGRDOcqppqbVmzkvWefxO/39PDPt64bUV9EvgbhgDCzA4kDIhnOfRB5Z5xwOGeccBifvWMDa7cWPjZ7UJ27ujgoPVfCAWFmB5JSHxj0ZNI7jKE28m5asZFXPPModnf1csOdv+FpR82kVeINpx/b0PbbdnXx1I4ZPPDwTl/qamYHFNcgkpHWIACmtrVy85IzePpRM/nrr93Pt+/fyg13/qahbTt3dXHkrKnMnjHVNQgzO6C4BpGMpA8i7+v3buGNpx/Hukd2csOdG3l45z52d1V46wufOuh2nbv2cfK8Q+mY2eWAMLMDimsQyf77IEa+j9YW8XeLn8V5zzmGBzt386/fX89Vy9YMcswqv9vdTcfMqRwxc6ovczWzA4oDImn0mdRD+cLPNnHGCYfzthedyMFTW/nsHRt49xfvZdOOPf3W3bG7mwg4YuZUOma6icnMDiylBoSksyWtk7Re0hUFyyXpk2n5akmnNrrtWOuN0TUx1Ttq1jTedtaJvGDBbL61eisv/Ocf8P6v3sfdG3/P3u5eYP+NcR0zp9IxYyo7dnf1BZWZ2UQrrQ9CUitwDfAyYDOwUtKyiPhFbrVzgAXpdTrwKeD0BrcdU5UGn0k9HO2tLZz9rKN53lNn84N12/jPlZu4ccVG2lvFGSccziHT24GsBrFt1jSqAT97aAdHzprKtPZWpra10NbaggQCJKW/IEQty/LT/dYbo8Azs+ZTZif1acD6iHgQQNLNwGIg/yW/GLg+slHq7pR0qKSjgfkNbDtmnvt3t/HY3h4g60cYa7Omt7P45Dm8+KQj2LRjDxt+t4cHHt7F9l1dtLWIlRt+z4IjZwBwwX/cOebHBwYMGfrmp+kDxIGWawdYccye4PAZU/nRe1805vstMyDmAJty05vJaglDrTOnwW0BkLQEWJImH5e0boTlnQ1s//OPjHDrUXjrh8f/mMlsYGyeevTk0Wzn3GznC016zvo/Iz7n4wZaUGZAFP3oqm9gH2idRrbNZkZcC1w7vKL1J2lVRCwa7X6eTHzOk1+znS/4nMdSmQGxGZiXm54LbGlwnSkNbGtmZiUq8yqmlcACScdLmgKcDyyrW2cZcFG6mukM4LGI2NrgtmZmVqLSahARUZF0KXAr0ApcFxFrJF2Sli8FlgPnAuuBPcDFg21bVlmTUTdTPQn5nCe/Zjtf8DmPGfkxl2ZmVsR3UpuZWSEHhJmZFXJAMP7DeowXSddJ2ibp/ty8wyTdJulX6e9TcsuuTJ/BOkmvmJhSj5ykeZJ+IGmtpDWS3pXmT8pzljRN0s8k/Tyd74fS/El5vnmSWiXdI+mbaXpSn7OkDZLuk3SvpFVpXvnnHBFN/SLrBP81cALZ5bU/BxZOdLnG6NxeAJwK3J+b91HgivT+CuAj6f3CdO5TgePTZ9I60ecwzPM9Gjg1vZ8J/DKd16Q8Z7L7hWak9+3ACuCMyXq+def+buAm4JtpelKfM7ABmF03r/Rzdg0iNyRIRHQDtWE9nvQi4kfAjrrZi4HPpfefA16Tm39zRHRFxENkV5adNh7lHCsRsTUi7k7vdwFrye7Kn5TnHJnH02R7egWT9HxrJM0FXgl8Ojd7Up/zAEo/ZwfEwMN9TFZHRnavCenvEWn+pPocJM0HTiH7VT1pzzk1tdwLbANui4hJfb7Jx4H3AtXcvMl+zgF8R9JdaXghGIdz9hPlhjGsxyQ3aT4HSTOALwOXRcTOQUa0fdKfc0T0AidLOhT4qqRnDbL6k/58Jb0K2BYRd0k6q5FNCuY9qc45OTMitkg6ArhN0gODrDtm5+waRGNDgkwmj6QRc0l/t6X5k+JzkNROFg43RsRX0uxJfc4AEfEocDtwNpP7fM8EzpO0gaw5+MWSbmBynzMRsSX93QZ8lazJqPRzdkA037Aey4A3p/dvBr6em3++pKmSjid7RsfPJqB8I6asqvAZYG1EXJ1bNCnPWVJHqjkgaTrwUuABJun5AkTElRExNyLmk/2/+v2IuJBJfM6SDpY0s/YeeDlwP+NxzhPdO38gvMiG+/glWW//+ye6PGN4Xl8AtgI9ZL8q/hQ4HPge8Kv097Dc+u9Pn8E64JyJLv8Izvf5ZFXp1cC96XXuZD1n4NnAPel87wc+mOZPyvMtOP+z2H8V06Q9Z7IrLH+eXmtq31Hjcc4easPMzAq5icnMzAo5IMzMrJADwszMCjkgzMyskAPCzMwKOSBs0pDUm0a7rL3ml3CMUyTFRI8KKuktko4ZZPnHJb2gbt5VddP/Q9JnyymhTQYOCJtM9kbEybnXhqE2UGY4/x9cAPwk/Z1IbwEKA0LSYcAZkQ3WiKTnS1oJXJKGB38xQETcB8yVdOw4ldmeZBwQNmlJmiHpe5LuTmPpL07z5yt7ZsS/AXcD8yT9laSVklbXnqtQsD8BryP7cn65pGm5/T0g6dOS7pd0o6SXSvrvNFb/aWm9wyR9LR3jTknPTvOvkvSe3HHuT/uslfM/lD3v4TuSpkt6HbAIuDHVlKbXFfV1wLdz01cDVwJLgZeRje5Z8w2yO5LN+nFA2GQyPde89FVgH/A/I+JU4EXAv2j/yH1PB66PiFPS+wVk49ucDDy3vnkmORN4KCJ+TTbu0bm5ZScCnyC7u/kk4A1kd3a/B3hfWudDwD0R8ew07/oGzmkBcE1EPBN4FHhtRNwCrALemGpKewvKeVduuhs4CiAiHouIjbllq4A/aqAc1oQ8mqtNJnsj4uTaRBq47x/Tl32VbMjjI9Pi30TEnen9y9PrnjQ9g+yL+Ud1+7+AbIA40t83AbUBAR9KTTZIWgN8LyJC0n3A/LTO84HXAkTE9yUdLumQIc7poYi4N72/K7evwRwNdOamlwAfAc6U9HTgylzz2zYGaKoyc0DYZPZGoAN4bkT0pBFAp6Vlu3PrCfhwRPz7QDuS1Er25X6epPenbQ6vDaIGdOVWr+amq+z//2ygYZgrPLE2Py33Pr/fXqC+OanI3vw+IuIXwKsl/QPwW7IBDV+SO1Z9DcQMcBOTTW6HkD07oEfSi4DjBljvVuBPlD1HAklz0rj7eS8Ffh4R8yJifkQcRzas+GuGUZ4fkYUW6VkG2yNiJ9njJE9N808le0zkUHaRPVa1yFqyJi/SPmvPiOgh63PJb/c0soH+zPpxQNhkdiOwSNlD3t9INhR2PxHxHbLnG/80NQndQv8v3wvIxuHP+zJZX0OjrkrlWQ38E/uHav4ycJiyJ8P9BdnIwkP5LLB0gE7qb5GNdFrzNkl3kI3mu5TsaWw1L0rrm/Xj0VzNJiFJPwFeFdmDhGrzroqIq3LTU4EfAs+PiMq4F9IOeK5BmE1OlwP19zfcXjd9LHCFw8EG4hqEmZkVcg3CzMwKOSDMzKyQA8LMzAo5IMzMrJADwszMCv1/WPLhS6ufiYQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Fare amount distribution\n",
    "# we use rice binning for fare amount distribution\n",
    "def rice(x):\n",
    "    return int(2 * x ** (1/3))\n",
    "    \n",
    "fare_amount_clean_df = clean_sdf.select('fare_amount').toPandas()\n",
    "sns.distplot(fare_amount_clean_df['fare_amount'], bins = rice(len(fare_amount_clean_df)))\n",
    "plt.title('Distribution of Fare Amount')\n",
    "plt.xlabel('Fare Amount ($)')\n",
    "plt.savefig('../plots/fare_amount_clean_distplot.png')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save clean data in parquet format\n",
    "clean_sdf.write.format('parquet').save('../preprocessed_data/clean_yellow.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>temp_ave</th>\n",
       "      <th>temp_max</th>\n",
       "      <th>temp_min</th>\n",
       "      <th>wind_ave</th>\n",
       "      <th>percip</th>\n",
       "      <th>snowfall</th>\n",
       "      <th>snowdepth</th>\n",
       "      <th>is_foggy</th>\n",
       "      <th>is_heavy_foggy</th>\n",
       "      <th>is_thunder</th>\n",
       "      <th>is_smoggy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2018-01-01</td>\n",
       "      <td>-10.555556</td>\n",
       "      <td>-7.222222</td>\n",
       "      <td>-13.888889</td>\n",
       "      <td>7.83</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2018-01-02</td>\n",
       "      <td>-6.944444</td>\n",
       "      <td>-3.333333</td>\n",
       "      <td>-10.555556</td>\n",
       "      <td>8.05</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2018-01-03</td>\n",
       "      <td>-5.000000</td>\n",
       "      <td>-1.111111</td>\n",
       "      <td>-8.888889</td>\n",
       "      <td>3.13</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2018-01-04</td>\n",
       "      <td>-4.444444</td>\n",
       "      <td>-1.666667</td>\n",
       "      <td>-7.222222</td>\n",
       "      <td>12.53</td>\n",
       "      <td>0.76</td>\n",
       "      <td>9.8</td>\n",
       "      <td>1.2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2018-01-05</td>\n",
       "      <td>-10.000000</td>\n",
       "      <td>-7.222222</td>\n",
       "      <td>-12.777778</td>\n",
       "      <td>12.97</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date   temp_ave  temp_max   temp_min  wind_ave  percip  snowfall  \\\n",
       "0 2018-01-01 -10.555556 -7.222222 -13.888889      7.83    0.00       0.0   \n",
       "1 2018-01-02  -6.944444 -3.333333 -10.555556      8.05    0.00       0.0   \n",
       "2 2018-01-03  -5.000000 -1.111111  -8.888889      3.13    0.00       0.0   \n",
       "3 2018-01-04  -4.444444 -1.666667  -7.222222     12.53    0.76       9.8   \n",
       "4 2018-01-05 -10.000000 -7.222222 -12.777778     12.97    0.00       0.0   \n",
       "\n",
       "   snowdepth  is_foggy  is_heavy_foggy  is_thunder  is_smoggy  \n",
       "0        0.0       0.0             0.0         0.0        0.0  \n",
       "1        0.0       0.0             0.0         0.0        0.0  \n",
       "2        0.0       0.0             0.0         0.0        0.0  \n",
       "3        1.2       1.0             1.0         0.0        1.0  \n",
       "4        7.1       0.0             0.0         0.0        0.0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "weather_df = pd.read_csv('../raw_data/nyc_weather_2018.csv',skiprows=[0])\n",
    "\n",
    "# change datetime datatype\n",
    "weather_df['DATE'] = pd.to_datetime(weather_df['DATE'])\n",
    "\n",
    "# select desired date\n",
    "weather_df = weather_df[((weather_df['DATE'] >= '2018-01-01') & (weather_df['DATE'] <= '2018-02-28')) | ((weather_df['DATE'] >= '2018-6-01') & (weather_df['DATE'] <= '2018-8-31')) | ((weather_df['DATE'] >= '2018-12-01') & (weather_df['DATE'] <= '2018-12-31'))]\n",
    "\n",
    "# temperature is in Fahrenheit, change to Celsius\n",
    "weather_df[['TMAX', 'TMIN']] = (weather_df[['TMAX', 'TMIN']] - 32) * 5/9\n",
    "\n",
    "# average temperature is missing, compute using (TMAX + TMIN) / 2\n",
    "weather_df['TAVG'] = (weather_df['TMAX'] + weather_df['TMIN']) / 2\n",
    "\n",
    "# select desired columns\n",
    "cols = ['DATE', 'TAVG', 'TMAX', 'TMIN','AWND','PRCP', 'SNOW', 'SNWD', 'WT01', 'WT02', 'WT03', 'WT08']\n",
    "weather_df[['WT01', 'WT02', 'WT03', 'WT08']] = weather_df[['WT01', 'WT02', 'WT03', 'WT08']].fillna(0)\n",
    "weather_df = weather_df[cols]\n",
    "\n",
    "# rename columns\n",
    "weather_df.columns = ['date', 'temp_ave', 'temp_max', 'temp_min', 'wind_ave', 'percip', 'snowfall', 'snowdepth', 'is_foggy', 'is_heavy_foggy', 'is_thunder', 'is_smoggy']\n",
    "\n",
    "weather_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "weather_df.to_csv('../preprocessed_data/nyc_weather_2018_clean.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "dce69896fdb445434427c12e791455610f9ef8e6bb07ea975426634cd43b3db3"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}