events_extraction_script.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "import eventful\n",
    "import pandas as pd\n",
    "from datetime import datetime, timedelta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the api key from http://api.eventful.com/\n",
    "\n",
    "api = eventful.API('<api key goes here>')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### For training we want the events from start time to end time of the data provided to us "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv(\"daily_counts_10-25-19.csv\")\n",
    "data['Time'] = pd.to_datetime(data['Time'])\n",
    "\n",
    "min_ts = min(data['Time'])\n",
    "max_ts = max(data['Time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Timestamp('2016-02-17 00:00:00'), Timestamp('2019-10-24 00:00:00'))"
      ]
     },
     "execution_count": 246,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_ts, max_ts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a list of dates from start time till end time\n",
    "\n",
    "start_date = datetime(2016, 2, 17)\n",
    "end_date = datetime(2019, 10, 24)\n",
    "\n",
    "def datespan(startDate, endDate, delta=timedelta(days=1)):\n",
    "    currentDate = startDate\n",
    "    while currentDate < endDate:\n",
    "        yield currentDate\n",
    "        currentDate += delta\n",
    "\n",
    "path_list = []\n",
    "for single_date in datespan(start_date, end_date):\n",
    "    path_list.append((str(single_date.year)) + \"/\" + (str(single_date.month)) + \"/\"\n",
    "                              + (str(single_date.day)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [],
   "source": [
    "title = []\n",
    "venue_name = []\n",
    "stop_time = []\n",
    "start_time = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dt in path_list:\n",
    "    events = api.call('/events/search', l='78701', t=dt, within='5', units='miles')\n",
    "    try:\n",
    "        for event in events['events']['event']:\n",
    "            title.append(event['title'])\n",
    "            venue_name.append(event['venue_name'])\n",
    "            stop_time.append(event['stop_time'])\n",
    "            start_time.append(event['start_time'])\n",
    "    # enters except statement when we did not find any events for a particular date \n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3935"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(venue_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "events_df = pd.DataFrame()\n",
    "\n",
    "events_df['title'] = title\n",
    "events_df['venue_name'] = venue_name\n",
    "events_df['start_time'] = start_time\n",
    "events_df['stop_time'] = stop_time\n",
    "\n",
    "events_df['start_time'] = pd.to_datetime(events_df['start_time'])\n",
    "\n",
    "events_df.head(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make the events date in the format of the trail data date format \n",
    "\n",
    "events_df['start_date'] = events_df['start_time'].dt.strftime('%m/%d/%Y')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Events weighting : For those events that have words in the word list, they are more likely to be events around the trail, so give a higher weight to these types of events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_list = ['bike','run','running','music','tour','hike','camp','camping','bird','yoga','kayak','kayaking', 'trail', 'walk','biking','hiking','walking']\n",
    "def type_of_event(x):\n",
    "    x = x.lower()\n",
    "    x = x.split()\n",
    "    if(len(set(x).intersection(word_list)) >=1):\n",
    "        return 2\n",
    "    return 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "events_df['event_weight'] = events_df['title'].apply(type_of_event)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>venue_name</th>\n",
       "      <th>start_time</th>\n",
       "      <th>stop_time</th>\n",
       "      <th>start_date</th>\n",
       "      <th>event_weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>SAFe® Advanced Scrum Master with SASM Certific...</td>\n",
       "      <td>Sheraton Austin Hotel @ the Capitol</td>\n",
       "      <td>2019-10-19 09:00:00</td>\n",
       "      <td>None</td>\n",
       "      <td>10/19/2019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Guide a Bike Tour!</td>\n",
       "      <td>Postal code 78741, US</td>\n",
       "      <td>2019-10-19 11:53:00</td>\n",
       "      <td>None</td>\n",
       "      <td>10/19/2019</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Early Bird Brunch</td>\n",
       "      <td>Cafe Blue</td>\n",
       "      <td>2019-10-19 09:00:00</td>\n",
       "      <td>None</td>\n",
       "      <td>10/19/2019</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Guide a Bike Tour!</td>\n",
       "      <td>Postal code 78741, US</td>\n",
       "      <td>2019-10-19 11:53:00</td>\n",
       "      <td>None</td>\n",
       "      <td>10/19/2019</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Energetic S@x Practicum &amp; Playshop w/ Reid Mih...</td>\n",
       "      <td>Austin</td>\n",
       "      <td>2019-10-19 10:30:00</td>\n",
       "      <td>None</td>\n",
       "      <td>10/19/2019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3930</th>\n",
       "      <td>Jennifer Fuentes</td>\n",
       "      <td>Parker Jazz Club</td>\n",
       "      <td>2019-10-23 22:00:00</td>\n",
       "      <td>None</td>\n",
       "      <td>10/23/2019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3931</th>\n",
       "      <td>Urban Roots Annual Seed to Harvest Kick-Off</td>\n",
       "      <td>Austin Children's Museum</td>\n",
       "      <td>2019-10-23 18:00:00</td>\n",
       "      <td>None</td>\n",
       "      <td>10/23/2019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3932</th>\n",
       "      <td>Preschool Storytime</td>\n",
       "      <td>Twin Oaks Branch</td>\n",
       "      <td>2019-10-23 00:00:00</td>\n",
       "      <td>None</td>\n",
       "      <td>10/23/2019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3933</th>\n",
       "      <td>All Ages Storytime</td>\n",
       "      <td>Howson Branch Library</td>\n",
       "      <td>2019-10-23 00:00:00</td>\n",
       "      <td>None</td>\n",
       "      <td>10/23/2019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3934</th>\n",
       "      <td>All Ages Storytime</td>\n",
       "      <td>University Hills Branch Library</td>\n",
       "      <td>2019-10-23 00:00:00</td>\n",
       "      <td>None</td>\n",
       "      <td>10/23/2019</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3935 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title  \\\n",
       "0     SAFe® Advanced Scrum Master with SASM Certific...   \n",
       "1                                    Guide a Bike Tour!   \n",
       "2                                     Early Bird Brunch   \n",
       "3                                    Guide a Bike Tour!   \n",
       "4     Energetic S@x Practicum & Playshop w/ Reid Mih...   \n",
       "...                                                 ...   \n",
       "3930                                   Jennifer Fuentes   \n",
       "3931        Urban Roots Annual Seed to Harvest Kick-Off   \n",
       "3932                                Preschool Storytime   \n",
       "3933                                 All Ages Storytime   \n",
       "3934                                 All Ages Storytime   \n",
       "\n",
       "                               venue_name          start_time stop_time  \\\n",
       "0     Sheraton Austin Hotel @ the Capitol 2019-10-19 09:00:00      None   \n",
       "1                   Postal code 78741, US 2019-10-19 11:53:00      None   \n",
       "2                               Cafe Blue 2019-10-19 09:00:00      None   \n",
       "3                   Postal code 78741, US 2019-10-19 11:53:00      None   \n",
       "4                                  Austin 2019-10-19 10:30:00      None   \n",
       "...                                   ...                 ...       ...   \n",
       "3930                     Parker Jazz Club 2019-10-23 22:00:00      None   \n",
       "3931             Austin Children's Museum 2019-10-23 18:00:00      None   \n",
       "3932                     Twin Oaks Branch 2019-10-23 00:00:00      None   \n",
       "3933                Howson Branch Library 2019-10-23 00:00:00      None   \n",
       "3934      University Hills Branch Library 2019-10-23 00:00:00      None   \n",
       "\n",
       "      start_date  event_weight  \n",
       "0     10/19/2019             1  \n",
       "1     10/19/2019             2  \n",
       "2     10/19/2019             2  \n",
       "3     10/19/2019             2  \n",
       "4     10/19/2019             1  \n",
       "...          ...           ...  \n",
       "3930  10/23/2019             1  \n",
       "3931  10/23/2019             1  \n",
       "3932  10/23/2019             1  \n",
       "3933  10/23/2019             1  \n",
       "3934  10/23/2019             1  \n",
       "\n",
       "[3935 rows x 6 columns]"
      ]
     },
     "execution_count": 223,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "events_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [],
   "source": [
    "events_grouped = events_df.groupby(['start_date'])[['event_weight']].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "events_grouped = events_grouped.rename(columns={'start_date' : 'weight'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event_weight</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>start_date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>01/01/2019</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01/02/2019</th>\n",
       "      <td>5</td>\n",
       "      <td>2019-01-02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01/03/2019</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-01-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01/04/2019</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-01-04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>01/05/2019</th>\n",
       "      <td>2</td>\n",
       "      <td>2019-01-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12/27/2018</th>\n",
       "      <td>4</td>\n",
       "      <td>2018-12-27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12/28/2018</th>\n",
       "      <td>3</td>\n",
       "      <td>2018-12-28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12/29/2018</th>\n",
       "      <td>4</td>\n",
       "      <td>2018-12-29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12/30/2018</th>\n",
       "      <td>3</td>\n",
       "      <td>2018-12-30</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12/31/2018</th>\n",
       "      <td>3</td>\n",
       "      <td>2018-12-31</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>433 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            event_weight       date\n",
       "start_date                         \n",
       "01/01/2019             2 2019-01-01\n",
       "01/02/2019             5 2019-01-02\n",
       "01/03/2019             2 2019-01-03\n",
       "01/04/2019             2 2019-01-04\n",
       "01/05/2019             2 2019-01-05\n",
       "...                  ...        ...\n",
       "12/27/2018             4 2018-12-27\n",
       "12/28/2018             3 2018-12-28\n",
       "12/29/2018             4 2018-12-29\n",
       "12/30/2018             3 2018-12-30\n",
       "12/31/2018             3 2018-12-31\n",
       "\n",
       "[433 rows x 2 columns]"
      ]
     },
     "execution_count": 218,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "events_grouped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "events_grouped['date'] = events_grouped.index\n",
    "events_grouped['date'] = pd.to_datetime(events_grouped['date'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Joining the events dataframe on weather and trail count data :"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [],
   "source": [
    "weather = pd.read_csv(\"weather_10-22-19.csv\")\n",
    "\n",
    "weather['DATE'] = pd.to_datetime(weather['DATE'])\n",
    "data['Time'] = pd.to_datetime(data['Time'])\n",
    "\n",
    "final_df = weather[['PRCP','TMAX','TMIN','DATE']].merge(data, left_on=['DATE'], right_on=['Time'], how='right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 236,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp = events_grouped.merge(final_df, left_on=['date'], right_on = ['Time'], how='right')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp.drop(['date','DATE'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 241,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp['event_weight'] = temp['event_weight'].fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['event_weight', 'PRCP', 'TMAX', 'TMIN', 'Time',\n",
       "       'Butler Trail : Crenshaw Bridge - Urban Trail',\n",
       "       'Butler Trail: South Lamar - Urban Trail',\n",
       "       'Butler Trail: North Congress - Urban Trail',\n",
       "       'Butler Trail : Longhorn Dam - Urban Trail',\n",
       "       'Shoal Creek Trail & 24th St- Urban Trail'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp.to_csv(\"events_df.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}