explore_athlete.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "import re\n",
    "import numpy as np\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Men's 100 Meters Finals\",\n",
       " \"Men's 200 Meters Finals\",\n",
       " \"Men's 400 Meters Finals\",\n",
       " \"Men's 800 Meters Finals\",\n",
       " \"Men's 1500 Meters Finals\",\n",
       " \"Men's 5000 Meters\",\n",
       " \"Men's 10,000 Meters\",\n",
       " \"Men's 110 Hurdles Finals\",\n",
       " \"Men's 400 Hurdles Finals\",\n",
       " \"Men's 3000 Steeplechase Finals\",\n",
       " 'Men High Jump',\n",
       " \"Men's  Pole Vault\",\n",
       " \"Men's  Long Jump\",\n",
       " \"Men's  Triple Jump\",\n",
       " \"Men's  Shot Put\",\n",
       " \"Men's  Discus\",\n",
       " \"Men's  Hammer\",\n",
       " \"Men's  Javelin\"]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url=\"https://www.tfrrs.org/results/75224/m/NCAA_Division_I_Outdoor_Track__Field_Championships\"\n",
    "headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.102 Safari/537.36\"}\n",
    "response = requests.get(url, headers=headers)\n",
    "\n",
    "if response.status_code < 300 and response.status_code >= 200:\n",
    "    # panda's read_html doesn't accept percent colspan arguments\n",
    "    HTML = response.text.replace('colspan=\"100%\"', 'colspan=\"3\"')\n",
    "else:\n",
    "    HTML = None\n",
    "    raise Exception(\"Could not retrieve\", response.status_code)\n",
    "\n",
    "soup = BeautifulSoup(HTML, \"html.parser\")\n",
    "event_names = [event.text.strip().replace(\"\\n\", \" \") for event in soup.find_all(\"h3\", class_=\"font-weight-500 pl-5\")]\n",
    "finals = [event for event in event_names if not any(x in event for x in [\"Preliminaries\", \"Jump Off\", \"Decathlon\", \"4 x\"])]\n",
    "finals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>all_american</th>\n",
       "      <th>event</th>\n",
       "      <th>place</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>JOSEPH FAHNBULLEH</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>FAVOUR ASHE</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>2</td>\n",
       "      <td>0.958333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JAVONTE HARDING</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>3</td>\n",
       "      <td>0.916667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>JOVAUGHN MARTIN</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>4</td>\n",
       "      <td>0.875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>DEDRICK VANOVER</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>5</td>\n",
       "      <td>0.833333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503</th>\n",
       "      <td>MICHAEL WHITTAKER</td>\n",
       "      <td>0</td>\n",
       "      <td>JT</td>\n",
       "      <td>20</td>\n",
       "      <td>0.208333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>504</th>\n",
       "      <td>JARED O'RILEY</td>\n",
       "      <td>0</td>\n",
       "      <td>JT</td>\n",
       "      <td>21</td>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>505</th>\n",
       "      <td>SCOTT CAMPBELL</td>\n",
       "      <td>0</td>\n",
       "      <td>JT</td>\n",
       "      <td>22</td>\n",
       "      <td>0.125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>506</th>\n",
       "      <td>JAMES MCNANEY</td>\n",
       "      <td>0</td>\n",
       "      <td>JT</td>\n",
       "      <td>23</td>\n",
       "      <td>0.083333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>LUKE JACKSON</td>\n",
       "      <td>0</td>\n",
       "      <td>JT</td>\n",
       "      <td>24</td>\n",
       "      <td>0.041667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>431 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                  name  all_american event  place     score\n",
       "0    JOSEPH FAHNBULLEH             1   100      1       1.0\n",
       "1          FAVOUR ASHE             1   100      2  0.958333\n",
       "2      JAVONTE HARDING             1   100      3  0.916667\n",
       "3      JOVAUGHN MARTIN             1   100      4     0.875\n",
       "4      DEDRICK VANOVER             1   100      5  0.833333\n",
       "..                 ...           ...   ...    ...       ...\n",
       "503  MICHAEL WHITTAKER             0    JT     20  0.208333\n",
       "504      JARED O'RILEY             0    JT     21  0.166667\n",
       "505     SCOTT CAMPBELL             0    JT     22     0.125\n",
       "506      JAMES MCNANEY             0    JT     23  0.083333\n",
       "507       LUKE JACKSON             0    JT     24  0.041667\n",
       "\n",
       "[431 rows x 5 columns]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.options.mode.chained_assignment = None\n",
    "\n",
    "finals_name_map = {\n",
    "    \"Men's 100 Meters Finals\": \"100\",\n",
    "    \"Men's 200 Meters Finals\": \"200\",\n",
    "    \"Men's 400 Meters Finals\": \"400\",\n",
    "    \"Men's 800 Meters Finals\": \"800\",\n",
    "    \"Men's 1500 Meters Finals\": \"1500\",\n",
    "    \"Men's 5000 Meters\": \"5000\",\n",
    "    \"Men's 10,000 Meters\": \"10000\",\n",
    "    \"Men's 110 Hurdles Finals\": \"110H\",\n",
    "    \"Men's 400 Hurdles Finals\": \"400H\",\n",
    "    \"Men's 3000 Steeplechase Finals\": \"3000S\",\n",
    "    \"Men High Jump\": \"HJ\",\n",
    "    \"Men's  Pole Vault\": \"PV\",\n",
    "    \"Men's  Long Jump\": \"LJ\",\n",
    "    \"Men's  Triple Jump\": \"TJ\",\n",
    "    \"Men's  Shot Put\": \"SP\",\n",
    "    \"Men's  Discus\": \"DT\",\n",
    "    \"Men's  Hammer\": \"HT\",\n",
    "    \"Men's  Javelin\": \"JT\"\n",
    "}\n",
    "\n",
    "prelims_name_map = {\n",
    "    # prelims\n",
    "    \"Men's 100 Meters Preliminaries\": \"100\",\n",
    "    \"Men's 200 Meters Preliminaries\": \"200\",\n",
    "    \"Men's 400 Meters Preliminaries\": \"400\",\n",
    "    \"Men's 800 Meters Preliminaries\": \"800\",\n",
    "    \"Men's 1500 Meters Preliminaries\": \"1500\",\n",
    "    \"Men's 110 Hurdles Preliminaries\": \"110H\",\n",
    "    \"Men's 400 Hurdles Preliminaries\": \"400H\",\n",
    "    \"Men's 3000 Steeplechase Preliminaries\": \"3000S\"\n",
    "}\n",
    "\n",
    "all_athletes = pd.DataFrame(\n",
    "    {\"name\": pd.Series(dtype=\"str\"),\n",
    "    \"all_american\": pd.Series(dtype=\"int\"),\n",
    "    \"event\": pd.Series(dtype=\"str\")}\n",
    ")\n",
    "\n",
    "for tag in soup.find_all(\"div\", class_=\"col-lg-12\"):\n",
    "    text = str(tag).replace(\"\\n\", \" \")\n",
    "    event = re.search(\"<h3\\s*.*>(\\s*.*)<\\/h3>\", text).group(1).strip()\n",
    "    \n",
    "    if event in prelims_name_map or event in finals_name_map:\n",
    "\n",
    "        dfs = pd.read_html(text)    \n",
    "        if len(dfs) > 1:\n",
    "            raise Exception(\"Too many HTML tables\")\n",
    "        df = dfs[0]\n",
    "        \n",
    "        df[\"valid_name\"] = df[\"NAME\"].apply(lambda x: re.sub(\"'|\\.|,|-|\\s|FOUL\", \"\", x).isalpha())\n",
    "        df = df[df[\"valid_name\"]]\n",
    "        df[\"NAME\"] = df[\"NAME\"].apply(lambda x: x.replace(\"  \", \" \").upper())\n",
    "        \n",
    "        if event in finals_name_map:\n",
    "            df[\"all_american\"] = df[\"PL\"].apply(lambda x: 1 if x <= 8 else 0)\n",
    "            df[\"event\"] = finals_name_map[event]\n",
    "        else:\n",
    "            df[\"all_american\"] = 0\n",
    "            df[\"event\"] = prelims_name_map[event]\n",
    "    \n",
    "        # df = df[[\"NAME\", \"all_american\", \"event\"]]\n",
    "        # df.columns = [\"name\", \"all_american\", \"event\"]\n",
    "        df = df[[\"NAME\", \"PL\", \"all_american\", \"event\"]]\n",
    "        df.columns = [\"name\", \"place\", \"all_american\", \"event\"]\n",
    "        df[\"place\"] = df[\"place\"].astype(pd.Int64Dtype())\n",
    "        df[\"score\"] = df[\"place\"].copy().apply(lambda x: 1-((x-1)/24))\n",
    "        \n",
    "        all_athletes = pd.concat([all_athletes, df], ignore_index=True)\n",
    "\n",
    "pd.options.mode.chained_assignment = \"warn\"\n",
    "all_athletes = all_athletes.drop_duplicates(subset=[\"name\", \"all_american\", \"event\"]) # there are duplicates for 9th place finisher in finals\n",
    "all_athletes[\"row_num\"] = all_athletes.groupby([\"name\", \"event\"])[\"all_american\"].transform('count') # row count over partition by name, event\n",
    "all_athletes = all_athletes[~((all_athletes[\"row_num\"] == 2) & (all_athletes[\"all_american\"] == 0))] # throw out prelims for all americans\n",
    "all_athletes = all_athletes.drop([\"row_num\"], axis=1)\n",
    "all_athletes[\"place\"] = all_athletes[\"place\"].fillna(24)\n",
    "all_athletes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>event</th>\n",
       "      <th>count</th>\n",
       "      <th>sum</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10000</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>110H</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1500</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>200</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3000S</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>400</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>400H</td>\n",
       "      <td>23</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>5000</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>800</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>DT</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>HJ</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>HT</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>JT</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>LJ</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>PV</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>SP</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>TJ</td>\n",
       "      <td>24</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    event  count  sum\n",
       "0     100     24    8\n",
       "1   10000     24    8\n",
       "2    110H     24    8\n",
       "3    1500     24    8\n",
       "4     200     24    8\n",
       "5   3000S     24    8\n",
       "6     400     24    8\n",
       "7    400H     23    8\n",
       "8    5000     24    8\n",
       "9     800     24    8\n",
       "10     DT     24    8\n",
       "11     HJ     24    8\n",
       "12     HT     24    8\n",
       "13     JT     24    8\n",
       "14     LJ     24    8\n",
       "15     PV     24    8\n",
       "16     SP     24    8\n",
       "17     TJ     24    8"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sum should be 8, count should be 24\n",
    "#all_athletes.to_csv(\"./data/processed/events_and_target.csv\", index=False)\n",
    "#all_athletes[all_athletes[\"event\"] == \"400H\"]\n",
    "all_athletes.groupby(\"event\").agg({\"all_american\": [\"count\", \"sum\"]})[\"all_american\"].reset_index() # only 23 participants in 400H"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_html(url):\n",
    "    headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.102 Safari/537.36\"}\n",
    "    response = requests.get(url, headers=headers)\n",
    "\n",
    "    if response.status_code < 300 and response.status_code >= 200:\n",
    "        # panda's read_html doesn't accept percent colspan arguments\n",
    "        HTML = response.text.replace('colspan=\"100%\"', 'colspan=\"3\"')\n",
    "    else:\n",
    "        HTML = None\n",
    "        raise Exception(\"Could not retrieve\", response.status_code)\n",
    "\n",
    "    soup = BeautifulSoup(HTML, \"html.parser\")\n",
    "    reduced = str(soup.find(\"div\", class_=\"panel panel-bordered\"))\n",
    "\n",
    "    highschool_cutoff = re.search('↓Competing for\\s+<a href=\"https:\\/\\/www.directathletics.com', reduced)\n",
    "    if highschool_cutoff:\n",
    "        print(url)\n",
    "        reduced = reduced[:highschool_cutoff.span(0)[0]]\n",
    "\n",
    "    return reduced\n",
    "\n",
    "# HTML = get_html(\"https://tfrrs.org/athletes/7531871/Florida/Joseph_Fahnbulleh.html\")\n",
    "#HTML = get_html(\"https://tfrrs.org/athletes/7546466/Texas_Arlington/Arthur_Petersen.html\")\n",
    "#HTML = get_html(\"https://tfrrs.org/athletes/7526270/Maryland/Caleb_Dean.html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_athlete_info(html_string, local=False):\n",
    "    if local:\n",
    "        with open(html_string, \"r\") as f:\n",
    "            html_string = f.read()\n",
    "\n",
    "    soup = BeautifulSoup(html_string, \"html.parser\")\n",
    "    info = soup.find(\"div\", class_=\"panel-heading\").get_text().replace(\"\\n\", \"\")\n",
    "\n",
    "    name, school = re.split(\"\\(.+?\\)\", info)[:2] # school MIAMI (FLA.) goes to MIAMI but idc\n",
    "    name = name.strip().replace(\"  \", \" \")\n",
    "    school = school.strip()\n",
    "\n",
    "    grade_year = re.search(\"\\(.+?\\)\", info).group()\n",
    "    grade, year = re.split(\"-|/\", grade_year[1:-1])\n",
    "\n",
    "    data = {\n",
    "        \"name\": name,\n",
    "        \"grade\": grade if grade != \"RS\" else \"SR\",\n",
    "        #\"year\": int(year) if year.isnumeric() else year,\n",
    "        \"year\": int(year) if year.isnumeric() else 4,\n",
    "        \"school\": school\n",
    "    }\n",
    "\n",
    "    return data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# valid_events = [\n",
    "#     '60', '100', '200', '400', '800', '1500', 'Mile', '3000S', '5000', '10000', '8k', '10k'\n",
    "#     '60H', '110H', '400H', '4x100', '4x400', 'DMR',\n",
    "#     'JT', 'HT', 'WT', 'DT', 'SP',\n",
    "#     'PV', 'HJ', 'LJ', 'TJ',\n",
    "#     'Dec', 'Hep', #'Pent'\n",
    "# ]\n",
    "\n",
    "events = {\n",
    "    \"races\": [\n",
    "        \"100\",\n",
    "        \"200\",\n",
    "        \"400\",\n",
    "        \"800\",\n",
    "        \"1500\",\n",
    "        \"3000S\",\n",
    "        \"5000\",\n",
    "        \"10000\",\n",
    "        \"110H\",\n",
    "        \"400H\",\n",
    "    ],\n",
    "    \"jumps\": [\"HJ\", \"PV\", \"LJ\", \"TJ\"],\n",
    "    \"throws\": [\"SP\", \"DT\", \"HT\", \"JT\"]\n",
    "    # \"multi\": [\n",
    "    #    \"Dec\", \"Hept\"\n",
    "    # ]\n",
    "}\n",
    "\n",
    "ncaa_champ_events = events[\"races\"] + events[\"jumps\"] + events[\"throws\"]\n",
    "\n",
    "# from https://en.wikipedia.org/wiki/List_of_United_States_collegiate_records_in_track_and_field\n",
    "# ignoring PS records if higher and accepting foreign born athletes\n",
    "ncaa_records = {\n",
    "    \"100\": 9.82,\n",
    "    \"200\": 19.69,\n",
    "    \"400\": 43.61,\n",
    "    \"800\": 60.0 + 43.25,\n",
    "    \"1500\": 3 * 60.0 + 33.74,\n",
    "    \"3000S\": 8 * 60.0 + 18.88,  # other record is hand timed and too fast\n",
    "    \"5000\": 13 * 60.0 + 6.32,\n",
    "    \"10000\": 27 * 60.0 + 8.49,\n",
    "    \"110H\": 12.98,\n",
    "    \"400H\": 47.02,\n",
    "    \"4x100\": 37.97,\n",
    "    \"4x400\": 2 * 60.0 + 58.53,\n",
    "    #'DMR': 9*60.0+20.10,\n",
    "    \"HJ\": 2.38,\n",
    "    \"PV\": 6.00,\n",
    "    \"LJ\": 8.74,\n",
    "    \"TJ\": 17.57,\n",
    "    \"JT\": 89.10,\n",
    "    \"SP\": 22.0,\n",
    "    \"DT\": 68.73,\n",
    "    \"HT\": 81.94,\n",
    "    #'Dec': 8720,\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_event_mark(mark):\n",
    "    # try to make pandas use float to avoid importing all of numpy\n",
    "    if isinstance(mark, np.\"float\") or isinstance(mark, float):\n",
    "        return float(mark)\n",
    "\n",
    "    # Some results are just the float\n",
    "    if mark.isalpha():\n",
    "        return mark\n",
    "\n",
    "    no_mark = re.match(\"([A-Z]+)\\s*(\\(-?\\d.\\d\\)|[A-Z]+)\", str(mark))\n",
    "    if no_mark:\n",
    "        return no_mark.group(1)\n",
    "\n",
    "    # possibly irrelevant\n",
    "    elif mark.replace(\".\", \"\").isnumeric():\n",
    "        return float(mark)\n",
    "\n",
    "    else:\n",
    "        # Don't want feet conversion or wind right now\n",
    "        endChars = [\"m\", \"W\", \"w\", \"(\", \"W\"]\n",
    "        for char in endChars:\n",
    "            if char in mark:\n",
    "                return float(mark[0 : mark.index(char)])\n",
    "\n",
    "    # Unaccounted for\n",
    "    return mark\n",
    "\n",
    "def convert_time_mark(mark):\n",
    "    match = re.match(\"(\\d{1,2}):(\\d{1,2}.\\d{1,2})\", str(mark))\n",
    "    return mark if not match else 60*float(match.group(1))+float(match.group(2))\n",
    "\n",
    "def scale_mark(df):\n",
    "    if str(df[\"raw_mark\"]).isalpha():\n",
    "        return df[\"raw_mark\"]\n",
    "    if df[\"event\"] in events[\"races\"]:\n",
    "        return ncaa_records[df[\"event\"]] / df[\"convert_mark\"]\n",
    "    else:\n",
    "        return df[\"convert_mark\"] / ncaa_records[df[\"event\"]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_event_history(HTML, local=False):\n",
    "    if local:\n",
    "        with open(HTML, \"r\") as f:\n",
    "            HTML = f.read()\n",
    "    dfs = pd.read_html(HTML)\n",
    "\n",
    "    l = [\n",
    "        df.values\n",
    "        for df in dfs[1:]\n",
    "        if not df.empty and str(df.values[0][0]).strip() in ncaa_champ_events# and int(df.columns[0][-4:]) >= 2017 \\# exclude highschool data\n",
    "    ]\n",
    "    flat_list = [item for sublist in l for item in sublist]\n",
    "\n",
    "    df = pd.DataFrame(flat_list, columns=[\"event\", \"raw_mark\", \"place\"])\n",
    "    df[\"event\"] = df[\"event\"].astype(str)\n",
    "    df = df[df[\"event\"].isin(ncaa_champ_events)] #re-apply incase they did multiple events in on meet\n",
    "\n",
    "    df[\"raw_mark\"] = df[\"raw_mark\"].apply(lambda x: parse_event_mark(x))\n",
    "    df[\"convert_mark\"] = df[\"raw_mark\"].apply(lambda x: convert_time_mark(x))\n",
    "    df[\"scaled_mark\"] = df.apply(lambda x: scale_mark(x), axis=1)\n",
    "\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define functions for descriptive statistics and feature extraction\n",
    "def no_mark_rate(x):\n",
    "    return sum([1 if str(mark).isalpha() else 0 for mark in x]) / len(x)\n",
    "\n",
    "\n",
    "def q1(x):\n",
    "    return x.quantile(0.25)\n",
    "\n",
    "\n",
    "def q3(x):\n",
    "    return x.quantile(0.75)\n",
    "\n",
    "\n",
    "def mean_three_recent(x):\n",
    "    num_marks = min(3, len(x))\n",
    "    return np.mean(x.loc[-num_marks:])\n",
    "\n",
    "\n",
    "def mean_three_best(x):\n",
    "    num_marks = min(3, len(x))\n",
    "    return np.mean(sorted(x)[-num_marks])\n",
    "\n",
    "\n",
    "def mean_three_worst(x):\n",
    "    num_marks = min(3, len(x))\n",
    "    return np.mean(sorted(x)[:num_marks])\n",
    "\n",
    "\n",
    "def compute_statistics(df):\n",
    "\n",
    "    # Get rate of no marking then remove no-mark columns\n",
    "    nm_df = (\n",
    "        df.groupby(\"event\")\n",
    "        .agg({\"scaled_mark\": [no_mark_rate, \"count\"]})[\"scaled_mark\"]\n",
    "        .reset_index()\n",
    "    )\n",
    "\n",
    "    df[\"convert_mark\"] = df[\"convert_mark\"].astype(str)\n",
    "    df = df[~df[\"convert_mark\"].str.contains(\"[A-Z]+\", na=False, regex=True)]\n",
    "    df = df.astype({\"convert_mark\": \"float\", \"scaled_mark\": \"float\"})\n",
    "\n",
    "    # add feature PB this season\n",
    "    stats_df = (\n",
    "        df.groupby(\"event\")\n",
    "        .agg({\n",
    "            \"scaled_mark\": [\n",
    "                \"min\",\n",
    "                \"max\",\n",
    "                \"mean\",\n",
    "                \"std\",\n",
    "                \"median\",\n",
    "                q1,\n",
    "                q3,\n",
    "                mean_three_recent,\n",
    "                mean_three_best,\n",
    "                mean_three_worst,\n",
    "            ]\n",
    "        })[\"scaled_mark\"]\n",
    "        .reset_index()\n",
    "    )\n",
    "    stats_df[\"percent_diff_recent_best\"] = abs(stats_df[\"mean_three_recent\"] - stats_df[\"mean_three_best\"]) / ((stats_df[\"mean_three_recent\"] + stats_df[\"mean_three_best\"]) / 2)\n",
    "    stats_df[\"percent_diff_recent_worst\"] = abs(stats_df[\"mean_three_recent\"] - stats_df[\"mean_three_worst\"]) / ((stats_df[\"mean_three_recent\"] + stats_df[\"mean_three_worst\"]) / 2)\n",
    "\n",
    "    return stats_df.merge(nm_df, how=\"left\", on=\"event\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Local stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_athlete_pages(meet_url):\n",
    "    headers = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.102 Safari/537.36\"}\n",
    "    response = requests.get(meet_url, headers=headers)\n",
    "    results_html = response.text.replace('colspan=\"100%\"', 'colspan=\"3\"')\n",
    "\n",
    "    results_soup = BeautifulSoup(results_html, \"html.parser\")\n",
    "    athlete_pages = [link[\"href\"] for link in results_soup.select(\"a[href*=athletes]\")]\n",
    "\n",
    "    return set(athlete_pages)\n",
    "\n",
    "athlete_pages = get_athlete_pages(\"https://tfrrs.org/results/75224/m/NCAA_Division_I_Outdoor_Track__Field_Championships\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.tfrrs.org/athletes/6986872/Houston/Jordan_Booker.html\n",
      "https://www.tfrrs.org/athletes/7984404/Houston/Adam_Mason.html\n",
      "https://tfrrs.org/athletes/6543213/Sam_Houston/Clayton_Fritsch.html\n",
      "https://tfrrs.org/athletes/6547485/Wichita_State/Taran_Taylor.html\n",
      "https://tfrrs.org/athletes/7526270/Maryland/Caleb_Dean.html\n",
      "https://www.tfrrs.org/athletes/7019840/Texas_Tech/Jacolby_Shelton.html\n",
      "https://tfrrs.org/athletes/6996705/Wichita_State/Michael_Bryan.html\n",
      "https://tfrrs.org/athletes/7495800/Georgia/Matthew_Boling.html\n",
      "https://tfrrs.org/athletes/6547850/Houston/Quivell_Jordan-Bacot.html\n",
      "https://www.tfrrs.org/athletes/7495800/Georgia/Matthew_Boling.html\n",
      "https://tfrrs.org/athletes/7388209/Arizona_State/Vincent__Mauri.html\n"
     ]
    }
   ],
   "source": [
    "#for page in list(athlete_pages):\n",
    "for page in athlete_pages:\n",
    "    file_name = page.split(\"/\")[6][:-5]\n",
    "    with open(\"./data/\"+file_name, \"w\") as f:\n",
    "        f.write(get_html(page))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# from the web\n",
    "# try:\n",
    "#     for i, athlete_page in enumerate(athlete_pages):\n",
    "#         if (i % 20) == 0:\n",
    "#             print(i, athlete_page)\n",
    "#         html = get_html(athlete_page)\n",
    "#         athlete_info = get_athlete_info(html)\n",
    "# except:\n",
    "#     print(athlete_page)\n",
    "\n",
    "# local\n",
    "X = pd.DataFrame({\n",
    "    'name': pd.Series(dtype=\"str\"),\n",
    "    'place': pd.Series(dtype=\"int\"),\n",
    "    'all_american': pd.Series(dtype=\"int\"),\n",
    "    'event': pd.Series(dtype=\"str\"),\n",
    "    'min': pd.Series(dtype=\"float\"),\n",
    "    'max': pd.Series(dtype=\"float\"),\n",
    "    'mean': pd.Series(dtype=\"float\"),\n",
    "    'std': pd.Series(dtype=\"float\"),\n",
    "    'median': pd.Series(dtype=\"float\"),\n",
    "    'q1': pd.Series(dtype=\"float\"),\n",
    "    'q3': pd.Series(dtype=\"float\"),\n",
    "    'mean_three_recent': pd.Series(dtype=\"float\"),\n",
    "    'mean_three_best': pd.Series(dtype=\"float\"),\n",
    "    'mean_three_worst': pd.Series(dtype=\"float\"),\n",
    "    'percent_diff_recent_best': pd.Series(dtype=\"float\"),\n",
    "    'percent_diff_recent_worst': pd.Series(dtype=\"float\"),\n",
    "    'no_mark_rate': pd.Series(dtype=\"float\"),\n",
    "    'count': pd.Series(dtype=\"int\"),\n",
    "    'grade': pd.Series(dtype=\"str\"),\n",
    "    'year': pd.Series(dtype=\"int\"),\n",
    "    'school': pd.Series(dtype=\"str\")\n",
    "})\n",
    "\n",
    "try:\n",
    "    for page in os.listdir(\"./data/html/\"):\n",
    "        info_dict = get_athlete_info(\"./data/html/\"+page, local=True)\n",
    "        event_df = get_event_history(\"./data/html/\"+page, local=True)\n",
    "        stats_df = compute_statistics(event_df)\n",
    "        info_df = pd.DataFrame(info_dict, index=range(len(stats_df)))\n",
    "        full_df = pd.merge(all_athletes, stats_df.join(info_df), on=[\"name\", \"event\"], how=\"inner\")\n",
    "        X = pd.concat([X, full_df], ignore_index=True)\n",
    "except Exception as e:\n",
    "    print(e)\n",
    "    print(page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Default data highest correlations to target\n",
      "[0.2388225799065916, 0.21672901806617573, 0.19566722208845774, 0.1619775394304868, 0.1591473098431233]\n",
      "\n",
      "Races data highest correlations to target\n",
      "[0.45971676480878604, 0.43555160664427806, 0.39455926378968326, 0.29382988561881695, 0.2854967814823146]\n",
      "\n",
      "Jumps data highest correlations to target\n",
      "[0.4906239489924711, 0.4814069139830118, 0.4705346286953792, 0.4361764528515468, 0.43617645285154605]\n",
      "\n",
      "Throws data highest correlations to target\n",
      "[0.4928573172847412, 0.4918374039661947, 0.48898830502945273, 0.4242505003943985, 0.41122150858457834]\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_7619/1435582171.py:1: FutureWarning: The default value of numeric_only in DataFrame.corr is deprecated. In a future version, it will default to False. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
      "  X.drop([\"q1\", \"q3\"], axis=1).corr()\n",
      "/tmp/ipykernel_7619/1435582171.py:3: FutureWarning: The default value of numeric_only in DataFrame.corr is deprecated. In a future version, it will default to False. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
      "  print(sorted(X.corr()[\"all_american\"], reverse=True)[1:6], sep=\" \", end=\"\\n\\n\")\n",
      "/tmp/ipykernel_7619/1435582171.py:6: FutureWarning: The default value of numeric_only in DataFrame.corr is deprecated. In a future version, it will default to False. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
      "  print(sorted(X[X[\"event\"].isin(events[\"races\"])].corr()[\"all_american\"], reverse=True)[1:6], sep=\" \", end=\"\\n\\n\")\n",
      "/tmp/ipykernel_7619/1435582171.py:9: FutureWarning: The default value of numeric_only in DataFrame.corr is deprecated. In a future version, it will default to False. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
      "  print(sorted(X[X[\"event\"].isin(events[\"jumps\"])].corr()[\"all_american\"], reverse=True)[1:6], sep=\" \", end=\"\\n\\n\")\n",
      "/tmp/ipykernel_7619/1435582171.py:12: FutureWarning: The default value of numeric_only in DataFrame.corr is deprecated. In a future version, it will default to False. Select only valid columns or specify the value of numeric_only to silence this warning.\n",
      "  print(sorted(X[X[\"event\"].isin(events[\"throws\"])].corr()[\"all_american\"], reverse=True)[1:6], sep=\" \", end=\"\\n\\n\")\n"
     ]
    }
   ],
   "source": [
    "X.drop([\"q1\", \"q3\"], axis=1).corr()\n",
    "print(\"Default data highest correlations to target\")\n",
    "print(sorted(X.corr()[\"all_american\"], reverse=True)[1:6], sep=\" \", end=\"\\n\\n\")\n",
    "\n",
    "print(\"Races data highest correlations to target\")\n",
    "print(sorted(X[X[\"event\"].isin(events[\"races\"])].corr()[\"all_american\"], reverse=True)[1:6], sep=\" \", end=\"\\n\\n\")\n",
    "\n",
    "print(\"Jumps data highest correlations to target\")\n",
    "print(sorted(X[X[\"event\"].isin(events[\"jumps\"])].corr()[\"all_american\"], reverse=True)[1:6], sep=\" \", end=\"\\n\\n\")\n",
    "\n",
    "print(\"Throws data highest correlations to target\")\n",
    "print(sorted(X[X[\"event\"].isin(events[\"throws\"])].corr()[\"all_american\"], reverse=True)[1:6], sep=\" \", end=\"\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'OMAMUYOWI ERHIRE'}"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df_names.difference({x.replace(\"  \", \" \") for x in info_names})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>all_american</th>\n",
       "      <th>event</th>\n",
       "      <th>min</th>\n",
       "      <th>max</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>median</th>\n",
       "      <th>q1</th>\n",
       "      <th>q3</th>\n",
       "      <th>mean_three_recent</th>\n",
       "      <th>mean_three_best</th>\n",
       "      <th>mean_three_worst</th>\n",
       "      <th>percent_diff_recent_best</th>\n",
       "      <th>percent_diff_recent_worst</th>\n",
       "      <th>no_mark_rate</th>\n",
       "      <th>count</th>\n",
       "      <th>grade</th>\n",
       "      <th>year</th>\n",
       "      <th>school</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FAVOUR ASHE</td>\n",
       "      <td>1</td>\n",
       "      <td>100</td>\n",
       "      <td>0.964637</td>\n",
       "      <td>1.003064</td>\n",
       "      <td>0.97443</td>\n",
       "      <td>0.011425</td>\n",
       "      <td>0.971316</td>\n",
       "      <td>0.968442</td>\n",
       "      <td>0.974206</td>\n",
       "      <td>0.97443</td>\n",
       "      <td>0.974206</td>\n",
       "      <td>0.966855</td>\n",
       "      <td>0.00023</td>\n",
       "      <td>0.007804</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9</td>\n",
       "      <td>FR</td>\n",
       "      <td>1</td>\n",
       "      <td>TENNESSEE</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name  all_american event       min       max     mean       std  \\\n",
       "0  FAVOUR ASHE             1   100  0.964637  1.003064  0.97443  0.011425   \n",
       "\n",
       "     median        q1        q3  mean_three_recent  mean_three_best  \\\n",
       "0  0.971316  0.968442  0.974206            0.97443         0.974206   \n",
       "\n",
       "   mean_three_worst  percent_diff_recent_best  percent_diff_recent_worst  \\\n",
       "0          0.966855                   0.00023                   0.007804   \n",
       "\n",
       "   no_mark_rate  count grade  year     school  \n",
       "0           0.0      9    FR     1  TENNESSEE  "
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#test_html = get_html(\"https://www.tfrrs.org/athletes/7871291/Cornell/Rhys_Hammond.html\")\n",
    "test_html = \"./data/html/Favour_Ashe\"\n",
    "info_dict = get_athlete_info(test_html, local=True)\n",
    "event_df = get_event_history(test_html, local=True)\n",
    "stats_df = compute_statistics(event_df)\n",
    "info_df = pd.DataFrame(info_dict, index=range(len(stats_df)))\n",
    "full_df = pd.merge(all_athletes, stats_df.join(info_df), on=[\"name\", \"event\"], how=\"inner\")\n",
    "full_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"./data/processed/raw.csv\")\n",
    "df.head()\n",
    "df[\"score\"] = df[\"place\"].copy().apply(lambda x: 1-((x-1)/24))\n",
    "df.to_csv(\"./data/processed/raw.csv\", index=False)\n",
    "# EMMANUEL BAMIDELE should be 9th place but is 8th"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.6 ('tfrrs': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "429b51c6b84d21d7646721574e539afba211ecf3c8246758e7ac99ea26dd71d1"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}