fix [home|away]_turnovers

nfldb/types.py

@@ -35,35 +35,24 @@ def _stat_categories():
 
 
 def _nflgame_start_time(schedule):
-    """
-    Given an entry in `nflgame.schedule`, return the start time of the
-    game in UTC.
-    """
-    # Hack to get around ambiugous times for weird London games.
-    if schedule['eid'] == '2015100400':
-        d = datetime.datetime(2015, 10, 4, 9, 30)
-        return pytz.timezone('US/Eastern').localize(d).astimezone(pytz.utc)
-    elif schedule['eid'] == '2015102500':
-        d = datetime.datetime(2015, 10, 25, 9, 30)
-        return pytz.timezone('US/Eastern').localize(d).astimezone(pytz.utc)
-    elif schedule['eid'] == '2015110100':
-        d = datetime.datetime(2015, 11, 1, 9, 30)
-        return pytz.timezone('US/Eastern').localize(d).astimezone(pytz.utc)
-
     # Year is always the season, so we bump it if the month is Jan-March.
     year, month, day = schedule['year'], schedule['month'], schedule['day']
     if 1 <= schedule['month'] <= 3:
         year += 1
 
-    # BUG: Getting the hour here will be wrong if a game starts before Noon
-    # EST. Not sure what to do about it...
     hour, minute = schedule['time'].strip().split(':')
-    minute = int(minute)
-    if hour == '12':
-        hour = 12
-    else:
-        hour = (int(hour) + 12) % 24
-    d = datetime.datetime(year, month, day, hour, minute)
+
+    # Format year, month, day, hour, minute for use with strptime
+    year = str(year)
+    month = str(month).zfill(2)
+    day = str(day).zfill(2)
+    hour = hour.zfill(2)
+    minute = minute.zfill(2)
+    meridiem = schedule['meridiem'] if 'meridiem' in schedule and schedule['meridiem'] else 'PM'
+
+    s = "{0}-{1}-{2} {3}:{4} {5}".format(year, month, day, hour, minute, meridiem)
+    d = datetime.datetime.strptime(s, '%Y-%m-%d %I:%M %p')
+
     return pytz.timezone('US/Eastern').localize(d).astimezone(pytz.utc)
 
 
@@ -2110,15 +2099,21 @@ def _from_nflgame(db, g):
         dbg.home_score_q3 = g.score_home_q3
         dbg.home_score_q4 = g.score_home_q4
         dbg.home_score_q5 = g.score_home_q5
-        dbg.home_turnovers = int(g.data['home']['to'])
+        try:
+            dbg.home_turnovers = int(g.data['home']['stats']['team']['trnovr'])
+        except KeyError:
+            dbg.home_turnovers = 0
         dbg.away_team = nfldb.team.standard_team(g.away)
         dbg.away_score = g.score_away
         dbg.away_score_q1 = g.score_away_q1
         dbg.away_score_q2 = g.score_away_q2
         dbg.away_score_q3 = g.score_away_q3
         dbg.away_score_q4 = g.score_away_q4
         dbg.away_score_q5 = g.score_away_q5
-        dbg.away_turnovers = int(g.data['away']['to'])
+        try:
+            dbg.away_turnovers = int(g.data['away']['stats']['team']['trnovr'])
+        except KeyError:
+            dbg.away_turnovers = 0
 
         # If it's been 8 hours since game start, we always conclude finished!
         if (now() - dbg.start_time).total_seconds() >= (60 * 60 * 8):

nfldb/update.py

@@ -293,6 +293,42 @@ def games_missing(cursor):
     return sorted(nada, key=int)
 
 
+def update_game_turnovers(db, since):
+    """
+    Updates the turnover data of every game in the database.
+    """
+    log('Updating all game turnovers... ', end='')
+
+    with nfldb.Tx(db) as cursor:
+        cursor.execute('SELECT MAX(season_year) as max, MIN(season_year) as min from game')
+        start_year = None
+        stop_year = None
+        for row in cursor.fetchall():
+            start_year = row['min']
+            stop_year = row['max']
+        if not start_year:
+            return
+
+        lock_tables(cursor)
+        cursor.execute("SET TIME ZONE 'UTC'")
+        for year in range(start_year, stop_year+1):
+            games = nflgame.games(year)
+            for game in games:
+                dbg = nfldb.Game.from_id(db,game.eid)
+                try:
+                    home = int(game.data['home']['stats']['team']['trnovr'])
+                except KeyError:
+                    home = 0
+                try:
+                    away = int(game.data['away']['stats']['team']['trnovr'])
+                except KeyError:
+                    away = 0
+                if home != db.home_turnovers or away != db.away_turnovers:
+                    dbg.home_turnovers = home
+                    dbg.away_turnovers = away
+                    dbg._save(cursor)
+
+
 def update_game_schedules(db):
     """
     Updates the schedule data of every game in the database.
@@ -442,10 +478,12 @@ def lock_tables(cursor):
 
 
 def run(player_interval=43200, interval=None, update_schedules=False,
-        batch_size=5, simulate=None):
+        batch_size=5, simulate=None, update_turnovers=None):
     global _simulate
 
     if simulate is not None:
+        assert not update_turnovers, \
+            "update_turnovers is incompatible with simulate"
         assert not update_schedules, \
             "update_schedules is incompatible with simulate"
 
@@ -507,7 +545,9 @@ def doit():
         nfldb.set_timezone(db, 'UTC')
         log('done.')
 
-        if update_schedules:
+        if update_turnovers:
+            update_game_turnovers(db, update_turnovers)
+        elif update_schedules:
             update_game_schedules(db)
         elif simulate is not None:
             done = update_simulate(db)

scripts/nfldb-update

@@ -34,6 +34,9 @@ if __name__ == '__main__':
             'e.g., A batch size of 150 seems to work well when building the '
             'database from scratch.')
     aa('--simulate', nargs='+', default=None)
+    aa('--update-turnovers', action='store_true',
+       help='When set, ALL games will have away_turnovers and home_turnovers '
+        'updated with the correct values from NFL.com')
     args = parser.parse_args()
 
     nfldb.update.run(**vars(args))

tests/test_query.py

@@ -18,6 +18,14 @@ def qgame(q):
     return q.game(gsis_id='2013090800')
 
 
+def test_num_turnovers(db):
+    import nfldb.update
+    with nfldb.Tx(db) as cursor:
+        g = nfldb.update.game_from_id(cursor, '2013090800')
+        assert g.home_turnovers == 2
+        assert g.away_turnovers == 3
+
+
 def test_num_games_in_season(q):
     assert len(q.as_games()) == (16 * 32) / 2