Fix baseball databank schemas, retrosheet data, add latest tags to build (#53)

droher · web-flow · commit 8a9adf423ed5 · 2020-05-22T20:49:02.000-04:00
* Fix schema issues

* Fix PK/charlen schema issues

* workin

* style

* tab

* add build time window

* bump

* separate build from up

* fix parallelism

* just build for now
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -35,12 +35,7 @@ jobs:
             environment:
                 BUILD_ENV: test
             command: |
-              docker-compose up -d
-              sleep 60s
-              total_exits=$(docker-compose ps -q | wc -l)
-              clean_exits=$(docker-compose ps -q | xargs docker inspect -f '{{ .State.ExitCode }}' | grep -c 0)
-              echo "Total containers: ${total_exits} Clean exits: ${clean_exits}"
-              ((${clean_exits}==${total_exits})) || (echo "At least one container had a non-zero exit" && exit 1);
+              docker-compose build
 
 
 # Orchestrate or schedule a set of jobs, see https://circleci.com/docs/2.0/workflows/
diff --git a/.env b/.env
@@ -1,7 +1,7 @@
 CHADWICK_VERSION=v0.7.2
 BASEBALLDATABANK_VERSION=bb19ecb78e6be5764da32497ec165eb0aaab66a9
-RETROSHEET_VERSION=0cd5f717c1ea5979a6eca43dff6ac543e155111c
+RETROSHEET_VERSION=75fe03a53e2add11441d1f012401e1aef299cf03
 
 EXTRACT_DIR=extract
 REPO=doublewick/boxball
-VERSION=2020.0.0
+VERSION=2020.0.1
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,111 +1,179 @@
 version: '3.7'
+x-extract:
+  &extract
+  build:
+    context: extract
+    args:
+      - CHADWICK_VERSION
+      - RETROSHEET_VERSION
+      - BASEBALLDATABANK_VERSION
+      - BUILD_ENV
+  image: ${REPO}:extract-${VERSION}
+x-ddl:
+  &ddl
+  build:
+    context: transform
+    dockerfile: ddl.Dockerfile
+    args:
+      - VERSION
+  image: ${REPO}:ddl-${VERSION}
+x-parquet:
+  &parquet
+  build:
+    context: transform
+    dockerfile: parquet.Dockerfile
+    args:
+      - VERSION
+  image: ${REPO}:parquet-${VERSION}
+  depends_on:
+    - extract
+x-csv:
+  &csv
+  build:
+    context: transform
+    dockerfile: csv.Dockerfile
+    args:
+      - VERSION
+  image: ${REPO}:csv-${VERSION}
+  depends_on:
+    - extract
+
+x-clickhouse:
+  &clickhouse
+  build:
+    context: load/clickhouse
+    args:
+      - VERSION
+  image: ${REPO}:clickhouse-${VERSION}
+  volumes:
+    - ~/boxball/clickhouse:/var/lib/clickhouse
+  depends_on:
+    - parquet
+    - ddl
+
+x-drill:
+  &drill
+  build:
+    context: load/drill
+    args:
+      - VERSION
+  image: ${REPO}:drill-${VERSION}
+  volumes:
+    - ~/boxball/drill:/data
+  depends_on:
+    - parquet
+    - ddl
+
+x-postgres:
+  &postgres
+  build:
+    context: load/postgres
+    args:
+      - VERSION
+  image: ${REPO}:postgres-${VERSION}
+  volumes:
+    - ~/boxball/postgres:/var/lib/postgresql/data
+  depends_on:
+    - csv
+    - ddl
+
+x-postgres-cstore-fdw:
+  &postgres-cstore-fdw
+  build:
+    context: load/postgres_cstore_fdw
+    args:
+      - VERSION
+  image: ${REPO}:postgres-cstore-fdw-${VERSION}
+  volumes:
+    - ~/boxball/postgres-cstore-fdw:/var/lib/postgresql/data
+  depends_on:
+    - csv
+    - ddl
+
+x-mysql:
+  &mysql
+  build:
+    context: load/mysql
+    args:
+      - VERSION
+  image: ${REPO}:mysql-${VERSION}
+  volumes:
+    - ~/boxball/mysql:/var/lib/mysql
+  depends_on:
+    - csv
+    - ddl
+
+x-sqlite:
+  &sqlite
+  build:
+    context: load/sqlite
+    args:
+      - VERSION
+  image: ${REPO}:sqlite-${VERSION}
+  volumes:
+    - ~/boxball/sqlite:/db
+  depends_on:
+    - csv
+    - ddl
+
+
 services:
   extract:
-    build:
-      context: extract
-      args:
-        - CHADWICK_VERSION
-        - RETROSHEET_VERSION
-        - BASEBALLDATABANK_VERSION
-        - BUILD_ENV
-    image: ${REPO}:extract-${VERSION}
+    << : *extract
+  extract-latest:
+    << : *extract
+    image: ${REPO}:extract-latest
 
   ddl:
-    build:
-      context: transform
-      dockerfile: ddl.Dockerfile
-      args:
-        - VERSION
-    image: ${REPO}:ddl-${VERSION}
+    << : *ddl
+  ddl-latest:
+    << : *ddl
+    image: ${REPO}:ddl-latest
 
   parquet:
-    build:
-      context: transform
-      dockerfile: parquet.Dockerfile
-      args:
-        - VERSION
-    image: ${REPO}:parquet-${VERSION}
-    depends_on:
-      - extract
+    << : *parquet
+  parquet-latest:
+    << : *parquet
+    image: ${REPO}:parquet-latest
 
   csv:
-    build:
-      context: transform
-      dockerfile: csv.Dockerfile
-      args:
-        - VERSION
-    image: ${REPO}:csv-${VERSION}
-    depends_on:
-      - extract
+    << : *csv
+  csv-latest:
+    << : *csv
+    image: ${REPO}:csv-latest
 
   clickhouse:
-    build:
-      context: load/clickhouse
-      args:
-        - VERSION
-    image: ${REPO}:clickhouse-${VERSION}
-    volumes:
-      - ~/boxball/clickhouse:/var/lib/clickhouse
-    depends_on:
-      - parquet
-      - ddl
+    << : *clickhouse
+  clickhouse-latest:
+    << : *clickhouse
+    image: ${REPO}:clickhouse-latest
 
   drill:
-    build:
-      context: load/drill
-      args:
-        - VERSION
-    image: ${REPO}:drill-${VERSION}
-    volumes:
-      - ~/boxball/drill:/data
-    depends_on:
-      - parquet
-      - ddl
+    << : *drill
+  drill-latest:
+    << : *drill
+    image: ${REPO}:drill-latest
 
   postgres:
-    build:
-      context: load/postgres
-      args:
-        - VERSION
-    image: ${REPO}:postgres-${VERSION}
-    volumes:
-      - ~/boxball/postgres:/var/lib/postgresql/data
-    depends_on:
-      - csv
-      - ddl
+    << : *postgres
+  postgres-latest:
+    << : *postgres
+    image: ${REPO}:postgres-latest
 
   postgres-cstore-fdw:
-    build:
-      context: load/postgres_cstore_fdw
-      args:
-        - VERSION
-    image: ${REPO}:postgres-cstore-fdw-${VERSION}
-    volumes:
-      - ~/boxball/postgres-cstore-fdw:/var/lib/postgresql/data
-    depends_on:
-      - csv
-      - ddl
+    << : *postgres-cstore-fdw
+  postgres-cstore-fdw-latest:
+    << : *postgres-cstore-fdw
+    image: ${REPO}:postgres-cstore-fdw-latest
 
   mysql:
-    build:
-      context: load/mysql
-      args:
-        - VERSION
-    image: ${REPO}:mysql-${VERSION}
-    volumes:
-      - ~/boxball/mysql:/var/lib/mysql
-    depends_on:
-      - csv
-      - ddl
+    << : *mysql
+  mysql-latest:
+    << : *mysql
+    image: ${REPO}:mysql-latest
 
   sqlite:
-    build:
-      context: load/sqlite
-      args:
-        - VERSION
-    image: ${REPO}:sqlite-${VERSION}
-    volumes:
-      - ~/boxball/sqlite:/db
-    depends_on:
-      - csv
-      - ddl
+    << : *sqlite
+  sqlite-latest:
+    << : *sqlite
+    image: ${REPO}:sqlite-latest
diff --git a/extract/Dockerfile b/extract/Dockerfile
@@ -22,7 +22,7 @@ ENV PYTHONPATH="/"
 # `prod` gets the full datasets, while `test` provides fixtures with small sample data for each file
 FROM build-common as get-retrosheet-prod
 ARG RETROSHEET_VERSION
-RUN wget https://github.com/chadwickbureau/retrosheet/archive/${RETROSHEET_VERSION}.zip -O retrosheet.zip
+RUN wget https://github.com/droher/retrosheet/archive/${RETROSHEET_VERSION}.zip -O retrosheet.zip
 
 FROM build-common as get-retrosheet-test
 COPY fixtures/raw/retrosheet.zip .
diff --git a/transform/src/schemas/baseballdatabank.py b/transform/src/schemas/baseballdatabank.py
@@ -8,14 +8,17 @@
 class AllstarFull(Base):
     __tablename__ = 'allstar_full'
 
-    player_id = Column(String(9), primary_key=True, nullable=False)
-    year_id = Column(SmallInteger, primary_key=True, nullable=False)
-    game_num = Column(SmallInteger, primary_key=True, nullable=False)
+    player_id = Column(String(9), nullable=False)
+    # Should be non-nullable, see https://github.com/chadwickbureau/baseballdatabank/issues/105
+    year_id = Column(SmallInteger)
+    game_num = Column(SmallInteger)
     game_id = Column(String(12))
     team_id = Column(String(3))
     lg_id = Column(String(2))
     gp = Column(SmallInteger)
     starting_pos = Column(SmallInteger)
+    # Note -- Billy Herman's 1934 record prevents us from using the true PK, player-year-gamenum
+    dummy_id = Column(Integer, autoincrement=True, primary_key=True)
 
 
 class Appearance(Base):
@@ -47,12 +50,14 @@ class Appearance(Base):
 class AwardsManager(Base):
     __tablename__ = 'awards_managers'
 
-    player_id = Column(String(10), primary_key=True, nullable=False)
-    award_id = Column(String(75), primary_key=True, nullable=False)
-    year_id = Column(SmallInteger, primary_key=True, nullable=False)
-    lg_id = Column(String(2), primary_key=True, nullable=False)
+    player_id = Column(String(10), nullable=False)
+    award_id = Column(String(75), nullable=False)
+    year_id = Column(SmallInteger, nullable=False)
+    lg_id = Column(String(2))
     tie = Column(String(1))
     notes = Column(String(100))
+    # PK should be player/award/year/lg, see https://github.com/chadwickbureau/baseballdatabank/issues/105
+    dummy_id = Column(Integer, autoincrement=True, primary_key=True)
 
 
 class AwardsPlayer(Base):
@@ -298,7 +303,7 @@ class Parks(Base):
 
     park_id = Column(String(5), primary_key=True, nullable=False)
     park_name = Column(String(40))
-    park_alias = Column(String(45))
+    park_alias = Column(String(55))
     city = Column(String(25))
     state = Column(String(16))
     country = Column(String(2))
diff --git a/transform/src/schemas/retrosheet.py b/transform/src/schemas/retrosheet.py
@@ -854,7 +854,7 @@ class Park(Base):
 
     park_id = Column(CHAR(5), primary_key=True, doc="Park ID")
     name = Column(String(41), doc="Park name")
-    aka = Column(String(40), doc="Common park alias")
+    aka = Column(String(55), doc="Common park alias")
     city = Column(String(17), doc="City")
     state = Column(String(9), doc="State")
     # TODO: Handle this MySQL edge case so these can be dates again
@@ -898,7 +898,7 @@ class Schedule(Base):
     home_team_league = Column(CHAR(2), doc="Home team league ID")
     home_team_game_number = Column(Integer, primary_key=True, doc="Home team game number")
     day_night = Column(CHAR(1), doc="D - day, N - night")
-    postponement_indicator = Column(String(30), doc="""
+    postponement_indicator = Column(String(120), doc="""
         This field will contain one or more phrases related to the game if it was
         not played as scheduled. If there is more than one phrase, they are separated
         by a semi-colon (";"). There are three possible outcomes for games not played
@@ -907,7 +907,7 @@ class Schedule(Base):
         -- The game was played on the original date but at another site
         -- The game was not played
         """)
-    makeup_dates = Column(String(20), doc="""
+    makeup_dates = Column(String(120), doc="""
         This field will contain a makeup date if the postponed game was played at
         another time or place. If an attempt was known to have been made on a date but
         postponed again, that date will be listed. In that case, there will be a second
