Daily: refresh NJSP data, update www #1591
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Daily: refresh NJSP data, update www" | |
| on: | |
| schedule: | |
| - cron: "30 15 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| force: | |
| type: boolean | |
| description: "Force re-run all stages (ignore freshness)" | |
| default: false | |
| targets: | |
| type: string | |
| description: "Specific DVX targets to run (space-separated .dvc paths, empty = all)" | |
| concurrency: | |
| group: daily | |
| cancel-in-progress: false | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-east-1 | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} | |
| SLACK_CI_CHANNEL_ID: ${{ vars.SLACK_CI_CHANNEL_ID }} | |
| jobs: | |
| run: | |
| name: DVX pipeline | |
| runs-on: ubuntu-latest | |
| env: | |
| DVX: dvx run --commit --push each ${{ inputs.force == true && '--force' || '' }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| token: ${{ secrets.GH_TOKEN }} | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - name: Install Python dependencies | |
| run: | | |
| uv sync | |
| echo "$PWD/.venv/bin" >> $GITHUB_PATH | |
| - uses: pnpm/action-setup@v5 | |
| with: | |
| version: 10 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 22 | |
| cache: 'pnpm' | |
| cache-dependency-path: www/pnpm-lock.yaml | |
| - name: Install www dependencies | |
| run: cd www && pnpm install | |
| - name: Configure Git author | |
| run: | | |
| git config --global user.name 'GitHub Actions' | |
| git config --global user.email 'ryan-williams@users.noreply.github.com' | |
| - name: Pull DVC-tracked deps from S3 | |
| run: | | |
| dvx pull njdot/data/crashes.parquet | |
| dvx pull njdot/data/vehicles.parquet.dvc | |
| dvx pull njdot/data/occupants.parquet.dvc | |
| dvx pull njdot/data/pedestrians.parquet.dvc | |
| dvx pull njdot/data/*/NewJersey*Accidents.pqt.dvc | |
| dvx pull njdot/data/aashto_combined_crashes.parquet.dvc | |
| dvx pull njdot/data/aashto_supplemented_occupants.parquet.dvc | |
| dvx pull njdot/data/aashto_supplemented_pedestrians.parquet.dvc | |
| dvx pull njdot/data/aashto_supplemented_vehicles.parquet.dvc | |
| # Per-year AASHTO `persons.parquet` + `vehicles.parquet` are | |
| # referenced as file deps in the person / vehicle supplements; | |
| # without them on disk, DVX reports "dep missing" and `dvx run` | |
| # re-tries the (uncrunched) cmd. `vehicles.parquet` is a side- | |
| # output of the same `normalize.py` invocation that produces | |
| # `crashes.parquet` for these years. | |
| dvx pull njdot/data/2023/persons.parquet.dvc | |
| dvx pull njdot/data/2024/persons.parquet.dvc | |
| dvx pull njdot/data/2025/persons.parquet.dvc | |
| dvx pull njdot/data/2023/vehicles.parquet.dvc | |
| dvx pull njdot/data/2024/vehicles.parquet.dvc | |
| dvx pull njdot/data/2025/vehicles.parquet.dvc | |
| dvx pull njsp/data/crash-log.parquet.dvc | |
| # NJSP outputs of `njsp update_pqts` (consumed by harmonize, csvs, | |
| # projections, match, three-way-match, d1-import). | |
| dvx pull njsp/data/crashes.parquet.dvc | |
| dvx pull www/public/njsp/crashes.db.dvc | |
| # Side-effect output of `njsp match_njdot`; needed by supplement_with_sp | |
| dvx pull njsp/data/njsp_njdot_residuals.parquet.dvc | |
| # Downstream parquets/dbs — pre-stage so `is_output_fresh` sees them on | |
| # disk and reports `up-to-date` instead of "output missing" → re-execute. | |
| dvx pull njdot/data/aashto_supplemented_crashes.parquet.dvc | |
| dvx pull www/public/njdot/cmymc.db.dvc | |
| dvx pull www/public/data/njdot/ymccmc.dvc | |
| dvx pull www/public/data/njdot/ymccmcs.dvc | |
| dvx pull www/public/Municipal_Boundaries_of_NJ.geojson | |
| - name: DVX status | |
| run: dvx status | |
| # === Pipeline stages (sequential, matching old daily.yml order) === | |
| - name: Refresh NJSP data | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/refresh.dvc | |
| - name: Update NJSP parquets | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/crashes.parquet.dvc www/public/njsp/crashes.db.dvc | |
| - name: Harmonize county/muni codes | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/harmonize.dvc | |
| - name: Update crash log | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/crash-log.parquet.dvc | |
| - name: Refresh annual summaries | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/summaries.dvc | |
| - name: "Fetch ≈1yr of git history" | |
| if: ${{ !inputs.targets }} | |
| run: | | |
| since="$(date --date="$(date +%Y-%m-%d) -375 day" +%Y-%m-%d)" | |
| git fetch --shallow-since "$since" origin ${{ github.ref_name }} || true | |
| - name: Update NJSP projections | |
| if: ${{ !inputs.targets }} | |
| run: $DVX www/public/njsp/projections.dvc | |
| - name: Re-match NJSP↔NJDOT (incl. AASHTO) | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/njsp_njdot_match.parquet.dvc | |
| - name: Supplement AASHTO with NJSP-only residuals | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njdot/data/aashto_supplemented_crashes.parquet.dvc | |
| # Person supplements (occupants/pedestrians) are NOT rebuilt in CI: | |
| # their deps are per-year `persons.parquet` side-effects of | |
| # `normalize.py`, which only runs locally after a manual `Crash.csv` | |
| # download. CI pulls these from S3 above (`dvx pull`) and feeds | |
| # them to the VTC enrichment + cmymc stages. | |
| - name: Rebuild cmymc.db (year-stats, victim-severity) | |
| if: ${{ !inputs.targets }} | |
| run: $DVX www/public/njdot/cmymc.db.dvc | |
| - name: Aggregate NJDOT crashes for FE | |
| if: ${{ !inputs.targets }} | |
| run: $DVX www/public/data/njdot/ymccmc.dvc www/public/data/njdot/ymccmcs.dvc | |
| - name: Update www CSVs | |
| if: ${{ !inputs.targets }} | |
| run: $DVX www/public/njsp/csvs.dvc | |
| - name: Post to Slack | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/slack_post.dvc | |
| - name: Install api dependencies | |
| if: ${{ !inputs.targets }} | |
| run: cd api && pnpm install | |
| - name: Sync NJSP crashes.db → D1 | |
| if: ${{ !inputs.targets }} | |
| run: $DVX api/d1-import.dvc | |
| - name: Regenerate homepage OG mosaic (→ S3) | |
| if: ${{ !inputs.targets }} | |
| run: $DVX www/og-image.dvc | |
| - name: Deploy www to CF Pages | |
| if: ${{ !inputs.targets }} | |
| run: $DVX www/deploy.dvc | |
| - name: Run custom targets | |
| if: ${{ inputs.targets }} | |
| run: $DVX ${{ inputs.targets }} | |
| - name: "Notify #crash-bot-ci on failure" | |
| if: failure() | |
| env: | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| EVENT_NAME: ${{ github.event_name }} | |
| run: bash .github/scripts/notify-ci-failure.sh |