daily.yml

name: "Daily: refresh NJSP data, update www"
on:
  schedule:
    - cron: "30 15 * * *"
  workflow_dispatch:
    inputs:
      force:
        type: boolean
        description: "Force re-run all stages (ignore freshness)"
        default: false
      targets:
        type: string
        description: "Specific DVX targets to run (space-separated .dvc paths, empty = all)"
concurrency:
  group: daily
  cancel-in-progress: false
env:
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_DEFAULT_REGION: us-east-1
  GH_TOKEN: ${{ secrets.GH_TOKEN }}
  CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
  CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
  SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }}
  SLACK_CI_CHANNEL_ID: ${{ vars.SLACK_CI_CHANNEL_ID }}
jobs:
  run:
    name: DVX pipeline
    runs-on: ubuntu-latest
    env:
      DVX: dvx run --commit --push each ${{ inputs.force == true && '--force' || '' }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ github.ref_name }}
          token: ${{ secrets.GH_TOKEN }}
      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
      - name: Install Python dependencies
        run: |
          uv sync
          echo "$PWD/.venv/bin" >> $GITHUB_PATH
      - uses: pnpm/action-setup@v5
        with:
          version: 10
      - uses: actions/setup-node@v6
        with:
          node-version: 22
          cache: 'pnpm'
          cache-dependency-path: www/pnpm-lock.yaml
      - name: Install www dependencies
        run: cd www && pnpm install
      - name: Configure Git author
        run: |
          git config --global user.name 'GitHub Actions'
          git config --global user.email 'ryan-williams@users.noreply.github.com'
      - name: Pull DVC-tracked deps from S3
        run: |
          dvx pull njdot/data/crashes.parquet
          dvx pull njdot/data/vehicles.parquet.dvc
          dvx pull njdot/data/occupants.parquet.dvc
          dvx pull njdot/data/pedestrians.parquet.dvc
          dvx pull njdot/data/*/NewJersey*Accidents.pqt.dvc
          dvx pull njdot/data/aashto_combined_crashes.parquet.dvc
          dvx pull njdot/data/aashto_supplemented_occupants.parquet.dvc
          dvx pull njdot/data/aashto_supplemented_pedestrians.parquet.dvc
          dvx pull njdot/data/aashto_supplemented_vehicles.parquet.dvc
          # Per-year AASHTO `persons.parquet` + `vehicles.parquet` are
          # referenced as file deps in the person / vehicle supplements;
          # without them on disk, DVX reports "dep missing" and `dvx run`
          # re-tries the (uncrunched) cmd. `vehicles.parquet` is a side-
          # output of the same `normalize.py` invocation that produces
          # `crashes.parquet` for these years.
          dvx pull njdot/data/2023/persons.parquet.dvc
          dvx pull njdot/data/2024/persons.parquet.dvc
          dvx pull njdot/data/2025/persons.parquet.dvc
          dvx pull njdot/data/2023/vehicles.parquet.dvc
          dvx pull njdot/data/2024/vehicles.parquet.dvc
          dvx pull njdot/data/2025/vehicles.parquet.dvc
          # Per-year AASHTO `crashes.parquet` (produced by `normalize.py` from
          # the corresponding `Crash.csv`). Same rationale as the persons /
          # vehicles pulls above — the outputs on disk let DVX report
          # up-to-date without traversing into the per-year stage. And
          # `Crash.csv` itself must be pulled too, otherwise the stage's dep
          # is missing → DVX still tries to re-run.
          dvx pull njdot/data/2023/crashes.parquet.dvc
          dvx pull njdot/data/2024/crashes.parquet.dvc
          dvx pull njdot/data/2025/crashes.parquet.dvc
          dvx pull njdot/data/2023/Crash.csv.dvc
          dvx pull njdot/data/2024/Crash.csv.dvc
          dvx pull njdot/data/2025/Crash.csv.dvc
          dvx pull njsp/data/crash-log.parquet.dvc
          # NJSP outputs of `njsp update_pqts` (consumed by harmonize, csvs,
          # projections, match, three-way-match, d1-import).
          dvx pull njsp/data/crashes.parquet.dvc
          dvx pull www/public/njsp/crashes.db.dvc
          # Side-effect output of `njsp match_njdot`; needed by supplement_with_sp
          dvx pull njsp/data/njsp_njdot_residuals.parquet.dvc
          # Downstream parquets/dbs — pre-stage so `is_output_fresh` sees them on
          # disk and reports `up-to-date` instead of "output missing" → re-execute.
          dvx pull njdot/data/aashto_supplemented_crashes.parquet.dvc
          dvx pull www/public/njdot/cmymc.db.dvc
          dvx pull www/public/data/njdot/ymccmc.dvc
          dvx pull www/public/data/njdot/ymccmcs.dvc
          dvx pull www/public/Municipal_Boundaries_of_NJ.geojson
      - name: DVX status
        run: dvx status
      # === Pipeline stages (sequential, matching old daily.yml order) ===
      - name: Refresh NJSP data
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/refresh.dvc
      - name: Update NJSP parquets
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/crashes.parquet.dvc www/public/njsp/crashes.db.dvc
      - name: Harmonize county/muni codes
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/harmonize.dvc
      - name: Update crash log
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/crash-log.parquet.dvc
      - name: Refresh annual summaries
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/summaries.dvc
      - name: "Fetch ≈1yr of git history"
        if: ${{ !inputs.targets }}
        run: |
          since="$(date --date="$(date +%Y-%m-%d) -375 day" +%Y-%m-%d)"
          git fetch --shallow-since "$since" origin ${{ github.ref_name }} || true
      - name: Update NJSP projections
        if: ${{ !inputs.targets }}
        run: $DVX www/public/njsp/projections.dvc
      - name: Re-match NJSP↔NJDOT (incl. AASHTO)
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/njsp_njdot_match.parquet.dvc
      - name: Supplement AASHTO with NJSP-only residuals
        if: ${{ !inputs.targets }}
        run: $DVX njdot/data/aashto_supplemented_crashes.parquet.dvc
      # Person supplements (occupants/pedestrians) are NOT rebuilt in CI:
      # their deps are per-year `persons.parquet` side-effects of
      # `normalize.py`, which only runs locally after a manual `Crash.csv`
      # download. CI pulls these from S3 above (`dvx pull`) and feeds
      # them to the VTC enrichment + cmymc stages.
      - name: Rebuild cmymc.db (year-stats, victim-severity)
        if: ${{ !inputs.targets }}
        run: $DVX www/public/njdot/cmymc.db.dvc
      - name: Aggregate NJDOT crashes for FE
        if: ${{ !inputs.targets }}
        run: $DVX www/public/data/njdot/ymccmc.dvc www/public/data/njdot/ymccmcs.dvc
      - name: Update www CSVs
        if: ${{ !inputs.targets }}
        run: $DVX www/public/njsp/csvs.dvc
      - name: Post to Slack
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/slack_post.dvc
      # Cells refresh for the map. DVX-gated: the raw H3 index + rollup DB only
      # rebuild on days the crash data actually changed. `cells.db` (the D1
      # rollup source) is built straight from the raw index via the duckdb `h3`
      # extension — no pyramid — so it stays memory-bounded and feeds the
      # d1-import below. The parquet pyramid (for year/severity-filtered queries)
      # and its R2 push are deferred: the pyramid build OOMs on the CI runner,
      # and the push needs R2 creds (see 4b).
      - name: Rebuild cells raw H3 index
        if: ${{ !inputs.targets }}
        run: $DVX data/cells/raw/h3_r15.dvc
      - name: Rebuild cells rollup DB (D1 source)
        if: ${{ !inputs.targets }}
        run: $DVX data/cells/cells.db.dvc
      - name: Install api dependencies
        if: ${{ !inputs.targets }}
        run: cd api && pnpm install
      - name: Sync crash DBs (NJSP, cmymc, cells) → D1
        if: ${{ !inputs.targets }}
        run: $DVX api/d1-import.dvc
      - name: Regenerate homepage OG mosaic (→ S3)
        if: ${{ !inputs.targets }}
        run: $DVX www/og-image.dvc
      - name: Deploy www to CF Pages
        if: ${{ !inputs.targets }}
        run: $DVX www/deploy.dvc
      - name: Run custom targets
        if: ${{ inputs.targets }}
        run: $DVX ${{ inputs.targets }}
      - name: "Notify #crash-bot-ci on failure"
        if: failure()
        env:
          GH_TOKEN: ${{ secrets.GH_TOKEN }}
          EVENT_NAME: ${{ github.event_name }}
        run: bash .github/scripts/notify-ci-failure.sh