daily.yml

name: "Daily: refresh NJSP data, update www"
on:
  schedule:
    - cron: "30 15 * * *"
  workflow_dispatch:
    inputs:
      force:
        type: boolean
        description: "Force re-run all stages (ignore freshness)"
        default: false
      targets:
        type: string
        description: "Specific DVX targets to run (space-separated .dvc paths, empty = all)"
concurrency:
  group: daily
  cancel-in-progress: false
env:
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AWS_DEFAULT_REGION: us-east-1
  GH_TOKEN: ${{ secrets.GH_TOKEN }}
  CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
  CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
  SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
  SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }}
jobs:
  run:
    name: DVX pipeline
    runs-on: ubuntu-latest
    env:
      DVX: dvx run --commit --push each ${{ inputs.force == true && '--force' || '' }}
    steps:
      - uses: actions/checkout@v6
        with:
          ref: ${{ github.ref_name }}
          token: ${{ secrets.GH_TOKEN }}
      - uses: astral-sh/setup-uv@v7
        with:
          enable-cache: true
      - name: Install Python dependencies
        run: |
          uv sync
          echo "$PWD/.venv/bin" >> $GITHUB_PATH
      - uses: pnpm/action-setup@v5
        with:
          version: 10
      - uses: actions/setup-node@v6
        with:
          node-version: 22
          cache: 'pnpm'
          cache-dependency-path: www/pnpm-lock.yaml
      - name: Install www dependencies
        run: cd www && pnpm install
      - name: Configure Git author
        run: |
          git config --global user.name 'GitHub Actions'
          git config --global user.email 'ryan-williams@users.noreply.github.com'
      - name: Pull DVC-tracked deps from S3
        run: |
          dvx pull njdot/data/crashes.parquet
          dvx pull njdot/data/vehicles.parquet.dvc
          dvx pull njdot/data/occupants.parquet.dvc
          dvx pull njdot/data/pedestrians.parquet.dvc
          dvx pull njdot/data/*/NewJersey*Accidents.pqt.dvc
          dvx pull njdot/data/aashto_combined_crashes.parquet.dvc
          dvx pull njdot/data/aashto_supplemented_occupants.parquet.dvc
          dvx pull njdot/data/aashto_supplemented_pedestrians.parquet.dvc
          dvx pull njdot/data/aashto_supplemented_vehicles.parquet.dvc
          # Per-year AASHTO `persons.parquet` + `vehicles.parquet` are
          # referenced as file deps in the person / vehicle supplements;
          # without them on disk, DVX reports "dep missing" and `dvx run`
          # re-tries the (uncrunched) cmd. `vehicles.parquet` is a side-
          # output of the same `normalize.py` invocation that produces
          # `crashes.parquet` for these years.
          dvx pull njdot/data/2023/persons.parquet.dvc
          dvx pull njdot/data/2024/persons.parquet.dvc
          dvx pull njdot/data/2025/persons.parquet.dvc
          dvx pull njdot/data/2023/vehicles.parquet.dvc
          dvx pull njdot/data/2024/vehicles.parquet.dvc
          dvx pull njdot/data/2025/vehicles.parquet.dvc
          dvx pull njsp/data/crash-log.parquet.dvc
          # NJSP outputs of `njsp update_pqts` (consumed by harmonize, csvs,
          # projections, match, three-way-match, d1-import).
          dvx pull njsp/data/crashes.parquet.dvc
          dvx pull www/public/njsp/crashes.db.dvc
          # Side-effect output of `njsp match_njdot`; needed by supplement_with_sp
          dvx pull njsp/data/njsp_njdot_residuals.parquet.dvc
          dvx pull www/public/Municipal_Boundaries_of_NJ.geojson
      - name: DVX status
        run: dvx status
      # === Pipeline stages (sequential, matching old daily.yml order) ===
      - name: Refresh NJSP data
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/refresh.dvc
      - name: Update NJSP parquets
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/crashes.parquet.dvc www/public/njsp/crashes.db.dvc
      - name: Harmonize county/muni codes
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/harmonize.dvc
      - name: Update crash log
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/crash-log.parquet.dvc
      - name: Refresh annual summaries
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/summaries.dvc
      - name: "Fetch ≈1yr of git history"
        if: ${{ !inputs.targets }}
        run: |
          since="$(date --date="$(date +%Y-%m-%d) -375 day" +%Y-%m-%d)"
          git fetch --shallow-since "$since" origin ${{ github.ref_name }} || true
      - name: Update NJSP projections
        if: ${{ !inputs.targets }}
        run: $DVX www/public/njsp/projections.dvc
      - name: Re-match NJSP↔NJDOT (incl. AASHTO)
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/njsp_njdot_match.parquet.dvc
      - name: Supplement AASHTO with NJSP-only residuals
        if: ${{ !inputs.targets }}
        run: $DVX njdot/data/aashto_supplemented_crashes.parquet.dvc
      # Person supplements (occupants/pedestrians) are NOT rebuilt in CI:
      # their deps are per-year `persons.parquet` side-effects of
      # `normalize.py`, which only runs locally after a manual `Crash.csv`
      # download. CI pulls these from S3 above (`dvx pull`) and feeds
      # them to the VTC enrichment + cmymc stages.
      - name: Rebuild cmymc.db (year-stats, victim-severity)
        if: ${{ !inputs.targets }}
        run: $DVX www/public/njdot/cmymc.db.dvc
      - name: Aggregate NJDOT crashes for FE
        if: ${{ !inputs.targets }}
        run: $DVX www/public/data/njdot/ymccmc.dvc www/public/data/njdot/ymccmcs.dvc
      - name: Update www CSVs
        if: ${{ !inputs.targets }}
        run: $DVX www/public/njsp/csvs.dvc
      - name: Post to Slack
        if: ${{ !inputs.targets }}
        run: $DVX njsp/data/slack_post.dvc
      - name: Install api dependencies
        if: ${{ !inputs.targets }}
        run: cd api && pnpm install
      - name: Sync NJSP crashes.db → D1
        if: ${{ !inputs.targets }}
        run: $DVX api/d1-import.dvc
      - name: Regenerate homepage OG mosaic (→ S3)
        if: ${{ !inputs.targets }}
        run: $DVX www/og-image.dvc
      - name: Deploy www to CF Pages
        if: ${{ !inputs.targets }}
        run: $DVX www/deploy.dvc
      - name: Run custom targets
        if: ${{ inputs.targets }}
        run: $DVX ${{ inputs.targets }}