Daily: refresh NJSP data, update www #1500
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Daily: refresh NJSP data, update www" | |
| on: | |
| schedule: | |
| - cron: "30 15 * * *" | |
| workflow_dispatch: | |
| inputs: | |
| force: | |
| type: boolean | |
| description: "Force re-run all stages (ignore freshness)" | |
| default: false | |
| targets: | |
| type: string | |
| description: "Specific DVX targets to run (space-separated .dvc paths, empty = all)" | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-east-1 | |
| GH_TOKEN: ${{ secrets.GH_TOKEN }} | |
| CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} | |
| CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| SLACK_CHANNEL_ID: ${{ secrets.SLACK_CHANNEL_ID }} | |
| jobs: | |
| run: | |
| name: DVX pipeline | |
| runs-on: ubuntu-latest | |
| env: | |
| DVX: dvx run --commit --push each ${{ inputs.force == true && '--force' || '' }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.ref_name }} | |
| token: ${{ secrets.GH_TOKEN }} | |
| - uses: astral-sh/setup-uv@v7 | |
| with: | |
| enable-cache: true | |
| - name: Install Python dependencies | |
| run: | | |
| uv sync | |
| echo "$PWD/.venv/bin" >> $GITHUB_PATH | |
| - uses: pnpm/action-setup@v5 | |
| with: | |
| version: 10 | |
| - uses: actions/setup-node@v6 | |
| with: | |
| node-version: 22 | |
| cache: 'pnpm' | |
| cache-dependency-path: www/pnpm-lock.yaml | |
| - name: Install www dependencies | |
| run: cd www && pnpm install | |
| - name: Configure Git author | |
| run: | | |
| git config --global user.name 'GitHub Actions' | |
| git config --global user.email 'ryan-williams@users.noreply.github.com' | |
| - name: Pull DVC-tracked deps from S3 | |
| run: | | |
| dvx pull njdot/data/crashes.parquet | |
| dvx pull njdot/data/*/NewJersey*Accidents.pqt.dvc | |
| dvx pull njsp/data/crash-log.parquet.dvc | |
| dvx pull www/public/Municipal_Boundaries_of_NJ.geojson | |
| - name: DVX status | |
| run: dvx status | |
| # === Pipeline stages (sequential, matching old daily.yml order) === | |
| - name: Refresh NJSP data | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/refresh.dvc | |
| - name: Update NJSP parquets | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/update_pqts.dvc | |
| - name: Harmonize county/muni codes | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/harmonize.dvc | |
| - name: Update crash log | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/crash-log.parquet.dvc | |
| - name: Refresh annual summaries | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/summaries.dvc | |
| - name: "Fetch ≈1yr of git history" | |
| if: ${{ !inputs.targets }} | |
| run: | | |
| since="$(date --date="$(date +%Y-%m-%d) -375 day" +%Y-%m-%d)" | |
| git fetch --shallow-since "$since" origin ${{ github.ref_name }} || true | |
| - name: Update NJSP projections | |
| if: ${{ !inputs.targets }} | |
| run: $DVX www/public/njsp/projections.dvc | |
| - name: Update www CSVs | |
| if: ${{ !inputs.targets }} | |
| run: $DVX www/public/njsp/csvs.dvc | |
| - name: Post to Slack | |
| if: ${{ !inputs.targets }} | |
| run: $DVX njsp/data/slack_post.dvc | |
| - name: Deploy www to CF Pages | |
| if: ${{ !inputs.targets }} | |
| run: $DVX www/deploy.dvc | |
| - name: Run custom targets | |
| if: ${{ inputs.targets }} | |
| run: $DVX ${{ inputs.targets }} |