Sync tripdata bucket #483
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: 'Sync tripdata bucket' | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 9 * * *' | |
| permissions: | |
| contents: write | |
| env: | |
| AWS_DEFAULT_REGION: us-east-1 | |
| jobs: | |
| sync: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v5 | |
| - uses: astral-sh/setup-uv@v6 | |
| with: | |
| python-version: '3.12' | |
| enable-cache: true | |
| - name: Sync deps + activate venv | |
| run: | | |
| uv sync --frozen | |
| echo "$GITHUB_WORKSPACE/.venv/bin" >> "$GITHUB_PATH" | |
| - name: Set Git author info | |
| run: | | |
| git config --global user.name 'GitHub Actions' | |
| git config --global user.email 'ryan-williams@users.noreply.github.com' | |
| - name: Sync tripdata bucket | |
| id: sync | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: ctbk import -ccdX html | |
| - name: Refresh tripdata summary | |
| if: always() | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| run: ctbk tripdata-summary --output tripdata-latest.json && cat tripdata-latest.json | |
| - name: Upload summary to R2 | |
| if: always() | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: auto | |
| AWS_ENDPOINT_URL: https://${{ secrets.CLOUDFLARE_ACCOUNT_ID }}.r2.cloudflarestorage.com | |
| run: | | |
| aws s3 cp tripdata-latest.json s3://ctbk/tripdata/latest.json \ | |
| --content-type application/json | |
| - name: Job summary | |
| if: always() | |
| run: | | |
| if [ -n "${{ steps.sync.outputs.new_files }}" ]; then | |
| { | |
| echo "## New tripdata imported" | |
| echo | |
| echo "Files: \`${{ steps.sync.outputs.new_files }}\`" | |
| echo | |
| echo "Month: \`${{ steps.sync.outputs.month }}\`" | |
| echo | |
| echo "[Run ingest pipeline →](https://github.com/${{ github.repository }}/actions/workflows/ci.yml)" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| { | |
| echo "## No new tripdata files" | |
| echo | |
| echo "All \`.zip\` files in \`s3://tripdata\` are already tracked." | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Notify Slack — new tripdata | |
| if: success() && steps.sync.outputs.new_files != '' | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| NEW_FILES: ${{ steps.sync.outputs.new_files }} | |
| MONTH: ${{ steps.sync.outputs.month }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| CI_URL: ${{ github.server_url }}/${{ github.repository }}/actions/workflows/ci.yml | |
| run: | | |
| uv pip install --quiet 'thrds==0.4.0' | |
| python - <<'PY' | |
| import os | |
| from thrds import SlackClient, Thread | |
| text = ( | |
| f":bike: *New Citi Bike tripdata imported* — `{os.environ['MONTH']}`\n" | |
| f"Files: `{os.environ['NEW_FILES']}`\n" | |
| f"<{os.environ['RUN_URL']}|sync run> · <{os.environ['CI_URL']}|ingest workflow>" | |
| ) | |
| SlackClient( | |
| token=os.environ['SLACK_BOT_TOKEN'], | |
| channel='C0B5MKF28NP', | |
| username='ctbk-bot', | |
| icon_emoji=':bike:', | |
| ).sync(Thread(messages=[text])) | |
| PY |