Collect metrics #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Collect metrics | |
| # Runs fetch_metrics.py once daily to snapshot sirius-db/sirius's stars, forks, | |
| # traffic, and issue/PR/commit activity for the most recent fully-elapsed day(s), | |
| # and refresh_traffic.py to keep already-collected snapshots' traffic fields in sync | |
| # with GitHub's own rolling 14-day breakdown (which can lag or later revise a day's | |
| # numbers -- see #14). Both are committed together in one commit to data/snapshots/. | |
| # This is the only source of truth for anything GitHub doesn't retain long-term -- | |
| # see README.md and BOOTSTRAP.md for the full picture. | |
| # | |
| # Deliberately never tries to collect the current, still-in-progress day -- only ever | |
| # finalizes fully-elapsed days, catching up every missing day within a 7-day lookback | |
| # in one firing (not just the oldest), so a backlog can actually close instead of | |
| # staying permanently one day behind. See #14 and the linked PR for the incident this | |
| # design replaced. | |
| on: | |
| schedule: | |
| - cron: "30 0 * * *" # 0030 UTC, single daily firing -- no retry window (see #14). | |
| workflow_dispatch: | |
| inputs: | |
| date: | |
| description: "Force-(re)collect a specific date (YYYY-MM-DD). Leave blank for the normal catch-up scan." | |
| required: false | |
| permissions: | |
| contents: write | |
| concurrency: | |
| # Prevents a manual workflow_dispatch run from racing the scheduled run's git push -- | |
| # queues instead of canceling, so a manual run never causes a scheduled snapshot to | |
| # be silently dropped. | |
| group: collect-metrics | |
| cancel-in-progress: false | |
| jobs: | |
| collect: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v5 | |
| with: | |
| # Default GITHUB_TOKEN can't push once main requires a PR -- this PAT's | |
| # owner is a repo admin, and enforce_admins is off, so the push bypasses | |
| # the PR requirement. See BOOTSTRAP.md. | |
| token: ${{ secrets.COLLECT_PUSH_TOKEN }} | |
| - name: Gate | |
| id: gate | |
| # A manual dispatch with a `date` input always force-collects exactly that | |
| # date. Otherwise: scan the last 7 days for every currently-missing snapshot | |
| # (not just the oldest -- one-per-firing can never close a backlog once | |
| # behind, since a new day becomes due every day at the same rate as | |
| # recovery) and collect all of them in this one firing. Also checks one day | |
| # past the lookback boundary so data that's about to become permanently | |
| # unrecoverable by this automated path is loudly flagged, not silently | |
| # dropped. | |
| env: | |
| DISPATCH_DATE: ${{ github.event.inputs.date }} | |
| run: | | |
| if [ -n "$DISPATCH_DATE" ]; then | |
| echo "run=true" >> "$GITHUB_OUTPUT" | |
| echo "targets=${DISPATCH_DATE}" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| targets="" | |
| for i in 7 6 5 4 3 2 1; do | |
| d="$(date -u -d "$i days ago" +%Y-%m-%d)" | |
| if [ ! -f "data/snapshots/${d}.json" ]; then | |
| targets="${targets}${d} " | |
| fi | |
| done | |
| aged_out="$(date -u -d "8 days ago" +%Y-%m-%d)" | |
| if [ ! -f "data/snapshots/${aged_out}.json" ]; then | |
| echo "::error::${aged_out} has fallen outside the 7-day catch-up window and will not be automatically recovered -- see BOOTSTRAP.md sec 5 for manual recovery via fetch_metrics.py --date" | |
| fi | |
| target_count="$(echo "$targets" | wc -w)" | |
| if [ "$target_count" -gt 1 ]; then | |
| echo "::warning::Catching up ${target_count} missing days in one firing (${targets}) -- pipeline fell behind; repo/labels fields on these days will reflect today's values, not that day's (see BOOTSTRAP.md's point-in-time limitation)" | |
| fi | |
| if [ -z "$targets" ]; then | |
| echo "run=false" >> "$GITHUB_OUTPUT" | |
| echo "nothing due -- all of the last 7 days are already collected" | |
| else | |
| echo "run=true" >> "$GITHUB_OUTPUT" | |
| echo "targets=${targets}" >> "$GITHUB_OUTPUT" | |
| fi | |
| - uses: astral-sh/setup-uv@v10.0.1 | |
| with: | |
| python-version: "3.12" | |
| - name: Fetch metrics | |
| if: steps.gate.outputs.run == 'true' | |
| env: | |
| SIRIUS_TRAFFIC_TOKEN: ${{ secrets.SIRIUS_TRAFFIC_TOKEN }} | |
| TARGETS: ${{ steps.gate.outputs.targets }} | |
| run: | | |
| for d in $TARGETS; do | |
| uv run scripts/fetch_metrics.py --date "$d" | |
| done | |
| - name: Refresh traffic | |
| # Always runs, regardless of whether Fetch metrics ran or succeeded -- | |
| # traffic-patching is independent of whether a new day was collected this | |
| # firing, and must not be silently skipped by GitHub Actions' default | |
| # if: success() just because an unrelated upstream step failed. | |
| if: always() | |
| env: | |
| SIRIUS_TRAFFIC_TOKEN: ${{ secrets.SIRIUS_TRAFFIC_TOKEN }} | |
| run: uv run scripts/refresh_traffic.py | |
| - name: Commit | |
| # Always runs (see Refresh traffic above). Combines whatever Fetch metrics | |
| # and Refresh traffic produced into a single commit -- never more than one | |
| # per firing. | |
| if: always() | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add data/snapshots/ | |
| if git diff --cached --quiet; then | |
| echo "no changes to commit" | |
| exit 0 | |
| fi | |
| dates="$(git diff --cached --name-only -- data/snapshots/ | sed -E 's#.*/([0-9-]+)\.json#\1#' | sort -u | tr '\n' ' ')" | |
| git commit -m "chore: update collected data (${dates})" | |
| git push |