Runner fleet metrics #41
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Runner fleet metrics | |
| # Every 30 minutes, count the self-hosted runners registered with the pytorch | |
| # org (macOS + B200 by default), then upload the counts to S3. The | |
| # clickhouse-replicator-s3 lambda ingests them into misc.runner_fleet_count, | |
| # which is graphed in Grafana via the ClickHouse datasource. | |
| # | |
| # No ClickHouse credentials live here: the workflow only authenticates to AWS | |
| # (OIDC) to write the object; the replicator holds the CH creds. | |
| on: | |
| schedule: | |
| - cron: "*/30 * * * *" | |
| workflow_dispatch: | |
| inputs: | |
| dry_run: | |
| description: "Collect and print counts without uploading to S3" | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: runner-fleet-metrics | |
| cancel-in-progress: false | |
| permissions: | |
| contents: read | |
| id-token: write # required for AWS OIDC role assumption | |
| jobs: | |
| collect: | |
| runs-on: ubuntu-latest | |
| # Protected environment restricted to the main branch; holds the runner-list | |
| # token. | |
| environment: runner-fleet-metrics | |
| env: | |
| RUNNER_ORG: pytorch | |
| S3_BUCKET: gha-artifacts | |
| S3_PREFIX: runner_fleet_count | |
| AWS_REGION: us-east-1 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - run: python -m pip install --no-cache-dir requests | |
| - name: Collect runner counts | |
| env: | |
| # GitHub reserves the "GITHUB_" prefix for secret *names*, so the | |
| # runner-list token is stored as RUNNER_LIST_GH_TOKEN and exposed to | |
| # the script as GITHUB_TOKEN. Needs org self-hosted-runners:read. | |
| GITHUB_TOKEN: ${{ secrets.RUNNER_LIST_GH_TOKEN }} | |
| TRACKED_LABEL_PREFIXES: "macos,linux.dgx.b200" | |
| OUTPUT_FILE: runner_fleet_count.jsonl | |
| run: python tools/runner_metrics/collect_runner_fleet.py | |
| - name: Configure AWS credentials | |
| if: ${{ !inputs.dry_run }} | |
| # Shared ARC role: trusts repo:pytorch/test-infra:* and can write | |
| # gha-artifacts (same role _linux-build.yml uses for artifact upload). | |
| uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| role-to-assume: arn:aws:iam::308535385114:role/arc | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Upload to S3 for ClickHouse ingestion | |
| if: ${{ !inputs.dry_run }} | |
| run: | | |
| set -euo pipefail | |
| ts="$(date -u +%Y%m%dT%H%M%SZ)" | |
| gzip -c runner_fleet_count.jsonl > runner_fleet_count.json.gz | |
| # NB: do NOT set --content-encoding gzip. ClickHouse's s3() reader | |
| # decompresses via the explicit compression arg in the replicator; a | |
| # gzip Content-Encoding header makes it mis-read the object and ingest | |
| # 0 rows silently (no error). Store plain gzip bytes instead. | |
| aws s3 cp runner_fleet_count.json.gz \ | |
| "s3://${S3_BUCKET}/${S3_PREFIX}/${RUNNER_ORG}/${ts}.json.gz" |