Cap bounded robustness worker resources #517
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| verify: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 | |
| - name: Install Nix | |
| uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 | |
| - name: Evaluate pinned environments | |
| run: | | |
| set -euo pipefail | |
| for attempt in 1 2 3; do | |
| if nix flake check --no-build --all-systems; then | |
| exit 0 | |
| fi | |
| if test "$attempt" -lt 3; then | |
| delay=$((attempt * 15)) | |
| echo "flake evaluation failed on attempt $attempt; retrying in ${delay}s" | |
| sleep "$delay" | |
| fi | |
| done | |
| echo "flake evaluation failed after 3 attempts; trying codeload fallback" >&2 | |
| fallback_root=$(mktemp -d) | |
| fallback_archive="$fallback_root/rust-overlay.tar.gz" | |
| fallback_source="$fallback_root/rust-overlay" | |
| rust_overlay_rev=$(jq -er '.nodes["rust-overlay"].locked.rev' flake.lock) | |
| curl --fail --location --retry 3 --retry-all-errors \ | |
| --connect-timeout 10 --max-time 120 \ | |
| -o "$fallback_archive" \ | |
| "https://codeload.github.com/oxalica/rust-overlay/tar.gz/$rust_overlay_rev" | |
| mkdir "$fallback_source" | |
| tar -xzf "$fallback_archive" --strip-components=1 -C "$fallback_source" | |
| nix flake check --no-build --all-systems \ | |
| --override-input rust-overlay "path:$fallback_source" | |
| echo "RUST_OVERLAY_SOURCE=$fallback_source" >> "$GITHUB_ENV" | |
| exit 0 | |
| - name: Synchronize frozen Python environment | |
| run: nix develop ${RUST_OVERLAY_SOURCE:+--override-input} ${RUST_OVERLAY_SOURCE:+rust-overlay} ${RUST_OVERLAY_SOURCE:+path:$RUST_OVERLAY_SOURCE} --command uv sync --frozen --reinstall-package format-bench | |
| - name: Run blocking Python quality gates | |
| run: | | |
| nix develop ${RUST_OVERLAY_SOURCE:+--override-input} ${RUST_OVERLAY_SOURCE:+rust-overlay} ${RUST_OVERLAY_SOURCE:+path:$RUST_OVERLAY_SOURCE} --command bash -euo pipefail -c ' | |
| ruff --version | |
| pyright --version | |
| ruff check src tests tools | |
| pyright | |
| python tools/audit_tracker.py validate | |
| ' | |
| # Native parallel steps share the prepared workspace and Nix store while | |
| # retaining separate logs and durations. Each child enters the pinned shell: | |
| # https://docs.github.com/actions/reference/workflows-and-actions/workflow-syntax#jobsjob_idstepsparallel | |
| - parallel: | |
| - name: Run pytest | |
| run: nix develop ${RUST_OVERLAY_SOURCE:+--override-input} ${RUST_OVERLAY_SOURCE:+rust-overlay} ${RUST_OVERLAY_SOURCE:+path:$RUST_OVERLAY_SOURCE} --command .venv/bin/pytest | |
| - name: Smoke fair lane | |
| run: nix develop ${RUST_OVERLAY_SOURCE:+--override-input} ${RUST_OVERLAY_SOURCE:+rust-overlay} ${RUST_OVERLAY_SOURCE:+path:$RUST_OVERLAY_SOURCE} --command bash tools/ci_smoke.sh fair | |
| - name: Smoke claims lane | |
| run: nix develop ${RUST_OVERLAY_SOURCE:+--override-input} ${RUST_OVERLAY_SOURCE:+rust-overlay} ${RUST_OVERLAY_SOURCE:+path:$RUST_OVERLAY_SOURCE} --command bash tools/ci_smoke.sh claims | |
| - name: Smoke prompt lane | |
| run: nix develop ${RUST_OVERLAY_SOURCE:+--override-input} ${RUST_OVERLAY_SOURCE:+rust-overlay} ${RUST_OVERLAY_SOURCE:+path:$RUST_OVERLAY_SOURCE} --command bash tools/ci_smoke.sh prompt | |
| - name: Smoke equivalence lane | |
| run: nix develop ${RUST_OVERLAY_SOURCE:+--override-input} ${RUST_OVERLAY_SOURCE:+rust-overlay} ${RUST_OVERLAY_SOURCE:+path:$RUST_OVERLAY_SOURCE} --command bash tools/ci_smoke.sh equivalence | |
| - name: Gate bounded robustness fixture | |
| run: | | |
| # Run separately from the concurrent smokes so subprocess timeouts remain meaningful. | |
| # shellcheck disable=SC2016 | |
| nix develop ${RUST_OVERLAY_SOURCE:+--override-input} ${RUST_OVERLAY_SOURCE:+rust-overlay} ${RUST_OVERLAY_SOURCE:+path:$RUST_OVERLAY_SOURCE} --command bash -euo pipefail -c ' | |
| run_dir="runs/ci-smoke-robustness-$BASHPID" | |
| result=$(.venv/bin/format-bench run \ | |
| --profile robustness \ | |
| --suite bounded \ | |
| --dataset github-stars-2026-07-03 \ | |
| --run-dir "$run_dir" \ | |
| --fixture \ | |
| --generated-cases 0 \ | |
| --mutations-per-target 1 \ | |
| --case-timeout-seconds 10 \ | |
| --artifact-budget-mib 64) | |
| run_dir=$(dirname "$result") | |
| report=$(.venv/bin/format-bench report --run-dir "$run_dir") | |
| before=$(sha256sum "$report") | |
| .venv/bin/format-bench report --run-dir "$run_dir" >/dev/null | |
| after=$(sha256sum "$report") | |
| test "$before" = "$after" | |
| .venv/bin/python -c \ | |
| "import json,sys; m=json.load(open(sys.argv[1])); assert m.get(\"fixture\") and not m.get(\"rankable\")" \ | |
| "$run_dir/manifest.json" | |
| .venv/bin/python -m format_bench.robustness.gate "$run_dir/results.json" | |
| ' | |
| - name: Gate implementation audit | |
| run: | | |
| # Keep the audit inside the pinned shell so PyArrow sees the same C++ runtime as every other gate. | |
| nix develop ${RUST_OVERLAY_SOURCE:+--override-input} ${RUST_OVERLAY_SOURCE:+rust-overlay} ${RUST_OVERLAY_SOURCE:+path:$RUST_OVERLAY_SOURCE} --command bash -euo pipefail -c ' | |
| .venv/bin/format-bench audit \ | |
| --dataset github-stars-2026-07-03 \ | |
| --output /tmp/implementation-audit.json | |
| .venv/bin/python -c \ | |
| "import json; assert json.load(open(\"/tmp/implementation-audit.json\"))[\"audit\"][\"status\"]==\"PASS\"" | |
| ' | |
| - name: Observe ty diagnostics | |
| continue-on-error: true | |
| run: | | |
| nix develop ${RUST_OVERLAY_SOURCE:+--override-input} ${RUST_OVERLAY_SOURCE:+rust-overlay} ${RUST_OVERLAY_SOURCE:+path:$RUST_OVERLAY_SOURCE} --command bash -euo pipefail -c ' | |
| ty version | |
| ty check | |
| ' |