Add OS-level resource metric collection to flow run subprocesses #25179
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmarks | |
| env: | |
| PY_COLORS: 1 | |
| on: | |
| pull_request: | |
| paths: | |
| - .github/workflows/benchmarks.yaml | |
| - .github/workflows/python-tests.yaml | |
| - benches/cli-bench.toml | |
| - "src/prefect/**/*.py" | |
| - pyproject.toml | |
| - uv.lock | |
| - Dockerfile | |
| push: | |
| branches: | |
| - main | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| run-benchmarks: | |
| name: Benchmark | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| fetch-depth: 0 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| cache-dependency-glob: "pyproject.toml" | |
| - name: Install the project | |
| run: uv sync --group benchmark --compile-bytecode --locked | |
| - name: Prepare benchmark comparisons | |
| # Note: We use a "cache" instead of artifacts because artifacts are not available | |
| # across workflow runs. | |
| id: bench-cache | |
| uses: actions/cache@v5 | |
| with: | |
| path: ./.benchmarks | |
| # Pushes benchmark results for this branch and sha, this will always be a cache miss | |
| # and `restore-keys` will be used to get the benchmarks for comparison | |
| key: ${{ runner.os }}-${{ github.head_ref || 'main' }}-${{ github.sha }} | |
| # Pulls benchmark results for the base branch | |
| restore-keys: | | |
| ${{ runner.os }}-${{ github.base_ref }}- | |
| ${{ runner.os }}-main- | |
| - name: Start server | |
| run: | | |
| PREFECT_HOME=$(pwd) uv run prefect server start& | |
| PREFECT_API_URL="http://127.0.0.1:4200/api" uv run ./scripts/wait-for-server.py | |
| # TODO: Replace `wait-for-server` with dedicated command | |
| # https://github.com/PrefectHQ/prefect/issues/6990 | |
| - name: Run benchmarks | |
| env: | |
| HEAD_REF: ${{ github.head_ref }} | |
| GITHUB_SHA: ${{ github.sha }} | |
| # Includes comparison to previous benchmarks if available | |
| # Import benchmark is ignored because because we run those | |
| # benchmarks via CodSpeed | |
| run: | | |
| if [[ -z "$HEAD_REF" ]]; then | |
| # HEAD_REF is unset or empty, use 'main' with the SHA | |
| uniquename="main-$GITHUB_SHA" | |
| else | |
| # HEAD_REF is set, use the branch name directly | |
| uniquename="$HEAD_REF" | |
| fi | |
| # Allow alphanumeric, underscores, and dashes, and replace other | |
| # characters with an underscore | |
| sanitized_uniquename="${uniquename//[^a-zA-Z0-9_\-]/_}" | |
| PREFECT_API_URL="http://127.0.0.1:4200/api" \ | |
| uv run python -m benches \ | |
| --ignore=benches/bench_import.py \ | |
| --timeout=180 \ | |
| --benchmark-save="${sanitized_uniquename}" \ | |
| ${{ steps.bench-cache.outputs.cache-hit && '--benchmark-compare' || '' }} | |
| detect-cli-benchmark-changes: | |
| name: Detect CLI benchmark scope | |
| if: ${{ github.event_name == 'pull_request' }} | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_run: ${{ steps.filter.outputs.should_run }} | |
| steps: | |
| - name: Detect relevant changes with path filters | |
| id: filter | |
| uses: dorny/paths-filter@v3 | |
| with: | |
| filters: | | |
| should_run: | |
| - "src/prefect/cli/**" | |
| - "benches/cli-bench.toml" | |
| - "pyproject.toml" | |
| - "uv.lock" | |
| - "Dockerfile" | |
| - ".github/workflows/benchmarks.yaml" | |
| cli-benchmark-shards: | |
| name: CLI startup benchmark shard ${{ matrix.shard_index }} | |
| if: ${{ github.event_name == 'pull_request' && needs.detect-cli-benchmark-changes.outputs.should_run == 'true' }} | |
| needs: [detect-cli-benchmark-changes] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| shard_index: [0, 1, 2, 3] | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up uv | |
| uses: astral-sh/setup-uv@v7 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| cache-dependency-glob: "pyproject.toml" | |
| - name: Install hyperfine | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y hyperfine | |
| - name: Extract cli-bench config from head | |
| id: full-config | |
| run: | | |
| git fetch origin ${{ github.event.pull_request.head.sha }} | |
| config_path="$RUNNER_TEMP/cli-bench.full.toml" | |
| git show ${{ github.event.pull_request.head.sha }}:benches/cli-bench.toml > "$config_path" | |
| echo "path=$config_path" >> "$GITHUB_OUTPUT" | |
| echo "Using benches/cli-bench.toml from ${{ github.event.pull_request.head.sha }}" | |
| cat "$config_path" | |
| - name: Build sharded benchmark config | |
| env: | |
| CLI_BENCH_SHARD_INDEX: ${{ matrix.shard_index }} | |
| CLI_BENCH_SHARD_TOTAL: "4" | |
| CLI_BENCH_CONFIG_SOURCE: ${{ steps.full-config.outputs.path }} | |
| CLI_BENCH_CONFIG_SHARD: ${{ runner.temp }}/cli-bench.shard.toml | |
| run: | | |
| uv run python - <<'PY' | |
| from __future__ import annotations | |
| import json | |
| import os | |
| import tomllib | |
| from pathlib import Path | |
| shard_index = int(os.environ["CLI_BENCH_SHARD_INDEX"]) | |
| shard_total = int(os.environ["CLI_BENCH_SHARD_TOTAL"]) | |
| source_path = Path(os.environ["CLI_BENCH_CONFIG_SOURCE"]) | |
| output_path = Path(os.environ["CLI_BENCH_CONFIG_SHARD"]) | |
| data = tomllib.loads(source_path.read_text()) | |
| commands = data.get("commands", []) | |
| selected = [ | |
| command | |
| for index, command in enumerate(commands) | |
| if index % shard_total == shard_index | |
| ] | |
| if not selected: | |
| raise SystemExit( | |
| f"No commands selected for shard {shard_index}/{shard_total}" | |
| ) | |
| lines: list[str] = [ | |
| "# generated shard config", | |
| f"# source: {source_path}", | |
| f"# shard: {shard_index}/{shard_total}", | |
| "", | |
| "[project]", | |
| ] | |
| for key, value in data.get("project", {}).items(): | |
| lines.append(f"{key} = {json.dumps(value)}") | |
| for command in selected: | |
| lines.append("") | |
| lines.append("[[commands]]") | |
| for key, value in command.items(): | |
| lines.append(f"{key} = {json.dumps(value)}") | |
| output_path.write_text("\n".join(lines) + "\n") | |
| print( | |
| f"Shard {shard_index}/{shard_total} selected {len(selected)} " | |
| f"commands out of {len(commands)}" | |
| ) | |
| print(output_path.read_text()) | |
| PY | |
| - name: Prepare worktrees | |
| run: | | |
| git fetch origin ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} | |
| git worktree add "$RUNNER_TEMP/base" ${{ github.event.pull_request.base.sha }} | |
| git worktree add "$RUNNER_TEMP/head" ${{ github.event.pull_request.head.sha }} | |
| - name: Install dependencies | |
| run: uv sync --group cli-bench --locked | |
| - name: Run base benchmarks | |
| run: | | |
| uv run --group cli-bench cli-bench \ | |
| --config "$RUNNER_TEMP/cli-bench.shard.toml" \ | |
| --project-root "$RUNNER_TEMP/base" \ | |
| run \ | |
| --runs 5 \ | |
| --category startup \ | |
| --output baseline.json | |
| - name: Run head benchmarks | |
| run: | | |
| uv run --group cli-bench cli-bench \ | |
| --config "$RUNNER_TEMP/cli-bench.shard.toml" \ | |
| --project-root "$RUNNER_TEMP/head" \ | |
| run \ | |
| --runs 5 \ | |
| --category startup \ | |
| --output comparison.json | |
| - name: Check shard for regressions | |
| run: | | |
| uv run --group cli-bench cli-bench compare \ | |
| baseline.json comparison.json \ | |
| --threshold 15 \ | |
| --fail-on-regression \ | |
| --summary-md cli-benchmark-compare.md \ | |
| --digest-json cli-benchmark-digest.json | |
| - name: Upload benchmark shard | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: cli-benchmark-shard-${{ matrix.shard_index }} | |
| path: | | |
| baseline.json | |
| comparison.json | |
| cli-benchmark-compare.md | |
| cli-benchmark-digest.json | |
| cli-benchmarks: | |
| name: CLI startup benchmarks | |
| if: ${{ always() && github.event_name == 'pull_request' }} | |
| needs: [detect-cli-benchmark-changes, cli-benchmark-shards] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Validate shard outcomes | |
| run: | | |
| echo "CLI benchmark scope check: ${{ needs.detect-cli-benchmark-changes.outputs.should_run }}" | |
| if [[ "${{ needs.detect-cli-benchmark-changes.outputs.should_run }}" != "true" ]]; then | |
| echo "No CLI/dependency changes detected; skipping CLI startup benchmark shards." | |
| exit 0 | |
| fi | |
| echo "Shard result: ${{ needs.cli-benchmark-shards.result }}" | |
| if [[ "${{ needs.cli-benchmark-shards.result }}" != "success" ]]; then | |
| echo "One or more CLI benchmark shards failed." | |
| exit 1 | |
| fi | |
| echo "All CLI benchmark shards passed." |