Skip to content

utoopm-e2e-bench

utoopm-e2e-bench #2676

Workflow file for this run

name: utoopm-e2e-bench
# Single workflow that builds utoo once per target, then fans out to
# every consumer (e2e, bench-phases, pm-bench-all, pm-bench-pcap).
# Replaces the old pm-e2e.yml + pm-bench.yml + pm-bench-phases.yml
# trio so each target builds exactly once per commit and downstream
# jobs reuse the uploaded artifact.
#
# Triggers (event → consumers):
# pull_request / push with `(pm)` in title/msg → e2e jobs
# pull_request labeled `benchmark` → bench-phases (utoo / utoo-npm / bun)
# workflow_dispatch with `target` choice → manual one-off runs
# (pm-bench-all = full PM shootout vs pnpm/pnpm12/yarn/bun/aube;
# pm-bench-phases = same as the labeled trigger but ad-hoc;
# pm-bench-pcap = tcpdump capture for resolve phase)
#
# Build job gates are the union of their downstream consumers' gates.
# Each downstream job is pure download-and-run, no `cargo build`.
on:
push:
branches: ["next", "e2e-pm"]
pull_request:
types: [opened, synchronize, reopened, edited, labeled]
workflow_dispatch:
inputs:
target:
description: 'Manual target to run'
required: true
type: choice
options:
- pm-bench-all
- pm-bench-phases
- pm-bench-pcap
default: pm-bench-all
# ----- pm-bench-all inputs -----
registry:
description: '[pm-bench-all] Registry to test'
required: false
default: 'both'
type: choice
options: [both, npmjs, npmmirror]
pm_list:
description: '[pm-bench-all] Comma-separated PMs (e.g. utoo,bun); "all" → utoo,pnpm,pnpm12,yarn,bun,aube'
required: false
default: 'utoo,bun'
type: string
baseline:
description: '[pm-bench-all] Compare with published utoo from npm'
required: false
default: true
type: boolean
build_next:
description: '[pm-bench-all] Also compare with utoo built from `next` branch'
required: false
default: false
type: boolean
cold_runs:
description: '[pm-bench-all] Number of cold install runs'
required: false
default: '3'
type: string
warm_runs:
description: '[pm-bench-all] Number of warm install runs'
required: false
default: '3'
type: string
# ----- pm-bench-phases / pcap inputs -----
project:
description: '[phases/pcap] Project to bench (under github.com/ant-design/<project>)'
required: false
default: 'ant-design'
type: string
bench_runs:
description: '[phases] Runs per phase'
required: false
default: '3'
type: string
utoo_alt_env:
description: '[phases] Extra env for a second utoo variant (ablation A/B), e.g. "UTOO_TARBALL_HTTP=h2" or "UTOO_CLONE_CONCURRENCY=8"'
required: false
default: ''
type: string
phases:
description: '[phases] Comma-separated phase filter (p0,p1,p3,p4) - ablations can run just the relevant phases'
required: false
default: 'p0,p1,p3,p4'
type: string
phases_pm_list:
description: '[phases] PMs to bench - an utoo vs utoo-alt ablation only needs "utoo"'
required: false
default: 'utoo,utoo-next,utoo-npm,bun'
type: string
phases_registry:
description: '[phases] Registry leg(s) to run'
required: false
default: 'both'
type: choice
options: [both, npmjs, npmmirror]
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
contents: read
jobs:
# ============================================================
# BUILD JOBS
# One per target, runs whenever any downstream consumer needs it.
# Downstream jobs `needs:` the relevant build job and download
# the uploaded artifact. No downstream rebuilds.
# ============================================================
build-linux:
# Linux serves: e2e-linux, bench-phases-linux, pm-bench-all-linux, pm-bench-pcap-linux.
# Gate is the union of all linux consumers' gates.
if: >
(github.event_name == 'push' && contains(github.event.head_commit.message, '(pm)')) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.title, '(pm)')) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmark')) ||
github.event_name == 'workflow_dispatch'
runs-on: ubuntu-latest
container:
image: ubuntu:20.04
steps:
- name: Install dependencies
timeout-minutes: 6
env:
DEBIAN_FRONTEND: noninteractive
# Short per-apt timeouts: on GHA runners that can't route to
# archive/security.ubuntu.com, apt otherwise sits in 90s × 8-IP SYN
# timeouts. Bail after 60s on the default mirror, swap to the
# Fastly-backed kernel.org mirror, and retry once.
run: |
install_pkgs() {
timeout 60 apt-get update \
&& timeout 60 apt-get install -y build-essential curl pkg-config git
}
install_pkgs && exit 0
echo "apt default mirror failed/timed out, falling back to mirrors.edge.kernel.org"
dpkg --configure -a || true
sed -i 's|http://archive\.ubuntu\.com|http://mirrors.edge.kernel.org|g; s|http://security\.ubuntu\.com|http://mirrors.edge.kernel.org|g' /etc/apt/sources.list
install_pkgs
- uses: actions/checkout@v4
- name: Configure git safe directory
run: git config --global --add safe.directory /__w/utoo/utoo
- name: Init git submodules
run: git submodule update --init --recursive --depth 1
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: nightly-2026-05-15
targets: x86_64-unknown-linux-gnu
- name: Cache cargo
uses: Swatinem/rust-cache@v2
continue-on-error: true
with:
shared-key: pm-build-linux
cache-bin: false
- name: Build utoo-pm
run: cargo build --release --target x86_64-unknown-linux-gnu --bin utoo -p utoo-pm
- name: Upload utoo binary
uses: actions/upload-artifact@v4
with:
name: utoo-linux-x64
path: target/x86_64-unknown-linux-gnu/release/utoo
retention-days: 1
# Piggyback on the already-built target/ from the step above: when the
# PR is labeled `benchmark`, overlay origin/next's tree onto the current
# workdir and re-run cargo build. cargo's incremental compile only
# rebuilds crates that actually differ between this PR and next — for
# bench-infra-only PRs that's a few seconds (just relink); for PRs that
# touch pm sources, only the affected crates rebuild. Either way far
# cheaper than a separate cold workspace, and the artifact route avoids
# competing with build-* rust-caches for the repo's 10GB cache budget.
- name: Build next branch utoo (bench baseline)
if: >
(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmark')) ||
(github.event_name == 'workflow_dispatch' && (inputs.target == 'pm-bench-phases' || inputs.target == 'pm-bench-pcap'))
run: |
# The previous Upload step has already shipped the PR binary; we
# can now overwrite target/.../release/utoo safely.
git fetch origin next --depth=1
# Overlay origin/next's working tree onto the current workdir
# without moving HEAD. `checkout -- .` restores next's tracked files
# but leaves files the PR adds that next lacks; drop those too, or new
# module dirs (e.g. registry/, driver/) shadow next's flat-file
# modules and the baseline build fails with E0761. Submodule pointers
# may differ so re-init.
git checkout origin/next -- .
# --no-renames so rename targets (e.g. registry.rs -> registry/mod.rs)
# count as Added and get removed too, not paired away as renames.
git diff --no-renames --diff-filter=A --name-only origin/next HEAD | xargs -r rm -f
git submodule update --init --recursive --depth 1
cargo build --release --target x86_64-unknown-linux-gnu --bin utoo -p utoo-pm
- name: Upload utoo-next binary
if: >
(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmark')) ||
(github.event_name == 'workflow_dispatch' && (inputs.target == 'pm-bench-phases' || inputs.target == 'pm-bench-pcap'))
uses: actions/upload-artifact@v4
with:
name: utoo-next-linux-x64
# Same workspace-relative path as utoo-linux-x64 above — but the
# content here is the next-branch binary because the cargo build
# in the previous step overwrote it. download-artifact then lays
# it out as `<dest>/utoo` (flat), matching utoo-linux-x64's shape.
path: target/x86_64-unknown-linux-gnu/release/utoo
retention-days: 1
build-mac-arm64:
# Mac arm64 serves: e2e-mac-arm64, bench-phases-mac (manual only),
# pm-bench-all-mac (manual only). Label-triggered benches now stick
# to linux, so we no longer build on the `benchmark` label.
if: >
(github.event_name == 'push' && contains(github.event.head_commit.message, '(pm)')) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.title, '(pm)')) ||
(github.event_name == 'workflow_dispatch' && inputs.target != 'pm-bench-pcap')
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Init git submodules
run: git submodule update --init --recursive --depth 1
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: nightly-2026-05-15
targets: aarch64-apple-darwin
- name: Cache cargo
uses: Swatinem/rust-cache@v2
continue-on-error: true
with:
shared-key: pm-build-mac-arm64
cache-bin: false
- name: Build utoo-pm
run: cargo build --release --target aarch64-apple-darwin --bin utoo -p utoo-pm
- name: Upload utoo binary
uses: actions/upload-artifact@v4
with:
name: utoo-mac-arm64
path: target/aarch64-apple-darwin/release/utoo
retention-days: 1
build-mac-x64:
# Mac x86_64 serves: e2e-mac-x64 only. Cross-compiled on macos-latest
# (arm64 host) via `rustup target add x86_64-apple-darwin`.
if: >
(github.event_name == 'push' && contains(github.event.head_commit.message, '(pm)')) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.title, '(pm)'))
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Init git submodules
run: git submodule update --init --recursive --depth 1
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: nightly-2026-05-15
targets: x86_64-apple-darwin
- name: Add cross-compilation target
run: rustup target add x86_64-apple-darwin
- name: Cache cargo
uses: Swatinem/rust-cache@v2
continue-on-error: true
with:
shared-key: pm-build-mac-x64
cache-bin: false
- name: Build utoo-pm
run: cargo build --release --target x86_64-apple-darwin --bin utoo -p utoo-pm
- name: Upload utoo binary
uses: actions/upload-artifact@v4
with:
name: utoo-mac-x64
path: target/x86_64-apple-darwin/release/utoo
retention-days: 1
build-windows:
# Windows serves: e2e-windows only.
if: >
(github.event_name == 'push' && contains(github.event.head_commit.message, '(pm)')) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.title, '(pm)'))
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
- name: Disable Windows Defender
shell: powershell
run: |
try {
Set-MpPreference -DisableRealtimeMonitoring $true -ErrorAction Stop
} catch {
Write-Warning "Unable to disable Windows Defender real-time monitoring: $_"
}
- name: Init git submodules
run: git submodule update --init --recursive --depth 1
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: nightly-2026-05-15
targets: x86_64-pc-windows-msvc
- name: Cache cargo
uses: Swatinem/rust-cache@v2
continue-on-error: true
with:
shared-key: pm-build-windows
cache-bin: false
- name: Build utoo-pm
run: cargo build --release --target x86_64-pc-windows-msvc --bin utoo -p utoo-pm
- name: Upload utoo binary
uses: actions/upload-artifact@v4
with:
name: utoo-windows-x64
path: target/x86_64-pc-windows-msvc/release/utoo.exe
retention-days: 1
# ============================================================
# E2E JOBS
# Auto-trigger on `(pm)` (strict) PR title / commit msg.
# ============================================================
e2e-linux:
needs: [build-linux]
name: utoopm-e2e-ubuntu-latest-x86_64-unknown-linux-gnu
if: >
(github.event_name == 'push' && contains(github.event.head_commit.message, '(pm)')) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.title, '(pm)'))
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup node
uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-utoo-bin
with:
artifact: utoo-linux-x64
- name: Verify tools
run: |
utoo --version
ut --version
- name: Run e2e tests
run: |
chmod +x e2e/utoo-pm.sh e2e/pm-arborist.sh
bash e2e/utoo-pm.sh
bash e2e/pm-arborist.sh
- name: Upload e2e artifacts on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: e2e-artifacts-ubuntu-latest-x86_64-unknown-linux-gnu
path: /tmp/utoo-*
if-no-files-found: ignore
e2e-mac-arm64:
needs: [build-mac-arm64]
name: utoopm-e2e-macos-latest-aarch64-apple-darwin
if: >
(github.event_name == 'push' && contains(github.event.head_commit.message, '(pm)')) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.title, '(pm)'))
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Setup node
uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-utoo-bin
with:
artifact: utoo-mac-arm64
- name: Verify tools
run: |
utoo --version
ut --version
- name: Run e2e tests
run: |
chmod +x e2e/utoo-pm.sh e2e/pm-arborist.sh
bash e2e/utoo-pm.sh
bash e2e/pm-arborist.sh
- name: Upload e2e artifacts on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: e2e-artifacts-macos-latest-aarch64-apple-darwin
path: /tmp/utoo-*
if-no-files-found: ignore
e2e-mac-x64:
needs: [build-mac-x64]
name: utoopm-e2e-macos-latest-x86_64-apple-darwin
if: >
(github.event_name == 'push' && contains(github.event.head_commit.message, '(pm)')) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.title, '(pm)'))
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Install x86_64 Node.js via nvm
run: |
arch -x86_64 bash -c '
curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
nvm install 22
nvm use 22
nvm alias default 22
echo "NVM_DIR=$HOME/.nvm" >> $GITHUB_ENV
NODE_PATH=$(find $HOME/.nvm/versions/node -name "v22.*" -type d | head -1)
echo "$NODE_PATH/bin" >> $GITHUB_PATH
node -p process.arch
npm -v
'
- uses: ./.github/actions/setup-utoo-bin
with:
artifact: utoo-mac-x64
- name: Verify tools
run: |
utoo --version
ut --version
- name: Run e2e tests (under arch -x86_64)
shell: bash
run: |
chmod +x e2e/utoo-pm.sh e2e/pm-arborist.sh
arch -x86_64 bash -c '
export NVM_DIR="$HOME/.nvm"
[ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
nvm use 22
bash e2e/utoo-pm.sh
bash e2e/pm-arborist.sh
'
- name: Upload e2e artifacts on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: e2e-artifacts-macos-latest-x86_64-apple-darwin
path: /tmp/utoo-*
if-no-files-found: ignore
e2e-windows:
needs: [build-windows]
name: utoopm-e2e-windows-latest-x86_64-pc-windows-msvc
if: >
(github.event_name == 'push' && contains(github.event.head_commit.message, '(pm)')) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.title, '(pm)'))
runs-on: windows-latest
steps:
- uses: actions/checkout@v4
- name: Disable Windows Defender
shell: powershell
run: |
try {
Set-MpPreference -DisableRealtimeMonitoring $true -ErrorAction Stop
} catch {
Write-Warning "Unable to disable Windows Defender real-time monitoring: $_"
}
- name: Setup node
uses: actions/setup-node@v4
with:
node-version: 22
- name: Download utoo binary
uses: actions/download-artifact@v4
with:
name: utoo-windows-x64
path: dist
- name: Install utoo
shell: pwsh
run: |
$binPath = "${{ github.workspace }}\dist"
# ut.exe is just a copy of utoo.exe (Windows has no symlinks).
Copy-Item "$binPath\utoo.exe" "$binPath\ut.exe" -Force
echo "$binPath" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
- name: Verify tools
shell: pwsh
run: |
$binPath = "${{ github.workspace }}\dist"
$env:PATH = "$binPath;$env:PATH"
Get-Command utoo | Select-Object -ExpandProperty Source
Get-Command ut | Select-Object -ExpandProperty Source
utoo --version
ut --version
- name: Run e2e tests
shell: pwsh
run: |
git config --system core.longpaths true
$binPath = "${{ github.workspace }}\dist"
$env:PATH = "$binPath;$env:PATH"
pwsh -File e2e/utoo-pm.ps1 --verbose
- name: Upload e2e artifacts on failure
if: failure()
uses: actions/upload-artifact@v4
with:
name: e2e-artifacts-windows-latest-x86_64-pc-windows-msvc
path: ${{ runner.temp }}/utoo-*
if-no-files-found: ignore
e2e-success:
name: utoopm-e2e-success
runs-on: ubuntu-latest
if: always()
needs: [build-linux, build-mac-arm64, build-mac-x64, build-windows, e2e-linux, e2e-mac-arm64, e2e-mac-x64, e2e-windows]
steps:
- name: Check success
run: |
for result in \
"${{ needs.build-linux.result }}" \
"${{ needs.build-mac-arm64.result }}" \
"${{ needs.build-mac-x64.result }}" \
"${{ needs.build-windows.result }}" \
"${{ needs.e2e-linux.result }}" \
"${{ needs.e2e-mac-arm64.result }}" \
"${{ needs.e2e-mac-x64.result }}" \
"${{ needs.e2e-windows.result }}"; do
if [ "$result" = "failure" ] || [ "$result" = "cancelled" ]; then
echo "Tests failed or were cancelled"
exit 1
fi
done
echo "Tests passed or were skipped"
# ============================================================
# BENCH-PHASES JOBS
# Auto-trigger on `(pm` (loose) — covers `(pm)`, `(pm-bench-phases)`,
# `(pm-foo)`, etc. Posts a sticky PR comment per platform with the
# phase-isolated bench results across npmjs + npmmirror.
# ============================================================
bench-phases-linux:
needs: [build-linux]
if: >
(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'benchmark')) ||
(github.event_name == 'workflow_dispatch' && inputs.target == 'pm-bench-phases')
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-utoo-bin
with:
artifact: utoo-linux-x64
with-ut-symlink: 'false'
- name: Install bun
uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install pnpm 12
id: pnpm12
if: contains(github.event.inputs.phases_pm_list, 'pnpm12')
uses: pnpm/setup@v2
env:
PNPM_CONFIG_PM_ON_FAIL: ignore
with:
version: next-12
dest: ~/setup-pnpm12
install: false
- name: Expose pnpm 12 binary
if: contains(github.event.inputs.phases_pm_list, 'pnpm12')
run: echo "PNPM12_BIN=${{ steps.pnpm12.outputs.dest }}/pnpm" >> "$GITHUB_ENV"
- name: Install pnpm
if: contains(github.event.inputs.phases_pm_list, 'pnpm')
uses: pnpm/action-setup@v4
with:
version: 10
- name: Install aube
if: contains(github.event.inputs.phases_pm_list, 'aube')
run: npm install -g @endevco/aube@1.29.1
- uses: ./.github/actions/install-bench-tools
with:
platform: linux
# utoo-npm (last published release) only changes on releases — bench it
# on manual dispatch only; label-triggered PR runs compare against
# utoo-next, which is the signal that can actually change per PR.
- if: github.event_name == 'workflow_dispatch'
uses: ./.github/actions/install-utoo-npm-baseline
# The utoo-next binary is built upstream by build-linux's
# "Build next branch utoo (bench baseline)" step. Just download.
- name: Download utoo-next binary
uses: actions/download-artifact@v4
with:
name: utoo-next-linux-x64
path: /tmp/utoo-next-dist
- name: Install utoo-next
run: |
chmod +x /tmp/utoo-next-dist/utoo
mv /tmp/utoo-next-dist/utoo /tmp/utoo-next
echo "Baseline utoo (next) version: $(/tmp/utoo-next --version)"
echo "UTOO_NEXT_BIN=/tmp/utoo-next" >> $GITHUB_ENV
- name: Verify tools
run: |
hyperfine --version
utoo --version
if [ -n "${UTOO_NPM_BIN:-}" ]; then "$UTOO_NPM_BIN" --version; fi
"$UTOO_NEXT_BIN" --version
bun --version
if command -v pnpm >/dev/null; then env PNPM_CONFIG_PM_ON_FAIL=ignore pnpm --version; fi
if [ -n "${PNPM12_BIN:-}" ]; then env PNPM_CONFIG_PM_ON_FAIL=ignore "$PNPM12_BIN" --version; fi
if command -v aube >/dev/null; then aube --version; fi
- name: Define cache wipe function
run: |
cat > /tmp/wipe-bench-state.sh <<'WIPE'
#!/usr/bin/env bash
rm -rf /tmp/pm-bench /tmp/pm-bench-results /tmp/pm-bench-output \
/tmp/utoo-bench-cache /tmp/utoo-npm-bench-cache \
/tmp/utoo-next-bench-cache /tmp/utoo-alt-bench-cache /tmp/bun-bench-cache \
/tmp/pnpm-bench-store /tmp/pnpm12-bench-store \
/tmp/aube-bench-data /tmp/aube-bench-cache \
~/.bun/install/cache ~/.cache/nm 2>/dev/null || true
mkdir -p /tmp/pm-bench-output
WIPE
chmod +x /tmp/wipe-bench-state.sh
- name: Wipe bench state before npmjs
run: bash /tmp/wipe-bench-state.sh
- name: Run phase-isolated benchmark · npmjs
if: github.event_name != 'workflow_dispatch' || github.event.inputs.phases_registry != 'npmmirror'
env:
PROJECT: ${{ github.event.inputs.project || 'ant-design' }}
REGISTRY: 'https://registry.npmjs.org'
BENCH_RUNS: ${{ github.event.inputs.bench_runs || '3' }}
# p1/p4 cells finish in seconds — extra runs there sharpen the
# significance gate of the PR comment at negligible cost.
BENCH_RUNS_CHEAP: '5'
UTOO_ALT_ENV: ${{ github.event.inputs.utoo_alt_env || '' }}
PHASES: ${{ github.event.inputs.phases || 'p0,p1,p3,p4' }}
BENCH_OUT_DIR: '/tmp/pm-bench-output'
PM_LIST: ${{ github.event_name == 'workflow_dispatch' && (github.event.inputs.phases_pm_list || 'utoo,utoo-next,utoo-npm,bun') || 'utoo,utoo-next,bun' }}
run: |
chmod +x bench/pm-bench-phases.sh
bash bench/pm-bench-phases.sh 2>&1 | tee /tmp/pm-bench-output/bench-phases-npmjs.log
# npmmirror only runs on manual dispatch — label-triggered runs stick to
# the npmjs leg to keep PR feedback loops short.
- name: Stash npmjs logs and wipe state before npmmirror
if: github.event_name == 'workflow_dispatch' && github.event.inputs.phases_registry != 'npmjs'
run: |
mkdir -p /tmp/bench-logs-stash
cp -r /tmp/pm-bench-output/. /tmp/bench-logs-stash/ 2>/dev/null || true
bash /tmp/wipe-bench-state.sh
mkdir -p /tmp/pm-bench-output
cp -r /tmp/bench-logs-stash/. /tmp/pm-bench-output/ 2>/dev/null || true
- name: Run phase-isolated benchmark · npmmirror
if: github.event_name == 'workflow_dispatch' && github.event.inputs.phases_registry != 'npmjs'
env:
PROJECT: ${{ github.event.inputs.project || 'ant-design' }}
REGISTRY: 'https://registry.npmmirror.com'
BENCH_RUNS: ${{ github.event.inputs.bench_runs || '3' }}
BENCH_RUNS_CHEAP: '5'
UTOO_ALT_ENV: ${{ github.event.inputs.utoo_alt_env || '' }}
PHASES: ${{ github.event.inputs.phases || 'p0,p1,p3,p4' }}
BENCH_OUT_DIR: '/tmp/pm-bench-output'
PM_LIST: ${{ github.event.inputs.phases_pm_list || 'utoo,utoo-next,utoo-npm,bun' }}
run: |
mkdir -p /tmp/pm-bench-output
bash bench/pm-bench-phases.sh 2>&1 | tee /tmp/pm-bench-output/bench-phases-npmmirror.log
- name: Upload bench logs
if: always()
uses: actions/upload-artifact@v4
with:
name: pm-bench-logs-linux
path: /tmp/pm-bench-output
retention-days: 7
- if: always()
uses: ./.github/actions/post-bench-phases-comment
with:
platform: linux
os: ubuntu-latest
project: ${{ github.event.inputs.project || 'ant-design' }}
github-token: ${{ secrets.GITHUB_TOKEN }}
bench-phases-mac:
# Mac runners are scarce and slow — only the manual dispatch path runs
# them. Label-triggered PR benches stick to linux to keep the feedback
# loop tight.
needs: [build-mac-arm64]
if: github.event_name == 'workflow_dispatch' && inputs.target == 'pm-bench-phases'
runs-on: macos-latest
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-utoo-bin
with:
artifact: utoo-mac-arm64
with-ut-symlink: 'false'
- name: Install bun
uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install pnpm 12
id: pnpm12
if: contains(github.event.inputs.phases_pm_list, 'pnpm12')
uses: pnpm/setup@v2
env:
PNPM_CONFIG_PM_ON_FAIL: ignore
with:
version: next-12
dest: ~/setup-pnpm12
install: false
- name: Expose pnpm 12 binary
if: contains(github.event.inputs.phases_pm_list, 'pnpm12')
run: echo "PNPM12_BIN=${{ steps.pnpm12.outputs.dest }}/pnpm" >> "$GITHUB_ENV"
- name: Install pnpm
if: contains(github.event.inputs.phases_pm_list, 'pnpm')
uses: pnpm/action-setup@v4
with:
version: 10
- name: Install aube
if: contains(github.event.inputs.phases_pm_list, 'aube')
run: npm install -g @endevco/aube@1.29.1
- uses: ./.github/actions/install-bench-tools
with:
platform: mac
- uses: ./.github/actions/install-utoo-npm-baseline
- name: Verify tools
run: |
hyperfine --version
utoo --version
"$UTOO_NPM_BIN" --version
bun --version
if command -v pnpm >/dev/null; then env PNPM_CONFIG_PM_ON_FAIL=ignore pnpm --version; fi
if [ -n "${PNPM12_BIN:-}" ]; then env PNPM_CONFIG_PM_ON_FAIL=ignore "$PNPM12_BIN" --version; fi
if command -v aube >/dev/null; then aube --version; fi
- name: Define cache wipe function
run: |
cat > /tmp/wipe-bench-state.sh <<'WIPE'
#!/usr/bin/env bash
rm -rf /tmp/pm-bench /tmp/pm-bench-results /tmp/pm-bench-output \
/tmp/utoo-bench-cache /tmp/utoo-npm-bench-cache /tmp/utoo-alt-bench-cache /tmp/bun-bench-cache \
/tmp/pnpm-bench-store /tmp/pnpm12-bench-store \
/tmp/aube-bench-data /tmp/aube-bench-cache \
~/.bun/install/cache ~/.cache/nm 2>/dev/null || true
mkdir -p /tmp/pm-bench-output
WIPE
chmod +x /tmp/wipe-bench-state.sh
- name: Wipe bench state before npmjs
run: bash /tmp/wipe-bench-state.sh
- name: Run phase-isolated benchmark · npmjs
if: github.event.inputs.phases_registry != 'npmmirror'
env:
PROJECT: ${{ github.event.inputs.project || 'ant-design' }}
REGISTRY: 'https://registry.npmjs.org'
BENCH_RUNS: ${{ github.event.inputs.bench_runs || '3' }}
BENCH_RUNS_CHEAP: '5'
UTOO_ALT_ENV: ${{ github.event.inputs.utoo_alt_env || '' }}
PHASES: ${{ github.event.inputs.phases || 'p0,p1,p3,p4' }}
BENCH_OUT_DIR: '/tmp/pm-bench-output'
PM_LIST: ${{ github.event.inputs.phases_pm_list || 'utoo,utoo-npm,bun' }}
run: |
chmod +x bench/pm-bench-phases.sh
bash bench/pm-bench-phases.sh 2>&1 | tee /tmp/pm-bench-output/bench-phases-npmjs.log
- name: Stash npmjs logs and wipe state before npmmirror
if: github.event.inputs.phases_registry != 'npmjs'
run: |
mkdir -p /tmp/bench-logs-stash
cp -r /tmp/pm-bench-output/. /tmp/bench-logs-stash/ 2>/dev/null || true
bash /tmp/wipe-bench-state.sh
mkdir -p /tmp/pm-bench-output
cp -r /tmp/bench-logs-stash/. /tmp/pm-bench-output/ 2>/dev/null || true
- name: Run phase-isolated benchmark · npmmirror
if: github.event.inputs.phases_registry != 'npmjs'
env:
PROJECT: ${{ github.event.inputs.project || 'ant-design' }}
REGISTRY: 'https://registry.npmmirror.com'
BENCH_RUNS: ${{ github.event.inputs.bench_runs || '3' }}
BENCH_RUNS_CHEAP: '5'
UTOO_ALT_ENV: ${{ github.event.inputs.utoo_alt_env || '' }}
PHASES: ${{ github.event.inputs.phases || 'p0,p1,p3,p4' }}
BENCH_OUT_DIR: '/tmp/pm-bench-output'
PM_LIST: ${{ github.event.inputs.phases_pm_list || 'utoo,utoo-npm,bun' }}
run: |
mkdir -p /tmp/pm-bench-output
bash bench/pm-bench-phases.sh 2>&1 | tee /tmp/pm-bench-output/bench-phases-npmmirror.log
- name: Upload bench logs
if: always()
uses: actions/upload-artifact@v4
with:
name: pm-bench-logs-mac
path: /tmp/pm-bench-output
retention-days: 7
- if: always()
uses: ./.github/actions/post-bench-phases-comment
with:
platform: mac
os: macos-latest
project: ${{ github.event.inputs.project || 'ant-design' }}
github-token: ${{ secrets.GITHUB_TOKEN }}
# ============================================================
# PM-BENCH-ALL (manual full PM shootout)
# workflow_dispatch with target=pm-bench-all only — comparing utoo
# against pnpm/pnpm12/yarn/bun/aube on cold + warm installs across
# configurable registries is slow, so it's never auto-triggered.
# ============================================================
pm-bench-all-linux:
needs: [build-linux]
if: github.event_name == 'workflow_dispatch' && inputs.target == 'pm-bench-all'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-utoo-bin
with:
artifact: utoo-linux-x64
- if: github.event.inputs.baseline != 'false'
uses: ./.github/actions/install-utoo-npm-baseline
- name: Build next branch utoo
if: github.event.inputs.build_next == 'true'
run: |
# The PATH already points at dist/utoo (current branch) — build
# `next` separately to /tmp so it doesn't shadow.
git fetch origin next --depth=1
WORKDIR=$(mktemp -d)
git --work-tree="$WORKDIR" --git-dir="$(pwd)/.git" checkout origin/next -- .
(cd "$WORKDIR" && git submodule update --init --recursive --depth 1 && cargo build --release --bin utoo -p utoo-pm)
cp "$WORKDIR/target/release/utoo" /tmp/utoo-next
chmod +x /tmp/utoo-next
echo "Baseline utoo (next) version: $(/tmp/utoo-next --version)"
echo "UTOO_NEXT_BIN=/tmp/utoo-next" >> $GITHUB_ENV
- name: Resolve pm_list
id: pm
run: |
PM_LIST="${{ github.event.inputs.pm_list || 'utoo,bun' }}"
if [ "$PM_LIST" = "all" ]; then
PM_LIST="utoo,pnpm,pnpm12,yarn,bun,aube"
fi
echo "list=$PM_LIST" >> $GITHUB_OUTPUT
- name: Install pnpm 12
id: pnpm12
if: contains(steps.pm.outputs.list, 'pnpm12')
uses: pnpm/setup@v2
env:
PNPM_CONFIG_PM_ON_FAIL: ignore
with:
version: next-12
dest: ~/setup-pnpm12
install: false
- name: Expose pnpm 12 binary
if: contains(steps.pm.outputs.list, 'pnpm12')
run: echo "PNPM12_BIN=${{ steps.pnpm12.outputs.dest }}/pnpm" >> "$GITHUB_ENV"
- name: Install pnpm
if: contains(steps.pm.outputs.list, 'pnpm')
uses: pnpm/action-setup@v4
with:
version: latest
- name: Install yarn
if: contains(steps.pm.outputs.list, 'yarn')
run: npm install -g yarn
- name: Install bun
if: contains(steps.pm.outputs.list, 'bun')
uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install aube
if: contains(steps.pm.outputs.list, 'aube')
run: npm install -g @endevco/aube
- name: Install hyperfine and GNU time
run: sudo apt-get update && sudo apt-get install -y hyperfine time
- name: Verify tools
run: |
hyperfine --version
IFS=',' read -ra PMS <<< "${{ steps.pm.outputs.list }}"
for pm in "${PMS[@]}"; do
if [ "$pm" = "pnpm12" ]; then
echo "$pm: $(env PNPM_CONFIG_PM_ON_FAIL=ignore "$PNPM12_BIN" --version 2>&1 || echo 'not installed')"
else
echo "$pm: $($pm --version 2>&1 || echo 'not installed')"
fi
done
- name: Run benchmark
run: |
chmod +x bench/pm-bench.sh
BENCH_COLD_RUNS=${{ github.event.inputs.cold_runs || '3' }} \
BENCH_WARM_RUNS=${{ github.event.inputs.warm_runs || '3' }} \
bash bench/pm-bench.sh ${{ github.event.inputs.registry || 'both' }} ${{ steps.pm.outputs.list }}
pm-bench-all-mac:
needs: [build-mac-arm64]
if: github.event_name == 'workflow_dispatch' && inputs.target == 'pm-bench-all'
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-utoo-bin
with:
artifact: utoo-mac-arm64
- if: github.event.inputs.baseline != 'false'
uses: ./.github/actions/install-utoo-npm-baseline
- name: Build next branch utoo
if: github.event.inputs.build_next == 'true'
run: |
git fetch origin next --depth=1
WORKDIR=$(mktemp -d)
git --work-tree="$WORKDIR" --git-dir="$(pwd)/.git" checkout origin/next -- .
(cd "$WORKDIR" && git submodule update --init --recursive --depth 1 && cargo build --release --bin utoo -p utoo-pm)
cp "$WORKDIR/target/release/utoo" /tmp/utoo-next
chmod +x /tmp/utoo-next
echo "Baseline utoo (next) version: $(/tmp/utoo-next --version)"
echo "UTOO_NEXT_BIN=/tmp/utoo-next" >> $GITHUB_ENV
- name: Resolve pm_list
id: pm
run: |
PM_LIST="${{ github.event.inputs.pm_list || 'utoo,bun' }}"
if [ "$PM_LIST" = "all" ]; then
PM_LIST="utoo,pnpm,pnpm12,yarn,bun,aube"
fi
echo "list=$PM_LIST" >> $GITHUB_OUTPUT
- name: Install pnpm 12
id: pnpm12
if: contains(steps.pm.outputs.list, 'pnpm12')
uses: pnpm/setup@v2
env:
PNPM_CONFIG_PM_ON_FAIL: ignore
with:
version: next-12
dest: ~/setup-pnpm12
install: false
- name: Expose pnpm 12 binary
if: contains(steps.pm.outputs.list, 'pnpm12')
run: echo "PNPM12_BIN=${{ steps.pnpm12.outputs.dest }}/pnpm" >> "$GITHUB_ENV"
- name: Install pnpm
if: contains(steps.pm.outputs.list, 'pnpm')
uses: pnpm/action-setup@v4
with:
version: latest
- name: Install yarn
if: contains(steps.pm.outputs.list, 'yarn')
run: npm install -g yarn
- name: Install bun
if: contains(steps.pm.outputs.list, 'bun')
uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install aube
if: contains(steps.pm.outputs.list, 'aube')
run: npm install -g @endevco/aube
- name: Install hyperfine
run: brew install hyperfine
- name: Verify tools
run: |
hyperfine --version
IFS=',' read -ra PMS <<< "${{ steps.pm.outputs.list }}"
for pm in "${PMS[@]}"; do
if [ "$pm" = "pnpm12" ]; then
echo "$pm: $(env PNPM_CONFIG_PM_ON_FAIL=ignore "$PNPM12_BIN" --version 2>&1 || echo 'not installed')"
else
echo "$pm: $($pm --version 2>&1 || echo 'not installed')"
fi
done
- name: Run benchmark
run: |
chmod +x bench/pm-bench.sh
BENCH_COLD_RUNS=${{ github.event.inputs.cold_runs || '3' }} \
BENCH_WARM_RUNS=${{ github.event.inputs.warm_runs || '3' }} \
bash bench/pm-bench.sh ${{ github.event.inputs.registry || 'both' }} ${{ steps.pm.outputs.list }}
# ============================================================
# PM-BENCH-PCAP (manual perf-debug only)
# workflow_dispatch with target=pm-bench-pcap. Captures tcpdump
# of the resolve phase per PM (linux only — sudo + raw sockets).
# ============================================================
pm-bench-pcap-linux:
needs: [build-linux]
if: github.event_name == 'workflow_dispatch' && inputs.target == 'pm-bench-pcap'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: 22
- uses: ./.github/actions/setup-utoo-bin
with:
artifact: utoo-linux-x64
with-ut-symlink: 'false'
- name: Install bun
uses: oven-sh/setup-bun@v2
with:
bun-version: latest
- name: Install pcap tools
# tcpdump captures, tshark + jq for post-capture metrics analysis,
# sysstat (iostat) for per-capture disk-saturation sampling.
# Pre-seed wireshark-common so tshark installs non-interactively
# (no need for setuid dumpcap; we only read existing pcaps).
run: |
echo "wireshark-common wireshark-common/install-setuid boolean false" | sudo debconf-set-selections
sudo DEBIAN_FRONTEND=noninteractive apt-get update
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y tcpdump tshark dnsutils hyperfine time jq sysstat
# The utoo-next binary is built upstream by build-linux's
# "Build next branch utoo (bench baseline)" step.
- name: Download utoo-next binary
uses: actions/download-artifact@v4
with:
name: utoo-next-linux-x64
path: /tmp/utoo-next-dist
- name: Install utoo-next
run: |
chmod +x /tmp/utoo-next-dist/utoo
mv /tmp/utoo-next-dist/utoo /tmp/utoo-next
echo "Baseline utoo (next) version: $(/tmp/utoo-next --version)"
echo "UTOO_NEXT_BIN=/tmp/utoo-next" >> $GITHUB_ENV
- name: Capture pcap
env:
PROJECT: ${{ github.event.inputs.project || 'ant-design' }}
REGISTRY: 'https://registry.npmjs.org'
run: |
chmod +x bench/pm-bench-pcap.sh
bash bench/pm-bench-pcap.sh
- name: Upload pcap artifact
if: always()
uses: actions/upload-artifact@v4
with:
name: pm-bench-pcap
path: /tmp/pm-bench-pcap
retention-days: 7