fix(ci): close the path-filter holes, and gate the class that made them (#3312) #3975
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This Source Code Form is subject to the terms of the Mozilla Public | |
| # License, v. 2.0. If a copy of the MPL was not distributed with this | |
| # file, You can obtain one at https://mozilla.org/MPL/2.0/. | |
| # Advisory viewer performance benchmark. Runs the real viewer (production | |
| # build + real WASM pipeline) against the two small fixtures and reports | |
| # per-metric deltas vs tests/benchmark/baseline.json as a PR comment + step | |
| # summary. A regression is still ADVISORY — surfaced, not blocking (promote to | |
| # blocking only once the runner-to-runner noise is understood). A HARNESS FAULT | |
| # is not: if the baseline cannot be read, or a model in VIEWER_BENCHMARK_FILES | |
| # has no baseline entry, this job goes RED. That distinction is the point -- | |
| # "no regressions" over a model that was never compared is not a result. This | |
| # header used to say the job could never fail, which stopped being true in the | |
| # same PR that added the refusal (#3200). | |
| # | |
| # The committed baseline is CI-recorded so this job diffs like-for-like: | |
| # dispatch this workflow with `record_baseline` to produce a refreshed | |
| # baseline.json as the `benchmark-baseline` artifact, then commit it via a | |
| # normal PR (see tests/benchmark/README.md). Local machines are a different | |
| # speed class — never commit locally recorded numbers. | |
| name: Benchmark | |
| on: | |
| pull_request: | |
| branches: [main] | |
| # Only paths that can plausibly move viewer load performance. The job is | |
| # advisory (not a required check), so skipping it entirely on unrelated | |
| # PRs is safe and free. | |
| paths: | |
| - 'rust/**' | |
| - 'packages/**' | |
| - 'apps/viewer/**' | |
| - 'Cargo.toml' | |
| - 'Cargo.lock' | |
| - 'rust-toolchain.toml' | |
| - 'scripts/build-wasm.sh' | |
| - 'scripts/check-benchmark-regression.js' | |
| - 'scripts/update-benchmark-baseline.mjs' | |
| - 'pnpm-lock.yaml' | |
| - 'pnpm-workspace.yaml' | |
| - 'turbo.json' | |
| - 'package.json' | |
| - 'playwright.config.ts' | |
| - 'tests/benchmark/**' | |
| - 'tests/models/manifest.json' | |
| - '.github/workflows/benchmark.yml' | |
| workflow_dispatch: | |
| inputs: | |
| record_baseline: | |
| description: 'Upload a refreshed baseline.json (CI-recorded numbers) as the benchmark-baseline artifact' | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| env: | |
| FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true | |
| jobs: | |
| benchmark: | |
| name: Viewer benchmark (advisory) | |
| # Free GitHub-hosted runner (public repo). The benchmark measures a noisy | |
| # shared VM anyway; the +50% thresholds in benchmark:check absorb the | |
| # jitter, and the job is advisory. When Rust changed we compile wasm32 | |
| # from source here too — slower than Depot but free, and correctness of | |
| # the measurement requires the PR's own WASM. | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: | |
| lfs: false | |
| persist-credentials: false | |
| # Same fast path as test.yml's build job: when the WASM source is | |
| # byte-identical to the published release tag, fetch the prebuilt | |
| # bundle and skip the Rust toolchain entirely. | |
| - name: Resolve prebuilt-WASM eligibility | |
| id: wasm | |
| run: echo "eligible=$(bash scripts/ci-wasm-prebuilt-eligible.sh)" >> "$GITHUB_OUTPUT" | |
| - name: Setup pnpm | |
| uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 | |
| with: | |
| node-version: 22 | |
| cache: pnpm | |
| - name: Setup WASM build toolchain | |
| if: steps.wasm.outputs.eligible != 'true' | |
| uses: ./.github/actions/setup-wasm-build | |
| with: | |
| cache-prefix: benchmark | |
| - name: Install dependencies | |
| run: pnpm install --frozen-lockfile | |
| - name: Fetch prebuilt WASM | |
| if: steps.wasm.outputs.eligible == 'true' | |
| run: | | |
| for attempt in 1 2 3; do | |
| if node scripts/fetch-prebuilt-wasm.mjs; then exit 0; fi | |
| echo "::warning::prebuilt WASM fetch attempt ${attempt} failed; retrying…" | |
| sleep $((attempt * 5)) | |
| done | |
| echo "::error::Could not fetch the prebuilt @ifc-lite/wasm bundle after 3 attempts." | |
| exit 1 | |
| - name: Build WASM | |
| if: steps.wasm.outputs.eligible != 'true' | |
| run: bash scripts/build-wasm.sh | |
| # Only the two small benchmark fixtures (~11 MB), not the full ~994 MiB | |
| # fixture tree — namespace the cache away from `ci-fixtures-…`. | |
| - name: Cache benchmark fixtures | |
| id: fixtures-cache | |
| uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 | |
| with: | |
| path: tests/models | |
| key: benchmark-fixtures-${{ runner.os }}-${{ hashFiles('tests/models/manifest.json') }} | |
| - name: Fetch benchmark fixtures | |
| if: steps.fixtures-cache.outputs.cache-hit != 'true' | |
| run: node scripts/fixtures/fetch-fixtures.mjs "ara3d/AC20-FZK-Haus.ifc" "various/01_Snowdon_Towers_Sample_Structural(1).ifc" | |
| # Unconditional — the fetch above is gated on a cache miss, so on a cache | |
| # hit nothing had ever confirmed the restored tree still matches the | |
| # manifest. Scoped to the same two paths: this cache holds only those. | |
| - name: Verify benchmark fixtures | |
| run: node scripts/fixtures/fetch-fixtures.mjs --check "ara3d/AC20-FZK-Haus.ifc" "various/01_Snowdon_Towers_Sample_Structural(1).ifc" | |
| # Production build: `vite preview` (the benchmark's webServer) serves | |
| # apps/viewer/dist. Turbo builds the workspace deps it needs. | |
| - name: Build viewer app | |
| run: pnpm turbo build --filter=@ifc-lite/viewer | |
| # benchmark:check reads every viewer-*.json in the results dir, so make | |
| # sure only this run's output is present (results are gitignored run | |
| # artifacts, but clear defensively in case one ever gets committed). | |
| - name: Clear stale benchmark results | |
| run: rm -f tests/benchmark/benchmark-results/viewer-*.json tests/benchmark/benchmark-results/viewer-*.console.log | |
| - name: Run viewer benchmark | |
| id: benchmark | |
| env: | |
| VIEWER_BENCHMARK_FILES: 'tests/models/ara3d/AC20-FZK-Haus.ifc,tests/models/various/01_Snowdon_Towers_Sample_Structural(1).ifc' | |
| # Advisory: the spec logs threshold violations instead of failing; | |
| # the verdict lives in benchmark:check + the PR comment. | |
| VIEWER_BENCHMARK_ADVISORY: '1' | |
| VIEWER_BENCHMARK_BUILD_MODE: 'production' | |
| # Divert the json reporter to a file so the step log stays readable. | |
| PLAYWRIGHT_JSON_OUTPUT_NAME: ${{ runner.temp }}/playwright-benchmark.json | |
| run: pnpm test:benchmark:viewer:ci | |
| # The check can now FAIL on a harness fault (#3200) -- a renamed fixture | |
| # that matches no baseline, or a metric the baseline had and this run | |
| # lost. Those are "the benchmark did not run", not "the benchmark is | |
| # slow", so `--advisory` does not cover them. | |
| # | |
| # The summary is written FIRST, unconditionally: the report is the | |
| # DIAGNOSIS of that failure, and a `bash -e` step that dies before the | |
| # `cat` would throw away exactly the output someone needs to fix it. | |
| - name: Check against baseline (regressions advisory, harness faults fatal) | |
| run: | | |
| set +e | |
| node scripts/check-benchmark-regression.js --advisory --markdown "$RUNNER_TEMP/benchmark-report.md" | |
| status=$? | |
| set -e | |
| [ -f "$RUNNER_TEMP/benchmark-report.md" ] && cat "$RUNNER_TEMP/benchmark-report.md" >> "$GITHUB_STEP_SUMMARY" | |
| exit $status | |
| # Sticky comment: update the existing benchmark comment if present, | |
| # else create one. Best-effort — fork PRs get a read-only token, and | |
| # the step summary above already carries the same report. | |
| - name: Comment benchmark report on PR | |
| if: always() && github.event_name == 'pull_request' | |
| continue-on-error: true | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| marker='<!-- viewer-benchmark-report -->' | |
| body_file="$RUNNER_TEMP/benchmark-report.md" | |
| # If the benchmark died before writing a report, `-F body=@` fails, | |
| # continue-on-error swallows it, and the PREVIOUS run's comment is | |
| # left standing -- a crashed run presenting as the last green one. | |
| # Say what happened instead. The marker must stay in the body or the | |
| # next run creates a second comment rather than updating this one. | |
| if [ ! -s "$body_file" ]; then | |
| printf '%s\n\n%s\n' "$marker" \ | |
| '⚠ The benchmark job produced no report — it failed or was cancelled before the check step. This replaces a stale result rather than leaving one standing; see the job log.' \ | |
| > "$body_file" | |
| fi | |
| # --paginate applies --jq per page, so a multi-page comment list can | |
| # emit one line per page — keep only the first id. | |
| existing=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" --paginate \ | |
| --jq "[.[] | select(.body | contains(\"$marker\")) | .id] | first // empty" | head -n1) | |
| if [ -n "$existing" ]; then | |
| gh api --method PATCH "repos/${{ github.repository }}/issues/comments/${existing}" -F body=@"$body_file" | |
| else | |
| gh api --method POST "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" -F body=@"$body_file" | |
| fi | |
| - name: Upload benchmark results | |
| if: always() | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-results | |
| path: | | |
| tests/benchmark/benchmark-results/viewer-*.json | |
| tests/benchmark/benchmark-results/viewer-*.console.log | |
| retention-days: 30 | |
| if-no-files-found: error | |
| # Baseline refresh (workflow_dispatch with record_baseline): merge this | |
| # run's numbers into baseline.json and upload it as an artifact. A human | |
| # (or agent) commits it via a normal PR so the change is reviewed like | |
| # any other — this job intentionally has no push access. | |
| # `always()` so this still runs when the baseline COMPARISON failed -- | |
| # that failure is often exactly why the dispatch was sent, and gating on | |
| # job success deadlocks the only supported repair path (#3200). | |
| # | |
| # But gated on the BENCHMARK step itself succeeding. The spec writes each | |
| # result JSON BEFORE asserting `streamCompleteMs` is present and | |
| # `totalMeshes > 0`, so a failed load leaves a JSON on disk that | |
| # `update-benchmark-baseline.mjs` would happily accept -- refreshing the | |
| # baseline from a run that did not load the model. That is a worse | |
| # outcome than a stale baseline, and it would look like a successful | |
| # refresh. | |
| - name: Record refreshed baseline | |
| id: record | |
| if: always() && github.event_name == 'workflow_dispatch' && inputs.record_baseline && steps.benchmark.outcome == 'success' | |
| run: node scripts/update-benchmark-baseline.mjs --environment "github-actions ubuntu-latest, viewer-benchmark-ci (headless Chrome, SwiftShader ANGLE), production build" | |
| # Gated on the RECORD step, not on the dispatch inputs: a bare `always()` | |
| # here would upload the COMMITTED baseline.json unchanged when the record | |
| # step failed or was skipped, publishing it as a `benchmark-baseline` | |
| # artifact that looks like a refresh of numbers never written. | |
| - name: Upload refreshed baseline | |
| if: always() && steps.record.outcome == 'success' | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: benchmark-baseline | |
| path: tests/benchmark/baseline.json | |
| retention-days: 30 | |
| if-no-files-found: error |