benchmark.yml

# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at https://mozilla.org/MPL/2.0/.

# Advisory viewer performance benchmark. Runs the real viewer (production
# build + real WASM pipeline) against the two small fixtures and reports
# per-metric deltas vs tests/benchmark/baseline.json as a PR comment + step
# summary. A regression is still ADVISORY — surfaced, not blocking (promote to
# blocking only once the runner-to-runner noise is understood). A HARNESS FAULT
# is not: if the baseline cannot be read, or a model in VIEWER_BENCHMARK_FILES
# has no baseline entry, this job goes RED. That distinction is the point --
# "no regressions" over a model that was never compared is not a result. This
# header used to say the job could never fail, which stopped being true in the
# same PR that added the refusal (#3200).
#
# The committed baseline is CI-recorded so this job diffs like-for-like:
# dispatch this workflow with `record_baseline` to produce a refreshed
# baseline.json as the `benchmark-baseline` artifact, then commit it via a
# normal PR (see tests/benchmark/README.md). Local machines are a different
# speed class — never commit locally recorded numbers.

name: Benchmark

on:
  pull_request:
    branches: [main]
    # Only paths that can plausibly move viewer load performance. The job is
    # advisory (not a required check), so skipping it entirely on unrelated
    # PRs is safe and free.
    paths:
      - 'rust/**'
      - 'packages/**'
      - 'apps/viewer/**'
      - 'Cargo.toml'
      - 'Cargo.lock'
      - 'rust-toolchain.toml'
      - 'scripts/build-wasm.sh'
      - 'scripts/check-benchmark-regression.js'
      - 'scripts/update-benchmark-baseline.mjs'
      - 'pnpm-lock.yaml'
      - 'pnpm-workspace.yaml'
      - 'turbo.json'
      - 'package.json'
      - 'playwright.config.ts'
      - 'tests/benchmark/**'
      - 'tests/models/manifest.json'
      - '.github/workflows/benchmark.yml'
  workflow_dispatch:
    inputs:
      record_baseline:
        description: 'Upload a refreshed baseline.json (CI-recorded numbers) as the benchmark-baseline artifact'
        type: boolean
        default: false

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  pull-requests: write

env:
  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

jobs:
  benchmark:
    name: Viewer benchmark (advisory)
    # Free GitHub-hosted runner (public repo). The benchmark measures a noisy
    # shared VM anyway; the +50% thresholds in benchmark:check absorb the
    # jitter, and the job is advisory. When Rust changed we compile wasm32
    # from source here too — slower than Depot but free, and correctness of
    # the measurement requires the PR's own WASM.
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          lfs: false
          persist-credentials: false

      # Same fast path as test.yml's build job: when the WASM source is
      # byte-identical to the published release tag, fetch the prebuilt
      # bundle and skip the Rust toolchain entirely.
      - name: Resolve prebuilt-WASM eligibility
        id: wasm
        run: echo "eligible=$(bash scripts/ci-wasm-prebuilt-eligible.sh)" >> "$GITHUB_OUTPUT"

      - name: Setup pnpm
        uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6.0.10

      - name: Setup Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: 22
          cache: pnpm

      - name: Setup WASM build toolchain
        if: steps.wasm.outputs.eligible != 'true'
        uses: ./.github/actions/setup-wasm-build
        with:
          cache-prefix: benchmark

      - name: Install dependencies
        run: pnpm install --frozen-lockfile

      - name: Fetch prebuilt WASM
        if: steps.wasm.outputs.eligible == 'true'
        run: |
          for attempt in 1 2 3; do
            if node scripts/fetch-prebuilt-wasm.mjs; then exit 0; fi
            echo "::warning::prebuilt WASM fetch attempt ${attempt} failed; retrying…"
            sleep $((attempt * 5))
          done
          echo "::error::Could not fetch the prebuilt @ifc-lite/wasm bundle after 3 attempts."
          exit 1

      - name: Build WASM
        if: steps.wasm.outputs.eligible != 'true'
        run: bash scripts/build-wasm.sh

      # Only the two small benchmark fixtures (~11 MB), not the full ~994 MiB
      # fixture tree — namespace the cache away from `ci-fixtures-…`.
      - name: Cache benchmark fixtures
        id: fixtures-cache
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: tests/models
          key: benchmark-fixtures-${{ runner.os }}-${{ hashFiles('tests/models/manifest.json') }}

      - name: Fetch benchmark fixtures
        if: steps.fixtures-cache.outputs.cache-hit != 'true'
        run: node scripts/fixtures/fetch-fixtures.mjs "ara3d/AC20-FZK-Haus.ifc" "various/01_Snowdon_Towers_Sample_Structural(1).ifc"

      # Unconditional — the fetch above is gated on a cache miss, so on a cache
      # hit nothing had ever confirmed the restored tree still matches the
      # manifest. Scoped to the same two paths: this cache holds only those.
      - name: Verify benchmark fixtures
        run: node scripts/fixtures/fetch-fixtures.mjs --check "ara3d/AC20-FZK-Haus.ifc" "various/01_Snowdon_Towers_Sample_Structural(1).ifc"

      # Production build: `vite preview` (the benchmark's webServer) serves
      # apps/viewer/dist. Turbo builds the workspace deps it needs.
      - name: Build viewer app
        run: pnpm turbo build --filter=@ifc-lite/viewer

      # benchmark:check reads every viewer-*.json in the results dir, so make
      # sure only this run's output is present (results are gitignored run
      # artifacts, but clear defensively in case one ever gets committed).
      - name: Clear stale benchmark results
        run: rm -f tests/benchmark/benchmark-results/viewer-*.json tests/benchmark/benchmark-results/viewer-*.console.log

      - name: Run viewer benchmark
        id: benchmark
        env:
          VIEWER_BENCHMARK_FILES: 'tests/models/ara3d/AC20-FZK-Haus.ifc,tests/models/various/01_Snowdon_Towers_Sample_Structural(1).ifc'
          # Advisory: the spec logs threshold violations instead of failing;
          # the verdict lives in benchmark:check + the PR comment.
          VIEWER_BENCHMARK_ADVISORY: '1'
          VIEWER_BENCHMARK_BUILD_MODE: 'production'
          # Divert the json reporter to a file so the step log stays readable.
          PLAYWRIGHT_JSON_OUTPUT_NAME: ${{ runner.temp }}/playwright-benchmark.json
        run: pnpm test:benchmark:viewer:ci

      # The check can now FAIL on a harness fault (#3200) -- a renamed fixture
      # that matches no baseline, or a metric the baseline had and this run
      # lost. Those are "the benchmark did not run", not "the benchmark is
      # slow", so `--advisory` does not cover them.
      #
      # The summary is written FIRST, unconditionally: the report is the
      # DIAGNOSIS of that failure, and a `bash -e` step that dies before the
      # `cat` would throw away exactly the output someone needs to fix it.
      - name: Check against baseline (regressions advisory, harness faults fatal)
        run: |
          set +e
          node scripts/check-benchmark-regression.js --advisory --markdown "$RUNNER_TEMP/benchmark-report.md"
          status=$?
          set -e
          [ -f "$RUNNER_TEMP/benchmark-report.md" ] && cat "$RUNNER_TEMP/benchmark-report.md" >> "$GITHUB_STEP_SUMMARY"
          exit $status

      # Sticky comment: update the existing benchmark comment if present,
      # else create one. Best-effort — fork PRs get a read-only token, and
      # the step summary above already carries the same report.
      - name: Comment benchmark report on PR
        if: always() && github.event_name == 'pull_request'
        continue-on-error: true
        env:
          GH_TOKEN: ${{ github.token }}
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          marker='<!-- viewer-benchmark-report -->'
          body_file="$RUNNER_TEMP/benchmark-report.md"
          # If the benchmark died before writing a report, `-F body=@` fails,
          # continue-on-error swallows it, and the PREVIOUS run's comment is
          # left standing -- a crashed run presenting as the last green one.
          # Say what happened instead. The marker must stay in the body or the
          # next run creates a second comment rather than updating this one.
          if [ ! -s "$body_file" ]; then
            printf '%s\n\n%s\n' "$marker" \
              '⚠ The benchmark job produced no report — it failed or was cancelled before the check step. This replaces a stale result rather than leaving one standing; see the job log.' \
              > "$body_file"
          fi
          # --paginate applies --jq per page, so a multi-page comment list can
          # emit one line per page — keep only the first id.
          existing=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" --paginate \
            --jq "[.[] | select(.body | contains(\"$marker\")) | .id] | first // empty" | head -n1)
          if [ -n "$existing" ]; then
            gh api --method PATCH "repos/${{ github.repository }}/issues/comments/${existing}" -F body=@"$body_file"
          else
            gh api --method POST "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" -F body=@"$body_file"
          fi

      - name: Upload benchmark results
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: benchmark-results
          path: |
            tests/benchmark/benchmark-results/viewer-*.json
            tests/benchmark/benchmark-results/viewer-*.console.log
          retention-days: 30
          if-no-files-found: error

      # Baseline refresh (workflow_dispatch with record_baseline): merge this
      # run's numbers into baseline.json and upload it as an artifact. A human
      # (or agent) commits it via a normal PR so the change is reviewed like
      # any other — this job intentionally has no push access.
      # `always()` so this still runs when the baseline COMPARISON failed --
      # that failure is often exactly why the dispatch was sent, and gating on
      # job success deadlocks the only supported repair path (#3200).
      #
      # But gated on the BENCHMARK step itself succeeding. The spec writes each
      # result JSON BEFORE asserting `streamCompleteMs` is present and
      # `totalMeshes > 0`, so a failed load leaves a JSON on disk that
      # `update-benchmark-baseline.mjs` would happily accept -- refreshing the
      # baseline from a run that did not load the model. That is a worse
      # outcome than a stale baseline, and it would look like a successful
      # refresh.
      - name: Record refreshed baseline
        id: record
        if: always() && github.event_name == 'workflow_dispatch' && inputs.record_baseline && steps.benchmark.outcome == 'success'
        run: node scripts/update-benchmark-baseline.mjs --environment "github-actions ubuntu-latest, viewer-benchmark-ci (headless Chrome, SwiftShader ANGLE), production build"

      # Gated on the RECORD step, not on the dispatch inputs: a bare `always()`
      # here would upload the COMMITTED baseline.json unchanged when the record
      # step failed or was skipped, publishing it as a `benchmark-baseline`
      # artifact that looks like a refresh of numbers never written.
      - name: Upload refreshed baseline
        if: always() && steps.record.outcome == 'success'
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: benchmark-baseline
          path: tests/benchmark/baseline.json
          retention-days: 30
          if-no-files-found: error