performance.yml

name: Performance

on:
  push:
    branches: [master, dev, 'release/**']
  pull_request:
    branches: [master, dev, 'release/**']

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

permissions:
  contents: read

env:
  DOTNET_NOLOGO: true
  DOTNET_CLI_TELEMETRY_OPTOUT: true
  DOTNET_TieredPGO: 0

jobs:
  allocations:
    # This job was named "Allocation budgets (B2, B6)" and asserted only B2, because
    # nothing under src/ constructed an ActivitySource or a Meter — there was no
    # telemetry to be zero-cost about, and naming B6 here made an unmeasured budget
    # look gated on every pull request.
    #
    # WP-90 (2026-08-01) built the telemetry, so B6 has a subject. The allocation half
    # is now asserted by TelemetryCostTests, in the same filter below: with no listener
    # the decorator is not created, and a decorator abandoned by its listener runs a
    # 4-step ephemeral flow for 0 B. That test found the defect it was written for —
    # StartActivity($"step {i} {id}") evaluates its interpolation before returning null,
    # at 68 B per step.
    #
    # The name still says B2 alone, and deliberately: B6 is "0 ns and 0 B", and only the
    # allocation half is asserted. The nanosecond half is not measured rather than
    # measured badly, which is this repository's standing position on timing here.
    name: Allocation budget (B2, and B6's allocation half)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4
      - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9  # v4
        with:
          global-json-file: global.json

      - name: Hard-zero allocation assertions
        # Unit tests, not benchmarks, running on every pull request. Allocation
        # counts are deterministic; timings are not — see docs/benchmarks §5, where
        # WP-3's assumption that ratios were reliable is withdrawn with evidence.
        # This is the gate that actually protects budget B2.
        run: |
          dotnet test tests/FlowX.Core.Tests --configuration Release \
            --filter "FullyQualifiedName~AllocationBudget" \
            --logger "console;verbosity=detailed"
          dotnet test tests/FlowX.Runtime.Tests --configuration Release \
            --filter "FullyQualifiedName~EngineAllocation|FullyQualifiedName~Concurrency|FullyQualifiedName~TelemetryCost" \
            --logger "console;verbosity=detailed"

  benchmarks:
    name: Benchmark budgets (B1, B3, B12 isolated)
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4
      - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9  # v4
        with:
          global-json-file: global.json

      - name: Run benchmarks
        run: >
          dotnet run --configuration Release --project tests/FlowX.Benchmarks
          -- --filter '*'

      - name: Gate against the committed baseline
        # Blocking: allocation changes and p95 over a documented budget.
        # Advisory: absolute and ratio drift — neither is reproducible on a shared
        # runner, as docs/benchmarks/README.md section 5 demonstrates with two runs
        # of the same commit. Add --strict once the baseline is recorded on
        # dedicated hardware (WP-11).
        run: python3 scripts/check-benchmark-budgets.py BenchmarkDotNet.Artifacts

      - name: Publish the report
        if: always()
        run: |
          if [ -d BenchmarkDotNet.Artifacts/results ]; then
            {
              echo "## Benchmark results"
              echo
              cat BenchmarkDotNet.Artifacts/results/*-report-github.md 2>/dev/null || true
            } >> "$GITHUB_STEP_SUMMARY"
          fi

      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
        if: always()
        with:
          name: benchmark-results
          path: BenchmarkDotNet.Artifacts/

  build-overhead:
    name: Budget B12 — build overhead
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4
      - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9  # v4
        with:
          global-json-file: global.json

      - name: Measure build overhead against identical non-FlowX code
        # Builds the reference sample twice per round, producing the same final
        # compilation both times and differing only in whether the generator ran.
        # The script exits non-zero above the +8 % budget, which is the number
        # ADR-0002's revisit clause depends on — see docs/benchmarks/B12.md.
        #
        # Fewer rounds than the local run: a hosted runner is noisier, and the
        # verdict rests on a margin of roughly twenty times the budget rather than
        # on precision.
        run: |
          # pipefail explicitly: the default shell is `bash -e`, which takes the exit
          # status of the LAST command in a pipeline. Without this, tee always succeeds
          # and a FAIL verdict would be reported as a passing job — a gate that cannot
          # fail, which is worse than no gate.
          set -o pipefail
          ./scripts/measure-build-overhead.sh 7 | tee -a "$GITHUB_STEP_SUMMARY"

  generator-cost:
    name: Generator cost vs the committed baseline
    runs-on: ubuntu-latest
    # BLOCKING, and since ADR-0014 it is the only gate on compile-time cost there is.
    #
    # IT ANSWERS TWO QUESTIONS NOW. Relative — "did this change make the generator more
    # expensive than the committed figure" — and, since the P1 criterion was re-expressed in
    # bytes per flow and per capability on 2026-08-10, absolute: `check-generator-cost.py`
    # fails the run when a ceiling in generator-cost-baseline.json is breached. Before that
    # the criterion was a wall-clock ratio measured by `scale-overhead` below, which this job
    # could only reprint. A criterion this job can evaluate is one it must evaluate; the old
    # one was reprinted for four working packages and stopped being watched.
    #
    # WHY THIS EXISTS. docs/benchmarks/B12-scale.md §5.2 bisects a commit that took
    # FlowPlanGenerator from 5.60 to 27.28 ms per flow — 4.9× — and merged unnoticed
    # along with three further working packages. The only gate on compile-time cost at
    # the time was `scale-overhead` below: absolute, against a +8 % budget the project
    # was already failing by a wide margin, and therefore advisory. An absolute gate you
    # are already failing says exactly as much about a 4.9× regression as it does about
    # a no-op commit, which is nothing.
    #
    # WHAT IT MEASURES. Bytes allocated by one run of FlowPlanGenerator over a synthetic
    # project, compared against docs/benchmarks/generator-cost-baseline.json. Not wall
    # clock: measured over twelve independent runs on a 4-core container at load averages
    # from 5.8 to 21.1, the gated figure moved by 0.069 % while wall clock over the same
    # runs moved by 123 %. That is why this job can be blocking on a hosted runner and
    # `scale-overhead` cannot.
    #
    # It is the split scripts/check-benchmark-budgets.py already makes at run time —
    # allocations are exact and gateable on shared hardware, timings are not — applied to
    # compile time. The threshold is +2 %, 29× the worst deviation ever observed of the
    # statistic it gates, and 1/50 of the +101 % the bisected commit produces.
    #
    # WHAT IT DOES NOT SAY. That the build is fast enough. It is not: P1's exit criterion
    # is a wall-clock question, `scale-overhead` is the instrument for it, and it fails.
    # The checker reprints that criterion on every run, including passing ones.
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4
      - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9  # v4
        with:
          global-json-file: global.json

      - name: Measure, then gate against the committed baseline
        # Exit codes are the gate, and each of the four means something different:
        #
        #   0  PASS          the generator costs what the baseline says.
        #   1  FAIL          it costs more, or the subject/toolchain moved and the
        #                    committed number no longer describes anything. Blocking.
        #   2  INCONCLUSIVE  the run could not judge. NOT blocking — but not silent
        #                    either; see below.
        #   3  broken        a flow failed to analyse, or a build failed. Blocking,
        #                    because that is a defect and not a measurement.
        #
        # INCONCLUSIVE is handled differently here than in `scale-overhead`, because it
        # means something different. There, exit 2 means the runner was too busy to
        # separate signal from scheduling noise — the weather, and failing a pull request
        # for the weather is what that job's comment refuses to do. Here the measurement
        # does not care how busy the runner is, so exit 2 means the same generator gave
        # different answers about the same sources: a broken instrument, not a busy one.
        # The right response to a broken instrument is to try it again and then say
        # loudly that the gate did not run — so this retries once, and on a second
        # INCONCLUSIVE raises a warning annotation that a reviewer sees on the pull
        # request while leaving the merge unblocked. A gate that cannot judge must not
        # pass quietly, and must not fail for something no author did.
        run: |
          set -o pipefail

          # `|| return` on the first line, not a bare sequence: a measurement that exits
          # non-zero has not written a usable samples file, and running the checker over a
          # stale one from the previous attempt would turn a broken run into a verdict.
          measure_and_check() {
            ./scripts/measure-generator-cost.py --json generator-cost.json \
              | tee -a "$GITHUB_STEP_SUMMARY" || return $?
            ./scripts/check-generator-cost.py generator-cost.json \
              | tee -a "$GITHUB_STEP_SUMMARY"
          }

          set +e
          measure_and_check
          rc=$?
          if [ "$rc" -eq 2 ]; then
            echo "::notice::Generator cost measurement was inconclusive. Retrying once."
            measure_and_check
            rc=$?
          fi
          set -e

          case "$rc" in
            0) ;;
            2) echo "::warning::The generator-cost gate did not run: the probe gave "\
                    "inconsistent byte counts for identical sources, twice. This is not a "\
                    "regression and it is not a pass. Investigate the probe before trusting "\
                    "the next green run." ;;
            *) exit "$rc" ;;
          esac

      - name: Publish a candidate baseline
        # Not applied — offered. The committed baseline is recorded on a container, and
        # nothing yet establishes that a hosted runner produces byte-identical counts;
        # docs/benchmarks/generator-cost-gate.md is explicit that this is the design's
        # one unverified assumption. If a runner ever disagrees by more than the
        # threshold, the fix is to re-record the baseline where the gate runs, not to
        # widen the threshold until the disagreement fits. This artifact is that
        # re-recording, ready to download and commit.
        if: always()
        run: |
          cp docs/benchmarks/generator-cost-baseline.json generator-cost-baseline.candidate.json
          if [ -f generator-cost.json ]; then
            ./scripts/check-generator-cost.py generator-cost.json \
              --baseline generator-cost-baseline.candidate.json --record || true
          fi

      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02  # v4
        if: always()
        with:
          name: generator-cost
          path: |
            generator-cost.json
            generator-cost-baseline.candidate.json

  generator-cost-self-test:
    name: The generator-cost gate can fail
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4

      - name: A regression under the threshold must fail the gate
        # A gate nobody has seen fail is an assumption, not a gate — the same argument
        # `gate-self-test` below makes for the allocation budgets. This one takes the
        # committed baseline, fabricates a measurement 2.5 % above it, and asserts the
        # checker rejects it. 2.5 % rather than 100 % on purpose: a self-test that only
        # proves a doubling is caught proves nothing about where the threshold actually
        # sits.
        run: |
          python3 - <<'PY'
          import json
          baseline = json.load(open('docs/benchmarks/generator-cost-baseline.json'))
          sizes = []
          for flows, entry in baseline['sizes'].items():
              sizes.append({
                  'flows': int(flows),
                  'allocated_bytes': int(entry['allocatedBytes'] * 1.025),
                  'allocated_bytes_per_flow': 0,
                  'spread_percent': 0.01,
                  'elapsed_ms': 0,
                  'elapsed_spread_percent': 0,
                  'generated_trees': entry['generatedTrees'],
                  'generated_chars': entry['generatedChars'],
                  'diagnostics': entry['diagnostics'],
                  'sources_sha256': entry['sourcesSha256'],
              })
          recorded = baseline['recordedOn']
          json.dump({
              'meta': {
                  'environment': {
                      'roslyn_product_version': recorded['roslyn'],
                      'roslyn_assembly_version': recorded['roslynAssemblyVersion'],
                      'framework': recorded['framework'],
                      'server_gc': False,
                  },
              },
              'sizes': sizes,
          }, open('/tmp/injected.json', 'w'))
          PY

          if python3 scripts/check-generator-cost.py /tmp/injected.json; then
            echo "::error::The gate accepted an injected 2.5 % generator cost regression."
            exit 1
          fi
          echo "Gate correctly rejected the injected regression."

  scale-overhead:
    name: P1 scale — 200-flow build overhead
    runs-on: ubuntu-latest
    # ADVISORY, AND NO LONGER P1'S EXIT CRITERION — see docs/benchmarks/B12-scale.md.
    #
    # It was the criterion, expressed as "a 200-flow solution builds within +8 %", and it
    # was advisory because the project missed it by fifty-nine points: blocking would have
    # redded every pull request for the whole of P1 over a defect none of them introduced.
    #
    # ADR-0014 replaced that ratio on 2026-08-10. A ratio's denominator is the user's own
    # code, so the same generator measured +0.4 % on the one-flow reference sample and
    # +67.1 % at 200 flows — a number that can be re-argued but not passed or failed. The
    # criterion is now generator allocation per flow and per capability, `generator-cost`
    # above measures it, and `check-generator-cost.py` fails the run on a breached ceiling.
    #
    # SO THIS JOB IS NOW INFORMATION RATHER THAN A GATE, and stays advisory for that reason
    # rather than the old one. What it measures is still worth seeing — wall-clock overhead
    # against a non-FlowX build is what a person actually waits for, and no allocation figure
    # substitutes for it. What it must not do is fail a pull request against a criterion the
    # repository has replaced.
    #
    # What is NOT advisory: the script's exit code. It exits 1 above +8 %, 2 when the
    # machine was too noisy to tell the two apart, pipefail is set below, and the step
    # is shown as failed either way. Only the job's effect on the pull request is
    # suppressed.
    #
    # WHAT DISCHARGED IT. `generator-cost` above is blocking, and it now answers both
    # questions rather than one. It was only ever *relative* — "did this change make the
    # generator more expensive than the committed figure" — which is the gate the regression
    # bisected in B12-scale.md §5.2 needed and did not have. Since ADR-0014 it also evaluates
    # the absolute criterion, because in bytes per flow and per capability the criterion is
    # exactly what its own samples already hold. "Is the build good enough" and "did this
    # change make it worse" are now both asked, in one place, by something that fails.
    #
    # DO NOT MAKE THIS JOB BLOCKING. It would be blocking on a superseded criterion. If
    # wall-clock overhead is ever to gate again, it needs a criterion of its own, stated in
    # a unit this runner can resolve — twelve identical runs of one tree disagreed on wall
    # clock by 139 % (docs/benchmarks/generator-cost-gate.md), which is why the criterion
    # that replaced it is not in milliseconds.
    #
    # WHEN IT IS MADE BLOCKING, exit code 2 must stay non-blocking. INCONCLUSIVE is
    # not evidence of a regression — it is the harness reporting that this runner
    # could not separate the generator's cost from its own scheduling noise, and
    # failing a pull request for that fails it for the weather. Blocking form:
    #
    #   ./scripts/measure-scale-overhead.sh --rounds 5 --sizes 50,200 \
    #     | tee -a "$GITHUB_STEP_SUMMARY"
    #   rc=$?
    #   case $rc in
    #     0) ;;
    #     2) echo "::warning::scale overhead inconclusive on this runner" ;;
    #     *) exit $rc ;;
    #   esac
    continue-on-error: true
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4
      - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9  # v4
        with:
          global-json-file: global.json

      - name: Measure build overhead on a synthetic 200-flow solution
        # Generates a synthetic solution of 50 and then 200 flows and builds each one
        # both ways, exactly as the B12 job does for the one-flow reference sample:
        # same final compilation, differing only in whether the generator ran.
        #
        # Two sizes because the ratio is the less interesting half. A generator whose
        # cost grows faster than the flow count fails at 2 000 flows whatever it does
        # at 200, and only a second point can tell the two apart. The script prints
        # the growth exponent together with a confidence interval for it, because a
        # fit through two noisy points reads far more convincing than it is.
        #
        # Five rounds, not the three this job ran before. Each round is a sandwich of
        # three builds per size — A B A, alternating which arm takes the outside slots
        # — whose outer pair doubles as an A/A control. The statistics that decide the
        # verdict need enough rounds to form an interval at all, and below five the
        # script says so in its own output rather than quoting one anyway.
        run: |
          # pipefail explicitly: the default shell is `bash -e`, which takes the exit
          # status of the LAST command in a pipeline. Without this, tee always succeeds
          # and a FAIL verdict would be reported as a passing step — a gate that cannot
          # fail, which is worse than no gate.
          set -o pipefail
          ./scripts/measure-scale-overhead.sh --rounds 5 --sizes 50,200 \
            | tee -a "$GITHUB_STEP_SUMMARY"

  gate-self-test:
    name: The gate can fail
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4

      - name: Injecting a regression must fail the gate
        # A gate nobody has seen fail is an assumption, not a gate. This builds a
        # synthetic report with a 64 B allocation where the baseline says zero and
        # asserts that check-benchmark-budgets.py rejects it.
        run: |
          python3 - <<'PY'
          import json, os
          os.makedirs('/tmp/synthetic/results', exist_ok=True)
          baseline = json.load(open('docs/benchmarks/baseline.json'))
          benchmarks = []
          for name, entry in baseline['benchmarks'].items():
              type_name, method = name.rsplit('.', 1)
              benchmarks.append({
                  'Type': type_name,
                  'Method': method,
                  'Statistics': {
                      'Mean': entry['absoluteNs'],
                      'Percentiles': {'P95': entry['absoluteNs'] * 1.02},
                  },
                  # The injected defect: a benchmark the baseline says is allocation-free.
                  # The injected defect targets a benchmark the baseline says is
                  # allocation-free, so it exercises the blocking check specifically.
                  'Memory': {'BytesAllocatedPerOperation':
                             64 if method == 'Query' else entry['allocatedBytes']},
              })
          by_type = {}
          for b in benchmarks:
              by_type.setdefault(b['Type'], []).append(b)
          for type_name, group in by_type.items():
              path = f"/tmp/synthetic/results/{type_name}-report-full.json"
              json.dump({'Benchmarks': group}, open(path, 'w'))
          PY

          if python3 scripts/check-benchmark-budgets.py /tmp/synthetic; then
            echo "::error::The gate accepted an injected 64 B allocation regression."
            exit 1
          fi
          echo "Gate correctly rejected the injected regression."

      - name: A run that measured nothing must be rejected as absent, not as a crash
        # The other failure mode, found on 2026-08-01 by running the benchmarks with a
        # duplicate .csproj on disk: BenchmarkDotNet exports a benchmark it could not
        # run with a null Statistics rather than omitting it. The gate did reject such
        # a run — but by raising TypeError inside a dict comprehension, so the log said
        # the script was broken and not that nothing had been measured. This asserts
        # both halves: non-zero exit, and the benchmark named as absent.
        run: |
          python3 - <<'PY'
          import json, os
          os.makedirs('/tmp/unmeasured/results', exist_ok=True)
          baseline = json.load(open('docs/benchmarks/baseline.json'))
          by_type = {}
          for name, entry in baseline['benchmarks'].items():
              type_name, method = name.rsplit('.', 1)
              # Every benchmark in one type fails to run; the rest measure cleanly, so
              # the gate has to keep reporting the good ones while blocking on these.
              failed = type_name == 'EngineBenchmarks'
              by_type.setdefault(type_name, []).append({
                  'Type': type_name,
                  'Method': method,
                  'Statistics': None if failed else {
                      'Mean': entry['absoluteNs'],
                      'Percentiles': {'P95': entry['absoluteNs'] * 1.02},
                  },
                  'Memory': None if failed else {
                      'BytesAllocatedPerOperation': entry['allocatedBytes'],
                  },
              })
          for type_name, group in by_type.items():
              path = f"/tmp/unmeasured/results/{type_name}-report-full.json"
              json.dump({'Benchmarks': group}, open(path, 'w'))
          PY

          set +e
          output=$(python3 scripts/check-benchmark-budgets.py /tmp/unmeasured 2>&1)
          status=$?
          set -e
          echo "$output"

          if [ "$status" -eq 0 ]; then
            echo "::error::The gate accepted a run in which no engine benchmark was measured."
            exit 1
          fi

          if echo "$output" | grep -q 'Traceback'; then
            echo "::error::The gate rejected the run by crashing rather than by reporting it."
            exit 1
          fi

          if ! echo "$output" | grep -q 'EngineBenchmarks.SagaFailure: in the baseline but absent'; then
            echo "::error::The gate did not name the unmeasured benchmark as absent."
            exit 1
          fi
          echo "Gate correctly reported the unmeasured benchmarks as absent."

  durability-self-test:
    name: The durability gate (B7, B8) can fail
    runs-on: ubuntu-latest
    # No database, and that is the point of putting it here rather than beside the rig.
    # tests/FlowX.Durability.Bench needs a PostgreSQL and minutes; its JUDGEMENT needs
    # neither, and the judgement is where a softening would happen unseen. The checker
    # downgrades a B7 latency breach to a warning when the offered rate was not sustained
    # — correct, because the queue is then the finding — and widening that by one
    # condition would switch the gate off silently.
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262  # v4

      - name: Nine verdicts, and only two of them green
        run: python3 scripts/selftest-durability-verdict.py

      - name: The recorded run is the one the self-test claims it is
        # The self-test builds every case by mutating docs/benchmarks/B7-B8-durability.json,
        # so a recorded run replaced by a fabricated one would take the whole suite with it.
        # Two facts are asserted directly: the commit was durable, and the arm the p99 is
        # read from actually ran.
        run: |
          python3 - <<'PY'
          import json, sys
          document = json.load(open('docs/benchmarks/B7-B8-durability.json'))
          machine = document['machine']
          if machine.get('synchronousCommit') != 'on':
              sys.exit("::error::The recorded run did not commit durably, so it is not a B7 record.")
          commit = next(a for a in document['budgets'] if a['budget'] == 'B7')
          if commit['response']['operations'] < 1000:
              sys.exit("::error::The recorded run measured too few commits to carry a p99.")
          print(f"Recorded run: {commit['response']['operations']} commits, "
                f"synchronous_commit {machine['synchronousCommit']}.")
          PY