Compare distributed revisions with one benchmark harness

.github/workflows/performance.yml

@@ -71,10 +71,8 @@ jobs:
       - name: Distributed load and recovery, baseline then candidate
         run: |
           epmd -daemon
-          for revision in baseline candidate; do
-            timeout 20m bash "$revision/priv/bench/run_distributed.sh" 2>&1 |
-              tee "reports/distributed-$revision.log"
-          done
+          bash candidate/priv/bench/compare_distributed.sh \
+            "$PWD/baseline" "$PWD/candidate" "$PWD/reports"
       - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         if: always()
         with:

priv/bench/README.md

@@ -45,6 +45,22 @@ Uses 3 separate BEAM VMs (coordinator + 2 replicas) as OS processes:
 The script compiles once, starts both replicas in the background, then launches
 the coordinator. Replicas are killed automatically on exit.
 
+For a baseline/candidate comparison, run the candidate's comparison launcher:
+
+```bash
+epmd -daemon
+bash /path/to/candidate/priv/bench/compare_distributed.sh \
+  /path/to/baseline /path/to/candidate /path/to/reports --shards 4
+```
+
+Both runs use that same candidate harness and its workload/timing boundaries;
+only the `GROUP_BENCH_GROUP_PATH` library dependency changes. Each run has an
+isolated build directory so the candidate cannot reuse compiled baseline code.
+The launcher requires GNU `timeout`, caps each revision at 20 minutes, and
+retains separate logs. This is also the distributed comparison used by CI.
+An older baseline that cannot complete the common workload fails the comparison;
+the launcher never substitutes that revision's older, weaker completion checks.
+
 To isolate the 10,000-cluster lifecycle scenario:
 
 ```bash

priv/bench/compare_distributed.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ $# -lt 3 ]]; then
+  echo "Usage: bash compare_distributed.sh BASELINE_ROOT CANDIDATE_ROOT REPORTS_DIR [benchmark options]" >&2
+  exit 2
+fi
+
+harness="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+baseline="$(cd "$1" && pwd)"
+candidate="$(cd "$2" && pwd)"
+mkdir -p "$3" "${harness}/_build"
+reports="$(cd "$3" && pwd)"
+shift 3
+
+# Reuse the harness, not compiled library artifacts. A unique build root also
+# prevents a later comparison against different refs from inheriting either VM.
+build_root="$(mktemp -d "${harness}/_build/comparison.XXXXXX")"
+
+for revision in baseline candidate; do
+  if [[ "${revision}" == "baseline" ]]; then
+    library="${baseline}"
+  else
+    library="${candidate}"
+  fi
+
+  GROUP_BENCH_GROUP_PATH="${library}" \
+    MIX_BUILD_PATH="${build_root}/${revision}" \
+    timeout 20m bash "${harness}/run_distributed.sh" "$@" 2>&1 |
+    tee "${reports}/distributed-${revision}.log"
+done

test/benchmark_comparison_test.exs

@@ -0,0 +1,118 @@
+defmodule Group.BenchmarkComparisonTest do
+  use ExUnit.Case, async: true
+
+  @moduletag :local
+  @moduletag :tmp_dir
+
+  setup %{tmp_dir: directory} do
+    baseline = Path.expand("old library", directory)
+    candidate = Path.expand("new library", directory)
+    harness = Path.join(candidate, "priv/bench")
+    bin = Path.join(directory, "bin")
+    calls = Path.expand("calls", directory)
+    reports = Path.expand("reports", directory)
+    File.mkdir_p!(Path.join(baseline, "priv/bench"))
+    File.mkdir_p!(harness)
+    File.mkdir_p!(bin)
+
+    script = Path.join(harness, "compare_distributed.sh")
+    File.cp!("priv/bench/compare_distributed.sh", script)
+
+    File.write!(Path.join(baseline, "priv/bench/run_distributed.sh"), "exit 99\n")
+
+    File.write!(Path.join(harness, "run_distributed.sh"), """
+    set -eu
+    printf '%s|%s|%s|%s\\n' "$0" "$GROUP_BENCH_GROUP_PATH" "$MIX_BUILD_PATH" "$*" >> "$BENCH_CALL_LOG"
+    mkdir -p "$MIX_BUILD_PATH"
+    test ! -e "$MIX_BUILD_PATH/library"
+    printf '%s\\n' "$GROUP_BENCH_GROUP_PATH" > "$MIX_BUILD_PATH/library"
+    if [ "$GROUP_BENCH_GROUP_PATH" = "${BENCH_FAIL_LIBRARY:-}" ]; then exit 23; fi
+    printf 'measured %s\\n' "$GROUP_BENCH_GROUP_PATH"
+    """)
+
+    timeout = Path.join(bin, "timeout")
+
+    File.write!(timeout, """
+    #!/bin/sh
+    test "$1" = 20m || exit 98
+    shift
+    exec "$@"
+    """)
+
+    File.chmod!(timeout, 0o755)
+
+    {:ok,
+     baseline: baseline,
+     candidate: candidate,
+     harness: harness,
+     script: script,
+     reports: reports,
+     calls: calls,
+     env: [
+       {"PATH", Path.expand(bin) <> ":" <> System.fetch_env!("PATH")},
+       {"BENCH_CALL_LOG", calls},
+       {"BENCH_FAIL_LIBRARY", nil}
+     ]}
+  end
+
+  test "both libraries use the candidate harness and separate clean builds", context do
+    assert {_, 0} = compare(context)
+    assert [baseline, candidate] = read_calls(context)
+    [baseline_harness, baseline_library, baseline_build, baseline_args] = baseline
+    [candidate_harness, candidate_library, candidate_build, candidate_args] = candidate
+    expected_harness = Path.join(context.harness, "run_distributed.sh")
+    assert baseline_harness == expected_harness
+    assert candidate_harness == expected_harness
+    assert baseline_library == context.baseline
+    assert candidate_library == context.candidate
+    assert baseline_args == "--shards 4"
+    assert candidate_args == baseline_args
+    refute baseline_build == candidate_build
+
+    assert File.read!(Path.join(baseline_build, "library")) == context.baseline <> "\n"
+    assert File.read!(Path.join(candidate_build, "library")) == context.candidate <> "\n"
+
+    for revision <- [:baseline, :candidate] do
+      log = File.read!(Path.join(context.reports, "distributed-#{revision}.log"))
+      assert log == "measured #{context[revision]}\n"
+    end
+
+    # A second comparison must not reuse stale artifacts from either revision.
+    assert {_, 0} = compare(context)
+    builds = Enum.map(read_calls(context), &Enum.at(&1, 2))
+    assert length(Enum.uniq(builds)) == 4
+  end
+
+  for failure <- [:baseline, :candidate] do
+    @tag failure: failure
+    test "a failing #{failure} propagates through tee", context do
+      env =
+        List.keyreplace(
+          context.env,
+          "BENCH_FAIL_LIBRARY",
+          0,
+          {"BENCH_FAIL_LIBRARY", context[context.failure]}
+        )
+
+      context = %{context | env: env}
+      assert {_, 23} = compare(context)
+      assert length(read_calls(context)) == if(context.failure == :baseline, do: 1, else: 2)
+    end
+  end
+
+  defp compare(context) do
+    System.cmd(
+      "bash",
+      [context.script, context.baseline, context.candidate, context.reports, "--shards", "4"],
+      env: context.env,
+      stderr_to_stdout: true
+    )
+  end
+
+  defp read_calls(context) do
+    context.calls
+    |> File.read!()
+    |> String.split("\n", trim: true)
+    |> Enum.map(&String.split(&1, "|"))
+  end
+end