fix(test-suite): freeze orchestrated fhevm baseline

.github/workflows/test-suite-e2e-tests.yml

@@ -267,6 +267,12 @@
         run: |
           echo "COMPAT_OVERRIDES=test-suite" >> "$GITHUB_ENV"
 
+      - name: Require frozen baseline lock for orchestrated runs
+        if: ${{ inputs.compat-test == '' && inputs.orchestrated && inputs.lock-artifact-name == '' && github.event_name != 'workflow_dispatch' }}
+        run: |
+          echo "Orchestrated e2e requires a frozen baseline lock artifact" >&2
+          exit 1
+
       - name: Resolve latest-main lock once
         if: ${{ inputs.compat-test == '' && inputs.lock-artifact-name == '' && !inputs.orchestrated }}
         working-directory: test-suite/fhevm
@@ -278,12 +284,26 @@
           fi
           echo "LOCK_FILE=$lock_file" >> "$GITHUB_ENV"
 
+      - name: Resolve orchestrated manual baseline lock once
+        if: ${{ inputs.compat-test == '' && inputs.lock-artifact-name == '' && inputs.orchestrated && github.event_name == 'workflow_dispatch' }}
+        working-directory: test-suite/fhevm
+        run: |
+          lock_file="$(./fhevm-cli resolve --target latest-main | tail -n1)"
+          if [ ! -f "$lock_file" ]; then
+            echo "Could not locate resolved lock file: $lock_file" >&2
+            exit 1
+          fi
+          echo "LOCK_FILE=$lock_file" >> "$GITHUB_ENV"
+
       - name: Resolve stack plan and display effective component versions
         working-directory: test-suite/fhevm
         run: |
           args=(--scenario "$SCENARIO")
           if [ -n "${LOCK_FILE:-}" ]; then
             args+=(--lock-file "$LOCK_FILE")
+          elif [ "${{ inputs.orchestrated }}" = "true" ]; then
+            echo "Orchestrated e2e requires a frozen baseline lock artifact" >&2
+            exit 1
           else
             args+=(--target latest-main)
           fi
@@ -304,6 +324,9 @@
           args=(--scenario "$SCENARIO")
           if [ -n "${LOCK_FILE:-}" ]; then
             args+=(--lock-file "$LOCK_FILE")
+          elif [ "${{ inputs.orchestrated }}" = "true" ]; then
+            echo "Orchestrated e2e requires a frozen baseline lock artifact" >&2
+            exit 1
           else
             args+=(--target latest-main)
           fi

.github/workflows/test-suite-orchestrate-e2e-tests.yml

@@ -14,9 +14,50 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
-  coprocessor-docker-build:
+  resolve-baseline:
+    name: resolve-baseline
     if: &build-trigger-condition |
       startsWith(github.head_ref, 'mergify/merge-queue/') || startsWith(github.base_ref, 'release/') || contains(github.event.pull_request.labels.*.name, 'e2e orchestrate test')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: 'read' # Required to checkout repository code
+    env:
+      BASE_SHA: ${{ github.event.pull_request.base.sha }}
+      BASE_REF: ${{ github.base_ref }}
+    outputs:
+      lock-artifact-name: ${{ steps.resolve-baseline.outputs.lock-artifact-name }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: 'false'
+          fetch-depth: 0
+
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2.1.3
+
+      - name: Install CLI deps
+        working-directory: test-suite/fhevm
+        run: bun install --frozen-lockfile
+
+      - id: resolve-baseline
+        name: Resolve base-sha baseline lock
+        working-directory: test-suite/fhevm
+        run: |
+          set -euo pipefail
+          lock_path="$(./fhevm-cli resolve --target sha --sha "$BASE_SHA" --ref "$BASE_REF" --reset | tail -n 1)"
+          artifact_name="fhevm-base-lock-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          echo "lock-path=$lock_path" >> "$GITHUB_OUTPUT"
+          echo "lock-artifact-name=$artifact_name" >> "$GITHUB_OUTPUT"
+
+      - name: Upload baseline lock
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: ${{ steps.resolve-baseline.outputs.lock-artifact-name }}
+          path: ${{ steps.resolve-baseline.outputs.lock-path }}
+
+  coprocessor-docker-build:
+    if: *build-trigger-condition
     uses: ./.github/workflows/coprocessor-docker-build.yml
     permissions: &docker_permissions
       actions: 'read' # Required to read workflow run information
@@ -61,6 +102,7 @@ jobs:
   create-e2e-tests-input:
     name: create-e2e-tests-input
     needs:
+      - resolve-baseline
       - coprocessor-docker-build
       - gateway-contracts-docker-build
       - host-contracts-docker-build
@@ -72,6 +114,7 @@ jobs:
       contents: 'read' # Required to read the checked-in release baseline defaults
     env:
       BASE_COMMIT_HASH: ${{ github.event.pull_request.base.sha }}
+      BASE_REF: ${{ github.base_ref }}
       NEW_COMMIT_HASH: ${{ github.event.pull_request.head.sha }}
       IS_RELEASE_PR: ${{ startsWith(github.base_ref, 'release/') }}
       DOCKER_BUILD_RESULTS: ${{ toJSON(needs) }}
@@ -91,6 +134,7 @@ jobs:
       gateway-version: ${{ steps.create-e2e-tests-input.outputs.gateway-version }}
       host-version: ${{ steps.create-e2e-tests-input.outputs.host-version }}
       kms-core-version: ${{ steps.create-e2e-tests-input.outputs.kms-core-version }}
+      lock-artifact-name: ${{ needs.resolve-baseline.outputs.lock-artifact-name }}
       relayer-migrate-version: ${{ steps.create-e2e-tests-input.outputs.relayer-migrate-version }}
       relayer-version: ${{ steps.create-e2e-tests-input.outputs.relayer-version }}
       test-suite-version: ${{ steps.create-e2e-tests-input.outputs.test-suite-version }}
@@ -124,24 +168,23 @@ jobs:
             gateway-version
             host-version
             kms-core-version
-            relayer-migrate-version
-            relayer-version
-            test-suite-version
           )
           for key in "${outputs[@]}"; do
             echo "$key=" >> "$GITHUB_OUTPUT"
           done
 
           if [ "$IS_RELEASE_PR" = "true" ]; then
             if git cat-file -e "${BASE_COMMIT_HASH}:test-suite/fhevm/profiles/latest-supported.json" 2>/dev/null; then
-              core_version="$(git show "${BASE_COMMIT_HASH}:test-suite/fhevm/profiles/latest-supported.json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["env"]["CORE_VERSION"])')"
+              defaults_json="$(git show "${BASE_COMMIT_HASH}:test-suite/fhevm/profiles/latest-supported.json")"
+              core_version="$(printf '%s' "$defaults_json" | python3 -c 'import json,sys; print(json.load(sys.stdin)["env"]["CORE_VERSION"])')"
             elif git cat-file -e "${BASE_COMMIT_HASH}:test-suite/fhevm/fhevm-cli" 2>/dev/null; then
-              core_version="$(git show "${BASE_COMMIT_HASH}:test-suite/fhevm/fhevm-cli" | sed -n 's/^export CORE_VERSION=.*:-"\\([^"]*\\)".*/\\1/p' | head -n 1)"
+              defaults_shell="$(git show "${BASE_COMMIT_HASH}:test-suite/fhevm/fhevm-cli")"
+              core_version="$(printf '%s' "$defaults_shell" | sed -n 's/^export CORE_VERSION=.*:-"\\([^"]*\\)".*/\\1/p' | head -n 1)"
             else
               core_version=""
             fi
             if [ -z "$core_version" ]; then
-              echo "Could not extract CORE_VERSION from release base ${BASE_COMMIT_HASH}" >&2
+              echo "Could not extract release CORE_VERSION from ${BASE_COMMIT_HASH}" >&2
               exit 1
             fi
             echo "kms-core-version=$core_version" >> "$GITHUB_OUTPUT"
@@ -175,16 +218,59 @@ jobs:
           kms-connector-docker-build|tx_sender_build_result|connector-tx-sender-version
           gateway-contracts-docker-build|build_result|gateway-version
           host-contracts-docker-build|build_result|host-version
-          relayer-docker-build|relayer_migrate_build_result|relayer-migrate-version
-          relayer-docker-build|relayer_build_result|relayer-version
-          test-suite-docker-build|build_result|test-suite-version
           EOF
 
           if [ "${#failures[@]}" -gt 0 ]; then
             echo "Required repo-owned build outputs failed: ${failures[*]}" >&2
             exit 1
           fi
 
+          relayer_result="$(jq -r '.["relayer-docker-build"].outputs["relayer_build_result"] // "missing"' "$needs_file")"
+          case "$relayer_result" in
+            success)
+              echo "relayer-version=$short_tag" >> "$GITHUB_OUTPUT"
+              ;;
+            skipped)
+              echo "relayer-version=" >> "$GITHUB_OUTPUT"
+              ;;
+            *)
+              echo "Required repo-owned build outputs failed: relayer-docker-build.relayer_build_result=$relayer_result" >&2
+              exit 1
+              ;;
+          esac
+
+          test_suite_result="$(jq -r '.["test-suite-docker-build"].outputs["build_result"] // "missing"' "$needs_file")"
+          case "$test_suite_result" in
+            success)
+              echo "test-suite-version=$short_tag" >> "$GITHUB_OUTPUT"
+              ;;
+            skipped)
+              if [ "$BASE_REF" = "release/0.11.x" ]; then
+                echo "test-suite-version=" >> "$GITHUB_OUTPUT"
+              else
+                echo "test-suite-version=$base_short_tag" >> "$GITHUB_OUTPUT"
+              fi
+              ;;
+            *)
+              echo "Required repo-owned build outputs failed: test-suite-docker-build.build_result=$test_suite_result" >&2
+              exit 1
+              ;;
+          esac
+
+          relayer_migrate_result="$(jq -r '.["relayer-docker-build"].outputs["relayer_migrate_build_result"] // "missing"' "$needs_file")"
+          case "$relayer_migrate_result" in
+            success)
+              echo "relayer-migrate-version=$short_tag" >> "$GITHUB_OUTPUT"
+              ;;
+            skipped)
+              echo "relayer-migrate-version=" >> "$GITHUB_OUTPUT"
+              ;;
+            *)
+              echo "Required repo-owned build outputs failed: relayer-docker-build.relayer_migrate_build_result=$relayer_migrate_result" >&2
+              exit 1
+              ;;
+          esac
+
   run-e2e-tests:
     needs: [create-e2e-tests-input]
     if: ${{ startsWith(github.head_ref, 'mergify/merge-queue/') || startsWith(github.base_ref, 'release/') || contains(github.event.pull_request.labels.*.name, 'e2e orchestrate test') }}
@@ -215,6 +301,7 @@ jobs:
       gateway-version: ${{ needs.create-e2e-tests-input.outputs.gateway-version }}
       host-version: ${{ needs.create-e2e-tests-input.outputs.host-version }}
       kms-core-version: ${{ needs.create-e2e-tests-input.outputs.kms-core-version }}
+      lock-artifact-name: ${{ needs.create-e2e-tests-input.outputs.lock-artifact-name }}
       relayer-migrate-version: ${{ needs.create-e2e-tests-input.outputs.relayer-migrate-version }}
       relayer-version: ${{ needs.create-e2e-tests-input.outputs.relayer-version }}
       scenario: ${{ startsWith(github.base_ref, 'release/') && 'two-of-two' || 'two-of-two-multi-chain' }}

test-suite/fhevm/src/cli.test.ts

@@ -141,6 +141,12 @@ describe("cli", () => {
     expect(result.stderr).toContain("--target sha requires --sha");
   });
 
+  test("rejects --ref without a sha target", async () => {
+    const result = await execCli(["up", "--target", "latest-main", "--ref", "release/0.12.x"]);
+    expect(result.code).toBe(1);
+    expect(result.stderr).toContain("--ref requires --target sha");
+  });
+
   test("rejects combining resume with an explicit target", async () => {
     const result = await execCli(["up", "--target", "latest-main", "--resume"]);
     expect(result.code).toBe(1);
@@ -165,6 +171,15 @@ describe("cli", () => {
     });
   });
 
+  test("sha target accepts ref before validating the sha value", async () => {
+    await withState(persistedState(), async (env) => {
+      const result = await execCli(["up", "--target", "sha", "--sha", "invalidhex", "--ref", "release/0.12.x"], env);
+      expect(result.code).toBe(1);
+      expect(result.stderr).toContain("Invalid sha invalidhex; expected 7 or 40 hex characters");
+      expect(result.stderr).not.toContain("--ref requires --target sha");
+    });
+  });
+
   test("validates pause scope", async () => {
     const result = await execCli(["pause", "nope"]);
     expect(result.code).toBe(1);

test-suite/fhevm/src/cli.ts

@@ -165,6 +165,7 @@ const upCommandDefinition = {
   args: {
     target: { type: "string", description: "Bundle source to boot." },
     sha: { type: "string", description: "Commit SHA to resolve when --target sha is used." },
+    ref: { type: "string", description: "Git ref used to validate and interpret --target sha." },
     override: { type: "string", description: "Build selected workspace groups locally. Use --override test-suite to run local e2e test changes.", alias: "o" },
     "from-step": { type: "string", description: `Start from a specific pipeline step when resuming or previewing. Valid: ${STEP_NAMES.join(", ")}.` },
     "lock-file": { type: "string", description: "Use an existing lock snapshot instead of resolving versions live." },
@@ -246,13 +247,14 @@ const root = defineCommand({
       args: {
         target: { type: "string", description: "Bundle source to resolve." },
         sha: { type: "string", description: "Commit SHA to resolve when --target sha is used." },
+        ref: { type: "string", description: "Git ref used to validate and interpret --target sha." },
         "lock-file": { type: "string", description: "Use an existing lock snapshot instead of resolving versions live." },
         reset: { type: "boolean", description: "Discard cached resolution and regenerate from scratch." },
       },
       async run({ args }) {
         const parsed = parseUpInput(args);
         if (parsed.resume || parsed.dryRun || parsed.fromStep || parsed.overrides.length || parsed.scenarioPath) {
-          throw new PreflightError("resolve only supports --target, --sha, --lock-file, and --reset");
+          throw new PreflightError("resolve only supports --target, --sha, --ref, --lock-file, and --reset");
         }
         const { lockPath } = await resolveBundle(parsed, process.env);
         console.log(lockPath);

test-suite/fhevm/src/input/up.ts

@@ -36,6 +36,7 @@ const parseLocalOverride = (value: string): LocalOverride[] => {
 export const parseUpInput = (args: Record<string, unknown>) => {
   const target = asString(args.target);
   const sha = asString(args.sha);
+  const ref = asString(args.ref);
   const fromStepRaw = asString(args["from-step"] ?? args.fromStep);
   const lockFile = asString(args["lock-file"] ?? args.lockFile);
   const scenarioPath = asString(args.scenario);
@@ -61,6 +62,9 @@ export const parseUpInput = (args: Record<string, unknown>) => {
   if (validTarget === "sha" && !sha) {
     throw new PreflightError("--target sha requires --sha");
   }
+  if (ref && validTarget !== "sha") {
+    throw new PreflightError("--ref requires --target sha");
+  }
   if (validTarget !== "sha" && sha) {
     throw new PreflightError("--sha requires --target sha");
   }
@@ -94,6 +98,7 @@ export const parseUpInput = (args: Record<string, unknown>) => {
     target: validTarget,
     requestedTarget: target as VersionTarget | undefined,
     sha,
+    ref,
     overrides,
     scenarioPath,
     fromStep,