chore(test-suite): sacrificial merge-queue validation for #2066

.dockerignore

@@ -97,6 +97,7 @@ charts/
 # -----------------------------------------------------------------------------
 # Git-related (keep .git/ for build metadata, exclude others)
 # -----------------------------------------------------------------------------
+.git/lfs/
 .gitignore
 .gitattributes
 **/.gitkeep

.github/workflows/test-suite-e2e-operators-tests.yml

@@ -82,6 +82,21 @@ jobs:
       contents: 'read' # Required to checkout repository code
       id-token: 'write' # Required for OIDC authentication
       packages: 'read' # Required to read GitHub packages/container registry
+    env:
+      GH_TOKEN: ${{ secrets.GHCR_READ_TOKEN || github.token }}
+      CONNECTOR_DB_MIGRATION_VERSION: ${{ inputs.connector_version }}
+      CONNECTOR_GW_LISTENER_VERSION: ${{ inputs.connector_version }}
+      CONNECTOR_KMS_WORKER_VERSION: ${{ inputs.connector_version }}
+      CONNECTOR_TX_SENDER_VERSION: ${{ inputs.connector_version }}
+      COPROCESSOR_DB_MIGRATION_VERSION: ${{ inputs.db_migration_version }}
+      HOST_VERSION: ${{ inputs.host_version }}
+      GATEWAY_VERSION: ${{ inputs.gateway_version }}
+      COPROCESSOR_HOST_LISTENER_VERSION: ${{ inputs.host_listener_version }}
+      COPROCESSOR_GW_LISTENER_VERSION: ${{ inputs.gateway_listener_version }}
+      COPROCESSOR_TX_SENDER_VERSION: ${{ inputs.tx_sender_version }}
+      COPROCESSOR_TFHE_WORKER_VERSION: ${{ inputs.tfhe_worker_version }}
+      COPROCESSOR_SNS_WORKER_VERSION: ${{ inputs.sns_worker_version }}
+      COPROCESSOR_ZKPROOF_WORKER_VERSION: ${{ inputs.zkproof_worker_version }}
     runs-on: ${{ needs.setup-instance.outputs.runner-name }}
     needs: setup-instance
     timeout-minutes: 1440
@@ -94,7 +109,10 @@ jobs:
       - name: Setup Docker
         uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
 
-      - name: Install Foundry
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2.1.3
+
+      - name: Install foundry
         uses: foundry-rs/foundry-toolchain@82dee4ba654bd2146511f85f0d013af94670c4de # v1.4.0
 
       - name: Login to GitHub Container Registry
@@ -111,24 +129,15 @@ jobs:
           username: ${{ secrets.CGR_USERNAME }}
           password: ${{ secrets.CGR_PASSWORD }}
 
+      - name: Install CLI deps
+        working-directory: test-suite/fhevm
+        run: |
+          bun install --frozen-lockfile
+
       - name: Deploy fhevm Stack
         working-directory: test-suite/fhevm
-        env:
-          CONNECTOR_DB_MIGRATION_VERSION: ${{ inputs.connector_version }}
-          CONNECTOR_GW_LISTENER_VERSION: ${{ inputs.connector_version }}
-          CONNECTOR_KMS_WORKER_VERSION: ${{ inputs.connector_version }}
-          CONNECTOR_TX_SENDER_VERSION: ${{ inputs.connector_version }}
-          DB_MIGRATION_VERSION: ${{ inputs.db_migration_version }}
-          HOST_VERSION: ${{ inputs.host_version }}
-          GATEWAY_VERSION: ${{ inputs.gateway_version }}
-          HOST_LISTENER_VERSION: ${{ inputs.host_listener_version }}
-          GW_LISTENER_VERSION: ${{ inputs.gateway_listener_version }}
-          TX_SENDER_VERSION: ${{ inputs.tx_sender_version }}
-          TFHE_WORKER_VERSION: ${{ inputs.tfhe_worker_version }}
-          SNS_WORKER_VERSION: ${{ inputs.sns_worker_version }}
-          ZKPROOF_WORKER_VERSION: ${{ inputs.zkproof_worker_version }}
         run: |
-          ./fhevm-cli deploy --coprocessors 2 --coprocessor-threshold 2
+          ./fhevm-cli up --target latest-release --coprocessors 2 --threshold 2
 
       - name: All operators tests
         working-directory: test-suite/fhevm
@@ -142,32 +151,39 @@ jobs:
 
       - name: Show logs on test failure
         working-directory: test-suite/fhevm
-        if: always()
+        if: failure()
         run: |
-          echo "::group::Relayer Logs"
-          ./fhevm-cli logs relayer
-          echo "::endgroup::"
-          echo "::group::SNS Worker Logs"
-          ./fhevm-cli logs sns-worker | grep -v "Selected 0 rows to process"
-          echo "::endgroup::"
-          echo "::group::Transaction Sender Logs (filtered)"
-          ./fhevm-cli logs transaction-sender | grep -v "Selected 0 rows to process"
-          echo "::endgroup::"
-          echo "::group::Coprocessor 2 - SNS Worker"
-          ./fhevm-cli logs coprocessor-2-sns-worker 2>/dev/null | grep -v "Selected 0 rows to process" || true
-          echo "::endgroup::"
-          echo "::group::Coprocessor 2 - Transaction Sender (filtered)"
-          ./fhevm-cli logs coprocessor-2-transaction-sender 2>/dev/null | grep -v "Selected 0 rows to process" || true
-          echo "::endgroup::"
-          echo "::group::Coprocessor 2 - TFHE Worker"
-          ./fhevm-cli logs coprocessor-2-tfhe-worker 2>/dev/null || true
-          echo "::endgroup::"
+          snapshot_logs() {
+            local group="$1"
+            local container="$2"
+            local filter="${3:-}"
+            echo "::group::${group}"
+            if [ -n "$filter" ]; then
+              docker logs --tail 200 "${container}" 2>&1 | grep -v "$filter" || true
+            else
+              docker logs --tail 200 "${container}" 2>&1 || true
+            fi
+            echo "::endgroup::"
+          }
+          snapshot_logs "Relayer Logs" fhevm-relayer
+          snapshot_logs "Host Listener" coprocessor-host-listener
+          snapshot_logs "Gateway Listener" coprocessor-gw-listener
+          snapshot_logs "SNS Worker Logs" coprocessor-sns-worker "Selected 0 rows to process"
+          snapshot_logs "Transaction Sender Logs (filtered)" coprocessor-transaction-sender "Selected 0 rows to process"
+          snapshot_logs "ZKProof Worker" coprocessor-zkproof-worker
+          snapshot_logs "TFHE Worker" coprocessor-tfhe-worker
+          snapshot_logs "Coprocessor 2 - Host Listener" coprocessor1-host-listener
+          snapshot_logs "Coprocessor 2 - Gateway Listener" coprocessor1-gw-listener
+          snapshot_logs "Coprocessor 2 - SNS Worker" coprocessor1-sns-worker "Selected 0 rows to process"
+          snapshot_logs "Coprocessor 2 - Transaction Sender (filtered)" coprocessor1-transaction-sender "Selected 0 rows to process"
+          snapshot_logs "Coprocessor 2 - ZKProof Worker" coprocessor1-zkproof-worker
+          snapshot_logs "Coprocessor 2 - TFHE Worker" coprocessor1-tfhe-worker
 
       - name: Cleanup
         working-directory: test-suite/fhevm
         if: always()
         run: |
-          ./fhevm-cli clean
+          ./fhevm-cli clean --images
 
   teardown-instance:
     name: test-suite-e2e-operators-tests/teardown

.github/workflows/test-suite-e2e-tests.yml

@@ -3,6 +3,18 @@ name: test-suite-e2e-tests
 on:
   workflow_dispatch:
     inputs: &workflow_inputs
+      image_mode:
+        description: "Use registry images or build workspace overrides on the runner"
+        default: "registry"
+        type: string
+      override:
+        description: "Comma-separated override values for workspace mode (for example: all, coprocessor, or coprocessor:host-listener)"
+        default: ""
+        type: string
+      stack-era:
+        description: "Stack era hint for workflow_call consumers (leave empty for manual dispatch)"
+        default: ""
+        type: string
       coprocessor-db-migration-version:
         description: "Coprocessor DB Migration Image Version"
         default: ""
@@ -63,14 +75,14 @@ on:
         description: "Relayer version"
         default: ""
         type: string
+      relayer-migrate-version:
+        description: "Relayer migrate version"
+        default: ""
+        type: string
       kms-core-version:
         description: "KMS Core version"
         default: ""
         type: string
-      deploy-build:
-        description: "Build local Docker images from the checked out repository before deploy"
-        default: false
-        type: boolean
   workflow_call:
     secrets:
       GHCR_READ_TOKEN:
@@ -95,6 +107,9 @@ jobs:
       id-token: 'write' # Required for OIDC authentication
       packages: 'read' # Required to read GitHub packages/container registry
     env:
+      GH_TOKEN: ${{ secrets.GHCR_READ_TOKEN || github.token }}
+      IMAGE_MODE: ${{ inputs.image_mode || 'registry' }}
+      WORKSPACE_OVERRIDE: ${{ inputs.override || '' }}
       COPROCESSOR_DB_MIGRATION_VERSION: ${{ inputs.coprocessor-db-migration-version }}
       COPROCESSOR_HOST_LISTENER_VERSION: ${{ inputs.coprocessor-host-listener-version }}
       COPROCESSOR_GW_LISTENER_VERSION: ${{ inputs.coprocessor-gw-listener-version }}
@@ -110,7 +125,9 @@ jobs:
       CONNECTOR_TX_SENDER_VERSION: ${{ inputs.connector-tx-sender-version }}
       TEST_SUITE_VERSION: ${{ inputs.test-suite-version }}
       RELAYER_VERSION: ${{ inputs.relayer-version }}
+      RELAYER_MIGRATE_VERSION: ${{ inputs.relayer-migrate-version }}
       CORE_VERSION: ${{ inputs.kms-core-version }}
+      STACK_ERA: ${{ inputs.stack-era }}
     runs-on: large_ubuntu_32
     steps:
       - name: Checkout code
@@ -121,7 +138,10 @@ jobs:
       - name: Setup Docker
         uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
 
-      - name: Install Foundry
+      - name: Setup Bun
+        uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2.1.3
+
+      - name: Install foundry
         uses: foundry-rs/foundry-toolchain@82dee4ba654bd2146511f85f0d013af94670c4de # v1.4.0
 
       - name: Login to GitHub Container Registry
@@ -138,22 +158,74 @@ jobs:
           username: ${{ secrets.CGR_USERNAME }}
           password: ${{ secrets.CGR_PASSWORD }}
 
-      - name: Display component versions
-        env:
-          JSON_INPUT: ${{ toJSON(inputs) }}
-        run: |
-          echo "Component versions: $JSON_INPUT"
+      - name: Install CLI deps
+        working-directory: test-suite/fhevm
+        run: bun install --frozen-lockfile
 
-      - name: Deploy fhevm Stack
+      - name: Resolve CLI up args
         working-directory: test-suite/fhevm
-        env:
-          DEPLOY_BUILD: ${{ inputs.deploy-build }}
         run: |
-          if [[ "$DEPLOY_BUILD" == 'true' ]]; then
-            ./fhevm-cli deploy --build --coprocessors 2 --coprocessor-threshold 2
-          else
-            ./fhevm-cli deploy --coprocessors 2 --coprocessor-threshold 2
+          set -euo pipefail
+          args_file="$RUNNER_TEMP/fhevm-up-args"
+          defaults=$(./fhevm-cli compat-defaults)
+          modern_relayer_version=$(echo "$defaults" | jq -r .externalDefaults.RELAYER_VERSION)
+          modern_relayer_migrate_version=$(echo "$defaults" | jq -r .externalDefaults.RELAYER_MIGRATE_VERSION)
+          workspace_all=false
+          printf '%s\n%s\n%s\n%s\n%s\n%s\n' --target latest-release --coprocessors 2 --threshold 2 > "$args_file"
+          case "$IMAGE_MODE" in
+            registry)
+              if [ -n "$WORKSPACE_OVERRIDE" ]; then
+                echo "::error::override requires image_mode=workspace"
+                exit 1
+              fi
+              ;;
+            workspace)
+              if [ -z "$WORKSPACE_OVERRIDE" ]; then
+                echo "::error::image_mode=workspace requires override"
+                exit 1
+              fi
+              IFS=',' read -r -a overrides <<< "$WORKSPACE_OVERRIDE"
+              for raw in "${overrides[@]}"; do
+                override="${raw#"${raw%%[![:space:]]*}"}"
+                override="${override%"${override##*[![:space:]]}"}"
+                [ -n "$override" ] || continue
+                if [ "$override" = "all" ]; then
+                  workspace_all=true
+                fi
+                printf '%s\n%s\n' --override "$override" >> "$args_file"
+              done
+              if ! grep -qx -- --override "$args_file"; then
+                echo "::error::override did not contain any usable values"
+                exit 1
+              fi
+              ;;
+            *)
+              echo "::error::image_mode must be registry or workspace"
+              exit 1
+              ;;
+          esac
+          if { [ "$workspace_all" = true ] || [ "${STACK_ERA:-}" = "modern" ]; } && [ -z "${RELAYER_VERSION:-}" ]; then
+            echo "RELAYER_VERSION=$modern_relayer_version" >> "$GITHUB_ENV"
           fi
+          if { [ "$workspace_all" = true ] || [ "${STACK_ERA:-}" = "modern" ]; } && [ -z "${RELAYER_MIGRATE_VERSION:-}" ]; then
+            echo "RELAYER_MIGRATE_VERSION=$modern_relayer_migrate_version" >> "$GITHUB_ENV"
+          fi
+          if [ "${RELAYER_VERSION:-}" = "$modern_relayer_version" ] && [ -z "${RELAYER_MIGRATE_VERSION:-}" ]; then
+            echo "RELAYER_MIGRATE_VERSION=$modern_relayer_migrate_version" >> "$GITHUB_ENV"
+          fi
+          echo "UP_ARGS_FILE=$args_file" >> "$GITHUB_ENV"
+
+      - name: Dry run
+        working-directory: test-suite/fhevm
+        run: |
+          mapfile -t up_args < "$UP_ARGS_FILE"
+          ./fhevm-cli up "${up_args[@]}" --dry-run
+
+      - name: Boot fhevm Stack
+        working-directory: test-suite/fhevm
+        run: |
+          mapfile -t up_args < "$UP_ARGS_FILE"
+          ./fhevm-cli up "${up_args[@]}"
 
       # E2E tests on pausing the Host contracts
       - name: Pause Host Contracts
@@ -242,41 +314,36 @@ jobs:
 
       - name: Show logs on test failure
         working-directory: test-suite/fhevm
-        if: always()
+        if: failure()
         run: |
-          echo "::group::Relayer Logs"
-          ./fhevm-cli logs fhevm-relayer
-          echo "::endgroup::"
-          echo "::group::SNS Worker Logs"
-          ./fhevm-cli logs coprocessor-sns-worker | grep -v "Selected 0 rows to process"
-          echo "::endgroup::"
-          echo "::group::Transaction Sender Logs (filtered)"
-          ./fhevm-cli logs coprocessor-transaction-sender | grep -v "Selected 0 rows to process"
-          echo "::endgroup::"
-          echo "::group::Host Listener"
-          ./fhevm-cli logs coprocessor-host-listener
-          echo "::endgroup::"
-          echo "::group::Gateway Listener"
-          ./fhevm-cli logs coprocessor-gw-listener
-          echo "::endgroup::"
-          echo "::group::ZKProof Worker"
-          ./fhevm-cli logs coprocessor-zkproof-worker
-          echo "::endgroup::"
-          echo "::group::TFHE Worker"
-          ./fhevm-cli logs coprocessor-tfhe-worker
-          echo "::endgroup::"
-          echo "::group::Coprocessor 2 - SNS Worker"
-          ./fhevm-cli logs coprocessor-2-sns-worker 2>/dev/null | grep -v "Selected 0 rows to process" || true
-          echo "::endgroup::"
-          echo "::group::Coprocessor 2 - Transaction Sender (filtered)"
-          ./fhevm-cli logs coprocessor-2-transaction-sender 2>/dev/null | grep -v "Selected 0 rows to process" || true
-          echo "::endgroup::"
-          echo "::group::Coprocessor 2 - TFHE Worker"
-          ./fhevm-cli logs coprocessor-2-tfhe-worker 2>/dev/null || true
-          echo "::endgroup::"
+          snapshot_logs() {
+            local group="$1"
+            local container="$2"
+            local filter="${3:-}"
+            echo "::group::${group}"
+            if [ -n "$filter" ]; then
+              docker logs --tail 200 "${container}" 2>&1 | grep -v "$filter" || true
+            else
+              docker logs --tail 200 "${container}" 2>&1 || true
+            fi
+            echo "::endgroup::"
+          }
+          snapshot_logs "Relayer Logs" fhevm-relayer
+          snapshot_logs "SNS Worker Logs" coprocessor-sns-worker "Selected 0 rows to process"
+          snapshot_logs "Transaction Sender Logs (filtered)" coprocessor-transaction-sender "Selected 0 rows to process"
+          snapshot_logs "Host Listener" coprocessor-host-listener
+          snapshot_logs "Gateway Listener" coprocessor-gw-listener
+          snapshot_logs "ZKProof Worker" coprocessor-zkproof-worker
+          snapshot_logs "TFHE Worker" coprocessor-tfhe-worker
+          snapshot_logs "Coprocessor 2 - Host Listener" coprocessor1-host-listener
+          snapshot_logs "Coprocessor 2 - Gateway Listener" coprocessor1-gw-listener
+          snapshot_logs "Coprocessor 2 - SNS Worker" coprocessor1-sns-worker "Selected 0 rows to process"
+          snapshot_logs "Coprocessor 2 - Transaction Sender (filtered)" coprocessor1-transaction-sender "Selected 0 rows to process"
+          snapshot_logs "Coprocessor 2 - ZKProof Worker" coprocessor1-zkproof-worker
+          snapshot_logs "Coprocessor 2 - TFHE Worker" coprocessor1-tfhe-worker
 
       - name: Cleanup
         working-directory: test-suite/fhevm
         if: always()
         run: |
-          ./fhevm-cli clean
+          ./fhevm-cli clean --images