test-suite-e2e-tests.yml

name: test-suite-e2e-tests

on:
  workflow_dispatch:
    inputs: &workflow_inputs
      image_mode:
        description: "Use registry images or build workspace overrides on the runner"
        default: "registry"
        type: string
      override:
        description: "Comma-separated override values for workspace mode (for example: all, coprocessor, or coprocessor:host-listener)"
        default: ""
        type: string
      stack-era:
        description: "Stack era hint for workflow_call consumers (leave empty for manual dispatch)"
        default: ""
        type: string
      coprocessor-db-migration-version:
        description: "Coprocessor DB Migration Image Version"
        default: ""
        type: string
      coprocessor-host-listener-version:
        description: "Coprocessor Host Listener Image Version"
        default: ""
        type: string
      coprocessor-gw-listener-version:
        description: "Coprocessor Gateway Listener Image Version"
        default: ""
        type: string
      coprocessor-tx-sender-version:
        description: "Coprocessor Transaction Sender Image Version"
        default: ""
        type: string
      coprocessor-tfhe-worker-version:
        description: "Coprocessor TFHE Worker Image Version"
        default: ""
        type: string
      coprocessor-sns-worker-version:
        description: "Coprocessor SNS Worker Image Version"
        default: ""
        type: string
      coprocessor-zkproof-worker-version:
        description: "Coprocessor ZKProof Worker Image Version"
        default: ""
        type: string
      gateway-version:
        description: "Gateway version"
        default: ""
        type: string
      host-version:
        description: "Host version"
        default: ""
        type: string
      connector-db-migration-version:
        description: "KMS Connector DB Migration Image Version"
        default: ""
        type: string
      connector-gw-listener-version:
        description: "KMS Connector Gateway Listener Image Version"
        default: ""
        type: string
      connector-kms-worker-version:
        description: "KMS Connector KMS Worker Image Version"
        default: ""
        type: string
      connector-tx-sender-version:
        description: "KMS Connector Transaction Sender Image Version"
        default: ""
        type: string
      test-suite-version:
        description: "Test suite version"
        default: ""
        type: string
      relayer-version:
        description: "Relayer version"
        default: ""
        type: string
      relayer-migrate-version:
        description: "Relayer migrate version"
        default: ""
        type: string
      kms-core-version:
        description: "KMS Core version"
        default: ""
        type: string
  workflow_call:
    secrets:
      GHCR_READ_TOKEN:
        required: true
      CGR_USERNAME:
        required: true
      CGR_PASSWORD:
        required: true
    inputs: *workflow_inputs

permissions: {}

# Allow to run multiple instances of the same workflow in parallel when triggered manually
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name == 'workflow_dispatch' && github.run_id || 'auto' }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}

jobs:
  fhevm-e2e-test:
    permissions:
      contents: 'read' # Required to checkout repository code
      id-token: 'write' # Required for OIDC authentication
      packages: 'read' # Required to read GitHub packages/container registry
    env:
      GH_TOKEN: ${{ secrets.GHCR_READ_TOKEN || github.token }}
      IMAGE_MODE: ${{ inputs.image_mode || 'registry' }}
      WORKSPACE_OVERRIDE: ${{ inputs.override || '' }}
      COPROCESSOR_DB_MIGRATION_VERSION: ${{ inputs.coprocessor-db-migration-version }}
      COPROCESSOR_HOST_LISTENER_VERSION: ${{ inputs.coprocessor-host-listener-version }}
      COPROCESSOR_GW_LISTENER_VERSION: ${{ inputs.coprocessor-gw-listener-version }}
      COPROCESSOR_TX_SENDER_VERSION: ${{ inputs.coprocessor-tx-sender-version }}
      COPROCESSOR_TFHE_WORKER_VERSION: ${{ inputs.coprocessor-tfhe-worker-version }}
      COPROCESSOR_SNS_WORKER_VERSION: ${{ inputs.coprocessor-sns-worker-version }}
      COPROCESSOR_ZKPROOF_WORKER_VERSION: ${{ inputs.coprocessor-zkproof-worker-version }}
      GATEWAY_VERSION: ${{ inputs.gateway-version }}
      HOST_VERSION: ${{ inputs.host-version }}
      CONNECTOR_DB_MIGRATION_VERSION: ${{ inputs.connector-db-migration-version }}
      CONNECTOR_GW_LISTENER_VERSION: ${{ inputs.connector-gw-listener-version }}
      CONNECTOR_KMS_WORKER_VERSION: ${{ inputs.connector-kms-worker-version }}
      CONNECTOR_TX_SENDER_VERSION: ${{ inputs.connector-tx-sender-version }}
      TEST_SUITE_VERSION: ${{ inputs.test-suite-version }}
      RELAYER_VERSION: ${{ inputs.relayer-version }}
      RELAYER_MIGRATE_VERSION: ${{ inputs.relayer-migrate-version }}
      CORE_VERSION: ${{ inputs.kms-core-version }}
      STACK_ERA: ${{ inputs.stack-era }}
    runs-on: large_ubuntu_32
    steps:
      - name: Checkout code
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          persist-credentials: 'false'
          fetch-depth: 0

      - name: Setup Docker
        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0

      - name: Setup Bun
        uses: oven-sh/setup-bun@ecf28ddc73e819eb6fa29df6b34ef8921c743461 # v2.1.3

      - name: Install foundry
        uses: foundry-rs/foundry-toolchain@82dee4ba654bd2146511f85f0d013af94670c4de # v1.4.0

      - name: Login to GitHub Container Registry
        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GHCR_READ_TOKEN }}

      - name: Login to Chainguard Registry
        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
        with:
          registry: cgr.dev
          username: ${{ secrets.CGR_USERNAME }}
          password: ${{ secrets.CGR_PASSWORD }}

      - name: Install CLI deps
        working-directory: test-suite/fhevm
        run: bun install --frozen-lockfile

      - name: Resolve CLI up args
        working-directory: test-suite/fhevm
        run: |
          set -euo pipefail
          args_file="$RUNNER_TEMP/fhevm-up-args"
          scenario_file="$RUNNER_TEMP/fhevm-scenario.yaml"
          workspace_all=false
          coprocessor_local=false
          coprocessor_local_services=()
          scenario_path="./scenarios/two-of-two.yaml"
          case "$IMAGE_MODE" in
            registry)
              if [ -n "$WORKSPACE_OVERRIDE" ]; then
                echo "::error::override requires image_mode=workspace"
                exit 1
              fi
              ;;
            workspace)
              if [ -z "$WORKSPACE_OVERRIDE" ]; then
                echo "::error::image_mode=workspace requires override"
                exit 1
              fi
              IFS=',' read -r -a overrides <<< "$WORKSPACE_OVERRIDE"
              for raw in "${overrides[@]}"; do
                override="${raw#"${raw%%[![:space:]]*}"}"
                override="${override%"${override##*[![:space:]]}"}"
                [ -n "$override" ] || continue
                if [ "$override" = "all" ]; then
                  workspace_all=true
                  coprocessor_local=true
                  continue
                fi
                if [ "$override" = "coprocessor" ]; then
                  coprocessor_local=true
                  continue
                fi
                if [[ "$override" == coprocessor:* ]]; then
                  coprocessor_local=true
                  suffixes="${override#coprocessor:}"
                  IFS=',' read -r -a services <<< "$suffixes"
                  for service in "${services[@]}"; do
                    service="${service#"${service%%[![:space:]]*}"}"
                    service="${service%"${service##*[![:space:]]}"}"
                    [ -n "$service" ] || continue
                    coprocessor_local_services+=("$service")
                  done
                  continue
                fi
              done
              if [ "$coprocessor_local" = false ] && [ "${#overrides[@]}" -eq 0 ]; then
                echo "::error::override did not contain any usable values"
                exit 1
              fi
              if [ "$coprocessor_local" = true ]; then
                {
                  echo "version: 1"
                  echo "kind: coprocessor-consensus"
                  echo "topology:"
                  echo "  count: 2"
                  echo "  threshold: 2"
                  echo "instances:"
                  echo "  - index: 1"
                  echo "    source:"
                  echo "      mode: local"
                  if [ "${#coprocessor_local_services[@]}" -gt 0 ]; then
                    echo "    localServices:"
                    for service in "${coprocessor_local_services[@]}"; do
                      echo "      - ${service}"
                    done
                  fi
                } > "$scenario_file"
                scenario_path="$scenario_file"
              fi
              if [ -z "$WORKSPACE_OVERRIDE" ]; then
                echo "::error::override did not contain any usable values"
                exit 1
              fi
              ;;
            *)
              echo "::error::image_mode must be registry or workspace"
              exit 1
              ;;
          esac
          printf '%s\n%s\n%s\n%s\n' --target latest-release --scenario "$scenario_path" > "$args_file"
          if [ "$IMAGE_MODE" = "workspace" ]; then
            IFS=',' read -r -a overrides <<< "$WORKSPACE_OVERRIDE"
            for raw in "${overrides[@]}"; do
              override="${raw#"${raw%%[![:space:]]*}"}"
              override="${override%"${override##*[![:space:]]}"}"
              [ -n "$override" ] || continue
              case "$override" in
                all)
                  printf '%s\n%s\n%s\n%s\n%s\n%s\n%s\n%s\n' \
                    --override gateway-contracts \
                    --override host-contracts \
                    --override kms-connector \
                    --override test-suite >> "$args_file"
                  ;;
                coprocessor|coprocessor:*)
                  ;;
                *)
                  printf '%s\n%s\n' --override "$override" >> "$args_file"
                  ;;
              esac
            done
          fi
          FORCE_MODERN_RELAYER=$workspace_all ./scripts/resolve-workflow-compat-env.sh \
            "${COPROCESSOR_DB_MIGRATION_VERSION:-}" \
            "${COPROCESSOR_HOST_LISTENER_VERSION:-}" \
            "${COPROCESSOR_GW_LISTENER_VERSION:-}" \
            "${COPROCESSOR_TX_SENDER_VERSION:-}" \
            "${COPROCESSOR_TFHE_WORKER_VERSION:-}" \
            "${COPROCESSOR_SNS_WORKER_VERSION:-}" \
            "${COPROCESSOR_ZKPROOF_WORKER_VERSION:-}" \
            "${CONNECTOR_DB_MIGRATION_VERSION:-}" \
            "${CONNECTOR_GW_LISTENER_VERSION:-}" \
            "${CONNECTOR_KMS_WORKER_VERSION:-}" \
            "${CONNECTOR_TX_SENDER_VERSION:-}" \
            "${GATEWAY_VERSION:-}" \
            "${HOST_VERSION:-}" \
            "${TEST_SUITE_VERSION:-}"
          echo "UP_ARGS_FILE=$args_file" >> "$GITHUB_ENV"

      - name: Dry run
        working-directory: test-suite/fhevm
        run: |
          mapfile -t up_args < "$UP_ARGS_FILE"
          ./fhevm-cli up "${up_args[@]}" --dry-run

      - name: Boot fhevm Stack
        working-directory: test-suite/fhevm
        run: |
          mapfile -t up_args < "$UP_ARGS_FILE"
          ./fhevm-cli up "${up_args[@]}"

      # E2E tests on pausing the Host contracts
      - name: Pause Host Contracts
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli pause host

      - name: Paused Host contracts test
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test paused-host-contracts

      - name: Unpause Host Contracts
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli unpause host

      # E2E tests on pausing the Gateway contracts
      - name: Pause Gateway Contracts
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli pause gateway

      - name: Paused Gateway contracts test
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test paused-gateway-contracts

      - name: Unpause Gateway Contracts
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli unpause gateway

      # E2E tests after unpausing the Host and Gateway contracts
      - name: Input proof test (uint64)
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test input-proof

      - name: Input proof test with compute and decrypt (uint64)
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test input-proof-compute-decrypt

      - name: User Decryption test
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test user-decryption

      - name: Delegated User Decryption test
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test delegated-user-decryption

      - name: ERC20 test
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test erc20

      - name: Public Decryption HTTP endpoint test (ebool)
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test public-decrypt-http-ebool

      - name: Public Decryption HTTP endpoint test (mixed)
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test public-decrypt-http-mixed

      - name: Negative ACL tests
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test negative-acl

      - name: Random operators test (subset)
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test random-subset

      - name: HCU block cap test
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test hcu-block-cap

      - name: Ciphertext drift test
        working-directory: test-suite/fhevm
        run: |
          ./fhevm-cli test ciphertext-drift

      - name: Host listener poller test
        working-directory: test-suite/fhevm
        run: |
          docker stop coprocessor-host-listener
          ./fhevm-cli test erc20
          docker start coprocessor-host-listener

      - name: Show logs on test failure
        working-directory: test-suite/fhevm
        if: failure()
        run: |
          snapshot_logs() {
            local group="$1"
            local container="$2"
            local filter="${3:-}"
            echo "::group::${group}"
            if [ -n "$filter" ]; then
              docker logs --tail 200 "${container}" 2>&1 | grep -v "$filter" || true
            else
              docker logs --tail 200 "${container}" 2>&1 || true
            fi
            echo "::endgroup::"
          }
          snapshot_logs "Relayer Logs" fhevm-relayer
          snapshot_logs "SNS Worker Logs" coprocessor-sns-worker "Selected 0 rows to process"
          snapshot_logs "Transaction Sender Logs (filtered)" coprocessor-transaction-sender "Selected 0 rows to process"
          snapshot_logs "Host Listener" coprocessor-host-listener
          snapshot_logs "Gateway Listener" coprocessor-gw-listener
          snapshot_logs "ZKProof Worker" coprocessor-zkproof-worker
          snapshot_logs "TFHE Worker" coprocessor-tfhe-worker
          snapshot_logs "Coprocessor 2 - Host Listener" coprocessor1-host-listener
          snapshot_logs "Coprocessor 2 - Gateway Listener" coprocessor1-gw-listener
          snapshot_logs "Coprocessor 2 - SNS Worker" coprocessor1-sns-worker "Selected 0 rows to process"
          snapshot_logs "Coprocessor 2 - Transaction Sender (filtered)" coprocessor1-transaction-sender "Selected 0 rows to process"
          snapshot_logs "Coprocessor 2 - ZKProof Worker" coprocessor1-zkproof-worker
          snapshot_logs "Coprocessor 2 - TFHE Worker" coprocessor1-tfhe-worker

      - name: Cleanup
        working-directory: test-suite/fhevm
        if: always()
        run: |
          ./fhevm-cli clean --images