test(host-listener): make slow-lane local validation deterministic

Author: Eikix

coprocessor/fhevm-engine/host-listener/README.md

@@ -58,7 +58,7 @@ Recommended (auditable) entrypoint:
 
 ```bash
 cd test-suite/fhevm
-./scripts/validate-slow-lane.sh --cap 1 --load-test operators
+./scripts/validate-slow-lane.sh --cap 2
 ```
 
 What this enforces:
@@ -76,7 +76,10 @@ Useful modes:
 ./scripts/validate-slow-lane.sh --integration-only
 
 # local stack only (assumes stack already up)
-./scripts/validate-slow-lane.sh --stack-only --cap 1 --load-test operators
+./scripts/validate-slow-lane.sh --stack-only --cap 2
+
+# optional fallback to fhevm-cli test flows
+./scripts/validate-slow-lane.sh --stack-only --cap 1 --load-grep "" --load-test operators
 ```
 
 1. Start local stack
@@ -140,19 +143,19 @@ Important:
 
 - Keep `-p fhevm` when restarting manually. Without it, containers can start on a different compose project/network and fail DNS (`host-node`, `db`).
 
-4. Generate dependent load
+4. Generate deterministic dependent load
+
+The validator defaults to a dedicated e2e case (`Slow lane deterministic contention`)
+that emits one heavy dependent chain and one light dependent chain.
 
-Use a real host-contract flow so host-listener ingests real TFHE logs:
+Manual equivalent:
 
 ```bash
-cd test-suite/fhevm
-./fhevm-cli test erc20
+docker exec fhevm-test-suite-e2e-debug \
+  ./run-tests.sh -n staging -g "Slow lane deterministic contention"
 ```
 
-If this does not trigger slow-lane in local runs, use forcing mode for validation:
-
-- set `--dependent-ops-max-per-chain=1` temporarily
-- rerun `./fhevm-cli test erc20`
+Recommended cap for this scenario: `2` (heavy chain > cap, light chain <= cap).
 
 5. Validate acceptance criteria in DB
 

test-suite/e2e/contracts/operations/SlowLaneContention.sol

@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: BSD-3-Clause-Clear
+
+pragma solidity ^0.8.24;
+
+import "@fhevm/solidity/lib/FHE.sol";
+import {E2ECoprocessorConfig} from "../E2ECoprocessorConfigLocal.sol";
+
+contract SlowLaneContention is E2ECoprocessorConfig {
+    euint64 public lastResult;
+
+    function runAddChain(externalEuint64 seed, bytes calldata inputProof, uint8 steps) external {
+        euint64 value = FHE.fromExternal(seed, inputProof);
+
+        for (uint8 i = 0; i < steps; i++) {
+            value = FHE.add(value, value);
+            FHE.allowThis(value);
+        }
+
+        lastResult = value;
+        FHE.allowThis(lastResult);
+        FHE.allow(lastResult, msg.sender);
+        FHE.makePubliclyDecryptable(lastResult);
+    }
+}

test-suite/e2e/test/slowlane/slowLaneContention.ts

@@ -0,0 +1,45 @@
+import { expect } from 'chai';
+import { ethers } from 'hardhat';
+
+import { createInstance } from '../instance';
+import { getSigner } from '../signers';
+
+describe('Slow lane deterministic contention', function () {
+  beforeEach(async function () {
+    this.signer = await getSigner(119);
+
+    const contractFactory = await ethers.getContractFactory('SlowLaneContention');
+    const contract = await contractFactory.connect(this.signer).deploy();
+    await contract.waitForDeployment();
+
+    this.contractAddress = await contract.getAddress();
+    this.contract = contract;
+    this.instance = await createInstance();
+  });
+
+  it('creates one heavy chain and one light chain', async function () {
+    const heavyInput = this.instance.createEncryptedInput(this.contractAddress, this.signer.address);
+    heavyInput.add64(1);
+    const heavyEncryptedInput = await heavyInput.encrypt();
+
+    const heavyTx = await this.contract.runAddChain(
+      heavyEncryptedInput.handles[0],
+      heavyEncryptedInput.inputProof,
+      8,
+    );
+    const heavyReceipt = await heavyTx.wait();
+    expect(heavyReceipt?.status).to.eq(1);
+
+    const lightInput = this.instance.createEncryptedInput(this.contractAddress, this.signer.address);
+    lightInput.add64(1);
+    const lightEncryptedInput = await lightInput.encrypt();
+
+    const lightTx = await this.contract.runAddChain(
+      lightEncryptedInput.handles[0],
+      lightEncryptedInput.inputProof,
+      1,
+    );
+    const lightReceipt = await lightTx.wait();
+    expect(lightReceipt?.status).to.eq(1);
+  });
+});

test-suite/fhevm/scripts/validate-slow-lane.sh

@@ -14,6 +14,7 @@ RUN_INTEGRATION=true
 RUN_STACK=true
 CAP=1
 LOAD_TEST="operators"
+LOAD_GREP="Slow lane deterministic contention"
 BOOTSTRAP_TIMEOUT_SECONDS=180
 
 RED='\033[0;31m'
@@ -33,7 +34,8 @@ Options:
   --integration-only       Run only deterministic Rust integration assertions.
   --stack-only             Run only local-stack multi-listener SQL assertions.
   --cap N                  Slow-lane cap for stack scenario (default: 1).
-  --load-test NAME         fhevm-cli test to generate load (default: operators).
+  --load-grep PATTERN      run-tests grep for deterministic load (default: "Slow lane deterministic contention").
+  --load-test NAME         fhevm-cli test fallback when --load-grep is empty (default: operators).
   --bootstrap-timeout SEC  Timeout for key-bootstrap gate (default: 180).
   -h, --help               Show this help.
 EOF
@@ -128,9 +130,11 @@ wait_for_bootstrap() {
     local has_activate_key
     local has_fetched_keyset
     local has_key_material
+    local sns_waiting_for_keys
 
     has_activate_key="$(docker logs --since=20m coprocessor-gw-listener 2>&1 | rg -c "ActivateKey event successful" || true)"
     has_fetched_keyset="$(docker logs --since=20m coprocessor-sns-worker 2>&1 | rg -c "Fetched keyset" || true)"
+    sns_waiting_for_keys="$(docker logs --since=2m coprocessor-sns-worker 2>&1 | rg -c "No keys available, retrying" || true)"
     has_key_material="$(db_query "
       SELECT COALESCE(bool_and(key_bytes > 0), false)
       FROM (
@@ -146,6 +150,10 @@ wait_for_bootstrap() {
       log "Bootstrap gate passed (ActivateKey + keyset + non-empty sns_pk)"
       return 0
     fi
+    if [[ "${has_key_material}" == "t" && "${sns_waiting_for_keys}" == "0" ]]; then
+      log "Bootstrap gate passed (non-empty sns_pk and sns-worker not waiting for keys)"
+      return 0
+    fi
     sleep 3
   done
 
@@ -155,11 +163,28 @@ wait_for_bootstrap() {
   local retry_deadline=$((SECONDS + BOOTSTRAP_TIMEOUT_SECONDS))
   while (( SECONDS < retry_deadline )); do
     local has_fetched_keyset
+    local has_key_material
+    local sns_waiting_for_keys
     has_fetched_keyset="$(docker logs --since=20m coprocessor-sns-worker 2>&1 | rg -c "Fetched keyset" || true)"
+    sns_waiting_for_keys="$(docker logs --since=2m coprocessor-sns-worker 2>&1 | rg -c "No keys available, retrying" || true)"
+    has_key_material="$(db_query "
+      SELECT COALESCE(bool_and(key_bytes > 0), false)
+      FROM (
+        SELECT COALESCE(SUM(octet_length(lo.data)), 0) AS key_bytes
+        FROM tenants t
+        LEFT JOIN pg_largeobject lo
+          ON lo.loid = t.sns_pk
+        GROUP BY t.tenant_id
+      ) s;
+    ")"
     if [[ "${has_fetched_keyset}" -gt 0 ]]; then
       log "Bootstrap recovered after gw-listener restart"
       return 0
     fi
+    if [[ "${has_key_material}" == "t" && "${sns_waiting_for_keys}" == "0" ]]; then
+      log "Bootstrap recovered from persisted key material state"
+      return 0
+    fi
     sleep 3
   done
 
@@ -215,7 +240,11 @@ run_integration_assertions() {
 }
 
 run_stack_assertions() {
-  log "Running local-stack assertions (cap=${CAP}, load=${LOAD_TEST})"
+  if [[ -n "${LOAD_GREP}" ]]; then
+    log "Running local-stack assertions (cap=${CAP}, grep=${LOAD_GREP})"
+  else
+    log "Running local-stack assertions (cap=${CAP}, load=${LOAD_TEST})"
+  fi
   wait_for_bootstrap
 
   local before_block_height
@@ -224,10 +253,14 @@ run_stack_assertions() {
 
   apply_listener_cap_override "${CAP}"
 
-  (
-    cd "${FHEVM_DIR}"
-    ./fhevm-cli test "${LOAD_TEST}"
-  )
+  if [[ -n "${LOAD_GREP}" ]]; then
+    docker exec fhevm-test-suite-e2e-debug ./run-tests.sh -n staging -g "${LOAD_GREP}"
+  else
+    (
+      cd "${FHEVM_DIR}"
+      ./fhevm-cli test "${LOAD_TEST}"
+    )
+  fi
 
   local counts
   counts="$(db_query "
@@ -287,6 +320,11 @@ while (( "$#" )); do
       LOAD_TEST="$2"
       shift 2
       ;;
+    --load-grep)
+      [[ $# -ge 2 ]] || die "--load-grep requires a value"
+      LOAD_GREP="$2"
+      shift 2
+      ;;
     --bootstrap-timeout)
       [[ $# -ge 2 ]] || die "--bootstrap-timeout requires a value"
       BOOTSTRAP_TIMEOUT_SECONDS="$2"