fix(test-suite): gateway keygen

Author: rudy-6-4

.github/workflows/test-suite-e2e-tests.yml

@@ -197,6 +197,9 @@ jobs:
           echo "::group::Host Listener"
           ./fhevm-cli logs host-listener
           echo "::endgroup::"
+          echo "::group::Gateway Listener"
+          ./fhevm-cli logs gw-listener
+          echo "::endgroup::"
 
       - name: Cleanup
         working-directory: test-suite/fhevm

coprocessor/fhevm-engine/db-migration/Dockerfile

@@ -20,18 +20,18 @@ WORKDIR /app/coprocessor/fhevm-engine
 # Build utils binary
 # Currently, this utils is used only for keys converting.
 # Later on, it will be extended to replace initialize_db.sh completely
-RUN mkdir /fhevm-keys && \
-    cargo fetch && \
-    SQLX_OFFLINE=true cargo build --release -p tfhe-worker --bin utils
+# RUN mkdir /fhevm-keys && \
+#     cargo fetch && \
+#     SQLX_OFFLINE=true cargo build --release -p tfhe-worker --bin utils
 
 # Stage 2: Runtime image
 FROM cgr.dev/zama.ai/postgres:17 AS prod
 
-COPY  --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/utils /usr/local/bin/utils
+# COPY  --from=builder --chown=fhevm:fhevm /app/coprocessor/fhevm-engine/target/release/utils /usr/local/bin/utils
 COPY  --from=builder --chown=fhevm:fhevm /usr/local/cargo/bin/sqlx /usr/local/bin/sqlx
 COPY  --from=builder --chown=fhevm:fhevm /app/initialize_db.sh /initialize_db.sh
 COPY  --from=builder --chown=fhevm:fhevm /app/migrations /migrations
-COPY  --from=builder --chown=fhevm:fhevm /fhevm-keys /fhevm-keys
+# COPY  --from=builder --chown=fhevm:fhevm /fhevm-keys /fhevm-keys
 COPY  --from=builder /etc/group /etc/group
 COPY  --from=builder /etc/passwd /etc/passwd
 

coprocessor/fhevm-engine/db-migration/initialize_db.sh

@@ -31,21 +31,22 @@ echo "-------------- Start inserting keys for tenant: $TENANT_API_KEY ----------
 
 
 CHAIN_ID=${CHAIN_ID:-"12345"}
-PKS_FILE=${PKS_FILE:-"$KEY_DIR/pks"}
-PUBLIC_PARAMS_FILE=${PUBLIC_PARAMS_FILE:-"$KEY_DIR/pp"}
-SNS_PK_FILE=${SNS_PK_FILE:-"$KEY_DIR/sns_pk"}
-KEY_ID=${KEY_ID:-"10f49fdf75a123370ce2e2b1c5cc0615fb6e78dd829d0d850470cdbc84f15c11"}
-KEY_ID_HEX="\\x${KEY_ID}"
-
-# Extract small ServerKey from ServerKey with noise squashing keys
-SKS_FILE="/tmp/sks"
-/usr/local/bin/utils extract-sks-without-noise --src-path $SNS_PK_FILE --dst-path $SKS_FILE
-
-for file in "$PKS_FILE" "$SKS_FILE" "$PUBLIC_PARAMS_FILE" "$SNS_PK_FILE"; do
-    if [[ ! -f $file ]]; then
-        echo "Error: Key file $file not found."; exit 1;
-    fi
-done
+# PKS_FILE=${PKS_FILE:-"$KEY_DIR/pks"}
+# PUBLIC_PARAMS_FILE=${PUBLIC_PARAMS_FILE:-"$KEY_DIR/pp"}
+# SNS_PK_FILE=${SNS_PK_FILE:-"$KEY_DIR/sns_pk"}
+# KEY_ID=${KEY_ID:-"10f49fdf75a123370ce2e2b1c5cc0615fb6e78dd829d0d850470cdbc84f15c11"}
+# KEY_ID_HEX="\\x${KEY_ID}"
+
+echo "Skip extract-sks-without-noise"
+# # Extract small ServerKey from ServerKey with noise squashing keys
+# SKS_FILE="/tmp/sks"
+# /usr/local/bin/utils extract-sks-without-noise --src-path $SNS_PK_FILE --dst-path $SKS_FILE
+
+# for file in "$PKS_FILE" "$SKS_FILE" "$PUBLIC_PARAMS_FILE" "$SNS_PK_FILE"; do
+#     if [[ ! -f $file ]]; then
+#         echo "Error: Key file $file not found."; exit 1;
+#     fi
+# done
 
 if [[ -z "$DATABASE_URL" || -z "$TENANT_API_KEY" || -z "$ACL_CONTRACT_ADDRESS" || -z "$INPUT_VERIFIER_ADDRESS" ]]; then
     echo "Error: One or more required environment variables are missing."; exit 1;
@@ -58,10 +59,6 @@ if [ "$TENANT_EXISTS" = "1" ]; then
     exit 0
 fi
 
-TMP_CSV="/tmp/tenant_data.csv"
-echo "tenant_api_key,chain_id,acl_contract_address,verifying_contract_address,pks_key,sks_key,public_params,sns_pk,key_id" > $TMP_CSV
-
-
 import_large_file() {
   local file="$1"
   local db_url="$2"
@@ -121,40 +118,18 @@ EOF
   echo "$oid"
 }
 
-echo "Importing large object from $SNS_PK_FILE ($(du -h "$SNS_PK_FILE" | cut -f1))..."
-SNS_PK_OID=$(import_large_file "$SNS_PK_FILE" "$DATABASE_URL")
- 
-
-echo "$TENANT_API_KEY,$CHAIN_ID,$ACL_CONTRACT_ADDRESS,$INPUT_VERIFIER_ADDRESS,\
-\"\\x$(< "$PKS_FILE" xxd -p | tr -d '\n')\",\
-\"\\x$(< "$SKS_FILE" xxd -p | tr -d '\n')\",\
-\"\\x$(< "$PUBLIC_PARAMS_FILE" xxd -p | tr -d '\n')\",\
-$SNS_PK_OID,\"$KEY_ID_HEX\"" >> $TMP_CSV
+echo "Fake OID"
+FAKE_KEY_FILE=$(mktemp)
+touch "$FAKE_KEY_FILE"
+SNS_PK_OID=$(import_large_file "$FAKE_KEY_FILE" "$DATABASE_URL")
 
 echo "----------- Tenant data prepared for insertion: $TMP_CSV -----------"
- 
-
 
 echo "Inserting tenant data from CSV using \COPY..."
-psql "$DATABASE_URL" -c "\COPY tenants (tenant_api_key, chain_id, acl_contract_address, verifying_contract_address, pks_key, sks_key, public_params, sns_pk, key_id) FROM '$TMP_CSV' CSV HEADER;" || {
+psql "$DATABASE_URL" -c \
+  "INSERT INTO tenants (tenant_api_key, chain_id, acl_contract_address, verifying_contract_address,pks_key,sks_key,public_params,sns_pk,key_id) \
+   VALUES ('$TENANT_API_KEY',$CHAIN_ID,'$ACL_CONTRACT_ADDRESS','$INPUT_VERIFIER_ADDRESS','','','',$SNS_PK_OID,'');" || {
     echo "Error: Failed to insert tenant data."; exit 1;
 }
 
-echo "Checking large object creation..."
-psql "$DATABASE_URL" -c "SELECT loid as oid,
-                        pg_size_pretty(SUM(octet_length(data))) as size
-                     FROM pg_largeobject
-                     GROUP BY loid;"
-
-echo "Checking tenant entry references correct OID..."
-psql "$DATABASE_URL" -c "SELECT t.tenant_id,
-                        t.tenant_api_key,
-                        t.sns_pk,
-                        pg_size_pretty((SELECT SUM(octet_length(lo.data))
-                                      FROM pg_largeobject lo
-                                      WHERE lo.loid = t.sns_pk)) as sns_pk_size
-                      FROM tenants t
-                      WHERE t.tenant_api_key = '$TENANT_API_KEY';"
-
-rm -f $TMP_CSV
 echo "Database initialization keys insertion complete successfully."

coprocessor/fhevm-engine/gw-listener/Dockerfile

@@ -9,7 +9,8 @@ COPY gateway-contracts ./gateway-contracts
 
 # Compiled gateway-contracts for gw-listener
 WORKDIR /app/gateway-contracts
-RUN npm install && DEPLOYER_PRIVATE_KEY="0x7136d8dc72f873124f4eded25f3525a20f6cee4296564c76b44f1d582c57640f" npx hardhat task:deployEmptyUUPSProxies && npx hardhat compile
+RUN npm install && \
+    DOTENV_CONFIG_PATH=.env.example npx hardhat task:deployAllGatewayContracts
 
 # Stage 1: Build GW Listener
 FROM ghcr.io/zama-ai/fhevm/gci/rust-glibc:1.90.0 AS builder

test-suite/fhevm/docker-compose/coprocessor-docker-compose.yml

@@ -1,21 +1,4 @@
 services:
-  ####################### DATABASE SETUP #######################
-  db:
-    container_name: fhevm-coprocessor-db
-    image: postgres:15.7
-    restart: always
-    env_file:
-      - ../env/staging/.env.coprocessor.local
-    ports:
-      - '5432:5432'
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U postgres"]
-      interval: 10s
-      timeout: 5s
-      retries: 3
-    volumes:
-      - db:/var/lib/postgresql/data
-
   db-migration:
     container_name: fhevm-db-migration
     image: ghcr.io/zama-ai/fhevm/coprocessor/db-migration:${DB_MIGRATION_VERSION}
@@ -34,11 +17,6 @@ services:
       - /initialize_db.sh
     volumes:
       - keys-cache:/fhevm-keys
-    depends_on:
-      db:
-        condition: service_healthy
-      key-downloader:
-        condition: service_completed_successfully
 
   ####################### COPROCESSOR SERVICES #######################
   host-listener:
@@ -62,8 +40,6 @@ services:
       - --url=${RPC_URL}
       - --initial-block-time=1
     depends_on:
-      db:
-        condition: service_healthy
       db-migration:
         condition: service_completed_successfully
 
@@ -95,8 +71,6 @@ services:
       - --error-sleep-initial-secs=1
       - --error-sleep-max-secs=10
     depends_on:
-      db:
-        condition: service_healthy
       db-migration:
         condition: service_completed_successfully
 
@@ -123,8 +97,6 @@ services:
       - --coprocessor-fhe-threads=8
       - --tokio-threads=4
     depends_on:
-      db:
-        condition: service_healthy
       db-migration:
         condition: service_completed_successfully
 
@@ -149,8 +121,6 @@ services:
       - --pg-pool-connections=5
       - --worker-thread-count=4
     depends_on:
-      db:
-        condition: service_healthy
       db-migration:
         condition: service_completed_successfully
 
@@ -188,8 +158,6 @@ services:
       - --s3-regular-recheck-duration=120s
       - --enable-compression
     depends_on:
-      db:
-        condition: service_healthy
       db-migration:
         condition: service_completed_successfully
 
@@ -222,11 +190,8 @@ services:
       - --verify-proof-remove-after-max-retries
       - --signer-type=private-key
     depends_on:
-      db:
-        condition: service_healthy
       db-migration:
         condition: service_completed_successfully
 
 volumes:
-  db:
   keys-cache:

test-suite/fhevm/env/staging/.env.relayer

@@ -8,9 +8,9 @@ APP_TRANSACTION__PRIVATE_KEY_GATEWAY=0xcb97ef45d352446a6adf810cf8f63c73ada027160
 
 # Key URL configuration
 APP_KEYURL__FHE_PUBLIC_KEY__DATA_ID="fhe-public-key-data-id"
-APP_KEYURL__FHE_PUBLIC_KEY__URL="http://minio:9000/kms-public/PUB/PublicKey/bdef3ae02c2a49472724c9f0caee201eb8ede7e29b9408f3b819eeb0082a9709"
+APP_KEYURL__FHE_PUBLIC_KEY__URL=http://minio:9000/kms-public/PUB/PublicKey/0400000000000000000000000000000000000000000000000000000000000001
 APP_KEYURL__CRS__DATA_ID="crs-data-id"
-APP_KEYURL__CRS__URL="http://minio:9000/kms-public/PUB/CRS/d76cca547619b54696469dfa08839fb1c94c0af017145bd85fb4208efeda9769"
+APP_KEYURL__CRS__URL=http://minio:9000/kms-public/PUB/CRS/0500000000000000000000000000000000000000000000000000000000000001
 
 
 # Network configuration

test-suite/fhevm/fhevm-cli

@@ -15,33 +15,27 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 PROJECT="fhevm"
 
 # Default versions for the fhevm stack.
-# export CONNECTOR_DB_MIGRATION_VERSION=${CONNECTOR_DB_MIGRATION_VERSION:-"v0.8.0"}
-# export CONNECTOR_GW_LISTENER_VERSION=${CONNECTOR_GW_LISTENER_VERSION:-"v0.8.0"}
-# export CONNECTOR_KMS_WORKER_VERSION=${CONNECTOR_KMS_WORKER_VERSION:-"v0.8.0"}
-# export CONNECTOR_TX_SENDER_VERSION=${CONNECTOR_TX_SENDER_VERSION:-"v0.8.0"}
 
 # KMS services.
-export CONNECTOR_DB_MIGRATION_VERSION="06603be"
-export CONNECTOR_GW_LISTENER_VERSION="06603be"
-export CONNECTOR_KMS_WORKER_VERSION="06603be"
-export CONNECTOR_TX_SENDER_VERSION="06603be"
+export CONNECTOR_DB_MIGRATION_VERSION=${CONNECTOR_DB_MIGRATION_VERSION:-"06603be"}
+export CONNECTOR_GW_LISTENER_VERSION=${CONNECTOR_GW_LISTENER_VERSION:-"06603be"}
+export CONNECTOR_KMS_WORKER_VERSION=${CONNECTOR_KMS_WORKER_VERSION:-"06603be"}
+export CONNECTOR_TX_SENDER_VERSION=${CONNECTOR_TX_SENDER_VERSION:-"06603be"}
 
 # Coprocessor services.
-export DB_MIGRATION_VERSION=${DB_MIGRATION_VERSION:-"v0.8.0"}
-export GW_LISTENER_VERSION=${GW_LISTENER_VERSION:-"v0.8.0"}
-export HOST_LISTENER_VERSION=${HOST_LISTENER_VERSION:-"v0.8.0"}
-export TX_SENDER_VERSION=${TX_SENDER_VERSION:-"v0.8.0"}
+export DB_MIGRATION_VERSION=${DB_MIGRATION_VERSION:-"b12a580"}
+export GW_LISTENER_VERSION=${GW_LISTENER_VERSION:-"b12a580"}
+export HOST_LISTENER_VERSION=${HOST_LISTENER_VERSION:-"b12a580"}
+export TX_SENDER_VERSION=${TX_SENDER_VERSION:-"b12a580"}
 
 # Other services.
-# export GATEWAY_VERSION=${GATEWAY_VERSION:-"v0.8.0"}
-export GATEWAY_VERSION="03401a4"
-export HOST_VERSION=${HOST_VERSION:-"v0.8.0"}
-export TFHE_WORKER_VERSION=${TFHE_WORKER_VERSION:-"v0.8.0"}
-export SNS_WORKER_VERSION=${SNS_WORKER_VERSION:-"v0.8.0"}
-export ZKPROOF_WORKER_VERSION=${ZKPROOF_WORKER_VERSION:-"v0.8.0"}
-export TEST_SUITE_VERSION=${TEST_SUITE_VERSION:-"v0.8.0"}
-# export CORE_VERSION=${CORE_VERSION:-"v0.11.0-26"}
-export CORE_VERSION="bc02d56"
+export GATEWAY_VERSION=${GATEWAY_VERSION:-"03401a4"}
+export HOST_VERSION=${HOST_VERSION:-"v0.8.0-6"}
+export TFHE_WORKER_VERSION=${TFHE_WORKER_VERSION:-"b12a580"}
+export SNS_WORKER_VERSION=${SNS_WORKER_VERSION:-"b12a580"}
+export ZKPROOF_WORKER_VERSION=${ZKPROOF_WORKER_VERSION:-"b12a580"}
+export TEST_SUITE_VERSION=${TEST_SUITE_VERSION:-"v0.8.0-6"}
+export CORE_VERSION=${CORE_VERSION:-"bc02d56"}
 export RELAYER_VERSION=${RELAYER_VERSION:-"v0.3.1"}