feat(coprocessor): parallelize CI tests across services using cargo-nextest

PanGan21 · PanGan21 · commit c388eaf84e30 · 2026-03-24T14:08:59.000+01:00
diff --git a/.github/workflows/coprocessor-cargo-tests.yml b/.github/workflows/coprocessor-cargo-tests.yml
@@ -32,15 +32,67 @@ jobs:
               - coprocessor/fhevm-engine/**
               - coprocessor/proto/**
   cargo-tests:
-    name: coprocessor-cargo-test/cargo-tests (bpr)
+    name: coprocessor-cargo-test/${{ matrix.service }}
     needs: check-changes
     if: ${{ needs.check-changes.outputs.changes-rust-files == 'true' }}
     permissions:
       contents: 'read' # Required to checkout repository code
       checks: 'write' # Required to create GitHub checks for test results
       packages: 'read' # Required to read GitHub packages/container registry
-      pull-requests: 'write' # Required to post coverage comment on PR
-    runs-on: large_ubuntu_16
+    runs-on: ${{ matrix.runner }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - service: tfhe-worker
+            package: tfhe-worker
+            needs_db: true
+            needs_db_reset: false
+            needs_localstack: false
+            needs_foundry: false
+            runner: large_ubuntu_16
+          - service: sns-worker
+            package: sns-worker
+            needs_db: true
+            needs_db_reset: true
+            needs_localstack: true
+            needs_foundry: false
+            runner: large_ubuntu_16
+          - service: zkproof-worker
+            package: zkproof-worker
+            needs_db: true
+            needs_db_reset: true
+            needs_localstack: false
+            needs_foundry: false
+            runner: large_ubuntu_16
+          - service: transaction-sender
+            package: transaction-sender
+            needs_db: true
+            needs_db_reset: true
+            needs_localstack: true
+            needs_foundry: true
+            runner: large_ubuntu_16
+          - service: gw-listener
+            package: gw-listener
+            needs_db: true
+            needs_db_reset: true
+            needs_localstack: false
+            needs_foundry: true
+            runner: large_ubuntu_16
+          - service: host-listener
+            package: host-listener
+            needs_db: true
+            needs_db_reset: true
+            needs_localstack: false
+            needs_foundry: true
+            runner: large_ubuntu_16
+          - service: common
+            package: fhevm-engine-common
+            needs_db: false
+            needs_db_reset: false
+            needs_localstack: false
+            needs_foundry: false
+            runner: ubuntu-latest
     steps:
     - name: Checkout code
       uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -65,10 +117,13 @@ jobs:
     - name: Install cargo dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install -y protobuf-compiler && \
-        cargo install sqlx-cli --version 0.7.2 --no-default-features --features postgres --locked
+        sudo apt-get install -y protobuf-compiler
+    - name: Install sqlx-cli
+      if: ${{ matrix.needs_db }}
+      run: cargo install sqlx-cli --version 0.7.2 --no-default-features --features postgres --locked
 
     - name: Install foundry
+      if: ${{ matrix.needs_foundry }}
       uses: foundry-rs/foundry-toolchain@de808b1eea699e761c404bda44ba8f21aba30b2c
 
     - name: Cache cargo
@@ -78,24 +133,27 @@ jobs:
           ~/.cargo/registry
           ~/.cargo/git
           target
-        key: ${{ runner.os }}-cargo-coverage-${{ hashFiles('**/Cargo.lock') }}
-        restore-keys: ${{ runner.os }}-cargo-coverage-
+        key: ${{ runner.os }}-cargo-coverage-${{ matrix.service }}-${{ hashFiles('**/Cargo.lock') }}
+        restore-keys: ${{ runner.os }}-cargo-coverage-${{ matrix.service }}-
 
     - name: Login to GitHub Container Registry
+      if: ${{ matrix.needs_db }}
       uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
       with:
         registry: ghcr.io
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Login to GitHub Chainguard Registry
+      if: ${{ matrix.needs_db }}
       uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
       with:
         registry: cgr.dev
         username: ${{ secrets.CGR_USERNAME }}
         password: ${{ secrets.CGR_PASSWORD }}
 
     - name: Init database
+      if: ${{ matrix.needs_db }}
       run: make init_db
       working-directory: coprocessor/fhevm-engine/tfhe-worker
 
@@ -105,47 +163,82 @@ jobs:
         node-version: 20.x
 
     - name: Start localstack
+      if: ${{ matrix.needs_localstack }}
       run: |
         docker run --rm -d -p 4566:4566 --name localstack localstack/localstack:4.14.0
 
-    - name: Clean previous coverage data
-      run: cargo llvm-cov clean --workspace --profile coverage
-      working-directory: coprocessor/fhevm-engine
-
     - name: Run tests with coverage
       run: |
+        cargo llvm-cov clean --workspace --profile coverage
         DATABASE_URL=postgresql://postgres:postgres@localhost:5432/coprocessor \
-        TEST_GLOBAL_LOCALSTACK=1 \
-        cargo llvm-cov --no-report --workspace --profile coverage
+        COPROCESSOR_TEST_LOCALHOST_RESET=${{ matrix.needs_db_reset && '1' || '0' }} \
+        TEST_GLOBAL_LOCALSTACK=${{ matrix.needs_localstack && '1' || '0' }} \
+        cargo llvm-cov --no-report -p ${{ matrix.package }} --profile coverage
       working-directory: coprocessor/fhevm-engine
 
-    - name: Generate coverage report
+    - name: Export LCOV coverage data
       if: ${{ !cancelled() }}
+      run: cargo llvm-cov report --lcov --profile coverage --output-path /tmp/lcov.info || true
+      working-directory: coprocessor/fhevm-engine
+
+    - name: Upload coverage artifact
+      if: ${{ !cancelled() }}
+      uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+      with:
+        name: lcov-${{ matrix.service }}
+        path: /tmp/lcov.info
+        retention-days: 1
+        if-no-files-found: ignore
+
+  coverage-report:
+    name: coprocessor-cargo-test/coverage-report
+    needs: [check-changes, cargo-tests]
+    if: ${{ !cancelled() && needs.check-changes.outputs.changes-rust-files == 'true' }}
+    permissions:
+      contents: 'read' # Required to checkout repository code
+      pull-requests: 'write' # Required to post coverage comment on PR
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      with:
+        persist-credentials: 'false'
+
+    - name: Install lcov
+      run: sudo apt-get update && sudo apt-get install -y lcov
+
+    - name: Download all coverage artifacts
+      uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
+      with:
+        pattern: lcov-*
+        path: /tmp/coverage
+
+    - name: Merge LCOV files
       run: |
-        if cargo llvm-cov report --profile coverage > /tmp/cov-report.txt 2>&1; then
-          REPORT=$(cat /tmp/cov-report.txt)
-        else
-          echo "cargo llvm-cov report failed:"
-          cat /tmp/cov-report.txt
-          REPORT=""
+        LCOV_FILES=$(find /tmp/coverage -name 'lcov.info' -size +0c)
+        if [ -z "$LCOV_FILES" ]; then
+          echo "No coverage data found"
+          exit 0
         fi
+        LCOV_ARGS=""
+        for f in $LCOV_FILES; do
+          LCOV_ARGS="$LCOV_ARGS -a $f"
+        done
+        lcov $LCOV_ARGS -o /tmp/lcov.info
+
+    - name: Generate coverage summary
+      if: ${{ !cancelled() }}
+      run: |
         {
           echo '## Coverage: coprocessor/fhevm-engine'
-          if [ -n "$REPORT" ]; then
+          if [ -f /tmp/lcov.info ]; then
             echo '```'
-            echo "$REPORT"
+            lcov --summary /tmp/lcov.info 2>&1 || true
             echo '```'
           else
             echo '*No coverage data available (tests may have failed before producing profiling data).*'
           fi
         } >> "$GITHUB_STEP_SUMMARY"
-        echo "$REPORT"
-      working-directory: coprocessor/fhevm-engine
-
-    - name: Export LCOV coverage data
-      if: ${{ !cancelled() }}
-      run: cargo llvm-cov report --lcov --profile coverage --output-path /tmp/lcov.info || true
-      working-directory: coprocessor/fhevm-engine
 
     - name: Diff coverage of changed lines
       if: ${{ !cancelled() }}
diff --git a/coprocessor/fhevm-engine/test-harness/src/instance.rs b/coprocessor/fhevm-engine/test-harness/src/instance.rs
@@ -55,6 +55,33 @@ pub async fn setup_test_db(mode: ImportMode) -> Result<DBInstance, Box<dyn std::
     }
 }
 
+// Extracts the database name from a PostgreSQL URL.
+// e.g. "postgresql://user:pass@host:port/mydb?opt=val" -> "mydb"
+// Panics if the extracted name contains characters other than alphanumeric, underscore, or hyphen.
+fn extract_db_name(db_url: &str) -> &str {
+    let after_slash = db_url
+        .rsplit('/')
+        .next()
+        .expect("database URL must contain /");
+    let name = after_slash
+        .split('?')
+        .next()
+        .expect("split always yields at least one element");
+    assert!(
+        !name.is_empty()
+            && name
+                .chars()
+                .all(|c| c.is_alphanumeric() || c == '_' || c == '-'),
+        "invalid database name extracted from URL: {name}"
+    );
+    name
+}
+
+fn admin_url_from(db_url: &str) -> String {
+    let last_slash = db_url.rfind('/').expect("database URL must contain /");
+    format!("{}postgres", &db_url[..=last_slash])
+}
+
 async fn setup_test_app_existing_localhost(
     with_reset: bool,
     mode: ImportMode,
@@ -63,7 +90,7 @@ async fn setup_test_app_existing_localhost(
 
     if with_reset {
         info!("Resetting local database at {db_url}");
-        let admin_db_url = db_url.as_str().replace("coprocessor", "postgres");
+        let admin_db_url = admin_url_from(db_url.as_str());
         create_database(&admin_db_url, db_url.as_str(), mode).await?;
     }
 
@@ -122,17 +149,18 @@ async fn create_database(
     db_url: &str,
     mode: ImportMode,
 ) -> Result<(), Box<dyn std::error::Error>> {
-    info!("Creating coprocessor db...");
+    let db_name = extract_db_name(db_url);
+    info!(db_name, "Creating database...");
     let admin_pool = sqlx::postgres::PgPoolOptions::new()
         .max_connections(1)
         .connect(admin_db_url)
         .await?;
 
-    sqlx::query!("DROP DATABASE IF EXISTS coprocessor;")
+    sqlx::query(&format!("DROP DATABASE IF EXISTS \"{db_name}\""))
         .execute(&admin_pool)
         .await?;
 
-    sqlx::query!("CREATE DATABASE coprocessor;")
+    sqlx::query(&format!("CREATE DATABASE \"{db_name}\""))
         .execute(&admin_pool)
         .await?;
 
diff --git a/coprocessor/fhevm-engine/transaction-sender/tests/common.rs b/coprocessor/fhevm-engine/transaction-sender/tests/common.rs
@@ -11,8 +11,8 @@ use alloy::{
     sol,
     transports::http::reqwest::Url,
 };
-use fhevm_engine_common::utils::DatabaseURL;
 use sqlx::{postgres::PgPoolOptions, Pool, Postgres};
+use test_harness::instance::{setup_test_db, DBInstance, ImportMode};
 use test_harness::localstack::{
     create_aws_aws_kms_client, create_localstack_kms_signing_key, start_localstack,
     LocalstackContainer, LOCALSTACK_PORT,
@@ -53,7 +53,7 @@ pub struct TestEnvironment {
     pub user_address: Address,
     anvil: Option<AnvilInstance>,
     pub wallet: EthereumWallet,
-    // Just keep the handle to destroy the container when it is dropped.
+    _db_instance: DBInstance,
     _localstack: Option<LocalstackContainer>,
 }
 
@@ -80,25 +80,14 @@ impl TestEnvironment {
             .with_test_writer()
             .try_init();
 
+        let db_instance = setup_test_db(ImportMode::None)
+            .await
+            .map_err(|e| anyhow::anyhow!("{e}"))?;
         let db_pool = PgPoolOptions::new()
             .max_connections(10)
-            .connect(DatabaseURL::default().as_str())
+            .connect(db_instance.db_url())
             .await?;
 
-        Self::truncate_tables(
-            &db_pool,
-            vec![
-                "verify_proofs",
-                "ciphertext_digest",
-                "allowed_handles",
-                "delegate_user_decrypt",
-                "keys",
-                "crs",
-                "host_chains",
-            ],
-        )
-        .await?;
-
         let anvil = Self::new_anvil()?;
         let chain_id =
             get_chain_id(anvil.ws_endpoint_url(), std::time::Duration::from_secs(1)).await;
@@ -141,6 +130,7 @@ impl TestEnvironment {
             user_address: PrivateKeySigner::random().address(),
             anvil: Some(anvil),
             wallet,
+            _db_instance: db_instance,
             _localstack: localstack,
         })
     }
@@ -177,12 +167,4 @@ impl TestEnvironment {
     fn new_anvil_with_port(port: u16) -> anyhow::Result<AnvilInstance> {
         Ok(Anvil::new().block_time(1).port(port).try_spawn()?)
     }
-
-    async fn truncate_tables(db_pool: &sqlx::PgPool, tables: Vec<&str>) -> Result<(), sqlx::Error> {
-        for table in tables {
-            let query = format!("TRUNCATE {}", table);
-            sqlx::query(&query).execute(db_pool).await?;
-        }
-        Ok(())
-    }
 }
