feat(coprocessor): add transaction dependence chains in HL (#1651)

* fix(coprocessor): host-listener, dependency chain

* fix(coprocessor): fix to squash, duplicated trivial encrypt

* fix(coprocessor): fix to squash, duplicated trivial encrypt, test

* fix(coprocessor): fix to squash, scalars are not handles

* fix(coprocessor): fix to squash, cargo fmt

* feat(coprocessor): topologic timestamp

* fix(coprocessor): host-listener, reject cycle and describe out of order dependencies

* fix(coprocessor): do not update dependence chain timestamp on row update

* fix(coprocessor): host-listener, bad condition for need to sort tx

* feat(coprocessor): host-listener, dependency_count for dependency_chain

* feat(coprocessor): host-listener, dependents for dependency_chain

* fix(coprocessor): restrict dependence counter to block scope

* fix(coprocessor): do not update dependence chain last_updated_at on release

* fix(coprocessor): emit warning only when dependence chain is missing dependences

* feat(coprocessor): host-listener, dependency_chain as connected component

* fix(coprocessor): host-listener, update last_updated_at de chain when already processed

* fix(coprocessor): deprecate schedule order in TFHE worker

* chore(coprocessor): fix CI

* fix(coprocessor): host-listener, params for dependency chain policy

* fix(coprocessor): hist-listener, dependency_chain, cycle detection

* chore(coprocessor): update charts for new params

* chore(coprocessor): fix TFHE worker CI test test_extend_or_release_lock_2

---------

Co-authored-by: rudy <rudy.sicard@zama.ai>

Author: antoniupop

.github/workflows/test-suite-e2e-operators-tests.yml

@@ -101,6 +101,13 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GHCR_READ_TOKEN }}
 
+      - name: Login to Chainguard Registry
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
+        with:
+          registry: cgr.dev
+          username: ${{ secrets.CGR_USERNAME }}
+          password: ${{ secrets.CGR_PASSWORD }}
+
       - name: Deploy fhevm Stack
         working-directory: test-suite/fhevm
         env:

charts/coprocessor/values.yaml

@@ -153,9 +153,13 @@ hostListener:
     - --initial-block-time=12  # it can switch to real blockTime when data is available
     - --log-level=INFO
     - --health-port=8080
-    - --dependence-cache-size=128
     - --reorg-maximum-duration-in-blocks=50
 
+    ### Dependence chains parameters
+    # - --dependence-cache-size=10000
+    # - --dependence-by-connexity   # Whether to build by connected components or linear chains only
+    # - --dependence-cross-block    # Do chains cross L1 block boundaries (default no) 
+
     ### Catchup parameters (optional)
     # - --catchup-margin
     # - --catchup-paging

…6ef6b1bad9654b4329cfe03bc1fa86ec41c.json …c5d9987d64701550eeb6d3b364f3acff0dc.jsoncoprocessor/fhevm-engine/.sqlx/query-b97f82da02242ae2916f28242ce4b6ef6b1bad9654b4329cfe03bc1fa86ec41c.json renamed to coprocessor/fhevm-engine/.sqlx/query-101db8494f4d2b3600130016f1dd0c5d9987d64701550eeb6d3b364f3acff0dc.json coprocessor/fhevm-engine/.sqlx/query-b97f82da02242ae2916f28242ce4b6ef6b1bad9654b4329cfe03bc1fa86ec41c.json renamed to coprocessor/fhevm-engine/.sqlx/query-101db8494f4d2b3600130016f1dd0c5d9987d64701550eeb6d3b364f3acff0dc.json

@@ -85,12 +85,15 @@ tfhe = { version = "=1.4.0-alpha.4", features = [
 ] }
 tfhe-versionable = "=0.6.2"
 tfhe-zk-pok = "=0.7.4"
+time = "0.3.43"
 tokio = { version = "1.45.0", features = ["full"] }
 tokio-util = "0.7.15"
 tonic = { version = "0.12.3", features = ["server"] }
 tonic-build = "0.12.3"
 tracing = "0.1.41"
 tracing-subscriber = { version = "0.3.20", features = ["fmt", "json"] }
+tracing-test = "0.2.5"
+union-find = "0.4.3"
 humantime = "2.2.0"
 bytesize = "2.0.1"
 http = "1.3.1"

coprocessor/fhevm-engine/.sqlx/query-1dde98bc8c1076c5708f985512b88082da81a35fd1411d6f9871a5414075a666.json

@@ -25,15 +25,17 @@ bigdecimal = { workspace = true }
 clap = { workspace = true }
 futures-util = { workspace = true }
 lru = { workspace = true }
+prometheus = { workspace = true }
 rustls = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
 sqlx = { workspace = true }
+time = { workspace = true }
 tokio = { workspace = true }
 tokio-util = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
-prometheus = { workspace = true }
+union-find = { workspace = true}
 
 # local dependencies
 fhevm-engine-common = { path = "../fhevm-engine-common" }
@@ -43,6 +45,7 @@ alloy = { workspace = true, features = ["node-bindings"] }
 anyhow = { workspace = true }
 serial_test = { workspace = true }
 test-harness = { path = "../test-harness" }
+tracing-test = { workspace = true }
 
 [build-dependencies]
 foundry-compilers = { workspace = true }

…8d0388c50a8b842be000d9c804fe17eba9e.json …dfa34bf30db25bcf69a333a341b17b5557a.jsoncoprocessor/fhevm-engine/.sqlx/query-aab520e6ce550c4dec799427739a68d0388c50a8b842be000d9c804fe17eba9e.json renamed to coprocessor/fhevm-engine/.sqlx/query-94e9cb426316068aa285da33e7fd1dfa34bf30db25bcf69a333a341b17b5557a.json coprocessor/fhevm-engine/.sqlx/query-aab520e6ce550c4dec799427739a68d0388c50a8b842be000d9c804fe17eba9e.json renamed to coprocessor/fhevm-engine/.sqlx/query-94e9cb426316068aa285da33e7fd1dfa34bf30db25bcf69a333a341b17b5557a.json

@@ -101,11 +101,25 @@ pub struct Args {
 
     #[arg(
         long,
-        default_value = "128",
+        default_value = "10000",
         help = "Pre-computation dependence chain cache size"
     )]
     pub dependence_cache_size: u16,
 
+    #[arg(
+        long,
+        default_value_t = false,
+        help = "Dependence chain are connected components"
+    )]
+    pub dependence_by_connexity: bool,
+
+    #[arg(
+        long,
+        default_value_t = false,
+        help = "Dependence chain are across blocks"
+    )]
+    pub dependence_cross_block: bool,
+
     #[arg(
         long,
         default_value = "50",
@@ -842,6 +856,7 @@ impl InfiniteLogIter {
                     warn!(
                         new_block = ?block_logs.summary,
                         block_time = self.block_time,
+                        nb_logs = block_logs.logs.len(),
                         "Block timeout, proceed with last block"
                     );
                     break block_logs;
@@ -875,6 +890,7 @@ async fn db_insert_block(
     block_logs: &BlockLogs<Log>,
     acl_contract_address: &Option<Address>,
     tfhe_contract_address: &Option<Address>,
+    args: &Args,
 ) -> anyhow::Result<()> {
     info!(
         block = ?block_logs.summary,
@@ -890,6 +906,8 @@ async fn db_insert_block(
             block_logs,
             acl_contract_address,
             tfhe_contract_address,
+            args.dependence_by_connexity,
+            args.dependence_cross_block,
         )
         .await;
         let Err(err) = res else {
@@ -1052,6 +1070,7 @@ pub async fn main(args: Args) -> anyhow::Result<()> {
                 &block_logs,
                 &acl_contract_address,
                 &tfhe_contract_address,
+                &args,
             )
             .await;
             if status.is_err() {