feat: Add clear_db option

If true (default), clears the DB and re syncs from start. If not:
- for the ranged test, it skips indexing and uses the current state
- for the full test, it starts syncing from the last block in the db

#time 18m

Author: dianacarvalho1

protocol-testing/Cargo.lock

@@ -55,7 +55,7 @@ impl FullTestCommand {
             args.db_url,
             args.rpc_url,
             args.vm_simulation_traces,
-            args.skip_indexing,
+            args.clear_db,
         )?
         .run()
     }
@@ -83,7 +83,7 @@ impl RangeTestCommand {
             args.db_url,
             args.rpc_url,
             args.vm_simulation_traces,
-            args.skip_indexing,
+            args.clear_db,
         )?
         .run()
     }
@@ -122,9 +122,11 @@ struct CommonArgs {
     #[arg(long, default_value_t = false)]
     execution_traces: bool,
 
-    /// Skip indexing and run directly against the database
-    #[arg(long, default_value_t = false)]
-    skip_indexing: bool,
+    /// If true (default), clears the DB and re-syncs from start. If false:
+    //   - for the ranged test, it skips indexing and uses the current state
+    //   - for the full test, it starts syncing from the last block in the db
+    #[arg(long, default_value_t = true)]
+    clear_db: bool,
 }
 
 impl CommonArgs {

protocol-testing/src/main.rs

@@ -97,7 +97,7 @@ pub struct TestRunner {
     runtime: Runtime,
     rpc_provider: RPCProvider,
     protocol_components: Arc<RwLock<HashMap<String, ProtocolComponentModel>>>,
-    skip_indexing: bool,
+    clear_db: bool,
 }
 
 impl TestRunner {
@@ -110,7 +110,7 @@ impl TestRunner {
         db_url: String,
         rpc_url: String,
         vm_simulation_traces: bool,
-        skip_indexing: bool,
+        clear_db: bool,
     ) -> miette::Result<Self> {
         let base_protocol = CLONE_TO_BASE_PROTOCOL
             .get(protocol.as_str())
@@ -150,7 +150,7 @@ impl TestRunner {
             adapter_contract_builder,
             runtime,
             rpc_provider,
-            skip_indexing,
+            clear_db,
             protocol_components: Arc::new(RwLock::new(HashMap::new())),
         })
     }
@@ -497,7 +497,7 @@ impl TestRunner {
             let tycho_runner = self
                 .runtime
                 .block_on(self.tycho_runner(initialized_accounts))?;
-            if !self.skip_indexing {
+            if self.clear_db {
                 let spkg_path = build_spkg(substreams_yaml_path, test.start_block)
                     .wrap_err("Failed to build spkg")?;
                 tycho_runner
@@ -511,7 +511,7 @@ impl TestRunner {
                     )
                     .wrap_err("Failed to run Tycho")?;
             } else {
-                info!("Skipping indexing")
+                info!("Skipping indexing and using existent DB")
             }
             let rpc_server = tycho_runner.start_rpc_server()?;
             match self.run_test(test, &config, test.stop_block) {
@@ -638,10 +638,10 @@ impl TestRunner {
     }
 
     async fn tycho_runner(&self, initialized_accounts: Vec<String>) -> miette::Result<TychoRunner> {
-        // If we skip indexing, reuse current db state
-        if !self.skip_indexing {
+        // If we want to clear db, reuse current db state
+        if self.clear_db {
             self.empty_database()
-            .await
+                .await
                 .into_diagnostic()
                 .wrap_err("Failed to empty the database")?;
         }

protocol-testing/src/test_runner.rs

@@ -67,11 +67,12 @@ impl TychoClient {
     ) -> Result<Vec<ProtocolComponent>, RpcError> {
         let request = ProtocolComponentsRequestBody::system_filtered(protocol_system, None, chain);
 
+        let chunk_size = 100;
         let concurrency = 1;
 
         let response = self
             .http_client
-            .get_protocol_components_paginated(&request, None, concurrency)
+            .get_protocol_components_paginated(&request, Some(chunk_size), concurrency)
             .await?;
 
         Ok(response.protocol_components)
@@ -90,7 +91,7 @@ impl TychoClient {
         #[allow(clippy::mutable_key_type)]
         let res = self
             .http_client
-            .get_all_tokens(chain, min_quality, max_days_since_last_trade, None, concurrency)
+            .get_all_tokens(chain, min_quality, max_days_since_last_trade, Some(3_000), concurrency)
             .await?
             .into_iter()
             .map(|token| {
@@ -147,11 +148,12 @@ impl TychoClient {
             SnapshotParameters::new(chain, protocol_system, components, contract_ids, block_number)
                 .entrypoints(entrypoints);
 
+        let chunk_size = 100;
         let concurrency = 1;
 
         let response = self
             .http_client
-            .get_snapshots(&params, None, concurrency)
+            .get_snapshots(&params, Some(chunk_size), concurrency)
             .await?;
 
         Ok(response)