Merge pull request #412 from namada-net/cometbft-service

Cometbft service

Author: Fraccaman

.github/workflows/deploy.yml

@@ -62,10 +62,10 @@ jobs:
             { image: governance, package: governance },
             { image: pos, package: pos },
             { image: rewards, package: rewards },
-            { image: seeder, package: seeder },
             { image: webserver, package: webserver },
             { image: parameters, package: parameters },
             { image: transactions, package: transactions },
+            { image: cometbft, package: cometbft },
           ]
 
     steps:

.github/workflows/queue_and_merge.yml

@@ -131,10 +131,10 @@ jobs:
             { image: governance, package: governance },
             { image: pos, package: pos },
             { image: rewards, package: rewards },
-            { image: seeder, package: seeder },
             { image: webserver, package: webserver },
             { image: parameters, package: parameters },
             { image: transactions, package: transactions },
+            { image: cometbft, package: cometbft },
           ]
 
     steps:

Cargo.lock

@@ -9,9 +9,9 @@ members = [
   "pos",
   "governance",
   "webserver",
-  "seeder",
   "parameters",
   "transactions",
+  "cometbft",
   "test_helpers",
   "utilities"
 ]
@@ -102,4 +102,5 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] }
 validator = { version = "0.16.0", features = ["derive"] }
 vergen = "8.0.0"
 bech32 = "0.11.0"
-reqwest = { version = "0.11.27", features = ["json", "cookies"] }
+reqwest = { version = "0.11.27", features = ["json", "cookies"] }
+itertools = "0.14.0"

Cargo.toml

@@ -29,6 +29,7 @@ use shared::block::Block;
 use shared::block_result::BlockResult;
 use shared::checksums::Checksums;
 use shared::client::Client;
+use shared::cometbft::CometbftBlock;
 use shared::crawler::crawl;
 use shared::crawler_state::ChainCrawlerState;
 use shared::error::{
@@ -277,8 +278,12 @@ async fn crawling_fn(
     let native_token_address: namada_sdk::address::Address =
         native_token.clone().into();
 
+    let cometbft_block =
+        get_cometbft_block_with_fallback(&conn, &client, block_height)
+            .await
+            .into_db_error()?;
     let (block, tm_block_response, epoch) =
-        get_block(block_height, &client, &checksums, &native_token_address)
+        get_block(cometbft_block, &client, &checksums, &native_token_address)
             .await?;
 
     let rate_limits = new_epoch.then(|| {
@@ -654,8 +659,14 @@ async fn try_initial_query(
             .await
             .into_rpc_error()?
             .into();
+
+    let cometbft_block =
+        get_cometbft_block_with_fallback(conn, client, block_height)
+            .await
+            .into_db_error()?;
+
     let (block, tm_block_response, epoch) =
-        get_block(block_height, client, &checksums, &native_token).await?;
+        get_block(cometbft_block, client, &checksums, &native_token).await?;
 
     let tokens = query_tokens(client).await.into_rpc_error()?;
 
@@ -846,36 +857,18 @@ async fn update_crawler_timestamp(
 }
 
 async fn get_block(
-    block_height: u32,
+    block: CometbftBlock,
     client: &HttpClient,
     checksums: &Checksums,
     native_token: &namada_sdk::address::Address,
 ) -> Result<(Block, TendermintBlockResponse, u32), MainError> {
-    tracing::debug!(block = block_height, "Query block...");
-    let tm_block_response =
-        tendermint_service::query_raw_block_at_height(client, block_height)
-            .await
-            .into_rpc_error()?;
-    tracing::debug!(
-        block = block_height,
-        "Raw block contains {} txs...",
-        tm_block_response.block.data.len()
-    );
+    let block_height = block.block_height;
 
-    tracing::debug!(block = block_height, "Query block results...");
-    let tm_block_results_response =
-        tendermint_service::query_raw_block_results_at_height(
-            client,
-            block_height,
-        )
-        .await
-        .into_rpc_error()?;
-    let block_results = BlockResult::from(tm_block_results_response);
+    let tm_block_response = block.block;
+    let tm_block_results_response = block.events;
+    let epoch = block.epoch;
 
-    tracing::debug!(block = block_height, "Query epoch...");
-    let epoch = namada_service::get_epoch_at_block_height(client, block_height)
-        .await
-        .into_rpc_error()?;
+    let block_results = BlockResult::from(tm_block_results_response);
 
     let proposer_address_namada = namada_service::get_validator_namada_address(
         client,
@@ -939,3 +932,46 @@ async fn query_token_supplies(
 
     Ok(buffer)
 }
+
+pub async fn get_cometbft_block_with_fallback(
+    conn: &Object,
+    client: &HttpClient,
+    block_height: u32,
+) -> anyhow::Result<CometbftBlock> {
+    let block = repository::cometbft::get_block(conn, block_height)
+        .await
+        .context("Failed to get block")?;
+
+    let block = match block {
+        Some(block) => block,
+        None => {
+            let block = tendermint_service::query_raw_block_at_height(
+                client,
+                block_height,
+            )
+            .await
+            .context("Failed to query block")?;
+
+            let events = tendermint_service::query_raw_block_results_at_height(
+                client,
+                block_height,
+            )
+            .await
+            .context("Failed to query block results")?;
+
+            let epoch =
+                namada_service::get_epoch_at_block_height(client, block_height)
+                    .await
+                    .context("Failed to query epoch")?;
+
+            CometbftBlock {
+                block_height,
+                block,
+                events,
+                epoch,
+            }
+        }
+    };
+
+    Ok(block)
+}

chain/src/main.rs

@@ -0,0 +1,21 @@
+use deadpool_diesel::postgres::Object;
+use diesel::{QueryDsl, RunQueryDsl, SelectableHelper};
+use orm::cometbft::CometbftBlock;
+use orm::schema::cometbft_block;
+use shared::cometbft::CometbftBlock as CometBlock;
+
+pub async fn get_block(
+    conn: &Object,
+    block_height: u32,
+) -> anyhow::Result<Option<CometBlock>> {
+    conn.interact(move |conn| {
+        cometbft_block::table
+            .find(block_height as i32)
+            .select(CometbftBlock::as_select())
+            .first(conn)
+            .ok()
+    })
+    .await
+    .map(|block| block.map(CometBlock::from))
+    .map_err(|e| anyhow::anyhow!(e.to_string()))
+}

chain/src/repository/cometbft.rs

@@ -1,5 +1,6 @@
 pub mod balance;
 pub mod block;
+pub mod cometbft;
 pub mod crawler_state;
 pub mod gov;
 pub mod pgf;

chain/src/repository/mod.rs

@@ -1,6 +1,6 @@
 [package]
-name = "seeder"
-description = "Database seeder."
+name = "cometbft"
+description = "Namada cometbft crawler."
 resolver = "2"
 authors.workspace = true
 edition.workspace = true
@@ -9,19 +9,33 @@ readme.workspace = true
 version.workspace = true
 
 [[bin]]
-name = "seeder"
+name = "cometbft"
 path = "src/main.rs"
 
 [dependencies]
 anyhow.workspace = true
+chrono.workspace = true
 clap.workspace = true
 deadpool-diesel.workspace = true
 diesel.workspace = true
+futures.workspace = true
+namada_core.workspace = true
+namada_governance.workspace = true
+namada_sdk.workspace = true
+namada_token.workspace = true
 orm.workspace = true
-rand.workspace = true
+rlimit.workspace = true
+serde_json.workspace = true
 shared.workspace = true
+subtle-encoding.workspace = true
+tendermint-rpc.workspace = true
 tokio.workspace = true
+tokio-retry.workspace = true
 tracing.workspace = true
+itertools.workspace = true
 
 [build-dependencies]
 vergen = { workspace = true, features = ["build", "git", "gitcl"] }
+
+[dev-dependencies]
+test_helpers.workspace = true

seeder/Cargo.toml cometbft/Cargo.tomlseeder/Cargo.toml renamed to cometbft/Cargo.toml

@@ -1,4 +1,4 @@
 . ../.env
 export TENDERMINT_URL
 export DATABASE_URL
-cargo run -- --cargo-env development
+cargo run --release