Merge pull request #226 from joel-u410/joel/1.0.0-maint-merge-main

Merge `1.0.0-maint` into `main`

Author: Fraccaman

.github/workflows/queue_and_merge.yml

@@ -107,7 +107,7 @@ jobs:
   docker:
     name: Docker
     runs-on: ubuntu-latest
-    if: ${{ github.event_name == 'merge_group' || github.event_name == 'pull_request' }}
+    # if: ${{ github.event_name == 'merge_group' || github.event_name == 'pull_request' }}
     env:
       GIT_LFS_SKIP_SMUDGE: 1
       REGISTRY_URL: ghcr.io
@@ -157,7 +157,7 @@ jobs:
           context: .
           file: Dockerfile
           build-args: PACKAGE=${{ matrix.docker.package }}
-          push: ${{ github.event.pull_request.head.repo.full_name == 'anoma/namada-indexer' }}
+          push: ${{ github.event.pull_request.head.repo.full_name == 'anoma/namada-indexer' || github.event_name == 'push'  }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
@@ -194,5 +194,4 @@ jobs:
         run: |
           echo "ALL_SUCCESS=$(echo "$NEEDS_JSON" | jq '. | to_entries | map([.value.result == "success", .value.result == "skipped"] | any) | all')" >> $GITHUB_ENV
       - name: check outcomes
-        run: "[ $ALL_SUCCESS == true ]"
-
+        run: "[ $ALL_SUCCESS == true ]"

.github/workflows/scripts/update-package.py

@@ -5,7 +5,8 @@
 
 configuration = {
   "npmName": "@namada/indexer-client",
-  "npmVersion": package_version
+  "npmVersion": package_version,
+  "httpUserAgent": ""
 }
 
 with open("swagger-codegen.json", 'w+', encoding='utf-8') as f:

Cargo.toml

@@ -8,7 +8,7 @@ authors = ["Heliax <hello@heliax.dev>"]
 edition = "2021"
 license = "GPL-3.0"
 readme = "README.md"
-version = "1.0.0"
+version = "1.1.6"
 
 [workspace.dependencies]
 clokwerk = "0.4.0"
@@ -40,14 +40,14 @@ axum-extra = { version = "0.9.3", features = ["query"] }
 chrono = { version = "0.4.30", features = ["serde"] }
 async-trait = "0.1.73"
 anyhow = "1.0.75"
-namada_core = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" }
-namada_sdk = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments", default-features = false, features = ["std", "async-send", "download-params"] }
-namada_tx = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" }
-namada_governance = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" }
-namada_ibc = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" }
-namada_token = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" }
-namada_parameters = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" }
-namada_proof_of_stake = { git = "https://github.com/anoma/namada", branch = "fraccaman/rpc-pgf-payments" }
+namada_core = { git = "https://github.com/anoma/namada", branch = "main" }
+namada_sdk = { git = "https://github.com/anoma/namada", branch = "main", default-features = false, features = ["std", "async-send", "download-params"] }
+namada_tx = { git = "https://github.com/anoma/namada", branch = "main" }
+namada_governance = { git = "https://github.com/anoma/namada", branch = "main" }
+namada_ibc = { git = "https://github.com/anoma/namada", branch = "main" }
+namada_token = { git = "https://github.com/anoma/namada", branch = "main" }
+namada_parameters = { git = "https://github.com/anoma/namada", branch = "main" }
+namada_proof_of_stake = { git = "https://github.com/anoma/namada", branch = "main" }
 tendermint = "0.38.0"
 tendermint-config = "0.38.0"
 tendermint-rpc = { version = "0.38.0", features = ["http-client"] }

Dockerfile

@@ -1,4 +1,4 @@
-FROM lukemathwalker/cargo-chef:latest-rust-1.79-bookworm AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.81-bookworm AS chef
 RUN apt-get update && apt-get install -y protobuf-compiler build-essential clang-tools-14
 
 FROM chef AS planner

chain/src/main.rs

@@ -28,7 +28,9 @@ use shared::crawler_state::ChainCrawlerState;
 use shared::error::{AsDbError, AsRpcError, ContextDbInteractError, MainError};
 use shared::id::Id;
 use shared::token::Token;
+use shared::utils::BalanceChange;
 use shared::validator::ValidatorSet;
+use tendermint_rpc::endpoint::block::Response as TendermintBlockResponse;
 use tendermint_rpc::HttpClient;
 use tokio_retry::strategy::{jitter, ExponentialBackoff};
 use tokio_retry::Retry;
@@ -140,6 +142,7 @@ async fn main() -> Result<(), MainError> {
             initial_query(
                 &client,
                 &conn,
+                checksums.clone(),
                 config.initial_query_retry_time,
                 config.initial_query_retry_attempts,
             )
@@ -186,46 +189,15 @@ async fn crawling_fn(
         return Err(MainError::NoAction);
     }
 
-    tracing::debug!(block = block_height, "Query block...");
-    let tm_block_response =
-        tendermint_service::query_raw_block_at_height(&client, block_height)
-            .await
-            .into_rpc_error()?;
-    tracing::debug!(
-        block = block_height,
-        "Raw block contains {} txs...",
-        tm_block_response.block.data.len()
-    );
-
-    tracing::debug!(block = block_height, "Query block results...");
-    let tm_block_results_response =
-        tendermint_service::query_raw_block_results_at_height(
-            &client,
-            block_height,
-        )
-        .await
-        .into_rpc_error()?;
-    let block_results = BlockResult::from(tm_block_results_response);
-
-    tracing::debug!(block = block_height, "Query epoch...");
-    let epoch =
-        namada_service::get_epoch_at_block_height(&client, block_height)
-            .await
-            .into_rpc_error()?;
-
     tracing::debug!(block = block_height, "Query first block in epoch...");
     let first_block_in_epoch =
         namada_service::get_first_block_in_epoch(&client)
             .await
             .into_rpc_error()?;
 
-    let block = Block::from(
-        tm_block_response,
-        &block_results,
-        checksums,
-        epoch,
-        block_height,
-    );
+    let (block, tm_block_response, epoch) =
+        get_block(block_height, &client, checksums).await?;
+
     tracing::debug!(
         block = block_height,
         txs = block.transactions.len(),
@@ -243,8 +215,33 @@ async fn crawling_fn(
         .map(Token::Ibc)
         .collect::<Vec<Token>>();
 
+    let native_addresses =
+        namada_service::query_native_addresses_balance_change(Token::Native(
+            native_token.clone(),
+        ));
     let addresses = block.addresses_with_balance_change(&native_token);
 
+    let validators_addresses = if first_block_in_epoch.eq(&block_height) {
+        namada_service::get_all_consensus_validators_addresses_at(
+            &client,
+            epoch - 1,
+            native_token.clone(),
+        )
+        .await
+        .into_rpc_error()?
+    } else {
+        HashSet::default()
+    };
+
+    let block_proposer_address = block
+        .header
+        .proposer_address_namada
+        .as_ref()
+        .map(|address| BalanceChange {
+            address: Id::Account(address.clone()),
+            token: Token::Native(native_token.clone()),
+        });
+
     let pgf_receipient_addresses = if first_block_in_epoch.eq(&block_height) {
         conn.interact(move |conn| {
             namada_pgf_repository::get_pgf_receipients_balance_changes(
@@ -260,8 +257,12 @@ async fn crawling_fn(
         HashSet::default()
     };
 
-    let all_balance_changed_addresses = pgf_receipient_addresses
-        .union(&addresses)
+    let all_balance_changed_addresses = addresses
+        .iter()
+        .chain(block_proposer_address.iter())
+        .chain(pgf_receipient_addresses.iter())
+        .chain(validators_addresses.iter())
+        .chain(native_addresses.iter())
         .cloned()
         .collect::<HashSet<_>>();
 
@@ -272,7 +273,13 @@ async fn crawling_fn(
     )
     .await
     .into_rpc_error()?;
-    tracing::info!("Updating balance for {} addresses...", addresses.len());
+
+    tracing::debug!(
+        block = block_height,
+        addresses = all_balance_changed_addresses.len(),
+        "Updating balance for {} addresses...",
+        all_balance_changed_addresses.len()
+    );
 
     let next_governance_proposal_id =
         namada_service::query_next_governance_id(&client, block_height)
@@ -298,12 +305,18 @@ async fn crawling_fn(
         proposals_votes.len()
     );
 
-    let validators = block.validators();
+    let validators = block.new_validators();
     let validator_set = ValidatorSet {
         validators: validators.clone(),
         epoch,
     };
 
+    let validators_state_change = block.update_validators_state();
+    tracing::debug!(
+        "Updating {} validators state",
+        validators_state_change.len()
+    );
+
     let addresses = block.bond_addresses();
     let bonds = query_bonds(&client, addresses).await.into_rpc_error()?;
     tracing::debug!(
@@ -368,6 +381,7 @@ async fn crawling_fn(
         withdraws = withdraw_addreses.len(),
         claimed_rewards = reward_claimers.len(),
         revealed_pks = revealed_pks.len(),
+        validator_state = validators_state_change.len(),
         epoch = epoch,
         first_block_in_epoch = first_block_in_epoch,
         block = block_height,
@@ -383,6 +397,12 @@ async fn crawling_fn(
                     ibc_tokens,
                 )?;
 
+                repository::block::upsert_block(
+                    transaction_conn,
+                    block,
+                    tm_block_response,
+                )?;
+
                 repository::balance::insert_balances(
                     transaction_conn,
                     balances,
@@ -402,6 +422,11 @@ async fn crawling_fn(
                     validator_set,
                 )?;
 
+                repository::pos::upsert_validator_state(
+                    transaction_conn,
+                    validators_state_change,
+                )?;
+
                 // We first remove all the bonds and then insert the new ones
                 repository::pos::clear_bonds(
                     transaction_conn,
@@ -453,29 +478,35 @@ async fn crawling_fn(
 async fn initial_query(
     client: &HttpClient,
     conn: &Object,
+    checksums: Checksums,
     retry_time: u64,
     retry_attempts: usize,
 ) -> Result<(), MainError> {
     let retry_strategy = ExponentialBackoff::from_millis(retry_time)
         .map(jitter)
         .take(retry_attempts);
-    Retry::spawn(retry_strategy, || try_initial_query(client, conn)).await
+    Retry::spawn(retry_strategy, || {
+        try_initial_query(client, conn, checksums.clone())
+    })
+    .await
 }
 
 async fn try_initial_query(
     client: &HttpClient,
     conn: &Object,
+    checksums: Checksums,
 ) -> Result<(), MainError> {
     tracing::debug!("Querying initial data...");
     let block_height =
         query_last_block_height(client).await.into_rpc_error()?;
-    let epoch = namada_service::get_epoch_at_block_height(client, block_height)
-        .await
-        .into_rpc_error()?;
+
     let first_block_in_epoch = namada_service::get_first_block_in_epoch(client)
         .await
         .into_rpc_error()?;
 
+    let (block, tm_block_response, epoch) =
+        get_block(block_height, client, checksums.clone()).await?;
+
     let tokens = query_tokens(client).await.into_rpc_error()?;
 
     // This can sometimes fail if the last block height in the node has moved
@@ -535,6 +566,12 @@ async fn try_initial_query(
             .run(|transaction_conn| {
                 repository::balance::insert_tokens(transaction_conn, tokens)?;
 
+                repository::block::upsert_block(
+                    transaction_conn,
+                    block,
+                    tm_block_response,
+                )?;
+
                 tracing::debug!(
                     block = block_height,
                     "Inserting {} balances...",
@@ -611,3 +648,60 @@ async fn update_crawler_timestamp(
     .and_then(identity)
     .into_db_error()
 }
+
+async fn get_block(
+    block_height: u32,
+    client: &HttpClient,
+    checksums: Checksums,
+) -> Result<(Block, TendermintBlockResponse, u32), MainError> {
+    tracing::debug!(block = block_height, "Query block...");
+    let tm_block_response =
+        tendermint_service::query_raw_block_at_height(client, block_height)
+            .await
+            .into_rpc_error()?;
+    tracing::debug!(
+        block = block_height,
+        "Raw block contains {} txs...",
+        tm_block_response.block.data.len()
+    );
+
+    tracing::debug!(block = block_height, "Query block results...");
+    let tm_block_results_response =
+        tendermint_service::query_raw_block_results_at_height(
+            client,
+            block_height,
+        )
+        .await
+        .into_rpc_error()?;
+    let block_results = BlockResult::from(tm_block_results_response);
+
+    tracing::debug!(block = block_height, "Query epoch...");
+    let epoch = namada_service::get_epoch_at_block_height(client, block_height)
+        .await
+        .into_rpc_error()?;
+
+    let proposer_address_namada = namada_service::get_validator_namada_address(
+        client,
+        &Id::from(&tm_block_response.block.header.proposer_address),
+    )
+    .await
+    .into_rpc_error()?;
+
+    tracing::info!(
+        block = block_height,
+        tm_address = tm_block_response.block.header.proposer_address.to_string(),
+        namada_address = ?proposer_address_namada,
+        "Got block proposer address"
+    );
+
+    let block = Block::from(
+        &tm_block_response,
+        &block_results,
+        &proposer_address_namada,
+        checksums,
+        epoch,
+        block_height,
+    );
+
+    Ok((block, tm_block_response, epoch))
+}