[Indexes] Add TxIndex: Transaction index, RPC, and native pruning

Cargo.toml

@@ -29,6 +29,7 @@ members = [
     "notify",
     "indexes/core",
     "indexes/processor",
+    "indexes/txindex",
     "indexes/utxoindex",
     "rpc/macros",
     "rpc/core",
@@ -119,6 +120,7 @@ kaspa-pow = { version = "1.1.0-rc.2", path = "consensus/pow" }
 kaspa-rpc-core = { version = "1.1.0-rc.2", path = "rpc/core" }
 kaspa-rpc-macros = { version = "1.1.0-rc.2", path = "rpc/macros" }
 kaspa-rpc-service = { version = "1.1.0-rc.2", path = "rpc/service" }
+kaspa-txindex = { version = "1.1.0-rc.2", path = "indexes/txindex" }
 kaspa-txscript = { version = "1.1.0-rc.2", path = "crypto/txscript" }
 kaspa-txscript-errors = { version = "1.1.0-rc.2", path = "crypto/txscript/errors" }
 kaspa-utils = { version = "1.1.0-rc.2", path = "utils" }
@@ -235,6 +237,7 @@ secp256k1 = { version = "0.29.0", features = [
     "rand-std",
     "serde",
 ] } # TODO "0.28.0"
+self_cell = "1.2.2"
 separator = "0.4.1"
 seqlock = "0.2.0"
 serde = { version = "1.0.190", features = ["derive", "rc"] }

cli/src/modules/rpc.rs

@@ -313,6 +313,47 @@ impl Rpc {
 
                 self.println(&ctx, result);
             }
+            RpcApiOps::GetTransaction => {
+                if argv.is_empty() {
+                    return Err(Error::custom("Missing transaction id argument"));
+                }
+                let transaction_id = argv.remove(0).parse::<TransactionId>()?;
+
+                argv.reverse(); // reverse to allow popping arguments in the correct order
+                let include_unaccepted = argv.pop().unwrap_or("false".to_string()).parse::<bool>().ok().unwrap_or_default();
+                // parse transaction verbosity: accept named (none|low|high|full) or numeric (0|1|2|3) verbosity levels, default to None if parsing fails or if not provided
+                let tx_verbosity_token = argv.pop().unwrap_or("none".to_string());
+                let transaction_verbosity = match tx_verbosity_token.to_lowercase().as_str() {
+                    "none" => Some(RpcDataVerbosityLevel::None),
+                    "low" => Some(RpcDataVerbosityLevel::Low),
+                    "high" => Some(RpcDataVerbosityLevel::High),
+                    "full" => Some(RpcDataVerbosityLevel::Full),
+                    other => {
+                        if let Ok(i) = other.parse::<i32>() {
+                            Some(RpcDataVerbosityLevel::try_from(i)?)
+                        } else {
+                            Some(RpcDataVerbosityLevel::None)
+                        }
+                    }
+                };
+                let include_inclusion_data = argv.pop().unwrap_or("false".to_string()).parse::<bool>().ok().unwrap_or_default();
+                let include_acceptance_data = argv.pop().unwrap_or("false".to_string()).parse::<bool>().ok().unwrap_or_default();
+                let include_conf_count = argv.pop().unwrap_or("false".to_string()).parse::<bool>().ok().unwrap_or_default();
+                let result = rpc
+                    .get_transaction_call(
+                        None,
+                        GetTransactionRequest {
+                            transaction_id,
+                            include_unaccepted,
+                            transaction_verbosity,
+                            include_inclusion_data,
+                            include_acceptance_data,
+                            include_conf_count,
+                        },
+                    )
+                    .await?;
+                self.println(&ctx, result);
+            }
             _ => {
                 tprintln!(ctx, "rpc method exists but is not supported by the cli: '{op_str}'\r\n");
                 return Ok(());

consensus/client/src/transaction.rs

@@ -23,7 +23,7 @@ use kaspa_utils::hex::*;
 const TS_TRANSACTION: &'static str = r#"
 /**
  * Interface defining the structure of a transaction.
- * 
+ *
  * @category Consensus
  */
 export interface ITransaction {
@@ -43,7 +43,7 @@ export interface ITransaction {
 
 /**
  * Optional transaction verbose data.
- * 
+ *
  * @category Node RPC
  */
 export interface ITransactionVerboseData {
@@ -53,6 +53,40 @@ export interface ITransactionVerboseData {
     blockHash : HexString;
     blockTime : bigint;
 }
+
+/**
+ * Transaction data returned by GetTransaction.
+ *
+ * @category Node RPC
+ */
+export interface ITransactionData {
+    transactions: ITransaction[];
+    inclusionData: ITransactionInclusionData[];
+    acceptanceData?: ITransactionAcceptanceData;
+    confCount?: bigint;
+}
+
+/**
+ * Transaction inclusion data
+ *
+ * @category Node RPC
+ */
+export interface ITransactionInclusionData {
+    includingBlockHash: HexString;
+    includingDaaScore: bigint;
+    indexWithinBlock: number;
+}
+
+/**
+ * Transaction acceptance data
+ *
+ * @category Node RPC
+ */
+export interface ITransactionAcceptanceData {
+    acceptingBlockHash: HexString;
+    acceptingBlueScore: bigint;
+    mergesetIndex: number;
+}
 "#;
 
 #[wasm_bindgen]

consensus/core/src/acceptance_data.rs

@@ -1,7 +1,10 @@
 use kaspa_hashes::Hash;
 use serde::{Deserialize, Serialize};
 
-use crate::tx::TransactionId;
+use crate::tx::{TransactionId, TransactionIndexType};
+
+// A mergeset currently cannot have more then 512 blocks.
+pub type MergesetIndexType = u16;
 
 /// Holds a mergeset acceptance data, a list of all its merged block with their accepted transactions
 pub type AcceptanceData = Vec<MergesetBlockAcceptanceData>;
@@ -16,5 +19,19 @@ pub struct MergesetBlockAcceptanceData {
 #[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct AcceptedTxEntry {
     pub transaction_id: TransactionId,
-    pub index_within_block: u32,
+    pub index_within_block: TransactionIndexType,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::config::params::ALL_PARAMS;
+
+    #[test]
+    fn test_mergeset_index_does_not_overflow() {
+        for param in ALL_PARAMS.iter() {
+            // make sure that MergesetIdx can hold mergeset_size_limit
+            MergesetIndexType::try_from(param.mergeset_size_limit()).unwrap();
+        }
+    }
 }

consensus/core/src/api/mod.rs

@@ -291,6 +291,14 @@ pub trait ConsensusApi: Send + Sync {
         unimplemented!()
     }
 
+    fn get_block_transaction_iterator(&self) -> Box<dyn Iterator<Item = (Hash, Arc<Vec<Transaction>>)> + Send + Sync + '_> {
+        unimplemented!()
+    }
+
+    fn get_acceptance_data_iterator(&self) -> Box<dyn Iterator<Item = (Hash, Arc<AcceptanceData>)> + Send + Sync + '_> {
+        unimplemented!()
+    }
+
     fn create_virtual_selected_chain_block_locator(&self, low: Option<Hash>, high: Option<Hash>) -> ConsensusResult<Vec<Hash>> {
         unimplemented!()
     }

consensus/core/src/config/mod.rs

@@ -42,6 +42,9 @@ pub struct Config {
     /// Enable the UTXO index
     pub utxoindex: bool,
 
+    /// Enable the TX index
+    pub txindex: bool,
+
     /// Enable RPC commands which affect the state of the node
     pub unsafe_rpc: bool,
 
@@ -86,6 +89,7 @@ impl Config {
             is_archival: false,
             enable_sanity_checks: false,
             utxoindex: false,
+            txindex: false,
             unsafe_rpc: false,
             enable_unsynced_mining: false,
             enable_mainnet_mining: false,

consensus/core/src/config/params.rs

@@ -16,6 +16,9 @@ use std::{
     ops::{Deref, DerefMut},
 };
 
+// All supported params
+pub const ALL_PARAMS: [Params; 4] = [MAINNET_PARAMS, TESTNET_PARAMS, DEVNET_PARAMS, SIMNET_PARAMS];
+
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
 pub struct ForkActivation(u64);
 
@@ -53,7 +56,7 @@ impl ForkActivation {
     }
 
     /// Checks if the fork is expected to be activated "soon", i.e., in the time frame of the provided range.
-    /// Returns the distance from activation if so, or `None` otherwise.  
+    /// Returns the distance from activation if so, or `None` otherwise.
     pub fn is_within_range_before_activation(self, current_daa_score: u64, range: u64) -> Option<u64> {
         if !self.is_active(current_daa_score) && current_daa_score + range > self.0 { Some(self.0 - current_daa_score) } else { None }
     }

consensus/core/src/errors/consensus.rs

@@ -48,6 +48,8 @@ pub enum ConsensusError {
 
     #[error("{0}")]
     GeneralOwned(String),
+    //#[error("store error: {0}")]
+    //StoreError(#[from] kaspa_database::prelude::StoreError),
 }
 
 pub type ConsensusResult<T> = std::result::Result<T, ConsensusError>;

consensus/notify/src/notification.rs

@@ -19,7 +19,7 @@ pub enum Notification {
     #[display(fmt = "BlockAdded notification: block hash {}", "_0.block.header.hash")]
     BlockAdded(BlockAddedNotification),
 
-    #[display(fmt = "VirtualChainChanged notification: {} removed blocks, {} added blocks, {} accepted transactions", "_0.removed_chain_block_hashes.len()", "_0.added_chain_block_hashes.len()", "_0.added_chain_blocks_acceptance_data.len()")]
+    #[display(fmt = "VirtualChainChanged notification: {} removed blocks, {} added blocks, {} accepted transactions, {} accepting blue scores", "_0.removed_chain_block_hashes.len()", "_0.added_chain_block_hashes.len()", "_0.added_chain_blocks_acceptance_data.len()", "_0.added_accepting_blue_scores.len()")]
     VirtualChainChanged(VirtualChainChangedNotification),
 
     #[display(fmt = "FinalityConflict notification: violating block hash {}", "_0.violating_block_hash")]
@@ -42,8 +42,10 @@ pub enum Notification {
 
     #[display(fmt = "NewBlockTemplate notification")]
     NewBlockTemplate(NewBlockTemplateNotification),
-}
-}
+
+    #[display(fmt = "RetentionRootChanged notification: retention root {}", "_0.retention_root")]
+    RetentionRootChanged(RetentionRootChangedNotification),
+}}
 
 impl NotificationTrait for Notification {
     fn apply_overall_subscription(&self, subscription: &OverallSubscription, _context: &SubscriptionContext) -> Option<Self> {
@@ -63,13 +65,20 @@ impl NotificationTrait for Notification {
                 // If the subscription excludes accepted transaction ids and the notification includes some
                 // then we must re-create the object and drop the ids, otherwise we can clone it as is.
                 if let Notification::VirtualChainChanged(payload) = self {
-                    if !subscription.include_accepted_transaction_ids() && !payload.added_chain_blocks_acceptance_data.is_empty() {
-                        return Some(Notification::VirtualChainChanged(VirtualChainChangedNotification {
-                            removed_chain_block_hashes: payload.removed_chain_block_hashes.clone(),
-                            added_chain_block_hashes: payload.added_chain_block_hashes.clone(),
-                            added_chain_blocks_acceptance_data: Arc::new(vec![]),
-                        }));
-                    }
+                    return Some(Notification::VirtualChainChanged(VirtualChainChangedNotification {
+                        removed_chain_block_hashes: payload.removed_chain_block_hashes.clone(),
+                        added_chain_block_hashes: payload.added_chain_block_hashes.clone(),
+                        added_chain_blocks_acceptance_data: if !subscription.include_accepted_transaction_ids() {
+                            Arc::new(Vec::new())
+                        } else {
+                            payload.added_chain_blocks_acceptance_data.clone()
+                        },
+                        added_accepting_blue_scores: if !subscription.include_accepting_blue_scores() {
+                            Arc::new(Vec::new())
+                        } else {
+                            payload.added_accepting_blue_scores.clone()
+                        },
+                    }));
                 }
                 Some(self.clone())
             }
@@ -92,7 +101,7 @@ impl NotificationTrait for Notification {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Clone, Debug)]
 pub struct BlockAddedNotification {
     pub block: Block,
 }
@@ -108,14 +117,16 @@ pub struct VirtualChainChangedNotification {
     pub added_chain_block_hashes: Arc<Vec<Hash>>,
     pub removed_chain_block_hashes: Arc<Vec<Hash>>,
     pub added_chain_blocks_acceptance_data: Arc<Vec<Arc<AcceptanceData>>>,
+    pub added_accepting_blue_scores: Arc<Vec<u64>>,
 }
 impl VirtualChainChangedNotification {
     pub fn new(
         added_chain_block_hashes: Arc<Vec<Hash>>,
         removed_chain_block_hashes: Arc<Vec<Hash>>,
         added_chain_blocks_acceptance_data: Arc<Vec<Arc<AcceptanceData>>>,
+        added_accepting_blue_scores: Arc<Vec<u64>>,
     ) -> Self {
-        Self { added_chain_block_hashes, removed_chain_block_hashes, added_chain_blocks_acceptance_data }
+        Self { added_chain_block_hashes, removed_chain_block_hashes, added_chain_blocks_acceptance_data, added_accepting_blue_scores }
     }
 }
 
@@ -181,3 +192,16 @@ pub struct PruningPointUtxoSetOverrideNotification {}
 
 #[derive(Debug, Clone)]
 pub struct NewBlockTemplateNotification {}
+
+#[derive(Debug, Clone)]
+pub struct RetentionRootChangedNotification {
+    pub retention_root: Hash,
+    pub retention_root_blue_score: u64,
+    pub retention_root_daa_score: u64,
+}
+
+impl RetentionRootChangedNotification {
+    pub fn new(retention_root: Hash, retention_root_blue_score: u64, retention_root_daa_score: u64) -> Self {
+        Self { retention_root, retention_root_blue_score, retention_root_daa_score }
+    }
+}