feat: add TxoCommitments softfork

Author: quake

Cargo.lock

@@ -787,6 +787,7 @@ impl ChainService {
                                     switch,
                                     Arc::clone(&txs_verify_cache),
                                     &mmr,
+                                    &txn,
                                 );
                                 contextual_block_verifier.verify(&resolved, b)
                             };

chain/src/chain.rs

@@ -3,7 +3,7 @@
 /// Column families alias type
 pub type Col = &'static str;
 /// Total column number
-pub const COLUMNS: u32 = 19;
+pub const COLUMNS: u32 = 20;
 /// Column store chain index
 pub const COLUMN_INDEX: Col = "0";
 /// Column store block's header
@@ -44,6 +44,8 @@ pub const COLUMN_CHAIN_ROOT_MMR: Col = "16";
 pub const COLUMN_BLOCK_FILTER: Col = "17";
 /// Column store filter data hash for client-side filtering
 pub const COLUMN_BLOCK_FILTER_HASH: Col = "18";
+/// Column store txo root MMR data
+pub const COLUMN_TXO_ROOT_MMR: Col = "19";
 
 /// META_TIP_HEADER_KEY tracks the latest known best block header
 pub const META_TIP_HEADER_KEY: &[u8] = b"TIP_HEADER";
@@ -52,6 +54,13 @@ pub const META_CURRENT_EPOCH_KEY: &[u8] = b"CURRENT_EPOCH";
 /// META_FILTER_DATA_KEY tracks the latest built filter data block hash
 pub const META_LATEST_BUILT_FILTER_DATA_KEY: &[u8] = b"LATEST_BUILT_FILTER_DATA";
 
+/// A key prefix records the txo root mmr size by block number
+pub const TXO_ROOT_MMR_SIZE_KEY_PREFIX: &[u8] = &[0];
+/// A key prefix records the status of out_point in txo root mmr
+pub const TXO_ROOT_MMR_STATUS_KEY_PREFIX: &[u8] = &[1];
+/// A key prefix records the txo root mmr element by position
+pub const TXO_ROOT_MMR_ELEMENT_KEY_PREFIX: &[u8] = &[2];
+
 /// CHAIN_SPEC_HASH_KEY tracks the hash of chain spec which created current database
 pub const CHAIN_SPEC_HASH_KEY: &[u8] = b"chain-spec-hash";
 /// MIGRATION_VERSION_KEY tracks the current database version.

db-schema/src/lib.rs

@@ -540,12 +540,31 @@ impl IntegrationTestRpc for IntegrationTestRpcImpl {
         Ok(tx_hash.unpack())
     }
 
-    fn generate_block_with_template(&self, block_template: BlockTemplate) -> Result<H256> {
+    fn generate_block_with_template(&self, mut block_template: BlockTemplate) -> Result<H256> {
         let dao_field = self.calculate_dao_field(block_template.clone())?;
 
-        let mut update_dao_template = block_template;
-        update_dao_template.dao = dao_field;
-        let block = update_dao_template.into();
+        let cellbase = packed::Transaction::from(block_template.cellbase.data.clone()).into_view();
+        let txs = block_template
+            .transactions
+            .iter()
+            .map(|t| packed::Transaction::from(t.data.clone()).into_view())
+            .collect::<Vec<_>>();
+        let extension = self
+            .shared
+            .snapshot()
+            .build_extension(&cellbase, txs.iter())
+            .map_err(|err| {
+                error!(
+                    "build extension error when generating block \
+                 with block template, error: {:?}",
+                    err
+                );
+                RPCError::invalid_params(err.to_string())
+            })?;
+
+        block_template.dao = dao_field;
+        block_template.extension = extension.map(|e| e.into());
+        let block = block_template.into();
         self.process_and_announce_block(block)
     }
 

rpc/src/module/test.rs

@@ -532,7 +532,17 @@ impl ChainSpec {
                     active_mode: ActiveMode::Always,
                     threshold: TESTNET_ACTIVATION_THRESHOLD,
                 };
+                let txo_commitments = Deployment {
+                    bit: 2,
+                    start: 0,
+                    timeout: 0,
+                    min_activation_epoch: 0,
+                    period: 10,
+                    active_mode: ActiveMode::Always,
+                    threshold: TESTNET_ACTIVATION_THRESHOLD,
+                };
                 deployments.insert(DeploymentPos::LightClient, light_client);
+                deployments.insert(DeploymentPos::TxoCommitments, txo_commitments);
                 Some(deployments)
             }
         }

spec/src/lib.rs

@@ -48,6 +48,7 @@ impl From<DeploymentPos> for JsonDeploymentPos {
         match pos {
             DeploymentPos::Testdummy => JsonDeploymentPos::Testdummy,
             DeploymentPos::LightClient => JsonDeploymentPos::LightClient,
+            DeploymentPos::TxoCommitments => JsonDeploymentPos::TxoCommitments,
         }
     }
 }

spec/src/versionbits/convert.rs

@@ -59,8 +59,10 @@ pub enum ActiveMode {
 pub enum DeploymentPos {
     /// Dummy
     Testdummy,
-    /// light client protocol
+    /// Light client protocol
     LightClient,
+    /// Tranaction output commitments
+    TxoCommitments,
 }
 
 /// VersionbitsIndexer

spec/src/versionbits/mod.rs

@@ -20,6 +20,7 @@ ckb-app-config = { path = "../util/app-config", version = "= 0.112.0-pre" }
 ckb-db-schema = { path = "../db-schema", version = "= 0.112.0-pre" }
 ckb-freezer = { path = "../freezer", version = "= 0.112.0-pre" }
 ckb-merkle-mountain-range =  { git = "https://github.com/quake/merkle-mountain-range", branch = "quake/updatable" }
+ckb-logger = { path = "../util/logger", version = "= 0.112.0-pre" }
 
 [dev-dependencies]
 tempfile.workspace = true

store/Cargo.toml

@@ -1,6 +1,11 @@
 use crate::{ChainStore, StoreTransaction};
-use ckb_error::Error;
-use ckb_types::{core::BlockView, packed, prelude::*};
+use ckb_error::{Error, InternalErrorKind};
+use ckb_types::{
+    core::{BlockNumber, BlockView},
+    packed,
+    prelude::*,
+    utilities::merkle_mountain_range::{hash_out_point_and_status, TxoStatus},
+};
 use std::collections::HashMap;
 
 /**
@@ -26,6 +31,39 @@ use std::collections::HashMap;
 pub fn attach_block_cell(txn: &StoreTransaction, block: &BlockView) -> Result<(), Error> {
     let transactions = block.transactions();
 
+    // update txo root mmr
+    let block_number = block.header().number();
+    let mut txo_root_mmr = txn.txo_root_mmr(block_number);
+    for tx in transactions.iter() {
+        for input in tx.inputs().into_iter() {
+            let out_point = input.previous_output();
+            // cellbase and genesis block's tx may not have previous output
+            if let Some(mut txo_status) = txn.get_txo_mmr_status(&out_point) {
+                txo_root_mmr
+                    .update(
+                        txo_status.mmr_position,
+                        hash_out_point_and_status(&out_point, txo_status.created_by, block_number),
+                    )
+                    .map_err(|e| InternalErrorKind::MMR.other(e))?;
+                txo_status.mark_as_consumed(block_number);
+                txn.insert_txo_root_mmr_status(&out_point, &txo_status)?;
+            }
+        }
+
+        for out_point in tx.output_pts().into_iter() {
+            let hash = hash_out_point_and_status(&out_point, block_number, BlockNumber::MAX);
+            let mmr_position = txo_root_mmr
+                .push(hash)
+                .map_err(|e| InternalErrorKind::MMR.other(e))?;
+            let txo_status = TxoStatus::new(mmr_position, block_number);
+            txn.insert_txo_root_mmr_status(&out_point, &txo_status)?;
+        }
+    }
+    txo_root_mmr
+        .commit()
+        .map_err(|e| InternalErrorKind::MMR.other(e))?;
+    txn.insert_txo_root_mmr_size(block_number, txo_root_mmr.mmr_size())?;
+
     // add new live cells
     let new_cells = transactions
         .iter()
@@ -84,6 +122,46 @@ pub fn attach_block_cell(txn: &StoreTransaction, block: &BlockView) -> Result<()
 /// Undoes the effects of this block on the live cell set.
 pub fn detach_block_cell(txn: &StoreTransaction, block: &BlockView) -> Result<(), Error> {
     let transactions = block.transactions();
+
+    // undo txo root mmr updates
+    let block_number = block.header().number();
+    let mut txo_root_mmr = txn.txo_root_mmr(block_number);
+
+    for tx in transactions.iter() {
+        for input in tx.inputs().into_iter() {
+            let out_point = input.previous_output();
+            if let Some(mut txo_status) = txn.get_txo_mmr_status(&out_point) {
+                txo_root_mmr
+                    .update(
+                        txo_status.mmr_position,
+                        hash_out_point_and_status(
+                            &out_point,
+                            txo_status.created_by,
+                            BlockNumber::MAX,
+                        ),
+                    )
+                    .map_err(|e| InternalErrorKind::MMR.other(e))?;
+                txo_status.mark_as_live();
+                txn.insert_txo_root_mmr_status(&out_point, &txo_status)?;
+            }
+        }
+
+        for out_point in tx.output_pts().into_iter() {
+            txn.delete_txo_root_mmr_status(&out_point)?;
+        }
+    }
+    txo_root_mmr
+        .commit()
+        .map_err(|e| InternalErrorKind::MMR.other(e))?;
+
+    let current_mmr_size = txn.get_txo_mmr_size(block_number);
+    let pre_mmr_size = txn.get_txo_mmr_size(block_number - 1);
+    for pos in pre_mmr_size..current_mmr_size {
+        txn.delete_txo_root_mmr_element(pos, block_number)?;
+    }
+    txn.delete_txo_root_mmr_size(block_number)?;
+
+    // restore inputs
     let mut input_pts = HashMap::with_capacity(transactions.len());
 
     for tx in transactions.iter().skip(1) {
@@ -95,7 +173,6 @@ pub fn detach_block_cell(txn: &StoreTransaction, block: &BlockView) -> Result<()
         }
     }
 
-    // restore inputs
     // skip cellbase
     let undo_deads = input_pts
         .iter()

store/src/cell.rs

@@ -9,7 +9,9 @@ use ckb_db_schema::{
     COLUMN_BLOCK_FILTER, COLUMN_BLOCK_FILTER_HASH, COLUMN_BLOCK_HEADER, COLUMN_BLOCK_PROPOSAL_IDS,
     COLUMN_BLOCK_UNCLE, COLUMN_CELL, COLUMN_CELL_DATA, COLUMN_CELL_DATA_HASH,
     COLUMN_CHAIN_ROOT_MMR, COLUMN_EPOCH, COLUMN_INDEX, COLUMN_META, COLUMN_TRANSACTION_INFO,
-    COLUMN_UNCLES, META_CURRENT_EPOCH_KEY, META_LATEST_BUILT_FILTER_DATA_KEY, META_TIP_HEADER_KEY,
+    COLUMN_TXO_ROOT_MMR, COLUMN_UNCLES, META_CURRENT_EPOCH_KEY, META_LATEST_BUILT_FILTER_DATA_KEY,
+    META_TIP_HEADER_KEY, TXO_ROOT_MMR_ELEMENT_KEY_PREFIX, TXO_ROOT_MMR_SIZE_KEY_PREFIX,
+    TXO_ROOT_MMR_STATUS_KEY_PREFIX,
 };
 use ckb_freezer::Freezer;
 use ckb_types::{
@@ -20,6 +22,8 @@ use ckb_types::{
     },
     packed::{self, OutPoint},
     prelude::*,
+    utilities::merkle_mountain_range::TxoStatus,
+    H256,
 };
 
 /// The `ChainStore` trait provides chain data store interface
@@ -576,6 +580,44 @@ pub trait ChainStore: Send + Sync + Sized {
             })
     }
 
+    /// Gets txo mmr size by block number
+    fn get_txo_mmr_size(&self, block_number: BlockNumber) -> u64 {
+        let start_key = [TXO_ROOT_MMR_SIZE_KEY_PREFIX, &block_number.to_be_bytes()].concat();
+        self.get_iter(
+            COLUMN_TXO_ROOT_MMR,
+            IteratorMode::From(&start_key, Direction::Reverse),
+        )
+        .take_while(|(key, _)| key.starts_with(TXO_ROOT_MMR_SIZE_KEY_PREFIX))
+        .next()
+        .map(|(_key, value)| u64::from_le_bytes(value.as_ref().try_into().expect("stored u64")))
+        .unwrap_or_default()
+    }
+
+    /// Gets txo mmr status by out point
+    fn get_txo_mmr_status(&self, out_point: &OutPoint) -> Option<TxoStatus> {
+        let key = [TXO_ROOT_MMR_STATUS_KEY_PREFIX, out_point.as_slice()].concat();
+        self.get(COLUMN_TXO_ROOT_MMR, &key)
+            .map(|slice| TxoStatus::new_unchecked(slice.as_ref()))
+    }
+
+    /// Gets txo mmr element by position and block number
+    fn get_txo_mmr_element(&self, position: u64, block_number: BlockNumber) -> Option<H256> {
+        // block_number is stored as big endian for prefix search in lexicographical order
+        let start_key = [
+            TXO_ROOT_MMR_ELEMENT_KEY_PREFIX,
+            &position.to_le_bytes(),
+            &block_number.to_be_bytes(),
+        ]
+        .concat();
+        self.get_iter(
+            COLUMN_TXO_ROOT_MMR,
+            IteratorMode::From(&start_key, Direction::Reverse),
+        )
+        .take_while(|(key, _)| key.starts_with(&start_key[0..9]))
+        .next()
+        .map(|(_key, value)| H256::from_slice(value.as_ref()).expect("stored H256"))
+    }
+
     /// Gets ancestor block header by a base block hash and number
     fn get_ancestor(&self, base: &packed::Byte32, number: BlockNumber) -> Option<HeaderView> {
         let header = self.get_block_header(base)?;