Test webhook on PR

doc/policy/mempool-replacements.md

@@ -12,12 +12,7 @@ other consensus and policy rules, each of the following conditions are met:
 
 1. (Removed)
 
-2. The replacement transaction only include an unconfirmed input if that input was included in
-   one of the directly conflicting transactions. An unconfirmed input spends an output from a
-   currently-unconfirmed transaction.
-
-   *Rationale*: When RBF was originally implemented, the mempool did not keep track of
-   ancestor feerates yet. This rule was suggested as a temporary restriction.
+2. [REDACTED]
 
 3. The replacement transaction pays an absolute fee of at least the sum paid by the original
    transactions.
@@ -38,23 +33,16 @@ other consensus and policy rules, each of the following conditions are met:
    *Rationale*: Try to prevent DoS attacks where an attacker causes the network to repeatedly relay
    transactions each paying a tiny additional amount in fees, e.g. just 1 satoshi.
 
-5. The number of original transactions does not exceed 100. More precisely, the sum of all
-   directly conflicting transactions' descendant counts (number of transactions inclusive of itself
-   and its descendants) must not exceed 100; it is possible that this overestimates the true number
-   of original transactions.
+5. The number of distinct clusters corresponding to conflicting transactions does not exceed 100.
 
-   *Rationale*: Try to prevent DoS attacks where an attacker is able to easily occupy and flush out
-   significant portions of the node's mempool using replacements with multiple directly conflicting
-   transactions, each with large descendant sets.
+   *Rationale*: Limit CPU usage required to update the mempool for so many transactions being
+   removed at once.
 
-6. The replacement transaction's feerate is greater than the feerates of all directly conflicting
-   transactions.
+6. The feerate diagram of the mempool must be strictly improved by the replacement transaction.
 
-   *Rationale*: This rule was originally intended to ensure that the replacement transaction is
-   preferable for block-inclusion, compared to what would be removed from the mempool. This rule
-   predates ancestor feerate-based transaction selection.
+   *Rationale*: This ensures that block fees in all future blocks will go up
+   after the replacement (ignoring tail effects at the end of a block).
 
-This set of rules is similar but distinct from BIP125.
 
 ## History
 
@@ -79,3 +67,5 @@ This set of rules is similar but distinct from BIP125.
 * Signaling for replace-by-fee is no longer required as of [PR 30592](https://github.com/bitcoin/bitcoin/pull/30592).
 
 * The incremental relay feerate default is 0.1sat/vB ([PR #33106](https://github.com/bitcoin/bitcoin/pull/33106)).
+
+* Feerate diagram policy enabled in conjunction with switch to cluster mempool as of **v??.0**.

src/bench/blockencodings.cpp

@@ -22,7 +22,7 @@
 static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs)
 {
     LockPoints lp;
-    AddToMempool(pool, CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp));
+    AddToMempool(pool, CTxMemPoolEntry(TxGraph::Ref(), tx, fee, /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp));
 }
 
 namespace {

src/bench/mempool_ephemeral_spends.cpp

@@ -29,7 +29,7 @@ static void AddTx(const CTransactionRef& tx, CTxMemPool& pool) EXCLUSIVE_LOCKS_R
     unsigned int sigOpCost{4};
     uint64_t fee{0};
     LockPoints lp;
-    AddToMempool(pool, CTxMemPoolEntry(
+    AddToMempool(pool, CTxMemPoolEntry(TxGraph::Ref(),
         tx, fee, nTime, nHeight, sequence,
         spendsCoinbase, sigOpCost, lp));
 }

src/bench/mempool_eviction.cpp

@@ -27,7 +27,7 @@ static void AddTx(const CTransactionRef& tx, const CAmount& nFee, CTxMemPool& po
     bool spendsCoinbase = false;
     unsigned int sigOpCost = 4;
     LockPoints lp;
-    AddToMempool(pool, CTxMemPoolEntry(
+    AddToMempool(pool, CTxMemPoolEntry(TxGraph::Ref(),
         tx, nFee, nTime, nHeight, sequence,
         spendsCoinbase, sigOpCost, lp));
 }

src/bench/mempool_stress.cpp

@@ -21,15 +21,15 @@
 
 class CCoinsViewCache;
 
-static void AddTx(const CTransactionRef& tx, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs)
+static void AddTx(const CTransactionRef& tx, CTxMemPool& pool, FastRandomContext& det_rand) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs)
 {
     int64_t nTime = 0;
     unsigned int nHeight = 1;
     uint64_t sequence = 0;
     bool spendsCoinbase = false;
     unsigned int sigOpCost = 4;
     LockPoints lp;
-    AddToMempool(pool, CTxMemPoolEntry(tx, 1000, nTime, nHeight, sequence, spendsCoinbase, sigOpCost, lp));
+    AddToMempool(pool, CTxMemPoolEntry(TxGraph::Ref(), tx, det_rand.randrange(10000)+1000, nTime, nHeight, sequence, spendsCoinbase, sigOpCost, lp));
 }
 
 struct Available {
@@ -39,15 +39,17 @@ struct Available {
     Available(CTransactionRef& ref, size_t tx_count) : ref(ref), tx_count(tx_count){}
 };
 
-static std::vector<CTransactionRef> CreateOrderedCoins(FastRandomContext& det_rand, int childTxs, int min_ancestors)
+// Create a cluster of transactions, randomly.
+static std::vector<CTransactionRef> CreateCoinCluster(FastRandomContext& det_rand, int childTxs, int min_ancestors)
 {
     std::vector<Available> available_coins;
     std::vector<CTransactionRef> ordered_coins;
     // Create some base transactions
     size_t tx_counter = 1;
-    for (auto x = 0; x < 100; ++x) {
+    for (auto x = 0; x < 10; ++x) {
         CMutableTransaction tx = CMutableTransaction();
         tx.vin.resize(1);
+        tx.vin[0].prevout = COutPoint(Txid::FromUint256(GetRandHash()), 1);
         tx.vin[0].scriptSig = CScript() << CScriptNum(tx_counter);
         tx.vin[0].scriptWitness.stack.push_back(CScriptNum(x).getvch());
         tx.vout.resize(det_rand.randrange(10)+2);
@@ -91,26 +93,106 @@ static std::vector<CTransactionRef> CreateOrderedCoins(FastRandomContext& det_ra
     return ordered_coins;
 }
 
+static void MemPoolAddTransactions(benchmark::Bench& bench)
+{
+    FastRandomContext det_rand{true};
+    int childTxs = 50;
+    if (bench.complexityN() > 1) {
+        childTxs = static_cast<int>(bench.complexityN());
+    }
+    const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN);
+    CTxMemPool& pool = *testing_setup.get()->m_node.mempool;
+
+    std::vector<CTransactionRef> transactions;
+    // Create 1000 clusters of 100 transactions each
+    for (int i=0; i<100; i++) {
+        auto new_txs = CreateCoinCluster(det_rand, childTxs, /*min_ancestors*/ 1);
+        transactions.insert(transactions.end(), new_txs.begin(), new_txs.end());
+    }
+
+    LOCK2(cs_main, pool.cs);
+
+    bench.run([&]() NO_THREAD_SAFETY_ANALYSIS {
+        for (auto& tx : transactions) {
+            AddTx(tx, pool, det_rand);
+        }
+        pool.TrimToSize(0, nullptr);
+    });
+}
+
 static void ComplexMemPool(benchmark::Bench& bench)
 {
     FastRandomContext det_rand{true};
-    int childTxs = 800;
+    int childTxs = 50;
     if (bench.complexityN() > 1) {
         childTxs = static_cast<int>(bench.complexityN());
     }
-    std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /*min_ancestors=*/1);
     const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN);
     CTxMemPool& pool = *testing_setup.get()->m_node.mempool;
+
+    std::vector<CTransactionRef> tx_remove_for_block;
+    std::vector<Txid> hashes_remove_for_block;
+
     LOCK2(cs_main, pool.cs);
+
+    for (int i=0; i<1000; i++) {
+        std::vector<CTransactionRef> transactions = CreateCoinCluster(det_rand, childTxs, /*min_ancestors=*/1);
+
+        // Add all transactions to the mempool.
+        // Also store the first 10 transactions from each cluster as the
+        // transactions we'll "mine" in the the benchmark.
+        int tx_count = 0;
+        for (auto& tx : transactions) {
+            if (tx_count < 10) {
+                tx_remove_for_block.push_back(tx);
+                ++tx_count;
+                hashes_remove_for_block.emplace_back(tx->GetHash());
+            }
+            AddTx(tx, pool, det_rand);
+        }
+    }
+
+    // Since the benchmark will be run repeatedly, we have to leave the mempool
+    // in the same state at the end of the function, so we benchmark both
+    // mining a block and reorging the block's contents back into the mempool.
     bench.run([&]() NO_THREAD_SAFETY_ANALYSIS {
-        for (auto& tx : ordered_coins) {
-            AddTx(tx, pool);
+        pool.removeForBlock(tx_remove_for_block, /*nBlockHeight*/100);
+        for (auto& tx: tx_remove_for_block) {
+            AddTx(tx, pool, det_rand);
         }
-        pool.TrimToSize(pool.DynamicMemoryUsage() * 3 / 4);
-        pool.TrimToSize(GetVirtualTransactionSize(*ordered_coins.front()));
+        pool.UpdateTransactionsFromBlock(hashes_remove_for_block);
+    });
+}
+
+static void MemPoolAncestorsDescendants(benchmark::Bench& bench)
+{
+    FastRandomContext det_rand{true};
+    int childTxs = 50;
+    if (bench.complexityN() > 1) {
+        childTxs = static_cast<int>(bench.complexityN());
+    }
+    const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN);
+    CTxMemPool& pool = *testing_setup.get()->m_node.mempool;
+
+    LOCK2(cs_main, pool.cs);
+
+    std::vector<CTransactionRef> transactions = CreateCoinCluster(det_rand, childTxs, /*min_ancestors=*/1);
+    for (auto& tx : transactions) {
+        AddTx(tx, pool, det_rand);
+    }
+
+    CTxMemPool::txiter first_tx = *pool.GetIter(transactions[0]->GetHash());
+    CTxMemPool::txiter last_tx = *pool.GetIter(transactions.back()->GetHash());
+
+    bench.run([&]() NO_THREAD_SAFETY_ANALYSIS {
+        CTxMemPool::setEntries dummy;
+        ankerl::nanobench::doNotOptimizeAway(dummy);
+        pool.CalculateDescendants({first_tx}, dummy);
+        ankerl::nanobench::doNotOptimizeAway(pool.CalculateMemPoolAncestors(*last_tx));
     });
 }
 
+
 static void MempoolCheck(benchmark::Bench& bench)
 {
     FastRandomContext det_rand{true};
@@ -126,5 +208,65 @@ static void MempoolCheck(benchmark::Bench& bench)
     });
 }
 
+#if 0
+static void MemPoolMiningScoreCheck(benchmark::Bench& bench)
+{
+    // Default test: each cluster is of size 20, and we'll try to RBF with a
+    // transaction that merges 10 clusters, evicting 10 transactions from each.
+
+    FastRandomContext det_rand{true};
+    int childTxs = 10;
+    if (bench.complexityN() > 1) {
+        childTxs = static_cast<int>(bench.complexityN());
+    }
+    const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN);
+    CTxMemPool& pool = *testing_setup.get()->m_node.mempool;
+
+    LOCK2(cs_main, pool.cs);
+
+    std::vector<CTransactionRef> parent_txs_for_rbf;
+    std::set<uint256> child_txs_to_conflict_with;
+
+    for (int i=0; i<10; i++) {
+        std::vector<CTransactionRef> transactions = CreateCoinCluster(det_rand, childTxs, /*min_ancestors=*/1);
+        parent_txs_for_rbf.push_back(transactions[0]);
+        // Conflict with everything after the first 10 transactions
+        for (size_t j=10; j<transactions.size(); ++j) {
+            child_txs_to_conflict_with.insert(transactions[j]->GetHash());
+        }
+
+        // Add all transactions to the mempool.
+        for (auto& tx : transactions) {
+            AddTx(tx, pool, det_rand);
+        }
+    }
+
+    // Construct a transaction that spends from each of the parent transactions
+    // selected.
+    CMutableTransaction tx = CMutableTransaction();
+    tx.vin.resize(10);
+    for (size_t i=0; i<parent_txs_for_rbf.size(); ++i) {
+        tx.vin[i].prevout = COutPoint(parent_txs_for_rbf[i]->GetHash(), 0);
+        tx.vin[i].scriptSig = CScript() << i;
+    }
+    tx.vout.resize(1);
+    for (auto& out : tx.vout) {
+        out.scriptPubKey = CScript() << CScriptNum(det_rand.randrange(19)+1) << OP_EQUAL;
+        out.nValue = 10 * COIN;
+    }
+
+    CTxMemPool::setEntries all_conflicts = pool.GetIterSet(child_txs_to_conflict_with);
+    CTxMemPoolEntry entry(MakeTransactionRef(tx), det_rand.randrange(10000)+1000, 0, 1, 0, false, 4, LockPoints());
+
+    bench.run([&]() NO_THREAD_SAFETY_ANALYSIS {
+        CTxMemPool::Limits limits(pool.m_limits);
+        pool.CalculateMiningScoreOfReplacementTx(entry, det_rand.randrange(30000)+1000, all_conflicts, limits);
+    });
+}
+#endif
+
+BENCHMARK(MemPoolAncestorsDescendants, benchmark::PriorityLevel::HIGH);
+BENCHMARK(MemPoolAddTransactions, benchmark::PriorityLevel::HIGH);
 BENCHMARK(ComplexMemPool, benchmark::PriorityLevel::HIGH);
 BENCHMARK(MempoolCheck, benchmark::PriorityLevel::HIGH);
+//BENCHMARK(MemPoolMiningScoreCheck, benchmark::PriorityLevel::HIGH);

src/bench/rpc_mempool.cpp

@@ -22,7 +22,7 @@
 static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs)
 {
     LockPoints lp;
-    AddToMempool(pool, CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp));
+    AddToMempool(pool, CTxMemPoolEntry(TxGraph::Ref(), tx, fee, /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp));
 }
 
 static void RpcMempool(benchmark::Bench& bench)

src/init.cpp

@@ -641,6 +641,8 @@ void SetupServerArgs(ArgsManager& argsman, bool can_listen_ipc)
     argsman.AddArg("-limitdescendantcount=<n>", strprintf("Do not accept transactions if any ancestor would have <n> or more in-mempool descendants (default: %u)", DEFAULT_DESCENDANT_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT_KVB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-test=<option>", "Pass a test-only option. Options include : " + Join(TEST_OPTIONS_DOC, ", ") + ".", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
+    argsman.AddArg("-limitclustercount=<n>", strprintf("Do not accept transactions connected to <n> or more existing in-mempool transactions (default: %u, maximum: %u)", DEFAULT_CLUSTER_LIMIT, MAX_CLUSTER_COUNT_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
+    argsman.AddArg("-limitclustersize=<n>", strprintf("Do not accept transactions whose size with all in-mempool connected transactions exceeds <n> kilobytes (default: %u)", DEFAULT_CLUSTER_SIZE_LIMIT_KVB), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);
     argsman.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_VALIDATION_CACHE_BYTES >> 20), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST);

src/kernel/CMakeLists.txt

@@ -58,6 +58,7 @@ add_library(bitcoinkernel
   ../support/lockedpool.cpp
   ../sync.cpp
   ../txdb.cpp
+  ../txgraph.cpp
   ../txmempool.cpp
   ../uint256.cpp
   ../util/chaintype.cpp