Merge branch 'master' into pkhry/external_transient_storage

Author: pkhry

.github/workflows/release-70_combined-publish-release.yml

@@ -106,6 +106,8 @@ jobs:
       version: ${{ inputs.version }}
       stable_tag: ${{ needs.promote-rc-to-final.outputs.final_tag }}
     secrets: inherit
+    permissions:
+      contents: write
 
   publish-docker-polkadot-parachain:
     name: Publish Docker image - polkadot-parachain
@@ -121,6 +123,8 @@ jobs:
       version: ${{ inputs.version }}
       stable_tag: ${{ needs.promote-rc-to-final.outputs.final_tag }}
     secrets: inherit
+    permissions:
+      contents: write
 
   publish-docker-polkadot-omni-node:
     name: Publish Docker image - polkadot-omni-node
@@ -136,6 +140,8 @@ jobs:
       version: ${{ inputs.version }}
       stable_tag: ${{ needs.promote-rc-to-final.outputs.final_tag }}
     secrets: inherit
+    permissions:
+      contents: write
 
   publish-docker-chain-spec-builder:
     name: Publish Docker image - chain-spec-builder
@@ -151,3 +157,5 @@ jobs:
       version: ${{ inputs.version }}
       stable_tag: ${{ needs.promote-rc-to-final.outputs.final_tag }}
     secrets: inherit
+    permissions:
+      contents: write

prdoc/pr_10880.prdoc

@@ -0,0 +1,13 @@
+title: Remove failing assertion related to VoterList count mismatch
+doc:
+  - audience: Runtime Dev
+    description:
+      Updated bags-list so that on_insert queues items into PendingRebag instead of failing,
+      and removed the invariant that required VoterList's count to equal the combined number
+      of Nominators and Validators. This is safe while bags-list is locked. After unlocking,
+      on_idle drains PendingRebag, and the counts converge back to consistency over time.
+crates:
+  - name: pallet-staking-async
+    bump: patch
+  - name: pallet-bags-list
+    bump: patch

prdoc/pr_10882.prdoc

@@ -0,0 +1,13 @@
+title: "statement-store: make encode/hash faster"
+doc:
+- audience: Node Dev
+  description: |
+    Optimizes statement encoding and hashing by pre-allocating memory for the encoded buffer.
+    This reduces allocation overhead and improves performance, particularly when receiving
+    statements from multiple peers. Benchmarks show ~16% speedup when receiving statements
+    from 16 peers.
+crates:
+- name: sp-statement-store
+  bump: minor
+- name: sc-network-statement
+  bump: patch

substrate/client/network/statement/benches/statement_network.rs

@@ -282,6 +282,7 @@ fn blocking_executor(
 fn bench_on_statements(c: &mut Criterion) {
 	let statement_counts = [100, 500, 1000, 2000];
 	let thread_counts = [1, 2, 4, 8];
+	let peer_counts = [1, 2, 4, 8, 16];
 	let max_runtime_instances = 8;
 	let executor_types = [("blocking", true), ("non_blocking", false)];
 
@@ -292,39 +293,45 @@ fn bench_on_statements(c: &mut Criterion) {
 	for &num_statements in &statement_counts {
 		for &num_threads in &thread_counts {
 			for &(executor_name, is_blocking) in &executor_types {
-				let statements: Vec<Statement> =
-					(0..num_statements).map(|i| create_signed_statement(i, &keypair)).collect();
-				let executor = if is_blocking {
-					blocking_executor(&handle)
-				} else {
-					non_blocking_executor(&handle)
-				};
-
-				let benchmark_name = format!(
-					"on_statements/statements_{}/threads_{}/{}",
-					num_statements, num_threads, executor_name
-				);
-
-				c.bench_function(&benchmark_name, |b| {
-					b.iter_batched(
-						|| build_handler(executor.clone(), num_threads, max_runtime_instances),
-						|(mut handler, peer_id, _temp_dir)| {
-							handler.on_statements(peer_id, statements.clone());
-
-							runtime.block_on(async {
-								while handler.pending_statements_mut().next().await.is_some() {}
-							});
-
-							let pending = handler.pending_statements_mut();
-							assert!(
-								pending.is_empty(),
-								"Pending statements not empty: {}",
-								pending.len()
-							);
-						},
-						criterion::BatchSize::LargeInput,
-					)
-				});
+				for num_peers in &peer_counts {
+					let statements: Vec<Statement> =
+						(0..num_statements).map(|i| create_signed_statement(i, &keypair)).collect();
+					let executor = if is_blocking {
+						blocking_executor(&handle)
+					} else {
+						non_blocking_executor(&handle)
+					};
+
+					let benchmark_name = format!(
+						"on_statements/statements_{}/peers_{}/threads_{}/{}",
+						num_statements, num_peers, num_threads, executor_name
+					);
+
+					c.bench_function(&benchmark_name, |b| {
+						b.iter_batched(
+							|| build_handler(executor.clone(), num_threads, max_runtime_instances),
+							|(mut handler, peer_id, _temp_dir)| {
+								// The number of peers determines how many times we might receive a
+								// statement.
+								for _ in 0..*num_peers {
+									handler.on_statements(peer_id, statements.clone());
+								}
+
+								runtime.block_on(async {
+									while handler.pending_statements_mut().next().await.is_some() {}
+								});
+
+								let pending = handler.pending_statements_mut();
+								assert!(
+									pending.is_empty(),
+									"Pending statements not empty: {}",
+									pending.len()
+								);
+							},
+							criterion::BatchSize::LargeInput,
+						)
+					});
+				}
 			}
 		}
 	}

substrate/client/network/statement/src/lib.rs

@@ -29,6 +29,8 @@
 use crate::config::*;
 
 use codec::{Compact, Decode, Encode, MaxEncodedLen};
+#[cfg(any(test, feature = "test-helpers"))]
+use futures::future::pending;
 use futures::{channel::oneshot, future::FusedFuture, prelude::*, stream::FuturesUnordered};
 use prometheus_endpoint::{
 	prometheus, register, Counter, Gauge, Histogram, HistogramOpts, PrometheusError, Registry, U64,
@@ -59,7 +61,6 @@ use std::{
 	sync::Arc,
 };
 use tokio::time::timeout;
-
 pub mod config;
 
 /// A set of statements.
@@ -432,7 +433,7 @@ where
 			statement_store,
 			queue_sender,
 			metrics: None,
-			initial_sync_timeout: Box::pin(tokio::time::sleep(INITIAL_SYNC_BURST_INTERVAL).fuse()),
+			initial_sync_timeout: Box::pin(pending().fuse()),
 			pending_initial_syncs: HashMap::new(),
 			initial_sync_peer_queue: VecDeque::new(),
 		}

substrate/frame/bags-list/src/lib.rs

@@ -708,13 +708,16 @@ impl<T: Config<I>, I: 'static> SortedListProvider<T::AccountId> for Pallet<T, I>
 	}
 
 	fn on_insert(id: T::AccountId, score: T::Score) -> Result<(), ListError> {
-		Pallet::<T, I>::ensure_unlocked().inspect_err(|_| {
+		if Pallet::<T, I>::ensure_unlocked().is_err() {
 			// Pallet is locked - store in PendingRebag for later processing
 			// Only queue if auto-rebagging is enabled
 			if T::MaxAutoRebagPerBlock::get() > 0u32 {
 				PendingRebag::<T, I>::insert(&id, ());
+				return Ok(());
 			}
-		})?;
+
+			return Err(ListError::Locked);
+		};
 		List::<T, I>::insert(id, score)
 	}
 

substrate/frame/bags-list/src/tests.rs

@@ -1041,12 +1041,12 @@ mod on_idle {
 			BagsList::lock();
 
 			// Try to insert 6 new nodes while locked - 5 regular + 1 that will lose staking status
-			assert_eq!(BagsList::on_insert(5, 15), Err(ListError::Locked));
-			assert_eq!(BagsList::on_insert(6, 45), Err(ListError::Locked));
-			assert_eq!(BagsList::on_insert(7, 55), Err(ListError::Locked));
-			assert_eq!(BagsList::on_insert(8, 1500), Err(ListError::Locked));
-			assert_eq!(BagsList::on_insert(11, 100), Err(ListError::Locked));
-			assert_eq!(BagsList::on_insert(99, 500), Err(ListError::Locked)); // Will lose staking
+			assert_ok!(BagsList::on_insert(5, 15));
+			assert_ok!(BagsList::on_insert(6, 45));
+			assert_ok!(BagsList::on_insert(7, 55));
+			assert_ok!(BagsList::on_insert(8, 1500));
+			assert_ok!(BagsList::on_insert(11, 100));
+			assert_ok!(BagsList::on_insert(99, 500)); // Will lose staking
 
 			// Verify they're in PendingRebag
 			let pending: Vec<_> = PendingRebag::<Runtime>::iter_keys().collect();
@@ -1184,7 +1184,7 @@ mod on_idle {
 
 			// Try to insert while locked - should go to PendingRebag
 			StakingMock::set_score_of(&1, 1000);
-			assert_eq!(BagsList::on_insert(1, 1000), Err(ListError::Locked));
+			assert_ok!(BagsList::on_insert(1, 1000));
 			assert!(PendingRebag::<Runtime>::contains_key(&1));
 			assert!(!List::<Runtime>::contains(&1));
 

substrate/frame/staking-async/src/pallet/impls.rs

@@ -1897,16 +1897,20 @@ impl<T: Config> Pallet<T> {
 	}
 
 	/// Invariants:
-	/// * Number of voters in `VoterList` match that of the number of Nominators and Validators in
-	/// the system (validator is both voter and target).
 	/// * Number of targets in `TargetList` matches the number of validators in the system.
 	/// * Current validator count is bounded by the election provider's max winners.
 	fn check_count() -> Result<(), TryRuntimeError> {
-		ensure!(
-			<T as Config>::VoterList::count() ==
-				Nominators::<T>::count() + Validators::<T>::count(),
-			"wrong external count"
+		// When the bags list is locked, nominators and validators may be temporarily
+		// missing from the voter set. If `PendingRebag` is enabled, it will later
+		// reconcile the mismatch.
+		crate::log!(
+			debug,
+			"VoterList count: {}, Nominators count: {}, Validators count: {}",
+			<T as Config>::VoterList::count(),
+			Nominators::<T>::count(),
+			Validators::<T>::count()
 		);
+
 		ensure!(
 			<T as Config>::TargetList::count() == Validators::<T>::count(),
 			"wrong external count"

substrate/frame/staking-async/src/tests/mod.rs

@@ -284,25 +284,6 @@ fn basic_setup_sessions_per_era() {
 
 mod try_state_assertions {
 	use super::*;
-	#[test]
-	#[should_panic]
-	fn count_check_works() {
-		ExtBuilder::default().build_and_execute(|| {
-			// We should never insert into the validators or nominators map directly as this will
-			// not keep track of the count. This test should panic as we verify the count is
-			// accurate after every test using the `post_checks` in `mock`.
-			Validators::<Test>::insert(987654321, ValidatorPrefs::default());
-			Nominators::<Test>::insert(
-				987654321,
-				Nominations {
-					targets: Default::default(),
-					submitted_in: Default::default(),
-					suppressed: false,
-				},
-			);
-		})
-	}
-
 	#[test]
 	#[should_panic = "called `Result::unwrap()` on an `Err` value: Other(\"number of entries in payee storage items does not match the number of bonded ledgers\")"]
 	fn check_payee_invariant1_works() {