Skip to content

Clean small values from Alpha map. #1635

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: devnet-ready
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion pallets/subtensor/src/coinbase/block_step.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,10 @@ impl<T: Config + pallet_drand::Config> Pallet<T> {
Self::run_coinbase(block_emission);
// --- 4. Set pending children on the epoch; but only after the coinbase has been run.
Self::try_set_pending_children(block_number);
// Return ok.

// Clear small batch of Alpha map entries if they are insignificant
Self::clear_small_nominations_batch();

Ok(())
}

Expand Down
15 changes: 15 additions & 0 deletions pallets/subtensor/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,11 @@ extern crate alloc;

pub const MAX_CRV3_COMMIT_SIZE_BYTES: u32 = 5000;

pub const ALPHA_MAP_CLEAN_BATCH_SIZE: usize = 10;

/// Min alpha level before removing from Alpha map
pub const MIN_ALPHA: u64 = 10u64;

#[deny(missing_docs)]
#[import_section(errors::errors)]
#[import_section(events::events)]
Expand Down Expand Up @@ -432,6 +437,11 @@ pub mod pallet {
T::InitialIssuance::get()
}
#[pallet::type_value]
/// Default last Alpha map key for cleaning
pub fn DefaultAlphaCleanLastKey<T: Config>() -> Option<Vec<u8>> {
None
}
#[pallet::type_value]
/// Default account, derived from zero trailing bytes.
pub fn DefaultAccount<T: Config>() -> T::AccountId {
T::AccountId::decode(&mut TrailingZeroInput::zeroes())
Expand Down Expand Up @@ -1176,6 +1186,10 @@ pub mod pallet {
pub type TokenSymbol<T: Config> =
StorageMap<_, Identity, u16, Vec<u8>, ValueQuery, DefaultUnicodeVecU8<T>>;

#[pallet::storage] // Contains last Alpha storage map key to clean (check first)
pub type AlphaMapCleanLastKey<T: Config> =
StorageValue<_, Option<Vec<u8>>, ValueQuery, DefaultAlphaCleanLastKey<T>>;

/// ============================
/// ==== Global Parameters =====
/// ============================
Expand Down Expand Up @@ -2616,6 +2630,7 @@ use sp_std::vec;
// used not 25 lines below
#[allow(unused)]
use sp_std::vec::Vec;
use substrate_fixed::types::U64F64;
use subtensor_macros::freeze_struct;

/// Trait for managing a membership pallet instance in the runtime
Expand Down
48 changes: 48 additions & 0 deletions pallets/subtensor/src/staking/helpers.rs
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,54 @@ impl<T: Config> Pallet<T> {
}
}

/// The function clears Alpha map in batches. Each run will check ALPHA_MAP_CLEAN_BATCH_SIZE
/// alphas. It keeps the alpha value stored when it's >= than MIN_ALPHA.
/// The function uses AlphaMapCleanLastKey as a storage for key iterator between runs.
pub fn clear_small_nominations_batch() {
// Get starting key for the batch. Get the first key if we restart the process.
let mut starting_raw_key = AlphaMapCleanLastKey::<T>::get();
let mut starting_key = None;
if starting_raw_key.is_none() {
starting_key = Alpha::<T>::iter_keys().next();
starting_raw_key = starting_key.as_ref().map(Alpha::<T>::hashed_key_for);
}

if let Some(starting_raw_key) = starting_raw_key {
// Get the key batch
let mut keys = Alpha::<T>::iter_keys_from(starting_raw_key)
.take(ALPHA_MAP_CLEAN_BATCH_SIZE)
.collect::<Vec<_>>();

// New iteration: insert the starting key in the batch if it's a new iteration
// iter_keys_from() skips the starting key
if let Some(starting_key) = starting_key {
if keys.len() == ALPHA_MAP_CLEAN_BATCH_SIZE {
keys.remove(keys.len().saturating_sub(1));
}
keys.insert(0, starting_key);
}

let mut new_starting_key = None;
let new_iteration = keys.len() < ALPHA_MAP_CLEAN_BATCH_SIZE;

// Check and remove alphas if necessary
for key in keys {
let alpha = Alpha::<T>::get(&key);
if alpha < U64F64::saturating_from_num(MIN_ALPHA) {
Alpha::<T>::remove(&key);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the ideal case when there are no rounding errors, the sum of all alpha for one hotkey will match TotalHotkeyShares, so TotalHotkeyShares should be also decreased here, but in practice we need to check (offline, not in the chain code) if the sum really matches.

}
new_starting_key = Some(Alpha::<T>::hashed_key_for(key));
}

// Restart the process if it's the last batch
if new_iteration {
new_starting_key = None;
}

AlphaMapCleanLastKey::<T>::put(new_starting_key);
}
}

pub fn add_balance_to_coldkey_account(
coldkey: &T::AccountId,
amount: <<T as Config>::Currency as fungible::Inspect<<T as system::Config>::AccountId>>::Balance,
Expand Down
107 changes: 107 additions & 0 deletions pallets/subtensor/src/tests/staking.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2404,6 +2404,113 @@ fn test_faucet_ok() {
});
}

#[test]
fn test_clear_small_nominations_batches() {
new_test_ext(0).execute_with(|| {
let cold = U256::from(3);
let netuid: u16 = 1;
let low_amount = U64F64::saturating_from_num(MIN_ALPHA.saturating_sub(1));
let enough_amount = U64F64::saturating_from_num(MIN_ALPHA);

// Set alpha to clear
for i in 0..ALPHA_MAP_CLEAN_BATCH_SIZE + 1 {
let hot = U256::from(i);
let key = (hot, cold, netuid);
Alpha::<Test>::insert(key, low_amount);
}

let keys = Alpha::<Test>::iter_keys().collect::<Vec<_>>();
assert_eq!(keys.len(), ALPHA_MAP_CLEAN_BATCH_SIZE + 1);
assert!(AlphaMapCleanLastKey::<Test>::get().is_none());

SubtensorModule::clear_small_nominations_batch();

let keys = Alpha::<Test>::iter_keys().collect::<Vec<_>>();
// First batch cleared
assert_eq!(keys.len(), 1);
// Pagination set
assert!(AlphaMapCleanLastKey::<Test>::get().is_some());

SubtensorModule::clear_small_nominations_batch();

let keys = Alpha::<Test>::iter_keys().collect::<Vec<_>>();
// Last batch cleared
assert_eq!(keys.len(), 0);
// Pagination removed (start over)
assert!(AlphaMapCleanLastKey::<Test>::get().is_none());

// Set alpha to keep
for i in 0..ALPHA_MAP_CLEAN_BATCH_SIZE + 1 {
let hot = U256::from(i);
let key = (hot, cold, netuid);
Alpha::<Test>::insert(key, enough_amount);
}

let keys = Alpha::<Test>::iter_keys().collect::<Vec<_>>();
assert_eq!(keys.len(), ALPHA_MAP_CLEAN_BATCH_SIZE + 1);
assert!(AlphaMapCleanLastKey::<Test>::get().is_none());

SubtensorModule::clear_small_nominations_batch();

let keys = Alpha::<Test>::iter_keys().collect::<Vec<_>>();
// First batch kept
assert_eq!(keys.len(), ALPHA_MAP_CLEAN_BATCH_SIZE + 1);
// Pagination set
assert!(AlphaMapCleanLastKey::<Test>::get().is_some());

SubtensorModule::clear_small_nominations_batch();

let keys = Alpha::<Test>::iter_keys().collect::<Vec<_>>();
// Last batch kept
assert_eq!(keys.len(), ALPHA_MAP_CLEAN_BATCH_SIZE + 1);
// Pagination removed (start over)
assert!(AlphaMapCleanLastKey::<Test>::get().is_none());
});
}

#[test]
fn test_clear_small_nominations_batches_by_blocks() {
new_test_ext(0).execute_with(|| {
let cold = U256::from(3);
let netuid: u16 = 1;
let low_amount = U64F64::saturating_from_num(MIN_ALPHA.saturating_sub(1));
let enough_amount = U64F64::saturating_from_num(MIN_ALPHA);

// Set alpha to keep and to remove
let extra = 5usize;
for i in 1..=(ALPHA_MAP_CLEAN_BATCH_SIZE * 2) {
let hot1 = U256::from(i);
let hot2 = U256::from(i * 100);
let key1 = (hot1, cold, netuid);
let key2 = (hot2, cold, netuid);
Alpha::<Test>::insert(key1, low_amount);
Alpha::<Test>::insert(key2, enough_amount);
}

for i in 1..=extra {
let hot = U256::from(i * 100000);
let key = (hot, cold, netuid);
Alpha::<Test>::insert(key, enough_amount);
}

// Check preconditions
let keys = Alpha::<Test>::iter_keys().collect::<Vec<_>>();
assert_eq!(keys.len(), ALPHA_MAP_CLEAN_BATCH_SIZE * 4 + extra);
assert!(AlphaMapCleanLastKey::<Test>::get().is_none());

run_to_block(4);
assert!(AlphaMapCleanLastKey::<Test>::get().is_some());
run_to_block(5);

let keys = Alpha::<Test>::iter_keys().collect::<Vec<_>>();
assert_eq!(keys.len(), 2 * ALPHA_MAP_CLEAN_BATCH_SIZE + extra);
assert!(AlphaMapCleanLastKey::<Test>::get().is_none());

run_to_block(6);
assert!(AlphaMapCleanLastKey::<Test>::get().is_some());
});
}

/// This test ensures that the clear_small_nominations function works as expected.
/// It creates a network with two hotkeys and two coldkeys, and then registers a nominator account for each hotkey.
/// When we call set_nominator_min_required_stake, it should clear all small nominations that are below the minimum required stake.
Expand Down
Loading