Skip to content

Commit b5938ef

Browse files
committed
feature(collator): remove cumulative stats recalculation
1 parent b02b785 commit b5938ef

File tree

3 files changed

+9
-62
lines changed

3 files changed

+9
-62
lines changed

collator/src/collator/messages_reader/internals_reader.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -356,7 +356,6 @@ impl<V: InternalMessageValue> InternalsPartitionReader<V> {
356356

357357
ranges.push(QueueShardBoundedRange {
358358
shard_ident: *shard_id,
359-
// TODO: may be we should pass `from` here because we should load stats for the full range
360359
from: Bound::Excluded(shard_reader_state.current_position),
361360
to: Bound::Included(shard_range_to),
362361
});

collator/src/collator/types.rs

Lines changed: 9 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ use tycho_block_util::state::{RefMcStateHandle, ShardStateStuff};
2424
use tycho_core::global_config::MempoolGlobalConfig;
2525
use tycho_executor::{AccountMeta, PublicLibraryChange, TransactionMeta};
2626
use tycho_network::PeerId;
27-
use tycho_util::metrics::HistogramGuard;
2827
use tycho_util::{DashMapEntry, FastDashMap, FastHashMap, FastHashSet};
2928

3029
use super::do_collate::work_units::PrepareMsgGroupsWu;
@@ -1290,9 +1289,6 @@ pub struct CumulativeStatistics {
12901289

12911290
/// The final aggregated statistics (across all shards) by partitions.
12921291
result: FastHashMap<QueuePartitionIdx, QueueStatisticsWithRemaning>,
1293-
1294-
/// A flag indicating that data has changed, and we need to recalculate before returning `result`.
1295-
dirty: bool,
12961292
}
12971293

12981294
impl CumulativeStatistics {
@@ -1306,7 +1302,6 @@ impl CumulativeStatistics {
13061302
all_shards_processed_to_by_partitions,
13071303
shards_stats_by_partitions: Default::default(),
13081304
result: Default::default(),
1309-
dirty: false,
13101305
}
13111306
}
13121307

@@ -1345,7 +1340,6 @@ impl CumulativeStatistics {
13451340
partitions: &FastHashSet<QueuePartitionIdx>,
13461341
ranges: Vec<QueueShardBoundedRange>,
13471342
) -> Result<()> {
1348-
self.dirty = true;
13491343
tracing::trace!(
13501344
target: tracing_targets::COLLATOR,
13511345
"cumulative_stats_partial_ranges: {:?}",
@@ -1405,9 +1399,6 @@ impl CumulativeStatistics {
14051399
by_partitions.retain(|_part, diffs| !diffs.is_empty());
14061400
!by_partitions.is_empty()
14071401
});
1408-
1409-
self.all_shards_processed_to_by_partitions = new_pt;
1410-
self.dirty = true;
14111402
}
14121403

14131404
fn compute_cumulative_stats_ranges(
@@ -1469,15 +1460,19 @@ impl CumulativeStatistics {
14691460
}
14701461
}
14711462

1463+
if !diff_partition_stats.is_empty() {
1464+
let part_stats = self.result.entry(partition).or_default();
1465+
part_stats.initial_stats.append(&diff_partition_stats);
1466+
part_stats.remaning_stats.append(&diff_partition_stats);
1467+
}
1468+
14721469
// finally add weeded stats
14731470
self.add_diff_partition_stats(
14741471
partition,
14751472
diff_shard,
14761473
diff_max_message,
14771474
diff_partition_stats,
14781475
);
1479-
1480-
self.dirty = true;
14811476
}
14821477

14831478
fn add_diff_partition_stats(
@@ -1541,6 +1536,9 @@ impl CumulativeStatistics {
15411536
cumulative_stats
15421537
.initial_stats
15431538
.decrement_for_account(dst_acc.clone(), *count);
1539+
cumulative_stats
1540+
.remaning_stats
1541+
.decrement_for_account(dst_acc.clone(), *count);
15441542
false
15451543
} else {
15461544
true
@@ -1571,15 +1569,12 @@ impl CumulativeStatistics {
15711569
/// Returns a reference to the aggregated stats by partitions.
15721570
/// If the data is marked as dirty, it triggers a lazy recalculation first.
15731571
pub fn result(&mut self) -> &FastHashMap<QueuePartitionIdx, QueueStatisticsWithRemaning> {
1574-
self.ensure_finalized();
15751572
&self.result
15761573
}
15771574

15781575
/// Calc aggregated stats among all partitions.
15791576
/// If the data is marked as dirty, it triggers a lazy recalculation first.
15801577
pub fn get_aggregated_result(&mut self) -> QueueStatistics {
1581-
self.ensure_finalized();
1582-
15831578
let mut res: Option<QueueStatistics> = None;
15841579
for stats in self.result.values() {
15851580
if let Some(aggregated) = res.as_mut() {
@@ -1590,42 +1585,6 @@ impl CumulativeStatistics {
15901585
}
15911586
res.unwrap_or_default()
15921587
}
1593-
1594-
/// A helper function to trigger a recalculation if `dirty` is set.
1595-
fn ensure_finalized(&mut self) {
1596-
if self.dirty {
1597-
self.recalculate();
1598-
}
1599-
}
1600-
1601-
/// Clears the existing result and aggregates all data from `shards_statistics`.
1602-
fn recalculate(&mut self) {
1603-
let _histogram = HistogramGuard::begin("tycho_do_collate_recalculate_statistics_time");
1604-
self.result.clear();
1605-
1606-
for shard_stats_by_partitions in self.shards_stats_by_partitions.values() {
1607-
for (&partition, diffs) in shard_stats_by_partitions {
1608-
let mut partition_stats = AccountStatistics::new();
1609-
for diff_stats in diffs.values() {
1610-
for (account, &count) in diff_stats {
1611-
*partition_stats.entry(account.clone()).or_default() += count;
1612-
}
1613-
}
1614-
self.result
1615-
.entry(partition)
1616-
.and_modify(|stats| {
1617-
stats.initial_stats.append(&partition_stats);
1618-
stats.remaning_stats.append(&partition_stats);
1619-
})
1620-
.or_insert(QueueStatisticsWithRemaning {
1621-
initial_stats: QueueStatistics::with_statistics(partition_stats.clone()),
1622-
remaning_stats: ConcurrentQueueStatistics::with_statistics(partition_stats),
1623-
});
1624-
}
1625-
}
1626-
1627-
self.dirty = false;
1628-
}
16291588
}
16301589

16311590
#[derive(Debug, Default, Clone)]
@@ -1634,12 +1593,6 @@ pub struct ConcurrentQueueStatistics {
16341593
}
16351594

16361595
impl ConcurrentQueueStatistics {
1637-
pub fn with_statistics(statistics: AccountStatistics) -> Self {
1638-
Self {
1639-
statistics: Arc::new(statistics.into_iter().collect()),
1640-
}
1641-
}
1642-
16431596
pub fn statistics(&self) -> &FastDashMap<IntAddr, u64> {
16441597
&self.statistics
16451598
}
@@ -1694,7 +1647,6 @@ impl fmt::Debug for CumulativeStatistics {
16941647
)
16951648
.field("shards_stats_by_partitions", &shards_summary)
16961649
.field("result", &format!("{} partitions", self.result.len()))
1697-
.field("dirty", &self.dirty)
16981650
.finish()
16991651
}
17001652
}

scripts/gen-dashboard.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1710,10 +1710,6 @@ def collator_core_operations_metrics() -> RowPanel:
17101710
"Handle block candidate",
17111711
labels=['workchain=~"$workchain"'],
17121712
),
1713-
create_heatmap_panel(
1714-
"tycho_do_collate_recalculate_statistics_time",
1715-
"Recalculate cumulative statistics"
1716-
),
17171713
]
17181714
return create_row("collator: Core Operations Metrics", metrics)
17191715

0 commit comments

Comments
 (0)