Skip to content

Commit 1f6047f

Browse files
g-talbotclaude
andcommitted
fix: check time-based maturity in ParquetMergePlanner (matches Tantivy planner)
Splits whose created_at + maturation_period has elapsed are effectively mature and should not participate in further merges. Without this check, old small splits would be re-merged indefinitely. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 8fe626c commit 1f6047f

1 file changed

Lines changed: 15 additions & 3 deletions

File tree

quickwit/quickwit-indexing/src/actors/metrics_pipeline/parquet_merge_planner.rs

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ use std::time::Instant;
3232
use async_trait::async_trait;
3333
use quickwit_actors::{Actor, ActorContext, ActorExitStatus, Handler, Mailbox, QueueCapacity};
3434
use quickwit_parquet_engine::merge::policy::{
35-
CompactionScope, ParquetMergeOperation, ParquetMergePolicy,
35+
CompactionScope, ParquetMergeOperation, ParquetMergePolicy, ParquetSplitMaturity,
3636
};
3737
use quickwit_parquet_engine::split::ParquetSplitMetadata;
3838
use tantivy::Inventory;
@@ -195,15 +195,27 @@ impl ParquetMergePlanner {
195195

196196
/// Filters and records incoming splits, skipping:
197197
/// - Mature splits (already at max merge ops or target size)
198+
/// - Time-matured splits (created_at + maturation_period has elapsed)
198199
/// - Splits we've already seen (dedup via `known_split_ids`)
199200
/// - Pre-Phase-31 splits without a window (can't participate in compaction)
200201
fn record_splits_if_necessary(&mut self, splits: Vec<ParquetSplitMetadata>) {
202+
let now = std::time::SystemTime::now();
201203
for split in splits {
202-
if let quickwit_parquet_engine::merge::policy::ParquetSplitMaturity::Mature = self
204+
match self
203205
.merge_policy
204206
.split_maturity(split.size_bytes, split.num_merge_ops)
205207
{
206-
continue;
208+
ParquetSplitMaturity::Mature => continue,
209+
ParquetSplitMaturity::Immature {
210+
maturation_period, ..
211+
} => {
212+
// A split that has lived past its maturation period is
213+
// effectively mature — no further merges needed. This
214+
// mirrors the Tantivy merge planner's `is_mature(now)`.
215+
if split.created_at + maturation_period <= now {
216+
continue;
217+
}
218+
}
207219
}
208220
if !self.acknowledge_split(split.split_id.as_str()) {
209221
continue;

0 commit comments

Comments
 (0)