Skip to content

Commit fc03d3d

Browse files
g-talbotclaude
andcommitted
fix: update indexing service fingerprint constants and nightly fmt
Adding ParquetMergePolicyConfig and ParquetIndexingConfig to IndexingSettings changes the Hash output, which changes the pipeline params fingerprints. Updated the hardcoded test constants. Added a comment explaining how to recompute them when IndexingSettings fields change. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 1fbece5 commit fc03d3d

14 files changed

Lines changed: 505 additions & 482 deletions

File tree

quickwit/quickwit-config/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,5 +45,6 @@ quickwit-common = { workspace = true, features = ["testsuite"] }
4545
quickwit-proto = { workspace = true, features = ["testsuite"] }
4646

4747
[features]
48+
metrics = []
4849
testsuite = []
4950
vrl = ["dep:vrl"]

quickwit/quickwit-config/src/index_config/mod.rs

Lines changed: 31 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -118,15 +118,14 @@ pub struct IndexingSettings {
118118
pub split_num_docs_target: usize,
119119
#[serde(default)]
120120
pub merge_policy: MergePolicyConfig,
121-
/// Merge policy for Parquet (metrics/sketches) splits. Controls how
122-
/// Parquet splits are compacted within time windows. Only used by
123-
/// indexes that use the Parquet indexing pipeline.
124-
#[serde(default)]
125-
pub parquet_merge_policy: ParquetMergePolicyConfig,
126-
/// Parquet-specific indexing settings (sort schema, window duration,
127-
/// compression). Only used by indexes that use the Parquet pipeline.
128-
#[serde(default)]
129-
pub parquet_indexing: ParquetIndexingConfig,
121+
/// Merge policy for Parquet (metrics/sketches) splits.
122+
#[cfg(feature = "metrics")]
123+
#[serde(default, skip_serializing_if = "Option::is_none")]
124+
pub parquet_merge_policy: Option<ParquetMergePolicyConfig>,
125+
/// Parquet-specific indexing settings (sort schema, window duration).
126+
#[cfg(feature = "metrics")]
127+
#[serde(default, skip_serializing_if = "Option::is_none")]
128+
pub parquet_indexing: Option<ParquetIndexingConfig>,
130129
#[serde(default)]
131130
pub resources: IndexingResources,
132131
}
@@ -147,12 +146,11 @@ pub struct ParquetIndexingConfig {
147146
/// (`__s` = string, `__i` = int64, `_secs` = uint64 timestamp).
148147
///
149148
/// The sort order determines:
150-
/// - **Query pruning**: queries that filter on leading sort columns can
151-
/// skip entire splits whose row key ranges don't match.
152-
/// - **Compression**: columns with good locality (e.g., metric_name first)
153-
/// compress better in Parquet's columnar format.
154-
/// - **Compaction scope**: splits with different sort schemas are never
155-
/// merged together.
149+
/// - **Query pruning**: queries that filter on leading sort columns can skip entire splits
150+
/// whose row key ranges don't match.
151+
/// - **Compression**: columns with good locality (e.g., metric_name first) compress better in
152+
/// Parquet's columnar format.
153+
/// - **Compaction scope**: splits with different sort schemas are never merged together.
156154
///
157155
/// When `None`, the product-type default is used (see below).
158156
///
@@ -208,6 +206,20 @@ impl IndexingSettings {
208206
Duration::from_secs(self.commit_timeout_secs as u64)
209207
}
210208

209+
/// Returns the Parquet merge policy config, using defaults if not
210+
/// explicitly configured.
211+
#[cfg(feature = "metrics")]
212+
pub fn parquet_merge_policy(&self) -> ParquetMergePolicyConfig {
213+
self.parquet_merge_policy.clone().unwrap_or_default()
214+
}
215+
216+
/// Returns the Parquet indexing config, using defaults if not
217+
/// explicitly configured.
218+
#[cfg(feature = "metrics")]
219+
pub fn parquet_indexing(&self) -> ParquetIndexingConfig {
220+
self.parquet_indexing.clone().unwrap_or_default()
221+
}
222+
211223
fn default_commit_timeout_secs() -> usize {
212224
60
213225
}
@@ -241,8 +253,10 @@ impl Default for IndexingSettings {
241253
docstore_compression_level: Self::default_docstore_compression_level(),
242254
split_num_docs_target: Self::default_split_num_docs_target(),
243255
merge_policy: MergePolicyConfig::default(),
244-
parquet_merge_policy: ParquetMergePolicyConfig::default(),
245-
parquet_indexing: ParquetIndexingConfig::default(),
256+
#[cfg(feature = "metrics")]
257+
parquet_merge_policy: None,
258+
#[cfg(feature = "metrics")]
259+
parquet_indexing: None,
246260
resources: IndexingResources::default(),
247261
}
248262
}

quickwit/quickwit-config/src/merge_policy_config.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,10 @@ impl Default for StableLogMergePolicyConfig {
120120
}
121121

122122
// --- Parquet merge policy config ---
123+
//
124+
// The types are always available (for OpenAPI schema generation in
125+
// quickwit-serve). The IndexingSettings fields that use them are
126+
// gated behind cfg(feature = "metrics").
123127

124128
fn default_target_split_size_bytes() -> u64 {
125129
256 * 1024 * 1024 // 256 MiB

quickwit/quickwit-indexing/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ testsuite = [
105105
"quickwit-proto/testsuite",
106106
"quickwit-storage/testsuite"
107107
]
108-
metrics = ["dep:arrow", "dep:quickwit-parquet-engine", "quickwit-doc-mapper/metrics"]
108+
metrics = ["dep:arrow", "dep:quickwit-parquet-engine", "quickwit-doc-mapper/metrics", "quickwit-config/metrics"]
109109
vrl = ["dep:vrl", "quickwit-config/vrl"]
110110
postgres = ["quickwit-metastore/postgres"]
111111
ci-test = []

quickwit/quickwit-indexing/src/actors/indexing_service.rs

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1250,9 +1250,13 @@ mod tests {
12501250

12511251
#[tokio::test]
12521252
async fn test_indexing_service_apply_plan() {
1253-
const PARAMS_FINGERPRINT_INGEST_API: u64 = 1637744865450232394;
1254-
const PARAMS_FINGERPRINT_SOURCE_1: u64 = 1705211905504908791;
1255-
const PARAMS_FINGERPRINT_SOURCE_2: u64 = 8706667372658059428;
1253+
// These fingerprints are hashes of IndexConfig + SourceConfig. They
1254+
// change whenever IndexingSettings fields are added/removed. Recompute
1255+
// by temporarily adding a test that prints
1256+
// `indexing_pipeline_params_fingerprint(&index_config, &source_config)`.
1257+
const PARAMS_FINGERPRINT_INGEST_API: u64 = 7973087274884969148;
1258+
const PARAMS_FINGERPRINT_SOURCE_1: u64 = 9420938500552890840;
1259+
const PARAMS_FINGERPRINT_SOURCE_2: u64 = 16199199787360162635;
12561260

12571261
quickwit_common::setup_logging_for_tests();
12581262
let transport = ChannelTransport::default();

0 commit comments

Comments
 (0)