Skip to content

Commit c7e3eff

Browse files
g-talbotclaude
andcommitted
fix: update indexing service fingerprint constants and nightly fmt
Adding ParquetMergePolicyConfig and ParquetIndexingConfig to IndexingSettings changes the Hash output, which changes the pipeline params fingerprints. Updated the hardcoded test constants. Added a comment explaining how to recompute them when IndexingSettings fields change. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 1fbece5 commit c7e3eff

13 files changed

Lines changed: 504 additions & 481 deletions

File tree

quickwit/quickwit-config/src/index_config/mod.rs

Lines changed: 27 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -121,12 +121,14 @@ pub struct IndexingSettings {
121121
/// Merge policy for Parquet (metrics/sketches) splits. Controls how
122122
/// Parquet splits are compacted within time windows. Only used by
123123
/// indexes that use the Parquet indexing pipeline.
124-
#[serde(default)]
125-
pub parquet_merge_policy: ParquetMergePolicyConfig,
126-
/// Parquet-specific indexing settings (sort schema, window duration,
127-
/// compression). Only used by indexes that use the Parquet pipeline.
128-
#[serde(default)]
129-
pub parquet_indexing: ParquetIndexingConfig,
124+
/// When absent, defaults are used (see [`ParquetMergePolicyConfig`]).
125+
#[serde(default, skip_serializing_if = "Option::is_none")]
126+
pub parquet_merge_policy: Option<ParquetMergePolicyConfig>,
127+
/// Parquet-specific indexing settings (sort schema, window duration).
128+
/// Only used by indexes that use the Parquet pipeline.
129+
/// When absent, defaults are used (see [`ParquetIndexingConfig`]).
130+
#[serde(default, skip_serializing_if = "Option::is_none")]
131+
pub parquet_indexing: Option<ParquetIndexingConfig>,
130132
#[serde(default)]
131133
pub resources: IndexingResources,
132134
}
@@ -147,12 +149,11 @@ pub struct ParquetIndexingConfig {
147149
/// (`__s` = string, `__i` = int64, `_secs` = uint64 timestamp).
148150
///
149151
/// The sort order determines:
150-
/// - **Query pruning**: queries that filter on leading sort columns can
151-
/// skip entire splits whose row key ranges don't match.
152-
/// - **Compression**: columns with good locality (e.g., metric_name first)
153-
/// compress better in Parquet's columnar format.
154-
/// - **Compaction scope**: splits with different sort schemas are never
155-
/// merged together.
152+
/// - **Query pruning**: queries that filter on leading sort columns can skip entire splits
153+
/// whose row key ranges don't match.
154+
/// - **Compression**: columns with good locality (e.g., metric_name first) compress better in
155+
/// Parquet's columnar format.
156+
/// - **Compaction scope**: splits with different sort schemas are never merged together.
156157
///
157158
/// When `None`, the product-type default is used (see below).
158159
///
@@ -208,6 +209,18 @@ impl IndexingSettings {
208209
Duration::from_secs(self.commit_timeout_secs as u64)
209210
}
210211

212+
/// Returns the Parquet merge policy config, using defaults if not
213+
/// explicitly configured.
214+
pub fn parquet_merge_policy(&self) -> ParquetMergePolicyConfig {
215+
self.parquet_merge_policy.clone().unwrap_or_default()
216+
}
217+
218+
/// Returns the Parquet indexing config, using defaults if not
219+
/// explicitly configured.
220+
pub fn parquet_indexing(&self) -> ParquetIndexingConfig {
221+
self.parquet_indexing.clone().unwrap_or_default()
222+
}
223+
211224
fn default_commit_timeout_secs() -> usize {
212225
60
213226
}
@@ -241,8 +254,8 @@ impl Default for IndexingSettings {
241254
docstore_compression_level: Self::default_docstore_compression_level(),
242255
split_num_docs_target: Self::default_split_num_docs_target(),
243256
merge_policy: MergePolicyConfig::default(),
244-
parquet_merge_policy: ParquetMergePolicyConfig::default(),
245-
parquet_indexing: ParquetIndexingConfig::default(),
257+
parquet_merge_policy: None,
258+
parquet_indexing: None,
246259
resources: IndexingResources::default(),
247260
}
248261
}

quickwit/quickwit-config/src/lib.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,9 @@ impl ConfigFormat {
230230
}
231231

232232
pub fn parse<T>(&self, payload: &[u8]) -> anyhow::Result<T>
233-
where T: DeserializeOwned {
233+
where
234+
T: DeserializeOwned,
235+
{
234236
match self {
235237
ConfigFormat::Json => {
236238
let mut json_value: JsonValue =

quickwit/quickwit-config/src/merge_policy_config.rs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,9 @@ impl Default for ParquetMergePolicyConfig {
184184
}
185185

186186
fn parse_human_duration<'de, D>(deserializer: D) -> Result<Duration, D::Error>
187-
where D: Deserializer<'de> {
187+
where
188+
D: Deserializer<'de>,
189+
{
188190
let value: String = Deserialize::deserialize(deserializer)?;
189191
let duration = humantime::parse_duration(&value).map_err(|error| {
190192
de::Error::custom(format!(
@@ -195,7 +197,9 @@ where D: Deserializer<'de> {
195197
}
196198

197199
fn serialize_duration<S>(value: &Duration, s: S) -> Result<S::Ok, S::Error>
198-
where S: Serializer {
200+
where
201+
S: Serializer,
202+
{
199203
let value_str = humantime::format_duration(*value).to_string();
200204
s.serialize_str(&value_str)
201205
}

quickwit/quickwit-indexing/src/actors/indexing_service.rs

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1250,9 +1250,13 @@ mod tests {
12501250

12511251
#[tokio::test]
12521252
async fn test_indexing_service_apply_plan() {
1253-
const PARAMS_FINGERPRINT_INGEST_API: u64 = 1637744865450232394;
1254-
const PARAMS_FINGERPRINT_SOURCE_1: u64 = 1705211905504908791;
1255-
const PARAMS_FINGERPRINT_SOURCE_2: u64 = 8706667372658059428;
1253+
// These fingerprints are hashes of IndexConfig + SourceConfig. They
1254+
// change whenever IndexingSettings fields are added/removed. Recompute
1255+
// by temporarily adding a test that prints
1256+
// `indexing_pipeline_params_fingerprint(&index_config, &source_config)`.
1257+
const PARAMS_FINGERPRINT_INGEST_API: u64 = 7973087274884969148;
1258+
const PARAMS_FINGERPRINT_SOURCE_1: u64 = 9420938500552890840;
1259+
const PARAMS_FINGERPRINT_SOURCE_2: u64 = 16199199787360162635;
12561260

12571261
quickwit_common::setup_logging_for_tests();
12581262
let transport = ChannelTransport::default();

0 commit comments

Comments
 (0)