From 86af6c069fadea54216c2e05f86cc7bfb74c797f Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Tue, 22 Apr 2025 15:04:50 -0700 Subject: [PATCH 1/9] add from and to to evaluate_condition and add_index --- lib/enrichment/src/vrl_util.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/enrichment/src/vrl_util.rs b/lib/enrichment/src/vrl_util.rs index 61e7044c53c99..df40824cb17c6 100644 --- a/lib/enrichment/src/vrl_util.rs +++ b/lib/enrichment/src/vrl_util.rs @@ -55,6 +55,22 @@ pub(crate) fn evaluate_condition(key: &str, value: Value) -> ExpressionResult Condition::FromDate { + field: key, + from: *map + .get("from") + .expect("should contain from") + .as_timestamp() + .ok_or("from in condition must be a timestamp")?, + }, + Value::Object(map) if map.contains_key("to") => Condition::ToDate { + field: key, + to: *map + .get("to") + .expect("should contain to") + .as_timestamp() + .ok_or("to in condition must be a timestamp")?, + }, _ => Condition::Equals { field: key, value }, }) } @@ -71,7 +87,12 @@ pub(crate) fn add_index( .filter_map(|(field, value)| match value { expression::Expr::Container(expression::Container { variant: expression::Variant::Object(map), - }) if map.contains_key("from") && map.contains_key("to") => None, + }) if (map.contains_key("from") && map.contains_key("to")) + || map.contains_key("from") + || map.contains_key("to") => + { + None + } _ => Some(field.as_ref()), }) .collect::>(); From b7a3f26b09083afaebdec9557423018c165b7e98 Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Tue, 22 Apr 2025 15:04:58 -0700 Subject: [PATCH 2/9] add FromDate and ToDate to enum Condition --- lib/enrichment/src/lib.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/enrichment/src/lib.rs b/lib/enrichment/src/lib.rs index 52f2a547f50f1..b84d899b8d25f 100644 --- a/lib/enrichment/src/lib.rs +++ b/lib/enrichment/src/lib.rs @@ -26,6 +26,16 @@ pub enum Condition<'a> { from: chrono::DateTime, to: chrono::DateTime, }, + /// The date in the field is greater than or equal to from. + FromDate { + field: &'a str, + from: chrono::DateTime, + }, + /// The date in the field is less than or equal to to. + ToDate { + field: &'a str, + to: chrono::DateTime, + }, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] From d895861db8a0f6026437b222a5290fb7dc02b78c Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Tue, 22 Apr 2025 15:05:04 -0700 Subject: [PATCH 3/9] add conditions to row_equals and unit tests --- src/enrichment_tables/file.rs | 142 +++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 1 deletion(-) diff --git a/src/enrichment_tables/file.rs b/src/enrichment_tables/file.rs index 32ec9318b88e0..20abb4fa8a4d1 100644 --- a/src/enrichment_tables/file.rs +++ b/src/enrichment_tables/file.rs @@ -317,6 +317,20 @@ impl File { _ => false, }, }, + Condition::FromDate { field, from } => match self.column_index(field) { + None => false, + Some(idx) => match row[idx] { + Value::Timestamp(date) => from <= &date, + _ => false, + }, + }, + Condition::ToDate { field, to } => match self.column_index(field) { + None => false, + Some(idx) => match row[idx] { + Value::Timestamp(date) => &date <= to, + _ => false, + }, + }, }) } @@ -1030,7 +1044,7 @@ mod tests { } #[test] - fn finds_row_with_dates() { + fn finds_row_between_dates() { let mut file = File::new( Default::default(), FileData { @@ -1096,6 +1110,132 @@ mod tests { ); } + #[test] + fn finds_row_from_date() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + + let conditions = [ + Condition::Equals { + field: "field1", + value: "zip".into(), + }, + Condition::FromDate { + field: "field2", + from: chrono::Utc + .with_ymd_and_hms(2016, 1, 1, 0, 0, 0) + .single() + .expect("invalid timestamp"), + }, + ]; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ( + "field2".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp") + ) + ) + ])), + file.find_table_row(Case::Sensitive, &conditions, None, Some(handle)) + ); + } + + #[test] + fn finds_row_to_date() { + let mut file = File::new( + Default::default(), + FileData { + modified: SystemTime::now(), + data: vec![ + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + vec![ + "zip".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2016, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp"), + ), + ], + ], + headers: vec!["field1".to_string(), "field2".to_string()], + }, + ); + + let handle = file.add_index(Case::Sensitive, &["field1"]).unwrap(); + + let conditions = [ + Condition::Equals { + field: "field1", + value: "zip".into(), + }, + Condition::ToDate { + field: "field2", + to: chrono::Utc + .with_ymd_and_hms(2016, 1, 1, 0, 0, 0) + .single() + .expect("invalid timestamp"), + }, + ]; + + assert_eq!( + Ok(ObjectMap::from([ + ("field1".into(), Value::from("zip")), + ( + "field2".into(), + Value::Timestamp( + chrono::Utc + .with_ymd_and_hms(2015, 12, 7, 0, 0, 0) + .single() + .expect("invalid timestamp") + ) + ) + ])), + file.find_table_row(Case::Sensitive, &conditions, None, Some(handle)) + ); + } + #[test] fn doesnt_find_row() { let file = File::new( From 2f66f31bdbaf706328a2945fa8157c231838cb33 Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Tue, 22 Apr 2025 15:01:30 -0700 Subject: [PATCH 4/9] update website --- website/cue/reference/remap/functions.cue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/cue/reference/remap/functions.cue b/website/cue/reference/remap/functions.cue index d739d474fec60..18ed9457a673c 100644 --- a/website/cue/reference/remap/functions.cue +++ b/website/cue/reference/remap/functions.cue @@ -74,7 +74,7 @@ remap: { performance perspective. 2. **Date range search**. The given field must be greater than or equal to the `from` date - and less than or equal to the `to` date. A date range search involves + and/or less than or equal to the `to` date. A date range search involves sequentially scanning through the rows that have been located using any exact match criteria. This can be an expensive operation if there are many rows returned by any exact match criteria. Therefore, use date ranges as the _only_ criteria when the enrichment From 2af062dae20c26fcb4197b971be8aa6244462b42 Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Mon, 5 May 2025 11:27:54 -0700 Subject: [PATCH 5/9] add changelog entry --- ...nt_function_single_bounded_date_range_search.enhancement.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md diff --git a/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md b/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md new file mode 100644 index 0000000000000..3e9b34fd0e2d6 --- /dev/null +++ b/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md @@ -0,0 +1,3 @@ +The [enrichment functions](https://vector.dev/docs/reference/vrl/functions/#enrichment-functions) now support single-bounded date range search. There are no changes to the function signatures. + +authors: nzxwang From d5b0a8c4ed755f243741e1f47cac9ca4d85cf977 Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Tue, 6 May 2025 12:56:51 -0700 Subject: [PATCH 6/9] Update comment in lib/enrichment/src/lib.rs Co-authored-by: Pavlos Rontidis --- lib/enrichment/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/enrichment/src/lib.rs b/lib/enrichment/src/lib.rs index b84d899b8d25f..1886c6455d7b6 100644 --- a/lib/enrichment/src/lib.rs +++ b/lib/enrichment/src/lib.rs @@ -26,7 +26,7 @@ pub enum Condition<'a> { from: chrono::DateTime, to: chrono::DateTime, }, - /// The date in the field is greater than or equal to from. + /// The date in the field is greater than or equal to `from`. FromDate { field: &'a str, from: chrono::DateTime, From 0a1ee7ebecd787c0486060fc80672c4de6558dd1 Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Tue, 6 May 2025 12:57:05 -0700 Subject: [PATCH 7/9] Update comment in lib/enrichment/src/lib.rs Co-authored-by: Pavlos Rontidis --- lib/enrichment/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/enrichment/src/lib.rs b/lib/enrichment/src/lib.rs index 1886c6455d7b6..91195aa77338b 100644 --- a/lib/enrichment/src/lib.rs +++ b/lib/enrichment/src/lib.rs @@ -31,7 +31,7 @@ pub enum Condition<'a> { field: &'a str, from: chrono::DateTime, }, - /// The date in the field is less than or equal to to. + /// The date in the field is less than or equal to `to`. ToDate { field: &'a str, to: chrono::DateTime, From 2c0a2356b35b24141c8bbb7ed7fc57938249a4f8 Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Tue, 6 May 2025 12:57:15 -0700 Subject: [PATCH 8/9] Update changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md Co-authored-by: Pavlos Rontidis --- ...ent_function_single_bounded_date_range_search.enhancement.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md b/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md index 3e9b34fd0e2d6..755436e4af598 100644 --- a/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md +++ b/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md @@ -1,3 +1,3 @@ -The [enrichment functions](https://vector.dev/docs/reference/vrl/functions/#enrichment-functions) now support single-bounded date range search. There are no changes to the function signatures. +The [enrichment functions](https://vector.dev/docs/reference/vrl/functions/#enrichment-functions) now support bounded date range filtering using optional `from` and `to` parameters. There are no changes to the function signatures. authors: nzxwang From 5ee8cfade7e10fb3230c50834ca6424ab1a06e25 Mon Sep 17 00:00:00 2001 From: Nick Wang Date: Fri, 9 May 2025 14:30:43 -0700 Subject: [PATCH 9/9] remove trailing space fixes failing check https://github.com/vectordotdev/vector/actions/runs/14868558637/job/41959755638 --- ...ent_function_single_bounded_date_range_search.enhancement.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md b/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md index 755436e4af598..e20a27c160a15 100644 --- a/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md +++ b/changelog.d/22926_enrichment_function_single_bounded_date_range_search.enhancement.md @@ -1,3 +1,3 @@ -The [enrichment functions](https://vector.dev/docs/reference/vrl/functions/#enrichment-functions) now support bounded date range filtering using optional `from` and `to` parameters. There are no changes to the function signatures. +The [enrichment functions](https://vector.dev/docs/reference/vrl/functions/#enrichment-functions) now support bounded date range filtering using optional `from` and `to` parameters. There are no changes to the function signatures. authors: nzxwang