From b79c8cdcb69db25a457e7614b59deedebe00c404 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 17:44:47 +0000 Subject: [PATCH 1/3] Initial plan From 654d403d2e7d083ebd0f0def124c7700420e6237 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 18:00:36 +0000 Subject: [PATCH 2/3] Add timestamp-based package filtering (exclude-newer / exclude-older) Agent-Logs-Url: https://github.com/conda-incubator/conda-mirror/sessions/6a079e7c-4576-40fd-9fd9-aab0f5bf2afc Co-authored-by: pavelzw <29506042+pavelzw@users.noreply.github.com> --- Cargo.lock | 1 + Cargo.toml | 1 + src/config.rs | 376 ++++++++++++++++++++++++-- src/lib.rs | 11 +- src/main.rs | 13 +- tests/resources/include-timestamp.yml | 15 + 6 files changed, 379 insertions(+), 38 deletions(-) create mode 100644 tests/resources/include-timestamp.yml diff --git a/Cargo.lock b/Cargo.lock index 9653d94..9e2cb35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1045,6 +1045,7 @@ version = "0.12.1" dependencies = [ "anyhow", "bytes", + "chrono", "clap", "clap-verbosity-flag", "console", diff --git a/Cargo.toml b/Cargo.toml index c8177a1..d8a5d96 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ rustls-tls = [ # https://github.com/conda/rattler/issues/1726 anyhow = "1.0.100" bytes = "1.11.1" +chrono = "0.4" clap = { version = "4.5.51", features = ["derive", "string", "env"] } clap-verbosity-flag = { version = "3.0.4", features = ["tracing"] } console = "0.16.1" diff --git a/src/config.rs b/src/config.rs index 49eb93a..97d8229 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,6 +1,7 @@ +use chrono::{DateTime, NaiveDate, Utc}; use miette::IntoDiagnostic; use rattler_conda_types::{ - Channel, ChannelConfig, MatchSpec, NamedChannelOrUrl, ParseStrictness, + Channel, ChannelConfig, Matches, MatchSpec, NamedChannelOrUrl, PackageRecord, ParseStrictness, ParseStrictnessWithNameMatcher, Platform, }; use serde::{Deserialize, Deserializer}; @@ -126,28 +127,157 @@ impl std::fmt::Debug for S3Credentials { /* -------------------------------------------- YAML ------------------------------------------- */ +/// Parse a datetime string in one of the supported formats: +/// - ISO 8601 with timezone: `"2025-10-01T00:00:00Z"` +/// - Date only (`YYYY-MM-DD`): `"2025-10-01"` (treated as midnight UTC) +/// - Relative duration: `"14d"` (14 days ago from now) +pub fn parse_datetime(s: &str) -> Result, String> { + // Try full RFC 3339 / ISO 8601 datetime with timezone + if let Ok(dt) = DateTime::parse_from_rfc3339(s) { + return Ok(dt.with_timezone(&Utc)); + } + + // Try date-only format YYYY-MM-DD (treat as midnight UTC) + if let Ok(date) = NaiveDate::parse_from_str(s, "%Y-%m-%d") { + if let Some(dt) = date.and_hms_opt(0, 0, 0) { + return Ok(dt.and_utc()); + } + } + + // Try relative duration suffix: e.g. "14d" = 14 days ago + if let Some(n) = s.strip_suffix('d').and_then(|n| n.parse::().ok()) { + return Ok(Utc::now() - chrono::Duration::days(n)); + } + + Err(format!( + "invalid datetime {s:?}: expected RFC 3339 (e.g. \"2025-01-01T00:00:00Z\"), \ + a date (e.g. \"2025-01-01\"), or a relative duration (e.g. \"14d\")" + )) +} + +fn parse_matchspec_str(s: &str) -> Result { + MatchSpec::from_str( + s, + ParseStrictnessWithNameMatcher { + parse_strictness: ParseStrictness::Strict, + exact_names_only: false, + }, + ) + .map_err(E::custom) +} + +/// A package filter combining a [`MatchSpec`] with optional timestamp bounds. +/// +/// Supports two YAML representations: +/// +/// 1. A plain string MatchSpec: +/// ```yaml +/// include: +/// - "python >=3.9" +/// - "jupyter*[license=MIT]" +/// ``` +/// +/// 2. An object with an explicit `matchspec` key plus optional timestamp bounds: +/// ```yaml +/// include: +/// - matchspec: "python*" +/// exclude-newer: "2025-10-01T00:00:00Z" +/// exclude-older: "2023-01-01" +/// ``` #[derive(Debug, Clone)] -#[repr(transparent)] -pub struct MatchSpecWrapper(pub MatchSpec); +pub struct PackageFilter { + pub matchspec: MatchSpec, + /// Exclude packages whose build timestamp is **strictly after** this datetime. + pub exclude_newer: Option>, + /// Exclude packages whose build timestamp is **strictly before** this datetime. + pub exclude_older: Option>, +} -impl<'de> Deserialize<'de> for MatchSpecWrapper { +impl PackageFilter { + /// Returns `true` if the package satisfies both the MatchSpec and any + /// configured timestamp bounds. + /// + /// Packages whose `timestamp` field is absent are always considered a match + /// with respect to the timestamp bounds (conservative / include-unknown approach). + pub fn matches(&self, pkg: &PackageRecord) -> bool { + if !self.matchspec.matches(pkg) { + return false; + } + if let Some(ref ts) = pkg.timestamp { + if let Some(ref exclude_newer) = self.exclude_newer { + if ts > exclude_newer { + return false; + } + } + if let Some(ref exclude_older) = self.exclude_older { + if ts < exclude_older { + return false; + } + } + } + true + } +} + +// Internal helper struct for the object form of a PackageFilter in YAML. +#[derive(Deserialize)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +struct PackageFilterObject { + matchspec: String, + exclude_newer: Option, + exclude_older: Option, +} + +// Two-way untagged: a bare string OR an object with matchspec + optional bounds. +#[derive(Deserialize)] +#[serde(untagged)] +enum PackageFilterRaw { + String(String), + Object(PackageFilterObject), +} + +impl<'de> Deserialize<'de> for PackageFilter { fn deserialize(deserializer: D) -> Result where D: Deserializer<'de>, { - let s = String::deserialize(deserializer)?; - let wrapper = MatchSpecWrapper( - MatchSpec::from_str( - &s, - ParseStrictnessWithNameMatcher { - parse_strictness: ParseStrictness::Strict, - exact_names_only: false, - }, - ) - .map_err(serde::de::Error::custom)?, - ); - tracing::trace!("Deserialized MatchSpec: {wrapper:?}"); - Ok(wrapper) + let raw = PackageFilterRaw::deserialize(deserializer)?; + + match raw { + PackageFilterRaw::String(s) => { + tracing::trace!("Deserializing PackageFilter from string: {s:?}"); + let matchspec = parse_matchspec_str(&s)?; + Ok(PackageFilter { + matchspec, + exclude_newer: None, + exclude_older: None, + }) + } + PackageFilterRaw::Object(obj) => { + tracing::trace!( + "Deserializing PackageFilter from object: matchspec={:?}", + obj.matchspec + ); + let matchspec = parse_matchspec_str(&obj.matchspec)?; + let exclude_newer = obj + .exclude_newer + .as_deref() + .map(parse_datetime) + .transpose() + .map_err(serde::de::Error::custom)?; + let exclude_older = obj + .exclude_older + .as_deref() + .map(parse_datetime) + .transpose() + .map_err(serde::de::Error::custom)?; + Ok(PackageFilter { + matchspec, + exclude_newer, + exclude_older, + }) + } + } } } @@ -177,8 +307,8 @@ pub struct CondaMirrorYamlConfig { pub max_retries: Option, pub max_parallel: Option, - pub include: Option>, - pub exclude: Option>, + pub include: Option>, + pub exclude: Option>, pub s3_config: Option, pub precondition_checks: Option, } @@ -190,12 +320,12 @@ pub enum MirrorMode { /// Mirror all packages. All, /// Mirror all packages except those matching the given patterns. - AllButExclude(Vec), + AllButExclude(Vec), /// Mirror only packages matching the given patterns. - OnlyInclude(Vec), + OnlyInclude(Vec), /// Mirror all packages except those matching the given patterns. /// Override excludes with include patterns. - IncludeExclude(Vec, Vec), + IncludeExclude(Vec, Vec), } #[derive(Clone, Debug)] @@ -273,3 +403,205 @@ impl CondaMirrorConfig { self.platform_url(platform).join(filename) } } + +#[cfg(test)] +mod tests { + use super::*; + use chrono::TimeZone; + use rattler_conda_types::{PackageName, Version}; + use std::str::FromStr; + + fn make_record(name: &str, timestamp_ms: Option) -> PackageRecord { + use rattler_conda_types::utils::TimestampMs; + let mut record = PackageRecord::new( + PackageName::new_unchecked(name), + Version::from_str("1.0.0").unwrap(), + "0".to_string(), + ); + record.timestamp = timestamp_ms.map(|ms| { + let dt = DateTime::from_timestamp_millis(ms).expect("valid timestamp"); + TimestampMs::from(dt) + }); + record + } + + fn filter(spec: &str) -> PackageFilter { + serde_yml::from_str(spec).expect("valid PackageFilter") + } + + // ── parse_datetime ────────────────────────────────────────────────────── + + #[test] + fn test_parse_datetime_rfc3339() { + let dt = parse_datetime("2025-01-01T00:00:00Z").unwrap(); + assert_eq!(dt, Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).unwrap()); + } + + #[test] + fn test_parse_datetime_date_only() { + let dt = parse_datetime("2025-01-01").unwrap(); + assert_eq!(dt, Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).unwrap()); + } + + #[test] + fn test_parse_datetime_relative_days() { + let now = Utc::now(); + let dt = parse_datetime("7d").unwrap(); + let expected = now - chrono::Duration::days(7); + // Allow a 5-second tolerance for test execution time. + let diff_secs = (dt - expected).num_seconds().abs(); + assert!( + diff_secs <= 5, + "Expected timestamp within 5s of 7 days ago, got diff={diff_secs}s" + ); + } + + #[test] + fn test_parse_datetime_invalid() { + assert!(parse_datetime("not-a-date").is_err()); + assert!(parse_datetime("14w").is_err()); // "w" weeks not supported + } + + // ── PackageFilter deserialization ─────────────────────────────────────── + + #[test] + fn test_deserialize_string_form() { + let f: PackageFilter = serde_yml::from_str("\"python >=3.9\"").unwrap(); + assert!(f.exclude_newer.is_none()); + assert!(f.exclude_older.is_none()); + } + + #[test] + fn test_deserialize_object_form() { + let yaml = r#" +matchspec: "python*" +exclude-newer: "2025-10-01T00:00:00Z" +exclude-older: "2023-01-01" +"#; + let f: PackageFilter = serde_yml::from_str(yaml).unwrap(); + assert!(f.exclude_newer.is_some()); + assert!(f.exclude_older.is_some()); + let newer = f.exclude_newer.unwrap(); + assert_eq!(newer, Utc.with_ymd_and_hms(2025, 10, 1, 0, 0, 0).unwrap()); + let older = f.exclude_older.unwrap(); + assert_eq!(older, Utc.with_ymd_and_hms(2023, 1, 1, 0, 0, 0).unwrap()); + } + + #[test] + fn test_deserialize_object_unknown_field_rejected() { + let yaml = r#" +matchspec: "python*" +unknown-field: "value" +"#; + let result: Result = serde_yml::from_str(yaml); + assert!(result.is_err()); + } + + // ── PackageFilter::matches ────────────────────────────────────────────── + + #[test] + fn test_matches_name_only() { + let f = filter("\"python*\""); + assert!(f.matches(&make_record("python", None))); + assert!(f.matches(&make_record("python-3.9", None))); + assert!(!f.matches(&make_record("numpy", None))); + } + + #[test] + fn test_matches_no_timestamp_always_passes_bounds() { + // A package without a timestamp should pass regardless of bounds. + let yaml = r#" +matchspec: "*" +exclude-newer: "2020-01-01T00:00:00Z" +exclude-older: "2030-01-01T00:00:00Z" +"#; + let f: PackageFilter = serde_yml::from_str(yaml).unwrap(); + assert!(f.matches(&make_record("anything", None))); + } + + #[test] + fn test_matches_exclude_newer() { + // exclude_newer = 2025-01-01: include packages built on or before 2025-01-01 + let yaml = "matchspec: \"*\"\nexclude-newer: \"2025-01-01T00:00:00Z\"\n"; + let f: PackageFilter = serde_yml::from_str(yaml).unwrap(); + + let cutoff_ms = Utc + .with_ymd_and_hms(2025, 1, 1, 0, 0, 0) + .unwrap() + .timestamp_millis(); + + // Exactly at the cutoff – should be included (not strictly after) + assert!(f.matches(&make_record("pkg", Some(cutoff_ms)))); + // One millisecond after the cutoff – excluded + assert!(!f.matches(&make_record("pkg", Some(cutoff_ms + 1)))); + // Before the cutoff – included + assert!(f.matches(&make_record("pkg", Some(cutoff_ms - 1_000)))); + } + + #[test] + fn test_matches_exclude_older() { + // exclude_older = 2023-01-01: include packages built on or after 2023-01-01 + let yaml = "matchspec: \"*\"\nexclude-older: \"2023-01-01T00:00:00Z\"\n"; + let f: PackageFilter = serde_yml::from_str(yaml).unwrap(); + + let cutoff_ms = Utc + .with_ymd_and_hms(2023, 1, 1, 0, 0, 0) + .unwrap() + .timestamp_millis(); + + // Exactly at the cutoff – included (not strictly before) + assert!(f.matches(&make_record("pkg", Some(cutoff_ms)))); + // One millisecond before the cutoff – excluded + assert!(!f.matches(&make_record("pkg", Some(cutoff_ms - 1)))); + // After the cutoff – included + assert!(f.matches(&make_record("pkg", Some(cutoff_ms + 1_000)))); + } + + #[test] + fn test_matches_both_bounds() { + let yaml = "matchspec: \"python*\"\nexclude-newer: \"2025-01-01T00:00:00Z\"\nexclude-older: \"2023-01-01T00:00:00Z\"\n"; + let f: PackageFilter = serde_yml::from_str(yaml).unwrap(); + + let ts_in_range = Utc + .with_ymd_and_hms(2024, 6, 1, 0, 0, 0) + .unwrap() + .timestamp_millis(); + let ts_too_old = Utc + .with_ymd_and_hms(2022, 1, 1, 0, 0, 0) + .unwrap() + .timestamp_millis(); + let ts_too_new = Utc + .with_ymd_and_hms(2026, 1, 1, 0, 0, 0) + .unwrap() + .timestamp_millis(); + + assert!(f.matches(&make_record("python", Some(ts_in_range)))); + assert!(!f.matches(&make_record("python", Some(ts_too_old)))); + assert!(!f.matches(&make_record("python", Some(ts_too_new)))); + // Wrong name: never matches + assert!(!f.matches(&make_record("numpy", Some(ts_in_range)))); + } + + // ── CondaMirrorYamlConfig deserialization ─────────────────────────────── + + #[test] + fn test_yaml_config_with_timestamp_filter() { + let yaml = r#" +source: conda-forge +destination: ./mirror + +include: + - "jupyter*[license=MIT]" + - matchspec: "python*" + exclude-newer: "2025-10-01T00:00:00Z" + exclude-older: "2023-01-01" +"#; + let config: CondaMirrorYamlConfig = serde_yml::from_str(yaml).unwrap(); + let include = config.include.unwrap(); + assert_eq!(include.len(), 2); + assert!(include[0].exclude_newer.is_none()); + assert!(include[0].exclude_older.is_none()); + assert!(include[1].exclude_newer.is_some()); + assert!(include[1].exclude_older.is_some()); + } +} diff --git a/src/lib.rs b/src/lib.rs index 8cad1a7..58730a0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,7 +4,7 @@ use miette::IntoDiagnostic; use number_prefix::NumberPrefix; use opendal::{Configurator, Operator, layers::RetryLayer}; use rattler_conda_types::{ - ChannelConfig, Matches, NamedChannelOrUrl, PackageRecord, Platform, RepoData, + ChannelConfig, NamedChannelOrUrl, PackageRecord, Platform, RepoData, package::ArchiveType, }; use rattler_digest::Sha256Hash; @@ -407,16 +407,15 @@ fn get_packages_to_mirror( match config.mode.clone() { MirrorMode::All => all_packages.collect(), MirrorMode::OnlyInclude(include) => all_packages - .filter(|pkg| include.iter().any(|i| i.matches(&pkg.1))) + .filter(|pkg| include.iter().any(|f| f.matches(&pkg.1))) .collect(), MirrorMode::AllButExclude(exclude) => all_packages - .filter(|pkg| !exclude.iter().any(|i| i.matches(&pkg.1))) + .filter(|pkg| !exclude.iter().any(|f| f.matches(&pkg.1))) .collect(), MirrorMode::IncludeExclude(include, exclude) => all_packages .filter(|pkg| { - !exclude - .iter() - .any(|i| i.matches(&pkg.1) || include.iter().any(|i| i.matches(&pkg.1))) + include.iter().any(|f| f.matches(&pkg.1)) + || !exclude.iter().any(|f| f.matches(&pkg.1)) }) .collect(), } diff --git a/src/main.rs b/src/main.rs index 58554e3..e6b3399 100644 --- a/src/main.rs +++ b/src/main.rs @@ -69,16 +69,9 @@ async fn main() -> miette::Result<()> { .unwrap_or(true); let mode = match (yaml_config.include, yaml_config.exclude) { - (Some(include), Some(exclude)) => MirrorMode::IncludeExclude( - include.into_iter().map(|spec| spec.0).collect(), - exclude.into_iter().map(|spec| spec.0).collect(), - ), - (Some(include), None) => { - MirrorMode::OnlyInclude(include.into_iter().map(|spec| spec.0).collect()) - } - (None, Some(exclude)) => { - MirrorMode::AllButExclude(exclude.into_iter().map(|spec| spec.0).collect()) - } + (Some(include), Some(exclude)) => MirrorMode::IncludeExclude(include, exclude), + (Some(include), None) => MirrorMode::OnlyInclude(include), + (None, Some(exclude)) => MirrorMode::AllButExclude(exclude), (None, None) => MirrorMode::All, }; diff --git a/tests/resources/include-timestamp.yml b/tests/resources/include-timestamp.yml new file mode 100644 index 0000000..20ca3c5 --- /dev/null +++ b/tests/resources/include-timestamp.yml @@ -0,0 +1,15 @@ +source: conda-forge +destination: ./conda-forge-fs-mirror + +# Mirror only Python packages built between 2023 and 2025. +# The plain-string form and the object form can be mixed freely. +include: + # Plain string MatchSpec (no timestamp bounds) + - "jupyter*[license=MIT]" + # Object form with timestamp bounds + - matchspec: "python*" + exclude-newer: "2025-10-01T00:00:00Z" + exclude-older: "2023-01-01" + # Relative bound: exclude packages built more than 90 days ago + - matchspec: "numpy*" + exclude-older: "90d" From c18f9a9ede4defd9636a6312df3c5312cc09910f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 2 Apr 2026 07:46:46 +0000 Subject: [PATCH 3/3] fix: collapse nested if-let blocks to satisfy clippy and apply rustfmt Agent-Logs-Url: https://github.com/conda-incubator/conda-mirror/sessions/90fb0126-16ac-43fd-86fd-f71280e2ac9a Co-authored-by: pavelzw <29506042+pavelzw@users.noreply.github.com> --- src/config.rs | 26 +++++++++++++------------- src/lib.rs | 3 +-- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/config.rs b/src/config.rs index 97d8229..a1a8168 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,7 +1,7 @@ use chrono::{DateTime, NaiveDate, Utc}; use miette::IntoDiagnostic; use rattler_conda_types::{ - Channel, ChannelConfig, Matches, MatchSpec, NamedChannelOrUrl, PackageRecord, ParseStrictness, + Channel, ChannelConfig, MatchSpec, Matches, NamedChannelOrUrl, PackageRecord, ParseStrictness, ParseStrictnessWithNameMatcher, Platform, }; use serde::{Deserialize, Deserializer}; @@ -138,10 +138,10 @@ pub fn parse_datetime(s: &str) -> Result, String> { } // Try date-only format YYYY-MM-DD (treat as midnight UTC) - if let Ok(date) = NaiveDate::parse_from_str(s, "%Y-%m-%d") { - if let Some(dt) = date.and_hms_opt(0, 0, 0) { - return Ok(dt.and_utc()); - } + if let Ok(date) = NaiveDate::parse_from_str(s, "%Y-%m-%d") + && let Some(dt) = date.and_hms_opt(0, 0, 0) + { + return Ok(dt.and_utc()); } // Try relative duration suffix: e.g. "14d" = 14 days ago @@ -204,15 +204,15 @@ impl PackageFilter { return false; } if let Some(ref ts) = pkg.timestamp { - if let Some(ref exclude_newer) = self.exclude_newer { - if ts > exclude_newer { - return false; - } + if let Some(ref exclude_newer) = self.exclude_newer + && ts > exclude_newer + { + return false; } - if let Some(ref exclude_older) = self.exclude_older { - if ts < exclude_older { - return false; - } + if let Some(ref exclude_older) = self.exclude_older + && ts < exclude_older + { + return false; } } true diff --git a/src/lib.rs b/src/lib.rs index 58730a0..bb3bd72 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,8 +4,7 @@ use miette::IntoDiagnostic; use number_prefix::NumberPrefix; use opendal::{Configurator, Operator, layers::RetryLayer}; use rattler_conda_types::{ - ChannelConfig, NamedChannelOrUrl, PackageRecord, Platform, RepoData, - package::ArchiveType, + ChannelConfig, NamedChannelOrUrl, PackageRecord, Platform, RepoData, package::ArchiveType, }; use rattler_digest::Sha256Hash; use rattler_index::{PreconditionChecks, RepodataMetadataCollection, write_repodata};