Keep files younger than days_to_keep

Author: syyyr

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "historyprovider"
-version = "2.2.1"
+version = "2.3.0"
 edition = "2024"
 
 [[bin]]

Cargo.toml

@@ -5,6 +5,8 @@ use tokio::io;
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 
+use crate::util::msec_to_log2_filename;
+
 #[derive(Debug, Clone)]
 pub(crate) struct LogFile {
     pub(crate) name: PathBuf,
@@ -40,11 +42,11 @@ pub(crate) async fn collect_log2_files(dir: impl AsRef<Path>) -> io::Result<Vec<
 }
 
 /// Prune `.log2` files while keeping the newest one per directory
-pub(crate) async fn cleanup_log2_files(dir: impl AsRef<Path>, size_limit: u64) -> io::Result<()> {
+pub(crate) async fn cleanup_log2_files(dir: impl AsRef<Path>, size_limit: u64, days_to_keep: i64) -> io::Result<()> {
     let files = collect_log2_files(dir.as_ref()).await?;
     let mut files_size: u64 = files.iter().map(|f| f.size).sum();
 
-    info!("log2 files size: {files_size}, size limit: {size_limit}");
+    info!("log2 files size: {files_size}, size limit: {size_limit}, days_to_keep: {days_to_keep}");
     if files_size < size_limit {
         info!("Size limit not hit, nothing to cleanup");
         return Ok(());
@@ -58,11 +60,15 @@ pub(crate) async fn cleanup_log2_files(dir: impl AsRef<Path>, size_limit: u64) -
 
     let mut deletable_files = Vec::new();
 
+    // This file doesn't have to exist, I'm only constructing the filename for log retention.
+    let oldest_file_to_keep = PathBuf::from(msec_to_log2_filename(shvproto::DateTime::now().add_days(-days_to_keep).epoch_msec()));
+    info!("keeping files younger than {filename}", filename = oldest_file_to_keep.to_string_lossy());
+
     for (_dir, mut group) in grouped {
         // Sort descending (newest first)
         group.sort_by(|a, b| b.name.cmp(&a.name));
-        // Keep the newest file
-        deletable_files.extend(group.into_iter().skip(1));
+        // Keep the newest file, and keep files newer than oldest_file_to_delete
+        deletable_files.extend(group.into_iter().skip(1).filter(|log_file| log_file.name < oldest_file_to_keep));
     }
 
     // Sort deletable files (oldest first for deletion)

src/cleanup.rs

@@ -38,6 +38,7 @@ pub struct HpConfig {
     max_sync_tasks: Option<usize>,
     max_journal_dir_size: Option<usize>,
     periodic_sync_interval: Option<u64>,
+    days_to_keep: Option<i64>,
 }
 
 impl HpConfig {

src/lib.rs

@@ -722,6 +722,7 @@ pub(crate) async fn sync_task(
     // The download size limit should be lower than the max_journal_dir_size, because it doesn't
     // count in the files synced by getLog.
     let max_journal_dir_size = log_size_limit(&app_state.config);
+    let days_to_keep = app_state.config.days_to_keep.unwrap_or(0);
 
     while let Some(cmd) = sync_cmd_rx.next().await {
         match cmd {
@@ -817,7 +818,7 @@ pub(crate) async fn sync_task(
                             panic!("Cannot send dirtylog Trim command for site {site}: {e}")
                         )
                     );
-                match cleanup_log2_files(&app_state.config.journal_dir, max_journal_dir_size).await {
+                match cleanup_log2_files(&app_state.config.journal_dir, max_journal_dir_size, days_to_keep).await {
                     Ok(_) => info!("Cleanup journal dir done"),
                     Err(err) => error!("Cleanup journal dir error: {err}"),
                 }
@@ -881,7 +882,7 @@ pub(crate) async fn sync_task(
                         .unwrap_or_else(|e|
                             panic!("Cannot send dirtylog Trim command for site {site}: {e}")
                         );
-                    match cleanup_log2_files(&app_state.config.journal_dir, max_journal_dir_size).await {
+                    match cleanup_log2_files(&app_state.config.journal_dir, max_journal_dir_size, days_to_keep).await {
                         Ok(_) => info!("Cleanup journal dir done"),
                         Err(err) => error!("Cleanup journal dir error: {err}"),
                     }
@@ -891,7 +892,7 @@ pub(crate) async fn sync_task(
             }
             SyncCommand::Cleanup => {
                 info!("Cleanup journal dir start");
-                match cleanup_log2_files(&app_state.config.journal_dir, max_journal_dir_size).await {
+                match cleanup_log2_files(&app_state.config.journal_dir, max_journal_dir_size, days_to_keep).await {
                     Ok(_) => info!("Cleanup journal dir done"),
                     Err(err) => error!("Cleanup journal dir error: {err}"),
                 }

src/sync.rs

@@ -406,6 +406,7 @@ pub mod testing {
                 journal_dir: journal_dir.path().to_str().expect("path must work").to_string(),
                 max_sync_tasks: None,
                 max_journal_dir_size: None,
+                days_to_keep: None,
                 periodic_sync_interval: Some(3),
             },
             dirtylog_cmd_tx,