added batchcleanupservice

Projects/Dotmim.Sync.Core/Batch/BatchInfo.cs

@@ -21,7 +21,9 @@ public BatchInfo()
         {
             this.BatchPartsInfo = [];
             this.DirectoryRoot = SyncOptions.GetDefaultUserBatchDirectory();
-            this.DirectoryName = string.Concat(DateTime.UtcNow.ToString("yyyy_MM_dd_ss", CultureInfo.InvariantCulture), Path.GetRandomFileName().Replace(".", string.Empty, SyncGlobalization.DataSourceStringComparison));
+
+            // Note: It is important for the folder to start with the date information yyyyMMddHHmm so the batch cleanup service can easily find expired batch data and clean it up!
+            this.DirectoryName = string.Concat(DateTime.UtcNow.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture), Path.GetRandomFileName().Replace(".", string.Empty, SyncGlobalization.DataSourceStringComparison));
         }
 
         /// <inheritdoc cref="BatchInfo"/>
@@ -30,8 +32,9 @@ public BatchInfo(string rootDirectory, string directoryName = null, string info
         {
             this.DirectoryRoot = rootDirectory;
 
-            var randomName = string.Concat(DateTime.UtcNow.ToString("yyyy_MM_dd_ss", CultureInfo.InvariantCulture), Path.GetRandomFileName().Replace(".", string.Empty, SyncGlobalization.DataSourceStringComparison));
-            randomName = string.IsNullOrEmpty(info) ? randomName : $"{info}_{randomName}";
+            // Note: It is important for the folder to start with the date information yyyyMMddHHmm so the batch cleanup service can easily find expired batch data and clean it up!
+            var randomName = string.Concat(DateTime.UtcNow.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture), Path.GetRandomFileName().Replace(".", string.Empty, SyncGlobalization.DataSourceStringComparison));
+            randomName = string.IsNullOrEmpty(info) ? randomName : $"{randomName}_{info}";
             this.DirectoryName = string.IsNullOrEmpty(directoryName) ? randomName : directoryName;
         }
 

Projects/Dotmim.Sync.Core/BatchCleanupService.cs

@@ -0,0 +1,120 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Dotmim.Sync
+{
+    /// <summary>
+    /// Default implementation of IBatchCleanupService for cleaning up orphaned batch directories.
+    /// </summary>
+    public class BatchCleanupService : IBatchCleanupService
+    {
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="options"></param>
+        /// <param name="cancellationToken"></param>
+        /// <returns></returns>
+        /// <exception cref="NotImplementedException"></exception>
+        public Task<int> CleanupExpiredBatchesAsync(SyncOptions options, CancellationToken cancellationToken = default)
+        {
+            return this.CleanupExpiredBatchesAsync(options.BatchDirectory, options.BatchRetentionPeriod, cancellationToken);
+        }
+
+        /// <inheritdoc />
+        public Task<int> CleanupExpiredBatchesAsync(SyncOptions options, TimeSpan retentionPeriod,
+            CancellationToken cancellationToken = default)
+        {
+            return this.CleanupExpiredBatchesAsync(options.BatchDirectory, retentionPeriod, cancellationToken);
+        }
+
+        /// <inheritdoc/>
+        public async Task<int> CleanupExpiredBatchesAsync(string batchDirectory, TimeSpan retentionPeriod, CancellationToken cancellationToken = default)
+        {
+            if (string.IsNullOrEmpty(batchDirectory))
+                throw new ArgumentException("Batch directory cannot be null or empty.", nameof(batchDirectory));
+
+            if (!Directory.Exists(batchDirectory))
+                return 0;
+
+            if (retentionPeriod == TimeSpan.Zero)
+                return 0; // Time-based cleanup disabled
+
+            var expiredDirectories = await GetExpiredBatchDirectoriesAsync(batchDirectory, retentionPeriod, cancellationToken).ConfigureAwait(false);
+
+            int cleanedCount = 0;
+
+            foreach (var directory in expiredDirectories)
+            {
+                cancellationToken.ThrowIfCancellationRequested();
+
+                try
+                {
+                    if (Directory.Exists(directory))
+                    {
+                        Directory.Delete(directory, recursive: true);
+                        cleanedCount++;
+                    }
+                }
+                catch (Exception)
+                {
+                    // Log if needed, but don't fail the entire operation for one directory
+                    // Individual directory cleanup failures should not break the batch cleanup
+                }
+            }
+
+            return cleanedCount;
+        }
+
+        /// <inheritdoc />
+        public Task<IList<string>> GetExpiredBatchDirectoriesAsync(SyncOptions options, TimeSpan retentionPeriod,
+            CancellationToken cancellationToken = default)
+        {
+            return this.GetExpiredBatchDirectoriesAsync(options.BatchDirectory, retentionPeriod, cancellationToken);
+        }
+
+        /// <inheritdoc/>
+        public async Task<IList<string>> GetExpiredBatchDirectoriesAsync(string batchDirectory, TimeSpan retentionPeriod, CancellationToken cancellationToken = default)
+        {
+            if (string.IsNullOrEmpty(batchDirectory))
+                throw new ArgumentException("Batch directory cannot be null or empty.", nameof(batchDirectory));
+
+            if (!Directory.Exists(batchDirectory))
+                return new List<string>();
+
+            if (retentionPeriod == TimeSpan.Zero)
+                return new List<string>(); // Time-based cleanup disabled
+
+            // Calculate cutoff timestamp and format it as yyyyMMddHHmm
+            var cutoffTime = DateTime.UtcNow - retentionPeriod;
+            var cutoffTimestamp = cutoffTime.ToString("yyyyMMddHHmm", CultureInfo.InvariantCulture);
+
+            return await Task.Run(() =>
+            {
+                try
+                {
+                    return Directory.EnumerateDirectories(batchDirectory)
+                        .Where(dir =>
+                        {
+                            var directoryName = Path.GetFileName(dir);
+                            // Directory must start with yyyyMMddHHmm format and be older than cutoff
+                            return directoryName.Length >= 12 &&
+                                   directoryName.Substring(0, 12).All(char.IsDigit) &&
+                                   string.Compare(directoryName.Substring(0, 12), cutoffTimestamp, StringComparison.Ordinal) <= 0;
+                        })
+                        .OrderBy(Path.GetFileName)
+                        .ToList();
+                }
+                catch (Exception)
+                {
+                    // Return empty list if directory enumeration fails
+                    return new List<string>();
+                }
+            }, cancellationToken).ConfigureAwait(false);
+        }
+    }
+}

Projects/Dotmim.Sync.Core/IBatchCleanupService.cs

@@ -0,0 +1,65 @@
+using System;
+using System.Collections.Generic;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace Dotmim.Sync
+{
+    /// <summary>
+    /// Service for cleaning up orphaned batch directories based on retention policies.
+    /// Helps prevent disk space issues from failed sync sessions and network interruptions.
+    /// </summary>
+    public interface IBatchCleanupService
+    {
+
+        /// <summary>
+        /// Cleans up batch directories older than the specified retention period.
+        /// Parses timestamps from directory names to determine age.
+        /// </summary>
+        /// <param name="options">The sync options we can get the batchDirectory from</param>
+        /// <param name="cancellationToken">Cancellation token</param>
+        /// <returns>Number of directories cleaned up</returns>
+        Task<int> CleanupExpiredBatchesAsync(SyncOptions options, CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Cleans up batch directories older than the specified retention period.
+        /// Parses timestamps from directory names to determine age.
+        /// </summary>
+        /// <param name="options">The sync options we can get the batchDirectory from</param>
+        /// <param name="retentionPeriod">Allows to override the retention period configured in SyncOptionsBatchRetentionPeriod.  Age threshold for cleanup (directories older than this will be removed)</param>
+        /// <param name="cancellationToken">Cancellation token</param>
+        /// <returns>Number of directories cleaned up</returns>
+        Task<int> CleanupExpiredBatchesAsync(SyncOptions options, TimeSpan retentionPeriod, CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Cleans up batch directories older than the specified retention period.
+        /// Parses timestamps from directory names to determine age.
+        /// </summary>
+        /// <param name="batchDirectory">Root batch directory path</param>
+        /// <param name="retentionPeriod">Age threshold for cleanup (directories older than this will be removed)</param>
+        /// <param name="cancellationToken">Cancellation token</param>
+        /// <returns>Number of directories cleaned up</returns>
+        Task<int> CleanupExpiredBatchesAsync(string batchDirectory, TimeSpan retentionPeriod, CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Gets a list of batch directories that are eligible for cleanup based on retention period.
+        /// Does not perform actual cleanup, only identifies candidates.
+        /// </summary>
+        /// <param name="options">The sync options we can get the batchDirectory from</param>
+        /// <param name="retentionPeriod">Age threshold for cleanup eligibility</param>
+        /// <param name="cancellationToken">Cancellation token</param>
+        /// <returns>List of directory paths eligible for cleanup</returns>
+        Task<IList<string>> GetExpiredBatchDirectoriesAsync(SyncOptions options, TimeSpan retentionPeriod, CancellationToken cancellationToken = default);
+
+        /// <summary>
+        /// Gets a list of batch directories that are eligible for cleanup based on retention period.
+        /// Does not perform actual cleanup, only identifies candidates.
+        /// </summary>
+        /// <param name="batchDirectory">Root batch directory path</param>
+        /// <param name="retentionPeriod">Age threshold for cleanup eligibility</param>
+        /// <param name="cancellationToken">Cancellation token</param>
+        /// <returns>List of directory paths eligible for cleanup</returns>
+        Task<IList<string>> GetExpiredBatchDirectoriesAsync(string batchDirectory, TimeSpan retentionPeriod, CancellationToken cancellationToken = default);
+
+    }
+}

Projects/Dotmim.Sync.Core/Orchestrators/LocalOrchestrator.cs

@@ -48,6 +48,11 @@ public LocalOrchestrator(CoreProvider provider)
             if (provider == null)
                 throw this.GetSyncError(null, new MissingProviderException(nameof(LocalOrchestrator)));
         }
+
+        /// <summary>
+        /// Gets or sets the service that is used to cleanup expired batch files when the session is started
+        /// </summary>
+        public IBatchCleanupService BatchCleanupService { get; set; } = new BatchCleanupService();
 
         /// <summary>
         /// Called when a new synchronization session has started. Initialize the SyncContext instance, used for this session.
@@ -82,6 +87,10 @@ internal async Task<SyncContext> InternalBeginSessionAsync(SyncContext context,
 
             // Progress & interceptor
             await this.InterceptAsync(new SessionBeginArgs(context, connection), progress, cancellationToken).ConfigureAwait(false);
+
+            // cleanup batches
+            if (this.BatchCleanupService is { } svc)
+                await svc.CleanupExpiredBatchesAsync(this.Options, cancellationToken).ConfigureAwait(false);
 
             return context;
         }

Projects/Dotmim.Sync.Core/Orchestrators/RemoteOrchestrator.cs

@@ -32,6 +32,11 @@ public RemoteOrchestrator(CoreProvider provider)
                 throw this.GetSyncError(null, new UnsupportedServerProviderException(this.Provider.GetProviderTypeName()));
         }
 
+        /// <summary>
+        /// Gets or sets the service that is used to cleanup expired batch files when the session is started
+        /// </summary>
+        public IBatchCleanupService BatchCleanupService { get; set; } = new BatchCleanupService();
+
         /// <summary>
         /// Called when a new synchronization session has started. Initialize the SyncContext instance, used for this session.
         /// </summary>
@@ -65,6 +70,10 @@ internal virtual async Task<SyncContext> InternalBeginSessionAsync(SyncContext c
 
             // Progress & interceptor
             await this.InterceptAsync(new SessionBeginArgs(context, connection), progress, cancellationToken).ConfigureAwait(false);
+
+            // cleanup batches
+            if (this.BatchCleanupService is { } svc)
+                await svc.CleanupExpiredBatchesAsync(this.Options, cancellationToken).ConfigureAwait(false);
 
             return context;
         }

Projects/Dotmim.Sync.Core/SyncOptions.cs

@@ -104,6 +104,18 @@ public int BatchSize
         /// </summary>
         public TransactionMode TransactionMode { get; set; }
 
+        /// <summary>
+        /// Gets or sets the batch retention period before automatic cleanup.
+        /// Default: 1 hour. Set to TimeSpan.Zero to disable time-based cleanup.
+        /// </summary>
+        public TimeSpan BatchRetentionPeriod { get; set; }
+
+        /// <summary>
+        /// Gets or sets whether to perform immediate cleanup on session end.
+        /// If false, relies on time-based cleanup. Default: false (deferred cleanup).
+        /// </summary>
+        public bool BatchCleanupWhenSessionEnds { get; set; }
+
         /// <summary>
         /// Initializes a new instance of the <see cref="SyncOptions"/> class.
         /// Create a new instance of options with default values.
@@ -121,6 +133,7 @@ public SyncOptions()
             this.Logger = new SyncLogger().AddDebug();
             this.ProgressLevel = SyncProgressLevel.Information;
             this.TransactionMode = TransactionMode.AllOrNothing;
+            this.BatchRetentionPeriod = TimeSpan.FromHours(1);
         }
 
         /// <summary>

Projects/Dotmim.Sync.Web.Server/DependencyInjection.cs

@@ -86,8 +86,10 @@ public static IServiceCollection AddSyncServer(this IServiceCollection serviceCo
             setup = setup ?? throw new ArgumentNullException(nameof(setup));
             scopeName ??= SyncOptions.DefaultScopeName;
 
+            serviceCollection.AddSingleton<IBatchCleanupService, BatchCleanupService>();
+
             // Create orchestrator
-            serviceCollection.AddScoped(sp => new WebServerAgent(provider, setup, options, webServerOptions, scopeName, identifier));
+            serviceCollection.AddScoped(sp => new WebServerAgent(provider, setup, options, webServerOptions, scopeName, identifier, sp.GetRequiredService<IBatchCleanupService>()));
 
             return serviceCollection;
         }

Projects/Dotmim.Sync.Web.Server/WebServerAgent.cs

@@ -27,25 +27,34 @@ public class WebServerAgent
 
         /// <inheritdoc cref="WebServerAgent"/>
         public WebServerAgent(CoreProvider provider, SyncSetup setup, SyncOptions options = null, WebServerOptions webServerOptions = null,
-            string scopeName = null, string identifier = null)
+            string scopeName = null, 
+            string identifier = null, 
+            IBatchCleanupService cleanupService = null)
         {
             this.Setup = setup;
             this.WebServerOptions = webServerOptions ?? new WebServerOptions();
             this.Provider = provider;
             this.ScopeName = string.IsNullOrEmpty(scopeName) ? SyncOptions.DefaultScopeName : scopeName;
-            this.RemoteOrchestrator = new RemoteOrchestrator(this.Provider, options ?? new SyncOptions());
+            this.RemoteOrchestrator = new RemoteOrchestrator(this.Provider, options ?? new SyncOptions())
+            {
+                BatchCleanupService = cleanupService??new BatchCleanupService()
+            };
             this.Identifier = identifier;
         }
 
         /// <inheritdoc cref="WebServerAgent"/>
         public WebServerAgent(CoreProvider provider, string[] tables, SyncOptions options = null, WebServerOptions webServerOptions = null,
             string scopeName = null,
-            string identifier = null)
+            string identifier = null,
+            IBatchCleanupService cleanupService = null)
         {
             this.Setup = new SyncSetup(tables);
             this.WebServerOptions = webServerOptions ?? new WebServerOptions();
             this.Provider = provider;
-            this.RemoteOrchestrator = new RemoteOrchestrator(this.Provider, options ?? new SyncOptions());
+            this.RemoteOrchestrator = new RemoteOrchestrator(this.Provider, options ?? new SyncOptions())
+            {
+                BatchCleanupService = cleanupService??new BatchCleanupService()
+            };
             this.ScopeName = string.IsNullOrEmpty(scopeName) ? SyncOptions.DefaultScopeName : scopeName;
             this.Identifier = identifier;
         }