Added Retention

Author: zbalkan

Apps/QueryLogsDuckDBApp/App.cs

@@ -49,15 +49,12 @@ public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs
 
         [ThreadStatic]
         private static StringBuilder? _sb;
-        Config _config;
-
         private Channel<LogEntry>? _channel;
+        Config? _config;
         private DuckDBConnection? _conn;
         private Task? _consumerTask;
         private bool _disposed;
         private IDnsServer? _dnsServer;
-        private bool _enableLogging;
-        private int _maxQueueSize; // Maximum number of log entries in the queue, default 200,000
         #endregion variables
 
         #region IDisposable
@@ -248,6 +245,83 @@ private async Task ProcessLogsAsync()
                 BulkInsert(batch);
         }
 
+        private async Task RetentionLoopAsync()
+        {
+            // Initial delay
+            await Task.Delay(TimeSpan.FromMinutes(1));
+
+            while (!_disposed)
+            {
+                try
+                {
+                    await RunRetentionAsync();
+                }
+                catch (Exception ex)
+                {
+                    _dnsServer?.WriteLog(ex);
+                }
+
+                await Task.Delay(TimeSpan.FromMinutes(15));
+            }
+        }
+
+        private async Task RunRetentionAsync()
+        {
+            if (_conn is null || _config is null)
+                return;
+
+            using DuckDBCommand cmd = _conn.CreateCommand();
+
+            long deleted = 0;
+
+            /* ---------------------------------
+               Max records
+               --------------------------------- */
+
+            if (_config.MaxLogRecords > 0)
+            {
+                cmd.CommandText = @"
+DELETE FROM dns_logs
+WHERE timestamp NOT IN (
+    SELECT timestamp
+    FROM dns_logs
+    ORDER BY timestamp DESC
+    LIMIT $limit
+);";
+
+                cmd.Parameters.Clear();
+                cmd.Parameters.Add(
+                    new DuckDBParameter("limit", _config.MaxLogRecords));
+
+                deleted += await cmd.ExecuteNonQueryAsync();
+            }
+
+            /* ---------------------------------
+               Max days
+               --------------------------------- */
+
+            if (_config.MaxLogDays > 0)
+            {
+                DateTime cutoff =
+                    DateTime.UtcNow.AddDays(-_config.MaxLogDays);
+
+                cmd.CommandText =
+                    "DELETE FROM dns_logs WHERE timestamp < $cutoff;";
+
+                cmd.Parameters.Clear();
+                cmd.Parameters.Add(
+                    new DuckDBParameter("cutoff", cutoff));
+
+                deleted += await cmd.ExecuteNonQueryAsync();
+            }
+
+            if (deleted > 0)
+            {
+                cmd.Parameters.Clear();
+                cmd.CommandText = "CHECKPOINT;";
+                await cmd.ExecuteNonQueryAsync();
+            }
+        }
         #endregion private
 
         #region public
@@ -258,6 +332,7 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config)
 
             JsonSerializerOptions options = new JsonSerializerOptions { PropertyNameCaseInsensitive = true };
             _config = JsonSerializer.Deserialize<Config>(config, options);
+            _config ??= new Config();
             Validator.ValidateObject(_config, new ValidationContext(_config), validateAllProperties: true);
 
             if (!System.IO.Path.IsPathRooted(_config.DbPath))
@@ -276,6 +351,7 @@ public async Task InitializeAsync(IDnsServer dnsServer, string config)
             await CreateSchemaAsync();
 
             _consumerTask = Task.Run(ProcessLogsAsync);
+            _ = Task.Run(RetentionLoopAsync);
         }
 
         public Task InsertLogAsync(
@@ -285,7 +361,7 @@ public Task InsertLogAsync(
             DnsTransportProtocol protocol,
             DnsDatagram response)
         {
-            if (_config.EnableLogging)
+            if (_config!.EnableLogging)
                 _channel!.Writer.TryWrite(
                     new LogEntry(timestamp, request, remoteEP, protocol, response));
 
@@ -588,11 +664,16 @@ public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, Dn
 
         private class Config
         {
+            [JsonPropertyName("dbPath")]
+            public string DbPath { get; set; } = "querylogs.db";
+
             [JsonPropertyName("enableLogging")]
             public bool EnableLogging { get; set; } = true;
+            [JsonPropertyName("maxLogDays")]
+            public int MaxLogDays { get; set; } = 30;
 
-            [JsonPropertyName("dbPath")]
-            public string DbPath { get; set; } = "querylogs.db";
+            [JsonPropertyName("maxLogRecords")]
+            public long MaxLogRecords { get; set; } = 1_000_000;
 
             [JsonPropertyName("maxQueueSize")]
             public int MaxQueueSize { get; set; } = 200_000;

Apps/QueryLogsDuckDBApp/dnsApp.config

@@ -1,5 +1,7 @@
 {
-  "enableLogging": true,
-  "dbPath": "querylogs.db",
-  "maxQueueSize": 200000
+	"enableLogging": true,
+	"dbPath": "querylogs.db",
+	"maxQueueSize": 200000,
+	"maxLogDays:" 30,
+	"maxLogRecords": 1000000
 }