feat: configurable data retention for usage and audit

internal/app/app.go

@@ -41,6 +41,7 @@ import (
 	"github.com/voidmind-io/voidllm/internal/proxy"
 	"github.com/voidmind-io/voidllm/internal/ratelimit"
 	voidredis "github.com/voidmind-io/voidllm/internal/redis"
+	"github.com/voidmind-io/voidllm/internal/retention"
 	"github.com/voidmind-io/voidllm/internal/router"
 	"github.com/voidmind-io/voidllm/internal/shutdown"
 	"github.com/voidmind-io/voidllm/internal/sso"
@@ -79,6 +80,7 @@ type Application struct {
 	usageLogger      *usage.Logger
 	mcpLogger        *usage.MCPLogger
 	auditLogger      *audit.Logger
+	retentionCleaner *retention.Cleaner
 	healthChecker    *health.Checker
 	mcpHealthChecker *health.MCPHealthChecker
 
@@ -183,10 +185,11 @@ func New(cfg *config.Config, log *slog.Logger, devMode bool) (*Application, erro
 	// Declare variables that the deferred cleanup needs to reference before
 	// they are assigned by the steps below.
 	var (
-		encKey      []byte
-		hmacSecret  []byte
-		usageLogger *usage.Logger
-		auditLogger *audit.Logger
+		encKey           []byte
+		hmacSecret       []byte
+		usageLogger      *usage.Logger
+		auditLogger      *audit.Logger
+		retentionCleaner *retention.Cleaner
 	)
 
 	// From this point on, any early return must clean up in reverse order.
@@ -197,6 +200,9 @@ func New(cfg *config.Config, log *slog.Logger, devMode bool) (*Application, erro
 		if success {
 			return
 		}
+		if retentionCleaner != nil {
+			retentionCleaner.Stop()
+		}
 		if usageLogger != nil {
 			usageLogger.Stop()
 		}
@@ -405,6 +411,9 @@ func New(cfg *config.Config, log *slog.Logger, devMode bool) (*Application, erro
 		log.LogAttrs(ctx, slog.LevelInfo, "audit logging enabled")
 	}
 
+	retentionCleaner = retention.New(database, cfg.Settings.Retention, log)
+	retentionCleaner.Start()
+
 	// Step 9: load model access cache and alias cache from DB.
 	accessCache := proxy.NewModelAccessCache()
 	orgA, teamA, keyA, err := database.LoadAllModelAccess(ctx)
@@ -984,6 +993,7 @@ func New(cfg *config.Config, log *slog.Logger, devMode bool) (*Application, erro
 		usageLogger:       usageLogger,
 		mcpLogger:         mcpLogger,
 		auditLogger:       auditLogger,
+		retentionCleaner:  retentionCleaner,
 		healthChecker:     healthChecker,
 		mcpHealthChecker:  mcpHealthChecker,
 		shutdownState:     shutdownState,
@@ -1065,6 +1075,10 @@ func (a *Application) Start() error {
 		}),
 	)
 
+	// Register retention cleaner stop so it is halted during graceful shutdown.
+	// LIFO ordering ensures retention stops before the usage and audit loggers.
+	a.stopFuncs = append(a.stopFuncs, a.retentionCleaner.Stop)
+
 	// The in-memory rate limiter accumulates counter entries that must be
 	// periodically evicted to reclaim memory. The Redis-backed checker uses
 	// TTL-keyed counters that self-expire, so no eviction goroutine is needed.

internal/config/config.go

@@ -415,6 +415,25 @@ type HealthProbeConfig struct {
 	Interval time.Duration `yaml:"interval"`
 }
 
+// RetentionConfig controls periodic deletion of old usage and audit records.
+// Retention is opt-in: a zero duration means "keep forever".
+type RetentionConfig struct {
+	// UsageEvents is the maximum age of rows in the usage_events table.
+	// A zero value means rows are kept forever.
+	UsageEvents time.Duration `yaml:"usage_events" json:"usage_events"`
+	// AuditLogs is the maximum age of rows in the audit_logs table.
+	// A zero value means rows are kept forever.
+	AuditLogs time.Duration `yaml:"audit_logs" json:"audit_logs"`
+	// Interval controls how often the cleanup job runs.
+	// Defaults to 24h when retention is enabled.
+	Interval time.Duration `yaml:"interval" json:"interval"`
+}
+
+// Enabled reports whether any retention job is active.
+func (r RetentionConfig) Enabled() bool {
+	return r.UsageEvents > 0 || r.AuditLogs > 0
+}
+
 // SettingsConfig holds application-level settings.
 type SettingsConfig struct {
 	AdminKey      string `yaml:"admin_key" json:"-"`
@@ -434,6 +453,7 @@ type SettingsConfig struct {
 	CircuitBreaker CircuitBreakerConfig `yaml:"circuit_breaker"`
 	HealthCheck    HealthCheckConfig    `yaml:"health_check"`
 	MCP            MCPConfig            `yaml:"mcp"`
+	Retention      RetentionConfig      `yaml:"retention"`
 	// SoftLimitThreshold uses *float64 so that an explicit 0.0 can be
 	// distinguished from the zero value after unmarshalling. Use
 	// GetSoftLimitThreshold to read the value.
@@ -657,6 +677,11 @@ func (c *Config) setDefaults() {
 		c.Settings.Audit.FlushInterval = 5 * time.Second
 	}
 
+	// Settings retention
+	if c.Settings.Retention.Interval <= 0 {
+		c.Settings.Retention.Interval = 24 * time.Hour
+	}
+
 	// Bootstrap
 	if c.Settings.Bootstrap.OrgName == "" {
 		c.Settings.Bootstrap.OrgName = "Default"

internal/config/config_test.go

@@ -706,6 +706,161 @@ models:
 ` + validModel,
 			wantErr: false,
 		},
+		{
+			name: "retention usage_events negative error",
+			yaml: `
+server:
+  proxy:
+    port: 8080
+database:
+  driver: sqlite
+  dsn: voidllm.db
+settings:
+  encryption_key: key
+  usage:
+    buffer_size: 100
+  retention:
+    usage_events: -1h
+`,
+			wantErr:     true,
+			errContains: "settings.retention.usage_events must be >= 0",
+		},
+		{
+			name: "retention audit_logs negative error",
+			yaml: `
+server:
+  proxy:
+    port: 8080
+database:
+  driver: sqlite
+  dsn: voidllm.db
+settings:
+  encryption_key: key
+  usage:
+    buffer_size: 100
+  retention:
+    audit_logs: -1h
+`,
+			wantErr:     true,
+			errContains: "settings.retention.audit_logs must be >= 0",
+		},
+		{
+			name: "retention usage_events exceeds 10 years error",
+			yaml: `
+server:
+  proxy:
+    port: 8080
+database:
+  driver: sqlite
+  dsn: voidllm.db
+settings:
+  encryption_key: key
+  usage:
+    buffer_size: 100
+  retention:
+    usage_events: 87660h
+`,
+			wantErr:     true,
+			errContains: "settings.retention.usage_events exceeds maximum",
+		},
+		{
+			name: "retention audit_logs exceeds 10 years error",
+			yaml: `
+server:
+  proxy:
+    port: 8080
+database:
+  driver: sqlite
+  dsn: voidllm.db
+settings:
+  encryption_key: key
+  usage:
+    buffer_size: 100
+  retention:
+    audit_logs: 87660h
+`,
+			wantErr:     true,
+			errContains: "settings.retention.audit_logs exceeds maximum",
+		},
+		{
+			name: "retention interval below minimum when enabled error",
+			yaml: `
+server:
+  proxy:
+    port: 8080
+database:
+  driver: sqlite
+  dsn: voidllm.db
+settings:
+  encryption_key: key
+  usage:
+    buffer_size: 100
+  retention:
+    usage_events: 24h
+    interval: 30s
+`,
+			wantErr:     true,
+			errContains: "settings.retention.interval must be >=",
+		},
+		{
+			name: "retention interval irrelevant when disabled no error",
+			yaml: `
+server:
+  proxy:
+    port: 8080
+database:
+  driver: sqlite
+  dsn: voidllm.db
+settings:
+  encryption_key: key
+  usage:
+    buffer_size: 100
+  retention:
+    usage_events: 0s
+    audit_logs: 0s
+    interval: 1ms
+`,
+			wantErr: false,
+		},
+		{
+			name: "retention all zeros no error",
+			yaml: `
+server:
+  proxy:
+    port: 8080
+database:
+  driver: sqlite
+  dsn: voidllm.db
+settings:
+  encryption_key: key
+  usage:
+    buffer_size: 100
+  retention:
+    usage_events: 0s
+    audit_logs: 0s
+`,
+			wantErr: false,
+		},
+		{
+			name: "retention valid configuration no error",
+			yaml: `
+server:
+  proxy:
+    port: 8080
+database:
+  driver: sqlite
+  dsn: voidllm.db
+settings:
+  encryption_key: key
+  usage:
+    buffer_size: 100
+  retention:
+    usage_events: 720h
+    audit_logs: 2160h
+    interval: 24h
+`,
+			wantErr: false,
+		},
 	}
 
 	for _, tc := range tests {

internal/config/validate.go

@@ -308,6 +308,25 @@ func (c *Config) validate() error {
 		errs = append(errs, fmt.Errorf("settings.soft_limit_threshold: must be between 0.0 and 1.0, got %g", t))
 	}
 
+	// --- settings.retention ---
+	const maxRetention = 10 * 365 * 24 * time.Hour // 10 years
+	if c.Settings.Retention.UsageEvents < 0 {
+		errs = append(errs, errors.New("settings.retention.usage_events must be >= 0"))
+	}
+	if c.Settings.Retention.AuditLogs < 0 {
+		errs = append(errs, errors.New("settings.retention.audit_logs must be >= 0"))
+	}
+	if c.Settings.Retention.UsageEvents > maxRetention {
+		errs = append(errs, errors.New("settings.retention.usage_events exceeds maximum (10 years)"))
+	}
+	if c.Settings.Retention.AuditLogs > maxRetention {
+		errs = append(errs, errors.New("settings.retention.audit_logs exceeds maximum (10 years)"))
+	}
+	const minRetentionInterval = 1 * time.Minute
+	if c.Settings.Retention.Enabled() && c.Settings.Retention.Interval < minRetentionInterval {
+		errs = append(errs, fmt.Errorf("settings.retention.interval must be >= %s when retention is enabled", minRetentionInterval))
+	}
+
 	// --- settings.sso.default_role ---
 	if c.Settings.SSO.Enabled {
 		switch c.Settings.SSO.DefaultRole {

internal/db/dialect.go

@@ -13,6 +13,15 @@ type Dialect interface {
 	// SupportsMigrationLock reports whether the dialect supports advisory locking
 	// during schema migrations. PostgreSQL supports pg_advisory_lock; SQLite does not.
 	SupportsMigrationLock() bool
+	// TimestampLessThan returns a SQL expression (without parameter binding) that
+	// compares the named TEXT-typed timestamp column to a parameter, handling
+	// format differences across drivers. The caller supplies the column name and
+	// placeholder string; the dialect wraps both sides in a driver-appropriate
+	// cast so string-level comparison is never used.
+	//
+	// Example (SQLite):   datetime(created_at) < datetime(?)
+	// Example (Postgres): created_at::timestamptz < ($1)::timestamptz
+	TimestampLessThan(column, placeholder string) string
 }
 
 // SQLiteDialect implements Dialect for SQLite.
@@ -31,6 +40,13 @@ func (SQLiteDialect) HourTrunc() string {
 // locks. SQLite's single-writer model makes migration locking unnecessary.
 func (SQLiteDialect) SupportsMigrationLock() bool { return false }
 
+// TimestampLessThan wraps both sides in datetime() which parses TEXT into a
+// comparable form. datetime() accepts ISO-8601 with or without subseconds and
+// both the "2006-01-02 15:04:05" and "2006-01-02T15:04:05Z" variants.
+func (SQLiteDialect) TimestampLessThan(column, placeholder string) string {
+	return "datetime(" + column + ") < datetime(" + placeholder + ")"
+}
+
 // PostgresDialect implements Dialect for PostgreSQL.
 type PostgresDialect struct{}
 
@@ -49,3 +65,9 @@ func (PostgresDialect) HourTrunc() string {
 // via pg_advisory_lock, which prevents concurrent migration runs in multi-replica
 // deployments.
 func (PostgresDialect) SupportsMigrationLock() bool { return true }
+
+// TimestampLessThan casts both sides to timestamptz which parses any reasonable
+// ISO-8601 variant and compares as absolute instants.
+func (PostgresDialect) TimestampLessThan(column, placeholder string) string {
+	return column + "::timestamptz < (" + placeholder + ")::timestamptz"
+}