feat: add database migration management with versioned schemas (Loop 38)

- Create versioned migration SQL files for SQLite (3 migrations) and
  PostgreSQL (1 migration) in crates/llmtrace-storage/migrations/
- Implement migration runner (migration.rs) with schema_version tracking
  table, transactional application, and idempotent re-runs
- Embed migration SQL at compile time via include_str! — no runtime
  filesystem access needed
- Replace inline SQL constants in sqlite.rs and postgres.rs with calls
  to the versioned migration runner
- Add 'migrate' CLI subcommand: llmtrace-proxy migrate --config config.yaml
- Add storage.auto_migrate config option (default true for dev)
- Add 7 new tests: migration ordering, idempotency, version tracking,
  table existence, partial resume
- All existing tests continue to pass unchanged

Author: Jenkins

config.example.yaml

@@ -56,6 +56,11 @@ storage:
   # Redis connection URL for hot-query cache and sessions.
   # redis_url: "redis://127.0.0.1:6379"
 
+  # Automatically run pending database migrations on startup.
+  # Default: true (convenient for development). Set to false in production
+  # and use `llmtrace-proxy migrate --config config.yaml` explicitly.
+  auto_migrate: true
+
 # ---------------------------------------------------------------------------
 # Logging
 # ---------------------------------------------------------------------------

crates/llmtrace-core/src/lib.rs

@@ -916,6 +916,12 @@ pub struct StorageConfig {
     /// Redis connection URL (used by the `"production"` profile).
     #[serde(default)]
     pub redis_url: Option<String>,
+    /// Automatically run pending database migrations on startup.
+    ///
+    /// Defaults to `true` for development (lite/memory profiles). Set to
+    /// `false` in production to require explicit `llmtrace-proxy migrate`.
+    #[serde(default = "default_auto_migrate")]
+    pub auto_migrate: bool,
 }
 
 fn default_storage_profile() -> String {
@@ -926,6 +932,10 @@ fn default_database_path() -> String {
     "llmtrace.db".to_string()
 }
 
+fn default_auto_migrate() -> bool {
+    true
+}
+
 impl Default for StorageConfig {
     fn default() -> Self {
         Self {
@@ -935,6 +945,7 @@ impl Default for StorageConfig {
             clickhouse_database: None,
             postgres_url: None,
             redis_url: None,
+            auto_migrate: default_auto_migrate(),
         }
     }
 }
@@ -2391,6 +2402,7 @@ mod tests {
                 clickhouse_database: None,
                 postgres_url: None,
                 redis_url: None,
+                auto_migrate: true,
             },
             timeout_ms: 60000,
             connection_timeout_ms: 10000,

crates/llmtrace-proxy/src/main.rs

@@ -53,6 +53,8 @@ struct Cli {
 enum Commands {
     /// Validate a configuration file and print resolved settings.
     Validate,
+    /// Run pending database migrations and exit.
+    Migrate,
 }
 
 // ---------------------------------------------------------------------------
@@ -68,6 +70,10 @@ async fn main() -> anyhow::Result<()> {
 
     match cli.command {
         Some(Commands::Validate) => run_validate(&config),
+        Some(Commands::Migrate) => {
+            init_logging(&config)?;
+            run_migrate(&config).await
+        }
         None => {
             init_logging(&config)?;
             config::validate_config(&config)?;
@@ -127,6 +133,46 @@ fn run_validate(config: &ProxyConfig) -> anyhow::Result<()> {
     Ok(())
 }
 
+// ---------------------------------------------------------------------------
+// Subcommand: migrate
+// ---------------------------------------------------------------------------
+
+/// Run pending database migrations and exit.
+async fn run_migrate(config: &ProxyConfig) -> anyhow::Result<()> {
+    info!(
+        profile = %config.storage.profile,
+        "Running database migrations"
+    );
+
+    match config.storage.profile.as_str() {
+        "lite" => {
+            let url = format!("sqlite:{}", config.storage.database_path);
+            let pool = llmtrace_storage::migration::open_sqlite_pool(&url).await?;
+            llmtrace_storage::migration::run_sqlite_migrations(&pool).await?;
+            info!("SQLite migrations complete");
+        }
+        "memory" => {
+            info!("Memory profile uses no persistent storage — nothing to migrate");
+        }
+        "production" => {
+            // Run PostgreSQL migrations for production metadata storage.
+            if let Some(ref pg_url) = config.storage.postgres_url {
+                let pool = llmtrace_storage::migration::open_pg_pool(pg_url).await?;
+                llmtrace_storage::migration::run_pg_migrations(&pool).await?;
+                info!("PostgreSQL migrations complete");
+            } else {
+                anyhow::bail!("storage.postgres_url is required for production profile migrations");
+            }
+        }
+        other => {
+            anyhow::bail!("Unknown storage profile: {other}");
+        }
+    }
+
+    println!("✓ All migrations applied successfully.");
+    Ok(())
+}
+
 // ---------------------------------------------------------------------------
 // Subcommand: run proxy (default)
 // ---------------------------------------------------------------------------

crates/llmtrace-storage/Cargo.toml

@@ -21,6 +21,7 @@ serde_json.workspace = true
 async-trait.workspace = true
 chrono.workspace = true
 uuid.workspace = true
+tracing.workspace = true
 sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "sqlite", "chrono", "uuid"] }
 dashmap = "6"
 bytes = "1"

crates/llmtrace-storage/migrations/postgres/001_initial_schema.sql

@@ -0,0 +1,42 @@
+-- 001_initial_schema.sql: tenants, tenant_configs, audit_events, api_keys for PostgreSQL
+
+CREATE TABLE IF NOT EXISTS tenants (
+    id UUID PRIMARY KEY,
+    name VARCHAR(255) NOT NULL,
+    plan VARCHAR(50) NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    config JSONB NOT NULL DEFAULT '{}'
+);
+
+CREATE TABLE IF NOT EXISTS tenant_configs (
+    tenant_id UUID PRIMARY KEY REFERENCES tenants(id),
+    security_thresholds JSONB NOT NULL DEFAULT '{}',
+    feature_flags JSONB NOT NULL DEFAULT '{}'
+);
+
+CREATE TABLE IF NOT EXISTS audit_events (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    tenant_id UUID NOT NULL REFERENCES tenants(id),
+    event_type VARCHAR(100) NOT NULL,
+    actor VARCHAR(255) NOT NULL,
+    resource VARCHAR(255) NOT NULL,
+    data JSONB NOT NULL DEFAULT '{}',
+    timestamp TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_audit_tenant_time ON audit_events(tenant_id, timestamp DESC);
+CREATE INDEX IF NOT EXISTS idx_audit_type ON audit_events(tenant_id, event_type);
+
+CREATE TABLE IF NOT EXISTS api_keys (
+    id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+    tenant_id UUID NOT NULL REFERENCES tenants(id),
+    name VARCHAR(255) NOT NULL,
+    key_hash VARCHAR(64) NOT NULL UNIQUE,
+    key_prefix VARCHAR(16) NOT NULL,
+    role VARCHAR(20) NOT NULL,
+    created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    revoked_at TIMESTAMPTZ
+);
+
+CREATE INDEX IF NOT EXISTS idx_api_keys_hash ON api_keys(key_hash);
+CREATE INDEX IF NOT EXISTS idx_api_keys_tenant ON api_keys(tenant_id);

crates/llmtrace-storage/migrations/sqlite/001_initial_schema.sql

@@ -0,0 +1,44 @@
+-- 001_initial_schema.sql: traces and spans tables for SQLite
+
+CREATE TABLE IF NOT EXISTS traces (
+    trace_id TEXT NOT NULL,
+    tenant_id TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    PRIMARY KEY (tenant_id, trace_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_traces_created ON traces(tenant_id, created_at);
+
+CREATE TABLE IF NOT EXISTS spans (
+    span_id TEXT NOT NULL,
+    trace_id TEXT NOT NULL,
+    parent_span_id TEXT,
+    tenant_id TEXT NOT NULL,
+    operation_name TEXT NOT NULL,
+    start_time TEXT NOT NULL,
+    end_time TEXT,
+    provider TEXT NOT NULL,
+    model_name TEXT NOT NULL,
+    prompt TEXT NOT NULL,
+    response TEXT,
+    prompt_tokens INTEGER,
+    completion_tokens INTEGER,
+    total_tokens INTEGER,
+    time_to_first_token_ms INTEGER,
+    duration_ms INTEGER,
+    status_code INTEGER,
+    error_message TEXT,
+    estimated_cost_usd REAL,
+    security_score INTEGER,
+    security_findings TEXT NOT NULL DEFAULT '[]',
+    tags TEXT NOT NULL DEFAULT '{}',
+    events TEXT NOT NULL DEFAULT '[]',
+    PRIMARY KEY (tenant_id, span_id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_spans_trace ON spans(tenant_id, trace_id);
+CREATE INDEX IF NOT EXISTS idx_spans_time ON spans(tenant_id, start_time);
+CREATE INDEX IF NOT EXISTS idx_spans_provider ON spans(tenant_id, provider);
+CREATE INDEX IF NOT EXISTS idx_spans_model ON spans(tenant_id, model_name);
+CREATE INDEX IF NOT EXISTS idx_spans_security ON spans(tenant_id, security_score);
+CREATE INDEX IF NOT EXISTS idx_spans_operation ON spans(tenant_id, operation_name);

crates/llmtrace-storage/migrations/sqlite/002_metadata_tables.sql

@@ -0,0 +1,42 @@
+-- 002_metadata_tables.sql: tenants, tenant_configs, audit_events, api_keys
+
+CREATE TABLE IF NOT EXISTS tenants (
+    id TEXT NOT NULL PRIMARY KEY,
+    name TEXT NOT NULL,
+    plan TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    config TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE TABLE IF NOT EXISTS tenant_configs (
+    tenant_id TEXT NOT NULL PRIMARY KEY,
+    security_thresholds TEXT NOT NULL DEFAULT '{}',
+    feature_flags TEXT NOT NULL DEFAULT '{}'
+);
+
+CREATE TABLE IF NOT EXISTS audit_events (
+    id TEXT NOT NULL PRIMARY KEY,
+    tenant_id TEXT NOT NULL,
+    event_type TEXT NOT NULL,
+    actor TEXT NOT NULL,
+    resource TEXT NOT NULL,
+    data TEXT NOT NULL DEFAULT '{}',
+    timestamp TEXT NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS idx_audit_tenant ON audit_events(tenant_id, timestamp);
+CREATE INDEX IF NOT EXISTS idx_audit_type ON audit_events(tenant_id, event_type);
+
+CREATE TABLE IF NOT EXISTS api_keys (
+    id TEXT NOT NULL PRIMARY KEY,
+    tenant_id TEXT NOT NULL,
+    name TEXT NOT NULL,
+    key_hash TEXT NOT NULL UNIQUE,
+    key_prefix TEXT NOT NULL,
+    role TEXT NOT NULL,
+    created_at TEXT NOT NULL,
+    revoked_at TEXT
+);
+
+CREATE INDEX IF NOT EXISTS idx_api_keys_hash ON api_keys(key_hash);
+CREATE INDEX IF NOT EXISTS idx_api_keys_tenant ON api_keys(tenant_id);

crates/llmtrace-storage/migrations/sqlite/003_add_agent_actions.sql

@@ -0,0 +1,3 @@
+-- 003_add_agent_actions.sql: agent_actions column on spans (Loop 18)
+
+ALTER TABLE spans ADD COLUMN agent_actions TEXT NOT NULL DEFAULT '[]';

crates/llmtrace-storage/src/lib.rs

@@ -8,6 +8,7 @@
 
 mod cache;
 mod memory;
+pub mod migration;
 mod sqlite;
 
 #[cfg(feature = "clickhouse")]