feat: enable Delta Lake catalog attachment by default (#553)

* feat: enable Delta Lake catalog attachment by default

Flips delta_catalog_enabled to true everywhere: CLI/env/YAML defaults
in DefaultServerConfig and the duckgres-worker flag, plus column-level
gorm:"default:true" on the configstore ManagedWarehouseS3 and
DuckLakeConfig fields. A one-shot migration tracked in
duckgres_schema_migrations backfills existing configstore rows so
upgraded clusters pick up the new default without admin intervention;
admins can still opt out per-warehouse via the admin API afterward
without being re-flipped on subsequent restarts.

To keep plain standalone deployments (no DuckLake metadata_store /
object_store / data_path) bootable under the new default,
AttachDeltaCatalog now silently no-ops when no catalog path is
derivable instead of erroring. The catalog is still attached as
'delta' (ATTACH '...' AS delta (TYPE delta), unchanged).

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fix(server): treat empty Delta location as benign skip

Under the always-on Delta default, a fresh DuckLake tenant has no
Delta data at the sibling delta/ prefix yet, and ATTACH returns
"No files in log segment" from DeltaKernel. That's the expected
state, not a failure — log it and continue without the catalog
attached. Fixes the k8s tenant isolation test where billing's
worker activation rejected client queries before the tenant ever
wrote Delta data.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

* fix(server): probe-and-detach when Delta path has no transaction log

ATTACH '...' (TYPE delta) is lazy — it succeeds against an empty path,
then every subsequent query against DuckLake fails at prepare time
because the planner walks all attached catalogs and Delta blows up
trying to read a missing _delta_log/. Earlier the empty-path check
was only at ATTACH time; that wasn't enough because ATTACH never
errored. Now we probe with SHOW TABLES FROM delta immediately after
ATTACH and DETACH if the catalog is unusable, leaving the connection
clean for ducklake-only queries until Delta data lands.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: fuziontech

cmd/duckgres-worker/main.go

@@ -75,7 +75,7 @@ func main() {
 	sessionInitTimeout := flag.String("session-init-timeout", "", "Session startup metadata/probe timeout (e.g., '10s', '30s') (env: DUCKGRES_SESSION_INIT_TIMEOUT)")
 
 	// DuckLake (workers attach DuckLake)
-	duckLakeDeltaCatalogEnabled := flag.Bool("ducklake-delta-catalog-enabled", false, "Attach a Delta Lake catalog during DuckLake worker boot (env: DUCKGRES_DUCKLAKE_DELTA_CATALOG_ENABLED)")
+	duckLakeDeltaCatalogEnabled := flag.Bool("ducklake-delta-catalog-enabled", true, "Attach a Delta Lake catalog during DuckLake worker boot (default true; use --ducklake-delta-catalog-enabled=false to disable; env: DUCKGRES_DUCKLAKE_DELTA_CATALOG_ENABLED)")
 	duckLakeDeltaCatalogPath := flag.String("ducklake-delta-catalog-path", "", "Delta Lake catalog/table path to attach (env: DUCKGRES_DUCKLAKE_DELTA_CATALOG_PATH)")
 	duckLakeDefaultSpecVersion := flag.String("ducklake-default-spec-version", "", "Default DuckLake spec version for migration checks (env: DUCKGRES_DUCKLAKE_DEFAULT_SPEC_VERSION)")
 

configresolve/cliflags.go

@@ -40,7 +40,7 @@ func RegisterCLIInputsFlags(fs *flag.FlagSet) func() CLIInputs {
 	threads := fs.Int("threads", 0, "DuckDB threads per session (env: DUCKGRES_THREADS)")
 	memoryBudget := fs.String("memory-budget", "", "Total memory for all DuckDB sessions (e.g., '24GB') (env: DUCKGRES_MEMORY_BUDGET)")
 	memoryRebalance := fs.Bool("memory-rebalance", false, "Enable dynamic per-connection memory reallocation (control-plane mode) (env: DUCKGRES_MEMORY_REBALANCE)")
-	duckLakeDeltaCatalogEnabled := fs.Bool("ducklake-delta-catalog-enabled", false, "Attach a Delta Lake catalog during DuckLake worker boot (env: DUCKGRES_DUCKLAKE_DELTA_CATALOG_ENABLED)")
+	duckLakeDeltaCatalogEnabled := fs.Bool("ducklake-delta-catalog-enabled", true, "Attach a Delta Lake catalog during DuckLake worker boot (default true; use --ducklake-delta-catalog-enabled=false to disable; env: DUCKGRES_DUCKLAKE_DELTA_CATALOG_ENABLED)")
 	duckLakeDeltaCatalogPath := fs.String("ducklake-delta-catalog-path", "", "Delta Lake catalog/table path to attach, defaults to sibling delta/ prefix at DuckLake object-store root (env: DUCKGRES_DUCKLAKE_DELTA_CATALOG_PATH)")
 	duckLakeDefaultSpecVersion := fs.String("ducklake-default-spec-version", "", "Default DuckLake spec version for migration checks (env: DUCKGRES_DUCKLAKE_DEFAULT_SPEC_VERSION)")
 	processMinWorkers := fs.Int("process-min-workers", 0, "Pre-warm worker count at startup for process workers (control-plane mode) (env: DUCKGRES_PROCESS_MIN_WORKERS)")

configresolve/resolve.go

@@ -145,6 +145,7 @@ func DefaultServerConfig() server.Config {
 			CheckpointInterval:              24 * time.Hour,
 			DataInliningRowLimit:            intPtr(0),
 			DisableMetadataThreadLocalCache: boolPtr(true),
+			DeltaCatalogEnabled:             true,
 		},
 		QueryLog: server.QueryLogConfig{
 			Enabled:              true,
@@ -1091,9 +1092,6 @@ func ResolveEffective(fileCfg *configloader.FileConfig, cli CLIInputs, getenv fu
 	}
 	if cfg.DuckLake.DeltaCatalogEnabled && cfg.DuckLake.DeltaCatalogPath == "" {
 		cfg.DuckLake.DeltaCatalogPath = ducklake.DefaultDeltaCatalogPath(cfg.DuckLake)
-		if cfg.DuckLake.DeltaCatalogPath == "" {
-			warn("ducklake.delta_catalog_enabled requires ducklake.delta_catalog_path when no ducklake.object_store or ducklake.data_path is configured")
-		}
 	}
 
 	return Resolved{

controlplane/configstore/models.go

@@ -104,7 +104,7 @@ type ManagedWarehouseS3 struct {
 	Endpoint            string `gorm:"size:512" json:"endpoint"`
 	UseSSL              bool   `json:"use_ssl"`
 	URLStyle            string `gorm:"size:16" json:"url_style"`
-	DeltaCatalogEnabled bool   `json:"delta_catalog_enabled"`
+	DeltaCatalogEnabled bool   `gorm:"default:true" json:"delta_catalog_enabled"`
 	DeltaCatalogPath    string `gorm:"size:1024" json:"delta_catalog_path"`
 }
 
@@ -187,7 +187,7 @@ type DuckLakeConfig struct {
 	S3URLStyle          string    `gorm:"size:16" json:"s3_url_style"`
 	S3Chain             string    `gorm:"size:255" json:"s3_chain"`
 	S3Profile           string    `gorm:"size:255" json:"s3_profile"`
-	DeltaCatalogEnabled bool      `json:"delta_catalog_enabled"`
+	DeltaCatalogEnabled bool      `gorm:"default:true" json:"delta_catalog_enabled"`
 	DeltaCatalogPath    string    `gorm:"size:1024" json:"delta_catalog_path"`
 	UpdatedAt           time.Time `json:"updated_at"`
 }
@@ -219,6 +219,16 @@ type QueryLogConfig struct {
 
 func (QueryLogConfig) TableName() string { return "duckgres_query_log_config" }
 
+// SchemaMigration tracks one-shot data migrations that aren't expressible
+// through GORM's AutoMigrate (e.g., backfills of new column defaults onto
+// existing rows). One row per migration name, inserted exactly once.
+type SchemaMigration struct {
+	Name      string    `gorm:"primaryKey;size:128" json:"name"`
+	AppliedAt time.Time `json:"applied_at"`
+}
+
+func (SchemaMigration) TableName() string { return "duckgres_schema_migrations" }
+
 // ControlPlaneInstanceState describes the liveness state of a control-plane instance.
 type ControlPlaneInstanceState string
 

controlplane/configstore/store.go

@@ -79,10 +79,16 @@ func NewConfigStore(connStr string, pollInterval time.Duration) (*ConfigStore, e
 		&DuckLakeConfig{},
 		&RateLimitConfig{},
 		&QueryLogConfig{},
+		&SchemaMigration{},
 	); err != nil {
 		return nil, fmt.Errorf("auto-migrate config store: %w", err)
 	}
 
+	// One-shot data migrations (idempotent — tracked in duckgres_schema_migrations).
+	if err := migrateDeltaCatalogDefaultEnabled(db); err != nil {
+		return nil, fmt.Errorf("migrate delta catalog default: %w", err)
+	}
+
 	runtimeSchema, err := resolveRuntimeSchema(db)
 	if err != nil {
 		return nil, fmt.Errorf("resolve runtime schema: %w", err)
@@ -415,6 +421,46 @@ func (cs *ConfigStore) UpdateWarehouseState(orgID string, expectedState ManagedW
 	return nil
 }
 
+// migrateDeltaCatalogDefaultEnabled is a one-shot backfill that flips existing
+// rows where delta_catalog_enabled was stored as false (the old default) to
+// true. New rows get true automatically via the gorm:"default:true" column
+// default. Tracked in duckgres_schema_migrations so it runs exactly once;
+// admins can disable per-warehouse via the admin API after the backfill
+// without it being re-flipped on subsequent restarts.
+const deltaCatalogDefaultMigrationName = "2026_05_delta_catalog_default_enabled"
+
+func migrateDeltaCatalogDefaultEnabled(db *gorm.DB) error {
+	var existing SchemaMigration
+	err := db.Where("name = ?", deltaCatalogDefaultMigrationName).First(&existing).Error
+	if err == nil {
+		return nil // already applied
+	}
+	if !errors.Is(err, gorm.ErrRecordNotFound) {
+		return fmt.Errorf("check schema migration: %w", err)
+	}
+
+	return db.Transaction(func(tx *gorm.DB) error {
+		if err := tx.Exec(
+			`UPDATE duckgres_ducklake_config SET delta_catalog_enabled = TRUE WHERE delta_catalog_enabled IS DISTINCT FROM TRUE`,
+		).Error; err != nil {
+			return fmt.Errorf("backfill ducklake config: %w", err)
+		}
+		if err := tx.Exec(
+			`UPDATE duckgres_managed_warehouses SET s3_delta_catalog_enabled = TRUE WHERE s3_delta_catalog_enabled IS DISTINCT FROM TRUE`,
+		).Error; err != nil {
+			return fmt.Errorf("backfill managed warehouses: %w", err)
+		}
+		if err := tx.Create(&SchemaMigration{
+			Name:      deltaCatalogDefaultMigrationName,
+			AppliedAt: time.Now().UTC(),
+		}).Error; err != nil {
+			return fmt.Errorf("record schema migration: %w", err)
+		}
+		slog.Info("Backfilled delta_catalog_enabled=true on existing config rows.")
+		return nil
+	})
+}
+
 func migrateOrgUserPK(db *gorm.DB) error {
 	// Check if the PK already has 2 columns (idempotent)
 	var count int64

main_test.go

@@ -214,6 +214,37 @@ func TestResolveEffectiveConfigDuckLakeDisableMetadataThreadLocalCacheDefaultsTr
 }
 
 func TestResolveEffectiveConfigDuckLakeDeltaCatalog(t *testing.T) {
+	// Default-on: with no file/env/CLI overrides, Delta is enabled by default.
+	resolvedDefault := configresolve.ResolveEffective(nil, configresolve.CLIInputs{}, envFromMap(nil), nil)
+	if !resolvedDefault.Server.DuckLake.DeltaCatalogEnabled {
+		t.Fatal("expected DeltaCatalogEnabled to default to true")
+	}
+
+	// Default-on with DuckLake configured: path auto-derives to a sibling delta/.
+	resolvedDuckLake := configresolve.ResolveEffective(&FileConfig{
+		DuckLake: DuckLakeFileConfig{
+			ObjectStore: "s3://warehouse/ducklake/",
+		},
+	}, configresolve.CLIInputs{}, envFromMap(nil), nil)
+	if !resolvedDuckLake.Server.DuckLake.DeltaCatalogEnabled {
+		t.Fatal("expected default DeltaCatalogEnabled with DuckLake configured")
+	}
+	if got, want := resolvedDuckLake.Server.DuckLake.DeltaCatalogPath, "s3://warehouse/delta/"; got != want {
+		t.Fatalf("expected derived Delta catalog path %q, got %q", want, got)
+	}
+
+	// Explicit YAML opt-out wins over the new default.
+	fileDisabled := false
+	resolvedOptOut := configresolve.ResolveEffective(&FileConfig{
+		DuckLake: DuckLakeFileConfig{
+			ObjectStore:         "s3://warehouse/ducklake/",
+			DeltaCatalogEnabled: &fileDisabled,
+		},
+	}, configresolve.CLIInputs{}, envFromMap(nil), nil)
+	if resolvedOptOut.Server.DuckLake.DeltaCatalogEnabled {
+		t.Fatal("expected YAML ducklake.delta_catalog_enabled=false to disable Delta catalog")
+	}
+
 	fileEnabled := true
 	resolved := configresolve.ResolveEffective(&FileConfig{
 		DuckLake: DuckLakeFileConfig{

server/server.go

@@ -1486,13 +1486,17 @@ func AttachDuckLake(db *sql.DB, dlCfg DuckLakeConfig, sem chan struct{}, dataDir
 // AttachDeltaCatalog attaches the configured Delta Lake catalog/table alongside
 // DuckLake. It reuses the DuckLake S3 secret settings so Delta scans can access
 // the same object store credentials.
+//
+// Delta is enabled by default. When no path is derivable (e.g. a plain
+// standalone DuckDB instance with no DuckLake object_store/data_path), this is
+// a benign no-op: there's no Delta sibling to attach.
 func AttachDeltaCatalog(db *sql.DB, dlCfg DuckLakeConfig, sem chan struct{}) error {
 	if !dlCfg.DeltaCatalogEnabled {
 		return nil
 	}
 	catalogPath := ducklake.DeltaCatalogPath(dlCfg)
 	if catalogPath == "" {
-		return fmt.Errorf("delta catalog path is empty")
+		return nil
 	}
 
 	select {
@@ -1521,12 +1525,49 @@ func AttachDeltaCatalog(db *sql.DB, dlCfg DuckLakeConfig, sem chan struct{}) err
 	attachStmt := ducklake.BuildDeltaAttachStmt(dlCfg)
 	slog.Info("Attaching Delta catalog.", "path", catalogPath)
 	if _, err := db.Exec(attachStmt); err != nil {
+		// Delta is enabled by default. A fresh DuckLake tenant won't have any
+		// Delta data at the sibling delta/ prefix yet, so DeltaKernel returns
+		// "No files in log segment". That's expected — the catalog will start
+		// resolving once Delta data lands at the path. Treat as a benign skip
+		// instead of failing connection setup.
+		if isDeltaCatalogEmptyError(err) {
+			slog.Info("Skipping Delta catalog attach: no Delta data at path yet.", "path", catalogPath)
+			return nil
+		}
 		return fmt.Errorf("failed to attach Delta catalog: %w", err)
 	}
+	// ATTACH '...' (TYPE delta) is lazy — it doesn't read the transaction log
+	// until something forces resolution. With Delta attached but the path
+	// empty, every unqualified-table query the user runs against DuckLake will
+	// fail at prepare time when the planner walks all attached catalogs and
+	// Delta tries to read a missing _delta_log/. Probe immediately and detach
+	// if there's no Delta data here yet, so the catalog only sticks around
+	// once it's actually queryable.
+	if _, err := db.Exec("SHOW TABLES FROM delta"); err != nil {
+		if isDeltaCatalogEmptyError(err) {
+			if _, derr := db.Exec("DETACH delta"); derr != nil {
+				slog.Warn("Failed to detach empty Delta catalog after attach probe.", "error", derr)
+			}
+			slog.Info("Detached Delta catalog: no Delta data at path yet.", "path", catalogPath)
+			return nil
+		}
+		return fmt.Errorf("failed to probe Delta catalog: %w", err)
+	}
 	slog.Info("Attached Delta catalog successfully.", "path", catalogPath)
 	return nil
 }
 
+// isDeltaCatalogEmptyError reports whether err indicates the Delta location
+// exists but contains no Delta transaction log yet (the common case for a
+// fresh DuckLake tenant under the always-on Delta default). The DuckDB Delta
+// extension surfaces this as "No files in log segment" via DeltaKernel.
+func isDeltaCatalogEmptyError(err error) bool {
+	if err == nil {
+		return false
+	}
+	return strings.Contains(err.Error(), "No files in log segment")
+}
+
 func deltaCatalogNeedsS3Secret(catalogPath string, dlCfg DuckLakeConfig) bool {
 	if !strings.Contains(catalogPath, "://") {
 		return false