fix(server): drop iceberg credential_chain fallback, fail loudly instead

Removes the credential_chain branch introduced in the previous commit on
this PR. The fallback was hedging — it dressed up the broken DuckDB chain
path (the very thing this PR exists to fix) as forward-compat. In the
multitenant deploy the worker pod has no AWS identity, so a silent
fallback would just defer the same failure to attach time and obscure
the real misconfiguration.

Explicit STS credentials minted by the control plane are the only
supported auth model. If iceberg.Enabled is true and the activation
payload lacks credentials, that is a configuration bug (broken STS
broker, missing per-tenant IAM role, race in the activator) and should
surface as a clear error at attach time, not a credential_chain SQL that
will fail in DuckDB with an opaque "Secret Validation Failure".

Changes:
- BuildIcebergSecretStmt: no more empty-keyID branch; always emit
  PROVIDER config.
- AttachIcebergCatalog: validate keyID/secret are non-empty when iceberg
  is enabled; return an error naming the table_bucket so logs identify
  which tenant is affected.
- Tests: drop the credential_chain-fallback case; keep the with/without
  session-token, region default, and quote-escaping coverage.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: bill-ph

server/iceberg/migration.go

@@ -25,32 +25,25 @@ const CatalogName = "iceberg"
 // the iceberg extension internally signs with SigV4 and pulls credentials
 // from any TYPE S3 secret in scope.
 //
-// When keyID is non-empty the secret is built with PROVIDER config and the
-// supplied short-lived credentials inlined directly. This is the multitenant
-// production path: the control plane assumes the per-tenant IAM role via
-// STS and ships the resulting temporary credentials in the worker
-// activation payload, identical to how the DuckLake S3 secret is built (see
+// The secret is always built with PROVIDER config and the supplied
+// short-lived credentials inlined directly. This is the only supported
+// auth model: the control plane assumes the per-tenant IAM role via STS
+// and ships the resulting temporary credentials in the worker activation
+// payload, identical to how the DuckLake S3 secret is built (see
 // buildConfigSecret in server/server.go). The same role has both s3:* and
-// s3tables:* permissions on the tenant's data and table buckets, so reusing
-// the credentials here is correct.
+// s3tables:* permissions on the tenant's data and table buckets, so
+// reusing the credentials here is correct.
 //
-// When keyID is empty the secret falls back to PROVIDER credential_chain.
-// This preserves the legacy/standalone path where the host running the
-// worker has its own AWS identity reachable via env vars, ~/.aws, or IMDS.
-// On EKS deployments without IRSA/Pod Identity on the worker pod (the
-// PostHog production topology) this fallback will fail at create time —
-// callers in that topology must supply explicit credentials.
+// keyID and secret are required — callers must validate upstream and emit
+// a clear error if the activation payload is missing them when iceberg is
+// enabled. sessionToken is optional (omitted from the DDL when empty) to
+// support the rare static-IAM-user case, though STS:AssumeRole always
+// returns one in production.
 func BuildIcebergSecretStmt(cfg Config, keyID, secret, sessionToken string) string {
 	region := cfg.Region
 	if region == "" {
 		region = DefaultRegion
 	}
-	if keyID == "" {
-		return fmt.Sprintf(
-			"CREATE OR REPLACE SECRET iceberg_sigv4 (TYPE S3, PROVIDER credential_chain, REGION '%s')",
-			escapeSQLStringLiteral(region),
-		)
-	}
 	stmt := fmt.Sprintf(
 		"CREATE OR REPLACE SECRET iceberg_sigv4 (TYPE S3, PROVIDER config, KEY_ID '%s', SECRET '%s', REGION '%s'",
 		escapeSQLStringLiteral(keyID),

server/iceberg/migration_test.go

@@ -39,17 +39,9 @@ func TestBuildIcebergSecretStmtWithoutSessionToken(t *testing.T) {
 	}
 }
 
-func TestBuildIcebergSecretStmtFallsBackToCredentialChain(t *testing.T) {
-	got := BuildIcebergSecretStmt(Config{Region: "us-west-2"}, "", "", "")
-	want := "CREATE OR REPLACE SECRET iceberg_sigv4 (TYPE S3, PROVIDER credential_chain, REGION 'us-west-2')"
-	if got != want {
-		t.Fatalf("BuildIcebergSecretStmt should fall back to credential_chain when keyID is empty:\n got: %s\nwant: %s", got, want)
-	}
-}
-
 func TestBuildIcebergSecretStmtDefaultsRegion(t *testing.T) {
-	got := BuildIcebergSecretStmt(Config{}, "", "", "")
-	want := "CREATE OR REPLACE SECRET iceberg_sigv4 (TYPE S3, PROVIDER credential_chain, REGION 'us-east-1')"
+	got := BuildIcebergSecretStmt(Config{}, "AKIA_TEST", "shh", "tok")
+	want := "CREATE OR REPLACE SECRET iceberg_sigv4 (TYPE S3, PROVIDER config, KEY_ID 'AKIA_TEST', SECRET 'shh', REGION 'us-east-1', SESSION_TOKEN 'tok')"
 	if got != want {
 		t.Fatalf("BuildIcebergSecretStmt should default to us-east-1:\n got: %s\nwant: %s", got, want)
 	}

server/server.go

@@ -1605,16 +1605,22 @@ func isDeltaCatalogEmptyError(err error) bool {
 // fail-soft for the "fresh tenant, no namespaces yet" case so a worker
 // activation isn't blocked by an empty catalog.
 //
-// keyID/secret/sessionToken are short-lived AWS credentials produced by the
-// control plane's STS AssumeRole on the per-tenant IAM role. They are the
-// same credentials DuckLake uses (see buildConfigSecret) — the per-tenant
+// keyID/secret/sessionToken are short-lived AWS credentials produced by
+// the control plane's STS AssumeRole on the per-tenant IAM role — the
+// same credentials DuckLake uses (see buildConfigSecret). The per-tenant
 // role has both s3:* and s3tables:* permissions, so reusing them here is
-// correct. When empty, BuildIcebergSecretStmt falls back to
-// PROVIDER credential_chain (for standalone/host-AWS-identity deployments).
+// correct. This is the only supported auth model for iceberg: we do NOT
+// fall back to DuckDB's credential_chain because the worker pod has no
+// AWS identity of its own on the multitenant production topology. Missing
+// credentials when iceberg is enabled is a configuration bug and surfaces
+// as an explicit error rather than a silent fallback.
 func AttachIcebergCatalog(db *sql.DB, ic IcebergConfig, sem chan struct{}, keyID, secret, sessionToken string) error {
 	if !ic.Enabled || ic.TableBucket == "" {
 		return nil
 	}
+	if keyID == "" || secret == "" {
+		return fmt.Errorf("iceberg catalog enabled (table_bucket=%q) but no AWS credentials in activation payload — control plane STS broker did not populate DuckLake S3 credentials", ic.TableBucket)
+	}
 
 	select {
 	case sem <- struct{}{}: