fix: force re-provision master tenant schemas on every bootstrap (#2044)

* fix: force re-provision master tenant schemas on every bootstrap

The provisioner's idempotency gate trusts the stored provisioning
status, but this can be stale after partial provisioning or DB reset.
The bootstrap now resets the status to 'pending' before invoking the
provisioner, ensuring schemas are always created in all service
databases. CREATE SCHEMA IF NOT EXISTS makes this safe.

This fixes intermittent "schema does not exist" and "column does not
exist" errors in the demo deploy bootstrap step.

* fix: fail fast on provisioning status reset error

If the reset fails due to a DB error, the state remains active and
provisioning is silently skipped - defeating the purpose of the fix.
Propagate the error instead of swallowing it.

---------

Co-authored-by: Ben Coombs <bjcoombs@users.noreply.github.com>

Author: bjcoombs

internal/bootstrap/master_tenant.go

@@ -91,6 +91,11 @@ func Run(ctx context.Context, cfg Config) error {
 }
 
 // provisionSchemas creates org_meridian_master schemas in all service databases.
+//
+// The provisioner's idempotency gate trusts the stored provisioning status,
+// which can be stale if schemas were partially created or if the database was
+// reset. To guarantee all schemas exist, we reset the status to pending before
+// invoking the provisioner. CREATE SCHEMA IF NOT EXISTS makes this safe.
 func provisionSchemas(ctx context.Context, cfg Config, logger *slog.Logger) error {
 	logger.Info("provisioning master tenant schemas")
 
@@ -106,16 +111,12 @@ func provisionSchemas(ctx context.Context, cfg Config, logger *slog.Logger) erro
 
 	tenantID := tenant.TenantID(MasterTenantID)
 
-	// Check if already provisioned
-	status, err := prov.GetProvisioningStatus(ctx, tenantID)
-	if err == nil && status.State == provisioner.StateActive {
-		logger.Info("master tenant schemas already provisioned")
-		return nil
-	}
-
-	// Initialize provisioning status if needed
-	if err := prov.InitializeProvisioningStatus(ctx, tenantID); err != nil {
-		logger.Warn("failed to initialize provisioning status (may already exist)", "error", err)
+	// Reset provisioning status to pending so the provisioner always runs.
+	// The provisioner's idempotency gate trusts the stored status, which can
+	// be stale if schemas were partially created or if the database was reset.
+	// CREATE SCHEMA IF NOT EXISTS and idempotent migrations make this safe.
+	if err := resetProvisioningToPending(ctx, cfg.PlatformDB, tenantID, logger); err != nil {
+		return fmt.Errorf("reset provisioning status: %w", err)
 	}
 
 	// Provision schemas
@@ -204,6 +205,25 @@ func seedPlatformManifest(ctx context.Context, db *gorm.DB, logger *slog.Logger)
 	return nil
 }
 
+// resetProvisioningToPending updates the provisioning status to "pending" so the
+// provisioner re-runs schema creation and migrations. This is necessary because
+// the provisioner skips tenants with "active" status, but the actual schemas may
+// be missing (partial provisioning, DB reset, or new service added).
+func resetProvisioningToPending(ctx context.Context, db *gorm.DB, tenantID tenant.TenantID, logger *slog.Logger) error {
+	result := db.WithContext(ctx).Exec(
+		`UPDATE tenant_provisioning SET state = 'pending', updated_at = NOW() WHERE tenant_id = ?`,
+		tenantID.String(),
+	)
+	if result.Error != nil {
+		return result.Error
+	}
+	if result.RowsAffected > 0 {
+		logger.Info("reset provisioning status to pending for re-provisioning",
+			"tenant_id", tenantID.String())
+	}
+	return nil
+}
+
 // LoadPlatformManifest loads the embedded platform economy manifest as a proto Manifest.
 func LoadPlatformManifest() (*controlplanev1.Manifest, error) {
 	// First verify it's valid JSON