feat: add table-existence verification to provisioning completeness (#2123)

bjcoombs · web-flow · commit 0264b80ebd7f · 2026-04-04T09:34:37.000+01:00
* feat: add table-existence verification to provisioning completeness check

After schema migrations run, verify expected tables exist in tenant schema
before transitioning to active state. This closes the partial-provisioning
gap where the schema namespace exists but migrations failed silently.

Each service can define a SentinelTable (the primary domain table created by
its first migration). If set, that specific table is checked. If not set,
the service is assumed to have no required tables (e.g., internal-account).
Default service configs now include sentinel tables for all services with
migrations.

* fix: align SentinelTable doc with implementation, remove dead COUNT query

The doc comment promised "at least one table exists" verification when
SentinelTable is empty, but the code intentionally allowed zero tables
for services without migrations. Fix the doc to match the behavior and
remove the unused COUNT query.

---------

Co-authored-by: Ben Coombs &lt;bjcoombs@users.noreply.github.com&gt;
diff --git a/services/tenant/provisioner/errors.go b/services/tenant/provisioner/errors.go
@@ -84,4 +84,9 @@ var (
 	// ErrHookPanic indicates a post-provisioning hook panicked.
 	// The recovered panic value is wrapped in the error chain.
 	ErrHookPanic = errors.New("post-provisioning hook panicked")
+
+	// ErrSchemaVerificationFailed indicates that post-migration table verification failed.
+	// The tenant schema exists but expected tables were not created, indicating
+	// migrations ran but did not produce the expected database objects.
+	ErrSchemaVerificationFailed = errors.New("schema provisioning verification failed: expected tables not found")
 )
diff --git a/services/tenant/provisioner/postgres_provisioner.go b/services/tenant/provisioner/postgres_provisioner.go
@@ -126,6 +126,14 @@ func (p *PostgresProvisioner) ProvisionSchemas(ctx context.Context, tenantID ten
 		return err
 	}
 
+	// Verify expected tables exist before transitioning to active.
+	// This catches partial provisioning where schema exists but migrations failed silently.
+	if err := p.verifySchemaProvisioned(ctx, tenantID.SchemaName(), logger); err != nil {
+		logger.Error("schema verification failed after migrations", "error", err)
+		p.markProvisioningFailed(ctx, status, err.Error())
+		return err
+	}
+
 	// Mark as active
 	status.State = StateActive
 	status.UpdatedAt = timeNow()
diff --git a/services/tenant/provisioner/postgres_provisioner_test.go b/services/tenant/provisioner/postgres_provisioner_test.go
@@ -1896,6 +1896,163 @@ func TestPostgresProvisioner_PostProvisioningHooks_PanicRecovery(t *testing.T) {
 	assert.Equal(t, StateActive, status.State)
 }
 
+func TestPostgresProvisioner_VerifySchemaProvisioned_SentinelTable(t *testing.T) {
+	tc := setupTestContainer(t)
+	defer tc.cleanup(t)
+
+	tenantID := tenant.MustNewTenantID("sentinel_test")
+	createTestTenant(t, tc.db, tenantID.String())
+
+	// Create migration that creates a specific table
+	svcDir := filepath.Join(tc.migDir, "sentinel-service")
+	require.NoError(t, os.MkdirAll(svcDir, 0o755))
+	createTestMigration(t, svcDir, "20251201000000_init.sql", `
+		CREATE TABLE my_sentinel (id UUID PRIMARY KEY DEFAULT gen_random_uuid());
+	`)
+
+	config := &Config{
+		Services: []ServiceConfig{
+			{
+				Name:          "sentinel-service",
+				MigrationPath: svcDir,
+				DatabaseURL:   tc.connStr,
+				SentinelTable: "my_sentinel",
+			},
+		},
+		ProvisioningTimeout: 30 * time.Second,
+	}
+
+	prov, err := NewPostgresProvisioner(tc.db, config)
+	require.NoError(t, err)
+	defer prov.Close()
+
+	// Provision should succeed - sentinel table is created by migration
+	err = prov.ProvisionSchemas(context.Background(), tenantID)
+	require.NoError(t, err)
+
+	status, err := prov.GetProvisioningStatus(context.Background(), tenantID)
+	require.NoError(t, err)
+	assert.Equal(t, StateActive, status.State)
+}
+
+func TestPostgresProvisioner_VerifySchemaProvisioned_MissingSentinelTable(t *testing.T) {
+	tc := setupTestContainer(t)
+	defer tc.cleanup(t)
+
+	tenantID := tenant.MustNewTenantID("missing_sentinel")
+	createTestTenant(t, tc.db, tenantID.String())
+
+	// Create migration that creates a table with a DIFFERENT name than the sentinel
+	svcDir := filepath.Join(tc.migDir, "wrong-table-service")
+	require.NoError(t, os.MkdirAll(svcDir, 0o755))
+	createTestMigration(t, svcDir, "20251201000000_init.sql", `
+		CREATE TABLE some_other_table (id UUID PRIMARY KEY DEFAULT gen_random_uuid());
+	`)
+
+	config := &Config{
+		Services: []ServiceConfig{
+			{
+				Name:          "wrong-table-service",
+				MigrationPath: svcDir,
+				DatabaseURL:   tc.connStr,
+				SentinelTable: "expected_table_that_doesnt_exist",
+			},
+		},
+		ProvisioningTimeout: 30 * time.Second,
+	}
+
+	prov, err := NewPostgresProvisioner(tc.db, config)
+	require.NoError(t, err)
+	defer prov.Close()
+
+	// Provision should fail verification - sentinel table doesn't exist
+	err = prov.ProvisionSchemas(context.Background(), tenantID)
+	require.Error(t, err)
+	assert.ErrorIs(t, err, ErrSchemaVerificationFailed)
+
+	// Status should be failed
+	status, err := prov.GetProvisioningStatus(context.Background(), tenantID)
+	require.NoError(t, err)
+	assert.Equal(t, StateFailed, status.State)
+	assert.Contains(t, status.ErrorMessage, "expected_table_that_doesnt_exist")
+}
+
+func TestPostgresProvisioner_VerifySchemaProvisioned_NoSentinelWithTables(t *testing.T) {
+	tc := setupTestContainer(t)
+	defer tc.cleanup(t)
+
+	tenantID := tenant.MustNewTenantID("no_sentinel_ok")
+	createTestTenant(t, tc.db, tenantID.String())
+
+	// Service with no sentinel table configured but has migrations
+	svcDir := filepath.Join(tc.migDir, "no-sentinel-service")
+	require.NoError(t, os.MkdirAll(svcDir, 0o755))
+	createTestMigration(t, svcDir, "20251201000000_init.sql", `
+		CREATE TABLE any_table (id UUID PRIMARY KEY DEFAULT gen_random_uuid());
+	`)
+
+	config := &Config{
+		Services: []ServiceConfig{
+			{
+				Name:          "no-sentinel-service",
+				MigrationPath: svcDir,
+				DatabaseURL:   tc.connStr,
+				// SentinelTable intentionally empty
+			},
+		},
+		ProvisioningTimeout: 30 * time.Second,
+	}
+
+	prov, err := NewPostgresProvisioner(tc.db, config)
+	require.NoError(t, err)
+	defer prov.Close()
+
+	// Should succeed - no sentinel check, tables exist
+	err = prov.ProvisionSchemas(context.Background(), tenantID)
+	require.NoError(t, err)
+
+	status, err := prov.GetProvisioningStatus(context.Background(), tenantID)
+	require.NoError(t, err)
+	assert.Equal(t, StateActive, status.State)
+}
+
+func TestPostgresProvisioner_VerifySchemaProvisioned_EmptySchemaNoSentinel(t *testing.T) {
+	tc := setupTestContainer(t)
+	defer tc.cleanup(t)
+
+	tenantID := tenant.MustNewTenantID("empty_schema")
+	createTestTenant(t, tc.db, tenantID.String())
+
+	// Service with no sentinel and no migrations - should succeed (e.g., internal-account)
+	svcDir := filepath.Join(tc.migDir, "empty-service")
+	require.NoError(t, os.MkdirAll(svcDir, 0o755))
+	// No migration files
+
+	config := &Config{
+		Services: []ServiceConfig{
+			{
+				Name:          "empty-service",
+				MigrationPath: svcDir,
+				DatabaseURL:   tc.connStr,
+				// SentinelTable intentionally empty
+			},
+		},
+		ProvisioningTimeout: 30 * time.Second,
+	}
+
+	prov, err := NewPostgresProvisioner(tc.db, config)
+	require.NoError(t, err)
+	defer prov.Close()
+
+	// Should succeed - empty service with no sentinel is OK
+	err = prov.ProvisionSchemas(context.Background(), tenantID)
+	require.NoError(t, err)
+
+	status, err := prov.GetProvisioningStatus(context.Background(), tenantID)
+	require.NoError(t, err)
+	assert.Equal(t, StateActive, status.State)
+}
+
 func TestNewPostgresProvisioner_NilPlatformDB(t *testing.T) {
 	config := &Config{
 		Services: []ServiceConfig{
diff --git a/services/tenant/provisioner/provisioner.go b/services/tenant/provisioner/provisioner.go
@@ -297,6 +297,13 @@ type ServiceConfig struct {
 	// If empty, falls back to a constructed URL based on service name:
 	// postgres://meridian_{service}_user@cockroachdb:26257/meridian_{service}?sslmode=disable
 	DatabaseURL string
+
+	// SentinelTable is the name of a table that must exist after migrations complete.
+	// Used for post-migration verification to detect partial provisioning where
+	// the schema exists but migrations failed silently.
+	// If empty, the service is assumed to have no required tables and verification
+	// is skipped (e.g., services that only need a schema namespace without migrations).
+	SentinelTable string
 }
 
 // PostProvisioningHook is called after successful schema provisioning for a tenant.
@@ -380,32 +387,38 @@ func DefaultConfig() *Config {
 	}
 }
 
-// defaultServiceNames lists all BIAN services that require schema provisioning.
+// defaultServiceDefs lists all BIAN services that require schema provisioning.
 // Order matters: services are provisioned in the order listed.
-var defaultServiceNames = []string{
-	"party",
-	"current-account",
-	"position-keeping",
-	"financial-accounting",
-	"payment-order",
-	"market-information",
-	"reference-data",
+// SentinelTable is the primary domain table created by the first migration;
+// its presence confirms migrations ran to completion.
+var defaultServiceDefs = []struct {
+	Name          string
+	SentinelTable string // empty for services with no provisioner-specific migrations
+}{
+	{"party", "party"},
+	{"current-account", "account"},
+	{"position-keeping", "financial_position_log"},
+	{"financial-accounting", "financial_booking_log"},
+	{"payment-order", "payment_order"},
+	{"market-information", "data_source"},
+	{"reference-data", "instrument_definition"},
 	// Services below require org_<tenant> schemas for tenant-scoped
 	// queries but have no provisioner-specific migrations.
-	"internal-account",
-	"reconciliation",
-	"identity",
-	"control-plane",
+	{"internal-account", ""},
+	{"reconciliation", ""},
+	{"identity", ""},
+	{"control-plane", ""},
 }
 
 // buildDefaultServiceConfigs constructs ServiceConfig entries for all default services.
 func buildDefaultServiceConfigs(basePath string) []ServiceConfig {
-	configs := make([]ServiceConfig, 0, len(defaultServiceNames))
-	for _, name := range defaultServiceNames {
+	configs := make([]ServiceConfig, 0, len(defaultServiceDefs))
+	for _, def := range defaultServiceDefs {
 		configs = append(configs, ServiceConfig{
-			Name:          name,
-			MigrationPath: basePath + "/" + name,
-			DatabaseURL:   getServiceDatabaseURL(name),
+			Name:          def.Name,
+			MigrationPath: basePath + "/" + def.Name,
+			DatabaseURL:   getServiceDatabaseURL(def.Name),
+			SentinelTable: def.SentinelTable,
 		})
 	}
 	return configs
diff --git a/services/tenant/provisioner/provisioner_helpers.go b/services/tenant/provisioner/provisioner_helpers.go
@@ -118,6 +118,61 @@ func (p *PostgresProvisioner) dropSchemaInAllDBs(ctx context.Context, schemaName
 	return nil
 }
 
+// verifySchemaProvisioned checks that expected tables exist in the tenant schema
+// after migrations have been applied. This closes the partial-provisioning gap
+// where the schema namespace exists but migrations failed halfway through.
+//
+// For each service:
+//   - If SentinelTable is set, verifies that specific table exists
+//   - If SentinelTable is empty, verification is skipped (service has no required tables)
+//
+// Returns nil if verification passes, or an error listing which services failed.
+func (p *PostgresProvisioner) verifySchemaProvisioned(ctx context.Context, schemaName string, logger *slog.Logger) error {
+	var failedServices []string
+
+	for _, svc := range p.config.Services {
+		serviceDB, ok := p.serviceDbs[svc.Name]
+		if !ok {
+			continue // Already caught by provisionSingleService
+		}
+
+		if svc.SentinelTable != "" {
+			// Check for specific sentinel table
+			var exists bool
+			err := serviceDB.WithContext(bypassCtx(ctx)).Raw(
+				`SELECT EXISTS(
+					SELECT 1 FROM information_schema.tables
+					WHERE table_schema = ? AND table_name = ?
+				)`, schemaName, svc.SentinelTable,
+			).Scan(&exists).Error
+			if err != nil {
+				logger.Error("failed to verify sentinel table",
+					"service", svc.Name,
+					"sentinel_table", svc.SentinelTable,
+					"error", err)
+				failedServices = append(failedServices, fmt.Sprintf("%s (query error: %v)", svc.Name, err))
+				continue
+			}
+			if !exists {
+				logger.Error("sentinel table missing after migration",
+					"service", svc.Name,
+					"schema", schemaName,
+					"sentinel_table", svc.SentinelTable)
+				failedServices = append(failedServices, fmt.Sprintf("%s (missing table: %s)", svc.Name, svc.SentinelTable))
+			}
+		} else {
+			// No sentinel table configured - service has no required tables
+			// (e.g., internal-account, reconciliation). Skip verification.
+			logger.Debug("no sentinel table configured, skipping verification", "service", svc.Name)
+		}
+	}
+
+	if len(failedServices) > 0 {
+		return fmt.Errorf("%w: %s", ErrSchemaVerificationFailed, strings.Join(failedServices, "; "))
+	}
+	return nil
+}
+
 // isAlreadyExistsError checks if the error indicates an object already exists.
 //
 // IDEMPOTENCY: This is a key idempotency mechanism for migrations. When a migration
