boilerplate

Author: Amogh-Bharadwaj

.env.example

@@ -29,6 +29,12 @@ PG_USER=postgres
 PG_PASSWORD=postgres
 PG_DATABASE=postgres
 
+PG2_HOST=host.docker.internal
+PG2_PORT=5437
+PG2_USER=postgres
+PG2_PASSWORD=postgres
+PG2_DATABASE=postgres
+
 PEERDB_CATALOG_HOST=host.docker.internal
 PEERDB_CATALOG_PORT=9901
 PEERDB_CATALOG_USER=postgres

Tiltfile

@@ -113,6 +113,13 @@ local_resource(
     resource_deps=['postgres']
 )
 
+local_resource(
+    'provision-postgres2',
+    cmd='./local_provision_scripts/postgres.sh peerdb-postgres2',
+    labels=['Ancillary-DB-Provisioning'],
+    resource_deps=['postgres2']
+)
+
 # This is not defined as a resource as we need the file to be present
 # when `docker_compose` loads the configuration (next line).
 local('./generate-test-environment.sh')
@@ -150,6 +157,10 @@ dc_resource('postgres', labels=['Ancillary-DB'], links=[
     link('http://localhost:5432', 'PostgreSQL'),
 ], auto_init=False)
 
+dc_resource('postgres2', labels=['Ancillary-DB'], links=[
+    link('http://localhost:5437', 'PostgreSQL (secondary)'),
+], auto_init=False)
+
 local_resource(
     'all-test-resources',
     cmd=' '.join([

ancillary-docker-compose.yml

@@ -125,6 +125,27 @@ services:
       timeout: 10s
       retries: 5
 
+  postgres2:
+    container_name: peerdb-postgres2
+    image: ${POSTGRES_IMAGE}
+    restart: unless-stopped
+    volumes:
+      - postgres2_data:/var/lib/postgresql/data
+    ports:
+      - "${PG2_PORT}:5432"
+    environment:
+      PGUSER: ${PG2_USER}
+      POSTGRES_PASSWORD: ${PG2_PASSWORD}
+      POSTGRES_DB: ${PG2_DATABASE}
+      POSTGRES_INITDB_ARGS: --locale=C.UTF-8
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    healthcheck:
+      test: ["CMD", "pg_isready"]
+      interval: 2s
+      timeout: 10s
+      retries: 5
+
   dozzle:
     container_name: dozzle-compose-monitor
     image: amir20/dozzle:latest
@@ -185,6 +206,7 @@ services:
 
 volumes:
   postgres_data:
+  postgres2_data:
   clickhouse_data:
   mongo_data:
   mysql_gtid_data:

flow/activities/flowable.go

@@ -2144,6 +2144,10 @@ func (a *FlowableActivity) MigratePostgresTableOIDs(
 	return nil
 }
 
+func (a *FlowableActivity) PeerDBPGAutomatedSchemaDump(ctx context.Context, env map[string]string) (bool, error) {
+	return internal.PeerDBPGAutomatedSchemaDump(ctx, env)
+}
+
 func (a *FlowableActivity) RunPgDumpSchema(
 	ctx context.Context,
 	input *protos.RunPgDumpSchemaInput,

flow/connectors/postgres/validate.go

@@ -278,6 +278,10 @@ func (c *PostgresConnector) ValidateMirrorDestination(
 		return nil // no need to validate schema for resync, as we will create or replace the tables
 	}
 
+	if cfg.System == protos.TypeSystem_PG && cfg.Env["PEERDB_PG_AUTOMATED_SCHEMA_DUMP"] == "true" {
+		return nil // pg_dump will create the schema and tables on the destination
+	}
+
 	// Validate that all source columns exist in destination tables
 	checkedSchemas := make(map[string]struct{})
 	for _, tableMapping := range cfg.TableMappings {