From 08a6514c98a342277be8af76872d2fcfb9251740 Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 10 Jun 2025 02:55:36 +0000 Subject: [PATCH 1/5] add delete for publication, replication slots command Signed-off-by: Stephen Compall --- cluster/pulumi/canton-network/src/bigQuery.ts | 88 ++++++++++++------- 1 file changed, 58 insertions(+), 30 deletions(-) diff --git a/cluster/pulumi/canton-network/src/bigQuery.ts b/cluster/pulumi/canton-network/src/bigQuery.ts index 3adbc454c3..1ee0a1d49e 100644 --- a/cluster/pulumi/canton-network/src/bigQuery.ts +++ b/cluster/pulumi/canton-network/src/bigQuery.ts @@ -322,11 +322,49 @@ function createPostgresReplicatorUser( parent: postgres, deletedWith: postgres.databaseInstance, protect: protectCloudSql, - dependsOn: [postgres.databaseInstance, password.secret], + dependsOn: [postgres.databaseInstance, password.secret, cleanup], } ); } +function databaseCommandBracket(postgres: CloudPostgres) { + return { + header: pulumi.interpolate` + set -e + TMP_BUCKET="da-cn-tmp-sql-$(date +%s)-$RANDOM" + TMP_SQL_FILE="$(mktemp tmp_pub_rep_slots_XXXXXXXXXX.sql --tmpdir)" + GCS_URI="gs://$TMP_BUCKET/$(basename "$TMP_SQL_FILE")" + + # create temporary bucket + gsutil mb --pap enforced -p "${privateNetwork.project}" \ + -l "${cloudsdkComputeRegion()}" "gs://$TMP_BUCKET" + + # grant DB service account access to the bucket + gsutil iam ch "serviceAccount:${postgres.databaseInstance.serviceAccountEmailAddress}:roles/storage.objectAdmin" \ + "gs://$TMP_BUCKET" + + cat > "$TMP_SQL_FILE" <<'EOT' + `, + footer: pulumi.interpolate` +EOT + + # upload SQL to temporary bucket + gsutil cp "$TMP_SQL_FILE" "$GCS_URI" + + # then import into Cloud SQL + gcloud sql import sql ${postgres.databaseInstance.name} "$GCS_URI" \ + --database="${scanAppDatabaseName(postgres)}" \ + --user="${postgres.user.name}" \ + --quiet + + # cleanup: remove the file from GCS, delete the bucket, remove the local file + gsutil rm "$GCS_URI" + gsutil rb "gs://$TMP_BUCKET" + rm "$TMP_SQL_FILE" + `, + }; +} + /* For the SQL below to apply, the user/operator applying the pulumi needs the 'Cloud SQL Editor' IAM role in the relevant GCP project @@ -339,6 +377,7 @@ function createPublicationAndReplicationSlots( ) { const dbName = scanAppDatabaseName(postgres); const schemaName = dbName; + const {header, footer} = databaseCommandBracket(postgres); return new command.local.Command( `${postgres.namespace.logicalName}-${replicatorUserName}-pub-replicate-slots`, { @@ -346,20 +385,7 @@ function createPublicationAndReplicationSlots( // ---- // from https://cloud.google.com/datastream/docs/configure-cloudsql-psql create: pulumi.interpolate` - set -e - TMP_BUCKET="da-cn-tmp-sql-$(date +%s)-$RANDOM" - TMP_SQL_FILE="$(mktemp tmp_pub_rep_slots_XXXXXXXXXX.sql --tmpdir)" - GCS_URI="gs://$TMP_BUCKET/$(basename "$TMP_SQL_FILE")" - - # create temporary bucket - gsutil mb --pap enforced -p "${privateNetwork.project}" \ - -l "${cloudsdkComputeRegion()}" "gs://$TMP_BUCKET" - - # grant DB service account access to the bucket - gsutil iam ch "serviceAccount:${postgres.databaseInstance.serviceAccountEmailAddress}:roles/storage.objectAdmin" \ - "gs://$TMP_BUCKET" - - cat > "$TMP_SQL_FILE" <<'EOT' + ${header} DO $$ DECLARE migration_complete BOOLEAN := FALSE; @@ -414,21 +440,23 @@ function createPublicationAndReplicationSlots( ALTER DEFAULT PRIVILEGES IN SCHEMA ${schemaName} GRANT SELECT ON TABLES TO ${replicatorUserName}; COMMIT; -EOT - - # upload SQL to temporary bucket - gsutil cp "$TMP_SQL_FILE" "$GCS_URI" - - # then import into Cloud SQL - gcloud sql import sql ${postgres.databaseInstance.name} "$GCS_URI" \ - --database="${scanAppDatabaseName(postgres)}" \ - --user="${postgres.user.name}" \ - --quiet - - # cleanup: remove the file from GCS, delete the bucket, remove the local file - gsutil rm "$GCS_URI" - gsutil rb "gs://$TMP_BUCKET" - rm "$TMP_SQL_FILE" + ${footer} + `, + delete: pulumi.interpolate` + ${header} + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_replication_slots WHERE slot_name = '${replicationSlotName}') THEN + PERFORM PG_DROP_REPLICATION_SLOT('${replicationSlotName}'); + END IF; + END $$; + DO $$ + BEGIN + IF EXISTS (SELECT 1 FROM pg_publication WHERE pubname = '${publicationName}') THEN + DROP PUBLICATION ${publicationName}; + END IF; + END $$; + ${footer} `, }, { From d6da8936f53fab0631d003e8c8194e83d205b066 Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 10 Jun 2025 02:56:09 +0000 Subject: [PATCH 2/5] add revocations to replicator user Signed-off-by: Stephen Compall --- cluster/pulumi/canton-network/src/bigQuery.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/cluster/pulumi/canton-network/src/bigQuery.ts b/cluster/pulumi/canton-network/src/bigQuery.ts index 1ee0a1d49e..4fd5ea527f 100644 --- a/cluster/pulumi/canton-network/src/bigQuery.ts +++ b/cluster/pulumi/canton-network/src/bigQuery.ts @@ -311,8 +311,24 @@ function createPostgresReplicatorUser( postgres: CloudPostgres, password: PostgresPassword ): gcp.sql.User { + const schemaName = scanAppDatabaseName(postgres); + const name = `${postgres.namespace.logicalName}-user-${replicatorUserName}`; + const {header, footer} = databaseCommandBracket(postgres); + const cleanup = new command.local.Command( + `${name}-cleanup`, + {delete: pulumi.interpolate` + ${header} + ALTER DEFAULT PRIVILEGES IN SCHEMA ${schemaName} + REVOKE SELECT ON TABLES TO ${replicatorUserName}; + REVOKE USAGE ON SCHEMA ${schemaName} TO ${replicatorUserName}; + REVOKE SELECT ON ALL TABLES + IN SCHEMA ${schemaName} TO ${replicatorUserName}; + ALTER USER ${replicatorUserName} WITH NOREPLICATION; + COMMIT; + ${footer} + `}); return new gcp.sql.User( - `${postgres.namespace.logicalName}-user-${replicatorUserName}`, + name, { instance: postgres.databaseInstance.name, name: replicatorUserName, From ad365b6aefcc804fe83dc680421297c505200169 Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 10 Jun 2025 02:56:43 +0000 Subject: [PATCH 3/5] SV1_SCAN_BIGQUERY env option Signed-off-by: Stephen Compall --- cluster/pulumi/common-sv/src/svConfigs.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster/pulumi/common-sv/src/svConfigs.ts b/cluster/pulumi/common-sv/src/svConfigs.ts index b35d541048..ede91cdf56 100644 --- a/cluster/pulumi/common-sv/src/svConfigs.ts +++ b/cluster/pulumi/common-sv/src/svConfigs.ts @@ -15,6 +15,8 @@ import { StaticSvConfig } from './config'; import { dsoSize } from './dsoConfig'; import { cometbftRetainBlocks } from './synchronizer/cometbftConfig'; +const sv1ScanBigQuery = spliceEnvConfig.envFlag('SV1_SCAN_BIGQUERY', false); + const svCometBftSecrets: pulumi.Output[] = isMainNet ? [svCometBftKeysFromSecret('sv1-cometbft-keys')] : [ @@ -58,6 +60,9 @@ export const svConfigs: StaticSvConfig[] = isMainNet }, }, sweep: sweepConfigFromEnv('SV1'), + ...(sv1ScanBigQuery + ? { scanBigQuery: { dataset: 'mainnet_da2_scan', prefix: 'da2' } } + : {}), }, ] : [ @@ -83,6 +88,7 @@ export const svConfigs: StaticSvConfig[] = isMainNet }, }, sweep: sweepConfigFromEnv('SV1'), + ...(sv1ScanBigQuery ? { scanBigQuery: { dataset: 'devnet_da2_scan', prefix: 'da2' } } : {}), }, { // TODO(#12169): consider making nodeName and ingressName the same (also for all other SVs) From 08c2c1a4471bf14718c84fa484b0d8d52aab7130 Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 10 Jun 2025 02:58:47 +0000 Subject: [PATCH 4/5] remove bqdatastream cleanup, just leave the user alive Revocations still not enough: - gcp:sql:User sv-1-user-bqdatastream deleting (1s) error: sdk-v2/provider2.go:566: sdk.helper_schema: Error, failed to deleteuser bqdatastream in instance sv-1-cn-apps-pg-2964ec6: googleapi: Error 400: Invalid request: failed to delete user bqdatastream: . role "bqdatastream" cannot be dropped because some objects depend on it Details: 32 objects in database scan_sv_1., invalid: provider=google-beta@8.32.1 Signed-off-by: Stephen Compall --- cluster/pulumi/canton-network/src/bigQuery.ts | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/cluster/pulumi/canton-network/src/bigQuery.ts b/cluster/pulumi/canton-network/src/bigQuery.ts index 4fd5ea527f..43f595a425 100644 --- a/cluster/pulumi/canton-network/src/bigQuery.ts +++ b/cluster/pulumi/canton-network/src/bigQuery.ts @@ -311,22 +311,7 @@ function createPostgresReplicatorUser( postgres: CloudPostgres, password: PostgresPassword ): gcp.sql.User { - const schemaName = scanAppDatabaseName(postgres); const name = `${postgres.namespace.logicalName}-user-${replicatorUserName}`; - const {header, footer} = databaseCommandBracket(postgres); - const cleanup = new command.local.Command( - `${name}-cleanup`, - {delete: pulumi.interpolate` - ${header} - ALTER DEFAULT PRIVILEGES IN SCHEMA ${schemaName} - REVOKE SELECT ON TABLES TO ${replicatorUserName}; - REVOKE USAGE ON SCHEMA ${schemaName} TO ${replicatorUserName}; - REVOKE SELECT ON ALL TABLES - IN SCHEMA ${schemaName} TO ${replicatorUserName}; - ALTER USER ${replicatorUserName} WITH NOREPLICATION; - COMMIT; - ${footer} - `}); return new gcp.sql.User( name, { @@ -337,8 +322,9 @@ function createPostgresReplicatorUser( { parent: postgres, deletedWith: postgres.databaseInstance, + retainOnDelete: true, protect: protectCloudSql, - dependsOn: [postgres.databaseInstance, password.secret, cleanup], + dependsOn: [postgres.databaseInstance, password.secret], } ); } From 712c282393adcba095df7ad8c56cf202ddeee6a5 Mon Sep 17 00:00:00 2001 From: Stephen Compall Date: Tue, 10 Jun 2025 03:04:09 +0000 Subject: [PATCH 5/5] npmFix [skip ci] Signed-off-by: Stephen Compall --- cluster/pulumi/canton-network/src/bigQuery.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster/pulumi/canton-network/src/bigQuery.ts b/cluster/pulumi/canton-network/src/bigQuery.ts index 43f595a425..2d66e02470 100644 --- a/cluster/pulumi/canton-network/src/bigQuery.ts +++ b/cluster/pulumi/canton-network/src/bigQuery.ts @@ -311,7 +311,7 @@ function createPostgresReplicatorUser( postgres: CloudPostgres, password: PostgresPassword ): gcp.sql.User { - const name = `${postgres.namespace.logicalName}-user-${replicatorUserName}`; + const name = `${postgres.namespace.logicalName}-user-${replicatorUserName}`; return new gcp.sql.User( name, { @@ -331,7 +331,7 @@ function createPostgresReplicatorUser( function databaseCommandBracket(postgres: CloudPostgres) { return { - header: pulumi.interpolate` + header: pulumi.interpolate` set -e TMP_BUCKET="da-cn-tmp-sql-$(date +%s)-$RANDOM" TMP_SQL_FILE="$(mktemp tmp_pub_rep_slots_XXXXXXXXXX.sql --tmpdir)" @@ -347,7 +347,7 @@ function databaseCommandBracket(postgres: CloudPostgres) { cat > "$TMP_SQL_FILE" <<'EOT' `, - footer: pulumi.interpolate` + footer: pulumi.interpolate` EOT # upload SQL to temporary bucket @@ -379,7 +379,7 @@ function createPublicationAndReplicationSlots( ) { const dbName = scanAppDatabaseName(postgres); const schemaName = dbName; - const {header, footer} = databaseCommandBracket(postgres); + const { header, footer } = databaseCommandBracket(postgres); return new command.local.Command( `${postgres.namespace.logicalName}-${replicatorUserName}-pub-replicate-slots`, {