diff --git a/cluster/expected/multi-validator/expected.json b/cluster/expected/multi-validator/expected.json index 73d684728a..c3993505e8 100644 --- a/cluster/expected/multi-validator/expected.json +++ b/cluster/expected/multi-validator/expected.json @@ -4877,12 +4877,7 @@ } } }, - "db": { - "maxConnections": 1000, - "volumeSize": "100Gi" - }, - "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker", - "nonHyperdiskAppsAffinityAndTolerations": { + "appsAffinityAndTolerations": { "affinity": { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -4914,6 +4909,11 @@ } ] }, + "db": { + "maxConnections": 1000, + "volumeSize": "100Gi" + }, + "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker", "persistence": { "secretName": "postgres-0-secret" }, @@ -4973,12 +4973,7 @@ } } }, - "db": { - "maxConnections": 1000, - "volumeSize": "100Gi" - }, - "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker", - "nonHyperdiskAppsAffinityAndTolerations": { + "appsAffinityAndTolerations": { "affinity": { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -5010,6 +5005,11 @@ } ] }, + "db": { + "maxConnections": 1000, + "volumeSize": "100Gi" + }, + "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker", "persistence": { "secretName": "postgres-1-secret" }, @@ -5069,12 +5069,7 @@ } } }, - "db": { - "maxConnections": 1000, - "volumeSize": "100Gi" - }, - "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker", - "nonHyperdiskAppsAffinityAndTolerations": { + "appsAffinityAndTolerations": { "affinity": { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -5106,6 +5101,11 @@ } ] }, + "db": { + "maxConnections": 1000, + "volumeSize": "100Gi" + }, + "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker", "persistence": { "secretName": "postgres-2-secret" }, @@ -5165,12 +5165,7 @@ } } }, - "db": { - "maxConnections": 1000, - "volumeSize": "100Gi" - }, - "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker", - "nonHyperdiskAppsAffinityAndTolerations": { + "appsAffinityAndTolerations": { "affinity": { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -5202,6 +5197,11 @@ } ] }, + "db": { + "maxConnections": 1000, + "volumeSize": "100Gi" + }, + "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker", "persistence": { "secretName": "postgres-3-secret" }, @@ -5261,12 +5261,7 @@ } } }, - "db": { - "maxConnections": 1000, - "volumeSize": "100Gi" - }, - "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker", - "nonHyperdiskAppsAffinityAndTolerations": { + "appsAffinityAndTolerations": { "affinity": { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -5298,6 +5293,11 @@ } ] }, + "db": { + "maxConnections": 1000, + "volumeSize": "100Gi" + }, + "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker", "persistence": { "secretName": "postgres-4-secret" }, diff --git a/cluster/helm/splice-postgres/templates/postgres.yaml b/cluster/helm/splice-postgres/templates/postgres.yaml index e846cb3f23..a6d5dbed68 100644 --- a/cluster/helm/splice-postgres/templates/postgres.yaml +++ b/cluster/helm/splice-postgres/templates/postgres.yaml @@ -105,6 +105,10 @@ spec: storage: {{ $.Values.db.volumeSize }} storageClassName: {{ $.Values.db.volumeStorageClass }} volumeMode: Filesystem + {{- with .Values.db.dataSource }} + dataSource: + {{- toYaml . | nindent 8 }} + {{- end }} --- apiVersion: v1 kind: Service diff --git a/cluster/pulumi/common/src/config/configSchema.ts b/cluster/pulumi/common/src/config/configSchema.ts index b4fbd29eee..80d02c453c 100644 --- a/cluster/pulumi/common/src/config/configSchema.ts +++ b/cluster/pulumi/common/src/config/configSchema.ts @@ -14,6 +14,7 @@ const PulumiProjectConfigSchema = z.object({ interAppsDependencies: z.boolean(), cloudSql: CloudSqlConfigSchema, allowDowngrade: z.boolean(), + replacePostgresStatefulSetOnChanges: z.boolean().default(false), }); export type PulumiProjectConfig = z.infer; export const ConfigSchema = z.object({ diff --git a/cluster/pulumi/common/src/postgres.ts b/cluster/pulumi/common/src/postgres.ts index 97e9951ea7..01f5b86499 100644 --- a/cluster/pulumi/common/src/postgres.ts +++ b/cluster/pulumi/common/src/postgres.ts @@ -4,17 +4,20 @@ import * as gcp from '@pulumi/gcp'; import * as pulumi from '@pulumi/pulumi'; import * as random from '@pulumi/random'; import * as _ from 'lodash'; +import { CustomResource } from '@pulumi/kubernetes/apiextensions'; import { Resource } from '@pulumi/pulumi'; import { CnChartVersion } from './artifacts'; import { clusterSmallDisk, CloudSqlConfig, config } from './config'; import { spliceConfig } from './config/config'; +import { hyperdiskSupportConfig } from './config/hyperdiskSupportConfig'; import { + appsAffinityAndTolerations, infraAffinityAndTolerations, installSpliceHelmChart, - nonHyperdiskAppsAffinityAndTolerations, } from './helm'; import { installPostgresPasswordSecret } from './secrets'; +import { standardStorageClassName } from './storage/storageClass'; import { ChartValues, CLUSTER_BASENAME, ExactNamespace, GCP_ZONE } from './utils'; const project = gcp.organizations.getProjectOutput({}); @@ -224,6 +227,35 @@ export class SplicePostgres extends pulumi.ComponentResource implements Postgres // an initial database named cantonnet is created automatically (configured in the Helm chart). const smallDiskSize = clusterSmallDisk ? '240Gi' : undefined; + const supportsHyperdisk = + hyperdiskSupportConfig.hyperdiskSupport.enabled && !useInfraAffinityAndTolerations; + let hyperdiskMigrationValues = {}; + if (supportsHyperdisk && hyperdiskSupportConfig.hyperdiskSupport.migrating) { + const pvcSnapshot = new CustomResource( + `pg-data-${xns.logicalName}-${instanceName}-snapshot`, + { + apiVersion: 'snapshot.storage.k8s.io/v1', + kind: 'VolumeSnapshot', + metadata: { + name: `pg-data-${instanceName}-snapshot`, + namespace: xns.logicalName, + }, + spec: { + volumeSnapshotClassName: 'dev-vsc', + source: { + persistentVolumeClaimName: `pg-data-${instanceName}-0`, + }, + }, + } + ); + hyperdiskMigrationValues = { + dataSource: { + kind: 'VolumeSnapshot', + name: pvcSnapshot.metadata.name, + apiGroup: 'snapshot.storage.k8s.io', + }, + }; + } const pg = installSpliceHelmChart( xns, instanceName, @@ -233,6 +265,13 @@ export class SplicePostgres extends pulumi.ComponentResource implements Postgres volumeSize: overrideDbSizeFromValues ? values?.db?.volumeSize || smallDiskSize : smallDiskSize, + ...(supportsHyperdisk + ? { + volumeStorageClass: standardStorageClassName, + pvcTemplateName: 'pg-data-hd', + ...hyperdiskMigrationValues, + } + : {}), }, persistence: { secretName: this.secretName, @@ -242,11 +281,19 @@ export class SplicePostgres extends pulumi.ComponentResource implements Postgres { aliases: [{ name: logicalNameAlias, type: 'kubernetes:helm.sh/v3:Release' }], dependsOn: [passwordSecret], + ...((supportsHyperdisk && + // during the migration we first delete the stateful set, which keeps the old pvcs (stateful sets always keep the pvcs), and then recreate with the new pvcs + // the stateful sets are immutable so they need to be recreated to force the change of the pvcs + hyperdiskSupportConfig.hyperdiskSupport.migrating) || + spliceConfig.pulumiProjectConfig.replacePostgresStatefulSetOnChanges + ? { + replaceOnChanges: ['*'], + deleteBeforeReplace: true, + } + : {}), }, true, - useInfraAffinityAndTolerations - ? infraAffinityAndTolerations - : nonHyperdiskAppsAffinityAndTolerations + useInfraAffinityAndTolerations ? infraAffinityAndTolerations : appsAffinityAndTolerations ); this.pg = pg; diff --git a/cluster/pulumi/multi-validator/src/postgres.ts b/cluster/pulumi/multi-validator/src/postgres.ts index 3fd8a98fdd..7d8b63ad89 100644 --- a/cluster/pulumi/multi-validator/src/postgres.ts +++ b/cluster/pulumi/multi-validator/src/postgres.ts @@ -4,14 +4,18 @@ import * as pulumi from '@pulumi/pulumi'; import * as random from '@pulumi/random'; import { activeVersion, + appsAffinityAndTolerations, CnInput, ExactNamespace, - installSpliceRunbookHelmChart, - installPostgresPasswordSecret, InstalledHelmChart, - nonHyperdiskAppsAffinityAndTolerations, + installPostgresPasswordSecret, + installSpliceRunbookHelmChart, + spliceConfig, + standardStorageClassName, } from '@lfdecentralizedtrust/splice-pulumi-common'; +import { CustomResource } from '@pulumi/kubernetes/apiextensions'; +import { hyperdiskSupportConfig } from '../../common/src/config/hyperdiskSupportConfig'; import { multiValidatorConfig } from './config'; export function installPostgres( @@ -32,17 +36,66 @@ export function installPostgres( } const config = multiValidatorConfig!; + let hyperdiskMigrationValues = {}; + if ( + hyperdiskSupportConfig.hyperdiskSupport.enabled && + hyperdiskSupportConfig.hyperdiskSupport.migrating + ) { + const pvcSnapshot = new CustomResource(`pg-data-${xns.logicalName}-${name}-snapshot`, { + apiVersion: 'snapshot.storage.k8s.io/v1', + kind: 'VolumeSnapshot', + metadata: { + name: `pg-data-${name}-snapshot`, + namespace: xns.logicalName, + }, + spec: { + volumeSnapshotClassName: 'dev-vsc', + source: { + persistentVolumeClaimName: `pg-data-${name}-0`, + }, + }, + }); + hyperdiskMigrationValues = { + dataSource: { + kind: 'VolumeSnapshot', + name: pvcSnapshot.metadata.name, + apiGroup: 'snapshot.storage.k8s.io', + }, + }; + } return installSpliceRunbookHelmChart( xns, name, 'splice-postgres', { persistence: { secretName }, - db: { volumeSize: config.postgresPvcSize, maxConnections: 1000 }, + db: { + volumeSize: config.postgresPvcSize, + maxConnections: 1000, + ...(hyperdiskSupportConfig.hyperdiskSupport.enabled + ? { + volumeStorageClass: standardStorageClassName, + pvcTemplateName: 'pg-data-hd', + ...hyperdiskMigrationValues, + } + : {}), + }, resources: config.resources?.postgres, - nonHyperdiskAppsAffinityAndTolerations, + appsAffinityAndTolerations, }, activeVersion, - { dependsOn: [passwordSecret, ...dependsOn] } + { + dependsOn: [passwordSecret, ...dependsOn], + ...((hyperdiskSupportConfig.hyperdiskSupport.enabled && + // during the migration we first delete the stateful set, which keeps the old pvcs, and the recreate with the new pvcs + // the stateful sets are immutable so they need to be recreated to force the change of the pvcs + hyperdiskSupportConfig.hyperdiskSupport.migrating) || + spliceConfig.pulumiProjectConfig.replacePostgresStatefulSetOnChanges + ? { + replaceOnChanges: ['*'], + deleteBeforeReplace: true, + } + : {}), + } ); }