add support for migrating postgres pvcs to hyperdisk

nicu-da · nicu-da · commit e9a620f86d1a · 2026-01-22T15:22:12.000Z
[static]

Signed-off-by: Nicu Reut &lt;nicu.reut@digitalasset.com&gt;
diff --git a/cluster/expected/multi-validator/expected.json b/cluster/expected/multi-validator/expected.json
@@ -4877,12 +4877,7 @@
             }
           }
         },
-        "db": {
-          "maxConnections": 1000,
-          "volumeSize": "100Gi"
-        },
-        "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
-        "nonHyperdiskAppsAffinityAndTolerations": {
+        "appsAffinityAndTolerations": {
           "affinity": {
             "nodeAffinity": {
               "requiredDuringSchedulingIgnoredDuringExecution": {
@@ -4914,6 +4909,11 @@
             }
           ]
         },
+        "db": {
+          "maxConnections": 1000,
+          "volumeSize": "100Gi"
+        },
+        "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
         "persistence": {
           "secretName": "postgres-0-secret"
         },
@@ -4973,12 +4973,7 @@
             }
           }
         },
-        "db": {
-          "maxConnections": 1000,
-          "volumeSize": "100Gi"
-        },
-        "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
-        "nonHyperdiskAppsAffinityAndTolerations": {
+        "appsAffinityAndTolerations": {
           "affinity": {
             "nodeAffinity": {
               "requiredDuringSchedulingIgnoredDuringExecution": {
@@ -5010,6 +5005,11 @@
             }
           ]
         },
+        "db": {
+          "maxConnections": 1000,
+          "volumeSize": "100Gi"
+        },
+        "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
         "persistence": {
           "secretName": "postgres-1-secret"
         },
@@ -5069,12 +5069,7 @@
             }
           }
         },
-        "db": {
-          "maxConnections": 1000,
-          "volumeSize": "100Gi"
-        },
-        "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
-        "nonHyperdiskAppsAffinityAndTolerations": {
+        "appsAffinityAndTolerations": {
           "affinity": {
             "nodeAffinity": {
               "requiredDuringSchedulingIgnoredDuringExecution": {
@@ -5106,6 +5101,11 @@
             }
           ]
         },
+        "db": {
+          "maxConnections": 1000,
+          "volumeSize": "100Gi"
+        },
+        "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
         "persistence": {
           "secretName": "postgres-2-secret"
         },
@@ -5165,12 +5165,7 @@
             }
           }
         },
-        "db": {
-          "maxConnections": 1000,
-          "volumeSize": "100Gi"
-        },
-        "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
-        "nonHyperdiskAppsAffinityAndTolerations": {
+        "appsAffinityAndTolerations": {
           "affinity": {
             "nodeAffinity": {
               "requiredDuringSchedulingIgnoredDuringExecution": {
@@ -5202,6 +5197,11 @@
             }
           ]
         },
+        "db": {
+          "maxConnections": 1000,
+          "volumeSize": "100Gi"
+        },
+        "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
         "persistence": {
           "secretName": "postgres-3-secret"
         },
@@ -5261,12 +5261,7 @@
             }
           }
         },
-        "db": {
-          "maxConnections": 1000,
-          "volumeSize": "100Gi"
-        },
-        "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
-        "nonHyperdiskAppsAffinityAndTolerations": {
+        "appsAffinityAndTolerations": {
           "affinity": {
             "nodeAffinity": {
               "requiredDuringSchedulingIgnoredDuringExecution": {
@@ -5298,6 +5293,11 @@
             }
           ]
         },
+        "db": {
+          "maxConnections": 1000,
+          "volumeSize": "100Gi"
+        },
+        "imageRepo": "us-central1-docker.pkg.dev/da-cn-shared/ghcr/digital-asset/decentralized-canton-sync-dev/docker",
         "persistence": {
           "secretName": "postgres-4-secret"
         },
diff --git a/cluster/helm/splice-postgres/templates/postgres.yaml b/cluster/helm/splice-postgres/templates/postgres.yaml
@@ -105,6 +105,10 @@ spec:
           storage: {{ $.Values.db.volumeSize }}
       storageClassName: {{ $.Values.db.volumeStorageClass }}
       volumeMode: Filesystem
+      {{- with .Values.db.dataSource }}
+      dataSource:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
 ---
 apiVersion: v1
 kind: Service
diff --git a/cluster/pulumi/common/src/config/configSchema.ts b/cluster/pulumi/common/src/config/configSchema.ts
@@ -14,6 +14,7 @@ const PulumiProjectConfigSchema = z.object({
   interAppsDependencies: z.boolean(),
   cloudSql: CloudSqlConfigSchema,
   allowDowngrade: z.boolean(),
+  replacePostgresStatefulSetOnChanges: z.boolean().default(false),
 });
 export type PulumiProjectConfig = z.infer<typeof PulumiProjectConfigSchema>;
 export const ConfigSchema = z.object({
diff --git a/cluster/pulumi/common/src/postgres.ts b/cluster/pulumi/common/src/postgres.ts
@@ -4,17 +4,20 @@ import * as gcp from '@pulumi/gcp';
 import * as pulumi from '@pulumi/pulumi';
 import * as random from '@pulumi/random';
 import * as _ from 'lodash';
+import { CustomResource } from '@pulumi/kubernetes/apiextensions';
 import { Resource } from '@pulumi/pulumi';
 
 import { CnChartVersion } from './artifacts';
 import { clusterSmallDisk, CloudSqlConfig, config } from './config';
 import { spliceConfig } from './config/config';
+import { hyperdiskSupportConfig } from './config/hyperdiskSupportConfig';
 import {
+  appsAffinityAndTolerations,
   infraAffinityAndTolerations,
   installSpliceHelmChart,
-  nonHyperdiskAppsAffinityAndTolerations,
 } from './helm';
 import { installPostgresPasswordSecret } from './secrets';
+import { standardStorageClassName } from './storage/storageClass';
 import { ChartValues, CLUSTER_BASENAME, ExactNamespace, GCP_ZONE } from './utils';
 
 const project = gcp.organizations.getProjectOutput({});
@@ -224,6 +227,35 @@ export class SplicePostgres extends pulumi.ComponentResource implements Postgres
 
     // an initial database named cantonnet is created automatically (configured in the Helm chart).
     const smallDiskSize = clusterSmallDisk ? '240Gi' : undefined;
+    const supportsHyperdisk =
+      hyperdiskSupportConfig.hyperdiskSupport.enabled && !useInfraAffinityAndTolerations;
+    let hyperdiskMigrationValues = {};
+    if (supportsHyperdisk && hyperdiskSupportConfig.hyperdiskSupport.migrating) {
+      const pvcSnapshot = new CustomResource(
+        `pg-data-${xns.logicalName}-${instanceName}-snapshot`,
+        {
+          apiVersion: 'snapshot.storage.k8s.io/v1',
+          kind: 'VolumeSnapshot',
+          metadata: {
+            name: `pg-data-${instanceName}-snapshot`,
+            namespace: xns.logicalName,
+          },
+          spec: {
+            volumeSnapshotClassName: 'dev-vsc',
+            source: {
+              persistentVolumeClaimName: `pg-data-${instanceName}-0`,
+            },
+          },
+        }
+      );
+      hyperdiskMigrationValues = {
+        dataSource: {
+          kind: 'VolumeSnapshot',
+          name: pvcSnapshot.metadata.name,
+          apiGroup: 'snapshot.storage.k8s.io',
+        },
+      };
+    }
     const pg = installSpliceHelmChart(
       xns,
       instanceName,
@@ -233,6 +265,13 @@ export class SplicePostgres extends pulumi.ComponentResource implements Postgres
           volumeSize: overrideDbSizeFromValues
             ? values?.db?.volumeSize || smallDiskSize
             : smallDiskSize,
+          ...(supportsHyperdisk
+            ? {
+                volumeStorageClass: standardStorageClassName,
+                pvcTemplateName: 'pg-data-hd',
+                ...hyperdiskMigrationValues,
+              }
+            : {}),
         },
         persistence: {
           secretName: this.secretName,
@@ -242,11 +281,19 @@ export class SplicePostgres extends pulumi.ComponentResource implements Postgres
       {
         aliases: [{ name: logicalNameAlias, type: 'kubernetes:helm.sh/v3:Release' }],
         dependsOn: [passwordSecret],
+        ...((supportsHyperdisk &&
+          // during the migration we first delete the stateful set, which keeps the old pvcs, and the recreate with the new pvcs
+          // the stateful sets are immutable so they need to be recreated to force the change of the pvcs
+          hyperdiskSupportConfig.hyperdiskSupport.migrating) ||
+        spliceConfig.pulumiProjectConfig.replacePostgresStatefulSetOnChanges
+          ? {
+              replaceOnChanges: ['*'],
+              deleteBeforeReplace: true,
+            }
+          : {}),
       },
       true,
-      useInfraAffinityAndTolerations
-        ? infraAffinityAndTolerations
-        : nonHyperdiskAppsAffinityAndTolerations
+      useInfraAffinityAndTolerations ? infraAffinityAndTolerations : appsAffinityAndTolerations
     );
     this.pg = pg;
 
diff --git a/cluster/pulumi/multi-validator/src/postgres.ts b/cluster/pulumi/multi-validator/src/postgres.ts
@@ -4,14 +4,18 @@ import * as pulumi from '@pulumi/pulumi';
 import * as random from '@pulumi/random';
 import {
   activeVersion,
+  appsAffinityAndTolerations,
   CnInput,
   ExactNamespace,
-  installSpliceRunbookHelmChart,
-  installPostgresPasswordSecret,
   InstalledHelmChart,
-  nonHyperdiskAppsAffinityAndTolerations,
+  installPostgresPasswordSecret,
+  installSpliceRunbookHelmChart,
+  spliceConfig,
+  standardStorageClassName,
 } from '@lfdecentralizedtrust/splice-pulumi-common';
+import { CustomResource } from '@pulumi/kubernetes/apiextensions';
 
+import { hyperdiskSupportConfig } from '../../common/src/config/hyperdiskSupportConfig';
 import { multiValidatorConfig } from './config';
 
 export function installPostgres(
@@ -32,17 +36,66 @@ export function installPostgres(
   }
   const config = multiValidatorConfig!;
 
+  let hyperdiskMigrationValues = {};
+  if (
+    hyperdiskSupportConfig.hyperdiskSupport.enabled &&
+    hyperdiskSupportConfig.hyperdiskSupport.migrating
+  ) {
+    const pvcSnapshot = new CustomResource(`pg-data-${xns.logicalName}-${name}-snapshot`, {
+      apiVersion: 'snapshot.storage.k8s.io/v1',
+      kind: 'VolumeSnapshot',
+      metadata: {
+        name: `pg-data-${name}-snapshot`,
+        namespace: xns.logicalName,
+      },
+      spec: {
+        volumeSnapshotClassName: 'dev-vsc',
+        source: {
+          persistentVolumeClaimName: `pg-data-${name}-0`,
+        },
+      },
+    });
+    hyperdiskMigrationValues = {
+      dataSource: {
+        kind: 'VolumeSnapshot',
+        name: pvcSnapshot.metadata.name,
+        apiGroup: 'snapshot.storage.k8s.io',
+      },
+    };
+  }
   return installSpliceRunbookHelmChart(
     xns,
     name,
     'splice-postgres',
     {
       persistence: { secretName },
-      db: { volumeSize: config.postgresPvcSize, maxConnections: 1000 },
+      db: {
+        volumeSize: config.postgresPvcSize,
+        maxConnections: 1000,
+        ...(hyperdiskSupportConfig.hyperdiskSupport.enabled
+          ? {
+              volumeStorageClass: standardStorageClassName,
+              pvcTemplateName: 'pg-data-hd',
+              ...hyperdiskMigrationValues,
+            }
+          : {}),
+      },
       resources: config.resources?.postgres,
-      nonHyperdiskAppsAffinityAndTolerations,
+      appsAffinityAndTolerations,
     },
     activeVersion,
-    { dependsOn: [passwordSecret, ...dependsOn] }
+    {
+      dependsOn: [passwordSecret, ...dependsOn],
+      ...((hyperdiskSupportConfig.hyperdiskSupport.enabled &&
+        // during the migration we first delete the stateful set, which keeps the old pvcs, and the recreate with the new pvcs
+        // the stateful sets are immutable so they need to be recreated to force the change of the pvcs
+        hyperdiskSupportConfig.hyperdiskSupport.migrating) ||
+      spliceConfig.pulumiProjectConfig.replacePostgresStatefulSetOnChanges
+        ? {
+            replaceOnChanges: ['*'],
+            deleteBeforeReplace: true,
+          }
+        : {}),
+    }
   );
 }
