add support for migrating cometbft pvcs to hyperdisk (#3658)

Signed-off-by: Nicu Reut <nicu.reut@digitalasset.com>

Author: nicu-da

cluster/helm/splice-cometbft/templates/deployment.yaml

@@ -125,7 +125,7 @@ spec:
           # It is fine to use a persistent volume claim with a deployment instead of a stateful set,
           # since we are not going to scale this deployment.
           persistentVolumeClaim:
-            claimName: {{ include "prefix" (list $.Values "cometbft-data") }}
+            claimName: {{ $.Values.db.pvcName | default (include "prefix" (list $.Values "cometbft-data")) }}
       {{- with $.Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

cluster/helm/splice-cometbft/templates/pvc.yaml

@@ -5,7 +5,7 @@
 apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
-  name: {{ include "prefix" (list .Values "cometbft-data") }}
+  name: {{ .Values.db.pvcName | default (include "prefix" (list .Values "cometbft-data")) }}
   namespace: {{ .Release.Namespace }}
   annotations:
     helm.sh/resource-policy: keep
@@ -18,4 +18,8 @@ spec:
     requests:
       storage: {{ $.Values.db.volumeSize }}
   storageClassName: {{ $.Values.db.volumeStorageClass }}
+  {{- with .Values.db.dataSource }}
+  dataSource:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
   volumeMode: Filesystem

cluster/helm/splice-cometbft/tests/cometbft_deployment_test.yaml

@@ -4,7 +4,6 @@
 suite: "cometbft deployment"
 templates:
   - deployment.yaml
-  - pvc.yaml
 release:
   # Set for testing labels
   name: global-domain-3
@@ -35,15 +34,18 @@ tests:
       - equal:
           path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[0].values[0]
           value: "cometbft"
-  - it: "deploys a PVC"
-    template: pvc.yaml
+      - equal:
+          path: spec.template.spec.volumes[4].persistentVolumeClaim.claimName
+          value: global-domain-3-cometbft-cometbft-data
+  - it: "uses custom pvcName in deployment when set"
+    template: deployment.yaml
+    set:
+      db:
+        pvcName: "custom-pvc-name"
     documentSelector:
       path: kind
-      value: PersistentVolumeClaim
+      value: Deployment
     asserts:
       - equal:
-          path: metadata.name
-          value: global-domain-3-cometbft-cometbft-data
-      - equal:
-          path: metadata.annotations["helm.sh/resource-policy"]
-          value: keep
+          path: spec.template.spec.volumes[4].persistentVolumeClaim.claimName
+          value: custom-pvc-name

cluster/helm/splice-cometbft/tests/cometbft_pvc_test.yaml

@@ -0,0 +1,66 @@
+# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+suite: "cometbft pvc"
+templates:
+  - pvc.yaml
+release:
+  # Set for testing labels
+  name: global-domain-3
+set:
+  node:
+    identifier: "global-domain-3-cometbft"
+tests:
+  - it: "deploys a PVC"
+    template: pvc.yaml
+    documentSelector:
+      path: kind
+      value: PersistentVolumeClaim
+    asserts:
+      - equal:
+          path: metadata.name
+          value: global-domain-3-cometbft-cometbft-data
+      - equal:
+          path: metadata.annotations["helm.sh/resource-policy"]
+          value: keep
+  - it: "uses custom pvcName in PVC when set"
+    template: pvc.yaml
+    set:
+      db:
+        pvcName: "custom-pvc-name"
+    documentSelector:
+      path: kind
+      value: PersistentVolumeClaim
+    asserts:
+      - equal:
+          path: metadata.name
+          value: custom-pvc-name
+  - it: "does not include dataSource in PVC when not set"
+    template: pvc.yaml
+    documentSelector:
+      path: kind
+      value: PersistentVolumeClaim
+    asserts:
+      - isNull:
+          path: spec.dataSource
+  - it: "includes dataSource in PVC when set"
+    template: pvc.yaml
+    set:
+      db:
+        dataSource:
+          kind: VolumeSnapshot
+          name: my-snapshot
+          apiGroup: snapshot.storage.k8s.io
+    documentSelector:
+      path: kind
+      value: PersistentVolumeClaim
+    asserts:
+      - equal:
+          path: spec.dataSource.kind
+          value: VolumeSnapshot
+      - equal:
+          path: spec.dataSource.name
+          value: my-snapshot
+      - equal:
+          path: spec.dataSource.apiGroup
+          value: snapshot.storage.k8s.io

cluster/pulumi/common-sv/src/singleSvConfig.ts

@@ -21,7 +21,6 @@ const SvCometbftConfigSchema = z
     validatorKeyAddress: z.string().optional(),
     // defaults to {svName}-cometbft-keys if not set
     keysGcpSecret: z.string().optional(),
-    snapshotName: z.string().optional(),
     resources: K8sResourceSchema,
   })
   .strict();

cluster/pulumi/common-sv/src/synchronizer/cometbft.ts

@@ -1,29 +1,30 @@
 // Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
-import * as gcp from '@pulumi/gcp';
 import * as k8s from '@pulumi/kubernetes';
 import * as _ from 'lodash';
 import {
   activeVersion,
+  appsAffinityAndTolerations,
   CLUSTER_BASENAME,
   CLUSTER_HOSTNAME,
   clusterSmallDisk,
   config,
   DomainMigrationIndex,
   ExactNamespace,
-  GCP_ZONE,
   InstalledHelmChart,
   installSpliceHelmChart,
   isDevNet,
   loadYamlFromFile,
-  nonHyperdiskAppsAffinityAndTolerations,
   SPLICE_ROOT,
   SpliceCustomResourceOptions,
+  standardStorageClassName,
   svCometBftKeysFromSecret,
   withAddedDependencies,
 } from '@lfdecentralizedtrust/splice-pulumi-common';
 import { CnChartVersion } from '@lfdecentralizedtrust/splice-pulumi-common/src/artifacts';
-import { jsonStringify, Output, Resource } from '@pulumi/pulumi';
+import { hyperdiskSupportConfig } from '@lfdecentralizedtrust/splice-pulumi-common/src/config/hyperdiskSupportConfig';
+import { CustomResource } from '@pulumi/kubernetes/apiextensions';
+import { jsonStringify, Output } from '@pulumi/pulumi';
 
 import { svsConfig } from '../config';
 import { SingleSvConfiguration } from '../singleSvConfig';
@@ -104,6 +105,41 @@ export function installCometBftNode(
       ? undefined
       : installCometBftKeysSecret(xns, nodeConfig.validator.keyAddress, migrationId);
 
+  let hyperdiskDbValues = {};
+  if (hyperdiskSupportConfig.hyperdiskSupport.enabled) {
+    hyperdiskDbValues = {
+      pvcName: `cometbft-migration-${migrationId}-hd-pvc`,
+      volumeStorageClass: standardStorageClassName,
+    };
+    if (hyperdiskSupportConfig.hyperdiskSupport.migrating) {
+      const pvcSnapshot = new CustomResource(
+        `cometbft-${xns.logicalName}-migration-${migrationId}-snapshot`,
+        {
+          apiVersion: 'snapshot.storage.k8s.io/v1',
+          kind: 'VolumeSnapshot',
+          metadata: {
+            name: `cometbft-migration-${migrationId}-pd-snapshot`,
+            namespace: xns.logicalName,
+          },
+          spec: {
+            volumeSnapshotClassName: 'dev-vsc',
+            source: {
+              persistentVolumeClaimName: `global-domain-${migrationId}-cometbft-cometbft-data`,
+            },
+          },
+        }
+      );
+      hyperdiskDbValues = {
+        ...hyperdiskDbValues,
+        dataSource: {
+          kind: 'VolumeSnapshot',
+          name: pvcSnapshot.metadata.name,
+          apiGroup: 'snapshot.storage.k8s.io',
+        },
+      };
+    }
+  }
+
   const cometbftChartValues = _.mergeWith(cometBftValues, {
     sv1: nodeConfigs.sv1,
     istioVirtualService: {
@@ -138,67 +174,12 @@ export function installCometBftNode(
     },
     db: {
       volumeSize: clusterSmallDisk ? '240Gi' : svsConfig?.cometbft?.volumeSize,
+      ...hyperdiskDbValues,
     },
     extraLogLevelFlags: svConfiguration.logging?.cometbftExtraLogLevelFlags,
     serviceAccountName: imagePullServiceAccountName,
     resources: svConfiguration.cometbft?.resources,
   });
-  const svIdentifier = nodeConfigs.selfSvNodeName;
-  const svIdentifierWithMigration = `${svIdentifier}-m${migrationId}`;
-  let volumeDependecies: Resource[] = [];
-  if (svConfiguration?.cometbft?.snapshotName) {
-    const volumeSize = cometbftChartValues.db.volumeSize;
-    const diskSnapshot = gcp.compute.getSnapshot({
-      name: svConfiguration.cometbft.snapshotName,
-    });
-
-    if (!GCP_ZONE) {
-      throw new Error('Zone is required to create a disk');
-    }
-    const restoredDisk = new gcp.compute.Disk(
-      `${svIdentifierWithMigration}-cometbft-restored-data`,
-      {
-        name: `${svIdentifierWithMigration}-cometbft-restored-disk`,
-        // eslint-disable-next-line promise/prefer-await-to-then
-        size: diskSnapshot.then(snapshot => snapshot.diskSizeGb),
-        // eslint-disable-next-line promise/prefer-await-to-then
-        snapshot: diskSnapshot.then(snapshot => snapshot.selfLink),
-        type: 'pd-ssd',
-        zone: GCP_ZONE,
-      },
-      opts
-    );
-
-    // create the underlying persistent volume that will be used by cometbft from the state of an existing PV
-    volumeDependecies = [
-      new k8s.core.v1.PersistentVolume(
-        `${svIdentifier}-cometbft-data`,
-        {
-          metadata: {
-            name: `${svIdentifier}-cometbft-data-pv`,
-          },
-          spec: {
-            capacity: {
-              storage: volumeSize,
-            },
-            volumeMode: 'Filesystem',
-            accessModes: ['ReadWriteOnce'],
-            persistentVolumeReclaimPolicy: 'Delete',
-            storageClassName: cometbftChartValues.db.volumeStorageClass,
-            claimRef: {
-              name: `global-domain-${migrationId}-cometbft-cometbft-data`,
-              namespace: xns.ns.metadata.name,
-            },
-            csi: {
-              driver: 'pd.csi.storage.gke.io',
-              volumeHandle: restoredDisk.id,
-            },
-          },
-        },
-        opts
-      ),
-    ];
-  }
   const protectCometBft = svsConfig?.cometbft?.protected ?? false;
   const release = installSpliceHelmChart(
     xns,
@@ -208,13 +189,13 @@ export function installCometBftNode(
     version,
     // support old runbook names, can be removed once the runbooks are all reset and latest release is >= 0.2.x
     {
-      ...withAddedDependencies(opts, volumeDependecies.concat(keysSecret ? [keysSecret] : [])),
+      ...withAddedDependencies(opts, keysSecret ? [keysSecret] : []),
       aliases: [{ name: `global-domain-${migrationId}-cometbft`, parent: undefined }],
       ignoreChanges: ['name'],
       protect: disableProtection ? false : protectCometBft,
     },
     true,
-    nonHyperdiskAppsAffinityAndTolerations
+    appsAffinityAndTolerations
   );
   return { rpcServiceName: `${nodeConfig.identifier}-cometbft-rpc`, release };
 }