diff --git a/cluster/helm/splice-cometbft/templates/deployment.yaml b/cluster/helm/splice-cometbft/templates/deployment.yaml index bc4c222dce..65d13d160e 100644 --- a/cluster/helm/splice-cometbft/templates/deployment.yaml +++ b/cluster/helm/splice-cometbft/templates/deployment.yaml @@ -125,7 +125,7 @@ spec: # It is fine to use a persistent volume claim with a deployment instead of a stateful set, # since we are not going to scale this deployment. persistentVolumeClaim: - claimName: {{ include "prefix" (list $.Values "cometbft-data") }} + claimName: {{ $.Values.db.pvcName | default (include "prefix" (list $.Values "cometbft-data")) }} {{- with $.Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/cluster/helm/splice-cometbft/templates/pvc.yaml b/cluster/helm/splice-cometbft/templates/pvc.yaml index 469b0589ce..4622b25308 100644 --- a/cluster/helm/splice-cometbft/templates/pvc.yaml +++ b/cluster/helm/splice-cometbft/templates/pvc.yaml @@ -5,7 +5,7 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: - name: {{ include "prefix" (list .Values "cometbft-data") }} + name: {{ .Values.db.pvcName | default (include "prefix" (list .Values "cometbft-data")) }} namespace: {{ .Release.Namespace }} annotations: helm.sh/resource-policy: keep @@ -18,4 +18,8 @@ spec: requests: storage: {{ $.Values.db.volumeSize }} storageClassName: {{ $.Values.db.volumeStorageClass }} + {{- with .Values.db.dataSource }} + dataSource: + {{- toYaml . | nindent 4 }} + {{- end }} volumeMode: Filesystem diff --git a/cluster/helm/splice-cometbft/tests/cometbft_deployment_test.yaml b/cluster/helm/splice-cometbft/tests/cometbft_deployment_test.yaml index ee5b940132..a50de5ef44 100644 --- a/cluster/helm/splice-cometbft/tests/cometbft_deployment_test.yaml +++ b/cluster/helm/splice-cometbft/tests/cometbft_deployment_test.yaml @@ -4,7 +4,6 @@ suite: "cometbft deployment" templates: - deployment.yaml - - pvc.yaml release: # Set for testing labels name: global-domain-3 @@ -35,15 +34,18 @@ tests: - equal: path: spec.template.spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].labelSelector.matchExpressions[0].values[0] value: "cometbft" - - it: "deploys a PVC" - template: pvc.yaml + - equal: + path: spec.template.spec.volumes[4].persistentVolumeClaim.claimName + value: global-domain-3-cometbft-cometbft-data + - it: "uses custom pvcName in deployment when set" + template: deployment.yaml + set: + db: + pvcName: "custom-pvc-name" documentSelector: path: kind - value: PersistentVolumeClaim + value: Deployment asserts: - equal: - path: metadata.name - value: global-domain-3-cometbft-cometbft-data - - equal: - path: metadata.annotations["helm.sh/resource-policy"] - value: keep + path: spec.template.spec.volumes[4].persistentVolumeClaim.claimName + value: custom-pvc-name diff --git a/cluster/helm/splice-cometbft/tests/cometbft_pvc_test.yaml b/cluster/helm/splice-cometbft/tests/cometbft_pvc_test.yaml new file mode 100644 index 0000000000..bbc4c8bf86 --- /dev/null +++ b/cluster/helm/splice-cometbft/tests/cometbft_pvc_test.yaml @@ -0,0 +1,66 @@ +# Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: "cometbft pvc" +templates: + - pvc.yaml +release: + # Set for testing labels + name: global-domain-3 +set: + node: + identifier: "global-domain-3-cometbft" +tests: + - it: "deploys a PVC" + template: pvc.yaml + documentSelector: + path: kind + value: PersistentVolumeClaim + asserts: + - equal: + path: metadata.name + value: global-domain-3-cometbft-cometbft-data + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - it: "uses custom pvcName in PVC when set" + template: pvc.yaml + set: + db: + pvcName: "custom-pvc-name" + documentSelector: + path: kind + value: PersistentVolumeClaim + asserts: + - equal: + path: metadata.name + value: custom-pvc-name + - it: "does not include dataSource in PVC when not set" + template: pvc.yaml + documentSelector: + path: kind + value: PersistentVolumeClaim + asserts: + - isNull: + path: spec.dataSource + - it: "includes dataSource in PVC when set" + template: pvc.yaml + set: + db: + dataSource: + kind: VolumeSnapshot + name: my-snapshot + apiGroup: snapshot.storage.k8s.io + documentSelector: + path: kind + value: PersistentVolumeClaim + asserts: + - equal: + path: spec.dataSource.kind + value: VolumeSnapshot + - equal: + path: spec.dataSource.name + value: my-snapshot + - equal: + path: spec.dataSource.apiGroup + value: snapshot.storage.k8s.io diff --git a/cluster/pulumi/common-sv/src/singleSvConfig.ts b/cluster/pulumi/common-sv/src/singleSvConfig.ts index 47790ca53d..0ee9933832 100644 --- a/cluster/pulumi/common-sv/src/singleSvConfig.ts +++ b/cluster/pulumi/common-sv/src/singleSvConfig.ts @@ -21,7 +21,6 @@ const SvCometbftConfigSchema = z validatorKeyAddress: z.string().optional(), // defaults to {svName}-cometbft-keys if not set keysGcpSecret: z.string().optional(), - snapshotName: z.string().optional(), resources: K8sResourceSchema, }) .strict(); diff --git a/cluster/pulumi/common-sv/src/synchronizer/cometbft.ts b/cluster/pulumi/common-sv/src/synchronizer/cometbft.ts index c92be43902..96ab189411 100644 --- a/cluster/pulumi/common-sv/src/synchronizer/cometbft.ts +++ b/cluster/pulumi/common-sv/src/synchronizer/cometbft.ts @@ -1,29 +1,30 @@ // Copyright (c) 2024 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import * as gcp from '@pulumi/gcp'; import * as k8s from '@pulumi/kubernetes'; import * as _ from 'lodash'; import { activeVersion, + appsAffinityAndTolerations, CLUSTER_BASENAME, CLUSTER_HOSTNAME, clusterSmallDisk, config, DomainMigrationIndex, ExactNamespace, - GCP_ZONE, InstalledHelmChart, installSpliceHelmChart, isDevNet, loadYamlFromFile, - nonHyperdiskAppsAffinityAndTolerations, SPLICE_ROOT, SpliceCustomResourceOptions, + standardStorageClassName, svCometBftKeysFromSecret, withAddedDependencies, } from '@lfdecentralizedtrust/splice-pulumi-common'; import { CnChartVersion } from '@lfdecentralizedtrust/splice-pulumi-common/src/artifacts'; -import { jsonStringify, Output, Resource } from '@pulumi/pulumi'; +import { hyperdiskSupportConfig } from '@lfdecentralizedtrust/splice-pulumi-common/src/config/hyperdiskSupportConfig'; +import { CustomResource } from '@pulumi/kubernetes/apiextensions'; +import { jsonStringify, Output } from '@pulumi/pulumi'; import { svsConfig } from '../config'; import { SingleSvConfiguration } from '../singleSvConfig'; @@ -104,6 +105,41 @@ export function installCometBftNode( ? undefined : installCometBftKeysSecret(xns, nodeConfig.validator.keyAddress, migrationId); + let hyperdiskDbValues = {}; + if (hyperdiskSupportConfig.hyperdiskSupport.enabled) { + hyperdiskDbValues = { + pvcName: `cometbft-migration-${migrationId}-hd-pvc`, + volumeStorageClass: standardStorageClassName, + }; + if (hyperdiskSupportConfig.hyperdiskSupport.migrating) { + const pvcSnapshot = new CustomResource( + `cometbft-${xns.logicalName}-migration-${migrationId}-snapshot`, + { + apiVersion: 'snapshot.storage.k8s.io/v1', + kind: 'VolumeSnapshot', + metadata: { + name: `cometbft-migration-${migrationId}-pd-snapshot`, + namespace: xns.logicalName, + }, + spec: { + volumeSnapshotClassName: 'dev-vsc', + source: { + persistentVolumeClaimName: `global-domain-${migrationId}-cometbft-cometbft-data`, + }, + }, + } + ); + hyperdiskDbValues = { + ...hyperdiskDbValues, + dataSource: { + kind: 'VolumeSnapshot', + name: pvcSnapshot.metadata.name, + apiGroup: 'snapshot.storage.k8s.io', + }, + }; + } + } + const cometbftChartValues = _.mergeWith(cometBftValues, { sv1: nodeConfigs.sv1, istioVirtualService: { @@ -138,67 +174,12 @@ export function installCometBftNode( }, db: { volumeSize: clusterSmallDisk ? '240Gi' : svsConfig?.cometbft?.volumeSize, + ...hyperdiskDbValues, }, extraLogLevelFlags: svConfiguration.logging?.cometbftExtraLogLevelFlags, serviceAccountName: imagePullServiceAccountName, resources: svConfiguration.cometbft?.resources, }); - const svIdentifier = nodeConfigs.selfSvNodeName; - const svIdentifierWithMigration = `${svIdentifier}-m${migrationId}`; - let volumeDependecies: Resource[] = []; - if (svConfiguration?.cometbft?.snapshotName) { - const volumeSize = cometbftChartValues.db.volumeSize; - const diskSnapshot = gcp.compute.getSnapshot({ - name: svConfiguration.cometbft.snapshotName, - }); - - if (!GCP_ZONE) { - throw new Error('Zone is required to create a disk'); - } - const restoredDisk = new gcp.compute.Disk( - `${svIdentifierWithMigration}-cometbft-restored-data`, - { - name: `${svIdentifierWithMigration}-cometbft-restored-disk`, - // eslint-disable-next-line promise/prefer-await-to-then - size: diskSnapshot.then(snapshot => snapshot.diskSizeGb), - // eslint-disable-next-line promise/prefer-await-to-then - snapshot: diskSnapshot.then(snapshot => snapshot.selfLink), - type: 'pd-ssd', - zone: GCP_ZONE, - }, - opts - ); - - // create the underlying persistent volume that will be used by cometbft from the state of an existing PV - volumeDependecies = [ - new k8s.core.v1.PersistentVolume( - `${svIdentifier}-cometbft-data`, - { - metadata: { - name: `${svIdentifier}-cometbft-data-pv`, - }, - spec: { - capacity: { - storage: volumeSize, - }, - volumeMode: 'Filesystem', - accessModes: ['ReadWriteOnce'], - persistentVolumeReclaimPolicy: 'Delete', - storageClassName: cometbftChartValues.db.volumeStorageClass, - claimRef: { - name: `global-domain-${migrationId}-cometbft-cometbft-data`, - namespace: xns.ns.metadata.name, - }, - csi: { - driver: 'pd.csi.storage.gke.io', - volumeHandle: restoredDisk.id, - }, - }, - }, - opts - ), - ]; - } const protectCometBft = svsConfig?.cometbft?.protected ?? false; const release = installSpliceHelmChart( xns, @@ -208,13 +189,13 @@ export function installCometBftNode( version, // support old runbook names, can be removed once the runbooks are all reset and latest release is >= 0.2.x { - ...withAddedDependencies(opts, volumeDependecies.concat(keysSecret ? [keysSecret] : [])), + ...withAddedDependencies(opts, keysSecret ? [keysSecret] : []), aliases: [{ name: `global-domain-${migrationId}-cometbft`, parent: undefined }], ignoreChanges: ['name'], protect: disableProtection ? false : protectCometBft, }, true, - nonHyperdiskAppsAffinityAndTolerations + appsAffinityAndTolerations ); return { rpcServiceName: `${nodeConfig.identifier}-cometbft-rpc`, release }; }