diff --git a/e2e-tests/pvc-resize/compare/select-1.sql b/e2e-tests/pvc-resize/compare/select-1.sql new file mode 100644 index 0000000000..8e738f4cf2 --- /dev/null +++ b/e2e-tests/pvc-resize/compare/select-1.sql @@ -0,0 +1 @@ +100500 diff --git a/e2e-tests/pvc-resize/run b/e2e-tests/pvc-resize/run new file mode 100755 index 0000000000..7756e7cd79 --- /dev/null +++ b/e2e-tests/pvc-resize/run @@ -0,0 +1,38 @@ +#!/bin/bash + +set -o errexit + +test_dir=$(realpath $(dirname $0)) +. ${test_dir}/../functions + +set_debug + +create_infra ${namespace} + +desc 'create first PXC cluster' +cluster="some-name" +spinup_pxc "${cluster}" "$conf_dir/$cluster.yml" "3" "10" "${conf_dir}/secrets.yml" + +kubectl_bin patch pxc "$cluster" --type=merge --patch '{ + "spec": { "pxc": { "volumeSpec": { "persistentVolumeClaim": { "resources": { "requests": { "storage": "4Gi" } } } } } } +}' +wait_cluster_consistency "$cluster" 3 2 + +for pvc in $(kubectl_bin get pvc -l app.kubernetes.io/component=pxc -o name); do + retry=0 + until [[ $(kubectl_bin get ${pvc} -o jsonpath={.status.capacity.storage}) == "4Gi" ]]; do + if [[ $retry -ge 60 ]]; then + echo "PVC ${pvc} was not resized, max retries exceeded" + exit 1 + fi + + echo "Waiting for PVC ${pvc} to be resized" + sleep 5 + + retry=$((retry + 1)) + done + echo "PVC ${pvc} was resized" +done + +destroy "${namespace}" +desc "test passed" \ No newline at end of file diff --git a/e2e-tests/run-pr.csv b/e2e-tests/run-pr.csv index fd90f6a65a..c7125bac0c 100644 --- a/e2e-tests/run-pr.csv +++ b/e2e-tests/run-pr.csv @@ -35,3 +35,5 @@ upgrade-proxysql,8.0 users,5.7 users,8.0 validation-hook,8.0 +pvc-resize,5.7 +pvc-resize,8.0 \ No newline at end of file diff --git a/pkg/controller/pxc/controller.go b/pkg/controller/pxc/controller.go index a1d4fd31b3..cced298f16 100644 --- a/pkg/controller/pxc/controller.go +++ b/pkg/controller/pxc/controller.go @@ -294,6 +294,10 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(ctx context.Context, request r log.Info("failed to ensure version, running with default", "error", err) } } + err = r.reconcilePersistentVolumes(ctx, o) + if err != nil { + return reconcile.Result{}, errors.Wrap(err, "reconcile persistent volumes") + } err = r.deploy(ctx, o) if err != nil { diff --git a/pkg/controller/pxc/volumes.go b/pkg/controller/pxc/volumes.go new file mode 100644 index 0000000000..9e09e899b9 --- /dev/null +++ b/pkg/controller/pxc/volumes.go @@ -0,0 +1,95 @@ +package pxc + +import ( + "context" + "strings" + + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + + api "github.com/percona/percona-xtradb-cluster-operator/pkg/apis/pxc/v1" + "github.com/percona/percona-xtradb-cluster-operator/pkg/pxc/app/statefulset" +) + +func (r *ReconcilePerconaXtraDBCluster) reconcilePersistentVolumes(ctx context.Context, cr *api.PerconaXtraDBCluster) error { + log := logf.FromContext(ctx) + + pxcSet := statefulset.NewNode(cr) + sts := pxcSet.StatefulSet() + + err := r.client.Get(ctx, client.ObjectKeyFromObject(sts), sts) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + return errors.Wrapf(err, "get statefulset/%s", sts.Name) + } + + if cr.Spec.PXC.VolumeSpec.PersistentVolumeClaim == nil { + return nil + } + + var volumeTemplate corev1.PersistentVolumeClaim + for _, vct := range sts.Spec.VolumeClaimTemplates { + if vct.Name == "datadir" { + volumeTemplate = vct + } + } + + requested := cr.Spec.PXC.VolumeSpec.PersistentVolumeClaim.Resources.Requests[corev1.ResourceStorage] + actual := volumeTemplate.Spec.Resources.Requests[corev1.ResourceStorage] + + if requested.Cmp(actual) < 0 { + return errors.Wrap(err, "requested storage is less than actual") + } + + if requested.Cmp(actual) == 0 { + return nil + } + + // Check storage class for allowVolumeExpansion field + + labels := map[string]string{ + "app.kubernetes.io/component": "pxc", + "app.kubernetes.io/instance": cr.Name, + "app.kubernetes.io/managed-by": "percona-xtradb-cluster-operator", + "app.kubernetes.io/name": "percona-xtradb-cluster", + "app.kubernetes.io/part-of": "percona-xtradb-cluster", + } + + pvcList := corev1.PersistentVolumeClaimList{} + if err := r.client.List(ctx, &pvcList, client.InNamespace(cr.Namespace), client.MatchingLabels(labels)); err != nil { + return errors.Wrap(err, "list persistentvolumeclaims") + } + + pvcNames := make([]string, 0, len(pvcList.Items)) + for _, pvc := range pvcList.Items { + pvcNames = append(pvcNames, pvc.Name) + } + + log.Info("Resizing PVCs", "requested", requested, "actual", actual, "pvcList", strings.Join(pvcNames, ",")) + + log.Info("Deleting statefulset", "name", sts.Name) + + if err := r.client.Delete(ctx, sts, client.PropagationPolicy("Orphan")); err != nil { + return errors.Wrapf(err, "delete statefulset/%s", sts.Name) + } + + for _, pvc := range pvcList.Items { + if !strings.HasPrefix(pvc.Name, "datadir-"+sts.Name) { + continue + } + + log.Info("Resizing PVC", "name", pvc.Name) + pvc.Spec.Resources.Requests[corev1.ResourceStorage] = requested + + if err := r.client.Update(ctx, &pvc); err != nil { + return errors.Wrapf(err, "update persistentvolumeclaim/%s", pvc.Name) + } + } + + return nil +}