e2e: add volume expansion tests for NVMe-oF

e2e/ceph.go

@@ -25,10 +25,10 @@ import (
 	"k8s.io/kubernetes/test/e2e/framework"
 )
 
-const (
-	CephMajorSquid    = 19
-	CephMajorTentacle = 20
-	CephMajorUmbrella = 21
+var (
+	CephVersionSquid    = &cephVersion{major: 19}
+	CephVersionTentacle = &cephVersion{major: 20}
+	CephVersionUmbrella = &cephVersion{major: 21}
 )
 
 // cephVersion is a helper type that converts the standard Ceph version string
@@ -118,6 +118,21 @@ func (cv *cephVersion) GetRelease() string {
 	return cv.release
 }
 
+func (cv *cephVersion) GreaterEquals(req *cephVersion) bool {
+	// If major not equal, check if >
+	if cv.major != req.major {
+		return cv.major > req.major
+	}
+	// If minor not equal, check if >
+	if cv.minor != req.minor {
+		return cv.minor > req.minor
+	}
+
+	// At this point, major and minor are already equal.
+	// Check if patch is >=
+	return cv.patch >= req.patch
+}
+
 func getCephVersion(f *framework.Framework) (*cephVersion, error) {
 	cmd := "ceph --format=json version"
 	stdout, stderr, err := execCommandInToolBoxPod(f, cmd, rookNamespace)

e2e/ceph_test.go

@@ -112,3 +112,91 @@ func TestCephVersionUnmarshalJSON(t *testing.T) {
 		})
 	}
 }
+
+func TestCephVersionGreaterEquals(t *testing.T) {
+	t.Parallel()
+	tests := []struct {
+		name     string
+		version  cephVersion
+		required cephVersion
+		expected bool
+	}{
+		{
+			name:     "equal versions",
+			version:  cephVersion{major: 19, minor: 2, patch: 1},
+			required: cephVersion{major: 19, minor: 2, patch: 1},
+			expected: true,
+		},
+		{
+			name:     "greater major version",
+			version:  cephVersion{major: 20, minor: 0, patch: 0},
+			required: cephVersion{major: 19, minor: 2, patch: 1},
+			expected: true,
+		},
+		{
+			name:     "lesser major version",
+			version:  cephVersion{major: 18, minor: 5, patch: 9},
+			required: cephVersion{major: 19, minor: 0, patch: 0},
+			expected: false,
+		},
+		{
+			name:     "equal major, greater minor",
+			version:  cephVersion{major: 19, minor: 3, patch: 0},
+			required: cephVersion{major: 19, minor: 2, patch: 1},
+			expected: true,
+		},
+		{
+			name:     "equal major, lesser minor",
+			version:  cephVersion{major: 19, minor: 1, patch: 5},
+			required: cephVersion{major: 19, minor: 2, patch: 0},
+			expected: false,
+		},
+		{
+			name:     "equal major and minor, greater patch",
+			version:  cephVersion{major: 19, minor: 2, patch: 5},
+			required: cephVersion{major: 19, minor: 2, patch: 1},
+			expected: true,
+		},
+		{
+			name:     "equal major and minor, lesser patch",
+			version:  cephVersion{major: 19, minor: 2, patch: 0},
+			required: cephVersion{major: 19, minor: 2, patch: 1},
+			expected: false,
+		},
+		{
+			name:     "zero version comparison",
+			version:  cephVersion{major: 1, minor: 0, patch: 0},
+			required: cephVersion{major: 0, minor: 0, patch: 0},
+			expected: true,
+		},
+		{
+			name:     "comparing to zero version",
+			version:  cephVersion{major: 0, minor: 0, patch: 0},
+			required: cephVersion{major: 1, minor: 0, patch: 0},
+			expected: false,
+		},
+		{
+			name:     "Squid version >= Reef",
+			version:  cephVersion{major: 19, minor: 2, patch: 1},
+			required: cephVersion{major: 18, minor: 2, patch: 0},
+			expected: true,
+		},
+		{
+			name:     "Reef version >= Squid",
+			version:  cephVersion{major: 18, minor: 2, patch: 4},
+			required: cephVersion{major: 19, minor: 0, patch: 0},
+			expected: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			result := tt.version.GreaterEquals(&tt.required)
+			if result != tt.expected {
+				t.Errorf("GreaterEquals(%s) = %v, expected %v (version: %s)",
+					tt.required.String(), result, tt.expected, tt.version.String())
+			}
+		})
+	}
+}

e2e/nvmeof.go

@@ -22,6 +22,7 @@ import (
 	"github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2edebug "k8s.io/kubernetes/test/e2e/framework/debug"
 	"k8s.io/pod-security-admission/api"
@@ -60,9 +61,9 @@ var _ = ginkgo.Describe("nvmeof", func() {
 		if err != nil {
 			logAndFail("failed to get Ceph cluster version: %v", err)
 		}
-		if version.GetMajor() < CephMajorTentacle {
+		if !version.GreaterEquals(CephVersionTentacle) {
 			deployNVMeoF = false
-			ginkgo.Skip("Skipping NVMe-oF E2E, requires Ceph 20 (Tentacle):" + version.String())
+			ginkgo.Skip("Skipping NVMe-oF E2E, requires Ceph v20 (Tentacle): " + version.String())
 		}
 
 		framework.Logf("NVMe-oF testing supported, Ceph version: %s", version)
@@ -108,6 +109,13 @@ var _ = ginkgo.Describe("nvmeof", func() {
 			// log node plugin
 			logsCSIPods("app="+nvmeofDaemonsetName, f.ClientSet)
 
+			// Gateway logs - need to search in rook-ceph namespace
+			opt := metav1.ListOptions{LabelSelector: "app=ceph-nvmeof-gateway"}
+			podList, _ := f.ClientSet.CoreV1().Pods(rookNamespace).List(context.TODO(), opt)
+			for i := range podList.Items {
+				kubectlLogPod(f.ClientSet, &podList.Items[i])
+			}
+
 			// log all details from the namespace where Ceph-CSI is deployed
 			e2edebug.DumpAllNamespaceInfo(context.TODO(), f.ClientSet, cephCSINamespace)
 		}
@@ -121,23 +129,22 @@ var _ = ginkgo.Describe("nvmeof", func() {
 
 		pvcPath := nvmeofExamplePath + "pvc.yaml"
 		appPath := nvmeofExamplePath + "pod.yaml"
+		rawPvcPath := nvmeofExamplePath + "raw-block-pvc.yaml"
+		rawAppPath := nvmeofExamplePath + "raw-block-pod.yaml"
 
 		ginkgo.It("create a PVC and delete it", func() {
-			ginkgo.By("prepare PVC")
 			pvc, err := loadPVC(pvcPath)
 			Expect(err).ShouldNot(HaveOccurred())
 
 			pvc.Namespace = f.UniqueName
 			pvc.Spec.StorageClassName = &nvmeofStorageClass
 
-			ginkgo.By("create the PVC")
 			err = createPVCAndvalidatePV(f.ClientSet, pvc, deployTimeout)
 			Expect(err).ShouldNot(HaveOccurred())
 
 			validateRBDImageCount(f, 1, nvmeofPool)
 			validateOmapCount(f, 1, rbdType, nvmeofPool, volumesType)
 
-			ginkgo.By("delete the PVC again")
 			err = deletePVCAndValidatePV(f.ClientSet, pvc, deployTimeout)
 			Expect(err).ShouldNot(HaveOccurred())
 
@@ -152,5 +159,35 @@ var _ = ginkgo.Describe("nvmeof", func() {
 			validateRBDImageCount(f, 0, nvmeofPool)
 			validateOmapCount(f, 0, rbdType, nvmeofPool, volumesType)
 		})
+
+		ginkgo.It("Resize Filesystem PVC and check application directory size", func() {
+			pvc, err := loadPVC(pvcPath)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			pvc.Namespace = f.UniqueName
+			pvc.Spec.StorageClassName = &nvmeofStorageClass
+
+			err = resizePVCAndValidateSize(pvc, appPath, f)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			// validate created backend rbd images
+			validateRBDImageCount(f, 0, nvmeofPool)
+			validateOmapCount(f, 0, rbdType, nvmeofPool, volumesType)
+		})
+
+		ginkgo.It("Resize Block PVC and check Device size", func() {
+			pvc, err := loadPVC(rawPvcPath)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			pvc.Namespace = f.UniqueName
+			pvc.Spec.StorageClassName = &nvmeofStorageClass
+
+			err = resizePVCAndValidateSize(pvc, rawAppPath, f)
+			Expect(err).ShouldNot(HaveOccurred())
+
+			// validate created backend rbd images
+			validateRBDImageCount(f, 0, nvmeofPool)
+			validateOmapCount(f, 0, rbdType, nvmeofPool, volumesType)
+		})
 	})
 })

e2e/resize.go

@@ -86,20 +86,41 @@ func expandPVCSize(c kubernetes.Interface, pvc *v1.PersistentVolumeClaim, size s
 	})
 }
 
-func resizePVCAndValidateSize(pvcPath, appPath string, f *framework.Framework) error {
+// resizePVCAndValidateSize creates a PVC and starts an application Pod with
+// it. Once the Pod is running, the PVC is expanded and the size of the PVC
+// inside the Pod is verified.
+//
+// pvcObj can be a string (path to a YAML file for a PVC), or a pointer to a
+// PVC in case the object has been pre-created with special parameters (like
+// a StorageClass).
+func resizePVCAndValidateSize(pvcObj any, appPath string, f *framework.Framework) error {
+	var (
+		err            error
+		pvc, resizePvc *v1.PersistentVolumeClaim
+	)
+
 	size := "1Gi"
 	expandSize := "10Gi"
-	pvc, err := loadPVC(pvcPath)
-	if err != nil {
-		return err
-	}
-	pvc.Namespace = f.UniqueName
 
-	resizePvc, err := loadPVC(pvcPath)
-	if err != nil {
-		return err
+	switch pvcObj.(type) {
+	case string:
+		pvc, err = loadPVC(pvcObj.(string))
+		if err != nil {
+			return err
+		}
+		pvc.Namespace = f.UniqueName
+
+		resizePvc, err = loadPVC(pvcObj.(string))
+		if err != nil {
+			return err
+		}
+		resizePvc.Namespace = f.UniqueName
+	case *v1.PersistentVolumeClaim:
+		pvc = pvcObj.(*v1.PersistentVolumeClaim)
+		resizePvc = pvc.DeepCopy()
+	default:
+		return fmt.Errorf("BUG: pvcObj should be a string or *PersistentVolumeClaim, not a %T", pvcObj)
 	}
-	resizePvc.Namespace = f.UniqueName
 
 	app, err := loadApp(appPath)
 	if err != nil {

examples/nvmeof/raw-block-pod.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-with-raw-block-volume
+spec:
+  containers:
+    - name: centos
+      image: quay.io/centos/centos:latest
+      command: ["/bin/sleep", "infinity"]
+      volumeDevices:
+        - name: data
+          devicePath: /dev/xvda
+  volumes:
+    - name: data
+      persistentVolumeClaim:
+        claimName: raw-block-pvc

examples/nvmeof/raw-block-pvc.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: raw-block-pvc
+spec:
+  accessModes:
+    - ReadWriteOnce
+  volumeMode: Block
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: csi-nvmeof-sc