cephfs: add SetSubVolCSIMetadata to set CSI metadata on subvolumes

Add support for setting CSI metadata (PV/PVC info) on CephFS
subvolumes. The PV controller now distinguishes between CephFS
and RBD PVs and dispatches accordingly.

Signed-off-by: Praveen M <m.praveen@ibm.com>

Author: iPraveenParihar

charts/ceph-csi-cephfs/templates/provisioner-deployment.yaml

@@ -283,6 +283,36 @@ spec:
           resources:
 {{ toYaml .Values.provisioner.resizer.resources | indent 12 }}
 {{- end }}
+{{- if .Values.provisioner.deployController }}
+        - name: csi-cephfsplugin-controller
+          image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"
+          imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}
+          args:
+            - "--type=controller"
+            - "--v={{ .Values.logLevel }}"
+            - "--drivername=$(DRIVER_NAME)"
+            - "--drivernamespace=$(DRIVER_NAMESPACE)"
+            {{- if .Values.provisioner.clustername }}
+            - "--clustername={{ .Values.provisioner.clustername }}"
+            {{- end }}
+            - "--setmetadata={{ .Values.provisioner.setmetadata }}"
+          env:
+            - name: DRIVER_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: DRIVER_NAME
+              value: {{ .Values.driverName }}
+          volumeMounts:
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
+            - name: keys-tmp-dir
+              mountPath: /tmp/csi/keys
+            - name: ceph-config
+              mountPath: /etc/ceph/
+          resources:
+{{ toYaml .Values.nodeplugin.plugin.resources | indent 12 }}
+{{- end }}
 {{- if .Values.provisioner.httpMetrics.enabled }}
         - name: liveness-prometheus
           image: "{{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}"

charts/ceph-csi-cephfs/values.yaml

@@ -247,6 +247,10 @@ provisioner:
   # set metadata on volume
   setmetadata: true
 
+  # deployController to enable or disable the deployment of controller which
+  # sets the CSI metadata on CephFS subvolumes.
+  deployController: true
+
   # enable fencing
   fencing: false
 

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -192,6 +192,27 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
+        - name: csi-cephfsplugin-controller
+          image: quay.io/cephcsi/cephcsi:canary
+          args:
+            - "--type=controller"
+            - "--v=5"
+            - "--drivername=cephfs.csi.ceph.com"
+            - "--drivernamespace=$(DRIVER_NAMESPACE)"
+            - "--setmetadata=true"
+          env:
+            - name: DRIVER_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: ceph-csi-config
+              mountPath: /etc/ceph-csi-config/
+            - name: keys-tmp-dir
+              mountPath: /tmp/csi/keys
+            - name: ceph-config
+              mountPath: /etc/ceph/
         - name: liveness-prometheus
           image: quay.io/cephcsi/cephcsi:canary
           args:

internal/cephfs/store/fsjournal.go

@@ -28,6 +28,7 @@ import (
 	cerrors "github.com/ceph/ceph-csi/internal/cephfs/errors"
 	"github.com/ceph/ceph-csi/internal/journal"
 	"github.com/ceph/ceph-csi/internal/util"
+	"github.com/ceph/ceph-csi/internal/util/k8s"
 	"github.com/ceph/ceph-csi/internal/util/log"
 	"github.com/ceph/ceph-csi/pkg/util/crypto"
 )
@@ -471,3 +472,56 @@ func CheckSnapExists(
 
 	return sid, nil
 }
+
+// SetSubVolCSIMetadata sets CSI metadata (PV/PVC info) on a CephFS subvolume.
+func SetSubVolCSIMetadata(
+	ctx context.Context,
+	volumeAttributes map[string]string,
+	volumeID,
+	pvName,
+	pvcName,
+	pvcNamespace,
+	clusterName string,
+	cr *util.Credentials,
+) error {
+	var vi util.CSIIdentifier
+	if err := vi.DecomposeCSIID(volumeID); err != nil {
+		return fmt.Errorf("%w: error decoding volume ID (%w) (%s)",
+			cerrors.ErrInvalidVolID, err, volumeID)
+	}
+
+	monitors, err := util.Mons(util.CsiConfigFile, vi.ClusterID)
+	if err != nil {
+		return fmt.Errorf("failed to fetch monitor list using clusterID (%s): %w", vi.ClusterID, err)
+	}
+
+	subvolumeGroup, err := util.CephFSSubvolumeGroup(util.CsiConfigFile, vi.ClusterID)
+	if err != nil {
+		return fmt.Errorf("failed to fetch subvolumegroup using clusterID (%s): %w", vi.ClusterID, err)
+	}
+
+	conn := &util.ClusterConnection{}
+	if err = conn.Connect(monitors, cr); err != nil {
+		return fmt.Errorf("failed to connect to cluster: %w", err)
+	}
+	defer conn.Destroy()
+
+	fsName := volumeAttributes["fsName"]
+	subvolName := volumeAttributes["subvolumeName"]
+
+	subVol := &core.SubVolume{
+		VolID:          subvolName,
+		FsName:         fsName,
+		SubvolumeGroup: subvolumeGroup,
+	}
+	volClient := core.NewSubVolume(conn, subVol, vi.ClusterID, clusterName, true)
+	parameters := k8s.PrepareVolumeMetadata(pvcName, pvcNamespace, pvName)
+	if err = volClient.SetAllMetadata(parameters); err != nil {
+		return fmt.Errorf("failed to set metadata on subvolume %s: %w", subvolName, err)
+	}
+
+	log.DebugLog(ctx, "cephfs: successfully set metadata on subvolume %s for PV %s",
+		subvolName, pvName)
+
+	return nil
+}

internal/controller/persistentvolume/persistentvolume.go

@@ -33,6 +33,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 
+	cephfsstore "github.com/ceph/ceph-csi/internal/cephfs/store"
 	ctrl "github.com/ceph/ceph-csi/internal/controller"
 	"github.com/ceph/ceph-csi/internal/rbd"
 	"github.com/ceph/ceph-csi/internal/util"
@@ -137,23 +138,44 @@ func (r *ReconcilePersistentVolume) reconcilePV(ctx context.Context, obj runtime
 	}
 	defer cr.DeleteCredentials()
 
-	rbdVolID, err := rbd.RegenerateJournal(
-		pv.Spec.CSI.VolumeAttributes,
-		pv.Spec.ClaimRef.Name,
-		volumeHandler,
-		requestName,
-		pvcNamespace,
-		r.config.ClusterName,
-		r.config.InstanceID,
-		r.config.SetMetadata,
-		cr)
-	if err != nil {
-		log.ErrorLogMsg("failed to regenerate journal %s", err)
-
-		return err
-	}
-	if rbdVolID != volumeHandler {
-		log.DebugLog(ctx, "volumeHandler changed from %s to %s", volumeHandler, rbdVolID)
+	// Determine PV type from volume attributes and dispatch accordingly.
+	// CephFS PVs always have "fsName" in their volume attributes (required
+	// StorageClass parameter), while RBD PVs do not.
+	_, isCephFS := pv.Spec.CSI.VolumeAttributes["fsName"]
+	if isCephFS {
+		err = cephfsstore.SetSubVolCSIMetadata(
+			ctx,
+			pv.Spec.CSI.VolumeAttributes,
+			volumeHandler,
+			requestName,
+			pv.Spec.ClaimRef.Name,
+			pvcNamespace,
+			r.config.ClusterName,
+			cr)
+		if err != nil {
+			log.ErrorLogMsg("failed to set CephFS subvolume metadata %s", err)
+
+			return err
+		}
+	} else {
+		rbdVolID, err := rbd.RegenerateJournal(
+			pv.Spec.CSI.VolumeAttributes,
+			pv.Spec.ClaimRef.Name,
+			volumeHandler,
+			requestName,
+			pvcNamespace,
+			r.config.ClusterName,
+			r.config.InstanceID,
+			r.config.SetMetadata,
+			cr)
+		if err != nil {
+			log.ErrorLogMsg("failed to regenerate journal %s", err)
+
+			return err
+		}
+		if rbdVolID != volumeHandler {
+			log.DebugLog(ctx, "volumeHandler changed from %s to %s", volumeHandler, rbdVolID)
+		}
 	}
 
 	return nil