rbd: use volumeattributesclass feature implement rbd volume qos

PendingReleaseNotes.md

@@ -21,5 +21,6 @@
 - nvmeof: add Kubernetes ServiceAccount based volume access restriction
 - cephfs: add Kubernetes ServiceAccount based volume access restriction
 - nfs: add Kubernetes ServiceAccount based volume access restriction
+- rbd-nbd: use VolumeAttributesClass feature implement rbd volume qos [PR](https://github.com/ceph/ceph-csi/pull/6160)
 
 ## NOTE

README.md

@@ -99,6 +99,7 @@ for its support details.
 |         | Volume/PV Metrics of File Mode Volume                     | GA             | >= v1.2.0          | >= v1.1.0        | Pacific (>=v16.2.0)  | >= v1.15.0         |
 |         | Volume/PV Metrics of Block Mode Volume                    | GA             | >= v1.2.0          | >= v1.1.0        | Pacific (>=v16.2.0)  | >= v1.21.0         |
 |         | Topology Aware Provisioning Support                       | Alpha          | >= v2.1.0          | >= v1.1.0        | Pacific (>=v16.2.0)  | >= v1.14.0         |
+|         | RBD-NBD QoS by VolumeAttributesClass                      | Alpha          | >= v3.17.0         | >= v1.12.0       | Pacific (>=v16.2.0)  | >= v1.34.0         |
 | CephFS  | Dynamically provision, de-provision File mode RWO volume  | GA             | >= v1.1.0          | >= v1.0.0        | Pacific (>=v16.2.0)  | >= v1.14.0         |
 |         | Dynamically provision, de-provision File mode RWX volume  | GA             | >= v1.1.0          | >= v1.0.0        | Pacific (>=v16.2.0)  | >= v1.14.0         |
 |         | Dynamically provision, de-provision File mode ROX volume  | Alpha          | >= v3.0.0          | >= v1.0.0        | Pacific (>=v16.2.0)  | >= v1.14.0         |

charts/ceph-csi-cephfs/README.md

@@ -161,7 +161,7 @@ charts and their default values.
 | `provisioner.imagePullSecrets`                | Specifies imagePullSecrets for containers                                                                                                        | `[]`                                            |
 | `provisioner.profiling.enabled`                | Specifies whether profiling should be enabled                                                                                                        | `false`                                            |
 | `provisioner.provisioner.image.repository`     | Specifies the csi-provisioner image repository URL                                                                                                   | `registry.k8s.io/sig-storage/csi-provisioner`      |
-| `provisioner.provisioner.image.tag`            | Specifies image tag                                                                                                                                  | `v6.1.1`                                           |
+| `provisioner.provisioner.image.tag`            | Specifies image tag                                                                                                                                  | `v6.2.0`                                           |
 | `provisioner.provisioner.image.pullPolicy`     | Specifies pull policy                                                                                                                                | `IfNotPresent`                                     |
 | `provisioner.provisioner.args.httpEndpointPort`    | Specifies http server port for diagnostics, health checks and metrics                                                                                    | `""`                                               |
 | `provisioner.provisioner.extraArgs`            | Specifies extra arguments for the provisioner sidecar                                                                                                | `[]`                                               |

charts/ceph-csi-cephfs/values.yaml

@@ -232,7 +232,7 @@ provisioner:
   provisioner:
     image:
       repository: registry.k8s.io/sig-storage/csi-provisioner
-      tag: v6.1.1
+      tag: v6.2.0
       pullPolicy: IfNotPresent
     resources: {}
     args:

charts/ceph-csi-rbd/templates/provisioner-clusterrole.yaml

@@ -96,5 +96,8 @@ rules:
   - apiGroups: [""]
     resources: ["serviceaccounts/token"]
     verbs: ["create"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattributesclasses"]
+    verbs: ["get", "list", "watch"]
 
 {{- end -}}

charts/ceph-csi-rbd/values.yaml

@@ -256,7 +256,7 @@ provisioner:
   provisioner:
     image:
       repository: registry.k8s.io/sig-storage/csi-provisioner
-      tag: v6.1.1
+      tag: v6.2.0
       pullPolicy: IfNotPresent
     resources: {}
     args:

deploy/cephfs/kubernetes/csi-cephfsplugin-provisioner.yaml

@@ -93,7 +93,7 @@ spec:
             - name: ceph-csi-encryption-kms-config
               mountPath: /etc/ceph-csi-encryption-kms-config/
         - name: csi-provisioner
-          image: registry.k8s.io/sig-storage/csi-provisioner:v6.1.1
+          image: registry.k8s.io/sig-storage/csi-provisioner:v6.2.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"

deploy/rbd/kubernetes/csi-provisioner-rbac.yaml

@@ -81,6 +81,9 @@ rules:
   - apiGroups: ["replication.storage.openshift.io"]
     resources: ["volumegroupreplicationclasses"]
     verbs: ["get"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattributesclasses"]
+    verbs: ["get", "list", "watch"]
 ---
 kind: ClusterRoleBinding
 apiVersion: rbac.authorization.k8s.io/v1

deploy/rbd/kubernetes/csi-rbdplugin-provisioner.yaml

@@ -105,7 +105,7 @@ spec:
               mountPath: /run/secrets/tokens
               readOnly: true
         - name: csi-provisioner
-          image: registry.k8s.io/sig-storage/csi-provisioner:v6.1.1
+          image: registry.k8s.io/sig-storage/csi-provisioner:v6.2.0
           args:
             - "--csi-address=$(ADDRESS)"
             - "--v=1"

docs/volumeattributesclass.md

@@ -64,3 +64,122 @@ parameters. This does not allow the Node-plugin accessing the Ceph cluster to
 fetch updated parameters when the Volume is _staged_ or _published_. The
 secrets for staging and publishing can not (easily) be updated after the fact,
 these are part of the fixed parameters in the PersistentVolume.
+
+## Use VolumeAttributesClass for rbd-nbd volume qos
+
+### Create a VolumeAttributesClass
+
+- Define a VolumeAttributesClass
+
+```console
+---
+apiVersion: storage.k8s.io/v1
+kind: VolumeAttributesClass
+metadata:
+  name: silver
+driverName: rbd.csi.ceph.com
+parameters:
+  baseIops: "1800"
+  maxIops: "10000"
+  baseBps: "104857600"
+  maxBps: "188743680"
+  iopsPerGiB: "12"
+  bpsPerGiB: "209715"
+  baseVolSizeBytes: "21474836480"
+```
+
+```console
+kubectl create -f volumeattributesclass.yaml
+```
+
+- Verify VolumeAttributesClass has been created
+
+```console
+$ kubectl get vac
+NAME     DRIVERNAME                   AGE
+silver   rbd.csi.ceph.com             2s
+```
+
+### Create PVC with VolumeAttributesClass
+
+- Define a PVC
+
+```console
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: rbd-pvc-vac
+spec:
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 30Gi
+  storageClassName: csi-rbd-sc
+  volumeMode: Block
+  volumeAttributesClassName: silver
+```
+
+```console
+$ kubectl create -f pvc.yaml
+persistentvolumeclaim/rbd-pvc-vac created
+```
+
+- Verify pvc by get
+
+```console
+$ kubectl get pvc
+NAME          STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      VOLUMEATTRIBUTESCLASS   AGE
+rbd-pvc-vac   Bound    pvc-8b2fcb47-233a-4bcf-bb94-f94e9aa1150a   30Gi       RWO            csi-rbd-sc        silver                  1m
+```
+
+### Modify PVC VolumeAttributesClass
+
+- Create another VolumeAttributesClass
+
+```console
+---
+apiVersion: storage.k8s.io/v1
+kind: VolumeAttributesClass
+metadata:
+  name: gold
+driverName: rbd.csi.ceph.com
+parameters:
+  baseIops: "1800"
+  maxIops: "50000"
+  baseBps: "104857600"
+  maxBps: "367001600"
+  iopsPerGiB: "50"
+  bpsPerGiB: "524288"
+  baseVolSizeBytes: "21474836480"
+```
+
+```console
+$ kubectl create -f volumeattributesclass.yaml
+volumeattributesclass.storage.k8s.io/silver created
+```
+
+- Verify VolumeAttributesClass by get
+
+```console
+$ kubectl get vac
+NAME     DRIVERNAME                   AGE
+gold     rbd.csi.ceph.com             1m
+silver   rbd.csi.ceph.com             2m
+```
+
+- Modify PVC VolumeAttributesClassName
+
+```console
+$ kubectl patch pvc rbd-pvc-vac -p '{"spec":{"volumeAttributesClassName": "gold"}}'
+persistentvolumeclaim/rbd-pvc-vac patched
+```
+
+- Verify modify pvc volumeAttributesClassName by get
+
+```console
+$ kubectl get pvc
+NAME          STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      VOLUMEATTRIBUTESCLASS   AGE
+rbd-pvc-vac   Bound    pvc-8b2fcb47-233a-4bcf-bb94-f94e9aa1150a   30Gi       RWO            csi-rbd-sc        gold                    2m
+```