support snapshot/clone

Author: 杨尚坡

README.md

@@ -25,15 +25,15 @@ in the Kubernetes documentation.
 ## Develop
 
 You can follow the [csi spec](https://github.com/container-storage-interface/spec/blob/master/spec.md)
-and [curve cli](docs/Curve%20Interface.md).
+and [curve interface](docs/curve-interface).
 
 ## Setup
 
 1. Deploy the `curve tool` on the CO "Master" Hosts and `curve-nbd tool` on CO "Node" Hosts by the curve cluster provider.
 2. Choose a way to deploy the plugin:
 
 - Using the helm chart: [helm installation](charts/curve-csi/README.md)
-- Using the kubernetes manifests: refer to the specific version in `deploy/kubernetes`
+- Using the kubernetes manifests: refer to the specific version in `deploy/manifests`
 
 ## Test and User Guide
 

charts/curve-csi/templates/csi-daemonset.yaml

@@ -52,9 +52,6 @@ spec:
         image: "{{ .Values.nodeplugin.plugin.image }}"
         args:
         - --endpoint=$(CSI_ENDPOINT)
-{{- if .Values.common.curveVolumeNamingPrefix }}
-        - "--curve-volume-prefix={{ .Values.common.curveVolumeNamingPrefix }}"
-{{- end }}
         - --drivername=curve.csi.netease.com
         - --nodeid=$(NODE_ID)
 {{- if .Values.nodeplugin.debug.enabled }}

charts/curve-csi/templates/csi-deployment.yaml

@@ -114,11 +114,9 @@ spec:
         image: "{{ .Values.controllerplugin.plugin.image }}"
         args:
         - --endpoint=$(CSI_ENDPOINT)
-{{- if .Values.common.curveVolumeNamingPrefix }}
-        - "--curve-volume-prefix={{ .Values.common.curveVolumeNamingPrefix }}"
-{{- end }}
         - --drivername=curve.csi.netease.com
         - --nodeid=$(NODE_ID)
+        - --snapshot-server={{ .Values.controllerplugin.snapshotServer }}
 {{- if .Values.controllerplugin.debug.enabled }}
         - "--debug-port={{ .Values.controllerplugin.debug.port }}"
 {{- end }}

charts/curve-csi/values.yaml

@@ -28,6 +28,8 @@ nodeplugin:
 controllerplugin:
   replicas: 2
 
+  snapshotServer: http://127.0.0.1:5555
+
   debug:
     enabled: true
     port: 9696
@@ -66,7 +68,3 @@ controllerplugin:
   tolerations: []
 
   affinity: {}
-
-# common variables
-common:
-  curveVolumeNamingPrefix: ""

cmd/curve-csi.go

@@ -48,8 +48,8 @@ func init() {
 	flag.BoolVar(&curveConf.IsNodeServer, "node-server", false, "start curve-csi node server")
 	flag.BoolVar(&curveConf.IsControllerServer, "controller-server", false, "start curve-csi controller server")
 
-	// curve volume name prefix
-	flag.StringVar(&curveConf.CurveVolumePrefix, "curve-volume-prefix", "csi-vol-", "curve volume name prefix")
+	// curve snashot/clone server
+	flag.StringVar(&curveConf.SnapshotServer, "snapshot-server", "http://127.0.0.1:5555", "curve snapshot/clone http server address, set empty to disable snapshot")
 
 	// debug
 	flag.IntVar(&curveConf.DebugPort, "debug-port", 0, "debug port, set 0 to disable")

cmd/options/options.go

@@ -26,8 +26,8 @@ type CurveConf struct {
 	IsControllerServer bool
 	IsNodeServer       bool
 
-	// curve volume namd prefix
-	CurveVolumePrefix string
+	// curve flags
+	SnapshotServer string
 
 	// debugs
 	DebugPort       int

deploy/manifests/provisioner-deploy.yaml

@@ -105,6 +105,7 @@ spec:
         - --endpoint=$(CSI_ENDPOINT)
         - --drivername=curve.csi.netease.com
         - --nodeid=$(NODE_ID)
+        - "--snapshot-server=http://127.0.0.1:5555"
         - --controller-server=true
         - --debug-port=9696
         - --logtostderr=false

docs/README.md

@@ -2,6 +2,21 @@
 
 This document provides more detail about curve-csi driver.
 
+- [Deploy](#deploy)
+  - [Requirements](#requirements)
+  - [Using the helm chart](#using-the-helm-chart)
+  - [Using the kubernetes manifests](#using-the-kubernetes-manifests)
+- [Debug](#debug)
+- [Examples](#examples)
+  - [Create StorageClass](#create-storageclass)
+  - [Create PersistentVolumeClaim](#create-persistentvolumeclaim)
+  - [Create Test Pod](#create-test-pod)
+  - [Test block volume](#test-block-volume)
+  - [Test volume expanding](#test-volume-expanding)
+  - [Test snapshot](#test-snapshot)
+  - [Test volume clone](#test-volume-clone)
+- [Test Using CSC Tool](#test-using-csc-tool)
+
 ## Deploy
 
 #### Requirements
@@ -36,7 +51,7 @@ and call:
 curl -XPUT http://127.0.0.1:<debugPort>/debug/flags/v -d '5'
 ```
 
-## Example
+## Examples
 
 #### Create StorageClass
 
@@ -93,8 +108,8 @@ spec:
 
 ```
 ## create the pvc and pod:
-kubectl create -f ../examples/block/pvc.yaml
-kubectl create -f ../examples/block/pod.yaml
+kubectl create -f ../examples/pvc-block.yaml
+kubectl create -f ../examples/pod-block.yaml
 
 ## waiting for the pod running
 kubectl exec -it csi-curve-test-block bash
@@ -111,8 +126,23 @@ kubectl exec -it csi-curve-test-block bash
 
 #### Test snapshot
 
+Prerequisite: [install snapshot-controller](https://kubernetes-csi.github.io/docs/snapshot-controller.html)
+
+
+Create snapshot:
+
+```
+kubectl create -f ../examples/snapshotclass.yaml
+kubectl create -f ../examples/snapshot.yaml
+```
+
 #### Test volume clone
 
+```
+kubectl create -f ../examples/pvc.yaml
+kubectl create -f ../examples/pvc-clone.yaml
+kubectl create -f ../examples/pvc-restore.yaml
+```
 
 ## Test Using CSC Tool
 
@@ -164,6 +194,8 @@ $ csc controller create --endpoint tcp://127.0.0.1:10000 \
 "0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46"  10737418240     "user"="k8s"
 ```
 
+If the volume is block type, set: `--cap 5,1`
+
 Check:
 
 ```text
@@ -190,16 +222,18 @@ $ csc node stage --endpoint tcp://127.0.0.1:10000 \
 0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46
 ```
 
+If the volume is block type, set: `--cap 5,1`
+
 Check:
 
 ```text
 $ sudo curve-nbd list-mapped
 id    image                                                      device
 97297 cbd:k8s//k8s/pvc-ce482926-91d8-11ea-bf6e-fa163e23ce53_k8s_ /dev/nbd0
 
-$ sudo findmnt /mnt/test-csi/volume-globalmount
+$ sudo findmnt /mnt/test-csi/volume-globalmount/0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46
 TARGET                          SOURCE    FSTYPE OPTIONS
-/mnt/test-csi/volume-globalmount /dev/nbd0 ext4   rw,relatime,data=ordered
+/mnt/test-csi/volume-globalmount/0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46 /dev/nbd0 ext4   rw,relatime,data=ordered
 ```
 
 #### NodePublish a volume
@@ -253,8 +287,8 @@ $ csc controller expand --endpoint tcp://127.0.0.1:10000 \
 
 $ # nodeExpand:
 $ csc node expand --endpoint tcp://127.0.0.1:10000 \
-    /mnt/test-csi/test-pod \
-    0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46
+    0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46 \
+    /mnt/test-csi/test-pod
 0
 ```
 
@@ -317,3 +351,18 @@ $ sudo curve stat --user k8s --filename /k8s/csi-vol-volume-fa0c04c9-2e93-487e-8
 E 2020-08-24T13:57:55.946636+0800 58360 mds_client.cpp:395] GetFileInfo: filename = /k8s/volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46, owner = k8s, errocde = 6, error msg = kFileNotExists, log id = 1
 stat fail, ret = -6
 ```
+
+#### Snapshot
+
+```
+## create a snapshot
+$ csc controller create-snapshot --endpoint tcp://127.0.0.1:10000 \
+	--source-volume 0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46 \
+	snapshot-215d24ff-c04c-4b08-a1fb-692c94627c63
+"0024-9ea1a8fc-160d-47ef-b2ef-f0e09677b066-0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46"	23622320128	0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46	seconds:1639297566 nanos:780828000 	true
+
+## delete a snapshot
+$ csc controller delete-snapshot --endpoint tcp://127.0.0.1:10000 \
+    0024-9ea1a8fc-160d-47ef-b2ef-f0e09677b066-0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46
+0024-9ea1a8fc-160d-47ef-b2ef-f0e09677b066-0003-k8s-csi-vol-volume-fa0c04c9-2e93-487e-8986-1e1625fd8c46
+```

docs/clone.md

@@ -0,0 +1,21 @@
+### Verify if PVC is in Bound state
+
+```bash
+$ kubectl get pvc
+NAME                   STATUS        VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
+curve-test-pvc         Bound         pvc-b2789e09-9854-4aa1-b556-d9b0e0569f87   30Gi       RWO            curve          35m
+```
+
+### Create a new cloned PVC
+
+```
+kubectl create -f ../examples/pvc-clone.yaml
+```
+
+### Get PVC
+
+```bash
+$ kubectl get pvc curve-pvc-clone
+NAME              STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
+curve-pvc-clone   Bound    pvc-755cfb47-9b03-41b5-bdf9-0772a1ae41ef   40Gi       RWO            curve          3s
+```