feat(hostpath):enforce xfs quota on the hostpath (#78)

almas33 · web-flow · commit 0b57dff951e8 · 2021-08-30T16:55:06.000+05:30
Initial implementation to enforce quota on the XFS volumes. 
- The feature is controlled by specifying the quota parameter in the StorageClass along with specifying the values for limits.

Note: Additional PRs will be raised to refactor the implementation and add integration tests. 

Signed-off-by : Almas Ahmad &lt;ahmadalmas.786.aa@gmail.com&gt;
diff --git a/cmd/provisioner-localpv/app/helper_hostpath.go b/cmd/provisioner-localpv/app/helper_hostpath.go
@@ -21,7 +21,10 @@ package app
 
 import (
 	"context"
+	"math"
 	"path/filepath"
+	"regexp"
+	"strconv"
 	"time"
 
 	errors "github.com/pkg/errors"
@@ -69,12 +72,21 @@ type HelperPodOptions struct {
 	//path is the volume hostpath directory
 	path string
 
-	// serviceAccountName is the service account with which the pod should be launched
+	//serviceAccountName is the service account with which the pod should be launched
 	serviceAccountName string
 
 	selectedNodeTaints []corev1.Taint
 
 	imagePullSecrets []corev1.LocalObjectReference
+
+	//softLimitGrace is the soft limit of quota on the project
+	softLimitGrace string
+
+	//hardLimitGrace is the hard limit of quota on the project
+	hardLimitGrace string
+
+	//pvcStorage is the storage requested for pv
+	pvcStorage int64
 }
 
 // validate checks that the required fields to launch
@@ -92,6 +104,64 @@ func (pOpts *HelperPodOptions) validate() error {
 	return nil
 }
 
+//validateLimits check that the limits to setup qouta are valid
+func (pOpts *HelperPodOptions) validateLimits() error {
+	if pOpts.softLimitGrace == "0k" &&
+		pOpts.hardLimitGrace == "0k" {
+		return errors.Errorf("both limits cannot be 0")
+	}
+
+	if pOpts.softLimitGrace == "0k" ||
+		pOpts.hardLimitGrace == "0k" {
+		return nil
+	}
+
+	if len(pOpts.softLimitGrace) > len(pOpts.hardLimitGrace) ||
+		(len(pOpts.softLimitGrace) == len(pOpts.hardLimitGrace) &&
+			pOpts.softLimitGrace > pOpts.hardLimitGrace) {
+		return errors.Errorf("hard limit cannot be smaller than soft limit")
+	}
+
+	return nil
+}
+
+//converToK converts the limits to kilobytes
+func convertToK(limit string, pvcStorage int64) (string, error) {
+
+	if len(limit) == 0 {
+		return "0k", nil
+	}
+
+	valueRegex := regexp.MustCompile(`[\d]*[\.]?[\d]*`)
+	valueString := valueRegex.FindString(limit)
+
+	if limit != valueString+"%" {
+		return "", errors.Errorf("invalid format for limit grace")
+	}
+
+	value, err := strconv.ParseFloat(valueString, 64)
+
+	if err != nil {
+		return "", errors.Errorf("invalid format, cannot parse")
+	}
+	if value > 100 {
+		value = 100
+	}
+
+	value *= float64(pvcStorage)
+	value /= 100
+	value += float64(pvcStorage)
+	value /= 1000
+
+	if value != math.Trunc(value) {
+		value++
+	}
+	value = math.Trunc(value)
+	valueString = strconv.FormatFloat(value, 'f', -1, 64)
+	valueString += "k"
+	return valueString, nil
+}
+
 // createInitPod launches a helper(busybox) pod, to create the host path.
 //  The local pv expect the hostpath to be already present before mounting
 //  into pod. Validate that the local pv host path is not created under root.
@@ -117,6 +187,8 @@ func (p *Provisioner) createInitPod(ctx context.Context, pOpts *HelperPodOptions
 	//Pass on the taints, to create tolerations.
 	config.taints = pOpts.selectedNodeTaints
 
+	config.pOpts.cmdsForPath = append(config.pOpts.cmdsForPath, filepath.Join("/data/", config.volumeDir))
+
 	iPod, err := p.launchPod(ctx, config)
 	if err != nil {
 		return err
@@ -142,7 +214,6 @@ func (p *Provisioner) createCleanupPod(ctx context.Context, pOpts *HelperPodOpti
 		return err
 	}
 
-	config.taints = pOpts.selectedNodeTaints
 	// Initialize HostPath builder and validate that
 	// volume directory is not directly under root.
 	// Extract the base path and the volume unique path.
@@ -154,6 +225,10 @@ func (p *Provisioner) createCleanupPod(ctx context.Context, pOpts *HelperPodOpti
 		return vErr
 	}
 
+	config.taints = pOpts.selectedNodeTaints
+
+	config.pOpts.cmdsForPath = append(config.pOpts.cmdsForPath, filepath.Join("/data/", config.volumeDir))
+
 	cPod, err := p.launchPod(ctx, config)
 	if err != nil {
 		return err
@@ -165,6 +240,72 @@ func (p *Provisioner) createCleanupPod(ctx context.Context, pOpts *HelperPodOpti
 	return nil
 }
 
+// createQuotaPod launches a helper(busybox) pod, to apply the quota.
+//  The local pv expect the hostpath to be already present before mounting
+//  into pod. Validate that the local pv host path is not created under root.
+func (p *Provisioner) createQuotaPod(ctx context.Context, pOpts *HelperPodOptions) error {
+	var config podConfig
+	config.pOpts, config.podName = pOpts, "quota"
+	//err := pOpts.validate()
+	if err := pOpts.validate(); err != nil {
+		return err
+	}
+
+	// Initialize HostPath builder and validate that
+	// volume directory is not directly under root.
+	// Extract the base path and the volume unique path.
+	var vErr error
+	config.parentDir, config.volumeDir, vErr = hostpath.NewBuilder().WithPath(pOpts.path).
+		WithCheckf(hostpath.IsNonRoot(), "volume directory {%v} should not be under root directory", pOpts.path).
+		ExtractSubPath()
+	if vErr != nil {
+		return vErr
+	}
+
+	//Pass on the taints, to create tolerations.
+	config.taints = pOpts.selectedNodeTaints
+
+	var lErr error
+	config.pOpts.softLimitGrace, lErr = convertToK(config.pOpts.softLimitGrace, config.pOpts.pvcStorage)
+	if lErr != nil {
+		return lErr
+	}
+	config.pOpts.hardLimitGrace, lErr = convertToK(config.pOpts.hardLimitGrace, config.pOpts.pvcStorage)
+	if lErr != nil {
+		return lErr
+	}
+
+	if err := pOpts.validateLimits(); err != nil {
+		return err
+	}
+
+	//fs stores the file system of mount
+	fs := "FS=`stat -f -c %T /data` ; "
+	//check if fs is xfs
+	checkXfs := "if [[ \"$FS\" != \"xfs\" ]]; then rm -rf " + filepath.Join("/data/", config.volumeDir) + " ;exit 1 ; else "
+	//lastPid finds last project Id in the directory
+	lastPid := "PID=`xfs_quota -x -c 'report -h' /data | tail -2 | awk 'NR==1{print substr ($1,2)}+0'` ;"
+	//newPid increments last project Id by 1
+	newPid := "PID=`expr $PID + 1` ;"
+	//initializeProject initializes project with newpid
+	initializeProject := "xfs_quota -x -c 'project -s -p " + filepath.Join("/data/", config.volumeDir) + " '$PID /data ;"
+	//setQuota sets the quota according to limits defined
+	setQuota := "xfs_quota -x -c 'limit -p bsoft=" + config.pOpts.softLimitGrace + " bhard=" + config.pOpts.hardLimitGrace + " '$PID /data ; fi"
+
+	config.pOpts.cmdsForPath = []string{"sh", "-c", fs + checkXfs + lastPid + newPid + initializeProject + setQuota}
+
+	qPod, err := p.launchPod(ctx, config)
+	if err != nil {
+		return err
+	}
+
+	if err := p.exitPod(ctx, qPod); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (p *Provisioner) launchPod(ctx context.Context, config podConfig) (*corev1.Pod, error) {
 	// the helper pod need to be launched in privileged mode. This is because in CoreOS
 	// nodes, pods without privileged access cannot write to the host directory.
@@ -182,7 +323,7 @@ func (p *Provisioner) launchPod(ctx context.Context, config podConfig) (*corev1.
 			container.NewBuilder().
 				WithName("local-path-" + config.podName).
 				WithImage(p.helperImage).
-				WithCommandNew(append(config.pOpts.cmdsForPath, filepath.Join("/data/", config.volumeDir))).
+				WithCommandNew(config.pOpts.cmdsForPath).
 				WithVolumeMountsNew([]corev1.VolumeMount{
 					{
 						Name:      "data",
diff --git a/cmd/provisioner-localpv/app/helper_test.go b/cmd/provisioner-localpv/app/helper_test.go
@@ -0,0 +1,98 @@
+/*
+Copyright 2019 The OpenEBS Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+This code was taken from https://github.com/rancher/local-path-provisioner
+and modified to work with the configuration options used by OpenEBS
+*/
+
+package app
+
+import (
+	"testing"
+)
+
+func TestConvertToK(t *testing.T) {
+	type args struct {
+		limit      string
+		pvcStorage int64
+	}
+	tests := map[string]struct {
+		name    string
+		args    args
+		want    string
+		wantErr bool
+	}{
+		"Missing limit grace": {
+			args: args{
+				limit:      "",
+				pvcStorage: 5000000000,
+			},
+			want:    "0k",
+			wantErr: false,
+		},
+		"Present limit with grace": {
+			args: args{
+				limit:      "0%",
+				pvcStorage: 5000,
+			},
+			want:    "5k",
+			wantErr: false,
+		},
+		"Present limit grace exceeding 100%": {
+			args: args{
+				limit:      "200%",
+				pvcStorage: 5000000,
+			},
+			want:    "10000k",
+			wantErr: false,
+		},
+		"Present limit grace with decimal%": {
+			args: args{
+				limit:      ".5%",
+				pvcStorage: 1000,
+			},
+			want:    "2k",
+			wantErr: false,
+		},
+		"Present limit grace with invalid pattern": {
+			args: args{
+				limit:      "10",
+				pvcStorage: 10000,
+			},
+			want:    "",
+			wantErr: true,
+		},
+		"Present limit grace with only %": {
+			args: args{
+				limit:      "%",
+				pvcStorage: 10000,
+			},
+			want:    "",
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := convertToK(tt.args.limit, tt.args.pvcStorage)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("convertToK() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("convertToK() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
diff --git a/cmd/provisioner-localpv/app/provisioner_hostpath.go b/cmd/provisioner-localpv/app/provisioner_hostpath.go
@@ -31,6 +31,12 @@ import (
 	pvController "sigs.k8s.io/sig-storage-lib-external-provisioner/v7/controller"
 )
 
+const (
+	EnableXfsQuota string = "enableXfsQuota"
+	SoftLimitGrace string = "softLimitGrace"
+	HardLimitGrace string = "hardLimitGrace"
+)
+
 // ProvisionHostPath is invoked by the Provisioner which expect HostPath PV
 //  to be provisioned and a valid PV spec returned.
 func (p *Provisioner) ProvisionHostPath(ctx context.Context, opts pvController.ProvisionOptions, volumeConfig *VolumeConfig) (*v1.PersistentVolume, pvController.ProvisioningState, error) {
@@ -87,6 +93,45 @@ func (p *Provisioner) ProvisionHostPath(ctx context.Context, opts pvController.P
 		return nil, pvController.ProvisioningFinished, iErr
 	}
 
+	enableXfsQuota := opts.StorageClass.Parameters[EnableXfsQuota]
+
+	if enableXfsQuota == "true" {
+		softLimitGrace := opts.StorageClass.Parameters[SoftLimitGrace]
+		hardLimitGrace := opts.StorageClass.Parameters[HardLimitGrace]
+		pvcStorage := opts.PVC.Spec.Resources.Requests.Storage().Value()
+
+		podOpts := &HelperPodOptions{
+			name:                   name,
+			path:                   path,
+			nodeAffinityLabelKey:   nodeAffinityKey,
+			nodeAffinityLabelValue: nodeAffinityValue,
+			serviceAccountName:     saName,
+			selectedNodeTaints:     taints,
+			imagePullSecrets:       imagePullSecrets,
+			softLimitGrace:         softLimitGrace,
+			hardLimitGrace:         hardLimitGrace,
+			pvcStorage:             pvcStorage,
+		}
+		iErr := p.createQuotaPod(ctx, podOpts)
+		if iErr != nil {
+			klog.Infof("Applying quota failed: %v", iErr)
+			alertlog.Logger.Errorw("",
+				"eventcode", "local.pv.provision.failure",
+				"msg", "Failed to provision Local PV",
+				"rname", opts.PVName,
+				"reason", "Quota enforcement failed",
+				"storagetype", stgType,
+			)
+			return nil, pvController.ProvisioningFinished, iErr
+		}
+		alertlog.Logger.Infow("",
+			"eventcode", "local.pv.quota.success",
+			"msg", "Successfully applied quota",
+			"rname", opts.PVName,
+			"storagetype", stgType,
+		)
+	}
+
 	// VolumeMode will always be specified as Filesystem for host path volume,
 	// and the value passed in from the PVC spec will be ignored.
 	fs := v1.PersistentVolumeFilesystem
