feat(*): support pod annotation verification

Author: JulyWindK

cmd/katalyst-agent/app/options/qrm/memory_plugin.go

@@ -42,6 +42,8 @@ type MemoryOptions struct {
 	EnableNonBindingShareCoresMemoryResourceCheck bool
 	EnableNUMAAllocationReactor                   bool
 	NUMABindResultResourceAllocationAnnotationKey string
+	EnableMemoryAnnotationValidator               bool
+	MemoryAnnotationValidatorDryRun               bool
 
 	SockMemOptions
 	LogCacheOptions
@@ -126,6 +128,8 @@ func NewMemoryOptions() *MemoryOptions {
 		EnableNonBindingShareCoresMemoryResourceCheck: true,
 		EnableNUMAAllocationReactor:                   false,
 		NUMABindResultResourceAllocationAnnotationKey: consts.QRMResourceAnnotationKeyNUMABindResult,
+		EnableMemoryAnnotationValidator:               true,
+		MemoryAnnotationValidatorDryRun:               true,
 		SockMemOptions: SockMemOptions{
 			EnableSettingSockMem: false,
 			SetGlobalTCPMemRatio: 20,  // default: 20% * {host total memory}
@@ -187,6 +191,10 @@ func (o *MemoryOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 		o.EnableNUMAAllocationReactor, "enable numa allocation reactor for numa binding pods to patch pod numa binding result annotation")
 	fs.StringVar(&o.NUMABindResultResourceAllocationAnnotationKey, "numa-bind-result-resource-allocation-annotation-key",
 		o.NUMABindResultResourceAllocationAnnotationKey, "the key of numa bind result resource allocation annotation")
+	fs.BoolVar(&o.EnableMemoryAnnotationValidator, "enable-memory-annotation-validator",
+		o.EnableMemoryAnnotationValidator, "enable the memory annotation validator for pods already have memory allocated by runtime")
+	fs.BoolVar(&o.MemoryAnnotationValidatorDryRun, "memory-annotation-validator-dry-run",
+		o.MemoryAnnotationValidatorDryRun, "enable the dry run mode for memory annotation validator")
 	fs.StringVar(&o.OOMPriorityPinnedMapAbsPath, "oom-priority-pinned-bpf-map-path",
 		o.OOMPriorityPinnedMapAbsPath, "the absolute path of oom priority pinned bpf map")
 	fs.BoolVar(&o.EnableSettingSockMem, "enable-setting-sockmem",
@@ -248,6 +256,8 @@ func (o *MemoryOptions) ApplyTo(conf *qrmconfig.MemoryQRMPluginConfig) error {
 	conf.EnableOOMPriority = o.EnableOOMPriority
 	conf.EnableNonBindingShareCoresMemoryResourceCheck = o.EnableNonBindingShareCoresMemoryResourceCheck
 	conf.EnableNUMAAllocationReactor = o.EnableNUMAAllocationReactor
+	conf.EnableMemoryAnnotationValidator = o.EnableMemoryAnnotationValidator
+	conf.MemoryAnnotationValidatorDryRun = o.MemoryAnnotationValidatorDryRun
 	conf.NUMABindResultResourceAllocationAnnotationKey = o.NUMABindResultResourceAllocationAnnotationKey
 	conf.OOMPriorityPinnedMapAbsPath = o.OOMPriorityPinnedMapAbsPath
 	conf.EnableSettingSockMem = o.EnableSettingSockMem

cmd/katalyst-agent/app/options/qrm/network_plugin.go

@@ -40,6 +40,8 @@ type NetworkOptions struct {
 	NetBandwidthResourceAllocationAnnotationKey     string
 	NICHealthCheckers                               []string
 	EnableNICAllocationReactor                      bool
+	EnableNICAnnotationValidator                    bool
+	AnnotationValidatorDryRun                       bool
 }
 
 type NetClassOptions struct {
@@ -69,6 +71,8 @@ func NewNetworkOptions() *NetworkOptions {
 		NetClassIDResourceAllocationAnnotationKey:       "qrm.katalyst.kubewharf.io/netcls_id",
 		NetBandwidthResourceAllocationAnnotationKey:     "qrm.katalyst.kubewharf.io/net_bandwidth",
 		EnableNICAllocationReactor:                      true,
+		EnableNICAnnotationValidator:                    true,
+		AnnotationValidatorDryRun:                       true,
 		NICHealthCheckers:                               []string{"*"},
 	}
 }
@@ -112,6 +116,10 @@ func (o *NetworkOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 		o.NetBandwidthResourceAllocationAnnotationKey, "The annotation key of allocated bandwidth for the container, which is ready by runtime")
 	fs.BoolVar(&o.EnableNICAllocationReactor, "enable-network-resource-plugin-nic-allocation-reactor",
 		o.EnableNICAllocationReactor, "enable network allocation reactor, default is true")
+	fs.BoolVar(&o.EnableNICAnnotationValidator, "enable-network-resource-plugin-nic-annotation-validator",
+		o.EnableNICAnnotationValidator, "enable network annotation validator, default is true")
+	fs.BoolVar(&o.AnnotationValidatorDryRun, "network-resource-plugin-annotation-validator-dry-run",
+		o.AnnotationValidatorDryRun, "enable the dry run mode for network annotation validator, default is true")
 	fs.StringSliceVar(&o.NICHealthCheckers, "network-resource-plugin-nic-health-checkers",
 		o.NICHealthCheckers, "list of nic health checkers, '*' run all on-by-default checkers,"+
 			"'ip' run checker 'ip', '-ip' not run checker 'ip'")
@@ -136,7 +144,9 @@ func (o *NetworkOptions) ApplyTo(conf *qrmconfig.NetworkQRMPluginConfig) error {
 	conf.NetClassIDResourceAllocationAnnotationKey = o.NetClassIDResourceAllocationAnnotationKey
 	conf.NetBandwidthResourceAllocationAnnotationKey = o.NetBandwidthResourceAllocationAnnotationKey
 	conf.EnableNICAllocationReactor = o.EnableNICAllocationReactor
+	conf.EnableNICAnnotationValidator = o.EnableNICAnnotationValidator
 	conf.NICHealthCheckers = o.NICHealthCheckers
+	conf.NICAnnotationValidatorDryRun = o.AnnotationValidatorDryRun
 
 	return nil
 }

pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go

@@ -43,12 +43,14 @@ import (
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/oom"
 	memoryreactor "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/reactor"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/state"
+	memoryvalidator "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/dynamicpolicy/validator"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/handlers/fragmem"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/handlers/hostwatermark"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/handlers/logcache"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/memory/handlers/sockmem"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util/reactor"
+	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util/validator"
 	"github.com/kubewharf/katalyst-core/pkg/agent/utilcomponent/featuregatenegotiation"
 	"github.com/kubewharf/katalyst-core/pkg/agent/utilcomponent/periodicalhandler"
 	"github.com/kubewharf/katalyst-core/pkg/config"
@@ -165,6 +167,9 @@ type DynamicPolicy struct {
 
 	numaAllocationReactor                         reactor.AllocationReactor
 	numaBindResultResourceAllocationAnnotationKey string
+
+	memoryAnnotationValidator validator.AnnotationValidator
+	annotationValidatorDryRun bool
 }
 
 func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration,
@@ -237,6 +242,7 @@ func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration
 		resctrlHinter:               newResctrlHinter(&conf.ResctrlConfig, wrappedEmitter),
 		enableNonBindingShareCoresMemoryResourceCheck: conf.EnableNonBindingShareCoresMemoryResourceCheck,
 		numaBindResultResourceAllocationAnnotationKey: conf.NUMABindResultResourceAllocationAnnotationKey,
+		annotationValidatorDryRun:                     conf.NICAnnotationValidatorDryRun,
 	}
 
 	policyImplement.allocationHandlers = map[string]util.AllocationHandler{
@@ -298,6 +304,13 @@ func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration
 			))
 	}
 
+	policyImplement.memoryAnnotationValidator = validator.DummyAnnotationValidator{}
+	if conf.EnableMemoryAnnotationValidator {
+		policyImplement.memoryAnnotationValidator = memoryvalidator.NewMemoryAnnotationValidator(conf,
+			agentCtx.Client.KubeClient,
+			agentCtx.MetaServer.PodFetcher)
+	}
+
 	return true, &agent.PluginWrapper{GenericPlugin: pluginWrapper}, nil
 }
 
@@ -920,6 +933,23 @@ func (p *DynamicPolicy) Allocate(ctx context.Context,
 	// we should do it before GetKatalystQoSLevelFromResourceReq.
 	isDebugPod := util.IsDebugPod(req.Annotations, p.podDebugAnnoKeys)
 
+	valid, err := p.memoryAnnotationValidator.ValidatePodAnnotation(ctx, req.PodUid, req.PodNamespace, req.PodName)
+	if !isDebugPod || !valid || err != nil {
+		general.Warningf("pod annotations verification failed: %v", err)
+
+		metricTags := []metrics.MetricTag{
+			{Key: "pod_uid", Val: req.PodUid},
+			{Key: "pod_namespace", Val: req.PodNamespace},
+			{Key: "pod_name", Val: req.PodName},
+			{Key: "error_message", Val: metric.MetricTagValueFormat(err)},
+		}
+		p.emitter.StoreInt64(util.MetricNamePodAnnotationVerificationFailed, 1, metrics.MetricTypeNameRaw, metricTags...)
+
+		if !p.annotationValidatorDryRun {
+			return nil, fmt.Errorf("pod annotations verification failed: %v", err)
+		}
+	}
+
 	existReallocAnno, isReallocation := util.IsReallocation(req.Annotations)
 
 	qosLevel, err := util.GetKatalystQoSLevelFromResourceReq(p.qosConfig, req, p.podAnnotationKeptKeys, p.podLabelKeptKeys)

pkg/agent/qrm-plugins/memory/dynamicpolicy/validator/memory_annotation_validator.go

@@ -0,0 +1,60 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validator
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util"
+	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util/validator"
+	"github.com/kubewharf/katalyst-core/pkg/config"
+	"github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod"
+)
+
+type memoryAnnotationValidator struct {
+	conf       *config.Configuration
+	client     kubernetes.Interface
+	podFetcher pod.PodFetcher
+}
+
+func NewMemoryAnnotationValidator(conf *config.Configuration, client kubernetes.Interface, podFetcher pod.PodFetcher) validator.AnnotationValidator {
+	return &memoryAnnotationValidator{
+		conf:       conf,
+		client:     client,
+		podFetcher: podFetcher,
+	}
+}
+
+func (n *memoryAnnotationValidator) ValidatePodAnnotation(ctx context.Context, podUID, podNameSpace, podName string) (bool, error) {
+	annoList := map[string]string{
+		util.AnnotationRdtClosID:           "",
+		util.AnnotationRdtNeedPodMonGroups: "",
+	}
+
+	getPod, err := n.podFetcher.GetPod(context.WithValue(ctx, pod.BypassCacheKey, pod.BypassCacheTrue), podUID)
+	if err != nil {
+		getPod, err = n.client.CoreV1().Pods(podNameSpace).Get(ctx, podName, metav1.GetOptions{ResourceVersion: "0"})
+		if err != nil {
+			return false, err
+		}
+	}
+
+	return validator.ValidatePodAnnotations(getPod, annoList)
+}

pkg/agent/qrm-plugins/network/staticpolicy/policy.go

@@ -40,8 +40,10 @@ import (
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/network/state"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/network/staticpolicy/nic"
 	networkreactor "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/network/staticpolicy/reactor"
+	networkvalidator "github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/network/staticpolicy/validator"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util"
 	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util/reactor"
+	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util/validator"
 	"github.com/kubewharf/katalyst-core/pkg/agent/utilcomponent/periodicalhandler"
 	"github.com/kubewharf/katalyst-core/pkg/config"
 	agentconfig "github.com/kubewharf/katalyst-core/pkg/config/agent"
@@ -109,7 +111,8 @@ type StaticPolicy struct {
 
 	lowPriorityGroups map[string]*qrmgeneral.NetworkGroup
 
-	nicAllocationReactor reactor.AllocationReactor
+	nicAllocationReactor   reactor.AllocationReactor
+	nicAnnotationValidator validator.AnnotationValidator
 
 	nicManager nic.NICManager
 
@@ -188,6 +191,13 @@ func NewStaticPolicy(agentCtx *agent.GenericContext, conf *config.Configuration,
 			))
 	}
 
+	policyImplement.nicAnnotationValidator = validator.DummyAnnotationValidator{}
+	if conf.EnableNICAnnotationValidator {
+		policyImplement.nicAnnotationValidator = networkvalidator.NewNICAnnotationValidator(conf,
+			agentCtx.Client.KubeClient,
+			agentCtx.MetaServer.PodFetcher)
+	}
+
 	pluginWrapper, err := skeleton.NewRegistrationPluginWrapper(policyImplement, conf.QRMPluginSocketDirs,
 		func(key string, value int64) {
 			_ = wrappedEmitter.StoreInt64(key, value, metrics.MetricTypeNameRaw)
@@ -668,6 +678,23 @@ func (p *StaticPolicy) Allocate(ctx context.Context,
 		return nil, fmt.Errorf("Allocate got nil req")
 	}
 
+	valid, err := p.nicAnnotationValidator.ValidatePodAnnotation(ctx, req.PodUid, req.PodNamespace, req.PodName)
+	if !valid || err != nil {
+		general.Warningf("pod annotations verification failed: %v", err)
+
+		metricTags := []metrics.MetricTag{
+			{Key: "pod_uid", Val: req.PodUid},
+			{Key: "pod_namespace", Val: req.PodNamespace},
+			{Key: "pod_name", Val: req.PodName},
+			{Key: "error_message", Val: metric.MetricTagValueFormat(err)},
+		}
+		p.emitter.StoreInt64(util.MetricNamePodAnnotationVerificationFailed, 1, metrics.MetricTypeNameRaw, metricTags...)
+
+		if !p.qrmConfig.NICAnnotationValidatorDryRun {
+			return nil, fmt.Errorf("pod annotations verification failed: %v", err)
+		}
+	}
+
 	existReallocAnno, isReallocation := util.IsReallocation(req.Annotations)
 
 	// since qos config util will filter out annotation keys not related to katalyst QoS,

pkg/agent/qrm-plugins/network/staticpolicy/validator/nic_annotation_validator.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validator
+
+import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/kubewharf/katalyst-core/pkg/agent/qrm-plugins/util/validator"
+	"github.com/kubewharf/katalyst-core/pkg/config"
+	"github.com/kubewharf/katalyst-core/pkg/metaserver/agent/pod"
+)
+
+type nicAnnotationValidator struct {
+	conf       *config.Configuration
+	client     kubernetes.Interface
+	podFetcher pod.PodFetcher
+}
+
+func NewNICAnnotationValidator(conf *config.Configuration, client kubernetes.Interface, podFetcher pod.PodFetcher) validator.AnnotationValidator {
+	return &nicAnnotationValidator{
+		conf:       conf,
+		client:     client,
+		podFetcher: podFetcher,
+	}
+}
+
+func (n *nicAnnotationValidator) ValidatePodAnnotation(ctx context.Context, podUID, podNameSpace, podName string) (bool, error) {
+	annoList := map[string]string{
+		n.conf.NetworkQRMPluginConfig.IPv4ResourceAllocationAnnotationKey:             "",
+		n.conf.NetworkQRMPluginConfig.IPv6ResourceAllocationAnnotationKey:             "",
+		n.conf.NetworkQRMPluginConfig.NetNSPathResourceAllocationAnnotationKey:        "",
+		n.conf.NetworkQRMPluginConfig.NetInterfaceNameResourceAllocationAnnotationKey: "",
+		n.conf.NetworkQRMPluginConfig.NetClassIDResourceAllocationAnnotationKey:       "",
+		n.conf.NetworkQRMPluginConfig.NetBandwidthResourceAllocationAnnotationKey:     "",
+	}
+
+	getPod, err := n.podFetcher.GetPod(context.WithValue(ctx, pod.BypassCacheKey, pod.BypassCacheTrue), podUID)
+	if err != nil {
+		getPod, err = n.client.CoreV1().Pods(podNameSpace).Get(ctx, podName, metav1.GetOptions{ResourceVersion: "0"})
+		if err != nil {
+			return false, err
+		}
+	}
+
+	return validator.ValidatePodAnnotations(getPod, annoList)
+}

pkg/agent/qrm-plugins/util/consts.go

@@ -95,6 +95,9 @@ const (
 	MetricNameIrqTuningNicExclusiveIrqCoresCpuUtilMin = "irq_tuning_nic_exclusive_irq_cores_cpu_util_Min"
 	MetricNameIrqTuningNicExclusiveIrqCoresCpuUsage   = "irq_tuning_nic_exclusive_irq_cores_cpu_usage"
 	MetricNameIrqTuningErr                            = "irq_tuning_err"
+
+	// metrics for pod annotation validator
+	MetricNamePodAnnotationVerificationFailed = "pod_annotation_verification_failed"
 )
 
 const (

pkg/agent/qrm-plugins/util/validator/validator.go

@@ -0,0 +1,51 @@
+/*
+Copyright 2022 The Katalyst Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package validator
+
+import (
+	"context"
+	"fmt"
+
+	v1 "k8s.io/api/core/v1"
+)
+
+type AnnotationValidator interface {
+	ValidatePodAnnotation(ctx context.Context, podUID, podNameSpace, podName string) (bool, error)
+}
+
+type DummyAnnotationValidator struct{}
+
+func (d DummyAnnotationValidator) ValidatePodAnnotation(_ context.Context, _, _, _ string) (bool, error) {
+	return true, nil
+}
+
+// ValidatePodAnnotations validates the pod annotations
+// if any annotation is forbidden, it returns true and the annotation name
+// otherwise, it returns false and empty string
+func ValidatePodAnnotations(pod *v1.Pod, forbiddenAnnos map[string]string) (bool, error) {
+	if pod == nil || pod.Annotations == nil {
+		return true, nil
+	}
+
+	for _, anno := range pod.Annotations {
+		if _, ok := forbiddenAnnos[anno]; ok {
+			return false, fmt.Errorf("the pod contains an invalid annotation: %s", anno)
+		}
+	}
+
+	return true, nil
+}