feat(agent): eviction enhancement

Author: funnydreamwinz

cmd/katalyst-agent/app/options/dynamic/adminqos/eviction/reclaimed_resources_eviction.go

@@ -26,6 +26,7 @@ import (
 
 type ReclaimedResourcesEvictionOptions struct {
 	EvictionThreshold             native.ResourceThreshold
+	SoftEvictionThreshold         native.ResourceThreshold
 	GracePeriod                   int64
 	ThresholdMetToleranceDuration int64
 }
@@ -36,6 +37,10 @@ func NewReclaimedResourcesEvictionOptions() *ReclaimedResourcesEvictionOptions {
 			consts.ReclaimedResourceMilliCPU: 5.0,
 			consts.ReclaimedResourceMemory:   5.0,
 		},
+		SoftEvictionThreshold: native.ResourceThreshold{
+			consts.ReclaimedResourceMilliCPU: 1.5,
+			consts.ReclaimedResourceMemory:   1.5,
+		},
 		GracePeriod:                   60,
 		ThresholdMetToleranceDuration: 0,
 	}
@@ -54,6 +59,7 @@ func (o *ReclaimedResourcesEvictionOptions) AddFlags(fss *cliflag.NamedFlagSets)
 
 func (o *ReclaimedResourcesEvictionOptions) ApplyTo(c *eviction.ReclaimedResourcesEvictionConfiguration) error {
 	c.EvictionThreshold = o.EvictionThreshold
+	c.SoftEvictionThreshold = o.SoftEvictionThreshold
 	c.DeletionGracePeriod = o.GracePeriod
 	c.ThresholdMetToleranceDuration = o.ThresholdMetToleranceDuration
 	return nil

cmd/katalyst-agent/app/options/eviction/eviction_base.go

@@ -52,6 +52,9 @@ type GenericEvictionOptions struct {
 
 	// PodMetricLabels defines the pod labels to be added into metric selector list.
 	PodMetricLabels []string
+
+	// HostPathNotifierPathRoot is the root path for host-path notifier
+	HostPathNotifierRootPath string
 }
 
 // NewGenericEvictionOptions creates a new Options with a default config.
@@ -63,6 +66,7 @@ func NewGenericEvictionOptions() *GenericEvictionOptions {
 		EvictionSkippedAnnotationKeys: []string{},
 		EvictionSkippedLabelKeys:      []string{},
 		EvictionBurst:                 3,
+		HostPathNotifierRootPath:      "/opt/katalyst",
 		PodKiller:                     consts.KillerNameEvictionKiller,
 		StrictAuthentication:          false,
 	}
@@ -99,6 +103,9 @@ func (o *GenericEvictionOptions) AddFlags(fss *cliflag.NamedFlagSets) {
 
 	fs.StringSliceVar(&o.PodMetricLabels, "eviction-pod-metric-labels", o.PodMetricLabels,
 		"The pod labels to be added into metric selector list")
+
+	fs.StringVar(&o.HostPathNotifierRootPath, "pod-notifier-root-path", o.HostPathNotifierRootPath,
+		"root path of host-path notifier")
 }
 
 // ApplyTo fills up config with options
@@ -112,6 +119,7 @@ func (o *GenericEvictionOptions) ApplyTo(c *evictionconfig.GenericEvictionConfig
 	c.PodKiller = o.PodKiller
 	c.StrictAuthentication = o.StrictAuthentication
 	c.PodMetricLabels.Insert(o.PodMetricLabels...)
+	c.HostPathNotifierRootPath = o.HostPathNotifierRootPath
 	return nil
 }
 

go.mod

@@ -173,6 +173,8 @@ require (
 )
 
 replace (
+	// FIXME
+	github.com/kubewharf/katalyst-api => github.com/funnydreamwinz/katalyst-api v0.0.0-20250811070245-94cac124cdd7
 	k8s.io/api => k8s.io/api v0.24.6
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.24.6
 	k8s.io/apimachinery => k8s.io/apimachinery v0.24.6

go.sum

@@ -279,6 +279,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
 github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
+github.com/funnydreamwinz/katalyst-api v0.0.0-20250811070245-94cac124cdd7 h1:cSYxLZ+taqOLV02NutItx9IhR4wEMBe/KLTTLPr66a4=
+github.com/funnydreamwinz/katalyst-api v0.0.0-20250811070245-94cac124cdd7/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
 github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
 github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=

pkg/agent/evictionmanager/eviction_resp_collector.go

@@ -158,6 +158,42 @@ func (e *evictionRespCollector) collectMetThreshold(dryRunPlugins []string, plug
 	}
 }
 
+func (e *evictionRespCollector) collectTopSoftEvictionPods(dryRunPlugins []string, pluginName string,
+	threshold *pluginapi.ThresholdMetResponse, resp *pluginapi.GetTopEvictionPodsResponse,
+) {
+	dryRun := e.isDryRun(dryRunPlugins, pluginName)
+
+	targetPods := make([]*v1.Pod, 0, len(resp.TargetPods))
+	for i, pod := range resp.TargetPods {
+		if pod == nil {
+			continue
+		}
+
+		general.Infof("%v plugin %v request to notify topN pod %v/%v, reason: met threshold in scope [%v]",
+			e.getLogPrefix(dryRun), pluginName, pod.Namespace, pod.Name, threshold.EvictionScope)
+		if dryRun {
+			metricsPodToEvict(e.emitter, e.conf.GenericConfiguration.QoSConfiguration, pluginName, pod, dryRun, e.conf.GenericEvictionConfiguration.PodMetricLabels)
+		} else {
+			targetPods = append(targetPods, resp.TargetPods[i])
+		}
+	}
+
+	for _, pod := range targetPods {
+		reason := fmt.Sprintf("plugin %s met threshold in scope %s, target %v, observed %v",
+			pluginName, threshold.EvictionScope, threshold.ThresholdValue, threshold.ObservedValue)
+
+		e.getSoftEvictPods()[string(pod.UID)] = &rule.RuledEvictPod{
+			EvictPod: &pluginapi.EvictPod{
+				Pod:                pod.DeepCopy(),
+				Reason:             reason,
+				ForceEvict:         false,
+				EvictionPluginName: pluginName,
+			},
+			Scope: threshold.EvictionScope,
+		}
+	}
+}
+
 func (e *evictionRespCollector) collectTopEvictionPods(dryRunPlugins []string, pluginName string,
 	threshold *pluginapi.ThresholdMetResponse, resp *pluginapi.GetTopEvictionPodsResponse,
 ) {

pkg/agent/evictionmanager/manager.go

@@ -45,6 +45,7 @@ import (
 	"github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/plugin/resource"
 	"github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/plugin/rootfs"
 	"github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/podkiller"
+	"github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/podnotifier"
 	"github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/rule"
 	"github.com/kubewharf/katalyst-core/pkg/client"
 	"github.com/kubewharf/katalyst-core/pkg/client/control"
@@ -104,7 +105,8 @@ type EvictionManger struct {
 	// easy to test the code.
 	clock clocks.WithTickerAndDelayedExecution
 
-	podKiller podkiller.PodKiller
+	podNotifier podnotifier.PodNotifier
+	podKiller   podkiller.PodKiller
 
 	killQueue    rule.EvictionQueue
 	killStrategy rule.EvictionStrategy
@@ -171,6 +173,12 @@ func NewEvictionManager(genericClient *client.GenericClientSet, recorder events.
 
 	podKiller := podkiller.NewAsynchronizedPodKiller(killer, metaServer.PodFetcher, genericClient.KubeClient)
 
+	notifier, err := podnotifier.NewHostPathPodNotifier(conf, genericClient.KubeClient, metaServer, recorder, emitter)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create pod notifier: %v", err)
+	}
+	podNotifier := podnotifier.NewSynchronizedPodNotifier(notifier)
+
 	cnrTaintReporter, err := control.NewGenericReporterPlugin(cnrTaintReporterPluginName, conf, emitter)
 	if err != nil {
 		return nil, fmt.Errorf("failed to initialize cnr taint reporter plugin: %v", err)
@@ -183,6 +191,7 @@ func NewEvictionManager(genericClient *client.GenericClientSet, recorder events.
 		metaGetter:                metaServer,
 		emitter:                   emitter,
 		podKiller:                 podKiller,
+		podNotifier:               podNotifier,
 		cnrTaintReporter:          cnrTaintReporter,
 		endpoints:                 make(map[string]endpointpkg.Endpoint),
 		conf:                      conf,
@@ -235,6 +244,7 @@ func (m *EvictionManger) Run(ctx context.Context) {
 	general.RegisterHeartbeatCheck(reportTaintHealthCheckName, reportTaintToleration,
 		general.HealthzCheckStateNotReady, reportTaintToleration)
 	m.podKiller.Start(ctx)
+	m.podNotifier.Start(ctx)
 	for _, endpoint := range m.endpoints {
 		endpoint.Start()
 	}
@@ -277,6 +287,11 @@ func (m *EvictionManger) sync(ctx context.Context) {
 	}
 
 	errList := make([]error, 0)
+	notifyErr := m.doNotify(collector.getSoftEvictPods())
+	if notifyErr != nil {
+		errList = append(errList, notifyErr)
+	}
+
 	evictErr := m.doEvict(collector.getSoftEvictPods(), collector.getForceEvictPods())
 	if evictErr != nil {
 		errList = append(errList, evictErr)
@@ -342,8 +357,8 @@ func (m *EvictionManger) collectEvictionResult(pods []*v1.Pod) (*evictionRespCol
 	m.conditionLock.Unlock()
 
 	for pluginName, threshold := range thresholdsMet {
-		if threshold.MetType != pluginapi.ThresholdMetType_HARD_MET {
-			general.Infof(" the type: %s of met threshold from plugin: %s isn't  %s", threshold.MetType.String(), pluginName, pluginapi.ThresholdMetType_HARD_MET.String())
+		if threshold.MetType == pluginapi.ThresholdMetType_NOT_MET {
+			general.Infof("resp from plugin: %s not met threshold", pluginName)
 			continue
 		}
 
@@ -352,12 +367,18 @@ func (m *EvictionManger) collectEvictionResult(pods []*v1.Pod) (*evictionRespCol
 			general.Errorf(" pluginName points to nil endpoint, can't handle threshold from it")
 		}
 
+		topN := uint64(0)
+		forceEvict := false
+		if threshold.MetType == pluginapi.ThresholdMetType_HARD_MET {
+			topN = 1
+			forceEvict = true
+		}
+
 		resp, err := m.endpoints[pluginName].GetTopEvictionPods(context.Background(), &pluginapi.GetTopEvictionPodsRequest{
 			ActivePods:    pods,
-			TopN:          1,
+			TopN:          topN,
 			EvictionScope: threshold.EvictionScope,
 		})
-
 		m.endpointLock.RUnlock()
 		if err != nil {
 			general.Errorf(" calling GetTopEvictionPods of plugin: %s failed with error: %v", pluginName, err)
@@ -371,12 +392,40 @@ func (m *EvictionManger) collectEvictionResult(pods []*v1.Pod) (*evictionRespCol
 			continue
 		}
 
-		collector.collectTopEvictionPods(dynamicConfig.DryRun, pluginName, threshold, resp)
+		if forceEvict {
+			collector.collectTopEvictionPods(dynamicConfig.DryRun, pluginName, threshold, resp)
+		} else {
+			collector.collectTopSoftEvictionPods(dynamicConfig.DryRun, pluginName, threshold, resp)
+		}
+
 	}
 
 	return collector, errors.NewAggregate(errList)
 }
 
+func (m *EvictionManger) doNotify(softEvictPods map[string]*rule.RuledEvictPod) error {
+	errList := make([]error, 0)
+
+	for _, pod := range softEvictPods {
+		if pod == nil || pod.EvictPod.Pod == nil {
+			continue
+		}
+
+		// TODO
+		// if _, ok := pod.EvictPod.Pod.Annotations[constapi.PodAnnotationSoftEvictNotificationKey]; !ok {
+		if _, ok := pod.EvictPod.Pod.Annotations["katalyst.kubewharf.io/soft_evict_notify"]; !ok {
+			continue
+		}
+
+		err := m.podNotifier.NotifyPod(pod)
+		if err != nil {
+			errList = append(errList, err)
+		}
+	}
+
+	return errors.NewAggregate(errList)
+}
+
 func (m *EvictionManger) doEvict(softEvictPods, forceEvictPods map[string]*rule.RuledEvictPod) error {
 	softEvictPods = filterOutCandidatePodsWithForcePods(softEvictPods, forceEvictPods)
 	bestSuitedCandidate := m.getEvictPodFromCandidates(softEvictPods)

pkg/agent/evictionmanager/plugin/resource/reclaimed_resources.go

@@ -62,6 +62,10 @@ func NewReclaimedResourcesEvictionPlugin(_ *client.GenericClientSet, _ events.Ev
 		}
 	}
 
+	reclaimedSoftThresholdGetter := func(resourceName v1.ResourceName) *float64 {
+		return nil
+	}
+
 	deletionGracePeriodGetter := func() int64 {
 		return conf.GetDynamicConfiguration().ReclaimedResourcesEvictionConfiguration.DeletionGracePeriod
 	}
@@ -75,6 +79,8 @@ func NewReclaimedResourcesEvictionPlugin(_ *client.GenericClientSet, _ events.Ev
 		emitter,
 		reclaimedResourcesGetter,
 		reclaimedThresholdGetter,
+		reclaimedSoftThresholdGetter,
+		nil,
 		deletionGracePeriodGetter,
 		thresholdMetToleranceDurationGetter,
 		conf.SkipZeroQuantityResourceNames,