Skip to content

Commit 7e28cbe

Browse files
funnydreamwinzfunnydreamwinz
committed
feat(agent): eviction enhancement
1 parent 305cceb commit 7e28cbe

File tree

23 files changed

+1189
-45
lines changed

23 files changed

+1189
-45
lines changed

‎cmd/katalyst-agent/app/options/dynamic/adminqos/eviction/reclaimed_resources_eviction.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626

2727
type ReclaimedResourcesEvictionOptions struct {
2828
EvictionThreshold native.ResourceThreshold
29+
SoftEvictionThreshold native.ResourceThreshold
2930
GracePeriod int64
3031
ThresholdMetToleranceDuration int64
3132
}
@@ -36,6 +37,10 @@ func NewReclaimedResourcesEvictionOptions() *ReclaimedResourcesEvictionOptions {
3637
consts.ReclaimedResourceMilliCPU: 5.0,
3738
consts.ReclaimedResourceMemory: 5.0,
3839
},
40+
SoftEvictionThreshold: native.ResourceThreshold{
41+
consts.ReclaimedResourceMilliCPU: 1.5,
42+
consts.ReclaimedResourceMemory: 1.5,
43+
},
3944
GracePeriod: 60,
4045
ThresholdMetToleranceDuration: 0,
4146
}
@@ -54,6 +59,7 @@ func (o *ReclaimedResourcesEvictionOptions) AddFlags(fss *cliflag.NamedFlagSets)
5459

5560
func (o *ReclaimedResourcesEvictionOptions) ApplyTo(c *eviction.ReclaimedResourcesEvictionConfiguration) error {
5661
c.EvictionThreshold = o.EvictionThreshold
62+
c.SoftEvictionThreshold = o.SoftEvictionThreshold
5763
c.DeletionGracePeriod = o.GracePeriod
5864
c.ThresholdMetToleranceDuration = o.ThresholdMetToleranceDuration
5965
return nil

‎cmd/katalyst-agent/app/options/eviction/eviction_base.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,9 @@ type GenericEvictionOptions struct {
5555

5656
// PodMetricLabels defines the pod labels to be added into metric selector list.
5757
PodMetricLabels []string
58+
59+
// HostPathNotifierPathRoot is the root path for host-path notifier
60+
HostPathNotifierRootPath string
5861
}
5962

6063
// NewGenericEvictionOptions creates a new Options with a default config.
@@ -66,6 +69,7 @@ func NewGenericEvictionOptions() *GenericEvictionOptions {
6669
EvictionSkippedAnnotationKeys: []string{},
6770
EvictionSkippedLabelKeys: []string{},
6871
EvictionBurst: 3,
72+
HostPathNotifierRootPath: "/opt/katalyst",
6973
PodKiller: consts.KillerNameEvictionKiller,
7074
StrictAuthentication: false,
7175
}
@@ -105,6 +109,9 @@ func (o *GenericEvictionOptions) AddFlags(fss *cliflag.NamedFlagSets) {
105109

106110
fs.StringSliceVar(&o.PodMetricLabels, "eviction-pod-metric-labels", o.PodMetricLabels,
107111
"The pod labels to be added into metric selector list")
112+
113+
fs.StringVar(&o.HostPathNotifierRootPath, "pod-notifier-root-path", o.HostPathNotifierRootPath,
114+
"root path of host-path notifier")
108115
}
109116

110117
// ApplyTo fills up config with options
@@ -119,6 +126,7 @@ func (o *GenericEvictionOptions) ApplyTo(c *evictionconfig.GenericEvictionConfig
119126
c.QoSPodKillers = o.QoSPodKillers
120127
c.StrictAuthentication = o.StrictAuthentication
121128
c.PodMetricLabels.Insert(o.PodMetricLabels...)
129+
c.HostPathNotifierRootPath = o.HostPathNotifierRootPath
122130
return nil
123131
}
124132

‎go.mod‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,8 @@ require (
173173
)
174174

175175
replace (
176+
// FIXME
177+
github.com/kubewharf/katalyst-api => github.com/funnydreamwinz/katalyst-api v0.0.0-20250811070245-94cac124cdd7
176178
k8s.io/api => k8s.io/api v0.24.6
177179
k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.24.6
178180
k8s.io/apimachinery => k8s.io/apimachinery v0.24.6

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
279279
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
280280
github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
281281
github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
282+
github.com/funnydreamwinz/katalyst-api v0.0.0-20250811070245-94cac124cdd7 h1:cSYxLZ+taqOLV02NutItx9IhR4wEMBe/KLTTLPr66a4=
283+
github.com/funnydreamwinz/katalyst-api v0.0.0-20250811070245-94cac124cdd7/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
282284
github.com/fvbommel/sortorder v1.0.1/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
283285
github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg=
284286
github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ=
@@ -573,8 +575,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
573575
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
574576
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
575577
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
576-
github.com/kubewharf/katalyst-api v0.5.6-0.20250723073136-24e693f5681c h1:klz9kaaT0KHR/9K2xzTiSAs5tDhcqYk7QfCf/tOWc+4=
577-
github.com/kubewharf/katalyst-api v0.5.6-0.20250723073136-24e693f5681c/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
578578
github.com/kubewharf/kubelet v1.24.6-kubewharf.9 h1:jOTYZt7h/J7I8xQMKMUcJjKf5UFBv37jHWvNp5VRFGc=
579579
github.com/kubewharf/kubelet v1.24.6-kubewharf.9/go.mod h1:MxbSZUx3wXztFneeelwWWlX7NAAStJ6expqq7gY2J3c=
580580
github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8=

‎pkg/agent/evictionmanager/eviction_resp_collector.go‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,42 @@ func (e *evictionRespCollector) collectMetThreshold(dryRunPlugins []string, plug
158158
}
159159
}
160160

161+
func (e *evictionRespCollector) collectTopSoftEvictionPods(dryRunPlugins []string, pluginName string,
162+
threshold *pluginapi.ThresholdMetResponse, resp *pluginapi.GetTopEvictionPodsResponse,
163+
) {
164+
dryRun := e.isDryRun(dryRunPlugins, pluginName)
165+
166+
targetPods := make([]*v1.Pod, 0, len(resp.TargetPods))
167+
for i, pod := range resp.TargetPods {
168+
if pod == nil {
169+
continue
170+
}
171+
172+
general.Infof("%v plugin %v request to notify topN pod %v/%v, reason: met threshold in scope [%v]",
173+
e.getLogPrefix(dryRun), pluginName, pod.Namespace, pod.Name, threshold.EvictionScope)
174+
if dryRun {
175+
metricsPodToEvict(e.emitter, e.conf.GenericConfiguration.QoSConfiguration, pluginName, pod, dryRun, e.conf.GenericEvictionConfiguration.PodMetricLabels)
176+
} else {
177+
targetPods = append(targetPods, resp.TargetPods[i])
178+
}
179+
}
180+
181+
for _, pod := range targetPods {
182+
reason := fmt.Sprintf("plugin %s met threshold in scope %s, target %v, observed %v",
183+
pluginName, threshold.EvictionScope, threshold.ThresholdValue, threshold.ObservedValue)
184+
185+
e.getSoftEvictPods()[string(pod.UID)] = &rule.RuledEvictPod{
186+
EvictPod: &pluginapi.EvictPod{
187+
Pod: pod.DeepCopy(),
188+
Reason: reason,
189+
ForceEvict: false,
190+
EvictionPluginName: pluginName,
191+
},
192+
Scope: threshold.EvictionScope,
193+
}
194+
}
195+
}
196+
161197
func (e *evictionRespCollector) collectTopEvictionPods(dryRunPlugins []string, pluginName string,
162198
threshold *pluginapi.ThresholdMetResponse, resp *pluginapi.GetTopEvictionPodsResponse,
163199
) {

‎pkg/agent/evictionmanager/manager.go‎

Lines changed: 58 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ import (
3737
clocks "k8s.io/utils/clock"
3838

3939
"github.com/kubewharf/katalyst-api/pkg/apis/node/v1alpha1"
40+
apiconsts "github.com/kubewharf/katalyst-api/pkg/consts"
4041
"github.com/kubewharf/katalyst-api/pkg/plugins/registration"
4142
pluginapi "github.com/kubewharf/katalyst-api/pkg/protocol/evictionplugin/v1alpha1"
4243
endpointpkg "github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/endpoint"
@@ -46,6 +47,7 @@ import (
4647
"github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/plugin/resource"
4748
"github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/plugin/rootfs"
4849
"github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/podkiller"
50+
"github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/podnotifier"
4951
"github.com/kubewharf/katalyst-core/pkg/agent/evictionmanager/rule"
5052
"github.com/kubewharf/katalyst-core/pkg/client"
5153
"github.com/kubewharf/katalyst-core/pkg/client/control"
@@ -105,7 +107,8 @@ type EvictionManger struct {
105107
// easy to test the code.
106108
clock clocks.WithTickerAndDelayedExecution
107109

108-
podKiller podkiller.PodKiller
110+
podNotifier podnotifier.PodNotifier
111+
podKiller podkiller.PodKiller
109112

110113
killQueue rule.EvictionQueue
111114
killStrategy rule.EvictionStrategy
@@ -230,6 +233,12 @@ func NewEvictionManager(genericClient *client.GenericClientSet, recorder events.
230233

231234
podKiller := podkiller.NewAsynchronizedPodKiller(killer, metaServer.PodFetcher, genericClient.KubeClient)
232235

236+
notifier, err := podnotifier.NewHostPathPodNotifier(conf, genericClient.KubeClient, metaServer, recorder, emitter)
237+
if err != nil {
238+
return nil, fmt.Errorf("failed to create pod notifier: %v", err)
239+
}
240+
podNotifier := podnotifier.NewSynchronizedPodNotifier(notifier)
241+
233242
cnrTaintReporter, err := control.NewGenericReporterPlugin(cnrTaintReporterPluginName, conf, emitter)
234243
if err != nil {
235244
return nil, fmt.Errorf("failed to initialize cnr taint reporter plugin: %v", err)
@@ -242,6 +251,7 @@ func NewEvictionManager(genericClient *client.GenericClientSet, recorder events.
242251
metaGetter: metaServer,
243252
emitter: emitter,
244253
podKiller: podKiller,
254+
podNotifier: podNotifier,
245255
cnrTaintReporter: cnrTaintReporter,
246256
endpoints: make(map[string]endpointpkg.Endpoint),
247257
conf: conf,
@@ -294,6 +304,7 @@ func (m *EvictionManger) Run(ctx context.Context) {
294304
general.RegisterHeartbeatCheck(reportTaintHealthCheckName, reportTaintToleration,
295305
general.HealthzCheckStateNotReady, reportTaintToleration)
296306
m.podKiller.Start(ctx)
307+
m.podNotifier.Start(ctx)
297308
for _, endpoint := range m.endpoints {
298309
endpoint.Start()
299310
}
@@ -336,6 +347,11 @@ func (m *EvictionManger) sync(ctx context.Context) {
336347
}
337348

338349
errList := make([]error, 0)
350+
notifyErr := m.doNotify(collector.getSoftEvictPods())
351+
if notifyErr != nil {
352+
errList = append(errList, notifyErr)
353+
}
354+
339355
evictErr := m.doEvict(collector.getSoftEvictPods(), collector.getForceEvictPods())
340356
if evictErr != nil {
341357
errList = append(errList, evictErr)
@@ -401,8 +417,8 @@ func (m *EvictionManger) collectEvictionResult(pods []*v1.Pod) (*evictionRespCol
401417
m.conditionLock.Unlock()
402418

403419
for pluginName, threshold := range thresholdsMet {
404-
if threshold.MetType != pluginapi.ThresholdMetType_HARD_MET {
405-
general.Infof(" the type: %s of met threshold from plugin: %s isn't %s", threshold.MetType.String(), pluginName, pluginapi.ThresholdMetType_HARD_MET.String())
420+
if threshold.MetType == pluginapi.ThresholdMetType_NOT_MET {
421+
general.Infof("resp from plugin: %s not met threshold", pluginName)
406422
continue
407423
}
408424

@@ -411,12 +427,18 @@ func (m *EvictionManger) collectEvictionResult(pods []*v1.Pod) (*evictionRespCol
411427
general.Errorf(" pluginName points to nil endpoint, can't handle threshold from it")
412428
}
413429

430+
topN := uint64(0)
431+
forceEvict := false
432+
if threshold.MetType == pluginapi.ThresholdMetType_HARD_MET {
433+
topN = 1
434+
forceEvict = true
435+
}
436+
414437
resp, err := m.endpoints[pluginName].GetTopEvictionPods(context.Background(), &pluginapi.GetTopEvictionPodsRequest{
415438
ActivePods: pods,
416-
TopN: 1,
439+
TopN: topN,
417440
EvictionScope: threshold.EvictionScope,
418441
})
419-
420442
m.endpointLock.RUnlock()
421443
if err != nil {
422444
general.Errorf(" calling GetTopEvictionPods of plugin: %s failed with error: %v", pluginName, err)
@@ -430,12 +452,38 @@ func (m *EvictionManger) collectEvictionResult(pods []*v1.Pod) (*evictionRespCol
430452
continue
431453
}
432454

433-
collector.collectTopEvictionPods(dynamicConfig.DryRun, pluginName, threshold, resp)
455+
if forceEvict {
456+
collector.collectTopEvictionPods(dynamicConfig.DryRun, pluginName, threshold, resp)
457+
} else {
458+
collector.collectTopSoftEvictionPods(dynamicConfig.DryRun, pluginName, threshold, resp)
459+
}
460+
434461
}
435462

436463
return collector, errors.NewAggregate(errList)
437464
}
438465

466+
func (m *EvictionManger) doNotify(softEvictPods map[string]*rule.RuledEvictPod) error {
467+
errList := make([]error, 0)
468+
469+
for _, pod := range softEvictPods {
470+
if pod == nil || pod.EvictPod.Pod == nil {
471+
continue
472+
}
473+
474+
if _, ok := pod.EvictPod.Pod.Annotations[apiconsts.PodAnnotationSoftEvictNotificationKey]; !ok {
475+
continue
476+
}
477+
478+
err := m.podNotifier.NotifyPod(pod)
479+
if err != nil {
480+
errList = append(errList, err)
481+
}
482+
}
483+
484+
return errors.NewAggregate(errList)
485+
}
486+
439487
func (m *EvictionManger) doEvict(softEvictPods, forceEvictPods map[string]*rule.RuledEvictPod) error {
440488
softEvictPods = filterOutCandidatePodsWithForcePods(softEvictPods, forceEvictPods)
441489
bestSuitedCandidate := m.getEvictPodFromCandidates(softEvictPods)
@@ -594,6 +642,10 @@ func (m *EvictionManger) getEvictPodFromCandidates(candidateEvictPods map[string
594642
for _, rp := range candidateEvictPods {
595643
// only killing pods that pass candidate validation
596644
if rp != nil && rp.Pod != nil && m.killStrategy.CandidateValidate(rp) {
645+
// do NOT select soft evict pod with notification-enable as candidate
646+
if _, ok := rp.Pod.Annotations[apiconsts.PodAnnotationSoftEvictNotificationKey]; ok {
647+
continue
648+
}
597649
rpList = append(rpList, rp)
598650
}
599651
}

‎pkg/agent/evictionmanager/manager_test.go‎

Lines changed: 47 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ func makeConf() *config.Configuration {
107107
conf.PodKiller = consts.KillerNameEvictionKiller
108108
conf.GenericConfiguration.AuthConfiguration.AuthType = credential.AuthTypeInsecure
109109
conf.GenericConfiguration.AuthConfiguration.AccessControlType = authorization.AccessControlTypeInsecure
110+
conf.HostPathNotifierRootPath = "/opt/katalyst"
110111

111112
return conf
112113
}
@@ -231,12 +232,46 @@ func (p plugin2) GetEvictPods(_ context.Context, _ *pluginapi.GetEvictPodsReques
231232
return &pluginapi.GetEvictPodsResponse{EvictPods: []*pluginapi.EvictPod{}}, nil
232233
}
233234

235+
type plugin3 struct {
236+
pluginSkeleton
237+
}
238+
239+
func (p plugin3) ThresholdMet(_ context.Context) (*pluginapi.ThresholdMetResponse, error) {
240+
return &pluginapi.ThresholdMetResponse{
241+
MetType: pluginapi.ThresholdMetType_SOFT_MET,
242+
ThresholdValue: 0.8,
243+
ObservedValue: 0.9,
244+
ThresholdOperator: pluginapi.ThresholdOperator_GREATER_THAN,
245+
EvictionScope: "plugin3_scope",
246+
GracePeriodSeconds: -1,
247+
}, nil
248+
}
249+
250+
func (p plugin3) GetTopEvictionPods(_ context.Context, _ *pluginapi.GetTopEvictionPodsRequest) (*pluginapi.GetTopEvictionPodsResponse, error) {
251+
return &pluginapi.GetTopEvictionPodsResponse{TargetPods: []*v1.Pod{
252+
{
253+
ObjectMeta: metav1.ObjectMeta{
254+
Name: "pod-3",
255+
UID: "pod-3",
256+
},
257+
Status: v1.PodStatus{
258+
Phase: v1.PodRunning,
259+
},
260+
},
261+
}}, nil
262+
}
263+
264+
func (p plugin3) GetEvictPods(_ context.Context, _ *pluginapi.GetEvictPodsRequest) (*pluginapi.GetEvictPodsResponse, error) {
265+
return &pluginapi.GetEvictPodsResponse{EvictPods: []*pluginapi.EvictPod{}}, nil
266+
}
267+
234268
func makeEvictionManager(t *testing.T) *EvictionManger {
235269
mgr, err := NewEvictionManager(&client.GenericClientSet{}, nil, makeMetaServer(), metrics.DummyMetrics{}, makeConf())
236270
assert.NoError(t, err)
237271
mgr.endpoints = map[string]endpointpkg.Endpoint{
238272
"plugin1": &plugin1{},
239273
"plugin2": &plugin2{},
274+
"plugin3": &plugin3{},
240275
}
241276

242277
return mgr
@@ -257,6 +292,7 @@ func TestEvictionManger_collectEvictionResult(t *testing.T) {
257292
dryrun: []string{},
258293
wantSoftEvictPods: sets.String{
259294
"pod-1": sets.Empty{},
295+
"pod-3": sets.Empty{},
260296
"pod-5": sets.Empty{},
261297
},
262298
wantForceEvictPods: sets.String{
@@ -268,9 +304,11 @@ func TestEvictionManger_collectEvictionResult(t *testing.T) {
268304
},
269305
},
270306
{
271-
name: "dryrun plugin1",
272-
dryrun: []string{"plugin1"},
273-
wantSoftEvictPods: sets.String{},
307+
name: "dryrun plugin1",
308+
dryrun: []string{"plugin1"},
309+
wantSoftEvictPods: sets.String{
310+
"pod-3": sets.Empty{},
311+
},
274312
wantForceEvictPods: sets.String{
275313
"pod-3": sets.Empty{},
276314
},
@@ -283,6 +321,7 @@ func TestEvictionManger_collectEvictionResult(t *testing.T) {
283321
dryrun: []string{"plugin2"},
284322
wantSoftEvictPods: sets.String{
285323
"pod-1": sets.Empty{},
324+
"pod-3": sets.Empty{},
286325
"pod-5": sets.Empty{},
287326
},
288327
wantForceEvictPods: sets.String{
@@ -291,9 +330,11 @@ func TestEvictionManger_collectEvictionResult(t *testing.T) {
291330
wantConditions: sets.String{},
292331
},
293332
{
294-
name: "dryrun plugin1 & plugin2",
295-
dryrun: []string{"plugin1", "plugin2"},
296-
wantSoftEvictPods: sets.String{},
333+
name: "dryrun plugin1 & plugin2",
334+
dryrun: []string{"plugin1", "plugin2"},
335+
wantSoftEvictPods: sets.String{
336+
"pod-3": sets.Empty{},
337+
},
297338
wantForceEvictPods: sets.String{},
298339
wantConditions: sets.String{},
299340
},

0 commit comments

Comments
 (0)