From f0efa8677a42ee0df3bd7c592c324b150af4b846 Mon Sep 17 00:00:00 2001 From: davidLif Date: Sun, 16 Aug 2026 22:55:27 +0300 Subject: [PATCH 1/4] Take the pod termination grace period into account when determining if the pod is still in a terminating state, or is it /StuckInReleasing/ Signed-off-by: davidLif --- pkg/scheduler/api/pod_info/pod_info.go | 6 +++++- pkg/scheduler/api/pod_info/pod_info_test.go | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/pkg/scheduler/api/pod_info/pod_info.go b/pkg/scheduler/api/pod_info/pod_info.go index 7b779ebb9..1eec73b48 100644 --- a/pkg/scheduler/api/pod_info/pod_info.go +++ b/pkg/scheduler/api/pod_info/pod_info.go @@ -466,7 +466,11 @@ func getTaskStatus(pod *v1.Pod, bindRequest *bindrequest_info.BindRequestInfo, s switch pod.Status.Phase { case v1.PodRunning: if pod.DeletionTimestamp != nil { - if time.Since(pod.DeletionTimestamp.Time) > stuckInReleasingThreshold { + terminationGracePeriod := time.Duration(0) + if pod.Spec.TerminationGracePeriodSeconds != nil { + terminationGracePeriod = time.Duration(*pod.Spec.TerminationGracePeriodSeconds) * time.Second + } + if time.Since(pod.DeletionTimestamp.Time) > stuckInReleasingThreshold+terminationGracePeriod { return pod_status.StuckInReleasing } return pod_status.Releasing diff --git a/pkg/scheduler/api/pod_info/pod_info_test.go b/pkg/scheduler/api/pod_info/pod_info_test.go index fbc6349a5..02c8f2934 100644 --- a/pkg/scheduler/api/pod_info/pod_info_test.go +++ b/pkg/scheduler/api/pod_info/pod_info_test.go @@ -869,6 +869,16 @@ func TestGetTaskStatusStuckInReleasing(t *testing.T) { } } + runningPodWithTerminationGracePeriod := func(deletionTime *metav1.Time) *v1.Pod { + return &v1.Pod{ + Status: v1.PodStatus{Phase: v1.PodRunning}, + ObjectMeta: metav1.ObjectMeta{DeletionTimestamp: deletionTime}, + Spec: v1.PodSpec{ + TerminationGracePeriodSeconds: ptr.To(int64(300)), + }, + } + } + tests := []struct { name string pod *v1.Pod @@ -879,6 +889,8 @@ func TestGetTaskStatusStuckInReleasing(t *testing.T) { {name: "old deletion over default threshold", pod: runningPod(&fiveMinAgo), threshold: 2 * time.Minute, expected: pod_status.StuckInReleasing}, {name: "old deletion under custom larger threshold", pod: runningPod(&fiveMinAgo), threshold: 10 * time.Minute, expected: pod_status.Releasing}, {name: "running without deletion", pod: runningPod(nil), threshold: 2 * time.Minute, expected: pod_status.Running}, + {name: "old deletion over default threshold but under due to termination grace period", + pod: runningPodWithTerminationGracePeriod(&fiveMinAgo), threshold: 2 * time.Minute, expected: pod_status.Releasing}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From bbc7c4e86ea312d055fb97faba69a44fe74c321a Mon Sep 17 00:00:00 2001 From: davidLif Date: Sun, 16 Aug 2026 23:00:00 +0300 Subject: [PATCH 2/4] chore: add changelog fragment for StuckInReleasing grace period fix Signed-off-by: davidLif --- .changes/unreleased/fixed-20260816-225948.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changes/unreleased/fixed-20260816-225948.yaml diff --git a/.changes/unreleased/fixed-20260816-225948.yaml b/.changes/unreleased/fixed-20260816-225948.yaml new file mode 100644 index 000000000..fb7baaeda --- /dev/null +++ b/.changes/unreleased/fixed-20260816-225948.yaml @@ -0,0 +1,3 @@ +kind: Fixed +body: |- + Consider termination grace period when marking pods StuckInReleasing From e844134637f03fb583d7b5d526d909149d686a9f Mon Sep 17 00:00:00 2001 From: davidLif Date: Sun, 16 Aug 2026 23:02:20 +0300 Subject: [PATCH 3/4] Uodate changelog fragment Signed-off-by: davidLif --- .changes/unreleased/fixed-20260816-225948.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changes/unreleased/fixed-20260816-225948.yaml b/.changes/unreleased/fixed-20260816-225948.yaml index fb7baaeda..7f1871499 100644 --- a/.changes/unreleased/fixed-20260816-225948.yaml +++ b/.changes/unreleased/fixed-20260816-225948.yaml @@ -1,3 +1,3 @@ kind: Fixed body: |- - Consider termination grace period when marking pods StuckInReleasing + Add the pod termination grace period to the "stuckInReleasing" threshold when determining if the pod is still in a terminating state, or is it "StuckInReleasing" From 68f5fa583a89ecceddb0bf0ed00fda05a6acda4e Mon Sep 17 00:00:00 2001 From: davidLif Date: Mon, 17 Aug 2026 14:22:45 +0300 Subject: [PATCH 4/4] Make the stuck-in-releasing-threshold 5m instead of 2m Signed-off-by: davidLif --- cmd/scheduler/app/options/options.go | 2 +- pkg/common/constants/constants.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/scheduler/app/options/options.go b/cmd/scheduler/app/options/options.go index fccd3b56e..4efdc9ee9 100644 --- a/cmd/scheduler/app/options/options.go +++ b/cmd/scheduler/app/options/options.go @@ -126,7 +126,7 @@ func (s *ServerOption) AddFlags(fs *pflag.FlagSet) { fs.BoolVar(&s.AllowConsolidatingReclaim, "allow-consolidating-reclaim", true, "Do not count pipelined pods towards 'reclaimed' resources") fs.IntVar(&s.NumOfStatusRecordingWorkers, "num-of-status-recording-workers", defaultNumOfStatusRecordingWorkers, "specifies the max number of go routines spawned to update pod and podgroups conditions and events. Defaults to 5") fs.DurationVar(&s.GlobalDefaultStalenessGracePeriod, "default-staleness-grace-period", defaultStalenessGracePeriod, "Global default staleness grace period duration. Negative values means infinite. Defaults to 60s") - fs.DurationVar(&s.StuckInReleasingThreshold, "stuck-in-releasing-threshold", constants.DefaultStuckInReleasingThreshold, "Duration after a Running pod's deletionTimestamp before its resources are excluded from pipelining. Defaults to 2m.") + fs.DurationVar(&s.StuckInReleasingThreshold, "stuck-in-releasing-threshold", constants.DefaultStuckInReleasingThreshold, "Duration after a Running pod's deletionTimestamp before its resources are excluded from pipelining. Defaults to 5m.") fs.IntVar(&s.PluginServerPort, "plugin-server-port", 8081, "The port to bind for plugin server requests") fs.StringVar(&s.CPUWorkerNodeLabelKey, "cpu-worker-node-label-key", constants.DefaultCPUWorkerNodeLabelKey, "The label key for CPU worker nodes") fs.StringVar(&s.GPUWorkerNodeLabelKey, "gpu-worker-node-label-key", constants.DefaultGPUWorkerNodeLabelKey, "The label key for GPU worker nodes") diff --git a/pkg/common/constants/constants.go b/pkg/common/constants/constants.go index 012def875..287d770cc 100644 --- a/pkg/common/constants/constants.go +++ b/pkg/common/constants/constants.go @@ -25,7 +25,7 @@ const ( DefaultKAIConfigSingeltonInstanceName = "kai-config" DefaultNodePoolLabelKey = "kai.scheduler/node-pool" DefaultRuntimeClassName = "nvidia" - DefaultStuckInReleasingThreshold = 2 * time.Minute + DefaultStuckInReleasingThreshold = 5 * time.Minute ActionDefault = "default" ActionReclaim = "reclaim"