fix(scheduler): Take termination grace period into account for "StuckInReleasing"

.changes/unreleased/fixed-20260816-225948.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: |-
+  Add the pod termination grace period to the "stuckInReleasing" threshold when determining if the pod is still in a terminating state, or is it "StuckInReleasing"

cmd/scheduler/app/options/options.go

@@ -126,7 +126,7 @@ func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 	fs.BoolVar(&s.AllowConsolidatingReclaim, "allow-consolidating-reclaim", true, "Do not count pipelined pods towards 'reclaimed' resources")
 	fs.IntVar(&s.NumOfStatusRecordingWorkers, "num-of-status-recording-workers", defaultNumOfStatusRecordingWorkers, "specifies the max number of go routines spawned to update pod and podgroups conditions and events. Defaults to 5")
 	fs.DurationVar(&s.GlobalDefaultStalenessGracePeriod, "default-staleness-grace-period", defaultStalenessGracePeriod, "Global default staleness grace period duration. Negative values means infinite. Defaults to 60s")
-	fs.DurationVar(&s.StuckInReleasingThreshold, "stuck-in-releasing-threshold", constants.DefaultStuckInReleasingThreshold, "Duration after a Running pod's deletionTimestamp before its resources are excluded from pipelining. Defaults to 2m.")
+	fs.DurationVar(&s.StuckInReleasingThreshold, "stuck-in-releasing-threshold", constants.DefaultStuckInReleasingThreshold, "Duration after a Running pod's deletionTimestamp before its resources are excluded from pipelining. Defaults to 5m.")
 	fs.IntVar(&s.PluginServerPort, "plugin-server-port", 8081, "The port to bind for plugin server requests")
 	fs.StringVar(&s.CPUWorkerNodeLabelKey, "cpu-worker-node-label-key", constants.DefaultCPUWorkerNodeLabelKey, "The label key for CPU worker nodes")
 	fs.StringVar(&s.GPUWorkerNodeLabelKey, "gpu-worker-node-label-key", constants.DefaultGPUWorkerNodeLabelKey, "The label key for GPU worker nodes")

pkg/common/constants/constants.go

@@ -25,7 +25,7 @@ const (
 	DefaultKAIConfigSingeltonInstanceName = "kai-config"
 	DefaultNodePoolLabelKey               = "kai.scheduler/node-pool"
 	DefaultRuntimeClassName               = "nvidia"
-	DefaultStuckInReleasingThreshold      = 2 * time.Minute
+	DefaultStuckInReleasingThreshold      = 5 * time.Minute
 
 	ActionDefault       = "default"
 	ActionReclaim       = "reclaim"

pkg/scheduler/api/pod_info/pod_info.go

@@ -466,7 +466,11 @@ func getTaskStatus(pod *v1.Pod, bindRequest *bindrequest_info.BindRequestInfo, s
 	switch pod.Status.Phase {
 	case v1.PodRunning:
 		if pod.DeletionTimestamp != nil {
-			if time.Since(pod.DeletionTimestamp.Time) > stuckInReleasingThreshold {
+			terminationGracePeriod := time.Duration(0)
+			if pod.Spec.TerminationGracePeriodSeconds != nil {
+				terminationGracePeriod = time.Duration(*pod.Spec.TerminationGracePeriodSeconds) * time.Second
+			}
+			if time.Since(pod.DeletionTimestamp.Time) > stuckInReleasingThreshold+terminationGracePeriod {
 				return pod_status.StuckInReleasing
 			}
 			return pod_status.Releasing

pkg/scheduler/api/pod_info/pod_info_test.go

@@ -869,6 +869,16 @@ func TestGetTaskStatusStuckInReleasing(t *testing.T) {
 		}
 	}
 
+	runningPodWithTerminationGracePeriod := func(deletionTime *metav1.Time) *v1.Pod {
+		return &v1.Pod{
+			Status:     v1.PodStatus{Phase: v1.PodRunning},
+			ObjectMeta: metav1.ObjectMeta{DeletionTimestamp: deletionTime},
+			Spec: v1.PodSpec{
+				TerminationGracePeriodSeconds: ptr.To(int64(300)),
+			},
+		}
+	}
+
 	tests := []struct {
 		name      string
 		pod       *v1.Pod
@@ -879,6 +889,8 @@ func TestGetTaskStatusStuckInReleasing(t *testing.T) {
 		{name: "old deletion over default threshold", pod: runningPod(&fiveMinAgo), threshold: 2 * time.Minute, expected: pod_status.StuckInReleasing},
 		{name: "old deletion under custom larger threshold", pod: runningPod(&fiveMinAgo), threshold: 10 * time.Minute, expected: pod_status.Releasing},
 		{name: "running without deletion", pod: runningPod(nil), threshold: 2 * time.Minute, expected: pod_status.Running},
+		{name: "old deletion over default threshold but under due to termination grace period",
+			pod: runningPodWithTerminationGracePeriod(&fiveMinAgo), threshold: 2 * time.Minute, expected: pod_status.Releasing},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {