feat(rescheduling): gpuFragmentation eligible by default + env kill switch

pkg/scheduler/plugins/rescheduling/gpu_fragmentation.go

@@ -19,6 +19,7 @@ package rescheduling
 import (
 	"context"
 	"fmt"
+	"os"
 	"sort"
 	"time"
 
@@ -32,7 +33,7 @@ import (
 	"volcano.sh/volcano/pkg/scheduler/api"
 )
 
-// GpuFragmentationStrategy evicts at most one opted-in GPU pod per node pool
+// GpuFragmentationStrategy evicts at most one eligible GPU pod per node pool
 // whose departure empties its node of GPU work and which provably fits on a
 // fuller node in the same pool. The replacement is recreated by the pod's
 // controller and scheduled normally; binpack scoring steers it to the fuller
@@ -44,23 +45,29 @@ var DefaultGpuFragmentationConf = map[string]interface{}{
 	"dryRun":            true,
 	"gpuResource":       "nvidia.com/gpu",
 	"poolLabel":         "karpenter.sh/nodepool",
-	"eligibleLabel":     "exa.ai/repack-eligible",
+	"optOutLabel":       "exa.ai/repack-eligible",
 	"cooldownSeconds":   1800,
 	"maxVictims":        1,
 	"maxVictimPriority": -1,
 }
 
+// KillSwitchEnv disables the strategy entirely when set to "true" on the
+// scheduler process, e.g. `kubectl -n volcano set env deploy/<scheduler>
+// EXA_GPU_REPACK_DISABLED=true`.
+const KillSwitchEnv = "EXA_GPU_REPACK_DISABLED"
+
 const (
 	lastEvictionAnnotation  = "exa.ai/repack-last-eviction"
 	groupEvictionAnnotation = "exa.ai/repack-evictions"
 	doNotDisruptAnnotation  = "karpenter.sh/do-not-disrupt"
 )
 
 type gpuFragmentationConf struct {
-	DryRun          bool   `mapstructure:"dryRun"`
-	GpuResource     string `mapstructure:"gpuResource"`
-	PoolLabel       string `mapstructure:"poolLabel"`
-	EligibleLabel   string `mapstructure:"eligibleLabel"`
+	DryRun      bool   `mapstructure:"dryRun"`
+	GpuResource string `mapstructure:"gpuResource"`
+	PoolLabel   string `mapstructure:"poolLabel"`
+	// OptOutLabel excludes a pod from repacking when set to "false".
+	OptOutLabel     string `mapstructure:"optOutLabel"`
 	CooldownSeconds int    `mapstructure:"cooldownSeconds"`
 	MaxVictims      int    `mapstructure:"maxVictims"`
 	// MaxVictimPriority is the highest pod priority still movable. Pods
@@ -89,6 +96,10 @@ var victimsFnForGpuFragmentation = func(tasks []*api.TaskInfo) []*api.TaskInfo {
 	if Session == nil {
 		return nil
 	}
+	if os.Getenv(KillSwitchEnv) == "true" {
+		klog.V(2).Infof("gpuFragmentation: disabled via %s", KillSwitchEnv)
+		return nil
+	}
 	conf := newGpuFragmentationConf()
 	if params, ok := RegisteredStrategyConfigs[GpuFragmentationStrategy].(map[string]interface{}); ok {
 		conf.parse(params)
@@ -229,7 +240,7 @@ func planGpuFragmentationMoves(
 }
 
 // movableSoleGpuTask returns the node's single GPU-consuming task iff that
-// task is safe to move: it is running, opted in, not protected, at or below
+// task is safe to move: it is running, not opted out, not protected, at or below
 // the movable priority ceiling, owned by a controller that will recreate it,
 // and its PodGroup has exactly one member.
 func movableSoleGpuTask(
@@ -259,7 +270,7 @@ func movableSoleGpuTask(
 	if _, isRunning := running[sole.Pod.UID]; !isRunning {
 		return nil
 	}
-	if sole.Pod.Labels[conf.EligibleLabel] != "true" {
+	if sole.Pod.Labels[conf.OptOutLabel] == "false" {
 		return nil
 	}
 	if sole.Pod.Annotations[doNotDisruptAnnotation] == "true" {

pkg/scheduler/plugins/rescheduling/gpu_fragmentation_test.go

@@ -115,6 +115,10 @@ func eligible() map[string]string {
 	return map[string]string{"exa.ai/repack-eligible": "true"}
 }
 
+func optedOut() map[string]string {
+	return map[string]string{"exa.ai/repack-eligible": "false"}
+}
+
 func TestPlanSelectsSoleEligiblePodWithFullerDestination(t *testing.T) {
 	f := newFixture(t)
 	source := f.addNode(gpuNode("source", 8, nil))
@@ -131,15 +135,28 @@ func TestPlanSelectsSoleEligiblePodWithFullerDestination(t *testing.T) {
 	}
 }
 
-func TestPlanSkipsWithoutOptIn(t *testing.T) {
+func TestPlanSelectsUnlabeledPodByDefault(t *testing.T) {
 	f := newFixture(t)
 	source := f.addNode(gpuNode("source", 8, nil))
 	dest := f.addNode(gpuNode("dest", 8, nil))
 	f.placePod(t, source, gpuPod("victim", "source", 1, nil, nil, true), 1, "")
 	f.placePod(t, dest, gpuPod("resident", "dest", 3, nil, nil, true), 1, "")
 
+	plans := f.plan(newGpuFragmentationConf(), nil)
+	if len(plans) != 1 || plans[0].victim.Name != "victim" {
+		t.Fatalf("expected unlabeled pod to be eligible by default, got %+v", plans)
+	}
+}
+
+func TestPlanSkipsOptedOutPod(t *testing.T) {
+	f := newFixture(t)
+	source := f.addNode(gpuNode("source", 8, nil))
+	dest := f.addNode(gpuNode("dest", 8, nil))
+	f.placePod(t, source, gpuPod("victim", "source", 1, optedOut(), nil, true), 1, "")
+	f.placePod(t, dest, gpuPod("resident", "dest", 3, nil, nil, true), 1, "")
+
 	if plans := f.plan(newGpuFragmentationConf(), nil); len(plans) != 0 {
-		t.Fatalf("expected no plans without opt-in, got %+v", plans)
+		t.Fatalf("expected no plans for opted-out pod, got %+v", plans)
 	}
 }
 
@@ -248,8 +265,8 @@ func TestPlanRequiresStrictlyFullerDestinationWithRoom(t *testing.T) {
 	emptier := f.addNode(gpuNode("emptier", 8, nil))
 	full := f.addNode(gpuNode("full", 8, nil))
 	f.placePod(t, source, gpuPod("victim", "source", 3, eligible(), nil, true), 1, "")
-	f.placePod(t, emptier, gpuPod("small", "emptier", 1, nil, nil, true), 1, "")
-	f.placePod(t, full, gpuPod("big", "full", 7, nil, nil, true), 1, "")
+	f.placePod(t, emptier, gpuPod("small", "emptier", 1, optedOut(), nil, true), 1, "")
+	f.placePod(t, full, gpuPod("big", "full", 7, optedOut(), nil, true), 1, "")
 
 	// emptier is less full than source; full has only 1 free GPU for a 3-GPU victim.
 	if plans := f.plan(newGpuFragmentationConf(), nil); len(plans) != 0 {