feat(scheduler): add gpujoborder plugin for configurable JobOrderFn tiebreak

.changes/unreleased/added-20260729-142722.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: |-
+  Add gpujoborder plugin for configurable GPU-based JobOrderFn tiebreak on priority ties

pkg/scheduler/actions/utils/job_order_by_queue.go

@@ -255,7 +255,7 @@ func (jo *JobsOrderByQueues) createLeafNode(queue *queue_info.QueueInfo) *queueN
 		queue: queue,
 		children: scheduler_util.NewPriorityQueue(func(l, r interface{}) bool {
 			if jo.options.VictimQueue {
-				return !jo.ssn.JobOrderFn(l, r)
+				return jo.ssn.VictimOrderFn(l, r)
 			}
 			return jo.ssn.JobOrderFn(l, r)
 		}, jo.options.MaxJobsQueueDepth),

pkg/scheduler/api/podgroup_info/job_info.go

@@ -93,6 +93,7 @@ type PodGroupInfo struct {
 	tasksToAllocateInitResourceVector resource_info.ResourceVector
 	PodStatusIndex                    map[pod_status.PodStatus]pod_info.PodsMap
 	activeAllocatedCount              *int
+	aliveTasksRequestedGPUs           *float64
 }
 
 func NewPodGroupInfo(uid common_info.PodGroupID, tasks ...*pod_info.PodInfo) *PodGroupInfo {
@@ -365,6 +366,7 @@ func (pgi *PodGroupInfo) invalidateTasksCache() {
 	pgi.allPodsMap = nil
 	pgi.tasksToAllocate = nil
 	pgi.tasksToAllocateInitResourceVector = nil
+	pgi.aliveTasksRequestedGPUs = nil
 }
 
 func (pgi *PodGroupInfo) GetActiveAllocatedTasksCount() int {
@@ -456,14 +458,16 @@ func (pgi *PodGroupInfo) GetNumGatedTasks() int {
 }
 
 func (pgi *PodGroupInfo) GetAliveTasksRequestedGPUs() float64 {
-	tasksTotalRequestedGPUs := float64(0)
-	for _, task := range pgi.GetAllPodsMap() {
-		if pod_status.IsAliveStatus(task.Status) {
-			tasksTotalRequestedGPUs += task.ResReqVector.Get(resource_info.GPUIndex)
+	if pgi.aliveTasksRequestedGPUs == nil {
+		tasksTotalRequestedGPUs := float64(0)
+		for _, task := range pgi.GetAllPodsMap() {
+			if pod_status.IsAliveStatus(task.Status) {
+				tasksTotalRequestedGPUs += task.ResReqVector.Get(resource_info.GPUIndex)
+			}
 		}
+		pgi.aliveTasksRequestedGPUs = ptr.To(tasksTotalRequestedGPUs)
 	}
-
-	return tasksTotalRequestedGPUs
+	return *pgi.aliveTasksRequestedGPUs
 }
 
 func (pgi *PodGroupInfo) GetTasksActiveAllocatedReqResourceVector() resource_info.ResourceVector {

pkg/scheduler/framework/session.go

@@ -76,6 +76,7 @@ type Session struct {
 	NodePreOrderFns                       []api.NodePreOrderFn
 	NodeOrderFns                          []api.NodeOrderFn
 	JobOrderFns                           []common_info.CompareFn
+	VictimOrderFns                        []common_info.CompareFn
 	SubGroupOrderFns                      []common_info.CompareFn
 	TaskOrderFns                          []common_info.CompareFn
 	QueueOrderFns                         []api.CompareQueueFn
@@ -434,6 +435,7 @@ func (ssn *Session) clear() {
 	ssn.NodePreOrderFns = nil
 	ssn.NodeOrderFns = nil
 	ssn.JobOrderFns = nil
+	ssn.VictimOrderFns = nil
 	ssn.SubGroupOrderFns = nil
 	ssn.TaskOrderFns = nil
 	ssn.QueueOrderFns = nil

pkg/scheduler/framework/session_plugins.go

@@ -68,6 +68,15 @@ func (ssn *Session) AddJobOrderFn(jof common_info.CompareFn) {
 	ssn.JobOrderFns = append(ssn.JobOrderFns, jof)
 }
 
+// AddVictimOrderFn registers a comparator that applies ONLY when ordering
+// candidates for eviction (the victim queue), not for regular pending-job
+// allocation ordering. Unlike JobOrderFn, no external inversion is applied
+// to this comparator's result: a negative return means "l is the BETTER
+// victim (should be evicted first)", directly.
+func (ssn *Session) AddVictimOrderFn(vof common_info.CompareFn) {
+	ssn.VictimOrderFns = append(ssn.VictimOrderFns, vof)
+}
+
 func (ssn *Session) AddTaskOrderFn(tof common_info.CompareFn) {
 	ssn.TaskOrderFns = append(ssn.TaskOrderFns, tof)
 }
@@ -268,21 +277,61 @@ func (ssn *Session) QueueAllocatedResources(queue *queue_info.QueueInfo) *resour
 	return nil
 }
 
+// jobOrderCreationFallback is the shared tiebreak used when no registered
+// comparator has an opinion: CreationTimestamp first, then UID. Extracted
+// so both JobOrderFn and VictimOrderFn can share it without duplicating
+// the comparison logic.
+func jobOrderCreationFallback(l, r interface{}) bool {
+	lv := l.(*podgroup_info.PodGroupInfo)
+	rv := r.(*podgroup_info.PodGroupInfo)
+	if lv.CreationTimestamp.Equal(&rv.CreationTimestamp) {
+		return lv.UID < rv.UID
+	}
+	return lv.CreationTimestamp.Before(&rv.CreationTimestamp)
+}
+
 func (ssn *Session) JobOrderFn(l, r interface{}) bool {
 	for _, jof := range ssn.JobOrderFns {
 		if j := jof(l, r); j != 0 {
 			return j < 0
 		}
 	}
-
-	// If no job order funcs, order job by CreationTimestamp first, then by UID.
-	lv := l.(*podgroup_info.PodGroupInfo)
-	rv := r.(*podgroup_info.PodGroupInfo)
-	if lv.CreationTimestamp.Equal(&rv.CreationTimestamp) {
-		return lv.UID < rv.UID
-	} else {
-		return lv.CreationTimestamp.Before(&rv.CreationTimestamp)
+	return jobOrderCreationFallback(l, r)
+}
+
+// VictimOrderFn composes registered victim-specific comparators, but only
+// as a tiebreak AFTER the existing JobOrderFns chain (priority.go,
+// elastic.go, etc.) has had its say -- fixed per @gshaibi's review: the
+// original version checked VictimOrderFns first unconditionally, which
+// let a raw resource-size comparator (e.g. gpujoborder) outrank
+// elastic's deliberate at-min/above-min protection. Since JobOrderFn
+// itself always resolves via its own CreationTimestamp/UID fallback, the
+// raw JobOrderFns slice is iterated directly here (not the composed
+// method) so a genuine "no opinion" state (all registered JobOrderFns
+// return 0) can be detected before falling through to victim-specific
+// comparators.
+//
+// A negative result from a JobOrderFn means "l ordered first for
+// allocation" -- inverted here (j > 0) since a job LESS preferred for
+// allocation should be MORE preferred as a victim. VictimOrderFns use
+// direct (non-inverted) semantics: negative means "l is the better
+// victim". If neither chain has an opinion, falls back to the inverted
+// creation-timestamp order, preserving old behavior exactly.
+//
+// Path without any VictimOrderFns registered is byte-identical to the
+// original !JobOrderFn(l, r) behavior.
+func (ssn *Session) VictimOrderFn(l, r interface{}) bool {
+	for _, jof := range ssn.JobOrderFns {
+		if j := jof(l, r); j != 0 {
+			return j > 0
+		}
+	}
+	for _, vof := range ssn.VictimOrderFns {
+		if v := vof(l, r); v != 0 {
+			return v < 0
+		}
 	}
+	return !jobOrderCreationFallback(l, r)
 }
 
 func (ssn *Session) TaskOrderFn(l, r interface{}) bool {

pkg/scheduler/plugins/factory.go

@@ -23,6 +23,7 @@ import (
 	"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/framework"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/plugins/dynamicresources"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/plugins/elastic"
+	"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/plugins/gpujoborder"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/plugins/gpupack"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/plugins/gpusharingorder"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/plugins/gpuspread"
@@ -51,6 +52,7 @@ func InitDefaultPlugins() {
 	// Plugins for PodGroupInfos
 	framework.RegisterPluginBuilder("predicates", predicates.New)
 	framework.RegisterPluginBuilder("priority", priority.New)
+	framework.RegisterPluginBuilder("gpujoborder", gpujoborder.New)
 	framework.RegisterPluginBuilder("nodeplacement", nodeplacement.New)
 	framework.RegisterPluginBuilder("nominatednode", nominatednode.New)
 	framework.RegisterPluginBuilder("numa", numa.New)

pkg/scheduler/plugins/gpujoborder/gpujoborder.go

@@ -0,0 +1,91 @@
+// Copyright 2026 NVIDIA CORPORATION
+// SPDX-License-Identifier: Apache-2.0
+
+// Package gpujoborder implements a GPU-count-based victim-selection
+// tiebreak, used only when two jobs of equal priority are otherwise tied
+// (including by any other registered JobOrderFn, such as elastic's
+// at-min/above-min protection) and the scheduler must choose which one
+// to evict. It has no effect on pending-job allocation ordering, and no
+// effect whenever an existing JobOrderFn-registered plugin already has an
+// opinion on the comparison.
+package gpujoborder
+
+import (
+	"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/api/podgroup_info"
+	"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/framework"
+	"github.com/kai-scheduler/KAI-scheduler/pkg/scheduler/log"
+)
+
+const (
+	// ModeEvictLargerFirst prefers evicting the job requesting MORE GPUs
+	// when a tie must be broken. Named for the eviction outcome directly
+	// (not "prefer-larger", which reads ambiguously as either "prefer to
+	// evict the larger job" or "prefer to keep/favor the larger job") --
+	// per review discussion, this is deliberately unambiguous since the
+	// config string is hard to rename once real users depend on it.
+	ModeEvictLargerFirst = "evict-larger-first"
+	// ModeEvictSmallerFirst prefers evicting the job requesting FEWER
+	// GPUs when a tie must be broken.
+	ModeEvictSmallerFirst = "evict-smaller-first"
+)
+
+type gpuJobOrderPlugin struct {
+	mode string
+}
+
+func New(arguments framework.PluginArguments) framework.Plugin {
+	mode := arguments.GetString("mode", ModeEvictLargerFirst)
+	if mode != ModeEvictLargerFirst && mode != ModeEvictSmallerFirst {
+		log.InfraLogger.Warningf("gpujoborder: unrecognized mode %q, defaulting to %s", mode, ModeEvictLargerFirst)
+		mode = ModeEvictLargerFirst
+	}
+	return &gpuJobOrderPlugin{mode: mode}
+}
+
+func (rp *gpuJobOrderPlugin) Name() string {
+	return "gpujoborder"
+}
+
+// OnSessionOpen registers this plugin's comparator via AddVictimOrderFn,
+// NOT AddJobOrderFn. This plugin is scoped to victim/eviction selection
+// only, and only applies as a tiebreak after every registered JobOrderFn
+// (priority.go, elastic.go, etc.) has already had a chance to decide --
+// see Session.VictimOrderFn for the real composition order.
+func (rp *gpuJobOrderPlugin) OnSessionOpen(ssn *framework.Session) {
+	ssn.AddVictimOrderFn(rp.VictimOrderFn)
+}
+
+// VictimOrderFn returns -1 when l is the BETTER victim (should be evicted
+// first), matching VictimOrderFn's direct (non-inverted) contract -- no
+// external sign flip is applied or needed here.
+func (rp *gpuJobOrderPlugin) VictimOrderFn(l, r interface{}) int {
+	lv := l.(*podgroup_info.PodGroupInfo)
+	rv := r.(*podgroup_info.PodGroupInfo)
+
+	if lv.Priority != rv.Priority {
+		return 0
+	}
+
+	lGPU := lv.GetAliveTasksRequestedGPUs()
+	rGPU := rv.GetAliveTasksRequestedGPUs()
+
+	switch rp.mode {
+	case ModeEvictSmallerFirst:
+		if lGPU < rGPU {
+			return -1
+		}
+		if lGPU > rGPU {
+			return 1
+		}
+	default: // ModeEvictLargerFirst
+		if lGPU > rGPU {
+			return -1
+		}
+		if lGPU < rGPU {
+			return 1
+		}
+	}
+	return 0
+}
+
+func (rp *gpuJobOrderPlugin) OnSessionClose(_ *framework.Session) {}