Skip to content

Commit baf5b06

Browse files
authored
perf(scheduler): Geometric sub scenario search (#1640)
Signed-off-by: itsomri <omric@nvidia.com>
1 parent 6069ab8 commit baf5b06

2 files changed

Lines changed: 28 additions & 10 deletions

File tree

CHANGELOG.md

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
66

77
## [Unreleased]
88

9+
### Fixed
10+
- Improved solver performance in some large reclaim scenarios [#1627](https://github.com/kai-scheduler/KAI-Scheduler/pull/1627) [itsomri](https://github.com/itsomri)
11+
912
## [v0.15.1] - 2026-06-01
1013

1114
### Added
1215

1316
### Changed
1417
- Updated Go toolchain and base build images to v1.26.3.
1518

16-
### Fixed
17-
1819
## [v0.15.0] - 2026-05-20
1920

2021
### Added

pkg/scheduler/actions/common/solvers/sub_scenario_emitter.go

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import (
1818
// potential victims, picked by node so the simulation only sees nodes that can plausibly
1919
// host a pending task. The emitter starts at the smallest top-K of victim-bearing nodes
2020
// whose cumulative post-eviction capacity (added to baseline) covers total pending demand,
21-
// and grows the set by one node each subsequent call.
21+
// and grows the set with exponentially increasing steps, clamped to the full candidate set.
2222
//
2323
// "Picking" a node selects every potential-victim *batch* that has any task on that node,
2424
// and includes the batch's tasks across all nodes it spans. This preserves gang semantics:
@@ -48,18 +48,19 @@ type subScenarioEmitter struct {
4848
nodeBatches map[string][]int // node -> indexes into batches, in insertion order
4949
batches []victimBatch
5050
nextK int
51+
dK int
5152
}
5253

5354
func newSubScenarioEmitter(
5455
session *framework.Session, base *scenario.ByNodeScenario,
55-
feasibleNodes map[string]*node_info.NodeInfo,
56+
baseNodes map[string]*node_info.NodeInfo,
5657
) *subScenarioEmitter {
5758
pendingDemand, minPendingTask := pendingTaskGpuStats(base)
5859
recordedFreed := recordedFreedByNode(base)
5960
batches, nodeBatches, nodeFirstSeenAt := buildVictimBatches(base)
6061
nodeCap := nodeCapacities(session, batches, nodeBatches, recordedFreed)
6162
candidates := sortViableCandidates(nodeBatches, nodeCap, nodeFirstSeenAt, minPendingTask)
62-
baseline := baselineCapacity(feasibleNodes, nodeBatches, recordedFreed)
63+
baseline := baselineCapacity(baseNodes, nodeBatches, recordedFreed)
6364

6465
remaining := pendingDemand - baseline
6566
if remaining < 0 {
@@ -74,25 +75,27 @@ func newSubScenarioEmitter(
7475
nodeBatches: nodeBatches,
7576
batches: batches,
7677
nextK: minK,
78+
dK: 1,
7779
}
7880
}
7981

8082
// next emits the next sub-scenario, or nil when no more sub-scenarios are worth trying.
81-
// Each call grows the picked-nodes prefix by one. Picking a node selects every victim
82-
// batch with a task on that node and includes the batch's full task set across all
83-
// nodes (gang-preserving).
83+
// Each call grows the picked-nodes prefix. Picking a node selects every victim batch
84+
// with a task on that node and includes the batch's full task set across all nodes
85+
// (gang-preserving).
8486
func (sse *subScenarioEmitter) next() *scenario.ByNodeScenario {
8587
if sse.nextK < 0 || sse.nextK > len(sse.sortedNodes) {
8688
return nil
8789
}
8890

91+
k := sse.nextK
8992
pickedBatches := map[int]bool{}
90-
for i := 0; i < sse.nextK; i++ {
93+
for i := 0; i < k; i++ {
9194
for _, bi := range sse.nodeBatches[sse.sortedNodes[i]] {
9295
pickedBatches[bi] = true
9396
}
9497
}
95-
sse.nextK++
98+
sse.advanceNextK(k)
9699

97100
sub := scenario.NewByNodeScenario(
98101
sse.session,
@@ -107,6 +110,20 @@ func (sse *subScenarioEmitter) next() *scenario.ByNodeScenario {
107110
return sub
108111
}
109112

113+
func (sse *subScenarioEmitter) advanceNextK(currentK int) {
114+
if currentK >= len(sse.sortedNodes) {
115+
sse.nextK = len(sse.sortedNodes) + 1
116+
return
117+
}
118+
119+
nextK := currentK + sse.dK
120+
if nextK > len(sse.sortedNodes) {
121+
nextK = len(sse.sortedNodes)
122+
}
123+
sse.nextK = nextK
124+
sse.dK *= 2
125+
}
126+
110127
// recordedFreedByNode returns the per-node sum of GPUs that will be freed when the
111128
// scenario's recorded victims (committed evictions from prior solver iterations) get
112129
// evicted. The recorded set is the same across every sub-scenario, so its

0 commit comments

Comments
 (0)