Skip to content

Commit d497773

Browse files
Backport of Scheduler: Add configurable NodeLimitForFeasibilityChecks into release/2.0.x (#27884)
Co-authored-by: Luke Palmer <luke@lukepalmer.net>
1 parent 508703e commit d497773

14 files changed

Lines changed: 164 additions & 35 deletions

.changelog/27650.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
```release-note:improvement
2+
scheduler: Add a configuration field for the number of nodes that the scheduler considers when spread or affinity is in use. This can improve scheduler performance for some cluster shapes.
3+
```

api/operator.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,13 @@ type SchedulerConfiguration struct {
185185
// until the configuration is updated and written to the Nomad servers.
186186
PauseEvalBroker bool
187187

188+
// NodeLimitForFeasibilityChecks limits the number of feasible nodes to consider when
189+
// scheduling a job that specifies spread and/or affinity. Defaults to 100 nodes if
190+
// unset. Lower numbers result in better scheduler performance and more randomization
191+
// of jobs across nodes. Higher numbers result in more deterministic application of
192+
// spread and/or affinity.
193+
NodeLimitForFeasibilityChecks uint
194+
188195
// CreateIndex/ModifyIndex store the create/modify indexes of this configuration.
189196
CreateIndex uint64
190197
ModifyIndex uint64

api/operator_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ func TestOperator_SchedulerSetConfiguration(t *testing.T) {
6464
MemoryOversubscriptionEnabled: true,
6565
RejectJobRegistration: true,
6666
PauseEvalBroker: true,
67+
NodeLimitForFeasibilityChecks: 123,
6768
}
6869

6970
schedulerConfigUpdateResp, _, err := c.Operator().SchedulerSetConfiguration(&newSchedulerConfig, nil)
@@ -78,6 +79,7 @@ func TestOperator_SchedulerSetConfiguration(t *testing.T) {
7879
must.True(t, schedulerConfig.SchedulerConfig.RejectJobRegistration)
7980
must.True(t, schedulerConfig.SchedulerConfig.MemoryOversubscriptionEnabled)
8081
must.Eq(t, schedulerConfig.SchedulerConfig.PreemptionConfig, newSchedulerConfig.PreemptionConfig)
82+
must.Eq(t, schedulerConfig.SchedulerConfig.NodeLimitForFeasibilityChecks, newSchedulerConfig.NodeLimitForFeasibilityChecks)
8183
}
8284

8385
func TestOperator_AutopilotState(t *testing.T) {

command/agent/operator_endpoint.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,7 @@ func (s *HTTPServer) schedulerUpdateConfig(resp http.ResponseWriter, req *http.R
311311
MemoryOversubscriptionEnabled: conf.MemoryOversubscriptionEnabled,
312312
RejectJobRegistration: conf.RejectJobRegistration,
313313
PauseEvalBroker: conf.PauseEvalBroker,
314+
NodeLimitForFeasibilityChecks: conf.NodeLimitForFeasibilityChecks,
314315
PreemptionConfig: structs.PreemptionConfig{
315316
SystemSchedulerEnabled: conf.PreemptionConfig.SystemSchedulerEnabled,
316317
SysBatchSchedulerEnabled: conf.PreemptionConfig.SysBatchSchedulerEnabled,

command/agent/operator_endpoint_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -496,6 +496,7 @@ func TestOperator_SchedulerSetConfiguration(t *testing.T) {
496496
{
497497
"MemoryOversubscriptionEnabled": true,
498498
"PauseEvalBroker": true,
499+
"NodeLimitForFeasibilityChecks": 123,
499500
"PreemptionConfig": {
500501
"SystemSchedulerEnabled": true,
501502
"ServiceSchedulerEnabled": true
@@ -525,6 +526,7 @@ func TestOperator_SchedulerSetConfiguration(t *testing.T) {
525526
require.True(t, reply.SchedulerConfig.PreemptionConfig.ServiceSchedulerEnabled)
526527
require.True(t, reply.SchedulerConfig.MemoryOversubscriptionEnabled)
527528
require.True(t, reply.SchedulerConfig.PauseEvalBroker)
529+
require.Equal(t, reply.SchedulerConfig.GetNodeLimitForFeasibilityChecks(), uint(123))
528530
})
529531
}
530532

command/operator_scheduler_get_config.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ func (o *OperatorSchedulerGetConfig) Run(args []string) int {
8686
fmt.Sprintf("Preemption Service Scheduler|%v", schedConfig.PreemptionConfig.ServiceSchedulerEnabled),
8787
fmt.Sprintf("Preemption Batch Scheduler|%v", schedConfig.PreemptionConfig.BatchSchedulerEnabled),
8888
fmt.Sprintf("Preemption SysBatch Scheduler|%v", schedConfig.PreemptionConfig.SysBatchSchedulerEnabled),
89+
fmt.Sprintf("Node Limit For Feasibility Checks|%v", schedConfig.NodeLimitForFeasibilityChecks),
8990
fmt.Sprintf("Modify Index|%v", resp.SchedulerConfig.ModifyIndex),
9091
}))
9192
return 0

command/operator_scheduler_get_config_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,8 @@ func TestOperatorSchedulerGetConfig_Run(t *testing.T) {
2525
// Run the command, so we get the default output and test this.
2626
must.Zero(t, c.Run([]string{"-address=" + addr}))
2727
s := ui.OutputWriter.String()
28-
must.StrContains(t, s, "Scheduler Algorithm = binpack")
29-
must.StrContains(t, s, "Preemption SysBatch Scheduler = false")
28+
must.StrContains(t, s, "Scheduler Algorithm = binpack")
29+
must.StrContains(t, s, "Preemption SysBatch Scheduler = false")
3030
ui.ErrorWriter.Reset()
3131
ui.OutputWriter.Reset()
3232

command/operator_scheduler_set_config.go

Lines changed: 27 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -22,15 +22,16 @@ type OperatorSchedulerSetConfig struct {
2222
// The scheduler configuration flags allow us to tell whether the user set
2323
// a value or not. This means we can safely merge the current configuration
2424
// with user supplied, selective updates.
25-
checkIndex string
26-
schedulerAlgorithm string
27-
memoryOversubscription flagHelper.BoolValue
28-
rejectJobRegistration flagHelper.BoolValue
29-
pauseEvalBroker flagHelper.BoolValue
30-
preemptBatchScheduler flagHelper.BoolValue
31-
preemptServiceScheduler flagHelper.BoolValue
32-
preemptSysBatchScheduler flagHelper.BoolValue
33-
preemptSystemScheduler flagHelper.BoolValue
25+
checkIndex string
26+
schedulerAlgorithm string
27+
memoryOversubscription flagHelper.BoolValue
28+
rejectJobRegistration flagHelper.BoolValue
29+
pauseEvalBroker flagHelper.BoolValue
30+
preemptBatchScheduler flagHelper.BoolValue
31+
preemptServiceScheduler flagHelper.BoolValue
32+
preemptSysBatchScheduler flagHelper.BoolValue
33+
preemptSystemScheduler flagHelper.BoolValue
34+
nodeLimitForFeasibilityChecks flagHelper.UintValue
3435
}
3536

3637
func (o *OperatorSchedulerSetConfig) AutocompleteFlags() complete.Flags {
@@ -41,13 +42,14 @@ func (o *OperatorSchedulerSetConfig) AutocompleteFlags() complete.Flags {
4142
string(api.SchedulerAlgorithmBinpack),
4243
string(api.SchedulerAlgorithmSpread),
4344
),
44-
"-memory-oversubscription": complete.PredictSet("true", "false"),
45-
"-reject-job-registration": complete.PredictSet("true", "false"),
46-
"-pause-eval-broker": complete.PredictSet("true", "false"),
47-
"-preempt-batch-scheduler": complete.PredictSet("true", "false"),
48-
"-preempt-service-scheduler": complete.PredictSet("true", "false"),
49-
"-preempt-sysbatch-scheduler": complete.PredictSet("true", "false"),
50-
"-preempt-system-scheduler": complete.PredictSet("true", "false"),
45+
"-memory-oversubscription": complete.PredictSet("true", "false"),
46+
"-reject-job-registration": complete.PredictSet("true", "false"),
47+
"-pause-eval-broker": complete.PredictSet("true", "false"),
48+
"-preempt-batch-scheduler": complete.PredictSet("true", "false"),
49+
"-preempt-service-scheduler": complete.PredictSet("true", "false"),
50+
"-preempt-sysbatch-scheduler": complete.PredictSet("true", "false"),
51+
"-preempt-system-scheduler": complete.PredictSet("true", "false"),
52+
"-node-limit-for-feasibility-checks": complete.PredictAnything,
5153
},
5254
)
5355
}
@@ -72,6 +74,7 @@ func (o *OperatorSchedulerSetConfig) Run(args []string) int {
7274
flags.Var(&o.preemptServiceScheduler, "preempt-service-scheduler", "")
7375
flags.Var(&o.preemptSysBatchScheduler, "preempt-sysbatch-scheduler", "")
7476
flags.Var(&o.preemptSystemScheduler, "preempt-system-scheduler", "")
77+
flags.Var(&o.nodeLimitForFeasibilityChecks, "node-limit-for-feasibility-checks", "")
7578

7679
if err := flags.Parse(args); err != nil {
7780
return 1
@@ -134,6 +137,7 @@ func (o *OperatorSchedulerSetConfig) Run(args []string) int {
134137
o.preemptServiceScheduler.Merge(&schedulerConfig.PreemptionConfig.ServiceSchedulerEnabled)
135138
o.preemptSysBatchScheduler.Merge(&schedulerConfig.PreemptionConfig.SysBatchSchedulerEnabled)
136139
o.preemptSystemScheduler.Merge(&schedulerConfig.PreemptionConfig.SystemSchedulerEnabled)
140+
o.nodeLimitForFeasibilityChecks.Merge(&schedulerConfig.NodeLimitForFeasibilityChecks)
137141

138142
// Check-and-set the new configuration.
139143
result, _, err := client.Operator().SchedulerCASConfiguration(schedulerConfig, nil)
@@ -208,6 +212,13 @@ Scheduler Set Config Options:
208212
-preempt-system-scheduler=[true|false]
209213
Specifies whether preemption for system jobs is enabled. Note that if this
210214
is set to true, then system jobs can preempt any other jobs.
215+
216+
-node-limit-for-feasibility-checks=<count>
217+
Limits the number of feasible nodes to consider when scheduling a job that
218+
specifies spread and/or affinity. Defaults to 100 nodes if unset. Lower
219+
numbers result in better scheduler performance and more randomization of jobs
220+
across nodes. Higher numbers result in more deterministic application of
221+
feasibility checks.
211222
`
212223
return strings.TrimSpace(helpText)
213224
}

command/operator_scheduler_set_config_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ func TestOperatorSchedulerSetConfig_Run(t *testing.T) {
5151
"-preempt-service-scheduler=true",
5252
"-preempt-sysbatch-scheduler=true",
5353
"-preempt-system-scheduler=false",
54+
"-node-limit-for-feasibility-checks=200",
5455
}
5556
must.Zero(t, c.Run(modifyingArgs))
5657
s := ui.OutputWriter.String()
@@ -69,6 +70,7 @@ func TestOperatorSchedulerSetConfig_Run(t *testing.T) {
6970
MemoryOversubscriptionEnabled: true,
7071
RejectJobRegistration: true,
7172
PauseEvalBroker: true,
73+
NodeLimitForFeasibilityChecks: 200,
7274
}, modifiedConfig.SchedulerConfig)
7375

7476
ui.ErrorWriter.Reset()
@@ -108,4 +110,5 @@ func schedulerConfigEquals(t *testing.T, expected, actual *api.SchedulerConfigur
108110
must.Eq(t, expected.MemoryOversubscriptionEnabled, actual.MemoryOversubscriptionEnabled)
109111
must.Eq(t, expected.PauseEvalBroker, actual.PauseEvalBroker)
110112
must.Eq(t, expected.PreemptionConfig, actual.PreemptionConfig)
113+
must.Eq(t, expected.NodeLimitForFeasibilityChecks, actual.NodeLimitForFeasibilityChecks)
111114
}

nomad/operator_endpoint_test.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -551,13 +551,14 @@ func TestOperator_SchedulerSetConfiguration(t *testing.T) {
551551
rpcCodec := rpcClient(t, s1)
552552
testutil.WaitForLeader(t, s1.RPC)
553553

554-
// Disable preemption and pause the eval broker.
554+
// Disable preemption, pause the eval broker, and set a custom node limit.
555555
arg := structs.SchedulerSetConfigRequest{
556556
Config: structs.SchedulerConfiguration{
557557
PreemptionConfig: structs.PreemptionConfig{
558558
SystemSchedulerEnabled: false,
559559
},
560-
PauseEvalBroker: true,
560+
PauseEvalBroker: true,
561+
NodeLimitForFeasibilityChecks: 200,
561562
},
562563
}
563564
arg.Region = s1.config.Region
@@ -581,6 +582,7 @@ func TestOperator_SchedulerSetConfiguration(t *testing.T) {
581582
require.NotZero(t, reply.Index)
582583
require.False(t, reply.SchedulerConfig.PreemptionConfig.SystemSchedulerEnabled)
583584
require.True(t, reply.SchedulerConfig.PauseEvalBroker)
585+
require.Equal(t, uint(200), reply.SchedulerConfig.GetNodeLimitForFeasibilityChecks())
584586

585587
require.False(t, s1.evalBroker.Enabled())
586588
require.False(t, s1.blockedEvals.Enabled())

0 commit comments

Comments
 (0)