Skip to content

Commit 50f1518

Browse files
adity1rautadity1raut
committed
fix(scheduler): compare the request type with the device vendor when scoring
Signed-off-by: Aditya Raut <araut7798@gmail.com>
1 parent 3452ead commit 50f1518

6 files changed

Lines changed: 306 additions & 75 deletions

File tree

‎pkg/device/devices.go‎

Lines changed: 14 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ type DeviceUsage struct {
119119
MigUsage MigInUse // Deprecated: unused by dynamic NVIDIA MIG.
120120
Numa int
121121
Type string
122+
DeviceVendor string
122123
Health bool
123124
PodInfos []*PodInfo
124125
CustomInfo map[string]any
@@ -263,18 +264,19 @@ func (d *DeviceUsage) DeepCopy() *DeviceUsage {
263264
return nil
264265
}
265266
dup := &DeviceUsage{
266-
ID: d.ID,
267-
Index: d.Index,
268-
Used: d.Used,
269-
Count: d.Count,
270-
Usedmem: d.Usedmem,
271-
Totalmem: d.Totalmem,
272-
Totalcore: d.Totalcore,
273-
Usedcores: d.Usedcores,
274-
Mode: d.Mode,
275-
Numa: d.Numa,
276-
Type: d.Type,
277-
Health: d.Health,
267+
ID: d.ID,
268+
Index: d.Index,
269+
Used: d.Used,
270+
Count: d.Count,
271+
Usedmem: d.Usedmem,
272+
Totalmem: d.Totalmem,
273+
Totalcore: d.Totalcore,
274+
Usedcores: d.Usedcores,
275+
Mode: d.Mode,
276+
Numa: d.Numa,
277+
Type: d.Type,
278+
DeviceVendor: d.DeviceVendor,
279+
Health: d.Health,
278280
}
279281

280282
if d.MigProfiles != nil {

‎pkg/device/devices_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1928,9 +1928,10 @@ func TestDeviceUsageDeepCopy(t *testing.T) {
19281928
{Name: "1g.5gb", Core: 1, Memory: 5, InUse: false},
19291929
},
19301930
},
1931-
Numa: 0,
1932-
Type: "NVIDIA",
1933-
Health: true,
1931+
Numa: 0,
1932+
Type: "NVIDIA A100-SXM4-40GB",
1933+
DeviceVendor: "NVIDIA",
1934+
Health: true,
19341935
PodInfos: []*PodInfo{
19351936
{
19361937
Pod: &corev1.Pod{

‎pkg/scheduler/policy/gpu_policy.go‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -203,10 +203,13 @@ func (ds *DeviceListsScore) ComputeScore(requests device.ContainerDeviceRequests
203203
return
204204
}
205205
request, core, mem := int32(0), int32(0), int32(0)
206-
// Here we are required to use the same type device
206+
// Here we are required to use the same type device. Compare against the
207+
// vendor rather than the registered type: backends such as NVIDIA and
208+
// Cambricon register a model name ("NVIDIA A100-SXM4-40GB", "MLU370-X8")
209+
// as the type, while the request carries the vendor common word.
207210
for _, container := range requests {
208211

209-
if container.Type != ds.Device.Type {
212+
if container.Type != ds.Device.DeviceVendor {
210213
continue
211214
}
212215

‎pkg/scheduler/policy/gpu_policy_test.go‎

Lines changed: 207 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -512,53 +512,57 @@ func TestComputeScore(t *testing.T) {
512512
{
513513
name: "Zero capacity device returns score 0 without panic",
514514
device: &device.DeviceUsage{
515-
ID: "test-device",
516-
Type: "type1",
517-
Count: 0,
518-
Totalcore: 0,
519-
Totalmem: 0,
515+
ID: "test-device",
516+
Type: "type1",
517+
DeviceVendor: "type1",
518+
Count: 0,
519+
Totalcore: 0,
520+
Totalmem: 0,
520521
},
521522
requests: make(device.ContainerDeviceRequests),
522523
expectedScore: 0,
523524
},
524525
{
525526
name: "Partial zero capacity (Count=0) returns score 0 without panic",
526527
device: &device.DeviceUsage{
527-
ID: "test-device",
528-
Type: "type1",
529-
Count: 0,
530-
Totalcore: 8,
531-
Totalmem: 4096,
528+
ID: "test-device",
529+
Type: "type1",
530+
DeviceVendor: "type1",
531+
Count: 0,
532+
Totalcore: 8,
533+
Totalmem: 4096,
532534
},
533535
requests: make(device.ContainerDeviceRequests),
534536
expectedScore: 0,
535537
},
536538
{
537539
name: "ContainerDeviceRequests has no data",
538540
device: &device.DeviceUsage{
539-
ID: "test-device",
540-
Type: "type1",
541-
Totalcore: 4,
542-
Totalmem: 8192,
543-
Count: 10,
544-
Used: 2,
545-
Usedcores: 1,
546-
Usedmem: 2048,
541+
ID: "test-device",
542+
Type: "type1",
543+
DeviceVendor: "type1",
544+
Totalcore: 4,
545+
Totalmem: 8192,
546+
Count: 10,
547+
Used: 2,
548+
Usedcores: 1,
549+
Usedmem: 2048,
547550
},
548551
requests: make(device.ContainerDeviceRequests),
549552
expectedScore: 7.0,
550553
},
551554
{
552555
name: "Filter other device types and ignore Nums for slots",
553556
device: &device.DeviceUsage{
554-
ID: "test-device",
555-
Type: "type1",
556-
Totalcore: 4,
557-
Totalmem: 8192,
558-
Count: 10,
559-
Used: 2,
560-
Usedcores: 1,
561-
Usedmem: 2048,
557+
ID: "test-device",
558+
Type: "type1",
559+
DeviceVendor: "type1",
560+
Totalcore: 4,
561+
Totalmem: 8192,
562+
Count: 10,
563+
Used: 2,
564+
Usedcores: 1,
565+
Usedmem: 2048,
562566
},
563567
requests: device.ContainerDeviceRequests{
564568
"type1": {
@@ -581,14 +585,15 @@ func TestComputeScore(t *testing.T) {
581585
{
582586
name: "MemPercentagereq calculation evaluates correctly",
583587
device: &device.DeviceUsage{
584-
ID: "test-device",
585-
Type: "type1",
586-
Totalcore: 10,
587-
Totalmem: 10000,
588-
Count: 10,
589-
Used: 0,
590-
Usedcores: 0,
591-
Usedmem: 0,
588+
ID: "test-device",
589+
Type: "type1",
590+
DeviceVendor: "type1",
591+
Totalcore: 10,
592+
Totalmem: 10000,
593+
Count: 10,
594+
Used: 0,
595+
Usedcores: 0,
596+
Usedmem: 0,
592597
},
593598
requests: device.ContainerDeviceRequests{
594599
"type1": {
@@ -604,14 +609,15 @@ func TestComputeScore(t *testing.T) {
604609
{
605610
name: "Filter other device types and aggregate same device types",
606611
device: &device.DeviceUsage{
607-
ID: "test-device",
608-
Type: "type1",
609-
Totalcore: 10,
610-
Totalmem: 10000,
611-
Count: 10,
612-
Used: 0,
613-
Usedcores: 0,
614-
Usedmem: 0,
612+
ID: "test-device",
613+
Type: "type1",
614+
DeviceVendor: "type1",
615+
Totalcore: 10,
616+
Totalmem: 10000,
617+
Count: 10,
618+
Used: 0,
619+
Usedcores: 0,
620+
Usedmem: 0,
615621
},
616622
requests: device.ContainerDeviceRequests{
617623
"container1": {
@@ -638,6 +644,164 @@ func TestComputeScore(t *testing.T) {
638644
},
639645
expectedScore: 8.0,
640646
},
647+
{
648+
// The NVIDIA device plugin registers the card's model name as the
649+
// device type, while the request carries the "NVIDIA" common word.
650+
name: "Model named device type still counts the request",
651+
device: &device.DeviceUsage{
652+
ID: "gpu-0",
653+
Type: "NVIDIA A100-SXM4-40GB",
654+
DeviceVendor: "NVIDIA",
655+
Totalcore: 100,
656+
Totalmem: 40960,
657+
Count: 10,
658+
},
659+
requests: device.ContainerDeviceRequests{
660+
"NVIDIA": {
661+
Nums: 1,
662+
Type: "NVIDIA",
663+
Memreq: 20480,
664+
MemPercentagereq: 101,
665+
Coresreq: 30,
666+
},
667+
},
668+
// slots 1/10 + cores 30/100 + memory 20480/40960
669+
expectedScore: float32(util.Weight) * 0.9,
670+
},
671+
{
672+
// Cambricon takes the device type off the node's Model label.
673+
name: "Model named device type still counts the request for cambricon",
674+
device: &device.DeviceUsage{
675+
ID: "mlu-0",
676+
Type: "MLU370-X8",
677+
DeviceVendor: "MLU",
678+
Totalcore: 100,
679+
Totalmem: 40960,
680+
Count: 10,
681+
},
682+
requests: device.ContainerDeviceRequests{
683+
"MLU": {
684+
Nums: 1,
685+
Type: "MLU",
686+
Memreq: 20480,
687+
MemPercentagereq: 101,
688+
Coresreq: 30,
689+
},
690+
},
691+
// slots 1/10 + cores 30/100 + memory 20480/40960
692+
expectedScore: float32(util.Weight) * 0.9,
693+
},
694+
{
695+
name: "Request of another vendor is not scored",
696+
device: &device.DeviceUsage{
697+
ID: "gpu-0",
698+
Type: "NVIDIA A100-SXM4-40GB",
699+
DeviceVendor: "NVIDIA",
700+
Totalcore: 100,
701+
Totalmem: 40960,
702+
Count: 10,
703+
},
704+
requests: device.ContainerDeviceRequests{
705+
"MLU": {
706+
Nums: 1,
707+
Type: "MLU",
708+
Memreq: 20480,
709+
MemPercentagereq: 101,
710+
Coresreq: 30,
711+
},
712+
},
713+
expectedScore: 0,
714+
},
715+
{
716+
// The chart ships both "Ascend910B4" and "Ascend910B4-1" as Ascend
717+
// common words, so a device must only be scored against requests
718+
// carrying its own vendor, never one whose name contains it.
719+
name: "Vendors sharing a prefix are not mixed on the longer vendor",
720+
device: &device.DeviceUsage{
721+
ID: "npu-0",
722+
Type: "Ascend910B4-1",
723+
DeviceVendor: "Ascend910B4-1",
724+
Totalcore: 100,
725+
Totalmem: 40960,
726+
Count: 10,
727+
},
728+
requests: device.ContainerDeviceRequests{
729+
"Ascend910B4": {
730+
Nums: 1,
731+
Type: "Ascend910B4",
732+
Memreq: 8192,
733+
MemPercentagereq: 101,
734+
Coresreq: 10,
735+
},
736+
"Ascend910B4-1": {
737+
Nums: 1,
738+
Type: "Ascend910B4-1",
739+
Memreq: 4096,
740+
MemPercentagereq: 101,
741+
Coresreq: 20,
742+
},
743+
},
744+
// slots 1/10 + cores 20/100 + memory 4096/40960
745+
expectedScore: float32(util.Weight) * 0.4,
746+
},
747+
{
748+
name: "Vendors sharing a prefix are not mixed on the shorter vendor",
749+
device: &device.DeviceUsage{
750+
ID: "npu-1",
751+
Type: "Ascend910B4",
752+
DeviceVendor: "Ascend910B4",
753+
Totalcore: 100,
754+
Totalmem: 40960,
755+
Count: 10,
756+
},
757+
requests: device.ContainerDeviceRequests{
758+
"Ascend910B4": {
759+
Nums: 1,
760+
Type: "Ascend910B4",
761+
Memreq: 8192,
762+
MemPercentagereq: 101,
763+
Coresreq: 10,
764+
},
765+
"Ascend910B4-1": {
766+
Nums: 1,
767+
Type: "Ascend910B4-1",
768+
Memreq: 4096,
769+
MemPercentagereq: 101,
770+
Coresreq: 20,
771+
},
772+
},
773+
// slots 1/10 + cores 10/100 + memory 8192/40960
774+
expectedScore: float32(util.Weight) * 0.4,
775+
},
776+
{
777+
name: "Requests of one vendor accumulate on a model named device",
778+
device: &device.DeviceUsage{
779+
ID: "gpu-0",
780+
Type: "NVIDIA A100-SXM4-40GB",
781+
DeviceVendor: "NVIDIA",
782+
Totalcore: 100,
783+
Totalmem: 40960,
784+
Count: 10,
785+
},
786+
requests: device.ContainerDeviceRequests{
787+
"container1": {
788+
Nums: 1,
789+
Type: "NVIDIA",
790+
Memreq: 4096,
791+
MemPercentagereq: 101,
792+
Coresreq: 10,
793+
},
794+
"container2": {
795+
Nums: 1,
796+
Type: "NVIDIA",
797+
Memreq: 8192,
798+
MemPercentagereq: 101,
799+
Coresreq: 20,
800+
},
801+
},
802+
// slots 2/10 + cores 30/100 + memory 12288/40960
803+
expectedScore: float32(util.Weight) * 0.8,
804+
},
641805
}
642806

643807
for _, tt := range tests {
@@ -666,7 +830,7 @@ func TestComputeScoreWithResourceWeights(t *testing.T) {
666830
}
667831
newDeviceScore := func(id string, used, usedCores, usedMemory int32) *DeviceListsScore {
668832
return &DeviceListsScore{Device: &device.DeviceUsage{
669-
ID: id, Type: "type1", Count: 10, Totalcore: 100, Totalmem: 100,
833+
ID: id, Type: "type1", DeviceVendor: "type1", Count: 10, Totalcore: 100, Totalmem: 100,
670834
Used: used, Usedcores: usedCores, Usedmem: usedMemory,
671835
}}
672836
}

0 commit comments

Comments
 (0)