Skip to content

Commit 40dae3c

Browse files
author
wawa0210
committed
skip old architecture version GPU settings time slice
Signed-off-by: wawa0210 <xiao.zhang@daocloud.io>
1 parent 0e01612 commit 40dae3c

1 file changed

Lines changed: 30 additions & 0 deletions

File tree

cmd/nvidia-dra-plugin/sharing.go

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"fmt"
2323
"os"
2424
"os/exec"
25+
"strconv"
2526
"strings"
2627
"text/template"
2728
"time"
@@ -102,6 +103,17 @@ func (t *TimeSlicingManager) SetTimeSlice(devices *PreparedDevices, config *shar
102103
return fmt.Errorf("setting a TimeSlice duration on MIG devices is unsupported")
103104
}
104105

106+
for _, gpu := range devices.Gpu.Devices {
107+
err, isSupportTimeSlice := detectSupportTimeSliceByCudaComputeCapability(gpu.cudaComputeCapability)
108+
if err != nil {
109+
return fmt.Errorf("failed to detectSupportTimeSliceByCudaComputeCapability : %w", err)
110+
}
111+
if !isSupportTimeSlice {
112+
klog.InfoS("the current card does not support setting time slices and will be ignored.", "arch", gpu.architecture, "uuid", gpu.uuid, "cudaComputeCapability", gpu.cudaComputeCapability)
113+
return fmt.Errorf("setting a TimeSlice duration on devices uuid=%v is unsupported", gpu.uuid)
114+
}
115+
}
116+
105117
timeSlice := sharing.DefaultTimeSlice
106118
if config != nil && config.TimeSlice != nil {
107119
timeSlice = *config.TimeSlice
@@ -390,3 +402,21 @@ func (m *MpsControlDaemon) Stop(ctx context.Context) error {
390402

391403
return nil
392404
}
405+
406+
// detactSupportTimeSliceByArch Determine whether the architecture series
407+
// supports setting time slices based on the gpu cudaComputeCapability.
408+
func detectSupportTimeSliceByCudaComputeCapability(cudaComputeCapability string) (error, bool) {
409+
// ref https://github.com/NVIDIA/k8s-dra-driver/pull/58#discussion_r1469338562
410+
// we believe time-slicing is available on Volta+ architectures, so the check would simply be cudaComputeCapability >= 7.0
411+
// by https://github.com/NVIDIA/go-nvlib/blob/main/pkg/nvlib/device/device.go#L149, We know that cuda major and minor versions are concatenated through `.` .
412+
413+
cudaVersion := strings.Split(cudaComputeCapability, ".")
414+
major, err := strconv.Atoi(cudaVersion[0])
415+
if err != nil {
416+
return fmt.Errorf("error to get cudaComputeCapability major version %v", cudaComputeCapability), false
417+
}
418+
if major >= 7 {
419+
return nil, true
420+
}
421+
return nil, false
422+
}

0 commit comments

Comments
 (0)