use fuser to check if gpu is free

Signed-off-by: Varun Ramachandra Sekar <[email protected]>

Author: varunrsekar

cmd/gpu-kubelet-plugin/device_state.go

@@ -98,7 +98,7 @@ func NewDeviceState(ctx context.Context, config *Config) (*DeviceState, error) {
 
 	var vfioPciManager *VfioPciManager
 	if featuregates.Enabled(featuregates.PassthroughSupport) {
-		manager := NewVfioPciManager(string(containerDriverRoot), nvdevlib, true /* nvidiaEnabled */)
+		manager := NewVfioPciManager(string(containerDriverRoot), string(hostDriverRoot), nvdevlib, true /* nvidiaEnabled */)
 		if err := manager.Prechecks(); err == nil {
 			vfioPciManager = manager
 		} else {

cmd/gpu-kubelet-plugin/vfio-device.go

@@ -47,18 +47,20 @@ const (
 )
 
 type VfioPciManager struct {
-	driverRoot    string
-	driver        string
-	nvlib         *deviceLib
-	nvidiaEnabled bool
+	containerDriverRoot string
+	hostDriverRoot      string
+	driver              string
+	nvlib               *deviceLib
+	nvidiaEnabled       bool
 }
 
-func NewVfioPciManager(driverRoot string, nvlib *deviceLib, nvidiaEnabled bool) *VfioPciManager {
+func NewVfioPciManager(containerDriverRoot string, hostDriverRoot string, nvlib *deviceLib, nvidiaEnabled bool) *VfioPciManager {
 	vm := &VfioPciManager{
-		driverRoot:    driverRoot,
-		driver:        vfioPciDriver,
-		nvlib:         nvlib,
-		nvidiaEnabled: nvidiaEnabled,
+		containerDriverRoot: containerDriverRoot,
+		hostDriverRoot:      hostDriverRoot,
+		driver:              vfioPciDriver,
+		nvlib:               nvlib,
+		nvidiaEnabled:       nvidiaEnabled,
 	}
 	if !vm.isVfioPCIModuleLoaded() {
 		err := vm.loadVfioPciModule()
@@ -135,20 +137,21 @@ func (vm *VfioPciManager) WaitForGPUFree(ctx context.Context, info *VfioDeviceIn
 	ticker := time.NewTicker(gpuFreeCheckInterval)
 	defer ticker.Stop()
 
-	gpuDeviceNode := filepath.Join(vm.driverRoot, "dev", fmt.Sprintf("nvidia%d", info.parent.minor))
+	gpuDeviceNode := filepath.Join(vm.hostDriverRoot, "dev", fmt.Sprintf("nvidia%d", info.parent.minor))
 	for {
 		select {
 		case <-timeout:
 			return fmt.Errorf("timed out waiting for gpu to be free")
 		case <-ticker.C:
-			out, err := execCommand("lsof", []string{gpuDeviceNode}) //nolint:gosec
-			klog.Infof("lsof output: %s, err: %v", string(out), err)
+			out, err := execCommandWithChroot(hostRoot, "fuser", []string{gpuDeviceNode}) //nolint:gosec
 			if err != nil {
 				if exitErr, ok := err.(*exec.ExitError); ok && exitErr.ExitCode() == 1 {
 					return nil
 				}
+				klog.Errorf("Unexpected error checking if gpu device %q is free: %v", info.pcieBusID, err)
 				continue
 			}
+			klog.Infof("gpu device %q has open fds by process(es): %s", info.pcieBusID, string(out))
 		}
 	}
 }

deployments/container/Dockerfile

@@ -43,8 +43,7 @@ RUN apt-get update && \
         make \
         git \
         gcc-aarch64-linux-gnu \
-        gcc \
-        lsof
+        gcc
 
 # Install dependencies for `bash-static` build.
 RUN apt-get install -y gpg curl autoconf file
@@ -148,16 +147,6 @@ COPY --from=build   /artifacts/webhook                        /usr/bin/webhook
 COPY --from=build   /bashbuild/bash                           /bin/bash
 COPY --from=build   /build/scripts/bind_to_driver.sh          /usr/bin/bind_to_driver.sh
 COPY --from=build   /build/scripts/unbind_from_driver.sh      /usr/bin/unbind_from_driver.sh
-COPY --from=build   /lib/x86_64-linux-gnu/libselinux.so.1     /lib/x86_64-linux-gnu/libselinux.so.1
-COPY --from=build   /lib/x86_64-linux-gnu/libtirpc.so.3       /lib/x86_64-linux-gnu/libtirpc.so.3
-COPY --from=build   /lib/x86_64-linux-gnu/libpcre2-8.so.0     /lib/x86_64-linux-gnu/libpcre2-8.so.0
-COPY --from=build   /lib/x86_64-linux-gnu/libgssapi_krb5.so.2 /lib/x86_64-linux-gnu/libgssapi_krb5.so.2
-COPY --from=build   /lib/x86_64-linux-gnu/libkrb5.so.3        /lib/x86_64-linux-gnu/libkrb5.so.3
-COPY --from=build   /lib/x86_64-linux-gnu/libk5crypto.so.3    /lib/x86_64-linux-gnu/libk5crypto.so.3
-COPY --from=build   /lib/x86_64-linux-gnu/libcom_err.so.2     /lib/x86_64-linux-gnu/libcom_err.so.2
-COPY --from=build   /lib/x86_64-linux-gnu/libkrb5support.so.0 /lib/x86_64-linux-gnu/libkrb5support.so.0
-COPY --from=build   /lib/x86_64-linux-gnu/libkeyutils.so.1    /lib/x86_64-linux-gnu/libkeyutils.so.1
-COPY --from=build   /usr/bin/lsof                             /usr/bin/lsof
 COPY /hack/kubelet-plugin-prestart.sh /usr/bin/kubelet-plugin-prestart.sh
 COPY /templates /templates