Skip to content

Commit efd6002

Browse files
committed
Add pkg/nvpassthrough for binding GPUs to the vfio-pci driver
This is mostly a direct port from https://github.com/NVIDIA/k8s-driver-manager/tree/fd043d8f5f74a26b04f83f1eb11b659d402e94de/internal/nvpassthrough Signed-off-by: Christopher Desiniotis <cdesiniotis@nvidia.com>
1 parent 182c9a1 commit efd6002

330 files changed

Lines changed: 198495 additions & 2 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

go.mod

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
module github.com/NVIDIA/go-nvlib
22

3-
go 1.20
3+
go 1.24.0
4+
5+
toolchain go1.24.12
46

57
require (
68
github.com/NVIDIA/go-nvml v0.13.0-1
79
github.com/google/uuid v1.6.0
810
github.com/stretchr/testify v1.11.1
11+
golang.org/x/sys v0.40.0
912
)
1013

1114
require (

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
88
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
99
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
1010
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
11+
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
12+
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
1113
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
1214
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
1315
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

pkg/nvpassthrough/kmod.go

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
/*
2+
* Copyright (c) NVIDIA CORPORATION. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package nvpassthrough
18+
19+
import (
20+
"bufio"
21+
"fmt"
22+
"os"
23+
"os/exec"
24+
"path/filepath"
25+
"strconv"
26+
"strings"
27+
)
28+
29+
const (
30+
procModules = "/proc/modules"
31+
)
32+
33+
type kernelModules struct {
34+
log basicLogger
35+
36+
root string
37+
}
38+
39+
func newKernelModules(log basicLogger, root string) *kernelModules {
40+
km := &kernelModules{
41+
log: log,
42+
root: root,
43+
}
44+
return km
45+
}
46+
47+
func (km *kernelModules) list(searchKey string) error {
48+
modsFilePath := filepath.Join(km.root, procModules)
49+
file, err := os.Open(modsFilePath)
50+
if err != nil {
51+
return fmt.Errorf("error opening file %s: %w", modsFilePath, err)
52+
}
53+
defer func(file *os.File) {
54+
err := file.Close()
55+
if err != nil {
56+
km.log.Warnf("error closing file %s: %v", modsFilePath, err)
57+
}
58+
}(file)
59+
60+
scanner := bufio.NewScanner(file)
61+
km.log.Infof("%-20s %-10s %-15s %s\n", "Module", "Size", "Ref Count", "Used by") // Header
62+
63+
for scanner.Scan() {
64+
line := scanner.Text()
65+
66+
if len(searchKey) > 0 && !strings.Contains(line, searchKey) {
67+
continue
68+
}
69+
70+
fields := strings.Fields(line)
71+
72+
if len(fields) >= 4 {
73+
name := fields[0]
74+
75+
size, err := strconv.Atoi(fields[1])
76+
if err != nil {
77+
km.log.Warnf("error parsing module size %s: %v", fields[1], err)
78+
continue
79+
}
80+
81+
refCnt, err := strconv.Atoi(fields[2])
82+
if err != nil {
83+
km.log.Warnf("error parsing module ref count %s: %v", fields[2], err)
84+
continue
85+
}
86+
87+
usedBy := fields[3]
88+
89+
km.log.Infof("%-20s %-10d %-15d %s\n", name, size, refCnt, usedBy)
90+
}
91+
}
92+
93+
if err := scanner.Err(); err != nil {
94+
km.log.Warnf("error reading /proc/modules: %v\n", err)
95+
return err
96+
}
97+
return nil
98+
}
99+
100+
func (km *kernelModules) load(module string) error {
101+
cmd := exec.Command("chroot", km.root, "modprobe", module)
102+
return cmd.Run()
103+
}

pkg/nvpassthrough/logger.go

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/**
2+
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
**/
16+
17+
package nvpassthrough
18+
19+
type basicLogger interface {
20+
Debugf(string, ...interface{})
21+
Warnf(string, ...interface{})
22+
Infof(string, ...interface{})
23+
}
24+
25+
type nullLogger struct{}
26+
27+
func (n *nullLogger) Debugf(string, ...interface{}) {}
28+
29+
func (n *nullLogger) Warnf(string, ...interface{}) {}
30+
31+
func (n *nullLogger) Infof(string, ...interface{}) {}

pkg/nvpassthrough/modalias.go

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
/*
2+
* Copyright (c) NVIDIA CORPORATION. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package nvpassthrough
18+
19+
import (
20+
"fmt"
21+
"math"
22+
"reflect"
23+
"strings"
24+
25+
"golang.org/x/sys/unix"
26+
)
27+
28+
const (
29+
vfioPciAliasPrefix string = "alias vfio_pci:"
30+
)
31+
32+
// modAlias is a decomposed version of string like this
33+
//
34+
// vNNNNNNNNdNNNNNNNNsvNNNNNNNNsdNNNNNNNNbcNNscNNiNN
35+
//
36+
// The "NNNN" are always of the length in the example
37+
// unless replaced with a wildcard ("*").
38+
type modAlias struct {
39+
vendor string // v
40+
device string // d
41+
subvendor string // sv
42+
subdevice string // sd
43+
baseClass string // bc
44+
subClass string // sc
45+
programmingInterface string // i
46+
}
47+
48+
// vfioAlias represents an entry from the modules.alias file for a vfio driver.
49+
type vfioAlias struct {
50+
modAlias *modAlias // The modalias pattern
51+
driver string // The vfio driver name
52+
}
53+
54+
func parseModAliasString(input string) (*modAlias, error) {
55+
if input == "" {
56+
return nil, fmt.Errorf("modalias string is empty")
57+
}
58+
59+
input = strings.TrimSpace(input)
60+
61+
// Trim the leading "pci:" prefix in the modalias file
62+
split := strings.SplitN(input, ":", 2)
63+
if len(split) != 2 {
64+
return nil, fmt.Errorf("unexpected number of parts in modalias after trimming 'pci:' prefix: %s", input)
65+
}
66+
input = split[1]
67+
68+
if !strings.HasPrefix(input, "v") {
69+
return nil, fmt.Errorf("modalias must start with 'v', got: %s", input)
70+
}
71+
72+
ma := &modAlias{}
73+
var before, after string
74+
var found bool
75+
after = input[1:] // cut leading 'v'
76+
77+
before, after, found = strings.Cut(after, "d")
78+
if !found {
79+
return nil, fmt.Errorf("failed to find delimiter 'd' in %q", input)
80+
}
81+
ma.vendor = before
82+
83+
before, after, found = strings.Cut(after, "sv")
84+
if !found {
85+
return nil, fmt.Errorf("failed to find delimiter 'sv' in %q", input)
86+
}
87+
ma.device = before
88+
89+
before, after, found = strings.Cut(after, "sd")
90+
if !found {
91+
return nil, fmt.Errorf("failed to find delimiter 'sd' in %q", input)
92+
}
93+
ma.subvendor = before
94+
95+
before, after, found = strings.Cut(after, "bc")
96+
if !found {
97+
return nil, fmt.Errorf("failed to find delimiter 'bc' in %q", input)
98+
}
99+
ma.subdevice = before
100+
101+
before, after, found = strings.Cut(after, "sc")
102+
if !found {
103+
return nil, fmt.Errorf("failed to find delimiter 'sc' in input %q", input)
104+
}
105+
ma.baseClass = before
106+
107+
before, after, found = strings.Cut(after, "i")
108+
if !found {
109+
return nil, fmt.Errorf("failed to find delimiter 'i' in %q", input)
110+
}
111+
ma.subClass = before
112+
ma.programmingInterface = after
113+
114+
return ma, nil
115+
}
116+
117+
func getKernelVersion() (string, error) {
118+
var uname unix.Utsname
119+
if err := unix.Uname(&uname); err != nil {
120+
return "", err
121+
}
122+
123+
// Convert C-style byte array to Go string
124+
release := make([]byte, 0, len(uname.Release))
125+
for _, c := range uname.Release {
126+
if c == 0 {
127+
break
128+
}
129+
release = append(release, c)
130+
}
131+
132+
return string(release), nil
133+
}
134+
135+
// getVFIOAliases returns the vfio driver aliases from the input string.
136+
// The input string is expected to be the content of a modules.alias file.
137+
// Only lines that begin with 'alias vfio_pci:' are parsed, with the
138+
// format being: alias vfio_pci:<modalias string> <driver_name>.
139+
func getVFIOAliases(input string) []vfioAlias {
140+
var aliases []vfioAlias
141+
142+
lines := strings.Split(input, "\n")
143+
for _, line := range lines {
144+
line = strings.TrimSpace(line)
145+
146+
if !strings.HasPrefix(line, vfioPciAliasPrefix) {
147+
continue
148+
}
149+
150+
split := strings.SplitN(line, " ", 3)
151+
if len(split) != 3 {
152+
continue
153+
}
154+
modAliasStr := split[1]
155+
modAlias, err := parseModAliasString(modAliasStr)
156+
if err != nil {
157+
continue
158+
}
159+
160+
driver := split[2]
161+
aliases = append(aliases, vfioAlias{
162+
modAlias: modAlias,
163+
driver: driver,
164+
})
165+
}
166+
167+
return aliases
168+
}
169+
170+
// findBestMatch finds the best matching VFIO driver for the given modalias
171+
// by comparing against all available vfio alias patterns. The best match
172+
// is the one with the fewest wildcard characters.
173+
func findBestMatch(deviceModAlias *modAlias, aliases []vfioAlias) string {
174+
var bestDriver string
175+
bestWildcardCount := math.MaxInt
176+
177+
for _, alias := range aliases {
178+
if matches, wildcardCount := matchModalias(deviceModAlias, alias.modAlias); matches {
179+
if wildcardCount < bestWildcardCount {
180+
bestDriver = alias.driver
181+
bestWildcardCount = wildcardCount
182+
}
183+
}
184+
}
185+
186+
return bestDriver
187+
}
188+
189+
// matchModalias checks if a device modalias matches a pattern from modules.alias
190+
// Returns true if it matches and the number of wildcards
191+
func matchModalias(deviceModAlias, patternModAlias *modAlias) (bool, int) {
192+
wildcardCount := 0
193+
194+
modAliasType := reflect.TypeOf(*deviceModAlias)
195+
deviceModAliasValue := reflect.ValueOf(*deviceModAlias)
196+
patternModAliasValue := reflect.ValueOf(*patternModAlias)
197+
198+
// iterate over both modAlias structs, comparing each field
199+
for i := 0; i < modAliasType.NumField(); i++ {
200+
deviceValue := deviceModAliasValue.Field(i).String()
201+
patternValue := patternModAliasValue.Field(i).String()
202+
203+
if patternValue == "*" {
204+
wildcardCount++
205+
continue
206+
}
207+
208+
if deviceValue != patternValue {
209+
return false, wildcardCount
210+
}
211+
}
212+
return true, wildcardCount
213+
}

0 commit comments

Comments
 (0)