Skip to content

Commit a5d0987

Browse files
committed
Add pkg/nvpassthrough for binding GPUs to the vfio-pci driver
This is mostly a direct port from https://github.com/NVIDIA/k8s-driver-manager/tree/fd043d8f5f74a26b04f83f1eb11b659d402e94de/internal/nvpassthrough Signed-off-by: Christopher Desiniotis <cdesiniotis@nvidia.com>
1 parent 182c9a1 commit a5d0987

330 files changed

Lines changed: 198481 additions & 2 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

go.mod

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
module github.com/NVIDIA/go-nvlib
22

3-
go 1.20
3+
go 1.24.0
4+
5+
toolchain go1.24.12
46

57
require (
68
github.com/NVIDIA/go-nvml v0.13.0-1
79
github.com/google/uuid v1.6.0
810
github.com/stretchr/testify v1.11.1
11+
golang.org/x/sys v0.40.0
912
)
1013

1114
require (

go.sum

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
88
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
99
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
1010
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
11+
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
12+
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
1113
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
1214
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
1315
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

pkg/nvpassthrough/kmod.go

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright (c) NVIDIA CORPORATION. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package nvpassthrough
18+
19+
import (
20+
"os/exec"
21+
)
22+
23+
type kernelModules struct {
24+
root string
25+
}
26+
27+
func newKernelModules(root string) *kernelModules {
28+
km := &kernelModules{
29+
root: root,
30+
}
31+
return km
32+
}
33+
34+
func (km *kernelModules) load(module string) error {
35+
cmd := exec.Command("chroot", km.root, "modprobe", module)
36+
return cmd.Run()
37+
}

pkg/nvpassthrough/logger.go

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/**
2+
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
**/
16+
17+
package nvpassthrough
18+
19+
type basicLogger interface {
20+
Debugf(string, ...interface{})
21+
Warnf(string, ...interface{})
22+
Infof(string, ...interface{})
23+
}
24+
25+
type nullLogger struct{}
26+
27+
func (n *nullLogger) Debugf(string, ...interface{}) {}
28+
29+
func (n *nullLogger) Warnf(string, ...interface{}) {}
30+
31+
func (n *nullLogger) Infof(string, ...interface{}) {}

pkg/nvpassthrough/modalias.go

Lines changed: 231 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
/*
2+
* Copyright (c) NVIDIA CORPORATION. All rights reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package nvpassthrough
18+
19+
import (
20+
"fmt"
21+
"math"
22+
"os"
23+
"path/filepath"
24+
"reflect"
25+
"strings"
26+
27+
"golang.org/x/sys/unix"
28+
)
29+
30+
const (
31+
vfioPciAliasPrefix string = "alias vfio_pci:"
32+
)
33+
34+
// modAlias is a decomposed version of string like this
35+
//
36+
// vNNNNNNNNdNNNNNNNNsvNNNNNNNNsdNNNNNNNNbcNNscNNiNN
37+
//
38+
// The "NNNN" are always of the length in the example
39+
// unless replaced with a wildcard ("*").
40+
type modAlias struct {
41+
vendor string // v
42+
device string // d
43+
subvendor string // sv
44+
subdevice string // sd
45+
baseClass string // bc
46+
subClass string // sc
47+
programmingInterface string // i
48+
}
49+
50+
// vfioAlias represents an entry from the modules.alias file for a vfio driver.
51+
type vfioAlias struct {
52+
modAlias *modAlias // The modalias pattern
53+
driver string // The vfio driver name
54+
}
55+
56+
func parseModAliasString(input string) (*modAlias, error) {
57+
if input == "" {
58+
return nil, fmt.Errorf("modalias string is empty")
59+
}
60+
61+
input = strings.TrimSpace(input)
62+
63+
// Trim the leading "pci:" prefix in the modalias file
64+
_, input, ok := strings.Cut(input, "pci:")
65+
if !ok {
66+
return nil, fmt.Errorf("unexpected number of parts in modalias after trimming 'pci:' prefix: %s", input)
67+
}
68+
69+
if !strings.HasPrefix(input, "v") {
70+
return nil, fmt.Errorf("modalias must start with 'v', got: %s", input)
71+
}
72+
ma := &modAlias{}
73+
var before, after string
74+
var found bool
75+
after = strings.TrimPrefix(input, "v")
76+
77+
before, after, found = strings.Cut(after, "d")
78+
if !found {
79+
return nil, fmt.Errorf("failed to find delimiter 'd' in %q", input)
80+
}
81+
ma.vendor = before
82+
83+
before, after, found = strings.Cut(after, "sv")
84+
if !found {
85+
return nil, fmt.Errorf("failed to find delimiter 'sv' in %q", input)
86+
}
87+
ma.device = before
88+
89+
before, after, found = strings.Cut(after, "sd")
90+
if !found {
91+
return nil, fmt.Errorf("failed to find delimiter 'sd' in %q", input)
92+
}
93+
ma.subvendor = before
94+
95+
before, after, found = strings.Cut(after, "bc")
96+
if !found {
97+
return nil, fmt.Errorf("failed to find delimiter 'bc' in %q", input)
98+
}
99+
ma.subdevice = before
100+
101+
before, after, found = strings.Cut(after, "sc")
102+
if !found {
103+
return nil, fmt.Errorf("failed to find delimiter 'sc' in input %q", input)
104+
}
105+
ma.baseClass = before
106+
107+
before, after, found = strings.Cut(after, "i")
108+
if !found {
109+
return nil, fmt.Errorf("failed to find delimiter 'i' in %q", input)
110+
}
111+
ma.subClass = before
112+
ma.programmingInterface = after
113+
114+
return ma, nil
115+
}
116+
117+
func getKernelVersion() (string, error) {
118+
var uname unix.Utsname
119+
if err := unix.Uname(&uname); err != nil {
120+
return "", err
121+
}
122+
123+
// Convert C-style byte array to Go string
124+
release := make([]byte, 0, len(uname.Release))
125+
for _, c := range uname.Release {
126+
if c == 0 {
127+
break
128+
}
129+
release = append(release, c)
130+
}
131+
132+
return string(release), nil
133+
}
134+
135+
func getVFIOAliases() ([]vfioAlias, error) {
136+
kernelVersion, err := getKernelVersion()
137+
if err != nil {
138+
return nil, fmt.Errorf("failed to get kernel version: %w", err)
139+
}
140+
141+
modulesAliasFilePath := filepath.Join(libModulesRoot, kernelVersion, "modules.alias")
142+
modulesAliasContent, err := os.ReadFile(modulesAliasFilePath)
143+
if err != nil {
144+
return nil, fmt.Errorf("failed to read file %s: %w", modulesAliasFilePath, err)
145+
}
146+
147+
// Get all vfio aliases from the modules.alias file
148+
// (all lines starting with 'alias vfio_pci:')
149+
return getVFIOAliasesFromString(string(modulesAliasContent)), nil
150+
151+
}
152+
153+
// getVFIOAliases returns the vfio driver aliases from the input string.
154+
// The input string is expected to be the content of a modules.alias file.
155+
// Only lines that begin with 'alias vfio_pci:' are parsed, with the
156+
// format being: alias vfio_pci:<modalias string> <driver_name>.
157+
func getVFIOAliasesFromString(input string) []vfioAlias {
158+
var aliases []vfioAlias
159+
160+
lines := strings.Split(input, "\n")
161+
for _, line := range lines {
162+
line = strings.TrimSpace(line)
163+
164+
if !strings.HasPrefix(line, vfioPciAliasPrefix) {
165+
continue
166+
}
167+
168+
split := strings.SplitN(line, " ", 3)
169+
if len(split) != 3 {
170+
continue
171+
}
172+
modAliasStr := split[1]
173+
modAlias, err := parseModAliasString(modAliasStr)
174+
if err != nil {
175+
continue
176+
}
177+
178+
driver := split[2]
179+
aliases = append(aliases, vfioAlias{
180+
modAlias: modAlias,
181+
driver: driver,
182+
})
183+
}
184+
185+
return aliases
186+
}
187+
188+
// findBestMatch finds the best matching VFIO driver for the given modalias
189+
// by comparing against all available vfio alias patterns. The best match
190+
// is the one with the fewest wildcard characters.
191+
func findBestMatch(deviceModAlias *modAlias, aliases []vfioAlias) string {
192+
var bestDriver string
193+
bestWildcardCount := math.MaxInt
194+
195+
for _, alias := range aliases {
196+
if matches, wildcardCount := matchModalias(deviceModAlias, alias.modAlias); matches {
197+
if wildcardCount < bestWildcardCount {
198+
bestDriver = alias.driver
199+
bestWildcardCount = wildcardCount
200+
}
201+
}
202+
}
203+
204+
return bestDriver
205+
}
206+
207+
// matchModalias checks if a device modalias matches a pattern from modules.alias
208+
// Returns true if it matches and the number of wildcards.
209+
func matchModalias(deviceModAlias, patternModAlias *modAlias) (bool, int) {
210+
wildcardCount := 0
211+
212+
modAliasType := reflect.TypeOf(*deviceModAlias)
213+
deviceModAliasValue := reflect.ValueOf(*deviceModAlias)
214+
patternModAliasValue := reflect.ValueOf(*patternModAlias)
215+
216+
// iterate over both modAlias structs, comparing each field
217+
for i := 0; i < modAliasType.NumField(); i++ {
218+
deviceValue := deviceModAliasValue.Field(i).String()
219+
patternValue := patternModAliasValue.Field(i).String()
220+
221+
if patternValue == "*" {
222+
wildcardCount++
223+
continue
224+
}
225+
226+
if deviceValue != patternValue {
227+
return false, wildcardCount
228+
}
229+
}
230+
return true, wildcardCount
231+
}

0 commit comments

Comments
 (0)