Skip to content

Commit 7b41e06

Browse files
moshe010moshe010
authored
Merge pull request #38 from ykulazhenkov/dpdk-support
Add support for VFs which use userspace drivers
2 parents 9dd229e + b2a4f6f commit 7b41e06

File tree

8 files changed

+136
-60
lines changed

8 files changed

+136
-60
lines changed

‎README.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,10 @@ A metaplugin such as [Multus](https://github.com/intel/multus-cni) gets the allo
2929

3030
Accelerated Bridge plugin assumes that Linux Bridge is already exist and correctly configured on nodes.
3131

32+
CNI plugin also supports VF bound to userspace driver (currently only vfio-pci) which may be utilized
33+
for virtualization use-case i.e [KubeVirt](https://github.com/kubevirt/kubevirt).
34+
If CNI plugin detects that VF bounded to a userspace driver, it will skip step with VF netdev configuration.
35+
3236
## Build
3337

3438
This plugin uses Go modules for dependency management and requires Go 1.13+ to build.

‎pkg/config/config.go‎

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -44,25 +44,27 @@ func (c *Config) ParseConf(bytes []byte, conf *localtypes.PluginConf) error {
4444
conf.MAC = conf.NetConf.MAC
4545

4646
// DeviceID takes precedence; if we are given a VF pciaddr then work from there
47-
if conf.DeviceID != "" {
48-
// Get rest of the VF information
49-
pfName, vfID, err := getVfInfo(conf.DeviceID)
50-
if err != nil {
51-
return fmt.Errorf("failed to get VF information: %q", err)
52-
}
53-
conf.VFID = vfID
54-
conf.PFName = pfName
55-
} else {
47+
if conf.DeviceID == "" {
5648
return fmt.Errorf("VF pci addr is required")
5749
}
5850

59-
// Assuming VF is netdev interface; Get interface name
60-
hostIFName, err := utils.GetVFLinkName(conf.DeviceID)
51+
// Get rest of the VF information
52+
var err error
53+
conf.PFName, conf.VFID, err = getVfInfo(conf.DeviceID)
6154
if err != nil {
62-
return fmt.Errorf("failed to get VF name: %s", err)
55+
return fmt.Errorf("failed to get VF information: %q", err)
6356
}
64-
if hostIFName == "" {
65-
return fmt.Errorf("VF name is empty")
57+
58+
// Assuming VF is netdev interface; Get interface name
59+
hostIFName, err := utils.GetVFLinkName(conf.DeviceID)
60+
if err != nil || hostIFName == "" {
61+
conf.IsUserspaceDriver, err = utils.HasUserspaceDriver(conf.DeviceID)
62+
if err != nil {
63+
return fmt.Errorf("failed to detect if VF %s has userspace driver %q", conf.DeviceID, err)
64+
}
65+
if !conf.IsUserspaceDriver {
66+
return fmt.Errorf("the VF %s does not have a interface name or a userspace driver", conf.DeviceID)
67+
}
6668
}
6769

6870
conf.OrigVfState.HostIFName = hostIFName

‎pkg/plugin/plugin.go‎

Lines changed: 26 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -112,20 +112,22 @@ func (p *Plugin) CmdAdd(args *skel.CmdArgs) error {
112112
}
113113

114114
var macAddr string
115-
macAddr, err = p.manager.SetupVF(pluginConf, args.IfName, args.ContainerID, cmdCtx.netNS)
116-
cmdCtx.registerErrorHandler(func() {
117-
netNSErr := cmdCtx.netNS.Do(func(_ ns.NetNS) error {
118-
_, intErr := netlink.LinkByName(args.IfName)
119-
return intErr
115+
if !pluginConf.IsUserspaceDriver {
116+
macAddr, err = p.manager.SetupVF(pluginConf, args.IfName, args.ContainerID, cmdCtx.netNS)
117+
cmdCtx.registerErrorHandler(func() {
118+
netNSErr := cmdCtx.netNS.Do(func(_ ns.NetNS) error {
119+
_, intErr := netlink.LinkByName(args.IfName)
120+
return intErr
121+
})
122+
if netNSErr == nil {
123+
_ = p.manager.ReleaseVF(pluginConf, args.IfName, args.ContainerID, cmdCtx.netNS)
124+
}
120125
})
121-
if netNSErr == nil {
122-
_ = p.manager.ReleaseVF(pluginConf, args.IfName, args.ContainerID, cmdCtx.netNS)
123-
}
124-
})
125126

126-
if err != nil {
127-
return fmt.Errorf("failed to set up pod interface %q from the device %q: %v",
128-
args.IfName, pluginConf.PFName, err)
127+
if err != nil {
128+
return fmt.Errorf("failed to set up pod interface %q from the device %q: %v",
129+
args.IfName, pluginConf.PFName, err)
130+
}
129131
}
130132

131133
// run the IPAM plugin
@@ -205,11 +207,13 @@ func (p *Plugin) configureIPAM(cmdCtx *cmdContext, macAddr string) error {
205207
ipc.Interface = current.Int(0)
206208
}
207209

208-
err = cmdCtx.netNS.Do(func(_ ns.NetNS) error {
209-
return p.ipam.ConfigureIface(args.IfName, newResult)
210-
})
211-
if err != nil {
212-
return err
210+
if !pluginConf.IsUserspaceDriver {
211+
err = cmdCtx.netNS.Do(func(_ ns.NetNS) error {
212+
return p.ipam.ConfigureIface(args.IfName, newResult)
213+
})
214+
if err != nil {
215+
return err
216+
}
213217
}
214218
cmdCtx.result = newResult
215219
return nil
@@ -283,9 +287,11 @@ func (p *Plugin) CmdDel(args *skel.CmdArgs) error {
283287
}
284288
defer netns.Close()
285289

286-
//nolint:gocritic
287-
if err = p.manager.ReleaseVF(pluginConf, args.IfName, args.ContainerID, netns); err != nil {
288-
return err
290+
if !pluginConf.IsUserspaceDriver {
291+
//nolint:gocritic
292+
if err = p.manager.ReleaseVF(pluginConf, args.IfName, args.ContainerID, netns); err != nil {
293+
return err
294+
}
289295
}
290296

291297
//nolint:gocritic

‎pkg/plugin/plugin_test.go‎

Lines changed: 48 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -132,37 +132,49 @@ var _ = Describe("Plugin - test CNI command flows", func() {
132132
})
133133

134134
Describe("CmdAdd", func() {
135-
successfullyParseConfig := func() {
135+
successfullyParseConfig := func(_ bool) {
136136
configMock.On("ParseConf", mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
137137
*args[1].(*localtypes.PluginConf) = *pluginConf
138138
}).Return(nil).Once()
139139
}
140-
successfullyGetNS := func() {
141-
successfullyParseConfig()
140+
successfullyGetNS := func(withDeps bool) {
141+
if withDeps {
142+
successfullyParseConfig(true)
143+
}
142144
nsMock.On("GetNS", testValidNSPath).Return(netNSMock, nil).Once()
143145
netNSMock.On("Path").Return(testValidNSPath).Once()
144146
}
145-
successfullyAttachRepresentor := func() {
146-
successfullyGetNS()
147+
successfullyAttachRepresentor := func(withDeps bool) {
148+
if withDeps {
149+
successfullyGetNS(true)
150+
}
147151
managerMock.On("AttachRepresentor", pluginConf).Return(nil).Once()
148152
}
149-
successfullyApplyVFConfig := func() {
150-
successfullyAttachRepresentor()
153+
successfullyApplyVFConfig := func(withDeps bool) {
154+
if withDeps {
155+
successfullyAttachRepresentor(true)
156+
}
151157
managerMock.On("ApplyVFConfig", pluginConf).Return(nil).Once()
152158
}
153-
successfullySetupVF := func() {
154-
successfullyApplyVFConfig()
159+
successfullySetupVF := func(withDeps bool) {
160+
if withDeps {
161+
successfullyApplyVFConfig(true)
162+
}
155163
managerMock.On("SetupVF",
156164
pluginConf, testValidContIFNames, testValidContainerID, netNSMock).
157165
Return(testValidMAC, nil).Once()
158166
}
159-
successfullyExecAdd := func() {
160-
successfullySetupVF()
167+
successfullyExecAdd := func(withDeps bool) {
168+
if withDeps {
169+
successfullySetupVF(true)
170+
}
161171
ipamMock.On("ExecAdd", pluginConf.IPAM.Type, cmdArgs.StdinData).
162172
Return(getValidIPAMResult(), nil).Once()
163173
}
164-
successfullyConfigureIface := func() {
165-
successfullyExecAdd()
174+
successfullyConfigureIface := func(withDeps bool) {
175+
if withDeps {
176+
successfullyExecAdd(true)
177+
}
166178
ipamMock.On("ConfigureIface", cmdArgs.IfName,
167179
mock.MatchedBy(func(conf *current.Result) bool {
168180
return len(conf.Interfaces) > 0 && conf.Interfaces[0].Mac == testValidMAC
@@ -178,8 +190,10 @@ var _ = Describe("Plugin - test CNI command flows", func() {
178190
cacheMock.On("Save", testValidCacheRef, pluginConf).
179191
Return(nil).Once()
180192
}
181-
successfullySave := func() {
182-
successfullyConfigureIface()
193+
successfullySave := func(withDeps bool) {
194+
if withDeps {
195+
successfullyConfigureIface(true)
196+
}
183197
configureCacheMock()
184198
}
185199
cleanupGetNS := func() {
@@ -205,39 +219,39 @@ var _ = Describe("Plugin - test CNI command flows", func() {
205219
Expect(plugin.CmdAdd(getValidCmdArgs())).To(HaveOccurred())
206220
})
207221
It("Failed to get NS", func() {
208-
successfullyParseConfig()
222+
successfullyParseConfig(true)
209223
nsMock.On("GetNS", testValidNSPath).Return(nil, errTest).Once()
210224
Expect(plugin.CmdAdd(cmdArgs)).To(HaveOccurred())
211225
})
212226
It("Failed to attach representor", func() {
213-
successfullyGetNS()
227+
successfullyGetNS(true)
214228
managerMock.On("AttachRepresentor", pluginConf).Return(errTest).Once()
215229
cleanupGetNS()
216230
Expect(plugin.CmdAdd(cmdArgs)).To(HaveOccurred())
217231
})
218232
It("Failed to ApplyVFConfig", func() {
219-
successfullyAttachRepresentor()
233+
successfullyAttachRepresentor(true)
220234
managerMock.On("ApplyVFConfig", pluginConf).Return(errTest).Once()
221235
cleanupAttachRepresentor()
222236
Expect(plugin.CmdAdd(cmdArgs)).To(HaveOccurred())
223237
})
224238
It("Failed to SetupVF", func() {
225-
successfullyApplyVFConfig()
239+
successfullyApplyVFConfig(true)
226240
managerMock.On("SetupVF",
227241
pluginConf, testValidContIFNames, testValidContainerID, netNSMock).
228242
Return("", errTest).Once()
229243
cleanupSetupVFConfig()
230244
Expect(plugin.CmdAdd(cmdArgs)).To(HaveOccurred())
231245
})
232246
It("Failed to IPAM Add", func() {
233-
successfullySetupVF()
247+
successfullySetupVF(true)
234248
ipamMock.On("ExecAdd", pluginConf.IPAM.Type, cmdArgs.StdinData).
235249
Return(nil, errTest).Once()
236250
cleanupSetupVFConfig()
237251
Expect(plugin.CmdAdd(cmdArgs)).To(HaveOccurred())
238252
})
239253
It("IPAM add returns no IPs", func() {
240-
successfullySetupVF()
254+
successfullySetupVF(true)
241255
result := getValidIPAMResult()
242256
result.(*current.Result).IPs = nil
243257
ipamMock.On("ExecAdd", pluginConf.IPAM.Type, cmdArgs.StdinData).
@@ -246,7 +260,7 @@ var _ = Describe("Plugin - test CNI command flows", func() {
246260
Expect(plugin.CmdAdd(cmdArgs)).To(HaveOccurred())
247261
})
248262
It("Failed to IPAM ConfigureIface", func() {
249-
successfullyExecAdd()
263+
successfullyExecAdd(true)
250264
ipamMock.On("ConfigureIface", cmdArgs.IfName, mock.Anything).
251265
Return(errTest).Once()
252266
netNSMock.On("Do", mock.Anything).Return(func(f func(ns.NetNS) error) error {
@@ -256,7 +270,7 @@ var _ = Describe("Plugin - test CNI command flows", func() {
256270
Expect(plugin.CmdAdd(cmdArgs)).To(HaveOccurred())
257271
})
258272
It("Failed save cache", func() {
259-
successfullyConfigureIface()
273+
successfullyConfigureIface(true)
260274
cacheMock.On("GetStateRef", pluginConf.Name, cmdArgs.ContainerID, cmdArgs.IfName).
261275
Return(testValidCacheRef).Once()
262276
cacheMock.On("Save", testValidCacheRef, pluginConf).
@@ -267,24 +281,32 @@ var _ = Describe("Plugin - test CNI command flows", func() {
267281
})
268282
Context("Successful scenarios", func() {
269283
It("with IPAM", func() {
270-
successfullySave()
284+
successfullySave(true)
271285
cleanupGetNS()
272286
Expect(plugin.CmdAdd(cmdArgs)).ToNot(HaveOccurred())
273287
})
274288
It("no IPAM", func() {
275289
pluginConf.IPAM = types.IPAM{}
276-
successfullySetupVF()
290+
successfullySetupVF(true)
277291
configureCacheMock()
278292
cleanupGetNS()
279293
Expect(plugin.CmdAdd(cmdArgs)).ToNot(HaveOccurred())
280294
})
295+
It("userspace driver", func() {
296+
pluginConf.IsUserspaceDriver = true
297+
successfullyApplyVFConfig(true)
298+
successfullyExecAdd(false)
299+
successfullySave(false)
300+
cleanupGetNS()
301+
Expect(plugin.CmdAdd(cmdArgs)).ToNot(HaveOccurred())
302+
})
281303
})
282304
Context("MAC address configuration", func() {
283305

284306
var updatedPluginConf *localtypes.PluginConf
285307

286308
JustBeforeEach(func() {
287-
successfullyGetNS()
309+
successfullyGetNS(true)
288310
cleanupGetNS()
289311
// workaround to access pluginConf
290312
managerMock.On("AttachRepresentor", mock.Anything).Run(func(args mock.Arguments) {

‎pkg/types/types.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ type NetConf struct {
4242
// PluginConf is a internal representation of config options and state
4343
type PluginConf struct {
4444
NetConf
45+
// IsUserspaceDriver indicate that VF using userspace driver
46+
IsUserspaceDriver bool
4547
// Stores the original VF state as it was prior to any operations done during cmdAdd flow
4648
OrigVfState VfState `json:"orig_vf_state"`
4749
// Name of the PF to which VF belongs

‎pkg/utils/testing.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,10 @@ var ts = tmpSysFs{
3636
"sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.1/net/enp175s7",
3737
"sys/devices/pci0000:00/0000:00:02.0/0000:05:00.0/net/ens1",
3838
"sys/devices/pci0000:00/0000:00:02.0/0000:05:00.0/net/ens1d1",
39+
"sys/bus/pci/devices/0000:11:00.0",
40+
"sys/bus/pci/devices/0000:12:00.0",
41+
"sys/bus/pci/drivers/mlx5_core",
42+
"sys/bus/pci/drivers/vfio-pci",
3943
},
4044
fileList: map[string][]byte{
4145
"sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1/sriov_numvfs": []byte("2"),
@@ -66,6 +70,8 @@ var ts = tmpSysFs{
6670

6771
"sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1/virtfn1": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.1",
6872
"sys/devices/pci0000:ae/0000:ae:00.0/0000:af:06.1/physfn": "sys/devices/pci0000:ae/0000:ae:00.0/0000:af:00.1",
73+
"sys/bus/pci/devices/0000:11:00.0/driver": "sys/bus/pci/drivers/vfio-pci",
74+
"sys/bus/pci/devices/0000:12:00.0/driver": "sys/bus/pci/drivers/mlx5_core",
6975
},
7076
}
7177

‎pkg/utils/utils.go‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ var (
1515
NetDirectory = "/sys/class/net"
1616
// SysBusPci is sysfs pci device directory
1717
SysBusPci = "/sys/bus/pci/devices"
18+
// UserspaceDrivers is a list of driver names that don't have netlink representation for their devices
19+
UserspaceDrivers = []string{"vfio-pci"}
1820
)
1921

2022
// GetSriovNumVfs takes in a PF name(ifName) as string and returns number of VF configured as int
@@ -112,3 +114,23 @@ func GetVFLinkName(pciAddr string) (string, error) {
112114

113115
return names[0], nil
114116
}
117+
118+
// HasUserspaceDriver checks if a device is attached to userspace driver
119+
func HasUserspaceDriver(pciAddr string) (bool, error) {
120+
driverLink := filepath.Join(SysBusPci, pciAddr, "driver")
121+
driverPath, err := filepath.EvalSymlinks(driverLink)
122+
if err != nil {
123+
return false, err
124+
}
125+
driverStat, err := os.Stat(driverPath)
126+
if err != nil {
127+
return false, err
128+
}
129+
driverName := driverStat.Name()
130+
for _, drv := range UserspaceDrivers {
131+
if driverName == drv {
132+
return true, nil
133+
}
134+
}
135+
return false, nil
136+
}

‎pkg/utils/utils_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,4 +41,16 @@ var _ = Describe("Utils", func() {
4141
Expect(err).To(HaveOccurred(), "Not existing VF should return an error")
4242
})
4343
})
44+
Context("Checking HasUserspaceDriver function", func() {
45+
It("Use userspace driver", func() {
46+
result, err := HasUserspaceDriver("0000:11:00.0")
47+
Expect(err).NotTo(HaveOccurred(), "HasUserspaceDriver should not return an error")
48+
Expect(result).To(BeTrue(), "HasUserspaceDriver should return true")
49+
})
50+
It("Has not userspace driver", func() {
51+
result, err := HasUserspaceDriver("0000:12:00.0")
52+
Expect(result).To(BeFalse())
53+
Expect(err).NotTo(HaveOccurred(), "HasUserspaceDriver should not return an error")
54+
})
55+
})
4456
})

0 commit comments

Comments
 (0)