Merge pull request #35 from adrianchiris/extend-sriovnet

Extend sriovnet with Functionality for Smart-NICs

Author: moshe010

go.mod

@@ -4,8 +4,7 @@ go 1.13
 
 require (
 	github.com/google/uuid v1.1.1
-	github.com/spf13/afero v1.2.2
+	github.com/spf13/afero v1.4.1
 	github.com/stretchr/testify v1.5.1
 	github.com/vishvananda/netlink v1.1.0
-	golang.org/x/text v0.3.3 // indirect
 )

go.sum

@@ -2,17 +2,26 @@ github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
-github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ=
+github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444 h1:/d2cWp6PSamH4jDPFLyO150psQdqvtoNX8Zjg3AQ31g=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=

pkg/utils/filesystem/defaultfs.go

@@ -49,6 +49,16 @@ func (DefaultFs) Remove(name string) error {
 	return os.Remove(name)
 }
 
+// Readlink via os.Readlink
+func (DefaultFs) Readlink(name string) (string, error) {
+	return os.Readlink(name)
+}
+
+// Symlink via os.Symlink
+func (DefaultFs) Symlink(oldname, newname string) error {
+	return os.Symlink(oldname, newname)
+}
+
 // ReadFile via ioutil.ReadFile
 func (DefaultFs) ReadFile(filename string) ([]byte, error) {
 	return ioutil.ReadFile(filename)

pkg/utils/filesystem/fakefs.go

@@ -1,6 +1,8 @@
+//nolint:gomnd
 package filesystem
 
 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"time"
@@ -13,9 +15,31 @@ type fakeFs struct {
 	a afero.Afero
 }
 
-// NewFakeFs returns a fake Filesystem that exists in-memory, useful for unit tests
-func NewFakeFs() Filesystem {
-	return &fakeFs{a: afero.Afero{Fs: afero.NewMemMapFs()}}
+// NewFakeFs returns a fake Filesystem that exists at fakeFsRoot as its base path, useful for unit tests.
+// Returns: Filesystem interface, teardown method (cleanup of provided root path) and error.
+// teardown method should be called at the end of each test to ensure environment is left clean.
+func NewFakeFs(fakeFsRoot string) (Filesystem, func(), error) {
+	_, err := os.Stat(fakeFsRoot)
+	// if fakeFsRoot dir exists remove it.
+	if err == nil {
+		err = os.RemoveAll(fakeFsRoot)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to cleanup fake root dir %s. %s", fakeFsRoot, err)
+		}
+	} else if !os.IsNotExist(err) {
+		return nil, nil, fmt.Errorf("failed to lstat fake root dir %s. %s", fakeFsRoot, err)
+	}
+
+	// create fakeFsRoot dir
+	if err = os.MkdirAll(fakeFsRoot, os.FileMode(0755)); err != nil {
+		return nil, nil, fmt.Errorf("failed to create fake root dir: %s. %s", fakeFsRoot, err)
+	}
+
+	return &fakeFs{a: afero.Afero{Fs: afero.NewBasePathFs(afero.NewOsFs(), fakeFsRoot)}},
+		func() {
+			os.RemoveAll(fakeFsRoot)
+		},
+		nil
 }
 
 // Stat via afero.Fs.Stat
@@ -86,6 +110,16 @@ func (fs *fakeFs) Remove(name string) error {
 	return fs.a.Remove(name)
 }
 
+// Readlink via afero.ReadlinkIfPossible
+func (fs *fakeFs) Readlink(name string) (string, error) {
+	return fs.a.Fs.(afero.Symlinker).ReadlinkIfPossible(name)
+}
+
+// Symlink via afero.FS.(Symlinker).SymlinkIfPossible
+func (fs *fakeFs) Symlink(oldname, newname string) error {
+	return fs.a.Fs.(afero.Symlinker).SymlinkIfPossible(oldname, newname)
+}
+
 // fakeFile implements File; for use with fakeFs
 type fakeFile struct {
 	file afero.File

pkg/utils/filesystem/filesystem.go

@@ -18,6 +18,8 @@ type Filesystem interface {
 	Chtimes(name string, atime time.Time, mtime time.Time) error
 	RemoveAll(path string) error
 	Remove(name string) error
+	Readlink(name string) (string, error)
+	Symlink(oldname, newname string) error
 
 	// from "io/ioutil"
 	ReadFile(filename string) ([]byte, error)

sriovnet.go

@@ -5,6 +5,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strconv"
@@ -437,3 +438,18 @@ func GetNetDevicesFromPci(pciAddress string) ([]string, error) {
 	}
 	return netDevices, nil
 }
+
+// GetPfPciFromVfPci retrieves the parent PF PCI address of the provided VF PCI address in D:B:D.f format
+func GetPfPciFromVfPci(vfPciAddress string) (string, error) {
+	pfPath := filepath.Join(PciSysDir, vfPciAddress, "physfn")
+	pciDevDir, err := utilfs.Fs.Readlink(pfPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to read physfn link, provided address may not be a VF. %v", err)
+	}
+
+	pf := path.Base(pciDevDir)
+	if pf == "" {
+		return pf, fmt.Errorf("could not find PF PCI Address")
+	}
+	return pf, err
+}

sriovnet_integration_test.go

@@ -1,5 +1,19 @@
 // +build integration
 
+/*
+NOTE:
+====
+This test although not usable as is, since netdev names and configuration differs from one setup to the next
+is useful for testing your changes on a real setup.
+
+for new functionality in sriovnet package add a new integration test case.
+to run an existing test modify the netdev/VF in the set to fit your setup and execute the test.
+
+Build and run integration test:
+==============================
+# go test --tags integration -v -run <TestName>
+ */
+
 package sriovnet
 
 import (
@@ -201,3 +215,25 @@ func TestGetVfNetdevName(t *testing.T) {
 		}
 	}
 }
+
+func TestIntegrationGetPfPciFromVfPci(t *testing.T) {
+	vf := "0000:05:00.6"
+	pf, err := GetPfPciFromVfPci(vf)
+	if err != nil {
+		t.Log("GetPfPciFromVfPci", "VF PCI: ", vf, "Error: ", err)
+		t.Fatal()
+	}
+	t.Log("VF: ", vf, "PF: ", pf)
+}
+
+func TestIntegrationGetVfRepresentorSmartNIC(t *testing.T) {
+	pfID := "0"
+	vfIdx := "2"
+	t.Log("GetVfRepresentorSmartNIC ", "PF ID: ", pfID, "VF Index: ", vfIdx)
+	rep, err := GetVfRepresentorSmartNIC(pfID, vfIdx)
+	if err != nil {
+		t.Log("GetVfRepresentorSmartNIC ", "Error: ", err)
+		t.Fatal()
+	}
+	t.Log("VF Representor: ", rep)
+}

sriovnet_switchdev.go

@@ -129,3 +129,47 @@ func getNetDevPhysPortName(netDev string) (string, error) {
 	}
 	return strings.TrimSpace(string(physPortName)), nil
 }
+
+// GetVfRepresentorSmartNIC returns VF representor on Smart-NIC for a host VF identified by pfID and vfIndex
+func GetVfRepresentorSmartNIC(pfID, vfIndex string) (string, error) {
+	// TODO(Adrianc): This method should change to get switchID and vfIndex as input, then common logic can
+	// be shared with GetVfRepresentor, backward compatibility should be preserved when this happens.
+
+	// pfID should be 0 or 1
+	if pfID != "0" && pfID != "1" {
+		return "", fmt.Errorf("unexpected pfID(%s). It should be 0 or 1", pfID)
+	}
+
+	// vfIndex should be an unsinged integer provided as a decimal number
+	if _, err := strconv.ParseUint(vfIndex, 10, 32); err != nil {
+		return "", fmt.Errorf("unexpected vfIndex(%s). It should be an unsigned decimal number", vfIndex)
+	}
+
+	netdevs, err := utilfs.Fs.ReadDir(NetSysDir)
+	if err != nil {
+		return "", err
+	}
+
+	// map for easy search of expected VF rep port name.
+	// Note: no supoport for Multi-Chassis Smart-NICs
+	expectedPhysPortNames := map[string]interface{}{
+		fmt.Sprintf("pf%svf%s", pfID, vfIndex):   nil,
+		fmt.Sprintf("c0pf%svf%s", pfID, vfIndex): nil,
+	}
+
+	// iterate all net devs and get phys port name
+	// if phys port name == pf<pfIndex>vf<vfIndex> or c0pf<pfIndex>vf<vfIndex> we have a match
+	for _, netdev := range netdevs {
+		// find matching VF representor
+		netdevName := netdev.Name()
+		portName, err := getNetDevPhysPortName(netdevName)
+		if err != nil {
+			// skip
+			continue
+		}
+		if _, ok := expectedPhysPortNames[portName]; ok {
+			return netdevName, nil
+		}
+	}
+	return "", fmt.Errorf("vf representor for pfID:%s, vfIndex: %s not found", pfID, vfIndex)
+}

sriovnet_switchdev_test.go

@@ -18,14 +18,16 @@ type repContext struct {
 }
 
 func setUpRepresentorLayout(vfPciAddress string, rep *repContext) error {
-	path := filepath.Join(PciSysDir, vfPciAddress, "physfn/net", rep.Name)
-	err := utilfs.Fs.MkdirAll(path, os.FileMode(0755))
-	if err != nil {
-		return err
+	if vfPciAddress != "" {
+		path := filepath.Join(PciSysDir, vfPciAddress, "physfn/net", rep.Name)
+		err := utilfs.Fs.MkdirAll(path, os.FileMode(0755))
+		if err != nil {
+			return err
+		}
 	}
 
-	path = filepath.Join(NetSysDir, rep.Name)
-	err = utilfs.Fs.MkdirAll(path, os.FileMode(0755))
+	path := filepath.Join(NetSysDir, rep.Name)
+	err := utilfs.Fs.MkdirAll(path, os.FileMode(0755))
 	if err != nil {
 		return err
 	}
@@ -54,19 +56,38 @@ func setUpRepresentorLayout(vfPciAddress string, rep *repContext) error {
 //nolint:unparam
 func setupUplinkRepresentorEnv(t *testing.T, uplink *repContext, vfPciAddress string, vfReps []*repContext) func() {
 	var err error
-	utilfs.Fs = utilfs.NewFakeFs()
+	teardown := setupRepresentorEnv(t, vfPciAddress, vfReps)
 	defer func() {
 		if err != nil {
+			teardown()
 			t.Errorf("setupUplinkRepresentorEnv, got %v", err)
 		}
 	}()
-
+	// Setup uplink
 	err = setUpRepresentorLayout(vfPciAddress, uplink)
+	if err != nil {
+		return nil
+	}
+
+	return teardown
+}
+
+func setupRepresentorEnv(t *testing.T, vfPciAddress string, vfReps []*repContext) func() {
+	var err error
+	teardown := setupFakeFs(t)
+
+	defer func() {
+		if err != nil {
+			teardown()
+			t.Errorf("setupRepresentorEnv, got %v", err)
+		}
+	}()
+
 	for _, rep := range vfReps {
 		err = setUpRepresentorLayout(vfPciAddress, rep)
 	}
 
-	return func() { utilfs.Fs.RemoveAll("/") } //nolint:errcheck
+	return teardown
 }
 
 func TestGetUplinkRepresentorWithPhysPortNameSuccess(t *testing.T) {
@@ -153,3 +174,62 @@ func TestGetUplinkRepresentorErrorMissingUplink(t *testing.T) {
 	assert.Equal(t, "", uplinkNetdev)
 	assert.Contains(t, err.Error(), expectedError)
 }
+
+func TestGetVfRepresentorSmartNIC(t *testing.T) {
+	vfReps := []*repContext{
+		{
+			Name:         "eth0",
+			PhysPortName: "pf0vf0",
+			PhysSwitchID: "c2cfc60003a1420c",
+		},
+		{
+			Name:         "eth1",
+			PhysPortName: "pf0vf1",
+			PhysSwitchID: "c2cfc60003a1420c",
+		},
+		{
+			Name:         "eth2",
+			PhysPortName: "pf0vf2",
+			PhysSwitchID: "c2cfc60003a1420c",
+		},
+	}
+	teardown := setupRepresentorEnv(t, "", vfReps)
+	defer teardown()
+
+	vfRep, err := GetVfRepresentorSmartNIC("0", "2")
+	assert.NoError(t, err)
+	assert.Equal(t, "eth2", vfRep)
+}
+
+func TestGetVfRepresentorSmartNICNoRep(t *testing.T) {
+	vfReps := []*repContext{
+		{
+			Name:         "eth0",
+			PhysPortName: "pf0vf0",
+			PhysSwitchID: "c2cfc60003a1420c",
+		},
+		{
+			Name:         "eth1",
+			PhysPortName: "pf0vf1",
+			PhysSwitchID: "c2cfc60003a1420c",
+		},
+	}
+	teardown := setupRepresentorEnv(t, "", vfReps)
+	defer teardown()
+
+	vfRep, err := GetVfRepresentorSmartNIC("1", "2")
+	assert.Error(t, err)
+	assert.Equal(t, "", vfRep)
+}
+
+func TestGetVfRepresentorSmartNICInvalidPfID(t *testing.T) {
+	vfRep, err := GetVfRepresentorSmartNIC("invalid", "2")
+	assert.Error(t, err)
+	assert.Equal(t, "", vfRep)
+}
+
+func TestGetVfRepresentorSmartNICInvalidVfIndex(t *testing.T) {
+	vfRep, err := GetVfRepresentorSmartNIC("1", "invalid")
+	assert.Error(t, err)
+	assert.Equal(t, "", vfRep)
+}