Skip to content

Commit 798d4ce

Browse files
committed
Add network interface management to vbmctl
Signed-off-by: Nuutti Hakala <nuutti.hakala@est.tech>
1 parent 7f3d82a commit 798d4ce

9 files changed

Lines changed: 215 additions & 25 deletions

File tree

hack/ci-e2e.sh

Lines changed: 5 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,6 @@ sudo sysctl fs.inotify.max_user_instances=8192
6969
# Build the container image with e2e tag (used in tests)
7070
IMG=quay.io/metal3-io/baremetal-operator IMG_TAG=e2e make docker
7171

72-
# Build vbmctl
73-
make build-vbmctl
74-
# Create VMs to act as BMHs in the tests and the libvirt network
75-
./bin/vbmctl -c "${REPO_ROOT}/test/e2e/config/vbmctl.yaml" create bml
7672

7773
# We need to create veth pair to connect metal3 net (defined above with vbmctl)
7874
# and kind docker subnet. Let us start by creating a docker network with
@@ -91,22 +87,11 @@ if ! docker network list | grep kind; then
9187
fi
9288
docker network list
9389

94-
# Next create the veth pair
95-
if ! ip a | grep metalend; then
96-
sudo ip link add metalend type veth peer name kindend
97-
sudo ip link set metalend master metal3
98-
sudo ip link set kindend master kind-bridge
99-
sudo ip link set metalend up
100-
sudo ip link set kindend up
101-
fi
102-
ip a
103-
104-
# Then we need to set routing rules as well
105-
if ! sudo iptables -L FORWARD -v -n | grep kind-bridge; then
106-
sudo iptables -I FORWARD -i kind-bridge -o metal3 -j ACCEPT
107-
sudo iptables -I FORWARD -i metal3 -o kind-bridge -j ACCEPT
108-
fi
109-
sudo iptables -L FORWARD -n -v
90+
# Build vbmctl
91+
make build-vbmctl
92+
sudo setcap cap_net_admin+epi ./bin/vbmctl
93+
# Create VMs to act as BMHs in the tests and the libvirt network
94+
./bin/vbmctl -c "${REPO_ROOT}/test/e2e/config/vbmctl.yaml" create bml
11095

11196
# This IP is defined by the network we created above. It is sushy-tools / image
11297
# server endpoint, not ironic.

test/e2e/config/vbmctl.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,8 @@ spec:
3434
bridge: metal3
3535
address: 192.168.222.1
3636
netmask: 255.255.255.0
37+
vethPairs:
38+
- master1: metal3
39+
master2: kind-bridge
40+
veth1: metalend
41+
veth2: kindend

test/go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ require (
1212
github.com/onsi/ginkgo/v2 v2.28.1
1313
github.com/onsi/gomega v1.39.1
1414
github.com/spf13/cobra v1.10.2
15+
github.com/vishvananda/netlink v1.3.0
1516
golang.org/x/crypto v0.50.0
1617
gopkg.in/yaml.v2 v2.4.0
1718
k8s.io/api v0.35.4
@@ -102,6 +103,7 @@ require (
102103
github.com/spf13/pflag v1.0.10 // indirect
103104
github.com/spf13/viper v1.21.0 // indirect
104105
github.com/subosito/gotenv v1.6.0 // indirect
106+
github.com/vishvananda/netns v0.0.5 // indirect
105107
github.com/x448/float16 v0.8.4 // indirect
106108
github.com/xlab/treeprint v1.2.0 // indirect
107109
go.opentelemetry.io/auto/sdk v1.2.1 // indirect

test/go.sum

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,11 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
259259
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
260260
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
261261
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
262+
github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk=
263+
github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs=
264+
github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
265+
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
266+
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
262267
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
263268
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
264269
github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
@@ -305,7 +310,9 @@ golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
305310
golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
306311
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
307312
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
313+
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
308314
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
315+
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
309316
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
310317
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
311318
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=

test/vbmctl/README.md

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ This tool is under active development.
1717
| `vbmctl create bml` / `vbmctl delete bml` | ✅ Implemented |
1818
| `vbmctl status` | ✅ Implemented (basic) |
1919
| Configurable volumes | ✅ Implemented |
20-
| Network management | ⚠️ Partially implemented (only libvirt networks) |
20+
| Network management | ⚠️ Partially implemented (container networking missing) |
2121
| BMC emulator support | ❌ TODO |
2222
| Image server | ✅ Implemented (basic) |
2323
| State management (persistent state) | ❌ TODO |
@@ -31,6 +31,8 @@ This tool is under active development.
3131
- **Library Support**: Can be imported as a Go module for programmatic use
3232
- **Libvirt network management**: create and delete libvirt networks
3333
- **Image Server Management**: Create, delete, list image server
34+
- **Creating/deleting veth pairs**: create and delete veth-pairs to connect
35+
virtual networks. Works only with config file.
3436

3537
## Build Tags
3638

@@ -168,6 +170,11 @@ spec:
168170
imageServer:
169171
dataDir: "/tmp"
170172
port: 80
173+
vethPairs:
174+
- master1: metal3
175+
master2: kind-bridge
176+
veth1: metalend
177+
veth2: kindend
171178
```
172179
173180
The `spec.vms` section defines the VMs that will be created when you run `vbmctl

test/vbmctl/cmd/vbmctl/main.go

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717
"github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/config"
1818
containers "github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/containers"
1919
"github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/libvirt"
20+
"github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/network"
2021
"github.com/spf13/cobra"
2122
libvirtgo "libvirt.org/go/libvirt"
2223
)
@@ -201,16 +202,21 @@ Example configuration:
201202
networkAttachments:
202203
- network: "baremetal-e2e"
203204
macAddress: "00:60:2f:31:81:01"
204-
networks:
205+
networks:
205206
- name: "baremetal-e2e"
206-
- bridge: "metal3"
207+
bridge: "metal3"
207208
imageServer:
208209
image: "nginxinc/nginx-unprivileged"
209210
port: 8080
210211
containerPort: 8080
211212
dataDir: "/var/lib/vbmctl/images"
212213
containerDataDir: "/usr/share/nginx/html",
213-
containerName: "vbmctl-image-server"`,
214+
containerName: "vbmctl-image-server"
215+
vethPairs:
216+
- master1: "metal3"
217+
master2: "kind-bridge"
218+
veth1: "metalend"
219+
veth2: "kindend"`,
214220
RunE: func(_ *cobra.Command, _ []string) error {
215221
ctx, cancel := contextWithSignal()
216222
defer cancel()
@@ -256,6 +262,18 @@ Example configuration:
256262
fmt.Printf(" - %s (UUID: %s)\n", network.Name, network.UUID)
257263
}
258264

265+
// Connect the specified networks
266+
err = network.ConnectAllWithVeth(ctx, cfg.Spec.VethPairs)
267+
if err != nil {
268+
return fmt.Errorf("failed to create veth pairs: %w", err)
269+
}
270+
//nolint:forbidigo // CLI output is intentional
271+
fmt.Println("\nCreated veth pairs:")
272+
for _, pair := range cfg.Spec.VethPairs {
273+
//nolint:forbidigo // CLI output is intentional
274+
fmt.Printf(" - between %s and %s\n", pair.Master1, pair.Master2)
275+
}
276+
259277
vmManager, err := libvirt.NewVMManager(conn, libvirt.VMManagerOptions{
260278
PoolName: cfg.Spec.Pool.Name,
261279
PoolPath: cfg.Spec.Pool.Path,
@@ -529,9 +547,24 @@ func newDeleteBMLCmd() *cobra.Command {
529547
fmt.Printf(" - %s\n", name)
530548
}
531549

550+
// Delete veth pairs
551+
err = network.DeleteAllVeth(ctx, cfg.Spec.VethPairs)
552+
if err != nil {
553+
// Don't fail whole command if veth deletion fails
554+
//nolint:forbidigo // CLI output is intentional
555+
fmt.Printf("failed to delete veth pairs: %v", err)
556+
} else {
557+
//nolint:forbidigo // CLI output is intentional
558+
fmt.Println("\nDeleted veth pairs:")
559+
for _, pair := range cfg.Spec.VethPairs {
560+
//nolint:forbidigo // CLI output is intentional
561+
fmt.Printf(" - between %s and %s\n", pair.Master1, pair.Master2)
562+
}
563+
}
564+
532565
networkManager, err := libvirt.NewNetworkManager(conn)
533566
if err != nil {
534-
return fmt.Errorf("failed to create Network manager: %w", err)
567+
return fmt.Errorf("failed to create libvirt network manager: %w", err)
535568
}
536569

537570
networks := make([]string, len(cfg.Spec.Networks))

test/vbmctl/pkg/api/types.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,20 @@ type NetworkConfig struct {
7878
Netmask string `json:"netmask,omitempty" yaml:"netmask,omitempty"`
7979
}
8080

81+
type VethPair struct {
82+
// Master 1 is the name of the first network interface to be connected
83+
Master1 string `json:"master1" yaml:"master1"`
84+
85+
// Master 2 is the name of the second network interface to be connected
86+
Master2 string `json:"master2" yaml:"master2"`
87+
88+
// Veth1 is the name of the veth pair to be pushed under Master1
89+
Veth1 string `json:"veth1" yaml:"veth1"`
90+
91+
// Veth2 is the name of the veth pair to be pushed under Master2
92+
Veth2 string `json:"veth2" yaml:"veth2"`
93+
}
94+
8195
// PoolConfig represents the configuration for a storage pool.
8296
type PoolConfig struct {
8397
// Name is the name of the storage pool.

test/vbmctl/pkg/config/config.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,9 @@ type Spec struct {
101101

102102
// ImageServer contains configuration for the image server.
103103
ImageServer *vbmctlapi.ImageServerConfig `json:"imageServer,omitempty" yaml:"imageServer,omitempty"`
104+
105+
// VethPairs is a list of interfaces that should be connected with a veth-pair.
106+
VethPairs []vbmctlapi.VethPair `json:"vethPairs,omitempty" yaml:"vethPairs,omitempty"`
104107
}
105108

106109
// LibvirtConfig contains libvirt connection settings.

test/vbmctl/pkg/network/ip.go

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
//go:build vbmctl
2+
// +build vbmctl
3+
4+
package network
5+
6+
import (
7+
"context"
8+
"errors"
9+
"fmt"
10+
"log"
11+
12+
vbmctlapi "github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/api"
13+
"github.com/vishvananda/netlink"
14+
)
15+
16+
var ErrVethExistsWithWrongParams = errors.New("veth interface exists with different parameters")
17+
18+
func ConnectWithVeth(_ context.Context, network1 string, network2 string, vethpeer1 string, vethpeer2 string) error {
19+
// Get masters
20+
master1, err := netlink.LinkByName(network1)
21+
if err != nil {
22+
return fmt.Errorf("failed to get network interface %s: %w", network1, err)
23+
}
24+
master2, err := netlink.LinkByName(network2)
25+
if err != nil {
26+
return fmt.Errorf("failed to get network interface %s: %w", network2, err)
27+
}
28+
29+
// Check if pair exists and has correct masters
30+
veth1, err1 := netlink.LinkByName(vethpeer1)
31+
veth2, err2 := netlink.LinkByName(vethpeer2)
32+
if err1 == nil && err2 == nil {
33+
// Return error if the masters is wrong. Someone has created and
34+
// configured the interfaces already, and we don't want to break their
35+
// configuration by modifying the interfaces.
36+
if veth1.Attrs().MasterIndex != master1.Attrs().Index {
37+
return ErrVethExistsWithWrongParams
38+
}
39+
if veth2.Attrs().MasterIndex != master2.Attrs().Index {
40+
return ErrVethExistsWithWrongParams
41+
}
42+
// Veth pair exists and has correct masters
43+
return nil
44+
}
45+
46+
// Check for unknown errors
47+
var notFound netlink.LinkNotFoundError
48+
if !errors.As(err1, &notFound) || !errors.As(err2, &notFound) {
49+
return fmt.Errorf("error checking interfaces: %w AND %w", err1, err2)
50+
}
51+
52+
la := netlink.NewLinkAttrs()
53+
la.Name = vethpeer1
54+
veth := &netlink.Veth{
55+
LinkAttrs: la,
56+
PeerName: vethpeer2,
57+
}
58+
err = netlink.LinkAdd(veth)
59+
if err != nil {
60+
return fmt.Errorf("could not add vethpair %s: %w", la.Name, err)
61+
}
62+
63+
// Get the newly created interfaces and configure them
64+
veth1, err = netlink.LinkByName(vethpeer1)
65+
if err != nil {
66+
return fmt.Errorf("failed to get veth interface %s: %w", vethpeer1, err)
67+
}
68+
veth2, err = netlink.LinkByName(vethpeer2)
69+
if err != nil {
70+
return fmt.Errorf("failed to get veth interface %s: %w", vethpeer2, err)
71+
}
72+
73+
if err := netlink.LinkSetUp(veth1); err != nil {
74+
return fmt.Errorf("failed to bring up veth interface %s: %w", vethpeer1, err)
75+
}
76+
if err := netlink.LinkSetUp(veth2); err != nil {
77+
return fmt.Errorf("failed to bring up veth interface %s: %w", vethpeer2, err)
78+
}
79+
80+
if err := netlink.LinkSetMaster(veth1, master1); err != nil {
81+
return fmt.Errorf("failed to set master %s for veth %s: %w", network1, vethpeer1, err)
82+
}
83+
if err := netlink.LinkSetMaster(veth2, master2); err != nil {
84+
return fmt.Errorf("failed to set master %s for veth %s: %w", network2, vethpeer2, err)
85+
}
86+
return nil
87+
}
88+
89+
func ConnectAllWithVeth(ctx context.Context, vethPairs []vbmctlapi.VethPair) error {
90+
createdPairs := make([]vbmctlapi.VethPair, 0, len(vethPairs))
91+
for _, pair := range vethPairs {
92+
err := ConnectWithVeth(ctx, pair.Master1, pair.Master2, pair.Veth1, pair.Veth2)
93+
if err != nil {
94+
// Clean up previously created pairs
95+
log.Printf("Failed to create veth pair %s, cleaning up %d previously created pair(s)\n", pair.Veth1, len(createdPairs))
96+
for _, created := range createdPairs {
97+
if delErr := DeleteLink(ctx, created.Veth1); delErr != nil {
98+
log.Printf("Warning: failed to clean up veth pair %s: %v\n", created.Veth1, delErr)
99+
}
100+
}
101+
return fmt.Errorf("failed to create veth pair %s: %w", pair.Veth1, err)
102+
}
103+
createdPairs = append(createdPairs, pair)
104+
}
105+
return nil
106+
}
107+
108+
func DeleteLink(_ context.Context, link string) error {
109+
l, err := netlink.LinkByName(link)
110+
var notFound netlink.LinkNotFoundError
111+
if !errors.As(err, &notFound) {
112+
log.Printf("cannot delete network interface, interface %s does not exist", link)
113+
return nil
114+
}
115+
if err != nil {
116+
return fmt.Errorf("failed to get network interface %s: %w", link, err)
117+
}
118+
119+
if err := netlink.LinkDel(l); err != nil {
120+
return fmt.Errorf("failed to delete network interface %s: %w", link, err)
121+
}
122+
return nil
123+
}
124+
125+
func DeleteAllVeth(ctx context.Context, vethPairs []vbmctlapi.VethPair) error {
126+
var lastErr error
127+
for _, pair := range vethPairs {
128+
if err := DeleteLink(ctx, pair.Veth1); err != nil {
129+
log.Printf("Error deleting veth pair %s: %v\n", pair.Veth1, err)
130+
lastErr = err
131+
}
132+
}
133+
return lastErr
134+
}

0 commit comments

Comments
 (0)