Skip to content

Commit 1dab330

Browse files
committed
Add ip and iptables management to vbmctl
Signed-off-by: Nuutti Hakala <nuutti.hakala@est.tech>
1 parent d846d68 commit 1dab330

10 files changed

Lines changed: 253 additions & 22 deletions

File tree

hack/ci-e2e.sh

Lines changed: 4 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -69,10 +69,6 @@ sudo sysctl fs.inotify.max_user_instances=8192
6969
# Build the container image with e2e tag (used in tests)
7070
IMG=quay.io/metal3-io/baremetal-operator IMG_TAG=e2e make docker
7171

72-
# Build vbmctl
73-
make build-vbmctl
74-
# Create VMs to act as BMHs in the tests and the libvirt network
75-
./bin/vbmctl -c "${REPO_ROOT}/test/e2e/config/vbmctl.yaml" create bml
7672

7773
# We need to create veth pair to connect metal3 net (defined above with vbmctl)
7874
# and kind docker subnet. Let us start by creating a docker network with
@@ -91,22 +87,10 @@ if ! docker network list | grep kind; then
9187
fi
9288
docker network list
9389

94-
# Next create the veth pair
95-
if ! ip a | grep metalend; then
96-
sudo ip link add metalend type veth peer name kindend
97-
sudo ip link set metalend master metal3
98-
sudo ip link set kindend master kind-bridge
99-
sudo ip link set metalend up
100-
sudo ip link set kindend up
101-
fi
102-
ip a
103-
104-
# Then we need to set routing rules as well
105-
if ! sudo iptables -L FORWARD -v -n | grep kind-bridge; then
106-
sudo iptables -I FORWARD -i kind-bridge -o metal3 -j ACCEPT
107-
sudo iptables -I FORWARD -i metal3 -o kind-bridge -j ACCEPT
108-
fi
109-
sudo iptables -L FORWARD -n -v
90+
# Build vbmctl
91+
make build-vbmctl
92+
# Create VMs to act as BMHs in the tests and the libvirt network
93+
./bin/vbmctl -c "${REPO_ROOT}/test/e2e/config/vbmctl.yaml" create bml
11094

11195
# This IP is defined by the network we created above. It is sushy-tools / image
11296
# server endpoint, not ironic.

test/e2e/config/vbmctl.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,3 +34,8 @@ spec:
3434
bridge: metal3
3535
address: 192.168.222.1
3636
netmask: 255.255.255.0
37+
vethPairs:
38+
- master1: metal3
39+
master2: kind-bridge
40+
veth1: metalend
41+
veth2: kindend

test/go.mod

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@ go 1.25.0
44

55
require (
66
github.com/cert-manager/cert-manager v1.20.1
7+
github.com/coreos/go-iptables v0.8.0
78
github.com/metal3-io/baremetal-operator/apis v0.5.1
89
github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.5.1
910
github.com/metal3-io/ironic-standalone-operator/api v0.8.1
1011
github.com/onsi/ginkgo/v2 v2.28.1
1112
github.com/onsi/gomega v1.39.1
1213
github.com/spf13/cobra v1.10.2
14+
github.com/vishvananda/netlink v1.3.1
1315
golang.org/x/crypto v0.49.0
1416
gopkg.in/yaml.v2 v2.4.0
1517
k8s.io/api v0.35.3
@@ -104,6 +106,7 @@ require (
104106
github.com/spf13/pflag v1.0.10 // indirect
105107
github.com/spf13/viper v1.21.0 // indirect
106108
github.com/subosito/gotenv v1.6.0 // indirect
109+
github.com/vishvananda/netns v0.0.5 // indirect
107110
github.com/x448/float16 v0.8.4 // indirect
108111
github.com/xlab/treeprint v1.2.0 // indirect
109112
go.opentelemetry.io/auto/sdk v1.2.1 // indirect

test/go.sum

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,8 @@ github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0=
4848
github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
4949
github.com/coredns/corefile-migration v1.0.30 h1:ljZNPGgna+4yKv81gfkvkgLEWdtz0NjBR1glaiPI140=
5050
github.com/coredns/corefile-migration v1.0.30/go.mod h1:56DPqONc3njpVPsdilEnfijCwNGC3/kTJLl7i7SPavY=
51+
github.com/coreos/go-iptables v0.8.0 h1:MPc2P89IhuVpLI7ETL/2tx3XZ61VeICZjYqDEgNsPRc=
52+
github.com/coreos/go-iptables v0.8.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
5153
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
5254
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5355
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -277,6 +279,10 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
277279
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
278280
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
279281
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
282+
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
283+
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
284+
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
285+
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
280286
github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
281287
github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
282288
github.com/xlab/treeprint v1.2.0 h1:HzHnuAF1plUN2zGlAFHbSQP2qJ0ZAD3XF5XD7OesXRQ=
@@ -330,7 +336,9 @@ golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
330336
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
331337
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
332338
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
339+
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
333340
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
341+
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
334342
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
335343
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
336344
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

test/vbmctl/README.md

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ This tool is under active development.
1717
| `vbmctl create bml` / `vbmctl delete bml` | ✅ Implemented |
1818
| `vbmctl status` | ✅ Implemented (basic) |
1919
| Configurable volumes | ❌ TODO (hard-coded to two per VM) |
20-
| Network management | ⚠️ Partially implemented (only libvirt networks) |
20+
| Network management | ⚠️ Partially implemented (container networking missing) |
2121
| BMC emulator support | ❌ TODO |
2222
| Image server | ❌ TODO |
2323
| State management (persistent state) | ❌ TODO |
@@ -30,6 +30,8 @@ This tool is under active development.
3030
libvirt networks
3131
- **Library Support**: Can be imported as a Go module for programmatic use
3232
- **Libvirt network management**: create and delete libvirt networks
33+
- **Creating/deleting veth pairs**: create and delete veth-pairs to connect
34+
virtual networks. Works only with config file.
3335

3436
## Build Tags
3537

@@ -157,6 +159,11 @@ spec:
157159
bridge: metal3
158160
address: 192.168.222.1
159161
netmask: 255.255.255.0
162+
vethPairs:
163+
- master1: metal3
164+
master2: kind-bridge
165+
veth1: metalend
166+
veth2: kindend
160167
```
161168
162169
The `spec.vms` section defines the VMs that will be created when you run `vbmctl

test/vbmctl/cmd/vbmctl/main.go

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import (
1616
vbmctlapi "github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/api"
1717
"github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/config"
1818
"github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/libvirt"
19+
"github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/network"
1920
"github.com/spf13/cobra"
2021
libvirtgo "libvirt.org/go/libvirt"
2122
)
@@ -204,7 +205,12 @@ Example configuration:
204205
macAddress: "00:60:2f:31:81:01"
205206
networks:
206207
- name: "baremetal-e2e"
207-
- bridge: "metal3"`,
208+
- bridge: "metal3"
209+
vethPairs:
210+
- master1: "metal3"
211+
master2: "kind-bridge"
212+
veth1: "metalend"
213+
veth2: "kindend"`,
208214
RunE: func(_ *cobra.Command, _ []string) error {
209215
ctx, cancel := contextWithSignal()
210216
defer cancel()
@@ -240,6 +246,17 @@ Example configuration:
240246
fmt.Printf(" - %s (UUID: %s)\n", network.Name, network.UUID)
241247
}
242248

249+
// Connect the specified networks
250+
network.ConnectAllWithVeth(ctx, cfg.Spec.VethPairs)
251+
// Add forward rules to allow fordawing between connected networks
252+
network.AddForwardRulesAll(ctx, cfg.Spec.VethPairs)
253+
//nolint:forbidigo // CLI output is intentional
254+
fmt.Println("\nCreated veth pairs:")
255+
for _, pair := range cfg.Spec.VethPairs {
256+
//nolint:forbidigo // CLI output is intentional
257+
fmt.Printf(" - between %s and %s\n", pair.Master1, pair.Master2)
258+
}
259+
243260
vmManager, err := libvirt.NewVMManager(conn, libvirt.VMManagerOptions{
244261
PoolName: cfg.Spec.Pool.Name,
245262
PoolPath: cfg.Spec.Pool.Path,
@@ -435,6 +452,17 @@ func newDeleteBMLCmd() *cobra.Command {
435452
fmt.Printf(" - %s\n", name)
436453
}
437454

455+
// Delete forward rules
456+
network.DeleteForwardRulesAll(ctx, cfg.Spec.VethPairs)
457+
// Delete veth pairs
458+
network.DeleteAllVeth(ctx, cfg.Spec.VethPairs)
459+
//nolint:forbidigo // CLI output is intentional
460+
fmt.Println("\nDeleted veth pairs:")
461+
for _, pair := range cfg.Spec.VethPairs {
462+
//nolint:forbidigo // CLI output is intentional
463+
fmt.Printf(" - between %s and %s\n", pair.Master1, pair.Master2)
464+
}
465+
438466
networkManager, err := libvirt.NewNetworkManager(conn)
439467
if err != nil {
440468
return fmt.Errorf("failed to create Network manager: %w", err)

test/vbmctl/pkg/api/types.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,20 @@ type NetworkConfig struct {
5757
Netmask string `json:"netmask,omitempty" yaml:"netmask,omitempty"`
5858
}
5959

60+
type VethPairs struct {
61+
// Name of the first network to be connected
62+
Master1 string `json:"master1" yaml:"master1"`
63+
64+
// Name of the second network to be connected
65+
Master2 string `json:"master2" yaml:"master2"`
66+
67+
// Name of the veth pair to be pushed to first network
68+
Veth1 string `json:"veth1" yaml:"veth1"`
69+
70+
// Name of the veth pair to be pushed to the second network
71+
Veth2 string `json:"veth2" yaml:"veth2"`
72+
}
73+
6074
// PoolConfig represents the configuration for a storage pool.
6175
type PoolConfig struct {
6276
// Name is the name of the storage pool.

test/vbmctl/pkg/config/config.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ type Spec struct {
7676
VMs []vbmctlapi.VMConfig `json:"vms,omitempty" yaml:"vms,omitempty"`
7777

7878
Networks []vbmctlapi.NetworkConfig `json:"networks,omitempty" yaml:"networks,omitempty"`
79+
80+
VethPairs []vbmctlapi.VethPairs `json:"vethPairs,omitempty" yaml:"vethPairs,omitempty"`
7981
}
8082

8183
// LibvirtConfig contains libvirt connection settings.

test/vbmctl/pkg/network/ip.go

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
package network
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"log"
7+
8+
vbmctlapi "github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/api"
9+
"github.com/vishvananda/netlink"
10+
)
11+
12+
func ConnectWithVeth(_ context.Context, network1 string, network2 string, vethpeer1 string, vethpeer2 string) error {
13+
// Create veth pair
14+
la := netlink.NewLinkAttrs()
15+
la.Name = vethpeer1
16+
veth := &netlink.Veth{
17+
LinkAttrs: la,
18+
PeerName: vethpeer2,
19+
}
20+
err := netlink.LinkAdd(veth)
21+
if err != nil {
22+
return fmt.Errorf("could not add vethpair %s: %v\n", la.Name, err)
23+
}
24+
25+
// Connect the networks with the veth pair
26+
master1, err := netlink.LinkByName(network1)
27+
if err != nil {
28+
return fmt.Errorf("failed to get network interface %s: %w", network1, err)
29+
}
30+
master2, err := netlink.LinkByName(network2)
31+
if err != nil {
32+
return fmt.Errorf("failed to get network interface %s: %w", network2, err)
33+
}
34+
veth1, err := netlink.LinkByName(vethpeer1)
35+
if err != nil {
36+
return fmt.Errorf("failed to get veth interface %s: %w", vethpeer1, err)
37+
}
38+
veth2, err := netlink.LinkByName(vethpeer2)
39+
if err != nil {
40+
return fmt.Errorf("failed to get veth interface %s: %w", vethpeer2, err)
41+
}
42+
43+
if err := netlink.LinkSetUp(veth1); err != nil {
44+
return fmt.Errorf("failed to bring up veth interface %s: %w", vethpeer1, err)
45+
}
46+
if err := netlink.LinkSetUp(veth2); err != nil {
47+
return fmt.Errorf("failed to bring up veth interface %s: %w", vethpeer2, err)
48+
}
49+
50+
if err := netlink.LinkSetMaster(veth1, master1); err != nil {
51+
return fmt.Errorf("failed to set master %s for veth %s: %w", network1, vethpeer1, err)
52+
}
53+
if err := netlink.LinkSetMaster(veth2, master2); err != nil {
54+
return fmt.Errorf("failed to set master %s for veth %s: %w", network2, vethpeer2, err)
55+
}
56+
return nil
57+
}
58+
59+
func ConnectAllWithVeth(ctx context.Context, vethPairs []vbmctlapi.VethPairs) error {
60+
createdPairs := make([]*vbmctlapi.VethPairs, 0, len(vethPairs))
61+
for _, pair := range vethPairs {
62+
err := ConnectWithVeth(ctx, pair.Master1, pair.Master2, pair.Veth1, pair.Veth2)
63+
if err != nil {
64+
// Clean up previously created pairs
65+
log.Printf("Failed to create veth pair %s, cleaning up %d previously created pair(s)\n", pair.Veth1, len(createdPairs))
66+
for _, created := range createdPairs {
67+
if delErr := DeleteLink(ctx, pair.Veth1); delErr != nil {
68+
log.Printf("Warning: failed to clean up veth pair %s: %v\n", created.Veth1, delErr)
69+
}
70+
}
71+
return fmt.Errorf("failed to create veth pair %s: %w", pair.Veth1, err)
72+
}
73+
createdPairs = append(createdPairs, &pair)
74+
}
75+
return nil
76+
}
77+
78+
func DeleteLink(_ context.Context, link string) error {
79+
l, err := netlink.LinkByName(link)
80+
if err != nil {
81+
return fmt.Errorf("failed to get network interface %s: %w", link, err)
82+
}
83+
if err := netlink.LinkDel(l); err != nil {
84+
return fmt.Errorf("failed to delete network interface %s: %w", link, err)
85+
}
86+
return nil
87+
}
88+
89+
func DeleteAllVeth(ctx context.Context, vethPairs []vbmctlapi.VethPairs) error {
90+
var lastErr error
91+
for _, pair := range vethPairs {
92+
if err := DeleteLink(ctx, pair.Veth1); err != nil {
93+
log.Printf("Error deleting veth pair %s: %v\n", pair.Veth1, err)
94+
lastErr = err
95+
}
96+
}
97+
return lastErr
98+
}
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
package network
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"log"
7+
8+
"github.com/coreos/go-iptables/iptables"
9+
vbmctlapi "github.com/metal3-io/baremetal-operator/test/vbmctl/pkg/api"
10+
)
11+
12+
func AddForwardRules(_ context.Context, iface1, iface2 string) error {
13+
ipt, err := iptables.New()
14+
if err != nil {
15+
return fmt.Errorf("failed to initialize iptables: %w", err)
16+
}
17+
18+
// Add rule: iptables -I FORWARD -i iface1 -o iface2 -j ACCEPT
19+
err = ipt.Insert("filter", "FORWARD", 1, "-i", iface1, "-o", iface2, "-j", "ACCEPT")
20+
if err != nil {
21+
return fmt.Errorf("failed to add forward rule from %s to %s: %w", iface1, iface2, err)
22+
}
23+
24+
// Add reverse
25+
err = ipt.Insert("filter", "FORWARD", 1, "-i", iface2, "-o", iface1, "-j", "ACCEPT")
26+
if err != nil {
27+
return fmt.Errorf("failed to add forward rule from %s to %s: %w", iface1, iface2, err)
28+
}
29+
30+
return nil
31+
}
32+
33+
func AddForwardRulesAll(ctx context.Context, vethPairs []vbmctlapi.VethPairs) error {
34+
createdRules := make([]*vbmctlapi.VethPairs, 0, len(vethPairs))
35+
for _, pair := range vethPairs {
36+
err := AddForwardRules(ctx, pair.Master1, pair.Master2)
37+
if err != nil {
38+
// Clean up previously created rules
39+
log.Printf("Failed to add forward rules between %s and %s, cleaning up %d previously created rule(s)\n", pair.Master1, pair.Master2, len(createdRules))
40+
for _, created := range createdRules {
41+
if delErr := DeleteForwardRules(ctx, pair.Master1, pair.Master2); delErr != nil {
42+
log.Printf("Warning: failed to clean up rule between %s and %s: %v\n", created.Master1, created.Master2, delErr)
43+
}
44+
}
45+
return fmt.Errorf("failed to create rule between %s and %s: %w", pair.Master1, pair.Master2, err)
46+
}
47+
createdRules = append(createdRules, &pair)
48+
}
49+
return nil
50+
}
51+
52+
func DeleteForwardRules(_ context.Context, iface1, iface2 string) error {
53+
ipt, err := iptables.New()
54+
if err != nil {
55+
return fmt.Errorf("failed to initialize iptables: %w", err)
56+
}
57+
58+
// Delete rule: iptables -D FORWARD -i iface1 -o iface2 -j ACCEPT
59+
err = ipt.Delete("filter", "FORWARD", "-i", iface1, "-o", iface2, "-j", "ACCEPT")
60+
if err != nil {
61+
return fmt.Errorf("failed to delete forward rule from %s to %s: %w", iface1, iface2, err)
62+
}
63+
64+
// Delete reverse
65+
err = ipt.Delete("filter", "FORWARD", "-i", iface2, "-o", iface1, "-j", "ACCEPT")
66+
if err != nil {
67+
return fmt.Errorf("failed to delete forward rule from %s to %s: %w", iface2, iface1, err)
68+
}
69+
70+
return nil
71+
}
72+
73+
func DeleteForwardRulesAll(ctx context.Context, vethPairs []vbmctlapi.VethPairs) error {
74+
var lastErr error
75+
for _, pair := range vethPairs {
76+
if err := DeleteForwardRules(ctx, pair.Master1, pair.Master2); err != nil {
77+
log.Printf("Error deleting forward rules between %s and %s: %v\n", pair.Master1, pair.Master2, err)
78+
lastErr = err
79+
}
80+
}
81+
return lastErr
82+
}

0 commit comments

Comments
 (0)