Skip to content

Commit da3ffc8

Browse files
nirsnirs
committed
vfkit: add vmnet-shared network
Add new network option for vfkit "vmnet-shared", connecting vfkit to the vmnet shared network. Clusters using this network can access other clusters in the same network, similar to socket_vmnet with QEMU driver. If network is not specified, we default to the "nat" network, keeping the previous behavior. If network is "vmnet-shared", the vfkit driver manages 2 processes: vfkit and vmnet-helper. Like vfkit, vmnet-helper is started in the background, in a new process group, so it not terminated if the minikube process group is terminate. Since vmnet-helper requires root to start the vmnet interface, we start it with sudo, creating 2 child processes. vmnet-helper drops privileges immediately after starting the vmnet interface, and run as the user and group running minikube. Stopping the cluster will stop sudo, which will stop the vmnet-helper process. Deleting the cluster kill both sudo and vmnet-helper by killing the process group. This change is not complete, but it is good enough to play with the new shared network. Example usage: 1. Install vmnet-helper: https://github.com/nirs/vmnet-helper?tab=readme-ov-file#installation 2. Setup vmnet-helper sudoers rule: https://github.com/nirs/vmnet-helper?tab=readme-ov-file#granting-permission-to-run-vmnet-helper 3. Start 2 clusters with vmnet-shared network: % minikube start -p c1 --driver vfkit --network vmnet-shared ... % minikube start -p c2 --driver vfkit --network vmnet-shared ... % minikube ip -p c1 192.168.105.18 % minikube ip -p c2 192.168.105.19 4. Both cluster can access the other cluster: % minikube -p c1 ssh -- ping -c 3 192.168.105.19 PING 192.168.105.19 (192.168.105.19): 56 data bytes 64 bytes from 192.168.105.19: seq=0 ttl=64 time=0.621 ms 64 bytes from 192.168.105.19: seq=1 ttl=64 time=0.989 ms 64 bytes from 192.168.105.19: seq=2 ttl=64 time=0.490 ms --- 192.168.105.19 ping statistics --- 3 packets transmitted, 3 packets received, 0% packet loss round-trip min/avg/max = 0.490/0.700/0.989 ms % minikube -p c2 ssh -- ping -c 3 192.168.105.18 PING 192.168.105.18 (192.168.105.18): 56 data bytes 64 bytes from 192.168.105.18: seq=0 ttl=64 time=0.289 ms 64 bytes from 192.168.105.18: seq=1 ttl=64 time=0.798 ms 64 bytes from 192.168.105.18: seq=2 ttl=64 time=0.993 ms --- 192.168.105.18 ping statistics --- 3 packets transmitted, 3 packets received, 0% packet loss round-trip min/avg/max = 0.289/0.693/0.993 ms
1 parent f794287 commit da3ffc8

File tree

2 files changed

+131
-22
lines changed

2 files changed

+131
-22
lines changed

‎pkg/drivers/vfkit/vfkit.go

+99-16
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343

4444
"k8s.io/klog/v2"
4545
pkgdrivers "k8s.io/minikube/pkg/drivers"
46+
"k8s.io/minikube/pkg/drivers/vmnet"
4647
"k8s.io/minikube/pkg/minikube/exit"
4748
"k8s.io/minikube/pkg/minikube/firewall"
4849
"k8s.io/minikube/pkg/minikube/out"
@@ -67,8 +68,10 @@ type Driver struct {
6768
CPU int
6869
Memory int
6970
Cmdline string
70-
MACAddress string
7171
ExtraDisks int
72+
Network string // "", "nat", "vmnet-shared"
73+
MACAddress string // For network=nat, network=""
74+
VmnetHelper *vmnet.Helper // For network=vmnet-shared
7275
}
7376

7477
func NewDriver(hostName, storePath string) drivers.Driver {
@@ -136,7 +139,7 @@ func (d *Driver) GetIP() (string, error) {
136139
return d.IPAddress, nil
137140
}
138141

139-
func (d *Driver) GetState() (state.State, error) {
142+
func (d *Driver) getVfkitState() (state.State, error) {
140143
pidfile := d.pidfilePath()
141144
pid, err := process.ReadPidfile(pidfile)
142145
if err != nil {
@@ -156,6 +159,24 @@ func (d *Driver) GetState() (state.State, error) {
156159
return state.Running, nil
157160
}
158161

162+
func (d *Driver) getVmnetHelperState() (state.State, error) {
163+
if d.VmnetHelper == nil {
164+
return state.Stopped, nil
165+
}
166+
return d.VmnetHelper.GetState()
167+
}
168+
169+
// GetState returns driver state. Since vfkit driver may use 2 processes
170+
// (vmnet-helper, vfkit), this returns combined state of both processes.
171+
func (d *Driver) GetState() (state.State, error) {
172+
if vfkitState, err := d.getVfkitState(); err != nil {
173+
return state.Error, err
174+
} else if vfkitState == state.Running {
175+
return state.Running, nil
176+
}
177+
return d.getVmnetHelperState()
178+
}
179+
159180
func (d *Driver) Create() error {
160181
var err error
161182
if d.SSHPort, err = d.GetSSHPort(); err != nil {
@@ -197,6 +218,42 @@ func (d *Driver) Create() error {
197218
}
198219

199220
func (d *Driver) Start() error {
221+
var helperSock, vfkitSock *os.File
222+
var err error
223+
224+
if d.VmnetHelper != nil {
225+
helperSock, vfkitSock, err = vmnet.Socketpair()
226+
if err != nil {
227+
return err
228+
}
229+
defer helperSock.Close()
230+
defer vfkitSock.Close()
231+
232+
if err := d.VmnetHelper.Start(helperSock); err != nil {
233+
return err
234+
}
235+
236+
d.MACAddress = d.VmnetHelper.GetMACAddress()
237+
}
238+
239+
if err := d.startVfkit(vfkitSock); err != nil {
240+
d.stopVmnetHelper()
241+
return err
242+
}
243+
244+
if err := d.setupIP(d.MACAddress); err != nil {
245+
d.stopVmnetHelper()
246+
return err
247+
}
248+
249+
log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
250+
251+
return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second)
252+
}
253+
254+
// startVfkit starts the vfkit child process. If vfkitSock is non nil, vfkit is
255+
// connected to the vmnet network via the socket instead of "nat" network.
256+
func (d *Driver) startVfkit(vfkitSock *os.File) error {
200257
machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName())
201258

202259
var startCmd []string
@@ -209,8 +266,15 @@ func (d *Driver) Start() error {
209266
startCmd = append(startCmd,
210267
"--device", fmt.Sprintf("virtio-blk,path=%s", isoPath))
211268

212-
startCmd = append(startCmd,
213-
"--device", fmt.Sprintf("virtio-net,nat,mac=%s", d.MACAddress))
269+
if vfkitSock != nil {
270+
// The guest will be able to access other guests in the vmnet network.
271+
startCmd = append(startCmd,
272+
"--device", fmt.Sprintf("virtio-net,fd=%d,mac=%s", vfkitSock.Fd(), d.MACAddress))
273+
} else {
274+
// The guest will not be able to access other guests.
275+
startCmd = append(startCmd,
276+
"--device", fmt.Sprintf("virtio-net,nat,mac=%s", d.MACAddress))
277+
}
214278

215279
startCmd = append(startCmd,
216280
"--device", "virtio-rng")
@@ -241,16 +305,7 @@ func (d *Driver) Start() error {
241305
if err := cmd.Start(); err != nil {
242306
return err
243307
}
244-
if err := process.WritePidfile(d.pidfilePath(), cmd.Process.Pid); err != nil {
245-
return err
246-
}
247-
if err := d.setupIP(d.MACAddress); err != nil {
248-
return err
249-
}
250-
251-
log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
252-
253-
return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second)
308+
return process.WritePidfile(d.pidfilePath(), cmd.Process.Pid)
254309
}
255310

256311
func (d *Driver) setupIP(mac string) error {
@@ -291,7 +346,7 @@ func isBootpdError(err error) bool {
291346
return strings.Contains(err.Error(), "could not find an IP address")
292347
}
293348

294-
func (d *Driver) Stop() error {
349+
func (d *Driver) stopVfkit() error {
295350
if err := d.SetVFKitState("Stop"); err != nil {
296351
// vfkit may be already stopped, shutting down, or not listening.
297352
// We don't fallback to "HardStop" since it typically fails due to
@@ -316,6 +371,20 @@ func (d *Driver) Stop() error {
316371
return nil
317372
}
318373

374+
func (d *Driver) stopVmnetHelper() error {
375+
if d.VmnetHelper == nil {
376+
return nil
377+
}
378+
return d.VmnetHelper.Stop()
379+
}
380+
381+
func (d *Driver) Stop() error {
382+
if err := d.stopVfkit(); err != nil {
383+
return err
384+
}
385+
return d.stopVmnetHelper()
386+
}
387+
319388
func (d *Driver) Remove() error {
320389
s, err := d.GetState()
321390
if err != nil {
@@ -359,7 +428,7 @@ func (d *Driver) extractKernel(isoPath string) error {
359428
return nil
360429
}
361430

362-
func (d *Driver) Kill() error {
431+
func (d *Driver) killVfkit() error {
363432
if err := d.SetVFKitState("HardStop"); err != nil {
364433
// Typically fails with EOF due to https://github.com/crc-org/vfkit/issues/277.
365434
log.Debugf("Failed to set vfkit state to 'HardStop': %s", err)
@@ -382,6 +451,20 @@ func (d *Driver) Kill() error {
382451
return nil
383452
}
384453

454+
func (d *Driver) killVmnetHelper() error {
455+
if d.VmnetHelper == nil {
456+
return nil
457+
}
458+
return d.VmnetHelper.Kill()
459+
}
460+
461+
func (d *Driver) Kill() error {
462+
if err := d.killVfkit(); err != nil {
463+
return err
464+
}
465+
return d.killVmnetHelper()
466+
}
467+
385468
func (d *Driver) StartDocker() error {
386469
return fmt.Errorf("hosts without a driver cannot start docker")
387470
}

‎pkg/minikube/registry/drvs/vfkit/vfkit.go

+32-6
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,13 @@ import (
2222
"crypto/rand"
2323
"fmt"
2424
"os/exec"
25+
"path/filepath"
2526

2627
"github.com/docker/machine/libmachine/drivers"
28+
"github.com/google/uuid"
2729

2830
"k8s.io/minikube/pkg/drivers/vfkit"
31+
"k8s.io/minikube/pkg/drivers/vmnet"
2932
"k8s.io/minikube/pkg/minikube/config"
3033
"k8s.io/minikube/pkg/minikube/download"
3134
"k8s.io/minikube/pkg/minikube/driver"
@@ -51,24 +54,47 @@ func init() {
5154
}
5255

5356
func configure(cfg config.ClusterConfig, n config.Node) (interface{}, error) {
54-
mac, err := generateMACAddress()
55-
if err != nil {
56-
return nil, fmt.Errorf("generating MAC address: %v", err)
57+
var mac string
58+
var helper *vmnet.Helper
59+
60+
machineName := config.MachineName(cfg, n)
61+
storePath := localpath.MiniPath()
62+
63+
switch cfg.Network {
64+
case "nat", "":
65+
var err error
66+
mac, err = generateMACAddress()
67+
if err != nil {
68+
return nil, fmt.Errorf("generating MAC address: %v", err)
69+
}
70+
case "vmnet-shared":
71+
u := cfg.UUID
72+
if u == "" {
73+
u = uuid.NewString()
74+
}
75+
helper = &vmnet.Helper{
76+
MachineDir: filepath.Join(storePath, "machines", machineName),
77+
InterfaceID: u,
78+
}
79+
default:
80+
return nil, fmt.Errorf("unsupported network: %q", cfg.Network)
5781
}
5882

5983
return &vfkit.Driver{
6084
BaseDriver: &drivers.BaseDriver{
61-
MachineName: config.MachineName(cfg, n),
62-
StorePath: localpath.MiniPath(),
85+
MachineName: machineName,
86+
StorePath: storePath,
6387
SSHUser: "docker",
6488
},
6589
Boot2DockerURL: download.LocalISOResource(cfg.MinikubeISO),
6690
DiskSize: cfg.DiskSize,
6791
Memory: cfg.Memory,
6892
CPU: cfg.CPUs,
69-
MACAddress: mac,
7093
Cmdline: "",
7194
ExtraDisks: cfg.ExtraDisks,
95+
Network: cfg.Network,
96+
MACAddress: mac,
97+
VmnetHelper: helper,
7298
}, nil
7399
}
74100

0 commit comments

Comments
 (0)