Skip to content

Commit db39ed4

Browse files
nirsnirs
committed
vfkit: add vmnet-shared network
Add new network option for vfkit "vment-shared", connecting vfkit to the vmnet shared network. Clusters using this network can access other clusters in the same network, similar to socket_vment with QEMU driver. If network is not specified, we default to the "nat" network, keeping the previous behavior. If network is "vment-shared", the vfkit driver manages 2 processes: vfkit and vmnet-helper. Like vfkit, vmnet-helper is started in the background, in a new process group, so it not terminated if the minikube process group is terminate. Since vment-helper requires root to start the vmnet interface, we start it with sudo, creating 2 child processes. vment-helper drops privileges immediately after starting the vment interface, and run as the user and group running minikube. Stopping the cluster will stop sudo, which will stop the vmnet-helper process. Deleting the cluster kill both sudo and vment-helper by killing the process group. This change is not complete, but it is good enough to play with the new shared network. Example usage: 1. Install vmnet-helper: https://github.com/nirs/vmnet-helper?tab=readme-ov-file#installation 2. Setup vment-helper sudoers rule: https://github.com/nirs/vmnet-helper?tab=readme-ov-file#granting-permission-to-run-vmnet-helper 3. Start 2 clusters with vmnet-shared network: % minikube start -p c1 --driver vfkit --network vment-shared ... % minikube start -p c2 --driver vfkit --network vmnet-shared ... % minikube ip -p c1 192.168.105.18 % minikube ip -p c2 192.168.105.19 4. Both cluster can access the other cluster: % minikube -p c1 ssh -- ping -c 3 192.168.105.19 PING 192.168.105.19 (192.168.105.19): 56 data bytes 64 bytes from 192.168.105.19: seq=0 ttl=64 time=0.621 ms 64 bytes from 192.168.105.19: seq=1 ttl=64 time=0.989 ms 64 bytes from 192.168.105.19: seq=2 ttl=64 time=0.490 ms --- 192.168.105.19 ping statistics --- 3 packets transmitted, 3 packets received, 0% packet loss round-trip min/avg/max = 0.490/0.700/0.989 ms % minikube -p c2 ssh -- ping -c 3 192.168.105.18 PING 192.168.105.18 (192.168.105.18): 56 data bytes 64 bytes from 192.168.105.18: seq=0 ttl=64 time=0.289 ms 64 bytes from 192.168.105.18: seq=1 ttl=64 time=0.798 ms 64 bytes from 192.168.105.18: seq=2 ttl=64 time=0.993 ms --- 192.168.105.18 ping statistics --- 3 packets transmitted, 3 packets received, 0% packet loss round-trip min/avg/max = 0.289/0.693/0.993 ms
1 parent 1efa44a commit db39ed4

File tree

2 files changed

+131
-22
lines changed

2 files changed

+131
-22
lines changed

‎pkg/drivers/vfkit/vfkit.go

+99-16
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343

4444
"k8s.io/klog/v2"
4545
pkgdrivers "k8s.io/minikube/pkg/drivers"
46+
"k8s.io/minikube/pkg/drivers/vmnet"
4647
"k8s.io/minikube/pkg/minikube/exit"
4748
"k8s.io/minikube/pkg/minikube/firewall"
4849
"k8s.io/minikube/pkg/minikube/out"
@@ -66,8 +67,10 @@ type Driver struct {
6667
CPU int
6768
Memory int
6869
Cmdline string
69-
MACAddress string
7070
ExtraDisks int
71+
Network string // "", "nat", "vmnet-shared"
72+
MACAddress string // For network=nat, network=""
73+
VmnetHelper *vmnet.Helper // For network=vmnet-shared
7174
}
7275

7376
func NewDriver(hostName, storePath string) drivers.Driver {
@@ -135,7 +138,7 @@ func (d *Driver) GetIP() (string, error) {
135138
return d.IPAddress, nil
136139
}
137140

138-
func (d *Driver) GetState() (state.State, error) {
141+
func (d *Driver) getVfkitState() (state.State, error) {
139142
pidfile := d.pidfilePath()
140143
pid, err := pkgdrivers.ReadPidfile(pidfile)
141144
if err != nil {
@@ -152,6 +155,24 @@ func (d *Driver) GetState() (state.State, error) {
152155
return state.Running, nil
153156
}
154157

158+
func (d *Driver) getVmnetHelperState() (state.State, error) {
159+
if d.VmnetHelper == nil {
160+
return state.Stopped, nil
161+
}
162+
return d.VmnetHelper.GetState()
163+
}
164+
165+
// GetState returns driver state. Since vfkit driver may use 2 processes
166+
// (vmnet-helper, vfkit), this returns combined state of both processes.
167+
func (d *Driver) GetState() (state.State, error) {
168+
if vfkitState, err := d.getVfkitState(); err != nil {
169+
return state.Error, err
170+
} else if vfkitState == state.Running {
171+
return state.Running, nil
172+
}
173+
return d.getVmnetHelperState()
174+
}
175+
155176
func (d *Driver) Create() error {
156177
var err error
157178
if d.SSHPort, err = d.GetSSHPort(); err != nil {
@@ -193,6 +214,42 @@ func (d *Driver) Create() error {
193214
}
194215

195216
func (d *Driver) Start() error {
217+
var helperSock, vfkitSock *os.File
218+
var err error
219+
220+
if d.VmnetHelper != nil {
221+
helperSock, vfkitSock, err = vmnet.Socketpair()
222+
if err != nil {
223+
return err
224+
}
225+
defer helperSock.Close()
226+
defer vfkitSock.Close()
227+
228+
if err := d.VmnetHelper.Start(helperSock); err != nil {
229+
return err
230+
}
231+
232+
d.MACAddress = d.VmnetHelper.GetMACAddress()
233+
}
234+
235+
if err := d.startVfkit(vfkitSock); err != nil {
236+
d.stopVmnetHelper()
237+
return err
238+
}
239+
240+
if err := d.setupIP(d.MACAddress); err != nil {
241+
d.stopVmnetHelper()
242+
return err
243+
}
244+
245+
log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
246+
247+
return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second)
248+
}
249+
250+
// startVfkit starts the vfkit child process. If vfkitSock is non nil, vfkit is
251+
// connected to the vmnet network via the socket instead of "nat" network.
252+
func (d *Driver) startVfkit(vfkitSock *os.File) error {
196253
machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName())
197254

198255
var startCmd []string
@@ -205,8 +262,15 @@ func (d *Driver) Start() error {
205262
startCmd = append(startCmd,
206263
"--device", fmt.Sprintf("virtio-blk,path=%s", isoPath))
207264

208-
startCmd = append(startCmd,
209-
"--device", fmt.Sprintf("virtio-net,nat,mac=%s", d.MACAddress))
265+
if vfkitSock != nil {
266+
// The guest will be able to access other guests in the vmnet network.
267+
startCmd = append(startCmd,
268+
"--device", fmt.Sprintf("virtio-net,fd=%d,mac=%s", vfkitSock.Fd(), d.MACAddress))
269+
} else {
270+
// The guest will not be able to access other guests.
271+
startCmd = append(startCmd,
272+
"--device", fmt.Sprintf("virtio-net,nat,mac=%s", d.MACAddress))
273+
}
210274

211275
startCmd = append(startCmd,
212276
"--device", "virtio-rng")
@@ -237,16 +301,7 @@ func (d *Driver) Start() error {
237301
if err := cmd.Start(); err != nil {
238302
return err
239303
}
240-
if err := pkgdrivers.WritePidfile(d.pidfilePath(), cmd.Process.Pid); err != nil {
241-
return err
242-
}
243-
if err := d.setupIP(d.MACAddress); err != nil {
244-
return err
245-
}
246-
247-
log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
248-
249-
return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second)
304+
return pkgdrivers.WritePidfile(d.pidfilePath(), cmd.Process.Pid)
250305
}
251306

252307
func (d *Driver) setupIP(mac string) error {
@@ -287,7 +342,7 @@ func isBootpdError(err error) bool {
287342
return strings.Contains(err.Error(), "could not find an IP address")
288343
}
289344

290-
func (d *Driver) Stop() error {
345+
func (d *Driver) stopVfkit() error {
291346
if err := d.SetVFKitState("Stop"); err != nil {
292347
// vfkit may be already stopped, shutting down, or not listening.
293348
log.Debugf("Failed to set vfkit state to 'Stop': %s", err)
@@ -296,6 +351,20 @@ func (d *Driver) Stop() error {
296351
return nil
297352
}
298353

354+
func (d *Driver) stopVmnetHelper() error {
355+
if d.VmnetHelper == nil {
356+
return nil
357+
}
358+
return d.VmnetHelper.Stop()
359+
}
360+
361+
func (d *Driver) Stop() error {
362+
if err := d.stopVfkit(); err != nil {
363+
return err
364+
}
365+
return d.stopVmnetHelper()
366+
}
367+
299368
func (d *Driver) Remove() error {
300369
s, err := d.GetState()
301370
if err != nil {
@@ -339,7 +408,7 @@ func (d *Driver) extractKernel(isoPath string) error {
339408
return nil
340409
}
341410

342-
func (d *Driver) Kill() error {
411+
func (d *Driver) killVfkit() error {
343412
if err := d.SetVFKitState("HardStop"); err != nil {
344413
// Typically fails with EOF.
345414
log.Debugf("Failed to set vfkit state to 'HardStop': %s", err)
@@ -348,6 +417,20 @@ func (d *Driver) Kill() error {
348417
return nil
349418
}
350419

420+
func (d *Driver) killVmnetHelper() error {
421+
if d.VmnetHelper == nil {
422+
return nil
423+
}
424+
return d.VmnetHelper.Kill()
425+
}
426+
427+
func (d *Driver) Kill() error {
428+
if err := d.killVfkit(); err != nil {
429+
return err
430+
}
431+
return d.killVmnetHelper()
432+
}
433+
351434
func (d *Driver) StartDocker() error {
352435
return fmt.Errorf("hosts without a driver cannot start docker")
353436
}

‎pkg/minikube/registry/drvs/vfkit/vfkit.go

+32-6
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,13 @@ import (
2222
"crypto/rand"
2323
"fmt"
2424
"os/exec"
25+
"path/filepath"
2526

2627
"github.com/docker/machine/libmachine/drivers"
28+
"github.com/google/uuid"
2729

2830
"k8s.io/minikube/pkg/drivers/vfkit"
31+
"k8s.io/minikube/pkg/drivers/vmnet"
2932
"k8s.io/minikube/pkg/minikube/config"
3033
"k8s.io/minikube/pkg/minikube/download"
3134
"k8s.io/minikube/pkg/minikube/driver"
@@ -51,24 +54,47 @@ func init() {
5154
}
5255

5356
func configure(cfg config.ClusterConfig, n config.Node) (interface{}, error) {
54-
mac, err := generateMACAddress()
55-
if err != nil {
56-
return nil, fmt.Errorf("generating MAC address: %v", err)
57+
var mac string
58+
var helper *vmnet.Helper
59+
60+
machineName := config.MachineName(cfg, n)
61+
storePath := localpath.MiniPath()
62+
63+
switch cfg.Network {
64+
case "nat", "":
65+
var err error
66+
mac, err = generateMACAddress()
67+
if err != nil {
68+
return nil, fmt.Errorf("generating MAC address: %v", err)
69+
}
70+
case "vmnet-shared":
71+
u := cfg.UUID
72+
if u == "" {
73+
u = uuid.NewString()
74+
}
75+
helper = &vmnet.Helper{
76+
MachineDir: filepath.Join(storePath, "machines", machineName),
77+
InterfaceID: u,
78+
}
79+
default:
80+
return nil, fmt.Errorf("unsupported network: %q", cfg.Network)
5781
}
5882

5983
return &vfkit.Driver{
6084
BaseDriver: &drivers.BaseDriver{
61-
MachineName: config.MachineName(cfg, n),
62-
StorePath: localpath.MiniPath(),
85+
MachineName: machineName,
86+
StorePath: storePath,
6387
SSHUser: "docker",
6488
},
6589
Boot2DockerURL: download.LocalISOResource(cfg.MinikubeISO),
6690
DiskSize: cfg.DiskSize,
6791
Memory: cfg.Memory,
6892
CPU: cfg.CPUs,
69-
MACAddress: mac,
7093
Cmdline: "",
7194
ExtraDisks: cfg.ExtraDisks,
95+
Network: cfg.Network,
96+
MACAddress: mac,
97+
VmnetHelper: helper,
7298
}, nil
7399
}
74100

0 commit comments

Comments
 (0)