Use vment-helper in vfkit driver

This is a quick proof of concept, using vment-helper to connect vfkit to
vment network shared mode instead of the "nat" network.

We should add network options for vfkit and use vmnet-helper only when
using the vmnet network, but this is good enough to play with
vmnet-helper and run stress tests.

We also need documentation, but for now here is how to play with
vmnet-helper:

1. Install latest release:
   https://github.com/nirs/vmnet-helper?tab=readme-ov-file#installation

2. Setup vment-helper sudoers rule:
   https://github.com/nirs/vmnet-helper?tab=readme-ov-file#granting-permission-to-run-vmnet-helper

At this point you can start clusters with vfkit driver:

    % out/minikube start --driver vfkit --container-runtime containerd
    😄  minikube v1.35.0 on Darwin 15.3.1 (arm64)
    ✨  Using the vfkit (experimental) driver based on user configuration
    👍  Starting "minikube" primary control-plane node in "minikube" cluster
    🔥  Creating vfkit VM (CPUs=2, Memory=6000MB, Disk=20000MB) ...
    📦  Preparing Kubernetes v1.32.2 on containerd 1.7.23 ...
        ▪ Generating certificates and keys ...
        ▪ Booting up control plane ...
        ▪ Configuring RBAC rules ...
    🔗  Configuring bridge CNI (Container Networking Interface) ...
    🔎  Verifying Kubernetes components...
        ▪ Using image gcr.io/k8s-minikube/storage-provisioner:v5
    🌟  Enabled addons: default-storageclass, storage-provisioner
    🏄  Done! kubectl is now configured to use "minikube" cluster and "default" namespace by default

The cluster IP will be in the vment shared network:

    % out/minikube profile list
    |----------|-----------|------------|---------------|------|---------|--------|-------|----------------|--------------------|
    | Profile  | VM Driver |  Runtime   |      IP       | Port | Version | Status | Nodes | Active Profile | Active Kubecontext |
    |----------|-----------|------------|---------------|------|---------|--------|-------|----------------|--------------------|
    | minikube | vfkit     | containerd | 192.168.105.8 | 8443 | v1.32.2 | OK     |     1 | *              | *                  |
    |----------|-----------|------------|---------------|------|---------|--------|-------|----------------|--------------------|

We use the same --interface-id (generated from the machine name) when
starting the vmnet interface, so we will always get the same MAC address
(assigned by the vmnet framework) and the same IP address (assigned by
the DHCP server).

The vmnet-helper process is started in the background with vfkit, and
must remain running while vfkit is running. Like vfkit, we start it in
the background, in a new process group, so it not terminated if the
minikube process group is terminate.

Since vment-helper requires root to start the vmnet interface, we start
it with sudo, creating 2 child processes. vment-helper drops privileges
immediately after starting the vment interface, and run as the user and
group running minikube.

Stopping the cluster will stop sudo, which will stop the vmnet-helper
process. Deleting the cluster kill both sudo and vment-helper by killing
the process group.

Author: nirs

pkg/drivers/vfkit/vfkit.go

@@ -44,6 +44,7 @@ import (
 
 	"k8s.io/klog/v2"
 	pkgdrivers "k8s.io/minikube/pkg/drivers"
+	"k8s.io/minikube/pkg/drivers/vmnet"
 	"k8s.io/minikube/pkg/minikube/exit"
 	"k8s.io/minikube/pkg/minikube/firewall"
 	"k8s.io/minikube/pkg/minikube/out"
@@ -69,6 +70,9 @@ type Driver struct {
 	Cmdline        string
 	MACAddress     string
 	ExtraDisks     int
+
+	// For network=vmnet-shared.
+	VmnetHelper *vmnet.Helper
 }
 
 func NewDriver(hostName, storePath string) drivers.Driver {
@@ -144,7 +148,7 @@ func checkPid(pid int) error {
 	return process.Signal(syscall.Signal(0))
 }
 
-func (d *Driver) GetState() (state.State, error) {
+func (d *Driver) getVfkitState() (state.State, error) {
 	if _, err := os.Stat(d.pidfilePath()); err != nil {
 		return state.Stopped, nil
 	}
@@ -174,6 +178,35 @@ func (d *Driver) GetState() (state.State, error) {
 	return state.None, nil
 }
 
+func (d *Driver) getVmnetHelperState() (state.State, error) {
+	if d.VmnetHelper == nil {
+		return state.None, nil
+	}
+	return d.VmnetHelper.GetState()
+}
+
+// GetState returns driver state. Since vfkit driver may use 2 processes
+// (vmnet-helper, vfkit), this may returns combined state of both processes.
+func (d *Driver) GetState() (state.State, error) {
+	helperState, helperErr := d.getVmnetHelperState()
+	vfkitState, vfkitErr := d.getVfkitState()
+
+	// If at least one is running, we need to treat state as running, nedeed by
+	// Stop(), Remove(), and Restart().
+	if helperState == state.Running || vfkitState == state.Running {
+		return state.Running, nil
+	}
+
+	// If at least one is in error state we treat the state as error.
+	if helperState == state.Error || vfkitState == state.Error {
+		return state.Error, fmt.Errorf("vfkit: %w, vmnet-helper: %w", vfkitErr, helperErr)
+	}
+
+	// We can have many combinations (state.None, state.Stopped) of the 2
+	// states. Letss simplify by treating vfkit state as the main state.
+	return vfkitState, nil
+}
+
 func (d *Driver) Create() error {
 	var err error
 	if d.SSHPort, err = d.GetSSHPort(); err != nil {
@@ -215,6 +248,42 @@ func (d *Driver) Create() error {
 }
 
 func (d *Driver) Start() error {
+	var helperSock, vfkitSock *os.File
+	var err error
+
+	if d.VmnetHelper != nil {
+		helperSock, vfkitSock, err = vmnet.Socketpair()
+		if err != nil {
+			return err
+		}
+		defer helperSock.Close()
+		defer vfkitSock.Close()
+
+		if err := d.VmnetHelper.Start(helperSock); err != nil {
+			return err
+		}
+
+		d.MACAddress = d.VmnetHelper.GetMACAddress()
+	}
+
+	if err := d.startVfkit(vfkitSock); err != nil {
+		d.stopVmnetHelper()
+		return err
+	}
+
+	if err := d.setupIP(d.MACAddress); err != nil {
+		d.stopVmnetHelper()
+		return err
+	}
+
+	log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
+
+	return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second)
+}
+
+// startVfkit starts the vfkit child process. If vfkitSock is non nil, vfkit is
+// connected to the vmnet network via the socket instead of "nat" network.
+func (d *Driver) startVfkit(vfkitSock *os.File) error {
 	machineDir := filepath.Join(d.StorePath, "machines", d.GetMachineName())
 
 	var startCmd []string
@@ -227,8 +296,15 @@ func (d *Driver) Start() error {
 	startCmd = append(startCmd,
 		"--device", fmt.Sprintf("virtio-blk,path=%s", isoPath))
 
-	startCmd = append(startCmd,
-		"--device", fmt.Sprintf("virtio-net,nat,mac=%s", d.MACAddress))
+	if vfkitSock != nil {
+		// The guest will be able to access other guests in the vmnet network.
+		startCmd = append(startCmd,
+			"--device", fmt.Sprintf("virtio-net,fd=%d,mac=%s", vfkitSock.Fd(), d.MACAddress))
+	} else {
+		// The guest will not be able to access other guests.
+		startCmd = append(startCmd,
+			"--device", fmt.Sprintf("virtio-net,nat,mac=%s", d.MACAddress))
+	}
 
 	startCmd = append(startCmd,
 		"--device", "virtio-rng")
@@ -263,14 +339,7 @@ func (d *Driver) Start() error {
 	if err := os.WriteFile(d.pidfilePath(), []byte(fmt.Sprintf("%v", pid)), 0600); err != nil {
 		return err
 	}
-
-	if err := d.setupIP(d.MACAddress); err != nil {
-		return err
-	}
-
-	log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
-
-	return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second)
+	return nil
 }
 
 func (d *Driver) setupIP(mac string) error {
@@ -311,7 +380,17 @@ func isBootpdError(err error) bool {
 	return strings.Contains(err.Error(), "could not find an IP address")
 }
 
+func (d *Driver) stopVmnetHelper() error {
+	if d.VmnetHelper == nil {
+		return nil
+	}
+	return d.VmnetHelper.Stop()
+}
+
 func (d *Driver) Stop() error {
+	if err := d.stopVmnetHelper(); err != nil {
+		return err
+	}
 	if err := d.SetVFKitState("HardStop"); err != nil {
 		return err
 	}
@@ -366,7 +445,17 @@ func (d *Driver) extractKernel(isoPath string) error {
 	return nil
 }
 
+func (d *Driver) killVmnetHelper() error {
+	if d.VmnetHelper == nil {
+		return nil
+	}
+	return d.VmnetHelper.Kill()
+}
+
 func (d *Driver) Kill() error {
+	if err := d.killVmnetHelper(); err != nil {
+		return err
+	}
 	if err := d.SetVFKitState("HardStop"); err != nil {
 		return err
 	}

pkg/drivers/vmnet/vmnet.go

@@ -63,10 +63,10 @@ type interfaceInfo struct {
 func HelperAvailable() bool {
 	version, err := exec.Command("sudo", "--non-interactive", executablePath, "--version").Output()
 	if err != nil {
-		log.Debugf("Failed to run vmnet-helper: %w", executablePath, err)
+		log.Debugf("Failed to run vmnet-helper: %w", err)
 		return false
 	}
-	log.Debugf("Using vmnet-helper version %q", executablePath, version)
+	log.Debugf("Using vmnet-helper version %q", version)
 	return true
 }
 

pkg/minikube/registry/drvs/vfkit/vfkit.go

@@ -26,6 +26,7 @@ import (
 	"github.com/docker/machine/libmachine/drivers"
 
 	"k8s.io/minikube/pkg/drivers/vfkit"
+	"k8s.io/minikube/pkg/drivers/vmnet"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/download"
 	"k8s.io/minikube/pkg/minikube/driver"
@@ -51,6 +52,7 @@ func init() {
 }
 
 func configure(cfg config.ClusterConfig, n config.Node) (interface{}, error) {
+	// XXX Not needed when using vmnet-helper.
 	mac, err := generateMACAddress()
 	if err != nil {
 		return nil, fmt.Errorf("generating MAC address: %v", err)
@@ -69,6 +71,12 @@ func configure(cfg config.ClusterConfig, n config.Node) (interface{}, error) {
 		MACAddress:     mac,
 		Cmdline:        "",
 		ExtraDisks:     cfg.ExtraDisks,
+
+		// XXX Create only when using vmnet network.
+		VmnetHelper: &vmnet.Helper{
+			MachineName: config.MachineName(cfg, n),
+			StorePath:   localpath.MiniPath(),
+		},
 	}, nil
 }