Support docker-env with cri-o runtime

It is very similar to containerd, but with podman for cri-o
instead of nerdctl for containerd. Both using ssh sockets.

Author: afbjorklund

cmd/minikube/cmd/docker-env.go

@@ -314,14 +314,20 @@ docker-cli install instructions: https://minikube.sigs.k8s.io/docs/tutorials/doc
 			exit.Message(reason.Usage, err.Error())
 		}
 
-		// for the sake of docker-env command, start nerdctl and nerdctld
-		if cr == constants.Containerd {
-			out.WarningT("Using the docker-env command with the containerd runtime is a highly experimental feature, please provide feedback or contribute to make it better")
+		// for the sake of docker-env command, start nerdctl and nerdctld (or the podman system service, "podmand")
+		if cr == constants.Containerd || cr == constants.CRIO {
+			if cr == constants.Containerd {
+				out.WarningT("Using the docker-env command with the containerd runtime is a highly experimental feature, please provide feedback or contribute to make it better")
+
+				startNerdctld(options)
+			} else if cr == constants.CRIO {
+				out.WarningT("Using the docker-env command with the cri-o runtime is a highly experimental feature, please provide feedback or contribute to make it better")
 
-			startNerdctld(options)
+				startPodmand(options)
+			}
 
-			// docker-env on containerd depends on nerdctld (https://github.com/afbjorklund/nerdctld) as "docker" daeomn
-			// and nerdctld daemon must be used with ssh connection (it is set in kicbase image's Dockerfile)
+			// docker-env on containerd depends on nerdctld (https://github.com/afbjorklund/nerdctld) as "docker" daemon
+			// and nerdctld (or podman) daemon must be used with ssh connection (it is set in kicbase image's Dockerfile)
 			// so directly set --ssh-host --ssh-add to true, even user didn't specify them
 			sshAdd = true
 			sshHost = true
@@ -672,9 +678,6 @@ func tryDockerConnectivity(bin string, ec DockerEnvConfig) ([]byte, error) {
 }
 
 func dockerEnvSupported(containerRuntime, driverName string) error {
-	if containerRuntime != constants.Docker && containerRuntime != constants.Containerd {
-		return fmt.Errorf("the docker-env command only supports the docker and containerd runtimes")
-	}
 	// we only support containerd-env on the Docker driver
 	if containerRuntime == constants.Containerd && driverName != driver.Docker {
 		return fmt.Errorf("the docker-env command only supports the containerd runtime with the docker driver")

cmd/minikube/cmd/start.go

@@ -2100,3 +2100,20 @@ func startNerdctld(options *run.CommandOptions) {
 		exit.Error(reason.StartNerdctld, fmt.Sprintf("Failed to set up DOCKER_HOST: %s", rest.Output()), err)
 	}
 }
+
+func startPodmand(options *run.CommandOptions) {
+	co := mustload.Running(ClusterFlagValue(), options)
+	runner := co.CP.Runner
+
+	// sudo systemctl start podman.socket
+	if rest, err := runner.RunCmd(exec.Command("sudo", "systemctl", "start", "podman.socket")); err != nil {
+		exit.Error(reason.StartPodmand, fmt.Sprintf("Failed to enable podman.socket: %s", rest.Output()), err)
+	}
+
+	// set up environment variable on remote machine. docker client uses 'non-login & non-interactive shell' therefore the only way is to modify .bashrc file of user 'docker'
+	// insert this at 4th line
+	envSetupCommand := exec.Command("/bin/bash", "-c", "sed -i '4i export DOCKER_HOST=unix:///run/podman/podman.sock' .bashrc")
+	if rest, err := runner.RunCmd(envSetupCommand); err != nil {
+		exit.Error(reason.StartPodmand, fmt.Sprintf("Failed to set up DOCKER_HOST: %s", rest.Output()), err)
+	}
+}

pkg/minikube/reason/reason.go

@@ -102,6 +102,8 @@ var (
 	InternalCommandRunner = Kind{ID: "MK_COMMAND_RUNNER", ExitCode: ExProgramError}
 	// minikube failed to start nerdctld
 	StartNerdctld = Kind{ID: "MK_START_NERDCTLD", ExitCode: ExProgramError}
+	// minikube failed to start podmand
+	StartPodmand = Kind{ID: "MK_START_PODMAND", ExitCode: ExProgramError}
 	// minikube failed to generate shell command completion for a supported shell
 	InternalCompletion = Kind{ID: "MK_COMPLETION", ExitCode: ExProgramError}
 	// minikube failed to set an internal config value

test/integration/docker_test.go

@@ -255,3 +255,86 @@ func TestDockerEnvContainerd(t *testing.T) {
 		t.Fatal("failed to detect image 'local/minikube-dockerenv-containerd-test' in output of docker image ls")
 	}
 }
+
+// TestDockerEnvCrio makes sure that minikube docker-env command works when the runtime is crio
+func TestDockerEnvCrio(t *testing.T) {
+	t.Log("running with", ContainerRuntime(), DockerDriver(), runtime.GOOS, runtime.GOARCH)
+	if ContainerRuntime() != constants.CRIO || !DockerDriver() || runtime.GOOS != "linux" {
+		t.Skip("skipping: TestDockerEnvCrio can only be run with the crio runtime on Docker driver")
+	}
+	profile := UniqueProfileName("dockerenv")
+	ctx, cancel := context.WithTimeout(context.Background(), Minutes(30))
+	defer CleanupWithLogs(t, profile, cancel)
+
+	// start the minikube with crio runtime
+	args := append([]string{"start", "-p", profile}, StartArgs()...)
+	cmd := exec.CommandContext(ctx, Target(), args...)
+	startResult, err := Run(t, cmd)
+	if err != nil {
+		t.Errorf("failed to start minikube with args: %q : %v", startResult.Command(), err)
+	}
+	time.Sleep(time.Second * 10)
+
+	// execute 'minikube docker-env --ssh-host --ssh-add' and extract the 'DOCKER_HOST' environment value
+	cmd = exec.CommandContext(ctx, "/bin/bash", "-c", fmt.Sprintf("%s docker-env --ssh-host --ssh-add -p %s", Target(), profile))
+	result, err := Run(t, cmd)
+	if err != nil {
+		t.Errorf("failed to execute minikube docker-env --ssh-host --ssh-add, error: %v, output: %s", err, result.Output())
+	}
+
+	output := result.Output()
+	groups := regexp.MustCompile(`DOCKER_HOST="(\S*)"`).FindStringSubmatch(output)
+	if len(groups) < 2 {
+		t.Errorf("DOCKER_HOST doesn't match expected format, output is %s", output)
+	}
+	dockerHost := groups[1]
+	segments := strings.Split(dockerHost, ":")
+	if len(segments) < 3 {
+		t.Errorf("DOCKER_HOST doesn't match expected format, output is %s", dockerHost)
+	}
+
+	// get SSH_AUTH_SOCK
+	groups = regexp.MustCompile(`SSH_AUTH_SOCK=(\S*)`).FindStringSubmatch(output)
+	if len(groups) < 2 {
+		t.Errorf("failed to acquire SSH_AUTH_SOCK, output is %s", output)
+	}
+	sshAuthSock := groups[1]
+	// get SSH_AGENT_PID
+	groups = regexp.MustCompile(`SSH_AGENT_PID=(\S*)`).FindStringSubmatch(output)
+	if len(groups) < 2 {
+		t.Errorf("failed to acquire SSH_AUTH_PID, output is %s", output)
+	}
+	sshAgentPid := groups[1]
+
+	cmd = exec.CommandContext(ctx, "/bin/bash", "-c", fmt.Sprintf("SSH_AUTH_SOCK=%s SSH_AGENT_PID=%s DOCKER_HOST=%s docker version", sshAuthSock, sshAgentPid, dockerHost))
+
+	result, err = Run(t, cmd)
+	if err != nil {
+		t.Fatalf("failed to execute 'docker version', error: %v, output: %s", err, result.Output())
+	}
+	// if we are really connecting to podman inside node, docker version output should have word 'Podman'
+	// If everything works properly, in the output of `docker version` you should be able to see something like
+	/*
+		Server:       Podman Engine
+	*/
+	if !strings.Contains(result.Output(), "Podman") {
+		t.Fatal("failed to detect keyword 'Podman' in output of docker version")
+	}
+
+	// now try to build an image
+	cmd = exec.CommandContext(ctx, "/bin/bash", "-c", fmt.Sprintf("SSH_AUTH_SOCK=%s SSH_AGENT_PID=%s DOCKER_HOST=%s DOCKER_BUILDKIT=0 docker build -t local/minikube-dockerenv-crio-test:latest testdata/docker-env", sshAuthSock, sshAgentPid, dockerHost))
+	result, err = Run(t, cmd)
+	if err != nil {
+		t.Errorf("failed to build images, error: %v, output:%s", err, result.Output())
+	}
+
+	// and check whether that image is really available
+	cmd = exec.CommandContext(ctx, "/bin/bash", "-c", fmt.Sprintf("SSH_AUTH_SOCK=%s SSH_AGENT_PID=%s DOCKER_HOST=%s docker image ls", sshAuthSock, sshAgentPid, dockerHost))
+	result, err = Run(t, cmd)
+	if err != nil {
+		t.Fatalf("failed to execute 'docker image ls', error: %v, output: %s", err, result.Output())
+	}
+	if !strings.Contains(result.Output(), "local/minikube-dockerenv-crio-test") {
+		t.Fatal("failed to detect image 'local/minikube-dockerenv-crio-test' in output of docker image ls")
+	}
+}