tests: kubernetes worker

Signed-off-by: CrazyMax <[email protected]>

Author: crazy-max

.github/workflows/build.yml

@@ -60,6 +60,7 @@ jobs:
         worker:
           - docker-container
           - remote
+          - kubernetes
         pkg:
           - ./tests
         mode:

Dockerfile

@@ -10,6 +10,7 @@ ARG GOTESTSUM_VERSION=v1.12.0
 ARG REGISTRY_VERSION=2.8.3
 ARG BUILDKIT_VERSION=v0.17.1
 ARG UNDOCK_VERSION=0.8.0
+ARG K3S_VERSION=v1.21.2-k3s1
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS golatest
@@ -18,6 +19,7 @@ FROM dockereng/cli-bin:$DOCKER_CLI_VERSION AS docker-cli
 FROM registry:$REGISTRY_VERSION AS registry
 FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
 FROM crazymax/undock:$UNDOCK_VERSION AS undock
+FROM rancher/k3s:${K3S_VERSION} AS k3s
 
 FROM golatest AS gobase
 COPY --from=xx / /
@@ -118,14 +120,44 @@ RUN apk add --no-cache \
       shadow-uidmap \
       xfsprogs \
       xz
+# k3s deps
+RUN apk add --no-cache \
+      busybox-binsh \
+      cni-plugins \
+      cni-plugin-flannel \
+      conntrack-tools \
+      coreutils \
+      dbus \
+      findutils \
+      ipset
+ENV PATH="/usr/libexec/cni:${PATH}"
 COPY --link --from=gotestsum /out /usr/bin/
 COPY --link --from=registry /bin/registry /usr/bin/
 COPY --link --from=docker-engine / /usr/bin/
 COPY --link --from=docker-cli / /usr/bin/
+COPY --link --from=k3s /bin/k3s /usr/bin/
+COPY --link --from=k3s /bin/kubectl /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
 COPY --link --from=undock /usr/local/bin/undock /usr/bin/
 COPY --link --from=binaries /buildx /usr/bin/
+COPY --chmod=755 <<-"EOF" /entrypoint.sh
+#!/bin/sh
+set -e
+# cgroup v2: enable nesting
+# https://github.com/moby/moby/blob/v25.0.0/hack/dind#L59-L69
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+  # move the processes from the root group to the /init group,
+  # otherwise writing subtree_control fails with EBUSY.
+  # An error during moving non-existent process (i.e., "cat") is ignored.
+  mkdir -p /sys/fs/cgroup/init
+  xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+  # enable controllers
+  sed -e 's/ / +/g' -e 's/^/+/' < /sys/fs/cgroup/cgroup.controllers > /sys/fs/cgroup/cgroup.subtree_control
+fi
+exec "$@"
+EOF
+ENTRYPOINT ["/entrypoint.sh"]
 
 FROM integration-test-base AS integration-test
 COPY . .

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.2.1
 	github.com/Microsoft/go-winio v0.6.2
 	github.com/aws/aws-sdk-go-v2/config v1.26.6
+	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/compose-spec/compose-go/v2 v2.4.2
 	github.com/containerd/console v1.0.4
 	github.com/containerd/containerd v1.7.22
@@ -82,7 +83,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sts v1.26.7 // indirect
 	github.com/aws/smithy-go v1.19.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/containerd/containerd/api v1.7.19 // indirect
 	github.com/containerd/ttrpc v1.2.5 // indirect

tests/helpers/k3s.go

@@ -0,0 +1,112 @@
+package helpers
+
+import (
+	"bytes"
+	"net"
+	"os"
+	"os/exec"
+	"strconv"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/pkg/errors"
+)
+
+const (
+	k3sBin     = "k3s"
+	kubeCtlBin = "kubectl"
+)
+
+func NewK3sServer(cfg *integration.BackendConfig) (kubeConfig string, cl func() error, err error) {
+	if _, err := exec.LookPath(k3sBin); err != nil {
+		return "", nil, errors.Wrapf(err, "failed to lookup %s binary", k3sBin)
+	}
+	if _, err := exec.LookPath(kubeCtlBin); err != nil {
+		return "", nil, errors.Wrapf(err, "failed to lookup %s binary", kubeCtlBin)
+	}
+
+	deferF := &integration.MultiCloser{}
+	cl = deferF.F()
+
+	defer func() {
+		if err != nil {
+			deferF.F()()
+			cl = nil
+		}
+	}()
+
+	cfgfile, err := os.CreateTemp("", "kubeconfig*.yml")
+	if err != nil {
+		return "", nil, err
+	}
+	kubeConfig = cfgfile.Name()
+	deferF.Append(func() error {
+		return os.Remove(cfgfile.Name())
+	})
+
+	k3sDataDir, err := os.MkdirTemp("", "kubedata")
+	if err != nil {
+		return "", nil, err
+	}
+	deferF.Append(func() error {
+		return os.RemoveAll(k3sDataDir)
+	})
+
+	l, err := net.Listen("tcp", "localhost:0")
+	if err != nil {
+		return "", nil, err
+	}
+	_ = l.Close()
+
+	lport := strconv.Itoa(l.Addr().(*net.TCPAddr).Port)
+	nodeName := "integrationk3s"
+
+	stop, err := integration.StartCmd(exec.Command(k3sBin, "server",
+		"--bind-address", "127.0.0.1",
+		"--https-listen-port", lport,
+		"--data-dir", k3sDataDir, // write to /tmp for overlayfs support
+		"--write-kubeconfig", cfgfile.Name(),
+		"--write-kubeconfig-mode", "666",
+		"--node-name", nodeName,
+	), cfg.Logs)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if err = waitK3s(cfg, kubeConfig, nodeName); err != nil {
+		stop()
+		return "", nil, errors.Wrapf(err, "k3s did not start up: %s", integration.FormatLogs(cfg.Logs))
+	}
+
+	deferF.Append(stop)
+	return
+}
+
+func waitK3s(cfg *integration.BackendConfig, kubeConfig string, nodeName string) error {
+	logbuf := new(bytes.Buffer)
+	defer func() {
+		if logbuf.Len() > 0 {
+			cfg.Logs["waitK3s: "] = logbuf
+		}
+	}()
+
+	boff := backoff.NewExponentialBackOff()
+	boff.InitialInterval = 3 * time.Second
+	boff.MaxInterval = 5 * time.Second
+	boff.MaxElapsedTime = 2 * time.Minute
+
+	if err := backoff.Retry(func() error {
+		cmd := exec.Command(kubeCtlBin, "--kubeconfig", kubeConfig, "wait", "--for=condition=Ready", "node/"+nodeName)
+		out, err := cmd.CombinedOutput()
+		if err == nil && bytes.Contains(out, []byte("condition met")) {
+			return nil
+		}
+		return errors.Wrapf(err, "node is not ready: %s %s", cmd.String(), string(out))
+	}, boff); err != nil {
+		logbuf.WriteString(errors.Unwrap(err).Error())
+		return err
+	}
+
+	return nil
+}

tests/integration_test.go

@@ -16,6 +16,7 @@ func init() {
 		workers.InitDockerContainerWorker()
 	} else {
 		workers.InitRemoteWorker()
+		workers.InitKubernetesWorker()
 	}
 }
 

tests/workers/kubernetes.go

@@ -0,0 +1,100 @@
+package workers
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+
+	"github.com/docker/buildx/tests/helpers"
+	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/util/testutil/integration"
+	"github.com/pkg/errors"
+)
+
+func InitKubernetesWorker() {
+	integration.Register(&kubernetesWorker{
+		id: "kubernetes",
+	})
+}
+
+type kubernetesWorker struct {
+	id string
+
+	unsupported []string
+
+	k3sConfig string
+	k3sClose  func() error
+	k3sErr    error
+	k3sOnce   sync.Once
+}
+
+func (w *kubernetesWorker) Name() string {
+	return w.id
+}
+
+func (w *kubernetesWorker) Rootless() bool {
+	return false
+}
+
+func (w *kubernetesWorker) NetNSDetached() bool {
+	return false
+}
+
+func (w *kubernetesWorker) New(ctx context.Context, cfg *integration.BackendConfig) (integration.Backend, func() error, error) {
+	var err error
+
+	w.k3sOnce.Do(func() {
+		w.k3sConfig, w.k3sClose, w.k3sErr = helpers.NewK3sServer(cfg)
+	})
+	if w.k3sErr != nil {
+		return nil, w.k3sClose, w.k3sErr
+	}
+
+	cfgfile, err := integration.WriteConfig(cfg.DaemonConfig)
+	if err != nil {
+		return nil, nil, err
+	}
+	defer os.RemoveAll(filepath.Dir(cfgfile))
+
+	name := "integration-kubernetes-" + identity.NewID()
+	cmd := exec.Command("buildx", "create",
+		"--bootstrap",
+		"--name="+name,
+		"--config="+cfgfile,
+		"--driver=kubernetes",
+	)
+	cmd.Env = append(
+		os.Environ(),
+		"BUILDX_CONFIG=/tmp/buildx-"+name,
+		"KUBECONFIG="+w.k3sConfig,
+	)
+	if err := cmd.Run(); err != nil {
+		return nil, nil, errors.Wrapf(err, "failed to create buildx instance %s", name)
+	}
+
+	cl := func() error {
+		cmd := exec.Command("buildx", "rm", "-f", name)
+		return cmd.Run()
+	}
+
+	return &backend{
+		builder:             name,
+		unsupportedFeatures: w.unsupported,
+	}, cl, nil
+}
+
+func (w *kubernetesWorker) Close() error {
+	if c := w.k3sClose; c != nil {
+		return c()
+	}
+
+	// reset the worker to be ready to go again
+	w.k3sConfig = ""
+	w.k3sClose = nil
+	w.k3sErr = nil
+	w.k3sOnce = sync.Once{}
+
+	return nil
+}