Merge pull request docker#2307 from crazy-max/test-docker-multi-ver

tests: handle multiple docker versions

Author: tonistiigi

.github/workflows/build.yml

@@ -76,6 +76,16 @@ jobs:
           - worker: docker+containerd  # same as docker, but with containerd snapshotter
             pkg: ./tests
             mode: experimental
+          - worker: "[email protected]"
+            pkg: ./tests
+          - worker: "[email protected]"  # same as docker, but with containerd snapshotter
+            pkg: ./tests
+          - worker: "[email protected]"
+            pkg: ./tests
+            mode: experimental
+          - worker: "[email protected]"  # same as docker, but with containerd snapshotter
+            pkg: ./tests
+            mode: experimental
     steps:
       -
         name: Prepare
@@ -86,7 +96,7 @@ jobs:
           fi
           testFlags="--run=//worker=$(echo "${{ matrix.worker }}" | sed 's/\+/\\+/g')$"
           case "${{ matrix.worker }}" in
-            docker | docker+containerd)
+            docker | docker+containerd | docker@* | docker+containerd@*)
               echo "TESTFLAGS=${{ env.TESTFLAGS_DOCKER }} $testFlags" >> $GITHUB_ENV
               ;;
             *)

Dockerfile

@@ -5,6 +5,7 @@ ARG XX_VERSION=1.5.0
 
 # for testing
 ARG DOCKER_VERSION=27.4.0-rc.2
+ARG DOCKER_VERSION_ALT_26=26.1.3
 ARG DOCKER_CLI_VERSION=${DOCKER_VERSION}
 ARG GOTESTSUM_VERSION=v1.12.0
 ARG REGISTRY_VERSION=2.8.3
@@ -15,6 +16,8 @@ FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx
 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS golatest
 FROM moby/moby-bin:$DOCKER_VERSION AS docker-engine
 FROM dockereng/cli-bin:$DOCKER_CLI_VERSION AS docker-cli
+FROM moby/moby-bin:$DOCKER_VERSION_ALT_26 AS docker-engine-alt
+FROM dockereng/cli-bin:$DOCKER_VERSION_ALT_26 AS docker-cli-alt
 FROM registry:$REGISTRY_VERSION AS registry
 FROM moby/buildkit:$BUILDKIT_VERSION AS buildkit
 FROM crazymax/undock:$UNDOCK_VERSION AS undock
@@ -123,10 +126,13 @@ COPY --link --from=gotestsum /out /usr/bin/
 COPY --link --from=registry /bin/registry /usr/bin/
 COPY --link --from=docker-engine / /usr/bin/
 COPY --link --from=docker-cli / /usr/bin/
+COPY --link --from=docker-engine-alt / /opt/docker-alt-26/
+COPY --link --from=docker-cli-alt / /opt/docker-alt-26/
 COPY --link --from=buildkit /usr/bin/buildkitd /usr/bin/
 COPY --link --from=buildkit /usr/bin/buildctl /usr/bin/
 COPY --link --from=undock /usr/local/bin/undock /usr/bin/
 COPY --link --from=binaries /buildx /usr/bin/
+ENV TEST_DOCKER_EXTRA="[email protected]=/opt/docker-alt-26"
 
 FROM integration-test-base AS integration-test
 COPY . .

tests/build.go

@@ -505,7 +505,7 @@ RUN exit 1`)
 
 func testBuildProgress(t *testing.T, sb integration.Sandbox) {
 	dir := createTestProject(t)
-	sbDriver, _ := driverName(sb.Name())
+	sbDriver, _, _ := driverName(sb.Name())
 	name := sb.Address()
 
 	// progress=tty

tests/inspect.go

@@ -45,7 +45,7 @@ func testInspect(t *testing.T, sb integration.Sandbox) {
 	}
 
 	require.Equal(t, sb.Address(), name)
-	sbDriver, _ := driverName(sb.Name())
+	sbDriver, _, _ := driverName(sb.Name())
 	require.Equal(t, sbDriver, driver)
 	if isDockerWorker(sb) {
 		require.NotEmpty(t, hostGatewayIP, "host-gateway-ip worker label should be set with docker driver")

tests/integration.go

@@ -95,33 +95,37 @@ func buildxConfig(sb integration.Sandbox) string {
 }
 
 func isMobyWorker(sb integration.Sandbox) bool {
-	name, hasFeature := driverName(sb.Name())
+	name, _, hasFeature := driverName(sb.Name())
 	return name == "docker" && !hasFeature
 }
 
 func isMobyContainerdSnapWorker(sb integration.Sandbox) bool {
-	name, hasFeature := driverName(sb.Name())
+	name, _, hasFeature := driverName(sb.Name())
 	return name == "docker" && hasFeature
 }
 
 func isDockerWorker(sb integration.Sandbox) bool {
-	name, _ := driverName(sb.Name())
+	name, _, _ := driverName(sb.Name())
 	return name == "docker"
 }
 
 func isDockerContainerWorker(sb integration.Sandbox) bool {
-	name, _ := driverName(sb.Name())
+	name, _, _ := driverName(sb.Name())
 	return name == "docker-container"
 }
 
-func driverName(sbName string) (string, bool) {
+func driverName(sbName string) (string, bool, bool) {
 	name := sbName
-	var hasFeature bool
+	var hasVersion, hasFeature bool
+	if b, _, ok := strings.Cut(sbName, "@"); ok {
+		name = b
+		hasVersion = true
+	}
 	if b, _, ok := strings.Cut(name, "+"); ok {
 		name = b
 		hasFeature = true
 	}
-	return name, hasFeature
+	return name, hasVersion, hasFeature
 }
 
 func isExperimental() bool {

tests/ls.go

@@ -34,7 +34,7 @@ func testLs(t *testing.T, sb integration.Sandbox) {
 		},
 	}
 
-	sbDriver, _ := driverName(sb.Name())
+	sbDriver, _, _ := driverName(sb.Name())
 	for _, tt := range tests {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {

tests/workers/docker.go

@@ -2,29 +2,76 @@ package workers
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
+	"strings"
 
 	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/util/testutil/dockerd"
 	"github.com/moby/buildkit/util/testutil/integration"
 	bkworkers "github.com/moby/buildkit/util/testutil/workers"
 	"github.com/pkg/errors"
 )
 
 func InitDockerWorker() {
 	integration.Register(&dockerWorker{
-		id: "docker",
+		id:     "docker",
+		binary: dockerd.DefaultDockerdBinary,
 	})
 	integration.Register(&dockerWorker{
 		id:                    "docker+containerd",
+		binary:                dockerd.DefaultDockerdBinary,
 		containerdSnapshotter: true,
 	})
+	// e.g. `[email protected]=/opt/docker-26.0,[email protected]=/opt/docker-25.0`
+	if s := os.Getenv("TEST_DOCKER_EXTRA"); s != "" {
+		entries := strings.Split(s, ",")
+		for _, entry := range entries {
+			ver, bin, err := func(entry string) (string, string, error) {
+				p1 := strings.Split(strings.TrimSpace(entry), "=")
+				if len(p1) != 2 {
+					return "", "", errors.Errorf("invalid entry: %q", entry)
+				}
+				name, bin := p1[0], p1[1]
+				if _, err := os.Stat(bin); err != nil {
+					return "", "", errors.Wrapf(err, "bin not found: %q", bin)
+				}
+				p2 := strings.Split(strings.TrimSpace(name), "@")
+				if len(p2) != 2 {
+					return "", "", errors.Errorf("invalid name: %q", name)
+				}
+				_, ver := p2[0], p2[1]
+				if ver == "" {
+					return "", "", errors.New("empty version")
+				}
+				return ver, bin, nil
+			}(entry)
+			if err != nil {
+				panic(errors.Wrapf(err, "unexpected TEST_DOCKER_EXTRA: %q", s))
+			}
+			integration.Register(&dockerWorker{
+				id:       fmt.Sprintf("docker@%s", ver),
+				binary:   filepath.Join(bin, "dockerd"),
+				extraEnv: []string{fmt.Sprintf("PATH=%s:%s", bin, os.Getenv("PATH"))},
+			})
+			integration.Register(&dockerWorker{
+				id:                    fmt.Sprintf("docker+containerd@%s", ver),
+				binary:                filepath.Join(bin, "dockerd"),
+				containerdSnapshotter: true,
+				extraEnv:              []string{fmt.Sprintf("PATH=%s:%s", bin, os.Getenv("PATH"))},
+			})
+		}
+	}
 }
 
 type dockerWorker struct {
 	id                    string
+	binary                string
 	containerdSnapshotter bool
 	unsupported           []string
+	extraEnv              []string
 }
 
 func (c dockerWorker) Name() string {
@@ -42,7 +89,9 @@ func (c *dockerWorker) NetNSDetached() bool {
 func (c dockerWorker) New(ctx context.Context, cfg *integration.BackendConfig) (b integration.Backend, cl func() error, err error) {
 	moby := bkworkers.Moby{
 		ID:                    c.id,
+		Binary:                c.binary,
 		ContainerdSnapshotter: c.containerdSnapshotter,
+		ExtraEnv:              c.extraEnv,
 	}
 	bk, bkclose, err := moby.New(ctx, cfg)
 	if err != nil {