mz507: normalize images

.github/workflows/integration-tests.yaml

@@ -49,9 +49,10 @@ jobs:
 
       - uses: docker/setup-docker-action@1a6edb0ba9ac496f6850236981f15d8f9a82254d # v5.0.0
         with:
-          version: 28.5.2
+          version: 29.2.1
 
       - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
 
-      - run: make install-diffoci install-crane k3s-setup
+      - run: sudo apt install -y libgpgme-dev libassuan-dev libbtrfs-dev pkg-config # skopeo dependencies
+      - run: make install-diffoci install-crane install-skopeo k3s-setup
       - run: make ${{ matrix.make-target }}

Makefile

@@ -64,6 +64,10 @@ install-diffoci:
 install-crane:
 	@ GOFLAGS="" go install github.com/google/go-containerregistry/cmd/crane@latest
 
+.PHONY: install-skopeo
+install-skopeo:
+	@ GOFLAGS="" go install github.com/containers/skopeo/cmd/skopeo@latest
+
 .PHONY: k3s-setup
 k3s-setup:
 	@ ./scripts/k3s-setup.sh

integration/images.go

@@ -387,6 +387,7 @@ func (d *DockerFileBuilder) BuildDockerImage(t *testing.T, imageRepo, dockerfile
 	if env, ok := envsMap[dockerfile]; ok {
 		dockerCmd.Env = append(dockerCmd.Env, env...)
 	}
+	dockerCmd.Env = append(dockerCmd.Env, "BUILDX_NO_DEFAULT_ATTESTATIONS=1")
 
 	out, err := RunCommandWithoutTest(dockerCmd)
 	if err != nil {
@@ -620,6 +621,7 @@ func (d *DockerFileBuilder) buildRelativePathsImage(logf logger, imageRepo, dock
 			"./context",
 		}...,
 	)
+	dockerCmd.Env = []string{"BUILDX_NO_DEFAULT_ATTESTATIONS=1"}
 
 	timer := timing.Start(dockerfile + "_docker")
 	out, err := RunCommandWithoutTest(dockerCmd)

integration/integration_test.go

@@ -215,7 +215,11 @@ func TestRun(t *testing.T) {
 			dockerImage := GetDockerImage(config.imageRepo, dockerfile)
 			kanikoImage := GetKanikoImage(config.imageRepo, dockerfile)
 
-			containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
+			pullImage(t, kanikoImage)
+			_dockerImage := normalizeImageFormat(t, dockerImage)
+			_kanikoImage := normalizeImageFormat(t, kanikoImage)
+
+			containerDiff(t, daemonPrefix+_dockerImage, daemonPrefix+_kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
 		})
 	}
 
@@ -279,6 +283,7 @@ func testGitBuildcontextHelper(t *testing.T, url string, commit string, branch s
 			"-f", dockerfile,
 			DockerGitRepo(url, commit, branch),
 		}...)
+	dockerCmd.Env = []string{"BUILDX_NO_DEFAULT_ATTESTATIONS=1"}
 	out, err := RunCommandWithoutTest(dockerCmd)
 	if err != nil {
 		t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out))
@@ -300,7 +305,11 @@ func testGitBuildcontextHelper(t *testing.T, url string, commit string, branch s
 		t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
 	}
 
-	containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
+	pullImage(t, kanikoImage)
+	_dockerImage := normalizeImageFormat(t, dockerImage)
+	_kanikoImage := normalizeImageFormat(t, kanikoImage)
+
+	containerDiff(t, daemonPrefix+_dockerImage, daemonPrefix+_kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
 }
 
 // TestGitBuildcontext explicitly names the main branch
@@ -348,6 +357,7 @@ func TestGitBuildcontextSubPath(t *testing.T) {
 			"-f", filepath.Join(integrationPath, dockerfilesPath, dockerfile),
 			DockerGitRepo(url, "", branch),
 		}...)
+	dockerCmd.Env = []string{"BUILDX_NO_DEFAULT_ATTESTATIONS=1"}
 	out, err := RunCommandWithoutTest(dockerCmd)
 	if err != nil {
 		t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out))
@@ -373,7 +383,11 @@ func TestGitBuildcontextSubPath(t *testing.T) {
 		t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
 	}
 
-	containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
+	pullImage(t, kanikoImage)
+	_dockerImage := normalizeImageFormat(t, dockerImage)
+	_kanikoImage := normalizeImageFormat(t, kanikoImage)
+
+	containerDiff(t, daemonPrefix+_dockerImage, daemonPrefix+_kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
 }
 
 func TestBuildViaRegistryMirrors(t *testing.T) {
@@ -390,6 +404,7 @@ func TestBuildViaRegistryMirrors(t *testing.T) {
 			"-f", dockerfile,
 			DockerGitRepo(url, "", branch),
 		}...)
+	dockerCmd.Env = []string{"BUILDX_NO_DEFAULT_ATTESTATIONS=1"}
 	out, err := RunCommandWithoutTest(dockerCmd)
 	if err != nil {
 		t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out))
@@ -413,7 +428,11 @@ func TestBuildViaRegistryMirrors(t *testing.T) {
 		t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
 	}
 
-	containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
+	pullImage(t, kanikoImage)
+	_dockerImage := normalizeImageFormat(t, dockerImage)
+	_kanikoImage := normalizeImageFormat(t, kanikoImage)
+
+	containerDiff(t, daemonPrefix+_dockerImage, daemonPrefix+_kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
 }
 
 func TestBuildViaRegistryMap(t *testing.T) {
@@ -430,6 +449,7 @@ func TestBuildViaRegistryMap(t *testing.T) {
 			"-f", dockerfile,
 			DockerGitRepo(url, "", branch),
 		}...)
+	dockerCmd.Env = []string{"BUILDX_NO_DEFAULT_ATTESTATIONS=1"}
 	out, err := RunCommandWithoutTest(dockerCmd)
 	if err != nil {
 		t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out))
@@ -453,7 +473,11 @@ func TestBuildViaRegistryMap(t *testing.T) {
 		t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
 	}
 
-	containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
+	pullImage(t, kanikoImage)
+	_dockerImage := normalizeImageFormat(t, dockerImage)
+	_kanikoImage := normalizeImageFormat(t, kanikoImage)
+
+	containerDiff(t, daemonPrefix+_dockerImage, daemonPrefix+_kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
 }
 
 func TestBuildSkipFallback(t *testing.T) {
@@ -495,6 +519,7 @@ func TestKanikoDir(t *testing.T) {
 			"-f", dockerfile,
 			DockerGitRepo(url, "", branch),
 		}...)
+	dockerCmd.Env = []string{"BUILDX_NO_DEFAULT_ATTESTATIONS=1"}
 	out, err := RunCommandWithoutTest(dockerCmd)
 	if err != nil {
 		t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out))
@@ -517,7 +542,11 @@ func TestKanikoDir(t *testing.T) {
 		t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
 	}
 
-	containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
+	pullImage(t, kanikoImage)
+	_dockerImage := normalizeImageFormat(t, dockerImage)
+	_kanikoImage := normalizeImageFormat(t, kanikoImage)
+
+	containerDiff(t, daemonPrefix+_dockerImage, daemonPrefix+_kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
 }
 
 func TestBuildWithLabels(t *testing.T) {
@@ -537,6 +566,7 @@ func TestBuildWithLabels(t *testing.T) {
 			"--label", testLabel,
 			DockerGitRepo(url, "", branch),
 		}...)
+	dockerCmd.Env = []string{"BUILDX_NO_DEFAULT_ATTESTATIONS=1"}
 	out, err := RunCommandWithoutTest(dockerCmd)
 	if err != nil {
 		t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out))
@@ -560,7 +590,11 @@ func TestBuildWithLabels(t *testing.T) {
 		t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
 	}
 
-	containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
+	pullImage(t, kanikoImage)
+	_dockerImage := normalizeImageFormat(t, dockerImage)
+	_kanikoImage := normalizeImageFormat(t, kanikoImage)
+
+	containerDiff(t, daemonPrefix+_dockerImage, daemonPrefix+_kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
 }
 
 func TestBuildWithHTTPError(t *testing.T) {
@@ -577,6 +611,7 @@ func TestBuildWithHTTPError(t *testing.T) {
 			"-f", dockerfile,
 			DockerGitRepo(url, "", branch),
 		}...)
+	dockerCmd.Env = []string{"BUILDX_NO_DEFAULT_ATTESTATIONS=1"}
 	out, err := RunCommandWithoutTest(dockerCmd)
 	if err == nil {
 		t.Errorf("an error was expected, got %s", string(out))
@@ -720,7 +755,12 @@ func TestWarmer(t *testing.T) {
 			kanikoVersion0 := GetKanikoImage(imageRepo, "test_warmer_"+dockerfile) + strconv.Itoa(0)
 			kanikoVersion1 := GetKanikoImage(imageRepo, "test_warmer_"+dockerfile) + strconv.Itoa(1)
 
-			containerDiff(t, kanikoVersion0, kanikoVersion1)
+			pullImage(t, kanikoVersion0)
+			_kanikoVersion0 := normalizeImageFormat(t, kanikoVersion0)
+			pullImage(t, kanikoVersion1)
+			_kanikoVersion1 := normalizeImageFormat(t, kanikoVersion1)
+
+			containerDiff(t, daemonPrefix+_kanikoVersion0, daemonPrefix+_kanikoVersion1)
 			layerDiff(t, kanikoVersion0, kanikoVersion1)
 			manifestDiff(t, kanikoVersion0, kanikoVersion1)
 		})
@@ -801,7 +841,12 @@ func verifyBuildWith(t *testing.T, cache, dockerfile string) {
 	kanikoVersion0 := GetVersionedKanikoImage(config.imageRepo, dockerfile, 0)
 	kanikoVersion1 := GetVersionedKanikoImage(config.imageRepo, dockerfile, 1)
 
-	containerDiff(t, kanikoVersion0, kanikoVersion1)
+	pullImage(t, kanikoVersion0)
+	_kanikoVersion0 := normalizeImageFormat(t, kanikoVersion0)
+	pullImage(t, kanikoVersion1)
+	_kanikoVersion1 := normalizeImageFormat(t, kanikoVersion1)
+
+	containerDiff(t, daemonPrefix+_kanikoVersion0, daemonPrefix+_kanikoVersion1)
 	layerDiff(t, kanikoVersion0, kanikoVersion1)
 }
 
@@ -829,7 +874,11 @@ func TestRelativePaths(t *testing.T) {
 		dockerImage := GetDockerImage(config.imageRepo, "test_relative_"+dockerfile)
 		kanikoImage := GetKanikoImage(config.imageRepo, "test_relative_"+dockerfile)
 
-		containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
+		pullImage(t, kanikoImage)
+		_dockerImage := normalizeImageFormat(t, dockerImage)
+		_kanikoImage := normalizeImageFormat(t, kanikoImage)
+
+		containerDiff(t, daemonPrefix+_dockerImage, daemonPrefix+_kanikoImage, "--semantic", "--extra-ignore-file-content", "--extra-ignore-layer-length-mismatch")
 	})
 }
 
@@ -921,6 +970,7 @@ func TestBuildWithAnnotations(t *testing.T) {
 		"-f", dockerfile,
 		DockerGitRepo(url, "", branch),
 	)
+	dockerCmd.Env = []string{"BUILDX_NO_DEFAULT_ATTESTATIONS=1"}
 	out, err := RunCommandWithoutTest(dockerCmd)
 	if err != nil {
 		t.Errorf("Failed to build image %s with docker command %q: %s %s", dockerImage, dockerCmd.Args, err, string(out))
@@ -953,7 +1003,13 @@ func TestBuildWithAnnotations(t *testing.T) {
 	if err != nil {
 		t.Errorf("Failed to build image %s with kaniko command %q: %v %s", dockerImage, kanikoCmd.Args, err, string(out))
 	}
-	containerDiff(t, daemonPrefix+dockerImage, kanikoImage, "--ignore-history")
+
+	pullImage(t, kanikoImage)
+	pullImage(t, dockerImage)
+	_dockerImage := normalizeImageFormat(t, dockerImage)
+	_kanikoImage := normalizeImageFormat(t, kanikoImage)
+
+	containerDiff(t, daemonPrefix+_dockerImage, daemonPrefix+_kanikoImage, "--ignore-history")
 
 	dockerAnnotations, err := getImageManifestAnnotations(t, dockerImage)
 	if err != nil {
@@ -1294,7 +1350,7 @@ func initIntegrationTestConfig() *integrationTestConfig {
 }
 
 func meetsRequirements() bool {
-	requiredTools := []string{"diffoci"}
+	requiredTools := []string{"diffoci", "skopeo"}
 	hasRequirements := true
 	for _, tool := range requiredTools {
 		_, err := exec.LookPath(tool)
@@ -1308,21 +1364,6 @@ func meetsRequirements() bool {
 
 // containerDiff compares the container images image1 and image2.
 func containerDiff(t *testing.T, image1, image2 string, flags ...string) []byte {
-	// workaround for container-diff OCI issue https://github.com/GoogleContainerTools/container-diff/issues/389
-	if !strings.HasPrefix(image1, daemonPrefix) {
-		dockerPullCmd := exec.Command("docker", "pull", image1)
-		out := RunCommand(dockerPullCmd, t)
-		t.Logf("docker pull cmd output for image1 = %s", string(out))
-		image1 = daemonPrefix + image1
-	}
-
-	if !strings.HasPrefix(image2, daemonPrefix) {
-		dockerPullCmd := exec.Command("docker", "pull", image2)
-		out := RunCommand(dockerPullCmd, t)
-		t.Logf("docker pull cmd output for image2 = %s", string(out))
-		image2 = daemonPrefix + image2
-	}
-
 	flags = append([]string{"diff"}, flags...)
 	flags = append(flags, image1, image2, "--ignore-image-name", "--ignore-image-timestamps")
 	flags = append(flags, diffArgsMap[t.Name()]...)
@@ -1333,3 +1374,32 @@ func containerDiff(t *testing.T, image1, image2 string, flags ...string) []byte
 
 	return diff
 }
+
+func pullImage(t *testing.T, image string) {
+	t.Helper()
+	cmd := exec.Command("docker", "pull", image)
+	out, err := RunCommandWithoutTest(cmd)
+	t.Logf("docker pull:\n%s", out)
+	if err != nil {
+		t.Fatalf("docker pull: %s", err)
+	}
+}
+
+// normalizeImageFormat pulls image to the Docker daemon (if not already present)
+// and converts it to Docker V2S2 format using skopeo. This ensures both images
+// are in the same format before comparison, replicating the normalization that
+// legacy Docker storage performs on pull.
+func normalizeImageFormat(t *testing.T, image string) string {
+	t.Helper()
+	taggedRef := image + ":latest"
+	normalized := taggedRef + "-v2s2"
+	cmd := exec.Command("skopeo", "copy", "--format", "v2s2",
+		"docker-daemon:"+taggedRef,
+		"docker-daemon:"+normalized)
+	out, err := RunCommandWithoutTest(cmd)
+	t.Logf("skopeo normalize %s: %s", image, string(out))
+	if err != nil {
+		t.Fatalf("failed to normalize image %s to v2s2: %v\n%s", image, err, string(out))
+	}
+	return normalized
+}