diff --git a/integration/images.go b/integration/images.go index 307f78ed3..e1b70a5a5 100644 --- a/integration/images.go +++ b/integration/images.go @@ -85,8 +85,12 @@ var envsMap = map[string][]string{ var KanikoEnv = []string{ "FF_KANIKO_COPY_AS_ROOT=1", "FF_KANIKO_OCI_STAGES=1", - "FF_KANIKO_IGNORE_CACHED_MANIFEST=1", "FF_KANIKO_RUN_MOUNT_SECRET=1", + "FF_KANIKO_OCI_WARMER=1", +} + +var WarmerEnv = []string{ + "FF_KANIKO_OCI_WARMER=1", } // Arguments to build Dockerfiles with when building with docker @@ -177,9 +181,12 @@ var outputChecks = map[string]func(string, []byte) error{ }, } +// Digest for debian:12.10 (see baseImageToCache) +const debian1210Digest = "6bc30d909583f38600edd6609e29eb3fb284ab8affce8d0389f332fc91c2dd91" + var warmerOutputChecks = map[string]func(string, []byte) error{ "Dockerfile_test_issue_mz320": func(_ string, out []byte) error { - s := "Found sha256:6bc30d909583f38600edd6609e29eb3fb284ab8affce8d0389f332fc91c2dd91 in local cache" + s := fmt.Sprintf("Found sha256:%s in local cache", debian1210Digest) if !strings.Contains(string(out), s) { return fmt.Errorf("output must contain %s", s) } @@ -272,6 +279,7 @@ type DockerFileBuilder struct { DockerfilesToIgnore map[string]struct{} TestCacheDockerfiles map[string]struct{} TestOCICacheDockerfiles map[string]struct{} + TestWarmerDockerfiles map[string]struct{} } type logger func(string, ...interface{}) @@ -302,7 +310,6 @@ func NewDockerFileBuilder() *DockerFileBuilder { "Dockerfile_test_issue_workdir": {}, "Dockerfile_test_issue_add": {}, "Dockerfile_test_issue_empty": {}, - "Dockerfile_test_issue_mz320": {}, } d.TestOCICacheDockerfiles = map[string]struct{}{ "Dockerfile_test_cache_oci": {}, @@ -310,6 +317,9 @@ func NewDockerFileBuilder() *DockerFileBuilder { "Dockerfile_test_cache_perm_oci": {}, "Dockerfile_test_cache_copy_oci": {}, } + d.TestWarmerDockerfiles = map[string]struct{}{ + "Dockerfile_test_issue_mz320": {}, + } return &d } @@ -433,24 +443,31 @@ func (d *DockerFileBuilder) BuildImageWithContext(t *testing.T, config *integrat return nil } -func populateVolumeCache() error { +func populateVolumeCache(logf logger, serviceAccount string) error { + fmt.Println("Populating warmer cache") _, ex, _, _ := runtime.Caller(0) cwd := filepath.Dir(ex) - warmerCmd := exec.Command("docker", - []string{ - "run", "--net=host", - "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", - "-v", cwd + ":/workspace", - WarmerImage, - "-c", cacheDir, - "-i", baseImageToCache, - }..., + cmd := []string{ + "run", "--net=host", + "-v", os.Getenv("HOME") + "/.config/gcloud:/root/.config/gcloud", + "-v", cwd + ":/workspace", + } + for _, envVariable := range WarmerEnv { + cmd = append(cmd, "-e", envVariable) + } + cmd = addServiceAccountFlags(cmd, serviceAccount) + cmd = append(cmd, + WarmerImage, + "-c", cacheDir, + "-i", baseImageToCache, ) - if _, err := RunCommandWithoutTest(warmerCmd); err != nil { + warmerCmd := exec.Command("docker", cmd...) + out, err := RunCommandWithoutTest(warmerCmd) + logf(string(out)) + if err != nil { return fmt.Errorf("failed to warm kaniko cache: %w", err) } - return nil } @@ -513,6 +530,58 @@ func (d *DockerFileBuilder) buildCachedImage(logf logger, config *integrationTes return nil } +// buildCachedImage builds the image for testing caching via kaniko warmer cache where version is the nth time this image has been built +func (d *DockerFileBuilder) buildWarmerImage(logf logger, config *integrationTestConfig, dockerfilesPath, dockerfile string, version int, args []string, cache bool) error { + imageRepo, serviceAccount := config.imageRepo, config.serviceAccount + _, ex, _, _ := runtime.Caller(0) + cwd := filepath.Dir(ex) + + kanikoImage := GetKanikoImage(imageRepo, "test_warmer_"+dockerfile) + strconv.Itoa(version) + + dockerRunFlags := []string{ + "run", "--net=host", + "-v", cwd + ":/workspace:ro", + } + for _, envVariable := range KanikoEnv { + dockerRunFlags = append(dockerRunFlags, "-e", envVariable) + } + dockerRunFlags = addServiceAccountFlags(dockerRunFlags, serviceAccount) + dockerRunFlags = append(dockerRunFlags, ExecutorImage, + "-f", path.Join(buildContextPath, dockerfilesPath, dockerfile), + "-d", kanikoImage, + "-c", buildContextPath, + fmt.Sprintf("--cache=%t", cache), + "--cache-dir", cacheDir, + "--cache-run-layers=false", + "--no-push-cache", + ) + dockerRunFlags = append(dockerRunFlags, args...) + kanikoCmd := exec.Command("docker", dockerRunFlags...) + + out, err := RunCommandWithoutTest(kanikoCmd) + logf(string(out)) + + if err != nil { + return fmt.Errorf("failed to build image %s with kaniko command \"%s\": %w", kanikoImage, kanikoCmd.Args, err) + } + if outputCheck := outputChecks[dockerfile]; outputCheck != nil { + if err := outputCheck(dockerfile, out); err != nil { + return fmt.Errorf("output check failed for image %s with kaniko command : %w", kanikoImage, err) + } + } + if cache { + if outputCheck := warmerOutputChecks[dockerfile]; outputCheck != nil { + if err := outputCheck(dockerfile, out); err != nil { + return fmt.Errorf("output check failed for image %s with kaniko command : %w", kanikoImage, err) + } + } + } + if err := checkNoWarnings(dockerfile, out); err != nil { + return err + } + return nil +} + // buildRelativePathsImage builds the images for testing passing relatives paths to Kaniko func (d *DockerFileBuilder) buildRelativePathsImage(logf logger, imageRepo, dockerfile, serviceAccount, buildContextPath string) error { _, ex, _, _ := runtime.Caller(0) diff --git a/integration/integration_test.go b/integration/integration_test.go index 2621fab83..16bd2fd14 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -28,7 +28,6 @@ import ( "os" "os/exec" "path/filepath" - "runtime" "strconv" "strings" "testing" @@ -195,6 +194,9 @@ func TestRun(t *testing.T) { if _, ok := imageBuilder.TestCacheDockerfiles[dockerfile]; ok { t.SkipNow() } + if _, ok := imageBuilder.TestWarmerDockerfiles[dockerfile]; ok { + t.SkipNow() + } buildImage(t, dockerfile, imageBuilder) @@ -655,8 +657,6 @@ func buildImage(t *testing.T, dockerfile string, imageBuilder *DockerFileBuilder // Build each image with kaniko twice, and then make sure they're exactly the same func TestCache(t *testing.T) { - populateVolumeCache() - // Build dockerfiles with registry cache for dockerfile := range imageBuilder.TestCacheDockerfiles { t.Run("test_cache_"+dockerfile, func(t *testing.T) { @@ -682,10 +682,41 @@ func TestCache(t *testing.T) { } } +func TestWarmer(t *testing.T) { + populateVolumeCache(t.Logf, config.serviceAccount) + + for dockerfile := range imageBuilder.TestWarmerDockerfiles { + t.Run("test_warmer_"+dockerfile, func(t *testing.T) { + t.Parallel() + args, ok := additionalKanikoFlagsMap[dockerfile] + imageRepo := config.imageRepo + if !ok { + args = []string{} + } + + // Build the initial without warmer + if err := imageBuilder.buildWarmerImage(t.Logf, config, dockerfilesPath, dockerfile, 0, args, false); err != nil { + t.Fatalf("error building cached image for the first time: %v", err) + } + + // Build the second with warmer + if err := imageBuilder.buildWarmerImage(t.Logf, config, dockerfilesPath, dockerfile, 1, args, true); err != nil { + t.Fatalf("error building cached image for the second time: %v", err) + } + + // Make sure both images are the same + kanikoVersion0 := GetKanikoImage(imageRepo, "test_warmer_"+dockerfile) + strconv.Itoa(0) + kanikoVersion1 := GetKanikoImage(imageRepo, "test_warmer_"+dockerfile) + strconv.Itoa(1) + + containerDiff(t, kanikoVersion0, kanikoVersion1) + layerDiff(t, kanikoVersion0, kanikoVersion1) + manifestDiff(t, kanikoVersion0, kanikoVersion1) + }) + } +} + // Attempt to warm an image two times : first time should populate the cache, second time should find the image in the cache. func TestWarmerTwice(t *testing.T) { - _, ex, _, _ := runtime.Caller(0) - tmpDir := filepath.Dir(ex) + "/tmpCache" dockerfiles := map[string]bool{ "debian:trixie-slim": true, "debian:12.10@sha256:264982ff4d18000fa74540837e2c43ca5137a53a83f8f62c7b3803c0f0bdcd56": true, // image-index requires remote lookup @@ -694,9 +725,18 @@ func TestWarmerTwice(t *testing.T) { for dockerfile, remoteLookup := range dockerfiles { t.Run("test_warmer_twice_"+dockerfile, func(t *testing.T) { t.Parallel() + tmpDir, err := os.MkdirTemp("", "") + if err != nil { + t.Fatal("failed to create tmpdir") + } + defer os.RemoveAll(tmpDir) + // Start a sleeping warmer container dockerRunFlags := []string{"run", "--net=host"} dockerRunFlags = addServiceAccountFlags(dockerRunFlags, config.serviceAccount) + for _, envVariable := range WarmerEnv { + dockerRunFlags = append(dockerRunFlags, "-e", envVariable) + } dockerRunFlags = append(dockerRunFlags, "-v", tmpDir+":/cache", WarmerImage, @@ -705,17 +745,17 @@ func TestWarmerTwice(t *testing.T) { warmCmd := exec.Command("docker", dockerRunFlags...) out, err := RunCommandWithoutTest(warmCmd) + t.Logf("First warm output:\n%s", out) if err != nil { t.Fatalf("Unable to perform first warming: %s", err) } - t.Logf("First warm output: %s", out) warmCmd = exec.Command("docker", dockerRunFlags...) out, err = RunCommandWithoutTest(warmCmd) + t.Logf("Second warm output:\n%s", out) if err != nil { t.Fatalf("Unable to perform second warming: %s", err) } - t.Logf("Second warm output: %s", out) s := fmt.Sprintf("Image already in cache: %s", dockerfile) if !strings.Contains(string(out), s) { @@ -1010,6 +1050,34 @@ func layerDiff(t *testing.T, image1, image2 string) { } } +func manifestDiff(t *testing.T, image1, image2 string) { + t.Helper() + + imgRef1, err := getImage(image1) + if err != nil { + t.Fatalf("Couldn't get image reference for (%s): %s", image1, err) + } + + imgRef2, err := getImage(image2) + if err != nil { + t.Fatalf("Couldn't get image reference for (%s): %s", image2, err) + } + + media1, err := imgRef1.MediaType() + if err != nil { + t.Fatalf("Couldn't get mediatype for (%s): %s", image1, err) + } + + media2, err := imgRef2.MediaType() + if err != nil { + t.Fatalf("Couldn't get mediatype for (%s): %s", image2, err) + } + + if media1 != media2 { + t.Fatalf("mediatype diff: %s != %s", media1, media2) + } +} + func checkLayers(t *testing.T, image1, image2 string, offset int) { t.Helper() img1, err := getImageDetails(image1) @@ -1062,12 +1130,16 @@ func resolveCreatedBy(image string, layerIndex int) (string, error) { return "", fmt.Errorf("LayerIndex %d not found in History of length %d", layerIndex, len(cfg.History)) } -func getImageLayers(image string) ([]v1.Layer, error) { +func getImage(image string) (v1.Image, error) { ref, err := name.ParseReference(image, name.WeakValidation) if err != nil { return nil, fmt.Errorf("Couldn't parse reference to image %s: %w", image, err) } - imgRef, err := remote.Image(ref) + return remote.Image(ref) +} + +func getImageLayers(image string) ([]v1.Layer, error) { + imgRef, err := getImage(image) if err != nil { return nil, fmt.Errorf("Couldn't get reference to image %s from remote: %w", image, err) } @@ -1079,11 +1151,7 @@ func getImageLayers(image string) ([]v1.Layer, error) { } func getImageDetails(image string) (*imageDetails, error) { - ref, err := name.ParseReference(image, name.WeakValidation) - if err != nil { - return nil, fmt.Errorf("Couldn't parse reference to image %s: %w", image, err) - } - imgRef, err := remote.Image(ref) + imgRef, err := getImage(image) if err != nil { return nil, fmt.Errorf("Couldn't get reference to image %s from remote: %w", image, err) } @@ -1103,12 +1171,7 @@ func getImageDetails(image string) (*imageDetails, error) { } func getLastLayerFiles(image string) ([]string, error) { - ref, err := name.ParseReference(image, name.WeakValidation) - if err != nil { - return nil, fmt.Errorf("Couldn't parse reference to image %s: %w", image, err) - } - - imgRef, err := remote.Image(ref) + imgRef, err := getImage(image) if err != nil { return nil, fmt.Errorf("Couldn't get reference to image %s from daemon: %w", image, err) } diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 3df34cc1a..fbfef96b7 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -195,7 +195,11 @@ func LocalSource(opts *config.CacheOptions, cacheKey string) (v1.Image, error) { } logrus.Infof("Found %s in local cache", cacheKey) - return cachedImageFromPath(path) + if config.EnvBool("FF_KANIKO_OCI_WARMER") { + return ociCachedImageFromPath(path) + } else { + return cachedImageFromPath(path) + } } // cachedImage represents a v1.Tarball that is cached locally in a CAS. @@ -255,3 +259,28 @@ func cachedImageFromPath(p string) (v1.Image, error) { mfst: mfst, }, nil } + +func ociCachedImageFromPath(tarPath string) (v1.Image, error) { + p, err := layout.FromPath(tarPath) + if err != nil { + return nil, err + } + idx, err := p.ImageIndex() + if err != nil { + return nil, err + } + idxManifest, err := idx.IndexManifest() + if err != nil { + return nil, err + } + + if len(idxManifest.Manifests) == 0 { + return nil, fmt.Errorf("no images found in OCI layout") + } + if len(idxManifest.Manifests) > 1 { + return nil, fmt.Errorf("expected one image, found %d", len(idxManifest.Manifests)) + } + + hash := idxManifest.Manifests[0].Digest + return p.Image(hash) +} diff --git a/pkg/cache/warm.go b/pkg/cache/warm.go index 1fcbe3f0d..32ed31797 100644 --- a/pkg/cache/warm.go +++ b/pkg/cache/warm.go @@ -27,6 +27,8 @@ import ( "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/empty" + "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/tarball" "github.com/osscontainertools/kaniko/pkg/config" "github.com/osscontainertools/kaniko/pkg/dockerfile" @@ -56,11 +58,21 @@ func WarmCache(opts *config.WarmerOptions) error { logrus.Debugf("%s\n", images) errs := 0 - for _, img := range images { - err := warmToFile(cacheDir, img, opts) - if err != nil { - logrus.Warnf("Error while trying to warm image: %v %v", img, err) - errs++ + if config.EnvBool("FF_KANIKO_OCI_WARMER") { + for _, img := range images { + err := ociWarmToFile(cacheDir, img, opts) + if err != nil { + logrus.Warnf("Error while trying to warm image: %v %v", img, err) + errs++ + } + } + } else { + for _, img := range images { + err := warmToFile(cacheDir, img, opts) + if err != nil { + logrus.Warnf("Error while trying to warm image: %v %v", img, err) + errs++ + } } } @@ -122,6 +134,41 @@ func warmToFile(cacheDir, img string, opts *config.WarmerOptions) error { return nil } +// Download image in temporary files then move files to final destination +func ociWarmToFile(cacheDir, img string, opts *config.WarmerOptions) error { + tmp, err := os.MkdirTemp(cacheDir, "") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + + cw := &OciWarmer{ + Remote: remote.RetrieveRemoteImage, + Local: LocalSource, + TmpDir: tmp, + } + + digest, err := cw.Warm(img, opts) + if err != nil { + if IsAlreadyCached(err) { + logrus.Infof("Image already in cache: %v", img) + return nil + } + logrus.Warnf("Error while trying to warm image: %v %v", img, err) + return err + } + + finalCachePath := path.Join(cacheDir, digest.String()) + + err = os.Rename(tmp, finalCachePath) + if err != nil { + return err + } + + logrus.Debugf("Wrote %s to cache", img) + return nil +} + // FetchRemoteImage retrieves a Docker image manifest from a remote source. // github.com/GoogleContainerTools/kaniko/image/remote.RetrieveRemoteImage can be used as // this type. @@ -212,6 +259,82 @@ func (w *Warmer) Warm(image string, opts *config.WarmerOptions) (v1.Hash, error) return digest, nil } +type OciWarmer struct { + Remote FetchRemoteImage + Local FetchLocalSource + TmpDir string +} + +// Warm retrieves a Docker image and populates the supplied buffer with the image content and manifest +// or returns an AlreadyCachedErr if the image is present in the cache. +func (w *OciWarmer) Warm(image string, opts *config.WarmerOptions) (v1.Hash, error) { + cacheRef, err := name.ParseReference(image, name.WeakValidation) + if err != nil { + return v1.Hash{}, fmt.Errorf("failed to verify image name: %s: %w", image, err) + } + + // mz320: If we have a digest reference, we can try a cache lookup directly. + var oldKey string + var oldErr error + if !opts.Force { + if d, ok := cacheRef.(name.Digest); ok { + cacheKey := d.DigestStr() + _, err := w.Local(&opts.CacheOptions, cacheKey) + if err == nil || IsExpired(err) { + return v1.Hash{}, AlreadyCachedErr{} + } else { + // mz320: But in case it is a cache miss, not all hope is lost. + // It could have also been the digest for an image-index. + // The thin wrapper that only points to the image-manifests for different archs. + // Unfortunately we can't tell a-priori and we only store the image manifests as keys. + // Therefore we don't return and instead try a remote lookup again. + oldKey = cacheKey + oldErr = err + } + } + } + + img, err := w.Remote(image, opts.RegistryOptions, opts.CustomPlatform) + if err != nil || img == nil { + return v1.Hash{}, fmt.Errorf("failed to retrieve image: %s: %w", image, err) + } + + digest, err := img.Digest() + if err != nil { + return v1.Hash{}, fmt.Errorf("failed to retrieve digest: %s: %w", image, err) + } + + if !opts.Force { + var err error + cacheKey := digest.String() + if oldKey != "" && cacheKey == oldKey { + // mz320: But if the cacheKey didn't change, we indeed were looking + // at an image manifest, we already confirmed it is not in cache, + // so we can short-circuit with the previous error here. + err = oldErr + } else { + _, err = w.Local(&opts.CacheOptions, cacheKey) + } + if err == nil || IsExpired(err) { + return v1.Hash{}, AlreadyCachedErr{} + } + } + + p, err := layout.Write(w.TmpDir, empty.Index) + if err != nil { + return v1.Hash{}, fmt.Errorf("failed to create ocilayout for: %s: %w", image, err) + } + + err = p.AppendImage(img, layout.WithAnnotations(map[string]string{ + "org.opencontainers.image.ref.name": cacheRef.Name(), + })) + if err != nil { + return v1.Hash{}, fmt.Errorf("failed to append image %s to ocilayout: %w", image, err) + } + + return digest, nil +} + func ParseDockerfile(opts *config.WarmerOptions) ([]string, error) { var err error var d []uint8