Add S3 Docker build cache support

Add support for S3-backed Docker build layer caching using BuildKit's
type=s3 cache backend. This enables faster Docker builds by caching
intermediate layers in S3.

Configuration via environment variables:
- LEEWAY_DOCKER_S3_CACHE_BUCKET: S3 bucket name (required)
- LEEWAY_DOCKER_S3_CACHE_REGION: AWS region (required)
- LEEWAY_DOCKER_S3_CACHE_PREFIX: Optional prefix for cache keys
- LEEWAY_DOCKER_S3_CACHE_MODE: Cache mode (min/max, default: max)
- LEEWAY_DOCKER_S3_CACHE_ENDPOINT: Custom S3 endpoint

Also adds corresponding CLI flags for all options.

When S3 cache is enabled, leeway automatically uses docker buildx build
with --cache-from and --cache-to flags.

Co-authored-by: Ona <no-reply@ona.com>

Author: WVerlaek

README.md

@@ -209,6 +209,39 @@ E.g. `component/nested:docker` becomes `COMPONENT_NESTED__DOCKER`.
 
 See `leeway build --help` for more details.
 
+#### S3 Docker Build Cache
+
+Leeway supports S3-backed Docker build layer caching using BuildKit's `type=s3` cache backend. This can significantly speed up Docker builds by caching intermediate layers in S3.
+
+**Configuration via environment variables:**
+```bash
+export LEEWAY_DOCKER_S3_CACHE_BUCKET=my-cache-bucket
+export LEEWAY_DOCKER_S3_CACHE_REGION=us-east-1
+export LEEWAY_DOCKER_S3_CACHE_PREFIX=docker-cache/  # optional
+export LEEWAY_DOCKER_S3_CACHE_MODE=max              # optional: 'min' or 'max' (default: max)
+export LEEWAY_DOCKER_S3_CACHE_ENDPOINT=https://...  # optional: for S3-compatible storage
+```
+
+**Configuration via CLI flags:**
+```bash
+leeway build \
+  --docker-s3-cache-bucket=my-cache-bucket \
+  --docker-s3-cache-region=us-east-1 \
+  --docker-s3-cache-prefix=docker-cache/ \
+  :my-docker-package
+```
+
+**Requirements:**
+- Docker Buildx (automatically used when S3 cache is configured)
+- AWS credentials configured (via environment variables, IAM role, or AWS config file)
+- S3 bucket with appropriate permissions
+
+**Cache modes:**
+- `max`: Cache all layers including intermediate layers (better cache hit rate, more storage)
+- `min`: Cache only the final layer (less storage, fewer cache hits)
+
+When S3 cache is enabled, leeway automatically switches to `docker buildx build` with `--cache-from` and `--cache-to` flags pointing to the configured S3 bucket.
+
 ### Generic packages
 ```YAML
 config:

cmd/build.go

@@ -241,6 +241,13 @@ func addBuildFlags(cmd *cobra.Command) {
 	cmd.Flags().Bool("report-github", os.Getenv("GITHUB_OUTPUT") != "", "Report package build success/failure to GitHub Actions using the GITHUB_OUTPUT environment variable")
 	cmd.Flags().Bool("fixed-build-dir", true, "Use a fixed build directory for each package, instead of based on the package version, to better utilize caches based on absolute paths (defaults to true)")
 	cmd.Flags().Bool("docker-export-to-cache", false, "Export Docker images to cache instead of pushing directly (enables SLSA L3 compliance)")
+
+	// Docker S3 build cache flags
+	cmd.Flags().String("docker-s3-cache-bucket", os.Getenv(leeway.EnvvarDockerS3CacheBucket), "S3 bucket for Docker build layer caching (defaults to $LEEWAY_DOCKER_S3_CACHE_BUCKET)")
+	cmd.Flags().String("docker-s3-cache-region", os.Getenv(leeway.EnvvarDockerS3CacheRegion), "AWS region for Docker S3 cache bucket (defaults to $LEEWAY_DOCKER_S3_CACHE_REGION)")
+	cmd.Flags().String("docker-s3-cache-prefix", os.Getenv(leeway.EnvvarDockerS3CachePrefix), "Prefix for Docker S3 cache keys (defaults to $LEEWAY_DOCKER_S3_CACHE_PREFIX)")
+	cmd.Flags().String("docker-s3-cache-mode", os.Getenv(leeway.EnvvarDockerS3CacheMode), "Docker S3 cache mode: 'min' or 'max' (defaults to 'max')")
+	cmd.Flags().String("docker-s3-cache-endpoint", os.Getenv(leeway.EnvvarDockerS3CacheEndpoint), "Custom S3 endpoint for Docker cache (for S3-compatible storage)")
 }
 
 func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache) {
@@ -412,6 +419,29 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache) {
 		dockerExportSet = true
 	}
 
+	// Get Docker S3 cache configuration from CLI flags (which default to env vars)
+	dockerS3CacheBucket, _ := cmd.Flags().GetString("docker-s3-cache-bucket")
+	dockerS3CacheRegion, _ := cmd.Flags().GetString("docker-s3-cache-region")
+	dockerS3CachePrefix, _ := cmd.Flags().GetString("docker-s3-cache-prefix")
+	dockerS3CacheMode, _ := cmd.Flags().GetString("docker-s3-cache-mode")
+	dockerS3CacheEndpoint, _ := cmd.Flags().GetString("docker-s3-cache-endpoint")
+
+	var dockerS3Cache *leeway.DockerS3CacheConfig
+	if dockerS3CacheBucket != "" && dockerS3CacheRegion != "" {
+		dockerS3Cache = &leeway.DockerS3CacheConfig{
+			Bucket:   dockerS3CacheBucket,
+			Region:   dockerS3CacheRegion,
+			Prefix:   dockerS3CachePrefix,
+			Mode:     dockerS3CacheMode,
+			Endpoint: dockerS3CacheEndpoint,
+		}
+		log.WithFields(log.Fields{
+			"bucket": dockerS3Cache.Bucket,
+			"region": dockerS3Cache.Region,
+			"prefix": dockerS3Cache.Prefix,
+		}).Info("Docker S3 build cache enabled")
+	}
+
 	return []leeway.BuildOption{
 		leeway.WithLocalCache(localCache),
 		leeway.WithRemoteCache(remoteCache),
@@ -430,6 +460,7 @@ func getBuildOpts(cmd *cobra.Command) ([]leeway.BuildOption, cache.LocalCache) {
 		leeway.WithInFlightChecksums(inFlightChecksums),
 		leeway.WithDockerExportToCache(dockerExportToCache, dockerExportSet),
 		leeway.WithDockerExportEnv(dockerExportEnvValue, dockerExportEnvSet),
+		leeway.WithDockerS3Cache(dockerS3Cache),
 	}, localCache
 }
 

pkg/leeway/build.go

@@ -102,6 +102,21 @@ const (
 	// EnvvarWorkspaceRoot names the environment variable for workspace root path
 	EnvvarWorkspaceRoot = "LEEWAY_WORKSPACE_ROOT"
 
+	// EnvvarDockerS3CacheBucket names the S3 bucket for Docker build layer caching
+	EnvvarDockerS3CacheBucket = "LEEWAY_DOCKER_S3_CACHE_BUCKET"
+
+	// EnvvarDockerS3CacheRegion names the AWS region for the S3 cache bucket
+	EnvvarDockerS3CacheRegion = "LEEWAY_DOCKER_S3_CACHE_REGION"
+
+	// EnvvarDockerS3CachePrefix names the prefix for S3 cache keys (optional)
+	EnvvarDockerS3CachePrefix = "LEEWAY_DOCKER_S3_CACHE_PREFIX"
+
+	// EnvvarDockerS3CacheMode controls the cache mode (min or max, defaults to max)
+	EnvvarDockerS3CacheMode = "LEEWAY_DOCKER_S3_CACHE_MODE"
+
+	// EnvvarDockerS3CacheEndpoint allows specifying a custom S3 endpoint (for S3-compatible storage)
+	EnvvarDockerS3CacheEndpoint = "LEEWAY_DOCKER_S3_CACHE_ENDPOINT"
+
 	// dockerImageNamesFiles is the name of the file store in poushed Docker build artifacts
 	// which contains the names of the Docker images we just pushed
 	dockerImageNamesFiles = "imgnames.txt"
@@ -492,12 +507,70 @@ type buildOptions struct {
 	DockerExportEnvValue bool // Value from explicit user env var
 	DockerExportEnvSet   bool // Whether user explicitly set env var (before workspace)
 
+	// Docker S3 build cache configuration
+	DockerS3Cache *DockerS3CacheConfig
+
 	context *buildContext
 }
 
 // DockerBuildOptions are options passed to "docker build"
 type DockerBuildOptions map[string]string
 
+// DockerS3CacheConfig configures S3-backed Docker build layer caching.
+// When configured, leeway adds --cache-from and --cache-to flags to docker buildx commands.
+type DockerS3CacheConfig struct {
+	// Bucket is the S3 bucket name (required)
+	Bucket string
+	// Region is the AWS region for the bucket (required)
+	Region string
+	// Prefix is an optional prefix for cache keys (e.g., "cache/myproject/")
+	Prefix string
+	// Mode controls cache export mode: "min" (default layers only) or "max" (all layers)
+	// Defaults to "max" for better cache hit rates
+	Mode string
+	// Endpoint is an optional custom S3 endpoint for S3-compatible storage
+	Endpoint string
+}
+
+// IsEnabled returns true if the S3 cache is properly configured
+func (c *DockerS3CacheConfig) IsEnabled() bool {
+	return c != nil && c.Bucket != "" && c.Region != ""
+}
+
+// CacheFromArg returns the --cache-from argument value for docker buildx
+func (c *DockerS3CacheConfig) CacheFromArg() string {
+	if !c.IsEnabled() {
+		return ""
+	}
+	arg := fmt.Sprintf("type=s3,region=%s,bucket=%s", c.Region, c.Bucket)
+	if c.Prefix != "" {
+		arg += fmt.Sprintf(",blobs_prefix=%s,manifests_prefix=%s", c.Prefix, c.Prefix)
+	}
+	if c.Endpoint != "" {
+		arg += fmt.Sprintf(",endpoint_url=%s", c.Endpoint)
+	}
+	return arg
+}
+
+// CacheToArg returns the --cache-to argument value for docker buildx
+func (c *DockerS3CacheConfig) CacheToArg() string {
+	if !c.IsEnabled() {
+		return ""
+	}
+	mode := c.Mode
+	if mode == "" {
+		mode = "max"
+	}
+	arg := fmt.Sprintf("type=s3,region=%s,bucket=%s,mode=%s", c.Region, c.Bucket, mode)
+	if c.Prefix != "" {
+		arg += fmt.Sprintf(",blobs_prefix=%s,manifests_prefix=%s", c.Prefix, c.Prefix)
+	}
+	if c.Endpoint != "" {
+		arg += fmt.Sprintf(",endpoint_url=%s", c.Endpoint)
+	}
+	return arg
+}
+
 // BuildOption configures the build behaviour
 type BuildOption func(*buildOptions) error
 
@@ -640,6 +713,33 @@ func WithDockerExportEnv(value, isSet bool) BuildOption {
 	}
 }
 
+// WithDockerS3Cache configures S3-backed Docker build layer caching
+func WithDockerS3Cache(cfg *DockerS3CacheConfig) BuildOption {
+	return func(opts *buildOptions) error {
+		opts.DockerS3Cache = cfg
+		return nil
+	}
+}
+
+// DockerS3CacheFromEnv creates a DockerS3CacheConfig from environment variables.
+// Returns nil if the required environment variables are not set.
+func DockerS3CacheFromEnv() *DockerS3CacheConfig {
+	bucket := os.Getenv(EnvvarDockerS3CacheBucket)
+	region := os.Getenv(EnvvarDockerS3CacheRegion)
+
+	if bucket == "" || region == "" {
+		return nil
+	}
+
+	return &DockerS3CacheConfig{
+		Bucket:   bucket,
+		Region:   region,
+		Prefix:   os.Getenv(EnvvarDockerS3CachePrefix),
+		Mode:     os.Getenv(EnvvarDockerS3CacheMode),
+		Endpoint: os.Getenv(EnvvarDockerS3CacheEndpoint),
+	}
+}
+
 func withBuildContext(ctx *buildContext) BuildOption {
 	return func(opts *buildOptions) error {
 		opts.context = ctx
@@ -2380,19 +2480,43 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
 		return nil, err
 	}
 
-	// Use buildx for OCI layout export when exporting to cache
+	// Determine if we need buildx (required for OCI export or S3 cache)
+	useS3Cache := buildctx.DockerS3Cache.IsEnabled()
+	useBuildx := *cfg.ExportToCache || useS3Cache
+
+	// Use buildx for OCI layout export when exporting to cache, or when S3 cache is enabled
 	var buildcmd []string
 	if *cfg.ExportToCache {
 		// Build with OCI layout export for deterministic caching
 		imageTarPath := filepath.Join(wd, "image.tar")
 		buildcmd = []string{"docker", "buildx", "build", "--pull"}
 		buildcmd = append(buildcmd, "--output", fmt.Sprintf("type=oci,dest=%s", imageTarPath))
 		buildcmd = append(buildcmd, "--tag", version)
+	} else if useBuildx {
+		// Use buildx for S3 cache support, but load to daemon for pushing
+		buildcmd = []string{"docker", "buildx", "build", "--pull", "--load", "-t", version}
 	} else {
 		// Normal build (load to daemon for pushing)
 		buildcmd = []string{"docker", "build", "--pull", "-t", version}
 	}
 
+	// Add S3 cache options if configured (only works with buildx)
+	if useS3Cache {
+		cacheFrom := buildctx.DockerS3Cache.CacheFromArg()
+		cacheTo := buildctx.DockerS3Cache.CacheToArg()
+		if cacheFrom != "" {
+			buildcmd = append(buildcmd, "--cache-from", cacheFrom)
+		}
+		if cacheTo != "" {
+			buildcmd = append(buildcmd, "--cache-to", cacheTo)
+		}
+		log.WithFields(log.Fields{
+			"bucket": buildctx.DockerS3Cache.Bucket,
+			"region": buildctx.DockerS3Cache.Region,
+			"prefix": buildctx.DockerS3Cache.Prefix,
+		}).Debug("Docker S3 build cache enabled")
+	}
+
 	for arg, val := range cfg.BuildArgs {
 		buildcmd = append(buildcmd, "--build-arg", fmt.Sprintf("%s=%s", arg, val))
 	}