[docker] Pass dependency image names as build args

Do enable chaining Docker packages and re-using
a prior build as base image.

Author: csweichel

pkg/leeway/build.go

@@ -2,12 +2,14 @@ package leeway
 
 import (
 	"archive/tar"
+	"compress/gzip"
 	"context"
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/base64"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -1239,7 +1241,10 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
 		return nil, err
 	}
 
-	var buildCommands [][]string
+	var (
+		buildCommands     [][]string
+		imageDependencies = make(map[string]string)
+	)
 	buildCommands = append(buildCommands, []string{"cp", dockerfile, "Dockerfile"})
 	for _, dep := range p.GetDependencies() {
 		fn, exists := buildctx.LocalCache.Location(dep)
@@ -1252,6 +1257,15 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
 			{"mkdir", tgt},
 			{"tar", "xfz", fn, "--no-same-owner", "-C", tgt},
 		}...)
+
+		if dep.Type != DockerPackage {
+			continue
+		}
+		depimg, err := extractImageNameFromCache(dep.Name, fn)
+		if err != nil {
+			return nil, err
+		}
+		imageDependencies[strings.ToUpper(strings.ReplaceAll(dep.FilesystemSafeName(), "-", "_"))] = depimg
 	}
 
 	buildCommands = append(buildCommands, p.PreparationCommands...)
@@ -1265,6 +1279,9 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
 	for arg, val := range cfg.BuildArgs {
 		buildcmd = append(buildcmd, "--build-arg", fmt.Sprintf("%s=%s", arg, val))
 	}
+	for arg, val := range imageDependencies {
+		buildcmd = append(buildcmd, "--build-arg", fmt.Sprintf("DEP_%s=%s", arg, val))
+	}
 	buildcmd = append(buildcmd, "--build-arg", fmt.Sprintf("__GIT_COMMIT=%s", p.C.Git().Commit))
 	if cfg.Squash {
 		buildcmd = append(buildcmd, "--squash")
@@ -1415,6 +1432,60 @@ func dockerExportPostBuild(builddir, result string) func(sources fileset) (subj
 	}
 }
 
+// extractImageNameFromCache extracts the Docker image name of a previously built package
+// from the cache tar.gz file of that package.
+func extractImageNameFromCache(pkgName, cacheBundleFN string) (imgname string, err error) {
+	defer func() {
+		if err != nil {
+			err = fmt.Errorf("cannot extract image ref from cache for %s: %w", pkgName, err)
+		}
+	}()
+
+	f, err := os.Open(cacheBundleFN)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+
+	gzin, err := gzip.NewReader(f)
+	if err != nil {
+		return "", err
+	}
+	defer gzin.Close()
+
+	tarin := tar.NewReader(gzin)
+	for {
+		hdr, err := tarin.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+
+		if hdr.Typeflag != tar.TypeReg {
+			continue
+		}
+		if filepath.Base(hdr.Name) != dockerImageNamesFiles {
+			continue
+		}
+
+		imgnames := make([]byte, hdr.Size)
+		n, err := io.ReadFull(tarin, imgnames)
+		if err != nil {
+			return "", err
+		}
+		if int64(n) != hdr.Size {
+			return "", fmt.Errorf("cannot read %s from cache: %w", dockerImageNamesFiles, io.ErrShortBuffer)
+		}
+
+		lines := strings.Split(string(imgnames), "\n")
+		return lines[0], nil
+	}
+
+	return "", nil
+}
+
 // buildGeneric implements the build process for generic packages.
 // If you change anything in this process that's not backwards compatible, make sure you increment BuildGenericProccessVersion.
 func (p *Package) buildGeneric(buildctx *buildContext, wd, result string) (res *packageBuild, err error) {

pkg/leeway/build_test.go

@@ -1,25 +1,102 @@
-package leeway
+package leeway_test
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
+
+	"github.com/gitpod-io/leeway/pkg/leeway"
+	"github.com/gitpod-io/leeway/pkg/testutil"
+	log "github.com/sirupsen/logrus"
 )
 
-func TestCodecovComponentName(t *testing.T) {
-	tests := []struct {
-		Test     string
-		Package  string
-		Expected string
-	}{
-		{"valid package format", "components/ee/ws-scheduler", "components-ee-ws-scheduler-coverage.out"},
-		{"lower case", "COMPONENTS/gitpod-cli:app", "components-gitpod-cli-app-coverage.out"},
-		{"special character", "components/~ü:app", "components-app-coverage.out"},
-		{"with numbers", "components/1icens0r:app", "components-1icens0r-app-coverage.out"},
+const dummyDocker = `#!/bin/bash
+
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -o)
+      OUTPUT="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    *)
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift # past argument
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
+
+if [ "${POSITIONAL_ARGS}" == "save" ]; then
+	tar cvvfz "${OUTPUT}" -T /dev/null
+fi
+`
+
+func TestBuildDockerDeps(t *testing.T) {
+	if *dut {
+		pth, err := os.MkdirTemp("", "")
+		if err != nil {
+			t.Fatal(err)
+		}
+		err = os.WriteFile(filepath.Join(pth, "docker"), []byte(dummyDocker), 0755)
+		if err != nil {
+			t.Fatal(err)
+		}
+		t.Cleanup(func() { os.RemoveAll(pth) })
+
+		os.Setenv("PATH", pth+":"+os.Getenv("PATH"))
+		log.WithField("path", os.Getenv("PATH")).Debug("modified path to use dummy docker")
+	}
+	runDUT()
+
+	tests := []*CommandFixtureTest{
+		{
+			Name:        "docker dependency",
+			T:           t,
+			Args:        []string{"build", "-v", "-c", "none", "comp:pkg1"},
+			StderrSub:   "DEP_COMP__PKG0=foobar:1234",
+			NoStdoutSub: "already built",
+			ExitCode:    0,
+			Fixture: &testutil.Setup{
+				Components: []testutil.Component{
+					{
+						Location: "comp",
+						Files: map[string]string{
+							"pkg0.Dockerfile": "FROM alpine:latest",
+							"pkg1.Dockerfile": "FROM ${DEP_COMP__PKG0}",
+						},
+						Packages: []leeway.Package{
+							{
+								PackageInternal: leeway.PackageInternal{
+									Name: "pkg0",
+									Type: leeway.DockerPackage,
+								},
+								Config: leeway.DockerPkgConfig{
+									Dockerfile: "pkg0.Dockerfile",
+									Image:      []string{"foobar:1234"},
+								},
+							},
+							{
+								PackageInternal: leeway.PackageInternal{
+									Name:         "pkg1",
+									Type:         leeway.DockerPackage,
+									Dependencies: []string{":pkg0"},
+								},
+								Config: leeway.DockerPkgConfig{
+									Dockerfile: "pkg1.Dockerfile",
+								},
+							},
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {
-		name := codecovComponentName(test.Package)
-		if name != test.Expected {
-			t.Errorf("%s: expected: %v, actual: %v", test.Test, test.Expected, name)
-		}
+		test.Run()
 	}
 }

pkg/leeway/package_test.go

@@ -309,3 +309,23 @@ func NewTestPackage(name string) *Package {
 		Config:       GenericPkgConfig{},
 	}
 }
+
+func TestCodecovComponentName(t *testing.T) {
+	tests := []struct {
+		Test     string
+		Package  string
+		Expected string
+	}{
+		{"valid package format", "components/ee/ws-scheduler", "components-ee-ws-scheduler-coverage.out"},
+		{"lower case", "COMPONENTS/gitpod-cli:app", "components-gitpod-cli-app-coverage.out"},
+		{"special character", "components/~ü:app", "components-app-coverage.out"},
+		{"with numbers", "components/1icens0r:app", "components-1icens0r-app-coverage.out"},
+	}
+
+	for _, test := range tests {
+		name := codecovComponentName(test.Package)
+		if name != test.Expected {
+			t.Errorf("%s: expected: %v, actual: %v", test.Test, test.Expected, name)
+		}
+	}
+}

pkg/leeway/scripts_test.go

@@ -141,7 +141,7 @@ func (ft *CommandFixtureTest) Run() {
 				t.Fatalf("cannot materialize fixture: %v", err)
 			}
 			t.Logf("materialized fixture workspace: %s", loc)
-			t.Cleanup(func() { os.RemoveAll(loc) })
+			// t.Cleanup(func() { os.RemoveAll(loc) })
 		}
 
 		env := os.Environ()