feat(wanda): Build dependencies in topological order (#348)

Extend Build() to automatically build all dependencies before building
the root spec. In local mode, specs are discovered by scanning the repo
for *.wanda.yaml files and built in topological order. For a slight
optimization, in RayCI mode, only the root spec is built (deps built by
prior pipeline steps).

Topic: wanda-build-deps
Relative: wanda-deps

Signed-off-by: andrew <andrew@anyscale.com>

Signed-off-by: andrew <andrew@anyscale.com>

Author: andrew-anyscale

wanda/forge.go

@@ -14,25 +14,51 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 )
 
-// Build builds a container image from the given specification file.
+// Build builds a container image from the given specification file, and builds
+// all its dependencies in topological order.
+// In RayCI mode, dependencies are assumed built by prior pipeline steps; only
+// the root is built.
 func Build(specFile string, config *ForgeConfig) error {
 	if config == nil {
 		config = &ForgeConfig{}
 	}
 
-	spec, err := parseSpecFile(specFile)
+	wandaSpecsFile := config.WandaSpecsFile
+	if wandaSpecsFile == "" {
+		wandaSpecsFile = filepath.Join(config.WorkDir, ".wandaspecs")
+	}
+
+	graph, err := buildDepGraph(specFile, os.LookupEnv, config.NamePrefix, wandaSpecsFile)
 	if err != nil {
-		return fmt.Errorf("parse spec file: %w", err)
+		return fmt.Errorf("build dep graph: %w", err)
 	}
 
-	// Expand env variable.
-	spec = spec.expandVar(os.LookupEnv)
+	if err := graph.validateDeps(); err != nil {
+		return fmt.Errorf("validate deps: %w", err)
+	}
 
 	forge, err := NewForge(config)
 	if err != nil {
 		return fmt.Errorf("make forge: %w", err)
 	}
-	return forge.Build(spec)
+
+	// In RayCI mode, only build the root (deps built by prior pipeline steps).
+	order := graph.Order
+	if config.RayCI {
+		order = []string{graph.Root}
+	}
+
+	for _, name := range order {
+		rs := graph.Specs[name]
+
+		log.Printf("building %s (from %s)", name, rs.Path)
+
+		if err := forge.Build(rs.Spec); err != nil {
+			return fmt.Errorf("build %s: %w", name, err)
+		}
+	}
+
+	return nil
 }
 
 // Forge is a forge to build container images.

wanda/forge_test.go

@@ -10,6 +10,7 @@ import (
 	"net"
 	"net/http/httptest"
 	"os"
+	"path/filepath"
 	"runtime"
 
 	"github.com/google/go-containerregistry/pkg/name"
@@ -455,6 +456,83 @@ func TestForgeWithRemoteWorkRepo(t *testing.T) {
 	}
 }
 
+func TestBuild_WithDeps(t *testing.T) {
+	// Test: dep-top -> dep-middle -> dep-base
+	// Build should build in order: dep-base, dep-middle, dep-top
+
+	// Create a wandaspecs file pointing to testdata directory.
+	wandaSpecs := filepath.Join(t.TempDir(), ".wandaspecs")
+	absTestdata, err := filepath.Abs("testdata")
+	if err != nil {
+		t.Fatalf("abs testdata: %v", err)
+	}
+	if err := os.WriteFile(wandaSpecs, []byte(absTestdata), 0644); err != nil {
+		t.Fatalf("write wandaspecs: %v", err)
+	}
+
+	config := &ForgeConfig{
+		WorkDir:        "testdata",
+		NamePrefix:     "cr.ray.io/rayproject/",
+		WandaSpecsFile: wandaSpecs,
+	}
+
+	if err := Build("testdata/dep-top.wanda.yaml", config); err != nil {
+		t.Fatalf("build with deps: %v", err)
+	}
+
+	// Verify dep-top was built and can be read
+	ref, err := name.ParseReference("cr.ray.io/rayproject/dep-top")
+	if err != nil {
+		t.Fatalf("parse reference: %v", err)
+	}
+
+	img, err := daemon.Image(ref)
+	if err != nil {
+		t.Fatalf("read dep-top image: %v", err)
+	}
+
+	layers, err := img.Layers()
+	if err != nil {
+		t.Fatalf("read layers: %v", err)
+	}
+
+	// Should have 3 layers: dep-base, dep-middle, dep-top
+	if got, want := len(layers), 3; got != want {
+		t.Errorf("got %d layers, want %d", got, want)
+	}
+}
+
+func TestBuild_NoDeps(t *testing.T) {
+	// Test backward compatibility: a spec with no deps should work
+	config := &ForgeConfig{
+		WorkDir:    "testdata",
+		NamePrefix: "cr.ray.io/rayproject/",
+	}
+
+	if err := Build("testdata/hello-test.wanda.yaml", config); err != nil {
+		t.Fatalf("build with deps: %v", err)
+	}
+
+	ref, err := name.ParseReference("cr.ray.io/rayproject/hello-test")
+	if err != nil {
+		t.Fatalf("parse reference: %v", err)
+	}
+
+	img, err := daemon.Image(ref)
+	if err != nil {
+		t.Fatalf("read hello image: %v", err)
+	}
+
+	layers, err := img.Layers()
+	if err != nil {
+		t.Fatalf("read layers: %v", err)
+	}
+
+	if got, want := len(layers), 1; got != want {
+		t.Errorf("got %d layers, want %d", got, want)
+	}
+}
+
 func TestForgeLocal_withNamePrefix(t *testing.T) {
 	if runtime.GOOS != "linux" {
 		t.Skip("skipping test on non-linux")

wanda/testdata/Dockerfile.dep-base

@@ -0,0 +1,2 @@
+FROM scratch
+COPY Dockerfile.dep-base /opt/

wanda/testdata/Dockerfile.dep-middle

@@ -0,0 +1,2 @@
+FROM cr.ray.io/rayproject/dep-base
+COPY world.txt /opt/

wanda/testdata/Dockerfile.dep-top

@@ -0,0 +1,2 @@
+FROM cr.ray.io/rayproject/dep-middle
+COPY Dockerfile.dep-top /opt/

wanda/testdata/dep-base.wanda.yaml

@@ -0,0 +1,2 @@
+name: dep-base
+dockerfile: Dockerfile.dep-base

wanda/testdata/dep-middle.wanda.yaml

@@ -0,0 +1,5 @@
+name: dep-middle
+froms: ["cr.ray.io/rayproject/dep-base"]
+dockerfile: Dockerfile.dep-middle
+srcs:
+  - world.txt

wanda/testdata/dep-top.wanda.yaml

@@ -0,0 +1,3 @@
+name: dep-top
+froms: ["cr.ray.io/rayproject/dep-middle"]
+dockerfile: Dockerfile.dep-top