feat(wanda): Build dependencies in topological order

Extend Build() to automatically build all @name dependencies before
building the root spec. In local mode, specs are discovered by scanning
the repo for *.wanda.yaml files and built in topological order. In
RayCI mode, only the root spec is built (deps built by prior pipeline
steps).

- Add CI mode @name resolution to work repo
- Tag images with plain name for @name resolution in local mode
- Add tests for dependency chain builds

Topic: wanda-build-deps
Relative: wanda-deps

Signed-off-by: andrew <andrew@anyscale.com>

Author: andrew-anyscale

wanda/forge.go

@@ -14,25 +14,46 @@ import (
 	"github.com/google/go-containerregistry/pkg/v1/remote"
 )
 
-// Build builds a container image from the given specification file.
+// Build builds a container image from the given specification file, and builds
+// all its dependencies in topological order.
+// In RayCI mode, dependencies are assumed built by prior pipeline steps; only
+// the root is built.
 func Build(specFile string, config *ForgeConfig) error {
 	if config == nil {
 		config = &ForgeConfig{}
 	}
 
-	spec, err := parseSpecFile(specFile)
+	graph, err := buildDepGraph(specFile, os.LookupEnv)
 	if err != nil {
-		return fmt.Errorf("parse spec file: %w", err)
+		return fmt.Errorf("build dep graph: %w", err)
 	}
 
-	// Expand env variable.
-	spec = spec.expandVar(os.LookupEnv)
+	if err := graph.validateDeps(); err != nil {
+		return fmt.Errorf("validate deps: %w", err)
+	}
 
 	forge, err := NewForge(config)
 	if err != nil {
 		return fmt.Errorf("make forge: %w", err)
 	}
-	return forge.Build(spec)
+
+	// In RayCI mode, only build the root (deps built by prior pipeline steps).
+	order := graph.Order
+	if config.RayCI {
+		order = []string{graph.Root}
+	}
+
+	for _, name := range order {
+		rs := graph.Specs[name]
+
+		log.Printf("building %s (from %s)", name, rs.Path)
+
+		if err := forge.Build(rs.Spec); err != nil {
+			return fmt.Errorf("build %s: %w", name, err)
+		}
+	}
+
+	return nil
 }
 
 // Forge is a forge to build container images.
@@ -95,13 +116,26 @@ func (f *Forge) resolveBases(froms []string) (map[string]*imageSource, error) {
 	namePrefix := f.config.NamePrefix
 
 	for _, from := range froms {
-		if strings.HasPrefix(from, "@") { // A local image.
+		if strings.HasPrefix(from, "@") { // A sourceable image name.
 			name := strings.TrimPrefix(from, "@")
-			src, err := resolveDockerImage(f.docker, from, name)
-			if err != nil {
-				return nil, fmt.Errorf("resolve local image %s: %w", from, err)
+
+			if f.isRemote() {
+				// CI mode: deps were built by prior pipeline steps and pushed
+				// to the work repo. Resolve @name to {work_repo}:{build_id}-{name}.
+				workTag := f.workTag(name)
+				src, err := resolveRemoteImage(from, workTag, f.remoteOpts...)
+				if err != nil {
+					return nil, fmt.Errorf("resolve remote dep @%s: %w", name, err)
+				}
+				m[from] = src
+			} else {
+				// Local mode: resolve from local docker
+				src, err := resolveDockerImage(f.docker, from, name)
+				if err != nil {
+					return nil, fmt.Errorf("resolve local dep @%s: %w", name, err)
+				}
+				m[from] = src
 			}
-			m[from] = src
 			continue
 		}
 
@@ -188,13 +222,18 @@ func (f *Forge) Build(spec *Spec) error {
 	in.addTag(cacheTag)
 
 	// When running on rayCI, we only need the workTag and the cacheTag.
-	// Otherwise, add extra tags.
+	// Otherwise, add extra tags for local use.
 	if !f.config.RayCI {
 		// Name tag is the tag we use to reference the image locally.
 		// It is also what can be referenced by following steps.
 		if f.config.NamePrefix != "" {
 			nameTag := f.config.NamePrefix + spec.Name
 			in.addTag(nameTag)
+		} else {
+			// Tag with plain name so @name references can resolve it.
+			// e.g., if spec.Name is "dep-base", a dependent spec with
+			// froms: ["@dep-base"] will look up the image by "dep-base".
+			in.addTag(spec.Name)
 		}
 		for _, tag := range spec.Tags { // And add extra tags.
 			in.addTag(tag)

wanda/forge_test.go

@@ -455,6 +455,65 @@ func TestForgeWithRemoteWorkRepo(t *testing.T) {
 	}
 }
 
+func TestBuild_WithDeps(t *testing.T) {
+	// Test: dep-top -> dep-middle -> dep-base
+	// Build should build in order: dep-base, dep-middle, dep-top
+	config := &ForgeConfig{WorkDir: "testdata"}
+
+	if err := Build("testdata/dep-top.wanda.yaml", config); err != nil {
+		t.Fatalf("build with deps: %v", err)
+	}
+
+	// Verify dep-top was built and can be read
+	ref, err := name.ParseReference("dep-top")
+	if err != nil {
+		t.Fatalf("parse reference: %v", err)
+	}
+
+	img, err := daemon.Image(ref)
+	if err != nil {
+		t.Fatalf("read dep-top image: %v", err)
+	}
+
+	layers, err := img.Layers()
+	if err != nil {
+		t.Fatalf("read layers: %v", err)
+	}
+
+	// Should have 3 layers: dep-base, dep-middle, dep-top
+	if got, want := len(layers), 3; got != want {
+		t.Errorf("got %d layers, want %d", got, want)
+	}
+}
+
+func TestBuild_NoDeps(t *testing.T) {
+	// Test backward compatibility: a spec with no deps should work
+	config := &ForgeConfig{WorkDir: "testdata"}
+
+	if err := Build("testdata/hello.wanda.yaml", config); err != nil {
+		t.Fatalf("build with deps: %v", err)
+	}
+
+	ref, err := name.ParseReference("hello")
+	if err != nil {
+		t.Fatalf("parse reference: %v", err)
+	}
+
+	img, err := daemon.Image(ref)
+	if err != nil {
+		t.Fatalf("read hello image: %v", err)
+	}
+
+	layers, err := img.Layers()
+	if err != nil {
+		t.Fatalf("read layers: %v", err)
+	}
+
+	if got, want := len(layers), 1; got != want {
+		t.Errorf("got %d layers, want %d", got, want)
+	}
+}
+
 func TestForgeLocal_withNamePrefix(t *testing.T) {
 	if runtime.GOOS != "linux" {
 		t.Skip("skipping test on non-linux")

wanda/testdata/localbase.wanda.yaml

@@ -1,3 +1,2 @@
-name: localbase
+name: mybase
 dockerfile: Dockerfile.localbase
-tags: [ "mybase" ]