feat(carvel): add BOSH link support via job-spec-overlay and values-overlay sidecars (#663)

## Summary

Introduce two per-packageinstall sidecar conventions that kiln
auto-detects in `packageinstalls/`:

| Sidecar | Purpose | When it runs |
|---|---|---|
| `<entry>.job-spec-overlay.yml` | Declares additional BOSH link
consumptions; kiln appends them to the generated `registry-data` job.MF
| `kiln carvel bake` time |
| `<entry>.values-overlay.erb` | ERB fragment injected before
`YAML.dump(values)`; mutates the values hash via BOSH link objects |
BOSH deploy time (ERB) |

Missing sidecar files are silently skipped (no-op). Both sidecars are
co-located with the packageinstall YAML they extend.

## Motivation

Carvel tiles need to consume BOSH links from co-deployed products (e.g.
the `binding_cache` link provided by CF's `loggr-syslog-binding-cache`
job) to inject environment-specific runtime values at BOSH deploy time.
Previously only the hardcoded `cluster-info` link was available in the
generated job spec. The binding-cache has no BOSH DNS alias, so its
address can only be discovered via a BOSH link.

## Approach — co-located sidecars (mirrors the values-overlay.erb
pattern)

Tile authors declare BOSH link consumptions in a
`<name>.job-spec-overlay.yml` sidecar alongside their packageinstall
YAML. kiln reads all such files and aggregates their `consumes:` entries
into the generated `registry-data` job spec. The schema mirrors the BOSH
job spec `consumes:` block directly.

```
packageinstalls/
  tnz-ear-runtime-package-install.yml
  tnz-ear-runtime-package-install.job-spec-overlay.yml   ← NEW: BOSH link declarations
  tnz-ear-runtime-package-install.values-overlay.erb     ← NEW: values mutations at deploy time
```

Example `job-spec-overlay.yml`:
```yaml
consumes:
- name: binding_cache
  type: binding_cache
  optional: false
```

Example `values-overlay.erb`:
```erb
<% addr = link("binding_cache").instances.first.address %>
<% values["syslog_agent"]["cache"]["url"] = "https://#{addr}:9000" %>
```

**Kilnfile is not modified.** It stays focused on dependency management
(release sources, releases, stemcell). The
`BOSHLinkConsumer`/`BOSHLinks` types and `BoshLinks` field from the
initial commit have been removed.

## Backward Compatibility

- Tiles with no `*.job-spec-overlay.yml` files produce identical output
to before (no-op)
- Tiles with no `*.values-overlay.erb` files produce identical output to
before (no-op)
- All existing `cluster-info` link behaviour is preserved
- Kilnfile schema unchanged

## Test Plan

- [x] `jobSpecOverlay` — parses a `consumes:` list, handles empty list,
handles missing key
- [x] `buildRegistryDataSpec` — includes user-declared links after
cluster-info
- [x] `buildRegistryDataSpec` — no additional links when no sidecars
present
- [x] `buildRegistryDataSpec` — optional flag rendered correctly
- [x] `generateManifestTemplate` with overlay — overlay content present
and before `YAML.dump`
- [x] `generateManifestTemplate` with empty overlay — still produces
valid template
- [x] Full suite: `go test ./...` — 0 failures (Docker integration test
skipped due to no daemon)

Made with [Cursor](https://cursor.com)

Author: drich10

internal/carvel/baker.go

@@ -255,6 +255,48 @@ func (b *baker) progress(message string) {
 	_, _ = fmt.Fprintln(b.progressWriter, message)
 }
 
+// deduplicateConsumes removes duplicate BOSH link consumer entries by name.
+// Identical duplicates are dropped silently. If two entries share a name but
+// differ in type or optional, the first is kept and a WARNING is emitted —
+// BOSH rejects duplicate link names in job.MF, so the second is always ignored.
+func (b *baker) deduplicateConsumes(consumes []boshLinkConsumer) []boshLinkConsumer {
+	seen := make(map[string]boshLinkConsumer)
+	var deduped []boshLinkConsumer
+	for _, c := range consumes {
+		existing, ok := seen[c.Name]
+		if !ok {
+			seen[c.Name] = c
+			deduped = append(deduped, c)
+			continue
+		}
+		if existing != c {
+			b.progress(fmt.Sprintf(
+				"WARNING: duplicate BOSH link consumer name %q found across packageinstalls.\n"+
+					"  Keeping:  {type: %s, optional: %v}\n"+
+					"  Ignoring: {type: %s, optional: %v}\n"+
+					"  Ensure all packageinstalls agree on the link definition.",
+				c.Name, existing.Type, existing.Optional, c.Type, c.Optional,
+			))
+		}
+	}
+	return deduped
+}
+
+// boshLinkConsumer declares a BOSH link the registry-data job should consume.
+// Populated from per-packageinstall *.job-spec-overlay.yml sidecar files.
+type boshLinkConsumer struct {
+	Name     string `yaml:"name"`
+	Type     string `yaml:"type"`
+	Optional bool   `yaml:"optional"`
+}
+
+// jobSpecOverlay is the schema for <entry>.job-spec-overlay.yml sidecar files.
+// kiln reads these from packageinstalls/ and merges the consumes entries into
+// the generated registry-data job.MF alongside the hardcoded cluster-info link.
+type jobSpecOverlay struct {
+	Consumes []boshLinkConsumer `yaml:"consumes"`
+}
+
 func validateVariables(vars []proofing.Variable) error {
 	var errs []error
 	for i, v := range vars {
@@ -313,6 +355,7 @@ files:
 
 	registryDataTemplates := ""
 	registryDataProperties := ""
+	var allConsumes []boshLinkConsumer
 
 	b.progress("  Configuring package installs")
 	for _, entry := range b.metadata.PackageInstalls {
@@ -362,7 +405,33 @@ files:
 			return err
 		}
 
-		manifestTemplate := generateManifestTemplate(entry)
+		// Read optional values-overlay ERB file alongside the packageinstall YAML.
+		overlayContent := ""
+		overlayData, overlayErr := os.ReadFile(path.Join(b.source, "packageinstalls", entry+".values-overlay.erb"))
+		if overlayErr != nil {
+			if !errors.Is(overlayErr, os.ErrNotExist) {
+				return overlayErr
+			}
+		} else {
+			overlayContent = string(overlayData)
+		}
+
+		// Read optional job-spec-overlay sidecar to discover additional BOSH link consumptions.
+		jobSpecOverlayPath := path.Join(b.source, "packageinstalls", entry+".job-spec-overlay.yml")
+		overlayData, overlayErr = os.ReadFile(jobSpecOverlayPath)
+		if overlayErr != nil {
+			if !errors.Is(overlayErr, os.ErrNotExist) {
+				return overlayErr
+			}
+		} else {
+			var overlay jobSpecOverlay
+			if parseErr := yaml.Unmarshal(overlayData, &overlay); parseErr != nil {
+				return fmt.Errorf("parsing %s: %w", jobSpecOverlayPath, parseErr)
+			}
+			allConsumes = append(allConsumes, overlay.Consumes...)
+		}
+
+		manifestTemplate := generateManifestTemplate(entry, overlayContent)
 
 		err = os.WriteFile(
 			path.Join(dirName, "jobs", "registry-data", "templates", "packageinstalls", entry+".yml.erb"),
@@ -374,18 +443,12 @@ files:
 		}
 	}
 
-	registryDataSpec := `---
-name: registry-data
-templates:
-` + registryDataTemplates +
-		`packages:
-- registry-data
-consumes:
-- name: cluster
-  type: cluster-info
-  optional: true
-properties:
-` + registryDataProperties
+	deduped := b.deduplicateConsumes(allConsumes)
+
+	registryDataSpec, err := buildRegistryDataSpec(registryDataTemplates, registryDataProperties, deduped)
+	if err != nil {
+		return err
+	}
 
 	err = os.WriteFile(path.Join(dirName, "jobs", "registry-data", "spec"), []byte(registryDataSpec), 0644)
 	if err != nil {
@@ -395,7 +458,35 @@ properties:
 	return nil
 }
 
-func generateManifestTemplate(entry string) string {
+// buildRegistryDataSpec constructs the job.MF content for the registry-data BOSH job.
+// It always includes the hardcoded cluster-info link and appends any additional links
+// collected from *.job-spec-overlay.yml sidecars in the packageinstalls/ directory.
+func buildRegistryDataSpec(templates, properties string, additionalLinks []boshLinkConsumer) (string, error) {
+	extraLinks := ""
+	if len(additionalLinks) > 0 {
+		data, err := yaml.Marshal(additionalLinks)
+		if err != nil {
+			return "", err
+		}
+		extraLinks = string(data)
+	}
+	return `---
+name: registry-data
+templates:
+` + templates + `packages:
+- registry-data
+consumes:
+- name: cluster
+  type: cluster-info
+  optional: true
+` + extraLinks + `properties:
+` + properties, nil
+}
+
+// generateManifestTemplate produces the ERB template for the registry-data BOSH job.
+// overlayContent is optional ERB code injected into the values manipulation block
+// before YAML.dump(values) is called, enabling BOSH link-based value overrides.
+func generateManifestTemplate(entry, overlayContent string) string {
 	return `---
 apiVersion: v1
 kind: ServiceAccount
@@ -442,6 +533,7 @@ stringData:
     values["context"]["namespace"] = link("cluster").p("content-namespace") rescue "default"
   end
 %>
+` + overlayContent + `
 <%= YAML.dump(values).split("\n").map { |line| "    " + line }.join("\n") %>
 ---
 apiVersion: packaging.carvel.dev/v1alpha1

internal/carvel/baker_test.go

@@ -58,7 +58,7 @@ var _ = Describe("Carvel Baker", func() {
 		var template string
 
 		BeforeEach(func() {
-			template = generateManifestTemplate("test-install")
+			template = generateManifestTemplate("test-install", "")
 		})
 
 		It("generates a ServiceAccount", func() {
@@ -103,6 +103,183 @@ var _ = Describe("Carvel Baker", func() {
 		It("handles YAML conversion for string values", func() {
 			Expect(template).To(ContainSubstring(`values = YAML.load(values) if values.is_a?(String)`))
 		})
+
+		Context("with overlay content", func() {
+			It("includes overlay content before YAML.dump", func() {
+				overlay := `<% values["syslog_agent"]["cache"]["url"] = "https://1.2.3.4:9000" %>`
+				tmpl := generateManifestTemplate("test-install", overlay)
+				Expect(tmpl).To(ContainSubstring(overlay))
+				overlayIdx := strings.Index(tmpl, overlay)
+				dumpIdx := strings.Index(tmpl, "YAML.dump(values)")
+				Expect(overlayIdx).To(BeNumerically("<", dumpIdx), "overlay must appear before YAML.dump")
+			})
+
+			It("produces valid output with empty overlay", func() {
+				tmpl := generateManifestTemplate("test-install", "")
+				Expect(tmpl).To(ContainSubstring("YAML.dump(values)"))
+				Expect(tmpl).NotTo(BeEmpty())
+			})
+		})
+	})
+
+	Context("buildRegistryDataSpec", func() {
+		It("includes user-declared additional links after cluster-info", func() {
+			links := []boshLinkConsumer{
+				{Name: "binding_cache", Type: "binding_cache", Optional: false},
+			}
+			spec, err := buildRegistryDataSpec("", "", links)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(spec).To(ContainSubstring("name: binding_cache"))
+			Expect(spec).To(ContainSubstring("type: binding_cache"))
+			Expect(spec).To(ContainSubstring("optional: false"))
+			Expect(spec).To(ContainSubstring("name: cluster"))
+		})
+
+		It("marks optional links correctly", func() {
+			links := []boshLinkConsumer{
+				{Name: "optional-link", Type: "some-type", Optional: true},
+			}
+			spec, err := buildRegistryDataSpec("", "", links)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(spec).To(ContainSubstring("optional: true"))
+		})
+
+		It("includes only cluster-info when no additional links are declared", func() {
+			spec, err := buildRegistryDataSpec("", "", nil)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(spec).To(ContainSubstring("name: cluster"))
+			Expect(spec).NotTo(ContainSubstring("name: binding_cache"))
+		})
+
+		It("safely encodes link names containing YAML-special characters", func() {
+			// yaml.Marshal quotes/blocks the value so it cannot inject extra YAML keys.
+			// The real type field ("legit-type") must still appear at the correct level.
+			links := []boshLinkConsumer{
+				{Name: "name: injected\ntype: evil", Type: "legit-type", Optional: false},
+			}
+			spec, err := buildRegistryDataSpec("", "", links)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(spec).To(ContainSubstring("type: legit-type"))
+		})
+
+		It("emits each unique link name only once given pre-deduplicated input", func() {
+			links := []boshLinkConsumer{
+				{Name: "binding_cache", Type: "binding_cache", Optional: false},
+			}
+			spec, err := buildRegistryDataSpec("", "", links)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(strings.Count(spec, "name: binding_cache")).To(Equal(1))
+		})
+	})
+
+	Context("deduplicateConsumes", func() {
+		var progressBuf strings.Builder
+		var b *baker
+
+		BeforeEach(func() {
+			progressBuf.Reset()
+			b = &baker{progressWriter: &progressBuf}
+		})
+
+		It("keeps all entries when names are unique", func() {
+			input := []boshLinkConsumer{
+				{Name: "link-a", Type: "type-a"},
+				{Name: "link-b", Type: "type-b"},
+			}
+			result := b.deduplicateConsumes(input)
+			Expect(result).To(HaveLen(2))
+			Expect(progressBuf.String()).To(BeEmpty())
+		})
+
+		It("silently drops exact duplicates without warning", func() {
+			input := []boshLinkConsumer{
+				{Name: "link-a", Type: "type-a", Optional: false},
+				{Name: "link-a", Type: "type-a", Optional: false},
+			}
+			result := b.deduplicateConsumes(input)
+			Expect(result).To(HaveLen(1))
+			Expect(progressBuf.String()).To(BeEmpty())
+		})
+
+		It("warns and keeps first when conflicting type definitions are found", func() {
+			input := []boshLinkConsumer{
+				{Name: "binding_cache", Type: "binding_cache"},
+				{Name: "binding_cache", Type: "binding-cache-v2"},
+			}
+			result := b.deduplicateConsumes(input)
+			Expect(result).To(HaveLen(1))
+			Expect(result[0].Type).To(Equal("binding_cache"))
+			Expect(progressBuf.String()).To(ContainSubstring("WARNING"))
+			Expect(progressBuf.String()).To(ContainSubstring(`"binding_cache"`))
+			Expect(progressBuf.String()).To(ContainSubstring("binding-cache-v2"))
+		})
+
+		It("warns and keeps first when optional flag differs", func() {
+			input := []boshLinkConsumer{
+				{Name: "link-a", Type: "type-a", Optional: false},
+				{Name: "link-a", Type: "type-a", Optional: true},
+			}
+			result := b.deduplicateConsumes(input)
+			Expect(result).To(HaveLen(1))
+			Expect(result[0].Optional).To(BeFalse())
+			Expect(progressBuf.String()).To(ContainSubstring("WARNING"))
+		})
+
+		It("emits one warning per conflict when multiple entries share a name", func() {
+			input := []boshLinkConsumer{
+				{Name: "link-a", Type: "type-a"},
+				{Name: "link-a", Type: "type-b"},
+				{Name: "link-a", Type: "type-c"},
+			}
+			result := b.deduplicateConsumes(input)
+			Expect(result).To(HaveLen(1))
+			Expect(result[0].Type).To(Equal("type-a"))
+			Expect(strings.Count(progressBuf.String(), "WARNING")).To(Equal(2))
+		})
+
+		It("returns nil without panicking when given a nil slice", func() {
+			result := b.deduplicateConsumes(nil)
+			Expect(result).To(BeNil())
+			Expect(progressBuf.String()).To(BeEmpty())
+		})
+	})
+
+	Context("jobSpecOverlay", func() {
+		It("parses a consumes list from YAML", func() {
+			content := `
+consumes:
+- name: binding_cache
+  type: binding_cache
+  optional: false
+`
+			var overlay jobSpecOverlay
+			err := yaml.Unmarshal([]byte(content), &overlay)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(overlay.Consumes).To(HaveLen(1))
+			Expect(overlay.Consumes[0].Name).To(Equal("binding_cache"))
+			Expect(overlay.Consumes[0].Type).To(Equal("binding_cache"))
+			Expect(overlay.Consumes[0].Optional).To(BeFalse())
+		})
+
+		It("handles an empty consumes list without error", func() {
+			var overlay jobSpecOverlay
+			err := yaml.Unmarshal([]byte("consumes: []"), &overlay)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(overlay.Consumes).To(BeEmpty())
+		})
+
+		It("handles a missing consumes key without error", func() {
+			var overlay jobSpecOverlay
+			err := yaml.Unmarshal([]byte("{}"), &overlay)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(overlay.Consumes).To(BeNil())
+		})
+
+		It("returns an error for malformed YAML", func() {
+			var overlay jobSpecOverlay
+			err := yaml.Unmarshal([]byte("consumes: [\ninvalid"), &overlay)
+			Expect(err).To(HaveOccurred())
+		})
 	})
 
 	Context("Bake", func() {
@@ -242,6 +419,8 @@ var _ = Describe("Carvel Baker", func() {
 					Expect(specStr).To(ContainSubstring("name: cluster"))
 					Expect(specStr).To(ContainSubstring("type: cluster-info"))
 					Expect(specStr).To(ContainSubstring("optional: true"))
+					Expect(specStr).To(ContainSubstring("name: binding_cache"))
+					Expect(specStr).To(ContainSubstring("type: binding_cache"))
 				})
 				It("generates runtime config referencing tanzu-content release", func() {
 					rcPath := filepath.Join(outputPath, "runtime_configs", "k8s-tile-test-pkgr.yml")
@@ -576,7 +755,7 @@ var _ = Describe("Carvel Baker", func() {
 
 	Context("generateManifestTemplate with different entry names", func() {
 		It("parameterizes the entry name throughout the template", func() {
-			template := generateManifestTemplate("my-custom-pkg")
+			template := generateManifestTemplate("my-custom-pkg", "")
 
 			Expect(template).To(ContainSubstring(`p("my-custom-pkg.name")`))
 			Expect(template).To(ContainSubstring(`p("my-custom-pkg.version")`))
@@ -585,7 +764,7 @@ var _ = Describe("Carvel Baker", func() {
 		})
 
 		It("contains exactly 6 K8s resource documents", func() {
-			template := generateManifestTemplate("pkg")
+			template := generateManifestTemplate("pkg", "")
 			docs := strings.Split(template, "---")
 			nonEmpty := 0
 			for _, doc := range docs {

internal/carvel/testdata/sample-tile/packageinstalls/test-install.job-spec-overlay.yml

@@ -0,0 +1,4 @@
+consumes:
+- name: binding_cache
+  type: binding_cache
+  optional: false