fix: variable file default discovery for nested tiles (#538)

There's a bug where Kiln `re-bake` wasn't accounting for the relative
tile / Kilnfile path when automatically discovering variable files.
Since `re-bake` relies heavily on convention in typical use, we'd like
to fix this.

### Reproducing the bug

I forked `hello-tile` and modified it to have a nested tile directory
with a single variable `$( variable "hellothere" )` to be interpolated
in its `base.yml`. I created a variables file under
`nest/variables/hellothere.yml` defining its namesake.

It's expected at this point that we need to pass the `--variables-file`
explicitly because `kiln` doesn't know about the nesting yet:

**Creating the record**
```sh
kiln bake --final --version 6 \
   --vr artifactory_host=<redacted> \
   --vr artifactory_repo=<redacted> \
   --vr artifactory_username=<redacted> \
   --vr artifactory_password=<redacted> \
--metadata nest/base.yml \
--icon=nest/icon.png \
--kilnfile nest/Kilnfile \
--variables-file nest/variables/hellothere.yml
Setting default credentials from ~/.kiln/credentials.yml. (hint: --variable-file overrides this default. --variable overrides both.)
Warning: The "allow-only-publishable-releases" flag was not set. Some fetched releases may be intended for development/testing only. EXERCISE CAUTION WHEN PUBLISHING A TILE WITH THESE RELEASES!
Gathering releases...
All releases already downloaded
Reading release manifests...
Reading stemcell criteria from Kilnfile.lock
Encoding icon...
Building tile-6.pivotal...
Adding metadata/metadata.yml to tile-6.pivotal...
Creating empty migrations folder in tile-6.pivotal...
Adding releases/hello-release-1.0.5-ubuntu-jammy-1.737.tgz to tile-6.pivotal…
```

This is expected behavior.

Then we find the bug:

**Re-baking the record**
```sh
mv nest/bake_records/6.json /tmp/6.json
mv tile-6.pivotal ~/workspace/tilediff/tile-6-new.zip

kiln re-bake --output-file /tmp/tile-6-rebake.pivotal /tmp/6.json
Setting default credentials from ~/.kiln/credentials.yml. (hint: --variable-file overrides this default. --variable overrides both.)
Warning: The "allow-only-publishable-releases" flag was not set. Some fetched releases may be intended for development/testing only. EXERCISE CAUTION WHEN PUBLISHING A TILE WITH THESE RELEASES!
Gathering releases...
All releases already downloaded
Reading release manifests...
Reading stemcell criteria from Kilnfile.lock
Encoding icon...
2025/03/13 13:45:30 could not execute "re-bake": failed when rendering a template: nest/base.yml:3:10: executing "nest/base.yml" at <variable "hellothere">: error calling variable: could not find variable with key 'hellothere'
```

### The fix

Essentially we're including the nested directory when available during
our variable files search. We define

```go
func getVariablesDir(fs flags.FileSystem, kilnfilePath string) string
```

and then use it during the search:

```go
func getVariablesFilePaths(fs flags.FileSystem, kilnfilePath string) ([]string, error) {
	variablesDirPath := getVariablesDir(fs, kilnfilePath)
	files, err := fs.ReadDir(variablesDirPath)
...
```

and 

```go
func variablesDirPresent(fs flags.FileSystem, kilnfilePath string) bool {
	variablesDirPath := getVariablesDir(fs, kilnfilePath)
	file, err := fs.Stat(variablesDirPath)
...
```

### Testing the fix

Let's build this branch as `kiln-dev` and then re-run the above commands
in an equivalent way:

**Creating the record**
```sh
kiln-dev bake --final --version 6 \
   --vr artifactory_host=<redacted> \
   --vr artifactory_repo=<redacted> \
   --vr artifactory_username=<redacted> \
   --vr artifactory_password=<redacted> \
--metadata nest/base.yml \
--icon=nest/icon.png \
--kilnfile nest/Kilnfile
Setting default credentials from ~/.kiln/credentials.yml. (hint: --variable-file overrides this default. --variable overrides both.)
Warning: The "allow-only-publishable-releases" flag was not set. Some fetched releases may be intended for development/testing only. EXERCISE CAUTION WHEN PUBLISHING A TILE WITH THESE RELEASES!
Gathering releases...
All releases already downloaded
Reading release manifests...
Reading stemcell criteria from Kilnfile.lock
Encoding icon...
Building tile-6.pivotal...
Adding metadata/metadata.yml to tile-6.pivotal...
Creating empty migrations folder in tile-6.pivotal...
Adding releases/hello-release-1.0.5-ubuntu-jammy-1.737.tgz to tile-6.pivotal...
```
 
**Re-baking the record**
```sh
kiln-dev re-bake --output-file /tmp/tile-6-rebake.pivotal /tmp/6.json
Setting default credentials from ~/.kiln/credentials.yml. (hint: --variable-file overrides this default. --variable overrides both.)
Warning: The "allow-only-publishable-releases" flag was not set. Some fetched releases may be intended for development/testing only. EXERCISE CAUTION WHEN PUBLISHING A TILE WITH THESE RELEASES!
Gathering releases...
All releases already downloaded
Reading release manifests...
Reading stemcell criteria from Kilnfile.lock
Encoding icon...
Building /tmp/tile-6-rebake.pivotal...
Adding metadata/metadata.yml to /tmp/tile-6-rebake.pivotal...
Creating empty migrations folder in /tmp/tile-6-rebake.pivotal...
Adding releases/hello-release-1.0.5-ubuntu-jammy-1.737.tgz to /tmp/tile-6-rebake.pivotal…
```

fixing the issue.

Author: davewalter

internal/commands/bake.go

@@ -299,25 +299,39 @@ func shouldNotUseDefaultKilnfileFlag(args []string) bool {
 		!flags.IsSet("kf", "kilnfile", args)
 }
 
-func variablesDirPresent(fs flags.FileSystem) bool {
-	file, err := fs.Stat("variables")
+func variablesDirPresent(fs flags.FileSystem, kilnfilePath string) bool {
+	variablesDirPath := getVariablesDir(fs, kilnfilePath)
+	file, err := fs.Stat(variablesDirPath)
 	return err == nil && file != nil
 }
 
-func getVariablesFilePaths(fs flags.FileSystem) ([]string, error) {
-	files, err := fs.ReadDir("variables")
+func getVariablesFilePaths(fs flags.FileSystem, kilnfilePath string) ([]string, error) {
+	variablesDirPath := getVariablesDir(fs, kilnfilePath)
+	files, err := fs.ReadDir(variablesDirPath)
 	if err != nil {
 		return nil, err
 	}
 	var varFiles []string
 	for _, file := range files {
 		if strings.HasSuffix(file.Name(), ".yml") {
-			varFiles = append(varFiles, "variables/"+file.Name())
+			varFiles = append(varFiles, filepath.Join(variablesDirPath, file.Name()))
 		}
 	}
 	return varFiles, nil
 }
 
+func getVariablesDir(fs flags.FileSystem, kilnfilePath string) string {
+	variablesDirPath := "variables"
+	if kilnfilePath != "" {
+		_, err := fs.Stat(kilnfilePath)
+		if err == nil {
+			kilnfileParentDir := filepath.Dir(kilnfilePath)
+			variablesDirPath = filepath.Join(kilnfileParentDir, variablesDirPath)
+		}
+	}
+	return variablesDirPath
+}
+
 func (b *Bake) loadFlags(args []string) error {
 	_, err := flags.LoadWithDefaultFilePaths(&b.Options, args, b.fs.Stat)
 	if err != nil {
@@ -338,8 +352,9 @@ func (b *Bake) loadFlags(args []string) error {
 	}
 
 	// setup default tile variables
-	if variablesDirPresent(b.fs) {
-		variablesFilePaths, err := getVariablesFilePaths(b.fs)
+	kilnfilePath := b.Options.Kilnfile
+	if variablesDirPresent(b.fs, kilnfilePath) {
+		variablesFilePaths, err := getVariablesFilePaths(b.fs, kilnfilePath)
 		if err == nil {
 			if noTileVariablesFileAlreadySet(b, variablesFilePaths) {
 				setADefaultTileVariablesFile(b, variablesFilePaths)

internal/commands/bake_test.go

@@ -583,6 +583,59 @@ var _ = Describe("Bake", func() {
 		})
 
 		Context("when Kilnfile is specified", func() {
+			Context("when variable files can be auto discovered", func() {
+				It("renders with the discovered variables file", func() {
+					f1 := &fakes.FileInfo{}
+					f1.NameReturns("variables-file-1.yml")
+					f1.SizeReturns(100)
+					f1.ModeReturns(os.ModePerm)
+					f1.ModTimeReturns(time.Now())
+					f1.IsDirReturns(false)
+
+					f2 := &fakes.FileInfo{}
+					f2.NameReturns("variables-file-2.yml")
+					f2.SizeReturns(100)
+					f2.ModeReturns(os.ModePerm)
+					f2.ModTimeReturns(time.Now())
+					f2.IsDirReturns(false)
+
+					fakeFilesystem.ReadDirReturns([]os.FileInfo{f1, f2}, nil)
+
+					outputFile := "some-output-dir/some-product-file-1.2.3-build.4"
+					err := bake.Execute([]string{
+						"--forms-directory", "some-forms-directory",
+						"--instance-groups-directory", "some-instance-groups-directory",
+						"--jobs-directory", "some-jobs-directory",
+						"--metadata", "some-metadata",
+						"--output-file", outputFile,
+						"--properties-directory", "some-properties-directory",
+						"--releases-directory", someReleasesDirectory,
+						"--runtime-configs-directory", "some-other-runtime-configs-directory",
+						"--kilnfile", "Kilnfile",
+						"--bosh-variables-directory", "some-variables-directory",
+						"--version", "1.2.3", "--migrations-directory", "some-migrations-directory",
+						"--migrations-directory", "some-other-migrations-directory",
+						"--variables-file", "variables/variables-file-1.yml",
+						"--variables-file", "variables/variables-file-2.yml",
+					})
+					Expect(err).NotTo(HaveOccurred())
+					Expect(fakeStemcellService.FromKilnfileCallCount()).To(Equal(1))
+					Expect(fakeStemcellService.FromKilnfileArgsForCall(0)).To(Equal("Kilnfile"))
+					Expect(fakeFetcher.ExecuteCallCount()).To(Equal(1))
+					executeArgsForFetch := fakeFetcher.ExecuteArgsForCall(0)
+					Expect(executeArgsForFetch).To(Equal([]string{
+						"--kilnfile", "Kilnfile",
+						"--variables-file", "variables/variables-file-1.yml",
+						"--variables-file", "variables/variables-file-2.yml",
+						"--variables-file", "/home/.kiln/credentials.yml",
+						"--download-threads", "0",
+						"--no-confirm",
+						"--releases-directory",
+						someReleasesDirectory,
+					}))
+				})
+			})
+
 			It("renders the stemcell criteria in tile metadata from that specified the Kilnfile.lock", func() {
 				outputFile := "some-output-dir/some-product-file-1.2.3-build.4"
 				err := bake.Execute([]string{
@@ -606,8 +659,7 @@ var _ = Describe("Bake", func() {
 				executeArgsForFetch := fakeFetcher.ExecuteArgsForCall(0)
 				Expect(executeArgsForFetch).To(Equal([]string{
 					"--kilnfile", "Kilnfile",
-					"--variables-file",
-					"/home/.kiln/credentials.yml",
+					"--variables-file", "/home/.kiln/credentials.yml",
 					"--download-threads", "0",
 					"--no-confirm",
 					"--releases-directory",