Route kiln test through docker-virtual registry and refactor test-script generation

README.md

@@ -520,56 +520,65 @@ Any variables that Kilnfile needs for the kiln re-bake command should be set in
 
 ### `test`
 
-The `test` command exercises to ginkgo tests under the `/<tile>/test/manifest` and `/<tile>/migrations` paths of the `pivotal/tas` repos (where `<tile>` is tas, ist, or tasw).
+The `test` command exercises the Ginkgo tests under the `/<tile>/test/manifest` and `/<tile>/migrations` paths of the `pivotal/tas` repos (where `<tile>` is tas, ist, or tasw).
 
-Running these tests requires a docker daemon. It also requires the user to
-provide Artifactory credentials via the ARTIFACTORY_USERNAME and
-ARTIFACTORY_PASSWORD environment variables to allow the ops-manifest gem to
-be installed. The credentials must have access to the `tas-rel-eng-gem-dev-local`
-repository within Broadcom's Artifactory.
+Running these tests requires a Docker daemon (or Podman API-compatible socket). You must provide **ARTIFACTORY_USERNAME** and **ARTIFACTORY_PASSWORD** using **`-e`** and/or **exported** environment variables. They are used for the **ops-manifest** gem (`tas-rel-eng-gem-dev-local`), for **Go module** downloads during **`go install ginkgo`** (via **`GOPROXY`** / **`GOSUMDB=off`** in the embedded Dockerfile), and Kiln sends the same credentials to the daemon as **registry auth** so base images can be pulled from **docker-virtual** (`tas-rel-eng-docker-virtual.usw1.packages.broadcom.com`) **without a separate `docker login`** for `kiln test`.
+
+The embedded Dockerfile pins **`FROM`** paths on that registry. The registry hostname in Kiln’s **`AuthConfigs`** must stay aligned with those **`FROM`** lines (see **`DockerVirtualRegistryHost`** in `internal/test/container.go`). Passwords with characters that are special in URLs may not behave the same as URL-encoded credentials when interpolated into **`GOPROXY`** inside the Dockerfile.
+
+If either credential is missing, `kiln test` exits with an error before talking to Docker.
 
 If you run into this docker error `could not execute "test": failed to connect to Docker daemon: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running`,
 then create a symlink `sudo ln -s ~/.docker/run/docker.sock /var/run/docker.sock`
 
-Here are command line examples:
+Examples:
 
 ```
 $ cd ~/workspace/tas/ist
 $ kiln test -e ARTIFACTORY_USERNAME=myuser -e ARTIFACTORY_PASSWORD=secretpassword
 ```
 
 ```
-cd ~
-$ kiln test --verbose -tp ~/workspace/tas/ist --ginkgo-manifest-flags "-p -nodes 8 -v"
+$ export ARTIFACTORY_USERNAME=myuser
+$ export ARTIFACTORY_PASSWORD=secretpassword
+$ kiln test --verbose -tp ~/workspace/tas/ist --ginkgo-flags "-p -nodes 8 -v"
 ```
 
 <details>
   <summary>Additional test options</summary>
 
-##### `--ginkgo-manifest-flags`
+##### `--ginkgo-flags`
+
+The `--ginkgo-flags` flag can be used to pass through Ginkgo test flags. The defaults being passed through are `-r -p -slowSpecThreshold 15`. Pass `help` as a flag to retrieve the available options for the embedded version of ginkgo.
 
-The `--ginkgo-manifest-flags` flag can be used to pass through Ginkgo test flags. The defaults being passed through are `-r -p -slowSpecThreshold 15`. Pass `help` as a flag to retrieve the available options for the embeded version of ginkgo.
+#### `--manifest`
 
-#### `--manifest-only`
+The `--manifest` flag can be used to run only Manifest tests.
 
-The `--manifest-only` flag can be used to run only Manifest tests. If not passed, `kiln test` will run both Manifest and Migration tests by default.
+#### `--migrations`
 
-#### `--migrations-only`
+The `--migrations` flag can be used to run only Migration tests.
 
-The `--migrations-only` flag can be used to run only Migration tests. If not passed, `kiln test` will run both Manifest and Migration tests by default.
+#### `--stability`
+
+The `--stability` flag can be used to run only Stability tests.
 
 ##### `--tile-path`
 
-The `--tile-path` (`-tp`) flag can be set the path the directory you wish to test. It defaults to the current working directory. For example
+The `--tile-path` (`-tp`) flag can be set to the directory you wish to test. It defaults to the current working directory. For example:
 
 ```
-$ kiln test -tp ~/workspace/tas/ist
+$ kiln test -e ARTIFACTORY_USERNAME=myuser -e ARTIFACTORY_PASSWORD=secret -tp ~/workspace/tas/ist
 ```
 
 ##### `--verbose`
 
 The `--verbose` (`-v`) flag will log additional debugging info.
 
+##### `--silent`
+
+The `--silent` (`-s`) flag hides Kiln info lines (not Ginkgo output).
+
 </details>
 
 ### `fetch`

internal/commands/test_tile.go

@@ -2,7 +2,6 @@ package commands
 
 import (
 	"context"
-	_ "embed"
 	"fmt"
 	"io"
 	"os"
@@ -17,15 +16,15 @@ type TileTestFunction func(ctx context.Context, w io.Writer, configuration test.
 
 type TileTest struct {
 	Options struct {
-		TilePath   string `             long:"tile-path"                default:"."                             description:"Path to the Tile directory (e.g., ~/workspace/tas/ist)."`
-		Verbose    bool   `short:"v"    long:"verbose"                  default:"true"                          description:"Print info lines. This doesn't affect Ginkgo output."`
+		TilePath   string `             long:"tile-path"                default:"."                             description:"Path to the Tile directory (e.g. ~/workspace/tas/ist)."`
+		Verbose    bool   `short:"v"    long:"verbose"                  default:"false"                         description:"Print container ID, per-suite start/end timestamps, and timestamped pass/fail summary. Also enables npm install output. Does not affect Ginkgo or npm test output."`
 		Silent     bool   `short:"s"    long:"silent"                   default:"false"                         description:"Hide info lines. This doesn't affect Ginkgo output."`
 		Manifest   bool   `             long:"manifest"                 default:"false"                         description:"Focus the Manifest tests."`
 		Migrations bool   `             long:"migrations"               default:"false"                         description:"Focus the Migration tests."`
 		Stability  bool   `             long:"stability"                default:"false"                         description:"Focus the Stability tests."`
 
-		EnvironmentVars []string `short:"e"    long:"environment-variable"                                             description:"Pass environment variable to the test suites. For example --stability -e 'PRODUCT=srt'."`
-		GingkoFlags     string   `             long:"ginkgo-flags"             default:"-r -p -slowSpecThreshold 15"   description:"Flags to pass to the Ginkgo Manifest and Stability test suites."`
+		EnvironmentVars []string `short:"e"    long:"environment-variable"                                             description:"Pass environment variables to the test suites (e.g. -e 'PRODUCT=srt'). Include -e ARTIFACTORY_USERNAME=... and -e ARTIFACTORY_PASSWORD=... unless they are exported."`
+		GinkgoFlags     string   `             long:"ginkgo-flags"             default:"-r -p -slowSpecThreshold 15"   description:"Flags to pass to the Ginkgo Manifest and Stability test suites."`
 	}
 	function TileTestFunction
 	output   io.Writer
@@ -64,22 +63,27 @@ func (cmd TileTest) configuration() (test.Configuration, error) {
 	if _, err := os.Stat(absPath); err != nil {
 		return test.Configuration{}, fmt.Errorf("failed to get information about --tile-path: %w", err)
 	}
-	return test.Configuration{
+	if _, _, err := test.RequiredArtifactoryCredentials(cmd.Options.EnvironmentVars); err != nil {
+		return test.Configuration{}, err
+	}
+	cfg := test.Configuration{
 		AbsoluteTileDirectory: absPath,
 
 		RunAll:        !cmd.Options.Migrations && !cmd.Options.Manifest && !cmd.Options.Stability,
 		RunManifest:   cmd.Options.Manifest,
-		RunMetadata:   cmd.Options.Stability,
+		RunStability:  cmd.Options.Stability,
 		RunMigrations: cmd.Options.Migrations,
 
-		GinkgoFlags: cmd.Options.GingkoFlags,
+		GinkgoFlags: cmd.Options.GinkgoFlags,
 		Environment: cmd.Options.EnvironmentVars,
-	}, absErr
+		Verbose:     cmd.Options.Verbose,
+	}
+	return cfg, absErr
 }
 
 func (cmd TileTest) Usage() jhanda.Usage {
 	return jhanda.Usage{
-		Description:      "Run the Manifest, Migrations, and Stability tests for a Tile in a Docker container. Requires a Docker daemon to be running and Artifactory credentials to be provided via the ARTIFACTORY_USERNAME and ARTIFACTORY_PASSWORD environment variables to install the ops-manifest gem.",
+		Description:      "Run the Manifest, Migrations, and Stability tests for a Tile in a Docker container. Requires a Docker daemon. Requires ARTIFACTORY_USERNAME and ARTIFACTORY_PASSWORD (via -e or your environment) for the test image build and ops-manifest gem. Kiln passes the same credentials to the Docker daemon for pulling base images from docker-virtual (no separate docker login needed for kiln test).",
 		ShortDescription: "Runs unit tests for a Tile.",
 		Flags:            cmd.Options,
 	}

internal/commands/test_tile_artifactory_test.go

@@ -0,0 +1,58 @@
+package commands
+
+import (
+	"context"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/pivotal-cf/kiln/internal/test"
+)
+
+func TestTileTest_RequiresArtifactoryCredentials(t *testing.T) {
+	t.Setenv("ARTIFACTORY_USERNAME", "")
+	t.Setenv("ARTIFACTORY_PASSWORD", "")
+
+	err := NewTileTest().Execute([]string{})
+	require.Error(t, err)
+	require.ErrorContains(t, err, "ARTIFACTORY_USERNAME")
+	require.ErrorContains(t, err, "kiln test")
+}
+
+func TestTileTest_RequiresArtifactoryPassword(t *testing.T) {
+	t.Setenv("ARTIFACTORY_PASSWORD", "")
+
+	err := NewTileTest().Execute([]string{"-e", "ARTIFACTORY_USERNAME=onlyuser"})
+	require.Error(t, err)
+	require.ErrorContains(t, err, "ARTIFACTORY_PASSWORD")
+}
+
+func TestTileTest_PassesArtifactoryViaEnvironmentToConfiguration(t *testing.T) {
+	var captured test.Configuration
+	stub := func(_ context.Context, _ io.Writer, c test.Configuration) error {
+		captured = c
+		return nil
+	}
+	err := NewTileTestWithCollaborators(io.Discard, stub).Execute([]string{
+		"-e", "ARTIFACTORY_USERNAME=u",
+		"-e", "ARTIFACTORY_PASSWORD=p",
+	})
+	require.NoError(t, err)
+	require.Contains(t, captured.Environment, "ARTIFACTORY_USERNAME=u")
+	require.Contains(t, captured.Environment, "ARTIFACTORY_PASSWORD=p")
+}
+
+func TestTileTest_UsesProcessEnvArtifactoryCredentials(t *testing.T) {
+	t.Setenv("ARTIFACTORY_USERNAME", "fromenv")
+	t.Setenv("ARTIFACTORY_PASSWORD", "frompass")
+
+	var captured test.Configuration
+	stub := func(_ context.Context, _ io.Writer, c test.Configuration) error {
+		captured = c
+		return nil
+	}
+	err := NewTileTestWithCollaborators(io.Discard, stub).Execute([]string{})
+	require.NoError(t, err)
+	require.Empty(t, captured.Environment)
+}

internal/commands/test_tile_test.go

@@ -34,6 +34,12 @@ func init() {
 var _ = Describe("kiln test", func() {
 	var output bytes.Buffer
 
+	BeforeEach(func() {
+		t := GinkgoT()
+		t.Setenv("ARTIFACTORY_USERNAME", "ginkgo-test-user")
+		t.Setenv("ARTIFACTORY_PASSWORD", "ginkgo-test-pass")
+	})
+
 	AfterEach(func() {
 		output.Reset()
 	})
@@ -161,7 +167,7 @@ var _ = Describe("kiln test", func() {
 			Expect(w).NotTo(BeNil())
 
 			Expect(configuration.RunManifest).To(BeTrue())
-			Expect(configuration.RunMetadata).To(BeFalse())
+			Expect(configuration.RunStability).To(BeFalse())
 			Expect(configuration.RunMigrations).To(BeFalse())
 		})
 	})
@@ -183,13 +189,13 @@ var _ = Describe("kiln test", func() {
 			Expect(w).NotTo(BeNil())
 
 			Expect(configuration.RunManifest).To(BeFalse())
-			Expect(configuration.RunMetadata).To(BeFalse())
+			Expect(configuration.RunStability).To(BeFalse())
 			Expect(configuration.RunMigrations).To(BeTrue())
 		})
 	})
 
 	When("when the stability test is enabled", func() {
-		It("it sets the RunMetadata configuration flag", func() {
+		It("it sets the RunStability configuration flag", func() {
 			args := []string{"--stability"}
 
 			fakeTestFunc := fakes.TestTileFunction{}
@@ -205,14 +211,14 @@ var _ = Describe("kiln test", func() {
 			Expect(w).NotTo(BeNil())
 
 			Expect(configuration.RunManifest).To(BeFalse())
-			Expect(configuration.RunMetadata).To(BeTrue())
+			Expect(configuration.RunStability).To(BeTrue())
 			Expect(configuration.RunMigrations).To(BeFalse())
 		})
 	})
 
-	When("when the stability test is enabled", func() {
-		It("it sets the RunMetadata configuration flag", func() {
-			args := []string{"--stability"}
+	When("when ginkgo/v2 flag arguments are passed", func() {
+		It("it sets the GinkgoFlags configuration", func() {
+			args := []string{"--ginkgo-flags=peach pair"}
 
 			fakeTestFunc := fakes.TestTileFunction{}
 			fakeTestFunc.Returns(nil)
@@ -226,29 +232,42 @@ var _ = Describe("kiln test", func() {
 			Expect(ctx).NotTo(BeNil())
 			Expect(w).NotTo(BeNil())
 
-			Expect(configuration.RunManifest).To(BeFalse())
-			Expect(configuration.RunMetadata).To(BeTrue())
-			Expect(configuration.RunMigrations).To(BeFalse())
+			Expect(configuration.GinkgoFlags).To(Equal("peach pair"))
 		})
 	})
 
-	When("when ginkgo/v2 flag arguments are passed", func() {
-		It("it sets the GinkgoFlags configuration", func() {
-			args := []string{"--ginkgo-flags=peach pair"}
+	When("when Artifactory credentials are provided via -e", func() {
+		It("invokes the test function with those variables in Environment", func() {
+			args := []string{
+				"-e", "ARTIFACTORY_USERNAME=u",
+				"-e", "ARTIFACTORY_PASSWORD=p",
+			}
 
 			fakeTestFunc := fakes.TestTileFunction{}
 			fakeTestFunc.Returns(nil)
 
 			err := commands.NewTileTestWithCollaborators(&output, fakeTestFunc.Spy).Execute(args)
 			Expect(err).NotTo(HaveOccurred())
 
-			Expect(fakeTestFunc.CallCount()).To(Equal(1))
+			_, _, configuration := fakeTestFunc.ArgsForCall(0)
+			Expect(configuration.Environment).To(ContainElement("ARTIFACTORY_USERNAME=u"))
+			Expect(configuration.Environment).To(ContainElement("ARTIFACTORY_PASSWORD=p"))
+		})
+	})
 
-			ctx, w, configuration := fakeTestFunc.ArgsForCall(0)
-			Expect(ctx).NotTo(BeNil())
-			Expect(w).NotTo(BeNil())
+	When("when Artifactory credentials are missing", func() {
+		It("returns an error before invoking the test function", func() {
+			t := GinkgoT()
+			t.Setenv("ARTIFACTORY_USERNAME", "")
+			t.Setenv("ARTIFACTORY_PASSWORD", "")
 
-			Expect(configuration.GinkgoFlags).To(Equal("peach pair"))
+			fakeTestFunc := fakes.TestTileFunction{}
+			fakeTestFunc.Returns(nil)
+
+			err := commands.NewTileTestWithCollaborators(&output, fakeTestFunc.Spy).Execute([]string{})
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("ARTIFACTORY_USERNAME"))
+			Expect(fakeTestFunc.CallCount()).To(Equal(0))
 		})
 	})
 

internal/test/Dockerfile

@@ -1,36 +1,52 @@
-FROM golang AS go-image
-FROM docker.io/pivotalcfreleng/kiln:v0.110.0-rc2 AS kiln
+# Base images: host must match DockerVirtualRegistryHost in internal/test/container.go (AuthConfigs).
+FROM tas-rel-eng-docker-virtual.usw1.packages.broadcom.com/golang AS go-image
+FROM tas-rel-eng-docker-virtual.usw1.packages.broadcom.com/pivotalcfreleng/kiln:v0.110.0-rc2 AS kiln
 
-FROM ruby:3.4.8 AS builder
+FROM tas-rel-eng-docker-virtual.usw1.packages.broadcom.com/ruby:3.4.8 AS builder
 RUN mkdir -p -m 0700 ~/.ssh && ssh-keyscan github.com >> ~/.ssh/known_hosts
 
-FROM ruby:3.4.8
+FROM tas-rel-eng-docker-virtual.usw1.packages.broadcom.com/ruby:3.4.8
 
-# Install Go
+# ── Stable tools — no credentials; these layers cache across credential rotation ──
+
+# Go runtime
 COPY --from=go-image /usr/local/go/ /usr/local/go/
 ENV GOROOT=/usr/local/go/
 ENV PATH="$GOROOT/bin:/root/go/bin:$PATH"
 
-# Install Kiln
+# Kiln binary (used by ops-manifest during manifest tests)
 COPY --from=kiln /kiln /usr/local/bin/kiln
 
-# Install JQ
-RUN apt-get update && apt-get install jq -y
+# System packages (consolidated to one layer; before credentials so they stay cached)
+RUN apt-get update \
+ && apt-get install --no-install-recommends -y jq nodejs npm \
+ && rm -rf /var/lib/apt/lists/*
 
-# Install Ginkgo
-RUN go install github.com/onsi/ginkgo/ginkgo@latest
+# Go toolchain settings — must appear before any `go install`
+# CGO_ENABLED=0: pure-Go build avoids gcc issues on arm64 (e.g. Podman on Apple Silicon).
+ENV GOTOOLCHAIN=local
+ENV CGO_ENABLED=0
 
-# Install NodeJS
-RUN apt-get update && apt-get install nodejs npm -y
+# ── Ginkgo — credentials scoped to this RUN only so layers above stay cached ──
+# Pinned to v1.16.5 for reproducibility (last stable v1; tiles using ginkgo v2 use their own binary).
 
-# Install OpsManifest from Artifactory
 ARG ARTIFACTORY_USERNAME
 ARG ARTIFACTORY_PASSWORD
 
+RUN GOPROXY=https://${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD}@usw1.packages.broadcom.com/artifactory/api/go/tas-rel-eng-go-virtual \
+    GOSUMDB=off \
+    go install github.com/onsi/ginkgo/ginkgo@v1.16.5
+
+# ── ops-manifest gem — credentials exported for gem source registration at build time ──
+
 ENV ARTIFACTORY_USERNAME=${ARTIFACTORY_USERNAME}
 ENV ARTIFACTORY_PASSWORD=${ARTIFACTORY_PASSWORD}
 
 RUN gem source -a https://${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD}@usw1.packages.broadcom.com/artifactory/api/gems/tas-rel-eng-gem-dev-local/
 RUN gem install --verbose ops-manifest -v 0.0.4.pre
 
 RUN which ops-manifest
+
+RUN printf '%s\n%s\n' \
+    'registry=https://${ARTIFACTORY_USERNAME}:${ARTIFACTORY_PASSWORD}@usw1.packages.broadcom.com/artifactory/api/npm/tis-npm-virtual/' \
+    'always-auth=true' > /root/.npmrc