From b4b14f0006a85d371aa8e269dc9c4af610fbab36 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Wed, 21 Jan 2026 12:34:57 -0600 Subject: [PATCH 01/18] Add a carvel package that handles baking carvel based packages --- go.mod | 2 +- internal/carvel/baker.go | 545 ++++++++++++++++++ internal/carvel/baker_suite_test.go | 13 + internal/carvel/baker_test.go | 159 +++++ internal/carvel/models/job.go | 7 + internal/carvel/models/metadata.go | 17 + internal/carvel/models/metadata_out.go | 31 + internal/carvel/models/package_install.go | 8 + .../carvel/models/package_install_props.go | 7 + internal/carvel/models/runtime_config.go | 22 + .../carvel/testdata/sample-tile/.gitignore | 3 + .../carvel/testdata/sample-tile/README.md | 4 + internal/carvel/testdata/sample-tile/base.yml | 19 + .../carvel/testdata/sample-tile/bundle.tar | Bin 0 -> 2048 bytes .../testdata/sample-tile/forms/.gitkeep | 0 .../testdata/sample-tile/forms/db_props.yml | 8 + internal/carvel/testdata/sample-tile/icon.png | Bin 0 -> 1185 bytes .../packageinstalls/test-install.yml | 6 + .../testdata/sample-tile/properties/.gitkeep | 0 .../sample-tile/properties/properties.yml | 9 + internal/carvel/testdata/sample-tile/version | 1 + internal/commands/carvel.go | 160 +++++ internal/commands/carvel_bake.go | 87 +++ internal/commands/carvel_bake_test.go | 106 ++++ main.go | 2 + 25 files changed, 1215 insertions(+), 1 deletion(-) create mode 100644 internal/carvel/baker.go create mode 100644 internal/carvel/baker_suite_test.go create mode 100644 internal/carvel/baker_test.go create mode 100644 internal/carvel/models/job.go create mode 100644 internal/carvel/models/metadata.go create mode 100644 internal/carvel/models/metadata_out.go create mode 100644 internal/carvel/models/package_install.go create mode 100644 internal/carvel/models/package_install_props.go create mode 100644 internal/carvel/models/runtime_config.go create mode 100644 internal/carvel/testdata/sample-tile/.gitignore create mode 100644 internal/carvel/testdata/sample-tile/README.md create mode 100644 internal/carvel/testdata/sample-tile/base.yml create mode 100644 internal/carvel/testdata/sample-tile/bundle.tar create mode 100644 internal/carvel/testdata/sample-tile/forms/.gitkeep create mode 100644 internal/carvel/testdata/sample-tile/forms/db_props.yml create mode 100644 internal/carvel/testdata/sample-tile/icon.png create mode 100644 internal/carvel/testdata/sample-tile/packageinstalls/test-install.yml create mode 100644 internal/carvel/testdata/sample-tile/properties/.gitkeep create mode 100644 internal/carvel/testdata/sample-tile/properties/properties.yml create mode 100644 internal/carvel/testdata/sample-tile/version create mode 100644 internal/commands/carvel.go create mode 100644 internal/commands/carvel_bake.go create mode 100644 internal/commands/carvel_bake_test.go diff --git a/go.mod b/go.mod index f86e8097d..7fd93569b 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/go-git/go-git/v5 v5.16.5 github.com/google/go-github/v50 v50.2.0 github.com/jfrog/jfrog-client-go v1.52.0 + github.com/hashicorp/go-version v1.8.0 github.com/julienschmidt/httprouter v1.3.0 github.com/masterminds/sprig v2.22.0+incompatible github.com/maxbrunsfeld/counterfeiter/v6 v6.12.1 @@ -110,7 +111,6 @@ require ( github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-memdb v1.3.5 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/hashicorp/go-version v1.8.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect diff --git a/internal/carvel/baker.go b/internal/carvel/baker.go new file mode 100644 index 000000000..afb92410a --- /dev/null +++ b/internal/carvel/baker.go @@ -0,0 +1,545 @@ +package carvel + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strings" + + "github.com/pivotal-cf/kiln/internal/carvel/models" + + "github.com/hashicorp/go-version" + "gopkg.in/yaml.v3" +) + +// Baker transforms an imgpkg bundle and tile metadata into a BOSH release +// and kiln-compatible tile structure that can be baked into a .pivotal file. +type Baker interface { + Bake(source string) error + KilnBake(destination string) error + GetName() string + GetVersion() (string, error) + SetWriter(w io.Writer) +} + +// NewBaker creates a new Baker for transforming imgpkg bundles into BOSH releases. +func NewBaker() Baker { + return &baker{ + writer: io.Discard, + } +} + +type baker struct { + metadata models.Metadata + source, destination string + writer io.Writer +} + +func (b *baker) KilnBake(destination string) error { + cmd := exec.Command("kiln", + "bake", + "--skip-fetch", + "--output-file", destination, + ) + cmd.Dir = b.destination + out, err := cmd.CombinedOutput() + if err != nil { + b.log("failed to invoke kiln: " + string(out)) + return err + } + + return nil +} + +func (b *baker) Bake(source string) error { + b.source = source + b.destination = path.Join(source, ".ezbake") + + yamlPath := path.Join(source, "base.yml") + yamlData, err := os.ReadFile(yamlPath) + if err != nil { + return err + } + + err = yaml.Unmarshal(yamlData, &b.metadata) + if err != nil { + return err + } + + _, err = b.GetVersion() + if err != nil { + return err + } + + metadataVersion, err := version.NewVersion(b.metadata.MetadataVersion) + if err != nil { + return err + } + minVersion, _ := version.NewVersion("3.2.0") + if metadataVersion.LessThan(minVersion) { + return errors.New("tile metadata_version too old for kubernetes support (must be >=3.2.0)") + } + + err = b.generateBoshReleaseDir() + if err != nil { + b.log(err.Error()) + return err + } + + err = b.generateOutputTile() + if err != nil { + b.log(err.Error()) + return err + } + + return nil +} + +func (b *baker) GetName() string { + return b.metadata.Name +} + +func (b *baker) GetVersion() (string, error) { + re := regexp.MustCompile(`\s+`) + + // Replace all occurrences of whitespace with an empty string + versionNoSpace := re.ReplaceAllString(b.metadata.ProductVersion, "") + if versionNoSpace != `$(version)` { + return versionNoSpace, nil + } else { + // find the version from a "version" file + version, err := os.ReadFile(path.Join(b.source, "version")) + return strings.Trim(string(version), " \t\n\r"), err + } +} + +func (b *baker) SetWriter(w io.Writer) { + b.writer = w +} + +func (b *baker) log(message string) { + fmt.Fprintln(b.writer, message) +} + +func (b *baker) generateBoshReleaseDir() error { + dirName := path.Join(b.source, ".boshrelease") + // first clean out any previous bosh release directory + err := os.RemoveAll(dirName) + if err != nil { + return err + } + + commands := []*exec.Cmd{ + exec.Command("bosh", "init-release", "--dir="+dirName), + exec.Command("bosh", "add-blob", "--dir="+dirName, path.Join(b.source, "bundle.tar"), "imgpkg/bundle.tar"), + exec.Command("bosh", "generate-package", "--dir="+dirName, "registry-data"), + exec.Command("bosh", "generate-job", "--dir="+dirName, "registry-data"), + exec.Command("bosh", "generate-job", "--dir="+dirName, "package-install"), + } + for _, cmd := range commands { + b.log("executing " + cmd.String()) + out, err := cmd.CombinedOutput() + if err != nil { + return err + } + + b.log("output: " + string(out)) + } + + // Now populate the specs for packages and jobs + fileContents := map[string]string{ + "packages/registry-data/packaging": `set -eu +mkdir -p ${BOSH_INSTALL_TARGET}/imgpkg +cp imgpkg/*.tar ${BOSH_INSTALL_TARGET}/imgpkg +`, + "packages/registry-data/spec": `--- +name: registry-data +dependencies: [] +files: +- imgpkg/bundle.tar +`, + "jobs/registry-data/spec": `--- +name: registry-data +templates: {} +packages: +- registry-data +`, + } + for outpath, contents := range fileContents { + err = os.WriteFile(path.Join(dirName, outpath), []byte(contents), 0644) + if err != nil { + return err + } + } + + jobTemplates := "" + jobProperties := "" + // we need one PackageInstall for each entry in the metadata. + for _, entry := range b.metadata.PackageInstalls { + entry = strings.Trim(entry, "$() ") + entry = strings.TrimPrefix(entry, "package") + entry = strings.Trim(entry, `"' `) + + b.log("looking for package install: " + entry) + + // find this entry in the packageinstalls directory + matches, err := filepath.Glob(path.Join(b.source, "packageinstalls/*.yml")) + if err != nil { + return err + } + + for _, match := range matches { + yamlData, err := os.ReadFile(match) + if err != nil { + return err + } + + var pi models.PackageInstall + err = yaml.Unmarshal(yamlData, &pi) + if err != nil { + return err + } + if pi.Name != entry { + continue + } + + b.log("found " + pi.Name + " at " + match) + } + + // accumulate templates + jobTemplates += fmt.Sprintf(" packageinstalls/%s/name.erb: packageinstalls/%s/name\n", entry, entry) + jobTemplates += fmt.Sprintf(" packageinstalls/%s/version.erb: packageinstalls/%s/version\n", entry, entry) + jobTemplates += fmt.Sprintf(" packageinstalls/%s/values.yml.erb: packageinstalls/%s/values.yml\n", entry, entry) + // accumulate properties + jobProperties += " " + entry + ":\n" + jobProperties += ` name: + description: "package name" + version: + description: "package version" + values: + description: "values.yml contents" +` + os.MkdirAll(path.Join(dirName, "jobs", "package-install", "templates", "packageinstalls", entry), 0755) + templates := map[string]string{ + "name.erb": `<%= p("` + entry + `.name") %>`, + "version.erb": `<%= p("` + entry + `.version") %>`, + "values.yml.erb": `<% require 'yaml' %>` + "\n" + `<%= p("` + entry + `.values").is_a?(String) ? p("` + entry + `.values") : YAML.dump(p("` + entry + `.values")) %>`, + } + for fileName, contents := range templates { + err = os.WriteFile(path.Join(dirName, "jobs", "package-install", "templates", "packageinstalls", entry, fileName), []byte(contents), 0644) + if err != nil { + return err + } + } + } + + // now that we've collected all the templates and properties, write out the spec file for the + // package-install job. + contents := `--- +name: package-install +templates: +` + jobTemplates + + `packages: [] +properties: +` + jobProperties + err = os.WriteFile(path.Join(dirName, "jobs", "package-install", "spec"), []byte(contents), 0644) + if err != nil { + return err + } + + return nil +} + +func (b *baker) generateOutputTile() error { + // first clean out any previous tile directory + // Note: this directory should only ever contain generated files, which we are about to regenerate. + err := os.RemoveAll(b.destination) + if err != nil { + return err + } + + err = os.MkdirAll(b.destination, 0755) + if err != nil { + return err + } + + err = b.generateBaseYaml() + if err != nil { + return err + } + + err = b.copyFiles() + if err != nil { + return err + } + + err = b.generateJobFiles() + if err != nil { + return err + } + + err = b.generateInstanceGroupFiles() + if err != nil { + return err + } + + err = b.generateRuntimeConfigs() + if err != nil { + return err + } + + err = b.createBoshRelease() + if err != nil { + return err + } + + return nil +} + +func (b *baker) generateBaseYaml() error { + meta := models.MetadataOut{} + meta.Name = b.metadata.Name + meta.Label = b.metadata.Label + meta.IconImage = b.metadata.IconImage + meta.ProductVersion = b.metadata.ProductVersion + meta.MetadataVersion = b.metadata.MetadataVersion + meta.Rank = b.metadata.Rank + meta.Serial = b.metadata.Serial + meta.CompatibleKubernetesDistributions = b.metadata.CompatibleKubernetesDistributions + meta.FormTypes = b.metadata.FormTypes + meta.PropertyBlueprints = b.metadata.PropertyBlueprints + meta.Variables = b.metadata.Variables + meta.MinimumVersionForUpgrade = b.metadata.MinimumVersionForUpgrade + meta.RequiresKubernetes = true + // stemcell criteria are dummy data that OM will ignore when the tile is folded into + // TKR, but we need them as kiln inputs. + meta.StemcellCriteria.Os = "ubuntu-jammy" + meta.StemcellCriteria.Version = "1.446" + meta.InstanceGroups = []string{} + meta.RuntimeConfigs = []string{ + `$( runtime_config "` + b.metadata.Name + `-pkgr" )`, + } + + // we will use the tile name and version as the bosh release name and version. + meta.Releases = []string{ + `$( release "` + b.metadata.Name + `" )`, + } + + yamlData, err := yaml.Marshal(&meta) + if err != nil { + return err + } + err = os.WriteFile(path.Join(b.destination, "base.yml"), yamlData, 0644) // 0644 sets file permissions + if err != nil { + return err + } + return nil +} + +// copyfiles schleps all the files that we can collect from the source directory without +// modification: +// - variables, properties and forms +// - the icon file +// - the version file +func (b *baker) copyFiles() error { + for _, subdir := range []string{"bosh_variables", "forms", "properties"} { + info, err := os.Stat(path.Join(b.source, subdir)) + if err == nil && info.IsDir() { + err = os.CopyFS(path.Join(b.destination, subdir), os.DirFS(path.Join(b.source, subdir))) + if err != nil { + return err + } + } + } + + for _, fn := range []string{"icon.png", "version"} { + info, err := os.Stat(path.Join(b.source, fn)) + if err == nil && !info.IsDir() { + err = copyFileContents(path.Join(b.source, fn), path.Join(b.destination, fn)) + } + } + + return nil +} + +// generateRuntimeConfigs creates a runtime config that colocates registry-data and package-install jobs onto the +// registry VM (or whatever instance has the corresponding errands that will ingest the data) +func (b *baker) generateRuntimeConfigs() error { + err := os.MkdirAll(path.Join(b.destination, "runtime_configs"), 0755) + if err != nil { + return err + } + + registryDataJob := models.Job{ + Name: "registry-data", + Release: b.metadata.Name, + } + + // create the "package-install" job + props := map[string]models.PackageInstallProps{} + // we need one PackageInstall for each entry in the metadata. + for _, entry := range b.metadata.PackageInstalls { + entry = strings.Trim(entry, "$() ") + entry = strings.TrimPrefix(entry, "package") + entry = strings.Trim(entry, `"' `) + + // find this entry in the packageinstalls directory + matches, err := filepath.Glob(path.Join(b.source, "packageinstalls/*.yml")) + if err != nil { + return err + } + + found := false + for _, match := range matches { + yamlData, err := os.ReadFile(match) + if err != nil { + return err + } + + var pi models.PackageInstall + err = yaml.Unmarshal(yamlData, &pi) + if err != nil { + return err + } + if pi.Name != entry { + continue + } + + found = true + props[entry] = models.PackageInstallProps{ + Name: pi.PackageName, + Version: pi.PackageVersion, + Values: pi.Values, + } + } + if !found { + return errors.New("package install not found: " + entry) + } + } + + packageInstallJob := models.Job{ + Name: "package-install", + Release: b.metadata.Name, + Properties: props, + } + + inner := models.RuntimeConfigInner{ + Releases: []string{ + `$( release "` + b.metadata.Name + `" )`, + }, + Addons: []models.Addon{ + { + Name: b.metadata.Name + "-pkgr", + Include: models.Inclusion{ + Deployments: []string{ + `(( ..` + b.metadata.Name + `.deployment_name ))`, + }, + Jobs: []models.Job{ + {Name: "apply-packagerepos", Release: "registry"}, + {Name: "install-packages", Release: "registry"}, + }, + }, + Jobs: []models.Job{ + registryDataJob, + packageInstallJob, + }, + }, + }, + } + yamlData, err := yaml.Marshal(&inner) + if err != nil { + return err + } + rc := models.RuntimeConfigOuter{ + Name: b.metadata.Name + "-pkgr", + RuntimeConfig: string(yamlData), + } + + yamlData, err = yaml.Marshal(&rc) + if err != nil { + return err + } + err = os.WriteFile(path.Join(b.destination, "runtime_configs", b.metadata.Name+"-pkgr.yml"), yamlData, 0644) + if err != nil { + return err + } + + return nil +} + +// generateInstanceGroups creates an empty instance group folder +func (b *baker) generateInstanceGroupFiles() error { + err := os.MkdirAll(path.Join(b.destination, "instance_groups"), 0755) + if err != nil { + return err + } + + return nil +} + +// generateJobFiles creates an empty jobs folder +func (b *baker) generateJobFiles() error { + err := os.MkdirAll(path.Join(b.destination, "jobs"), 0755) + if err != nil { + return err + } + + return nil +} + +func (b *baker) createBoshRelease() error { + err := os.MkdirAll(path.Join(b.destination, "releases"), 0755) + if err != nil { + return err + } + + version, err := b.GetVersion() + if err != nil { + return err + } + + dirName := path.Join(b.source, ".boshrelease") + cmd := exec.Command("bosh", + "create-release", + "--dir="+dirName, + "--force", + "--name", b.metadata.Name, + "--version", version, + "--tarball", path.Join(b.destination, "releases", b.metadata.Name+"-"+version+".tgz")) + b.log("executing " + cmd.String()) + out, err := cmd.CombinedOutput() + b.log("output: " + string(out)) + if err != nil { + return err + } + + return nil +} + +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} diff --git a/internal/carvel/baker_suite_test.go b/internal/carvel/baker_suite_test.go new file mode 100644 index 000000000..6ce6cf99a --- /dev/null +++ b/internal/carvel/baker_suite_test.go @@ -0,0 +1,13 @@ +package carvel + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBaker(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Carvel Baker Suite") +} diff --git a/internal/carvel/baker_test.go b/internal/carvel/baker_test.go new file mode 100644 index 000000000..6a52238e6 --- /dev/null +++ b/internal/carvel/baker_test.go @@ -0,0 +1,159 @@ +package carvel + +import ( + "os" + "os/exec" + "path" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/carvel/models" + "gopkg.in/yaml.v3" +) + +func boshInstalled() bool { + _, err := exec.LookPath("bosh") + return err == nil +} + +func kilnInstalled() bool { + _, err := exec.LookPath("kiln") + return err == nil +} + +var _ = Describe("Carvel Baker", func() { + Context("Bake", func() { + When("the input directory contains k8s tile data", func() { + BeforeEach(func() { + if !boshInstalled() { + Skip("bosh CLI not installed - skipping integration test") + } + }) + var ( + inputPath, outputPath string + subject Baker + err error + ) + BeforeEach(func() { + var err error + inputPath, err = os.MkdirTemp("", "testinput-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + outputPath = path.Join(inputPath, ".ezbake") + err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + // create an initial git commit in the input directory + commands := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range commands { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + subject = NewBaker() + subject.SetWriter(GinkgoWriter) + }) + AfterEach(func() { + // Clean up the temp directory + if inputPath != "" { + os.RemoveAll(filepath.Dir(inputPath)) + } + }) + JustBeforeEach(func() { + err = subject.Bake(inputPath) + }) + When("the tile data is valid", func() { + JustBeforeEach(func() { + Expect(err).NotTo(HaveOccurred()) + }) + It("populates the output metadata", func() { + outMeta := models.MetadataOut{} + yamlPath := path.Join(outputPath, "base.yml") + yamlData, err := os.ReadFile(yamlPath) + Expect(err).NotTo(HaveOccurred()) + + err = yaml.Unmarshal(yamlData, &outMeta) + Expect(err).NotTo(HaveOccurred()) + + Expect(outMeta.Name).To(Equal("k8s-tile-test")) + Expect(outMeta.ProductVersion).To(Equal(`$( version )`)) + Expect(outMeta.MetadataVersion).To(Equal("3.2.0")) + Expect(outMeta.Rank).To(Equal(1)) + Expect(outMeta.Serial).To(BeFalse()) + Expect(outMeta.PropertyBlueprints).To(HaveLen(2)) + Expect(outMeta.FormTypes).To(HaveLen(1)) + Expect(outMeta.Variables).To(BeEmpty()) + Expect(outMeta.Releases).To(HaveLen(1)) + Expect(outMeta.Releases[0]).To(ContainSubstring("k8s-tile-test")) + Expect(outMeta.InstanceGroups).To(HaveLen(0)) + Expect(outMeta.RuntimeConfigs).To(HaveLen(1)) + Expect(outMeta.RuntimeConfigs[0]).To(Equal(`$( runtime_config "k8s-tile-test-pkgr" )`)) + Expect(outMeta.CompatibleKubernetesDistributions).To(HaveLen(1)) + Expect(outMeta.CompatibleKubernetesDistributions[0].Name).To(Equal("k0s")) + Expect(outMeta.CompatibleKubernetesDistributions[0].Version).To(Equal(">0.0.0")) + Expect(outMeta.RequiresKubernetes).To(BeTrue()) + }) + It("creates empty instance_group and jobs directories", func() { + Expect(filepath.Join(outputPath, "instance_groups")).To(BeADirectory()) + Expect(filepath.Join(outputPath, "jobs")).To(BeADirectory()) + }) + It("creates a runtime config", func() { + Expect(filepath.Join(outputPath, "runtime_configs")).To(BeADirectory()) + Expect(filepath.Join(outputPath, "runtime_configs", "k8s-tile-test-pkgr.yml")).To(BeAnExistingFile()) + }) + It("copies forms, properties, icon, version from the input", func() { + Expect(filepath.Join(outputPath, "properties", "properties.yml")).To(BeAnExistingFile()) + Expect(filepath.Join(outputPath, "forms", "db_props.yml")).To(BeAnExistingFile()) + Expect(filepath.Join(outputPath, "icon.png")).To(BeAnExistingFile()) + Expect(filepath.Join(outputPath, "version")).To(BeAnExistingFile()) + }) + It("Generates a bosh release tarball", func() { + Expect(filepath.Join(outputPath, "releases", "k8s-tile-test-0.1.1.tgz")).To(BeAnExistingFile()) + }) + It("can be kiln baked", func() { + if !kilnInstalled() { + Skip("kiln CLI not installed - skipping integration test") + } + err := subject.KilnBake(filepath.Join(outputPath, "my-tile.pivotal")) + Expect(err).NotTo(HaveOccurred()) + Expect(filepath.Join(outputPath, "my-tile.pivotal")).To(BeAnExistingFile()) + }) + }) + When("the tile metadata version is too old", func() { + BeforeEach(func() { + m := models.Metadata{ + Name: "k8s-tile-test", + Label: "test tile", + IconImage: "$( icon )", + MetadataVersion: "3.1.0", + MinimumVersionForUpgrade: "0.0.0", + ProductVersion: "$( version )", + Rank: 1, + Serial: false, + PropertyBlueprints: []string{ + `$( property "database_name" )`, + `$( property "admin_password" )`, + }, + FormTypes: []string{`$( form "db_props" )`}, + Variables: []string{}, + PackageInstalls: []string{`$( package "test-install" )`}, + } + yamlData, err := yaml.Marshal(&m) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(path.Join(inputPath, "base.yml"), yamlData, 0644) // 0644 sets file permissions + Expect(err).NotTo(HaveOccurred()) + }) + + It("fails to bake with an error", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("tile metadata_version too old")) + }) + }) + }) + }) +}) diff --git a/internal/carvel/models/job.go b/internal/carvel/models/job.go new file mode 100644 index 000000000..9b8ef81b1 --- /dev/null +++ b/internal/carvel/models/job.go @@ -0,0 +1,7 @@ +package models + +type Job struct { + Name string `yaml:"name"` + Release string `yaml:"release"` + Properties map[string]PackageInstallProps `yaml:"properties,omitempty"` +} diff --git a/internal/carvel/models/metadata.go b/internal/carvel/models/metadata.go new file mode 100644 index 000000000..a22b21d3c --- /dev/null +++ b/internal/carvel/models/metadata.go @@ -0,0 +1,17 @@ +package models + +type Metadata struct { + Name string `yaml:"name"` + ProductVersion string `yaml:"product_version"` + IconImage string `yaml:"icon_image"` + Label string `yaml:"label"` + MetadataVersion string `yaml:"metadata_version"` + MinimumVersionForUpgrade string `yaml:"minimum_version_for_upgrade"` + Rank int `yaml:"rank"` + Serial bool `yaml:"serial"` + PropertyBlueprints []string `yaml:"property_blueprints"` + FormTypes []string `yaml:"form_types"` + Variables []string `yaml:"variables"` + PackageInstalls []string `yaml:"package_installs"` + CompatibleKubernetesDistributions []ProductVersion `yaml:"compatible_kubernetes_distributions,omitempty"` +} diff --git a/internal/carvel/models/metadata_out.go b/internal/carvel/models/metadata_out.go new file mode 100644 index 000000000..88b652e96 --- /dev/null +++ b/internal/carvel/models/metadata_out.go @@ -0,0 +1,31 @@ +package models + +type MetadataOut struct { + Name string `yaml:"name"` + ProductVersion string `yaml:"product_version"` + IconImage string `yaml:"icon_image"` + Label string `yaml:"label"` + MetadataVersion string `yaml:"metadata_version"` + MinimumVersionForUpgrade string `yaml:"minimum_version_for_upgrade"` + Rank int `yaml:"rank"` + Serial bool `yaml:"serial"` + PropertyBlueprints []string `yaml:"property_blueprints"` + FormTypes []string `yaml:"form_types"` + Variables []string `yaml:"variables"` + InstanceGroups []string `yaml:"job_types"` + StemcellCriteria StemcellCriteria `yaml:"stemcell_criteria"` + Releases []string `yaml:"releases"` + RuntimeConfigs []string `yaml:"runtime_configs"` + RequiresKubernetes bool `yaml:"requires_kubernetes"` + CompatibleKubernetesDistributions []ProductVersion `yaml:"compatible_kubernetes_distributions"` +} + +type StemcellCriteria struct { + Os string `yaml:"os"` + Version string `yaml:"version"` +} + +type ProductVersion struct { + Name string `yaml:"name"` + Version string `yaml:"version"` +} diff --git a/internal/carvel/models/package_install.go b/internal/carvel/models/package_install.go new file mode 100644 index 000000000..e3e9ef348 --- /dev/null +++ b/internal/carvel/models/package_install.go @@ -0,0 +1,8 @@ +package models + +type PackageInstall struct { + Name string `yaml:"name"` + PackageName string `yaml:"packageName"` + PackageVersion string `yaml:"packageVersion"` + Values interface{} `yaml:"values,omitempty"` +} diff --git a/internal/carvel/models/package_install_props.go b/internal/carvel/models/package_install_props.go new file mode 100644 index 000000000..88895fb93 --- /dev/null +++ b/internal/carvel/models/package_install_props.go @@ -0,0 +1,7 @@ +package models + +type PackageInstallProps struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Values interface{} `yaml:"values,omitempty"` +} diff --git a/internal/carvel/models/runtime_config.go b/internal/carvel/models/runtime_config.go new file mode 100644 index 000000000..c19a68fd8 --- /dev/null +++ b/internal/carvel/models/runtime_config.go @@ -0,0 +1,22 @@ +package models + +type RuntimeConfigOuter struct { + Name string `yaml:"name"` + RuntimeConfig string `yaml:"runtime_config"` +} + +type RuntimeConfigInner struct { + Releases []string `yaml:"releases"` + Addons []Addon `yaml:"addons"` +} + +type Addon struct { + Name string `yaml:"name"` + Include Inclusion `yaml:"include"` + Jobs []Job `yaml:"jobs"` +} + +type Inclusion struct { + Deployments []string `yaml:"deployments"` + Jobs []Job `yaml:"jobs"` +} diff --git a/internal/carvel/testdata/sample-tile/.gitignore b/internal/carvel/testdata/sample-tile/.gitignore new file mode 100644 index 000000000..bd4fa3a61 --- /dev/null +++ b/internal/carvel/testdata/sample-tile/.gitignore @@ -0,0 +1,3 @@ +.boshrelease +.ezbake + diff --git a/internal/carvel/testdata/sample-tile/README.md b/internal/carvel/testdata/sample-tile/README.md new file mode 100644 index 000000000..1501223a8 --- /dev/null +++ b/internal/carvel/testdata/sample-tile/README.md @@ -0,0 +1,4 @@ +# Sample Kubernetes Tile +This is a test "unbaked" kubernetes tile. It includes a fake imgpkg bundle tarball that doesn't actually contain a packagereop. + +It is only for integration testing purposes and cannot be deployed. diff --git a/internal/carvel/testdata/sample-tile/base.yml b/internal/carvel/testdata/sample-tile/base.yml new file mode 100644 index 000000000..d91f08bba --- /dev/null +++ b/internal/carvel/testdata/sample-tile/base.yml @@ -0,0 +1,19 @@ +name: k8s-tile-test +label: "test tile" +icon_image: $( icon ) +metadata_version: "3.2.0" +minimum_version_for_upgrade: 0.0.0 +product_version: $( version ) +rank: 1 +serial: false +property_blueprints: +- $( property "database_name" ) +- $( property "admin_password" ) +form_types: +- $( form "db_props" ) +variables: [] +package_installs: +- $( package "test-install" ) +compatible_kubernetes_distributions: +- name: k0s + version: '>0.0.0' diff --git a/internal/carvel/testdata/sample-tile/bundle.tar b/internal/carvel/testdata/sample-tile/bundle.tar new file mode 100644 index 0000000000000000000000000000000000000000..6c4cdbde9ad04fe00f3e8afefd8f55cac6fae095 GIT binary patch literal 2048 zcmeH@K?=k$2t~7=Q+R^%Z5(@^ZbHdMXj7QuPukE$4iM-_0l5JrZ2Zl6b4_Zn@>wwGhGoS#XrbuFd7)w%Xadk&y( W_F|LFJ}-{v(VT!2Z~{(X5P?4%x*2Z( literal 0 HcmV?d00001 diff --git a/internal/carvel/testdata/sample-tile/forms/.gitkeep b/internal/carvel/testdata/sample-tile/forms/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/carvel/testdata/sample-tile/forms/db_props.yml b/internal/carvel/testdata/sample-tile/forms/db_props.yml new file mode 100644 index 000000000..2d41e9aa7 --- /dev/null +++ b/internal/carvel/testdata/sample-tile/forms/db_props.yml @@ -0,0 +1,8 @@ +--- +name: "db_props" +label: "Postgres Database Properties" +description: "Postgres Database Properties" +markdown: 'Postgres Database Properties' +property_inputs: +- reference: ".properties.database_name" + label: "Database name" diff --git a/internal/carvel/testdata/sample-tile/icon.png b/internal/carvel/testdata/sample-tile/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..d556b41cc5f57115791a330d4416d81ae2a3b342 GIT binary patch literal 1185 zcmV;S1YY}zP)C0000pP)t-sM{rC4 z00000000000000000000000000000000000000000000000000000000002nK`tr) z000nlQchCqo|T>Y#fG%8C53JXUT-3zec$>jZ9uKZZEo0KR*RNmGJjDZ`YWnzM6YNscoy-G}YtLLPzWy>XWkC_t*WUsc zK-fS8b>>Ny3&19fm+H)u{KOWYAZp&(Uvht?*hxTa`Fz+8hd!|uUGg+Qa?!8t`}t3+ zod$_rUu}VxV;-EBeBD;oX{fAq3wMHc2V13k`n4eO5QwP?z>fytYXD*u)gY;H(f z43`FgE9Do!FMwDxjVs00e8Gh>pCvFC$j@-kr>oc3`=IdlK~j&C(DDrU2)f+Ir+SuJ zg)k>EBcN#E{Nz-;k;~@LZpZP(* zKS4a%joxO$YxoF{164Q*mnie2rfvj`0EtQbZoc4OXR&>JNXBK*w|MHWdvy1Z25vx2 zc|K}o;u`@##=7#lD1b-{pwEw>@vBx<21y~#mGB6lX~I~##-Ja1z^61xb^!Sg1!HQ#-6}L5{XINZZ{+1 z3N;9z{j9l?UF0^0qth6nQV2`S4-K%)k4Av#^52_S^Q9{W%gso` z8EAqTONbVXr8-+&wYb8PrpCG&ap!Jr$@KM!v!RWcq&8{Wx(Yz+LNy~}w+k**GmxZp zK!9RLTGh8UdS!r+m^h&a_dnnsOLv#;q9XP1qBV6&PrPVdhydPxzd*coLhdwb+4VIU zfP1eIh<6a_;{Z>2hs5aKjSRqdlW3LDJ4Qtr79=wivoOx@phrDpL8c?86MX~^zQ865 zOf~V;$g)&eVj_h-)x@32VNW%2r;^t9MI&g@8WzRXA+B_4vD~d&Hj`fNi@%vb>f&?% z)oYPM;9S>|K)|~$mf4}93H00000NkvXXu0mjfGiM-^ literal 0 HcmV?d00001 diff --git a/internal/carvel/testdata/sample-tile/packageinstalls/test-install.yml b/internal/carvel/testdata/sample-tile/packageinstalls/test-install.yml new file mode 100644 index 000000000..eccf4450a --- /dev/null +++ b/internal/carvel/testdata/sample-tile/packageinstalls/test-install.yml @@ -0,0 +1,6 @@ +name: test-install +packageName: something-test.tanzu.vmware.com +packageVersion: "0.1.5" +values: + db_name: (( .properties.database_name.value )) + password: (( .properties.admin_password.value )) diff --git a/internal/carvel/testdata/sample-tile/properties/.gitkeep b/internal/carvel/testdata/sample-tile/properties/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/carvel/testdata/sample-tile/properties/properties.yml b/internal/carvel/testdata/sample-tile/properties/properties.yml new file mode 100644 index 000000000..de36e0aa6 --- /dev/null +++ b/internal/carvel/testdata/sample-tile/properties/properties.yml @@ -0,0 +1,9 @@ +- name: admin_password + type: secret + configurable: false + optional: false +- name: database_name + type: string + configurable: true + optional: false + default: my-db diff --git a/internal/carvel/testdata/sample-tile/version b/internal/carvel/testdata/sample-tile/version new file mode 100644 index 000000000..17e51c385 --- /dev/null +++ b/internal/carvel/testdata/sample-tile/version @@ -0,0 +1 @@ +0.1.1 diff --git a/internal/commands/carvel.go b/internal/commands/carvel.go new file mode 100644 index 000000000..ae98d040d --- /dev/null +++ b/internal/commands/carvel.go @@ -0,0 +1,160 @@ +package commands + +import ( + "fmt" + "log" + "sort" + "strings" + + "github.com/pivotal-cf/jhanda" +) + +// Carvel is a command group for Carvel/Kubernetes tile operations +type Carvel struct { + outLogger *log.Logger + errLogger *log.Logger + commands jhanda.CommandSet +} + +func NewCarvel(outLogger, errLogger *log.Logger) Carvel { + c := Carvel{ + outLogger: outLogger, + errLogger: errLogger, + commands: jhanda.CommandSet{}, + } + + // Register subcommands + c.commands["bake"] = NewCarvelBake(outLogger, errLogger) + + return c +} + +func (c Carvel) Execute(args []string) error { + if len(args) == 0 { + return c.printHelp() + } + + subcommand := args[0] + subargs := args[1:] + + if subcommand == "help" || subcommand == "-h" || subcommand == "--help" { + if len(subargs) > 0 { + return c.printSubcommandHelp(subargs[0]) + } + return c.printHelp() + } + + // Check if subargs contains help flags - this handles cases like + // "kiln carvel bake --help" where --help was passed through from main + for _, arg := range subargs { + if arg == "-h" || arg == "--help" || arg == "help" { + return c.printSubcommandHelp(subcommand) + } + } + + return c.commands.Execute(subcommand, subargs) +} + +func (c Carvel) Usage() jhanda.Usage { + // Build subcommand list for the description + var subcommandList strings.Builder + subcommandList.WriteString("Commands for working with Carvel/Kubernetes tiles.\n\n") + subcommandList.WriteString("Subcommands:\n") + + var names []string + var length int + for name := range c.commands { + names = append(names, name) + if len(name) > length { + length = len(name) + } + } + sort.Strings(names) + + for _, name := range names { + cmd := c.commands[name] + paddedName := c.pad(name, " ", length) + subcommandList.WriteString(fmt.Sprintf(" %s %s\n", paddedName, cmd.Usage().ShortDescription)) + } + subcommandList.WriteString("\nUse 'kiln carvel help ' for more information about a subcommand.") + + return jhanda.Usage{ + Description: subcommandList.String(), + ShortDescription: "commands for Carvel/Kubernetes tiles", + Flags: nil, + } +} + +func (c Carvel) printHelp() error { + var ( + length int + names []string + ) + + for name := range c.commands { + names = append(names, name) + if len(name) > length { + length = len(name) + } + } + + sort.Strings(names) + + fmt.Println("kiln carvel - commands for Carvel/Kubernetes tiles") + fmt.Println() + fmt.Println("Usage: kiln carvel []") + fmt.Println() + fmt.Println("Subcommands:") + for _, name := range names { + cmd := c.commands[name] + paddedName := c.pad(name, " ", length) + fmt.Printf(" %s %s\n", paddedName, cmd.Usage().ShortDescription) + } + fmt.Println() + fmt.Println("Use 'kiln carvel help ' for more information about a subcommand.") + + return nil +} + +func (c Carvel) printSubcommandHelp(subcommand string) error { + cmd, ok := c.commands[subcommand] + if !ok { + return fmt.Errorf("unknown subcommand: %s", subcommand) + } + + usage := cmd.Usage() + fmt.Printf("kiln carvel %s - %s\n", subcommand, usage.ShortDescription) + fmt.Println() + fmt.Println(usage.Description) + fmt.Println() + fmt.Printf("Usage: kiln carvel %s []\n", subcommand) + + if usage.Flags != nil { + flagUsage, err := jhanda.PrintUsage(usage.Flags) + if err != nil { + return err + } + + flagList := strings.Split(flagUsage, "\n") + if len(flagList) > 0 { + fmt.Println() + fmt.Println("Arguments:") + for _, flag := range flagList { + if flag != "" { + fmt.Printf(" %s\n", flag) + } + } + } + } + + return nil +} + +func (c Carvel) pad(str, pad string, length int) string { + for { + str += pad + if len(str) > length { + return str[0:length] + } + } +} diff --git a/internal/commands/carvel_bake.go b/internal/commands/carvel_bake.go new file mode 100644 index 000000000..8799106d4 --- /dev/null +++ b/internal/commands/carvel_bake.go @@ -0,0 +1,87 @@ +package commands + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/pivotal-cf/jhanda" + "github.com/pivotal-cf/kiln/internal/carvel" +) + +type CarvelBake struct { + outLogger *log.Logger + errLogger *log.Logger + Options CarvelBakeOptions +} + +type CarvelBakeOptions struct { + SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` + OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` +} + +func NewCarvelBake(outLogger, errLogger *log.Logger) CarvelBake { + return CarvelBake{ + outLogger: outLogger, + errLogger: errLogger, + } +} + +func (c CarvelBake) Execute(args []string) error { + _, err := jhanda.Parse(&c.Options, args) + if err != nil { + return err + } + + sourcePath := c.Options.SourceDirectory + if sourcePath == "" { + sourcePath, err = os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %w", err) + } + } else { + sourcePath, err = filepath.Abs(sourcePath) + if err != nil { + return fmt.Errorf("failed to resolve source directory: %w", err) + } + } + + targetPath, err := filepath.Abs(c.Options.OutputFile) + if err != nil { + return fmt.Errorf("failed to resolve output file path: %w", err) + } + + baker := carvel.NewBaker() + if c.Options.Verbose { + baker.SetWriter(os.Stdout) + } + + c.outLogger.Printf("Baking Carvel tile from %s into %s/.ezbake", sourcePath, sourcePath) + err = baker.Bake(sourcePath) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile: %w", err) + } + + v, err := baker.GetVersion() + if err != nil { + return fmt.Errorf("failed to get tile version: %w", err) + } + + err = baker.KilnBake(targetPath) + if err != nil { + return fmt.Errorf("failed to bake tile: %w", err) + } + + c.outLogger.Printf("Baked %s version %s to %s", baker.GetName(), v, targetPath) + return nil +} + +func (c CarvelBake) Usage() jhanda.Usage { + return jhanda.Usage{ + Description: "Bakes a Carvel/Kubernetes tile into a .pivotal file. This command transforms a Kubernetes tile (using imgpkg bundles and Carvel packages) into a BOSH-compatible format, then bakes it into a .pivotal file that can be consumed by Operations Manager.", + ShortDescription: "bakes a Carvel/Kubernetes tile", + Flags: c.Options, + } +} diff --git a/internal/commands/carvel_bake_test.go b/internal/commands/carvel_bake_test.go new file mode 100644 index 000000000..cbe3ea6a8 --- /dev/null +++ b/internal/commands/carvel_bake_test.go @@ -0,0 +1,106 @@ +package commands_test + +import ( + "log" + "os" + "os/exec" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/commands" +) + +func boshInstalled() bool { + _, err := exec.LookPath("bosh") + return err == nil +} + +func kilnInstalled() bool { + _, err := exec.LookPath("kiln") + return err == nil +} + +var _ = Describe("CarvelBake", func() { + var ( + outLogger *log.Logger + errLogger *log.Logger + command commands.CarvelBake + ) + + BeforeEach(func() { + outLogger = log.New(GinkgoWriter, "", 0) + errLogger = log.New(GinkgoWriter, "", 0) + command = commands.NewCarvelBake(outLogger, errLogger) + }) + + Describe("Usage", func() { + It("returns usage information", func() { + usage := command.Usage() + Expect(usage.ShortDescription).To(Equal("bakes a Carvel/Kubernetes tile")) + Expect(usage.Description).To(ContainSubstring("Carvel/Kubernetes tile")) + }) + }) + + Describe("Execute", func() { + var ( + inputPath string + outputPath string + ) + + BeforeEach(func() { + var err error + inputPath, err = os.MkdirTemp("", "testinput-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + // create an initial git commit in the input directory + cmds := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range cmds { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + outputPath = filepath.Join(inputPath, "output.pivotal") + }) + + AfterEach(func() { + if inputPath != "" { + os.RemoveAll(filepath.Dir(inputPath)) + } + }) + + When("required arguments are missing", func() { + It("returns an error when output-file is not provided", func() { + err := command.Execute([]string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("output-file")) + }) + }) + + When("valid arguments are provided", func() { + It("successfully bakes a tile", func() { + if !boshInstalled() { + Skip("bosh CLI not installed - skipping integration test") + } + if !kilnInstalled() { + Skip("kiln CLI not installed - skipping integration test") + } + err := command.Execute([]string{ + "--source-directory", inputPath, + "--output-file", outputPath, + "--verbose", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(outputPath).To(BeAnExistingFile()) + }) + }) + }) +}) diff --git a/main.go b/main.go index fb6c5fe58..6cf14562d 100644 --- a/main.go +++ b/main.go @@ -101,6 +101,8 @@ func main() { log.Fatal(err) } + commandSet["carvel"] = commands.NewCarvel(outLogger, errLogger) + err = commandSet.Execute(command, args) if err != nil { log.Fatal(err) From 44d4e841642ca1699e136f1052eb74e89c855b19 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Wed, 21 Jan 2026 12:52:37 -0600 Subject: [PATCH 02/18] Add an integration test --- .../acceptance/carvel/carvel_bake_test.go | 200 ++++++++++++++++++ .../carvel/fixtures/sample-tile/.gitignore | 3 + .../carvel/fixtures/sample-tile/README.md | 4 + .../carvel/fixtures/sample-tile/base.yml | 19 ++ .../carvel/fixtures/sample-tile/bundle.tar | Bin 0 -> 2048 bytes .../fixtures/sample-tile/forms/.gitkeep | 0 .../fixtures/sample-tile/forms/db_props.yml | 8 + .../carvel/fixtures/sample-tile/icon.png | Bin 0 -> 1185 bytes .../packageinstalls/test-install.yml | 6 + .../fixtures/sample-tile/properties/.gitkeep | 0 .../sample-tile/properties/properties.yml | 9 + .../carvel/fixtures/sample-tile/version | 1 + 12 files changed, 250 insertions(+) create mode 100644 internal/acceptance/carvel/carvel_bake_test.go create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/.gitignore create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/README.md create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/base.yml create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/bundle.tar create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/forms/.gitkeep create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/forms/db_props.yml create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/icon.png create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/packageinstalls/test-install.yml create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/properties/.gitkeep create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/properties/properties.yml create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/version diff --git a/internal/acceptance/carvel/carvel_bake_test.go b/internal/acceptance/carvel/carvel_bake_test.go new file mode 100644 index 000000000..9c105fae7 --- /dev/null +++ b/internal/acceptance/carvel/carvel_bake_test.go @@ -0,0 +1,200 @@ +package acceptance_test + +import ( + "archive/zip" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" + "github.com/onsi/gomega/gexec" +) + +var pathToMain string + +func TestCarvelAcceptance(t *testing.T) { + SetDefaultEventuallyTimeout(time.Minute) + RegisterFailHandler(Fail) + RunSpecs(t, "carvel acceptance") +} + +var _ = BeforeSuite(func() { + if _, err := exec.LookPath("bosh"); err != nil { + Skip("bosh CLI not installed - skipping carvel acceptance tests") + } + + var err error + pathToMain, err = gexec.Build("github.com/pivotal-cf/kiln") + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + gexec.CleanupBuildArtifacts() +}) + +var _ = Describe("carvel bake command", func() { + var ( + outputFile string + tmpDir string + inputPath string + commandWithArgs []string + ) + + const ( + sampleTileFixture = "fixtures/sample-tile" + ) + + BeforeEach(func() { + var err error + tmpDir, err = os.MkdirTemp("", "kiln-carvel-test") + Expect(err).NotTo(HaveOccurred()) + + // Copy the sample-tile fixture to a temp directory + inputPath = filepath.Join(tmpDir, "tile") + err = os.CopyFS(inputPath, os.DirFS(sampleTileFixture)) + Expect(err).NotTo(HaveOccurred()) + + // Initialize git repo (required by kiln for metadata) + gitCommands := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "config", "user.email", "test@test.com"), + exec.Command("git", "config", "user.name", "Test"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range gitCommands { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + outputFile = filepath.Join(tmpDir, "k8s-tile-test-0.0.1.pivotal") + + commandWithArgs = []string{ + "carvel", "bake", + "--source-directory", inputPath, + "--output-file", outputFile, + } + }) + + AfterEach(func() { + _ = os.RemoveAll(tmpDir) + }) + + It("generates a tile with the correct structure", func() { + commandWithArgs = append(commandWithArgs, "--verbose") + + command := exec.Command(pathToMain, commandWithArgs...) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(session, "60s").Should(gexec.Exit(0)) + + archive, err := os.Open(outputFile) + Expect(err).NotTo(HaveOccurred()) + defer archive.Close() + + archiveInfo, err := archive.Stat() + Expect(err).NotTo(HaveOccurred()) + + bakedTile, err := zip.NewReader(archive, archiveInfo.Size()) + Expect(err).NotTo(HaveOccurred()) + + // Verify metadata exists + _, err = bakedTile.Open("metadata/metadata.yml") + Expect(err).NotTo(HaveOccurred()) + + // Verify releases directory contains a tgz + var foundRelease bool + for _, f := range bakedTile.File { + if filepath.Dir(f.Name) == "releases" && filepath.Ext(f.Name) == ".tgz" { + foundRelease = true + break + } + } + Expect(foundRelease).To(BeTrue(), "releases/*.tgz should be in the tile") + + // Verify migrations directory exists (even if empty) + var foundMigrations bool + for _, f := range bakedTile.File { + if filepath.Dir(f.Name) == "migrations" || f.Name == "migrations/v1/" { + foundMigrations = true + break + } + } + Expect(foundMigrations).To(BeTrue(), "migrations directory should be in the tile") + + Eventually(session.Out).Should(gbytes.Say("Baked")) + Eventually(session.Out).Should(gbytes.Say("k8s-tile-test")) + }) + + It("produces a valid zip archive", func() { + command := exec.Command(pathToMain, commandWithArgs...) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(session, "60s").Should(gexec.Exit(0)) + + // Verify using unzip -t + verifyCmd := exec.Command("unzip", "-t", outputFile) + verifySession, err := gexec.Start(verifyCmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(verifySession, "10s").Should(gexec.Exit(0)) + Eventually(verifySession.Out).Should(gbytes.Say("No errors detected")) + }) + + Context("failure cases", func() { + Context("when the output-file flag is not provided", func() { + It("prints an error and exits 1", func() { + command := exec.Command(pathToMain, "carvel", "bake", + "--source-directory", inputPath, + ) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(session).Should(gexec.Exit(1)) + Eventually(session.Err).Should(gbytes.Say("output-file")) + }) + }) + + Context("when the source directory does not exist", func() { + It("prints an error and exits 1", func() { + command := exec.Command(pathToMain, "carvel", "bake", + "--source-directory", "/non/existent/path", + "--output-file", outputFile, + ) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(session).Should(gexec.Exit(1)) + }) + }) + + Context("when the source directory is missing base.yml", func() { + It("prints an error and exits 1", func() { + emptyDir, err := os.MkdirTemp(tmpDir, "empty-tile") + Expect(err).NotTo(HaveOccurred()) + + command := exec.Command(pathToMain, "carvel", "bake", + "--source-directory", emptyDir, + "--output-file", outputFile, + ) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(session).Should(gexec.Exit(1)) + Eventually(session.Err).Should(gbytes.Say("base.yml")) + }) + }) + }) +}) diff --git a/internal/acceptance/carvel/fixtures/sample-tile/.gitignore b/internal/acceptance/carvel/fixtures/sample-tile/.gitignore new file mode 100644 index 000000000..bd4fa3a61 --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/.gitignore @@ -0,0 +1,3 @@ +.boshrelease +.ezbake + diff --git a/internal/acceptance/carvel/fixtures/sample-tile/README.md b/internal/acceptance/carvel/fixtures/sample-tile/README.md new file mode 100644 index 000000000..1501223a8 --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/README.md @@ -0,0 +1,4 @@ +# Sample Kubernetes Tile +This is a test "unbaked" kubernetes tile. It includes a fake imgpkg bundle tarball that doesn't actually contain a packagereop. + +It is only for integration testing purposes and cannot be deployed. diff --git a/internal/acceptance/carvel/fixtures/sample-tile/base.yml b/internal/acceptance/carvel/fixtures/sample-tile/base.yml new file mode 100644 index 000000000..d91f08bba --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/base.yml @@ -0,0 +1,19 @@ +name: k8s-tile-test +label: "test tile" +icon_image: $( icon ) +metadata_version: "3.2.0" +minimum_version_for_upgrade: 0.0.0 +product_version: $( version ) +rank: 1 +serial: false +property_blueprints: +- $( property "database_name" ) +- $( property "admin_password" ) +form_types: +- $( form "db_props" ) +variables: [] +package_installs: +- $( package "test-install" ) +compatible_kubernetes_distributions: +- name: k0s + version: '>0.0.0' diff --git a/internal/acceptance/carvel/fixtures/sample-tile/bundle.tar b/internal/acceptance/carvel/fixtures/sample-tile/bundle.tar new file mode 100644 index 0000000000000000000000000000000000000000..6c4cdbde9ad04fe00f3e8afefd8f55cac6fae095 GIT binary patch literal 2048 zcmeH@K?=k$2t~7=Q+R^%Z5(@^ZbHdMXj7QuPukE$4iM-_0l5JrZ2Zl6b4_Zn@>wwGhGoS#XrbuFd7)w%Xadk&y( W_F|LFJ}-{v(VT!2Z~{(X5P?4%x*2Z( literal 0 HcmV?d00001 diff --git a/internal/acceptance/carvel/fixtures/sample-tile/forms/.gitkeep b/internal/acceptance/carvel/fixtures/sample-tile/forms/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/acceptance/carvel/fixtures/sample-tile/forms/db_props.yml b/internal/acceptance/carvel/fixtures/sample-tile/forms/db_props.yml new file mode 100644 index 000000000..2d41e9aa7 --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/forms/db_props.yml @@ -0,0 +1,8 @@ +--- +name: "db_props" +label: "Postgres Database Properties" +description: "Postgres Database Properties" +markdown: 'Postgres Database Properties' +property_inputs: +- reference: ".properties.database_name" + label: "Database name" diff --git a/internal/acceptance/carvel/fixtures/sample-tile/icon.png b/internal/acceptance/carvel/fixtures/sample-tile/icon.png new file mode 100644 index 0000000000000000000000000000000000000000..d556b41cc5f57115791a330d4416d81ae2a3b342 GIT binary patch literal 1185 zcmV;S1YY}zP)C0000pP)t-sM{rC4 z00000000000000000000000000000000000000000000000000000000002nK`tr) z000nlQchCqo|T>Y#fG%8C53JXUT-3zec$>jZ9uKZZEo0KR*RNmGJjDZ`YWnzM6YNscoy-G}YtLLPzWy>XWkC_t*WUsc zK-fS8b>>Ny3&19fm+H)u{KOWYAZp&(Uvht?*hxTa`Fz+8hd!|uUGg+Qa?!8t`}t3+ zod$_rUu}VxV;-EBeBD;oX{fAq3wMHc2V13k`n4eO5QwP?z>fytYXD*u)gY;H(f z43`FgE9Do!FMwDxjVs00e8Gh>pCvFC$j@-kr>oc3`=IdlK~j&C(DDrU2)f+Ir+SuJ zg)k>EBcN#E{Nz-;k;~@LZpZP(* zKS4a%joxO$YxoF{164Q*mnie2rfvj`0EtQbZoc4OXR&>JNXBK*w|MHWdvy1Z25vx2 zc|K}o;u`@##=7#lD1b-{pwEw>@vBx<21y~#mGB6lX~I~##-Ja1z^61xb^!Sg1!HQ#-6}L5{XINZZ{+1 z3N;9z{j9l?UF0^0qth6nQV2`S4-K%)k4Av#^52_S^Q9{W%gso` z8EAqTONbVXr8-+&wYb8PrpCG&ap!Jr$@KM!v!RWcq&8{Wx(Yz+LNy~}w+k**GmxZp zK!9RLTGh8UdS!r+m^h&a_dnnsOLv#;q9XP1qBV6&PrPVdhydPxzd*coLhdwb+4VIU zfP1eIh<6a_;{Z>2hs5aKjSRqdlW3LDJ4Qtr79=wivoOx@phrDpL8c?86MX~^zQ865 zOf~V;$g)&eVj_h-)x@32VNW%2r;^t9MI&g@8WzRXA+B_4vD~d&Hj`fNi@%vb>f&?% z)oYPM;9S>|K)|~$mf4}93H00000NkvXXu0mjfGiM-^ literal 0 HcmV?d00001 diff --git a/internal/acceptance/carvel/fixtures/sample-tile/packageinstalls/test-install.yml b/internal/acceptance/carvel/fixtures/sample-tile/packageinstalls/test-install.yml new file mode 100644 index 000000000..eccf4450a --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/packageinstalls/test-install.yml @@ -0,0 +1,6 @@ +name: test-install +packageName: something-test.tanzu.vmware.com +packageVersion: "0.1.5" +values: + db_name: (( .properties.database_name.value )) + password: (( .properties.admin_password.value )) diff --git a/internal/acceptance/carvel/fixtures/sample-tile/properties/.gitkeep b/internal/acceptance/carvel/fixtures/sample-tile/properties/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/acceptance/carvel/fixtures/sample-tile/properties/properties.yml b/internal/acceptance/carvel/fixtures/sample-tile/properties/properties.yml new file mode 100644 index 000000000..de36e0aa6 --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/properties/properties.yml @@ -0,0 +1,9 @@ +- name: admin_password + type: secret + configurable: false + optional: false +- name: database_name + type: string + configurable: true + optional: false + default: my-db diff --git a/internal/acceptance/carvel/fixtures/sample-tile/version b/internal/acceptance/carvel/fixtures/sample-tile/version new file mode 100644 index 000000000..17e51c385 --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/version @@ -0,0 +1 @@ +0.1.1 From 5e76c663cf1f2c577b99af3e89ea51a9e9a88a95 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Wed, 21 Jan 2026 13:02:25 -0600 Subject: [PATCH 03/18] Fix linting errors --- internal/acceptance/carvel/carvel_bake_test.go | 2 +- internal/carvel/baker.go | 16 ++++++++++------ internal/carvel/baker_test.go | 12 ++++++------ internal/commands/carvel_bake_test.go | 2 +- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/internal/acceptance/carvel/carvel_bake_test.go b/internal/acceptance/carvel/carvel_bake_test.go index 9c105fae7..21272728b 100644 --- a/internal/acceptance/carvel/carvel_bake_test.go +++ b/internal/acceptance/carvel/carvel_bake_test.go @@ -97,7 +97,7 @@ var _ = Describe("carvel bake command", func() { archive, err := os.Open(outputFile) Expect(err).NotTo(HaveOccurred()) - defer archive.Close() + defer func() { _ = archive.Close() }() archiveInfo, err := archive.Stat() Expect(err).NotTo(HaveOccurred()) diff --git a/internal/carvel/baker.go b/internal/carvel/baker.go index afb92410a..df6387147 100644 --- a/internal/carvel/baker.go +++ b/internal/carvel/baker.go @@ -123,7 +123,7 @@ func (b *baker) SetWriter(w io.Writer) { } func (b *baker) log(message string) { - fmt.Fprintln(b.writer, message) + _, _ = fmt.Fprintln(b.writer, message) } func (b *baker) generateBoshReleaseDir() error { @@ -224,7 +224,9 @@ packages: values: description: "values.yml contents" ` - os.MkdirAll(path.Join(dirName, "jobs", "package-install", "templates", "packageinstalls", entry), 0755) + if err = os.MkdirAll(path.Join(dirName, "jobs", "package-install", "templates", "packageinstalls", entry), 0755); err != nil { + return err + } templates := map[string]string{ "name.erb": `<%= p("` + entry + `.name") %>`, "version.erb": `<%= p("` + entry + `.version") %>`, @@ -358,9 +360,11 @@ func (b *baker) copyFiles() error { } for _, fn := range []string{"icon.png", "version"} { - info, err := os.Stat(path.Join(b.source, fn)) - if err == nil && !info.IsDir() { - err = copyFileContents(path.Join(b.source, fn), path.Join(b.destination, fn)) + info, statErr := os.Stat(path.Join(b.source, fn)) + if statErr == nil && !info.IsDir() { + if err := copyFileContents(path.Join(b.source, fn), path.Join(b.destination, fn)); err != nil { + return err + } } } @@ -526,7 +530,7 @@ func copyFileContents(src, dst string) (err error) { if err != nil { return } - defer in.Close() + defer func() { _ = in.Close() }() out, err := os.Create(dst) if err != nil { return diff --git a/internal/carvel/baker_test.go b/internal/carvel/baker_test.go index 6a52238e6..f11ff7cbb 100644 --- a/internal/carvel/baker_test.go +++ b/internal/carvel/baker_test.go @@ -58,12 +58,12 @@ var _ = Describe("Carvel Baker", func() { subject = NewBaker() subject.SetWriter(GinkgoWriter) }) - AfterEach(func() { - // Clean up the temp directory - if inputPath != "" { - os.RemoveAll(filepath.Dir(inputPath)) - } - }) + AfterEach(func() { + // Clean up the temp directory + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + }) JustBeforeEach(func() { err = subject.Bake(inputPath) }) diff --git a/internal/commands/carvel_bake_test.go b/internal/commands/carvel_bake_test.go index cbe3ea6a8..e16e78673 100644 --- a/internal/commands/carvel_bake_test.go +++ b/internal/commands/carvel_bake_test.go @@ -73,7 +73,7 @@ var _ = Describe("CarvelBake", func() { AfterEach(func() { if inputPath != "" { - os.RemoveAll(filepath.Dir(inputPath)) + _ = os.RemoveAll(filepath.Dir(inputPath)) } }) From a04069f1c07c5cab9cf9beceeda2838d1f3da2b1 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Thu, 5 Mar 2026 18:52:42 -0600 Subject: [PATCH 04/18] Port ezbake VKS runtime changes to kiln carvel baker Bring in changes from ezbake's yogesh/vks-tile-changes branch that make the carvel baker work with any k8s runtime (VKS, TKR) instead of being tied to the registry release. - Remove separate package-install job; consolidate into registry-data - Generate full K8s manifest templates (ServiceAccount, ClusterRole, ClusterRoleBinding, Secret, PackageInstall) per package install - Add BOSH link consumer (cluster/cluster-info) for namespace resolution - Switch runtime config from registry to tanzu-content release - Inject content-namespace from BOSH link into values context - Backfill tests for all new behaviors ai-assisted=yes Co-authored-by: Yogesh Katreddy Veera Co-authored-by: Praveen Rewar Made-with: Cursor --- internal/carvel/baker.go | 156 +++++++++++++++++++++++----------- internal/carvel/baker_test.go | 150 +++++++++++++++++++++++++++++++- 2 files changed, 252 insertions(+), 54 deletions(-) diff --git a/internal/carvel/baker.go b/internal/carvel/baker.go index df6387147..5b474f738 100644 --- a/internal/carvel/baker.go +++ b/internal/carvel/baker.go @@ -139,7 +139,6 @@ func (b *baker) generateBoshReleaseDir() error { exec.Command("bosh", "add-blob", "--dir="+dirName, path.Join(b.source, "bundle.tar"), "imgpkg/bundle.tar"), exec.Command("bosh", "generate-package", "--dir="+dirName, "registry-data"), exec.Command("bosh", "generate-job", "--dir="+dirName, "registry-data"), - exec.Command("bosh", "generate-job", "--dir="+dirName, "package-install"), } for _, cmd := range commands { b.log("executing " + cmd.String()) @@ -162,12 +161,6 @@ name: registry-data dependencies: [] files: - imgpkg/bundle.tar -`, - "jobs/registry-data/spec": `--- -name: registry-data -templates: {} -packages: -- registry-data `, } for outpath, contents := range fileContents { @@ -177,9 +170,10 @@ packages: } } - jobTemplates := "" - jobProperties := "" - // we need one PackageInstall for each entry in the metadata. + registryDataTemplates := "" + registryDataProperties := "" + + // we need one PackageInstall YAML manifest for each entry in the metadata. for _, entry := range b.metadata.PackageInstalls { entry = strings.Trim(entry, "$() ") entry = strings.TrimPrefix(entry, "package") @@ -211,45 +205,47 @@ packages: b.log("found " + pi.Name + " at " + match) } - // accumulate templates - jobTemplates += fmt.Sprintf(" packageinstalls/%s/name.erb: packageinstalls/%s/name\n", entry, entry) - jobTemplates += fmt.Sprintf(" packageinstalls/%s/version.erb: packageinstalls/%s/version\n", entry, entry) - jobTemplates += fmt.Sprintf(" packageinstalls/%s/values.yml.erb: packageinstalls/%s/values.yml\n", entry, entry) - // accumulate properties - jobProperties += " " + entry + ":\n" - jobProperties += ` name: + registryDataTemplates += fmt.Sprintf(" packageinstalls/%s.yml.erb: packageinstalls/%s.yml\n", entry, entry) + + registryDataProperties += " " + entry + ":\n" + registryDataProperties += ` name: description: "package name" version: description: "package version" values: description: "values.yml contents" ` - if err = os.MkdirAll(path.Join(dirName, "jobs", "package-install", "templates", "packageinstalls", entry), 0755); err != nil { + + if err = os.MkdirAll(path.Join(dirName, "jobs", "registry-data", "templates", "packageinstalls"), 0755); err != nil { return err } - templates := map[string]string{ - "name.erb": `<%= p("` + entry + `.name") %>`, - "version.erb": `<%= p("` + entry + `.version") %>`, - "values.yml.erb": `<% require 'yaml' %>` + "\n" + `<%= p("` + entry + `.values").is_a?(String) ? p("` + entry + `.values") : YAML.dump(p("` + entry + `.values")) %>`, - } - for fileName, contents := range templates { - err = os.WriteFile(path.Join(dirName, "jobs", "package-install", "templates", "packageinstalls", entry, fileName), []byte(contents), 0644) - if err != nil { - return err - } + + manifestTemplate := generateManifestTemplate(entry) + + err = os.WriteFile( + path.Join(dirName, "jobs", "registry-data", "templates", "packageinstalls", entry+".yml.erb"), + []byte(manifestTemplate), + 0644, + ) + if err != nil { + return err } } - // now that we've collected all the templates and properties, write out the spec file for the - // package-install job. - contents := `--- -name: package-install + registryDataSpec := `--- +name: registry-data templates: -` + jobTemplates + - `packages: [] +` + registryDataTemplates + + `packages: +- registry-data +consumes: +- name: cluster + type: cluster-info + optional: true properties: -` + jobProperties - err = os.WriteFile(path.Join(dirName, "jobs", "package-install", "spec"), []byte(contents), 0644) +` + registryDataProperties + + err = os.WriteFile(path.Join(dirName, "jobs", "registry-data", "spec"), []byte(registryDataSpec), 0644) if err != nil { return err } @@ -257,6 +253,72 @@ properties: return nil } +func generateManifestTemplate(entry string) string { + return `--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: <%= p("` + entry + `.name") %>-sa + namespace: <%= link("cluster").p("content-namespace") rescue "default" %> +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: <%= p("` + entry + `.name") %>-sa-cluster-role +rules: +- apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: <%= p("` + entry + `.name") %>-sa-cluster-role-binding +subjects: +- kind: ServiceAccount + name: <%= p("` + entry + `.name") %>-sa + namespace: <%= link("cluster").p("content-namespace") rescue "default" %> +roleRef: + kind: ClusterRole + name: <%= p("` + entry + `.name") %>-sa-cluster-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Secret +metadata: + name: <%= p("` + entry + `.name") %>-values + namespace: <%= link("cluster").p("content-namespace") rescue "default" %> +type: Opaque +stringData: + values.yaml: | +<% require 'yaml' %> +<% + values = p("` + entry + `.values") + values = YAML.load(values) if values.is_a?(String) + # Inject namespace from BOSH link into context + if values.is_a?(Hash) && values["context"].is_a?(Hash) + values["context"]["namespace"] = link("cluster").p("content-namespace") rescue "default" + end +%> +<%= YAML.dump(values).split("\n").map { |line| " " + line }.join("\n") %> +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: <%= p("` + entry + `.name") %> + namespace: <%= link("cluster").p("content-namespace") rescue "default" %> +spec: + serviceAccountName: <%= p("` + entry + `.name") %>-sa + packageRef: + refName: <%= p("` + entry + `.name") %> + versionSelection: + constraints: <%= p("` + entry + `.version") %> + values: + - secretRef: + name: <%= p("` + entry + `.name") %>-values +` +} + func (b *baker) generateOutputTile() error { // first clean out any previous tile directory // Note: this directory should only ever contain generated files, which we are about to regenerate. @@ -371,21 +433,14 @@ func (b *baker) copyFiles() error { return nil } -// generateRuntimeConfigs creates a runtime config that colocates registry-data and package-install jobs onto the -// registry VM (or whatever instance has the corresponding errands that will ingest the data) func (b *baker) generateRuntimeConfigs() error { err := os.MkdirAll(path.Join(b.destination, "runtime_configs"), 0755) if err != nil { return err } - registryDataJob := models.Job{ - Name: "registry-data", - Release: b.metadata.Name, - } + registryDataProps := map[string]models.PackageInstallProps{} - // create the "package-install" job - props := map[string]models.PackageInstallProps{} // we need one PackageInstall for each entry in the metadata. for _, entry := range b.metadata.PackageInstalls { entry = strings.Trim(entry, "$() ") @@ -415,7 +470,7 @@ func (b *baker) generateRuntimeConfigs() error { } found = true - props[entry] = models.PackageInstallProps{ + registryDataProps[entry] = models.PackageInstallProps{ Name: pi.PackageName, Version: pi.PackageVersion, Values: pi.Values, @@ -426,10 +481,10 @@ func (b *baker) generateRuntimeConfigs() error { } } - packageInstallJob := models.Job{ - Name: "package-install", + registryDataJob := models.Job{ + Name: "registry-data", Release: b.metadata.Name, - Properties: props, + Properties: registryDataProps, } inner := models.RuntimeConfigInner{ @@ -444,13 +499,12 @@ func (b *baker) generateRuntimeConfigs() error { `(( ..` + b.metadata.Name + `.deployment_name ))`, }, Jobs: []models.Job{ - {Name: "apply-packagerepos", Release: "registry"}, - {Name: "install-packages", Release: "registry"}, + {Name: "install-package-repository", Release: "tanzu-content"}, + {Name: "install-packages", Release: "tanzu-content"}, }, }, Jobs: []models.Job{ registryDataJob, - packageInstallJob, }, }, }, diff --git a/internal/carvel/baker_test.go b/internal/carvel/baker_test.go index f11ff7cbb..d585fd732 100644 --- a/internal/carvel/baker_test.go +++ b/internal/carvel/baker_test.go @@ -5,6 +5,7 @@ import ( "os/exec" "path" "path/filepath" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -23,6 +24,57 @@ func kilnInstalled() bool { } var _ = Describe("Carvel Baker", func() { + Context("generateManifestTemplate", func() { + var template string + + BeforeEach(func() { + template = generateManifestTemplate("test-install") + }) + + It("generates a ServiceAccount", func() { + Expect(template).To(ContainSubstring("kind: ServiceAccount")) + Expect(template).To(ContainSubstring(`name: <%= p("test-install.name") %>-sa`)) + }) + + It("generates a ClusterRole (not a namespaced Role)", func() { + Expect(template).To(ContainSubstring("kind: ClusterRole")) + Expect(template).NotTo(ContainSubstring("kind: Role\n")) + Expect(template).To(ContainSubstring(`name: <%= p("test-install.name") %>-sa-cluster-role`)) + }) + + It("generates a ClusterRoleBinding (not a namespaced RoleBinding)", func() { + Expect(template).To(ContainSubstring("kind: ClusterRoleBinding")) + Expect(template).NotTo(ContainSubstring("kind: RoleBinding\n")) + Expect(template).To(ContainSubstring(`name: <%= p("test-install.name") %>-sa-cluster-role-binding`)) + }) + + It("generates a Secret for values", func() { + Expect(template).To(ContainSubstring("kind: Secret")) + Expect(template).To(ContainSubstring(`name: <%= p("test-install.name") %>-values`)) + Expect(template).To(ContainSubstring("stringData:")) + Expect(template).To(ContainSubstring("values.yaml: |")) + }) + + It("generates a PackageInstall resource", func() { + Expect(template).To(ContainSubstring("kind: PackageInstall")) + Expect(template).To(ContainSubstring("apiVersion: packaging.carvel.dev/v1alpha1")) + Expect(template).To(ContainSubstring(`name: <%= p("test-install.name") %>`)) + Expect(template).To(ContainSubstring(`serviceAccountName: <%= p("test-install.name") %>-sa`)) + }) + + It("uses BOSH link for content-namespace with fallback to default", func() { + Expect(template).To(ContainSubstring(`<%= link("cluster").p("content-namespace") rescue "default" %>`)) + }) + + It("injects content-namespace from BOSH link into values context", func() { + Expect(template).To(ContainSubstring(`values["context"]["namespace"] = link("cluster").p("content-namespace") rescue "default"`)) + }) + + It("handles YAML conversion for string values", func() { + Expect(template).To(ContainSubstring(`values = YAML.load(values) if values.is_a?(String)`)) + }) + }) + Context("Bake", func() { When("the input directory contains k8s tile data", func() { BeforeEach(func() { @@ -31,9 +83,9 @@ var _ = Describe("Carvel Baker", func() { } }) var ( - inputPath, outputPath string - subject Baker - err error + inputPath, outputPath, boshReleasePath string + subject Baker + err error ) BeforeEach(func() { var err error @@ -41,6 +93,7 @@ var _ = Describe("Carvel Baker", func() { Expect(err).NotTo(HaveOccurred()) inputPath += "/tile" outputPath = path.Join(inputPath, ".ezbake") + boshReleasePath = path.Join(inputPath, ".boshrelease") err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) Expect(err).NotTo(HaveOccurred()) // create an initial git commit in the input directory @@ -115,6 +168,74 @@ var _ = Describe("Carvel Baker", func() { It("Generates a bosh release tarball", func() { Expect(filepath.Join(outputPath, "releases", "k8s-tile-test-0.1.1.tgz")).To(BeAnExistingFile()) }) + It("does not generate a separate package-install job", func() { + Expect(filepath.Join(boshReleasePath, "jobs", "package-install")).NotTo(BeADirectory()) + }) + It("generates manifest templates under registry-data job", func() { + templatePath := filepath.Join(boshReleasePath, "jobs", "registry-data", "templates", "packageinstalls", "test-install.yml.erb") + Expect(templatePath).To(BeAnExistingFile()) + + contents, err := os.ReadFile(templatePath) + Expect(err).NotTo(HaveOccurred()) + templateStr := string(contents) + + Expect(templateStr).To(ContainSubstring("kind: ServiceAccount")) + Expect(templateStr).To(ContainSubstring("kind: ClusterRole")) + Expect(templateStr).To(ContainSubstring("kind: ClusterRoleBinding")) + Expect(templateStr).To(ContainSubstring("kind: Secret")) + Expect(templateStr).To(ContainSubstring("kind: PackageInstall")) + Expect(templateStr).To(ContainSubstring(`link("cluster").p("content-namespace")`)) + }) + It("generates registry-data job spec with BOSH link consumer and templates", func() { + specPath := filepath.Join(boshReleasePath, "jobs", "registry-data", "spec") + Expect(specPath).To(BeAnExistingFile()) + + contents, err := os.ReadFile(specPath) + Expect(err).NotTo(HaveOccurred()) + specStr := string(contents) + + Expect(specStr).To(ContainSubstring("name: registry-data")) + Expect(specStr).To(ContainSubstring("packageinstalls/test-install.yml.erb: packageinstalls/test-install.yml")) + Expect(specStr).To(ContainSubstring("packages:\n- registry-data")) + Expect(specStr).To(ContainSubstring("consumes:")) + Expect(specStr).To(ContainSubstring("name: cluster")) + Expect(specStr).To(ContainSubstring("type: cluster-info")) + Expect(specStr).To(ContainSubstring("optional: true")) + }) + It("generates runtime config referencing tanzu-content release", func() { + rcPath := filepath.Join(outputPath, "runtime_configs", "k8s-tile-test-pkgr.yml") + rcData, err := os.ReadFile(rcPath) + Expect(err).NotTo(HaveOccurred()) + + var rc models.RuntimeConfigOuter + err = yaml.Unmarshal(rcData, &rc) + Expect(err).NotTo(HaveOccurred()) + + var inner models.RuntimeConfigInner + err = yaml.Unmarshal([]byte(rc.RuntimeConfig), &inner) + Expect(err).NotTo(HaveOccurred()) + + Expect(inner.Addons).To(HaveLen(1)) + addon := inner.Addons[0] + + By("referencing tanzu-content release instead of registry") + Expect(addon.Include.Jobs).To(HaveLen(2)) + Expect(addon.Include.Jobs[0].Name).To(Equal("install-package-repository")) + Expect(addon.Include.Jobs[0].Release).To(Equal("tanzu-content")) + Expect(addon.Include.Jobs[1].Name).To(Equal("install-packages")) + Expect(addon.Include.Jobs[1].Release).To(Equal("tanzu-content")) + + By("having only the registry-data job (no separate package-install job)") + Expect(addon.Jobs).To(HaveLen(1)) + Expect(addon.Jobs[0].Name).To(Equal("registry-data")) + Expect(addon.Jobs[0].Release).To(Equal("k8s-tile-test")) + + By("carrying package install properties on the registry-data job") + Expect(addon.Jobs[0].Properties).To(HaveKey("test-install")) + props := addon.Jobs[0].Properties["test-install"] + Expect(props.Name).To(Equal("something-test.tanzu.vmware.com")) + Expect(props.Version).To(Equal("0.1.5")) + }) It("can be kiln baked", func() { if !kilnInstalled() { Skip("kiln CLI not installed - skipping integration test") @@ -156,4 +277,27 @@ var _ = Describe("Carvel Baker", func() { }) }) }) + + Context("generateManifestTemplate with different entry names", func() { + It("parameterizes the entry name throughout the template", func() { + template := generateManifestTemplate("my-custom-pkg") + + Expect(template).To(ContainSubstring(`p("my-custom-pkg.name")`)) + Expect(template).To(ContainSubstring(`p("my-custom-pkg.version")`)) + Expect(template).To(ContainSubstring(`p("my-custom-pkg.values")`)) + Expect(template).NotTo(ContainSubstring("test-install")) + }) + + It("contains exactly 6 K8s resource documents", func() { + template := generateManifestTemplate("pkg") + docs := strings.Split(template, "---") + nonEmpty := 0 + for _, doc := range docs { + if strings.TrimSpace(doc) != "" { + nonEmpty++ + } + } + Expect(nonEmpty).To(Equal(5)) + }) + }) }) From 9af312380c0dc2d2489b312343f25b0ee45c3ddb Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Tue, 10 Mar 2026 15:52:30 -0500 Subject: [PATCH 05/18] Spec complete implementation of kiln carvel commands For more details: https://docs.google.com/document/d/1Hf_T721E18fFDVTrqNZE4m_udo3eSW30VoUU1Zrxl-A/edit?tab=t.0 ai-assisted=true --- .../carvel/fixtures/sample-tile/.gitignore | 2 +- internal/carvel/TODO.md | 26 +++ internal/carvel/baker.go | 152 ++++++++++++++- internal/carvel/baker_test.go | 124 ++++++++++++- internal/carvel/models/lockfile.go | 39 ++++ internal/carvel/models/lockfile_test.go | 56 ++++++ .../carvel/testdata/sample-tile/.gitignore | 2 +- internal/commands/carvel.go | 3 + internal/commands/carvel_bake.go | 22 ++- internal/commands/carvel_publish.go | 162 ++++++++++++++++ internal/commands/carvel_publish_test.go | 123 +++++++++++++ internal/commands/carvel_rebake.go | 137 ++++++++++++++ internal/commands/carvel_rebake_test.go | 119 ++++++++++++ internal/commands/carvel_upload.go | 174 ++++++++++++++++++ internal/commands/carvel_upload_test.go | 117 ++++++++++++ 15 files changed, 1243 insertions(+), 15 deletions(-) create mode 100644 internal/carvel/TODO.md create mode 100644 internal/carvel/models/lockfile.go create mode 100644 internal/carvel/models/lockfile_test.go create mode 100644 internal/commands/carvel_publish.go create mode 100644 internal/commands/carvel_publish_test.go create mode 100644 internal/commands/carvel_rebake.go create mode 100644 internal/commands/carvel_rebake_test.go create mode 100644 internal/commands/carvel_upload.go create mode 100644 internal/commands/carvel_upload_test.go diff --git a/internal/acceptance/carvel/fixtures/sample-tile/.gitignore b/internal/acceptance/carvel/fixtures/sample-tile/.gitignore index bd4fa3a61..95270bce3 100644 --- a/internal/acceptance/carvel/fixtures/sample-tile/.gitignore +++ b/internal/acceptance/carvel/fixtures/sample-tile/.gitignore @@ -1,3 +1,3 @@ .boshrelease -.ezbake +.carvel-tile diff --git a/internal/carvel/TODO.md b/internal/carvel/TODO.md new file mode 100644 index 000000000..6fb4cce77 --- /dev/null +++ b/internal/carvel/TODO.md @@ -0,0 +1,26 @@ +# Carvel Package TODOs + +## Refactor: Replace exec.Command("kiln bake") with internal function call + +**File:** `baker.go` (line 46) + +**Current behavior:** `KilnBake()` shells out to the system `kiln` binary via +`exec.Command("kiln", "bake", "--skip-fetch", "--output-file", destination)`. + +**Problem:** +- The inner `kiln bake` resolves to whatever binary is on PATH, which may be a + different version than the running `kiln carvel bake`. +- Integration tests must manipulate PATH to ensure the correct binary is used. +- Spawning a subprocess for logic that exists in the same codebase is unnecessary + overhead. +- Error propagation across the process boundary is lossy. + +**Desired behavior:** `KilnBake()` should call the internal bake logic directly +(e.g. instantiate and invoke `commands.Bake` or the underlying `BakeService`) +instead of shelling out. This guarantees version consistency, improves +testability, and removes the PATH dependency. + +**Complexity note:** The `Bake` command has a non-trivial setup (BakeService, +fetchers, template evaluators, checksummers). The wiring will need to be +extracted into a reusable helper or the relevant subset of bake logic factored +out for in-process use. diff --git a/internal/carvel/baker.go b/internal/carvel/baker.go index 5b474f738..df0ed56c7 100644 --- a/internal/carvel/baker.go +++ b/internal/carvel/baker.go @@ -21,16 +21,20 @@ import ( // and kiln-compatible tile structure that can be baked into a .pivotal file. type Baker interface { Bake(source string) error + BakeFromLockfile(source string, lockfilePath string) error KilnBake(destination string) error GetName() string GetVersion() (string, error) + GetReleaseTarball() (string, error) SetWriter(w io.Writer) + SetProgressWriter(w io.Writer) } // NewBaker creates a new Baker for transforming imgpkg bundles into BOSH releases. func NewBaker() Baker { return &baker{ - writer: io.Discard, + writer: io.Discard, + progressWriter: io.Discard, } } @@ -38,9 +42,15 @@ type baker struct { metadata models.Metadata source, destination string writer io.Writer + progressWriter io.Writer } func (b *baker) KilnBake(destination string) error { + if err := b.ensureGitRepo(); err != nil { + return fmt.Errorf("failed to initialize git repo for kiln bake: %w", err) + } + + b.progress("Assembling final .pivotal file...") cmd := exec.Command("kiln", "bake", "--skip-fetch", @@ -48,6 +58,7 @@ func (b *baker) KilnBake(destination string) error { ) cmd.Dir = b.destination out, err := cmd.CombinedOutput() + b.log(string(out)) if err != nil { b.log("failed to invoke kiln: " + string(out)) return err @@ -56,10 +67,29 @@ func (b *baker) KilnBake(destination string) error { return nil } +// ensureGitRepo initializes a git repo with an empty commit in the +// generated tile directory so that `kiln bake` (which runs git status +// and git rev-parse HEAD) can operate on it without failing. +func (b *baker) ensureGitRepo() error { + commands := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "commit", "--allow-empty", "-m", "carvel tile build"), + } + for _, cmd := range commands { + cmd.Dir = b.destination + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("command %q failed: %s: %w", cmd.String(), string(out), err) + } + } + return nil +} + func (b *baker) Bake(source string) error { b.source = source - b.destination = path.Join(source, ".ezbake") + b.destination = path.Join(source, ".carvel-tile") + b.progress("Reading tile metadata from " + path.Join(source, "base.yml")) yamlPath := path.Join(source, "base.yml") yamlData, err := os.ReadFile(yamlPath) if err != nil { @@ -71,10 +101,11 @@ func (b *baker) Bake(source string) error { return err } - _, err = b.GetVersion() + ver, err := b.GetVersion() if err != nil { return err } + b.progress(fmt.Sprintf("Tile: %s version %s (metadata_version %s)", b.metadata.Name, ver, b.metadata.MetadataVersion)) metadataVersion, err := version.NewVersion(b.metadata.MetadataVersion) if err != nil { @@ -85,12 +116,14 @@ func (b *baker) Bake(source string) error { return errors.New("tile metadata_version too old for kubernetes support (must be >=3.2.0)") } + b.progress("Generating BOSH release structure...") err = b.generateBoshReleaseDir() if err != nil { b.log(err.Error()) return err } + b.progress("Generating tile layout in " + b.destination) err = b.generateOutputTile() if err != nil { b.log(err.Error()) @@ -100,6 +133,100 @@ func (b *baker) Bake(source string) error { return nil } +func (b *baker) BakeFromLockfile(source string, lockfilePath string) error { + b.source = source + b.destination = path.Join(source, ".carvel-tile") + + b.progress("Reading tile metadata from " + path.Join(source, "base.yml")) + yamlPath := path.Join(source, "base.yml") + yamlData, err := os.ReadFile(yamlPath) + if err != nil { + return err + } + + err = yaml.Unmarshal(yamlData, &b.metadata) + if err != nil { + return err + } + + ver, err := b.GetVersion() + if err != nil { + return err + } + b.progress(fmt.Sprintf("Tile: %s version %s (metadata_version %s)", b.metadata.Name, ver, b.metadata.MetadataVersion)) + + b.progress("Reading lockfile from " + lockfilePath) + lf, err := models.ReadCarvelLockfile(lockfilePath) + if err != nil { + return fmt.Errorf("failed to read lockfile: %w", err) + } + + if lf.Release.Name != b.metadata.Name { + return fmt.Errorf("lockfile release name %q does not match tile name %q", lf.Release.Name, b.metadata.Name) + } + + err = os.RemoveAll(b.destination) + if err != nil { + return err + } + err = os.MkdirAll(b.destination, 0755) + if err != nil { + return err + } + + b.progress("Generating tile layout in " + b.destination) + err = b.generateBaseYaml() + if err != nil { + return err + } + err = b.copyFiles() + if err != nil { + return err + } + err = b.generateJobFiles() + if err != nil { + return err + } + err = b.generateInstanceGroupFiles() + if err != nil { + return err + } + err = b.generateRuntimeConfigs() + if err != nil { + return err + } + + releasesDir := path.Join(b.destination, "releases") + err = os.MkdirAll(releasesDir, 0755) + if err != nil { + return err + } + + cachedTarball := lf.Release.RemotePath + destTarball := path.Join(releasesDir, b.metadata.Name+"-"+ver+".tgz") + + b.progress("Copying cached BOSH release from " + cachedTarball) + b.log("copying cached BOSH release from " + cachedTarball) + err = copyFileContents(cachedTarball, destTarball) + if err != nil { + return fmt.Errorf("failed to copy cached release tarball: %w", err) + } + + return nil +} + +func (b *baker) GetReleaseTarball() (string, error) { + ver, err := b.GetVersion() + if err != nil { + return "", err + } + tarball := path.Join(b.destination, "releases", b.metadata.Name+"-"+ver+".tgz") + if _, err := os.Stat(tarball); err != nil { + return "", fmt.Errorf("release tarball not found at %s: %w", tarball, err) + } + return tarball, nil +} + func (b *baker) GetName() string { return b.metadata.Name } @@ -122,18 +249,26 @@ func (b *baker) SetWriter(w io.Writer) { b.writer = w } +func (b *baker) SetProgressWriter(w io.Writer) { + b.progressWriter = w +} + func (b *baker) log(message string) { _, _ = fmt.Fprintln(b.writer, message) } +func (b *baker) progress(message string) { + _, _ = fmt.Fprintln(b.progressWriter, message) +} + func (b *baker) generateBoshReleaseDir() error { dirName := path.Join(b.source, ".boshrelease") - // first clean out any previous bosh release directory err := os.RemoveAll(dirName) if err != nil { return err } + b.progress(" Initializing BOSH release") commands := []*exec.Cmd{ exec.Command("bosh", "init-release", "--dir="+dirName), exec.Command("bosh", "add-blob", "--dir="+dirName, path.Join(b.source, "bundle.tar"), "imgpkg/bundle.tar"), @@ -173,12 +308,13 @@ files: registryDataTemplates := "" registryDataProperties := "" - // we need one PackageInstall YAML manifest for each entry in the metadata. + b.progress(" Configuring package installs") for _, entry := range b.metadata.PackageInstalls { entry = strings.Trim(entry, "$() ") entry = strings.TrimPrefix(entry, "package") entry = strings.Trim(entry, `"' `) + b.progress(" - " + entry) b.log("looking for package install: " + entry) // find this entry in the packageinstalls directory @@ -320,8 +456,6 @@ spec: } func (b *baker) generateOutputTile() error { - // first clean out any previous tile directory - // Note: this directory should only ever contain generated files, which we are about to regenerate. err := os.RemoveAll(b.destination) if err != nil { return err @@ -332,11 +466,13 @@ func (b *baker) generateOutputTile() error { return err } + b.progress(" Generating base.yml") err = b.generateBaseYaml() if err != nil { return err } + b.progress(" Copying forms, properties, and static assets") err = b.copyFiles() if err != nil { return err @@ -352,11 +488,13 @@ func (b *baker) generateOutputTile() error { return err } + b.progress(" Generating runtime configs") err = b.generateRuntimeConfigs() if err != nil { return err } + b.progress(" Creating BOSH release tarball (this may take a while)...") err = b.createBoshRelease() if err != nil { return err diff --git a/internal/carvel/baker_test.go b/internal/carvel/baker_test.go index d585fd732..db2744abd 100644 --- a/internal/carvel/baker_test.go +++ b/internal/carvel/baker_test.go @@ -1,6 +1,7 @@ package carvel import ( + "io" "os" "os/exec" "path" @@ -13,6 +14,21 @@ import ( "gopkg.in/yaml.v3" ) +func copyTestFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.Create(dst) + if err != nil { + return err + } + defer func() { _ = out.Close() }() + _, err = io.Copy(out, in) + return err +} + func boshInstalled() bool { _, err := exec.LookPath("bosh") return err == nil @@ -92,7 +108,7 @@ var _ = Describe("Carvel Baker", func() { inputPath, err = os.MkdirTemp("", "testinput-*") Expect(err).NotTo(HaveOccurred()) inputPath += "/tile" - outputPath = path.Join(inputPath, ".ezbake") + outputPath = path.Join(inputPath, ".carvel-tile") boshReleasePath = path.Join(inputPath, ".boshrelease") err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) Expect(err).NotTo(HaveOccurred()) @@ -278,6 +294,112 @@ var _ = Describe("Carvel Baker", func() { }) }) + Context("BakeFromLockfile", func() { + When("a valid lockfile references a pre-built release", func() { + BeforeEach(func() { + if !boshInstalled() { + Skip("bosh CLI not installed - skipping integration test") + } + }) + + It("produces tile output without regenerating the BOSH release", func() { + inputPath, err := os.MkdirTemp("", "lockfile-test-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + defer func() { _ = os.RemoveAll(filepath.Dir(inputPath)) }() + + err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + commands := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range commands { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + // First do a normal bake to produce a real BOSH release tarball + subject := NewBaker() + subject.SetWriter(GinkgoWriter) + err = subject.Bake(inputPath) + Expect(err).NotTo(HaveOccurred()) + + tarball, err := subject.GetReleaseTarball() + Expect(err).NotTo(HaveOccurred()) + + // Copy the tarball to a temp location (simulating Artifactory cache) + cachedTarball := filepath.Join(filepath.Dir(inputPath), "cached-release.tgz") + err = copyTestFile(tarball, cachedTarball) + Expect(err).NotTo(HaveOccurred()) + + // Write a lockfile pointing to the cached tarball + lf := models.CarvelLockfile{ + Release: models.CarvelReleaseLock{ + Name: "k8s-tile-test", + Version: "0.1.1", + RemotePath: cachedTarball, + SHA256: "test-sha", + }, + } + lockfilePath := filepath.Join(filepath.Dir(inputPath), "Kilnfile.lock") + err = lf.WriteFile(lockfilePath) + Expect(err).NotTo(HaveOccurred()) + + // Now bake from lockfile + subject2 := NewBaker() + subject2.SetWriter(GinkgoWriter) + err = subject2.BakeFromLockfile(inputPath, lockfilePath) + Expect(err).NotTo(HaveOccurred()) + + outputPath := path.Join(inputPath, ".carvel-tile") + Expect(filepath.Join(outputPath, "base.yml")).To(BeAnExistingFile()) + Expect(filepath.Join(outputPath, "releases", "k8s-tile-test-0.1.1.tgz")).To(BeAnExistingFile()) + Expect(filepath.Join(outputPath, "runtime_configs")).To(BeADirectory()) + }) + }) + + When("the lockfile release name does not match", func() { + It("returns an error", func() { + inputPath, err := os.MkdirTemp("", "lockfile-mismatch-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + defer func() { _ = os.RemoveAll(filepath.Dir(inputPath)) }() + + err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + lf := models.CarvelLockfile{ + Release: models.CarvelReleaseLock{ + Name: "wrong-name", + Version: "0.1.1", + }, + } + lockfilePath := filepath.Join(filepath.Dir(inputPath), "Kilnfile.lock") + err = lf.WriteFile(lockfilePath) + Expect(err).NotTo(HaveOccurred()) + + subject := NewBaker() + err = subject.BakeFromLockfile(inputPath, lockfilePath) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not match tile name")) + }) + }) + }) + + Context("GetReleaseTarball", func() { + When("called before bake", func() { + It("returns an error", func() { + subject := NewBaker() + _, err := subject.GetReleaseTarball() + Expect(err).To(HaveOccurred()) + }) + }) + }) + Context("generateManifestTemplate with different entry names", func() { It("parameterizes the entry name throughout the template", func() { template := generateManifestTemplate("my-custom-pkg") diff --git a/internal/carvel/models/lockfile.go b/internal/carvel/models/lockfile.go new file mode 100644 index 000000000..d0f4a1665 --- /dev/null +++ b/internal/carvel/models/lockfile.go @@ -0,0 +1,39 @@ +package models + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +type CarvelLockfile struct { + Release CarvelReleaseLock `yaml:"release"` +} + +type CarvelReleaseLock struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + RemotePath string `yaml:"remote_path"` + SHA256 string `yaml:"sha256"` +} + +func ReadCarvelLockfile(path string) (CarvelLockfile, error) { + data, err := os.ReadFile(path) + if err != nil { + return CarvelLockfile{}, fmt.Errorf("failed to read lockfile: %w", err) + } + var lf CarvelLockfile + if err := yaml.Unmarshal(data, &lf); err != nil { + return CarvelLockfile{}, fmt.Errorf("failed to parse lockfile: %w", err) + } + return lf, nil +} + +func (lf CarvelLockfile) WriteFile(path string) error { + data, err := yaml.Marshal(&lf) + if err != nil { + return fmt.Errorf("failed to marshal lockfile: %w", err) + } + return os.WriteFile(path, data, 0644) +} diff --git a/internal/carvel/models/lockfile_test.go b/internal/carvel/models/lockfile_test.go new file mode 100644 index 000000000..242f07c54 --- /dev/null +++ b/internal/carvel/models/lockfile_test.go @@ -0,0 +1,56 @@ +package models_test + +import ( + "os" + "path/filepath" + "testing" + + . "github.com/onsi/gomega" + + "github.com/pivotal-cf/kiln/internal/carvel/models" +) + +func TestCarvelLockfileRoundTrip(t *testing.T) { + g := NewWithT(t) + + dir := t.TempDir() + lockPath := filepath.Join(dir, "Kilnfile.lock") + + original := models.CarvelLockfile{ + Release: models.CarvelReleaseLock{ + Name: "my-tile", + Version: "1.2.3", + RemotePath: "bosh-releases/my-tile/my-tile-1.2.3.tgz", + SHA256: "abc123def456", + }, + } + + err := original.WriteFile(lockPath) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(lockPath).To(BeAnExistingFile()) + + loaded, err := models.ReadCarvelLockfile(lockPath) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(loaded).To(Equal(original)) +} + +func TestReadCarvelLockfileNotFound(t *testing.T) { + g := NewWithT(t) + + _, err := models.ReadCarvelLockfile("/nonexistent/path/Kilnfile.lock") + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("failed to read lockfile")) +} + +func TestReadCarvelLockfileInvalidYAML(t *testing.T) { + g := NewWithT(t) + + dir := t.TempDir() + lockPath := filepath.Join(dir, "Kilnfile.lock") + err := os.WriteFile(lockPath, []byte("release:\n name: [unterminated"), 0644) + g.Expect(err).NotTo(HaveOccurred()) + + _, err = models.ReadCarvelLockfile(lockPath) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("failed to parse lockfile")) +} diff --git a/internal/carvel/testdata/sample-tile/.gitignore b/internal/carvel/testdata/sample-tile/.gitignore index bd4fa3a61..95270bce3 100644 --- a/internal/carvel/testdata/sample-tile/.gitignore +++ b/internal/carvel/testdata/sample-tile/.gitignore @@ -1,3 +1,3 @@ .boshrelease -.ezbake +.carvel-tile diff --git a/internal/commands/carvel.go b/internal/commands/carvel.go index ae98d040d..1b71b1c48 100644 --- a/internal/commands/carvel.go +++ b/internal/commands/carvel.go @@ -25,6 +25,9 @@ func NewCarvel(outLogger, errLogger *log.Logger) Carvel { // Register subcommands c.commands["bake"] = NewCarvelBake(outLogger, errLogger) + c.commands["upload"] = NewCarvelUpload(outLogger, errLogger) + c.commands["publish"] = NewCarvelPublish(outLogger, errLogger) + c.commands["rebake"] = NewCarvelReBake(outLogger, errLogger) return c } diff --git a/internal/commands/carvel_bake.go b/internal/commands/carvel_bake.go index 8799106d4..d1ca4c272 100644 --- a/internal/commands/carvel_bake.go +++ b/internal/commands/carvel_bake.go @@ -19,6 +19,7 @@ type CarvelBake struct { type CarvelBakeOptions struct { SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` + Lockfile string `short:"l" long:"lockfile" description:"path to Kilnfile.lock for using a cached BOSH release"` Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` } @@ -54,14 +55,25 @@ func (c CarvelBake) Execute(args []string) error { } baker := carvel.NewBaker() + baker.SetProgressWriter(os.Stdout) if c.Options.Verbose { baker.SetWriter(os.Stdout) } - c.outLogger.Printf("Baking Carvel tile from %s into %s/.ezbake", sourcePath, sourcePath) - err = baker.Bake(sourcePath) - if err != nil { - return fmt.Errorf("failed to prepare Carvel tile: %w", err) + if c.Options.Lockfile != "" { + lockfilePath, err := filepath.Abs(c.Options.Lockfile) + if err != nil { + return fmt.Errorf("failed to resolve lockfile path: %w", err) + } + err = baker.BakeFromLockfile(sourcePath, lockfilePath) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile from lockfile: %w", err) + } + } else { + err = baker.Bake(sourcePath) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile: %w", err) + } } v, err := baker.GetVersion() @@ -74,7 +86,7 @@ func (c CarvelBake) Execute(args []string) error { return fmt.Errorf("failed to bake tile: %w", err) } - c.outLogger.Printf("Baked %s version %s to %s", baker.GetName(), v, targetPath) + c.outLogger.Printf("Done! Baked %s version %s to %s", baker.GetName(), v, targetPath) return nil } diff --git a/internal/commands/carvel_publish.go b/internal/commands/carvel_publish.go new file mode 100644 index 000000000..8e5d06f31 --- /dev/null +++ b/internal/commands/carvel_publish.go @@ -0,0 +1,162 @@ +package commands + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log" + "os" + "path/filepath" + + "github.com/pivotal-cf/jhanda" + "github.com/pivotal-cf/kiln/internal/builder" + "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/pkg/bake" +) + +type CarvelPublish struct { + outLogger *log.Logger + errLogger *log.Logger + KilnVersion string + Options CarvelPublishOptions +} + +type CarvelPublishOptions struct { + SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` + OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` + Version string ` long:"version" description:"tile version for the final release"` + Lockfile string `short:"l" long:"lockfile" description:"path to Kilnfile.lock for using a cached BOSH release"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` + IsFinal bool ` long:"final" description:"create a bake record for this build"` +} + +func NewCarvelPublish(outLogger, errLogger *log.Logger) CarvelPublish { + return CarvelPublish{ + outLogger: outLogger, + errLogger: errLogger, + } +} + +func (c CarvelPublish) Execute(args []string) error { + _, err := jhanda.Parse(&c.Options, args) + if err != nil { + return err + } + + sourcePath := c.Options.SourceDirectory + if sourcePath == "" { + sourcePath, err = os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %w", err) + } + } else { + sourcePath, err = filepath.Abs(sourcePath) + if err != nil { + return fmt.Errorf("failed to resolve source directory: %w", err) + } + } + + targetPath, err := filepath.Abs(c.Options.OutputFile) + if err != nil { + return fmt.Errorf("failed to resolve output file path: %w", err) + } + + b := carvel.NewBaker() + if c.Options.Verbose { + b.SetWriter(os.Stdout) + } + + if c.Options.Lockfile != "" { + lockfilePath, err := filepath.Abs(c.Options.Lockfile) + if err != nil { + return fmt.Errorf("failed to resolve lockfile path: %w", err) + } + c.outLogger.Printf("Publishing Carvel tile from %s using lockfile %s", sourcePath, lockfilePath) + err = b.BakeFromLockfile(sourcePath, lockfilePath) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile from lockfile: %w", err) + } + } else { + c.outLogger.Printf("Publishing Carvel tile from %s", sourcePath) + err = b.Bake(sourcePath) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile: %w", err) + } + } + + ver, err := b.GetVersion() + if err != nil { + return fmt.Errorf("failed to get tile version: %w", err) + } + if c.Options.Version != "" { + ver = c.Options.Version + } + + err = b.KilnBake(targetPath) + if err != nil { + return fmt.Errorf("failed to bake tile: %w", err) + } + + c.outLogger.Printf("Baked %s version %s to %s", b.GetName(), ver, targetPath) + + if c.Options.IsFinal { + // Resolve symlinks so git's toplevel and our absolute path match (macOS /var -> /private/var) + resolvedSourcePath, err := filepath.EvalSymlinks(sourcePath) + if err != nil { + resolvedSourcePath = sourcePath + } + + sha, err := builder.GitMetadataSHA(resolvedSourcePath, false) + if err != nil { + return fmt.Errorf("failed to get git SHA: %w", err) + } + + checksum, err := tileFileChecksum(targetPath) + if err != nil { + return fmt.Errorf("failed to checksum tile: %w", err) + } + + record := bake.Record{ + SourceRevision: sha, + Version: ver, + KilnVersion: c.KilnVersion, + FileChecksum: checksum, + } + + record, err = record.SetTileDirectory(resolvedSourcePath) + if err != nil { + return fmt.Errorf("failed to set tile directory on bake record: %w", err) + } + + err = record.WriteFile(resolvedSourcePath) + if err != nil { + return fmt.Errorf("failed to write bake record: %w", err) + } + + c.outLogger.Printf("Wrote bake record for version %s", ver) + } + + return nil +} + +func (c CarvelPublish) Usage() jhanda.Usage { + return jhanda.Usage{ + Description: "Publishes a Carvel/Kubernetes tile as a .pivotal file. When --final is specified, creates a bake record that can be used with 'kiln carvel rebake' for reproducible builds.", + ShortDescription: "publishes a Carvel/Kubernetes tile", + Flags: c.Options, + } +} + +func tileFileChecksum(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/commands/carvel_publish_test.go b/internal/commands/carvel_publish_test.go new file mode 100644 index 000000000..844addd17 --- /dev/null +++ b/internal/commands/carvel_publish_test.go @@ -0,0 +1,123 @@ +package commands_test + +import ( + "encoding/json" + "log" + "os" + "os/exec" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/commands" + "github.com/pivotal-cf/kiln/pkg/bake" +) + +var _ = Describe("CarvelPublish", func() { + var ( + outLogger *log.Logger + errLogger *log.Logger + command commands.CarvelPublish + ) + + BeforeEach(func() { + outLogger = log.New(GinkgoWriter, "", 0) + errLogger = log.New(GinkgoWriter, "", 0) + command = commands.NewCarvelPublish(outLogger, errLogger) + }) + + Describe("Usage", func() { + It("returns usage information", func() { + usage := command.Usage() + Expect(usage.ShortDescription).To(Equal("publishes a Carvel/Kubernetes tile")) + Expect(usage.Description).To(ContainSubstring("bake record")) + }) + }) + + Describe("Execute", func() { + When("required arguments are missing", func() { + It("returns an error when output-file is not provided", func() { + err := command.Execute([]string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("output-file")) + }) + }) + + When("--final flag is used", func() { + var ( + inputPath string + outputPath string + ) + + BeforeEach(func() { + if !boshInstalled() { + Skip("bosh CLI not installed - skipping integration test") + } + if !kilnInstalled() { + Skip("kiln CLI not installed - skipping integration test") + } + + var err error + inputPath, err = os.MkdirTemp("", "publish-test-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + cmds := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range cmds { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + outputPath = filepath.Join(filepath.Dir(inputPath), "output.pivotal") + }) + + AfterEach(func() { + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + }) + + It("bakes the tile and creates a bake record", func() { + err := command.Execute([]string{ + "--source-directory", inputPath, + "--output-file", outputPath, + "--final", + "--verbose", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(outputPath).To(BeAnExistingFile()) + + // Resolve symlinks (macOS /var -> /private/var) + resolvedInput, resolveErr := filepath.EvalSymlinks(inputPath) + if resolveErr != nil { + resolvedInput = inputPath + } + + recordsDir := filepath.Join(resolvedInput, "bake_records") + Expect(recordsDir).To(BeADirectory()) + + entries, err := os.ReadDir(recordsDir) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(HaveLen(1)) + Expect(entries[0].Name()).To(Equal("0.1.1.json")) + + recordData, err := os.ReadFile(filepath.Join(recordsDir, "0.1.1.json")) + Expect(err).NotTo(HaveOccurred()) + + var record bake.Record + err = json.Unmarshal(recordData, &record) + Expect(err).NotTo(HaveOccurred()) + Expect(record.Version).To(Equal("0.1.1")) + Expect(record.SourceRevision).NotTo(BeEmpty()) + Expect(record.FileChecksum).NotTo(BeEmpty()) + }) + }) + }) +}) diff --git a/internal/commands/carvel_rebake.go b/internal/commands/carvel_rebake.go new file mode 100644 index 000000000..d948c21b8 --- /dev/null +++ b/internal/commands/carvel_rebake.go @@ -0,0 +1,137 @@ +package commands + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "os" + "path/filepath" + + "github.com/pivotal-cf/jhanda" + "github.com/pivotal-cf/kiln/internal/builder" + "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/pkg/bake" +) + +type CarvelReBake struct { + outLogger *log.Logger + errLogger *log.Logger + Options CarvelReBakeOptions +} + +type CarvelReBakeOptions struct { + OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` +} + +func NewCarvelReBake(outLogger, errLogger *log.Logger) CarvelReBake { + return CarvelReBake{ + outLogger: outLogger, + errLogger: errLogger, + } +} + +func (c CarvelReBake) Execute(args []string) error { + remaining, err := jhanda.Parse(&c.Options, args) + if err != nil { + return err + } + if len(remaining) != 1 { + return fmt.Errorf("exactly one bake record argument is required, got %d", len(remaining)) + } + + recordPath := remaining[0] + recordBuf, err := os.ReadFile(recordPath) + if err != nil { + return fmt.Errorf("failed to read bake record file: %w", err) + } + + var record bake.Record + if err := json.Unmarshal(recordBuf, &record); err != nil { + return fmt.Errorf("failed to parse bake record: %w", err) + } + + tileDir := filepath.FromSlash(record.TileDirectory) + if tileDir == "" { + tileDir = "." + } + + sourcePath, err := filepath.Abs(tileDir) + if err != nil { + return fmt.Errorf("failed to resolve tile directory: %w", err) + } + + workingDirectorySHA, err := builder.GitMetadataSHA(sourcePath, false) + if err != nil { + return err + } + + if got, exp := workingDirectorySHA, record.SourceRevision; got != exp { + return fmt.Errorf("expected worktree at source revision %s but current HEAD is %s", exp, got) + } + + targetPath, err := filepath.Abs(c.Options.OutputFile) + if err != nil { + return fmt.Errorf("failed to resolve output file path: %w", err) + } + + b := carvel.NewBaker() + if c.Options.Verbose { + b.SetWriter(os.Stdout) + } + + lockfilePath := filepath.Join(sourcePath, "Kilnfile.lock") + if _, statErr := os.Stat(lockfilePath); statErr == nil { + c.outLogger.Printf("Re-baking Carvel tile from %s using lockfile", sourcePath) + err = b.BakeFromLockfile(sourcePath, lockfilePath) + } else { + c.outLogger.Printf("Re-baking Carvel tile from %s", sourcePath) + err = b.Bake(sourcePath) + } + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile: %w", err) + } + + err = b.KilnBake(targetPath) + if err != nil { + return fmt.Errorf("failed to bake tile: %w", err) + } + + checksum, err := rebakeFileChecksum(targetPath) + if err != nil { + return fmt.Errorf("failed to checksum tile: %w", err) + } + + if record.FileChecksum != "" && record.FileChecksum != checksum { + return fmt.Errorf("tile checksum mismatch: record has %s, rebake produced %s", record.FileChecksum, checksum) + } + + ver, _ := b.GetVersion() + c.outLogger.Printf("Re-baked %s version %s to %s", b.GetName(), ver, targetPath) + + return nil +} + +func (c CarvelReBake) Usage() jhanda.Usage { + return jhanda.Usage{ + Description: "Re-bakes a Carvel tile from a bake record for reproducible builds. The repository must be checked out at the source_revision specified in the bake record.", + ShortDescription: "re-bakes a Carvel tile from a bake record", + Flags: c.Options, + } +} + +func rebakeFileChecksum(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/commands/carvel_rebake_test.go b/internal/commands/carvel_rebake_test.go new file mode 100644 index 000000000..3e4151642 --- /dev/null +++ b/internal/commands/carvel_rebake_test.go @@ -0,0 +1,119 @@ +package commands_test + +import ( + "encoding/json" + "log" + "os" + "os/exec" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/commands" + "github.com/pivotal-cf/kiln/pkg/bake" +) + +var _ = Describe("CarvelReBake", func() { + var ( + outLogger *log.Logger + errLogger *log.Logger + command commands.CarvelReBake + ) + + BeforeEach(func() { + outLogger = log.New(GinkgoWriter, "", 0) + errLogger = log.New(GinkgoWriter, "", 0) + command = commands.NewCarvelReBake(outLogger, errLogger) + }) + + Describe("Usage", func() { + It("returns usage information", func() { + usage := command.Usage() + Expect(usage.ShortDescription).To(Equal("re-bakes a Carvel tile from a bake record")) + Expect(usage.Description).To(ContainSubstring("bake record")) + }) + }) + + Describe("Execute", func() { + When("required arguments are missing", func() { + It("returns an error when no bake record is provided", func() { + err := command.Execute([]string{ + "--output-file", "/tmp/out.pivotal", + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("exactly one bake record argument")) + }) + + It("returns an error when output-file is not provided", func() { + err := command.Execute([]string{"some-record.json"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("output-file")) + }) + }) + + When("the bake record file does not exist", func() { + It("returns an error", func() { + err := command.Execute([]string{ + "--output-file", "/tmp/out.pivotal", + "/nonexistent/record.json", + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to read bake record")) + }) + }) + + When("the bake record has a mismatched source revision", func() { + var ( + inputPath string + recordPath string + ) + + BeforeEach(func() { + var err error + inputPath, err = os.MkdirTemp("", "rebake-test-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + cmds := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range cmds { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + record := bake.Record{ + SourceRevision: "0000000000000000000000000000000000000000", + Version: "0.1.1", + TileDirectory: inputPath, + } + buf, err := json.Marshal(record) + Expect(err).NotTo(HaveOccurred()) + + recordPath = filepath.Join(filepath.Dir(inputPath), "record.json") + err = os.WriteFile(recordPath, buf, 0644) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + }) + + It("returns a source revision mismatch error", func() { + err := command.Execute([]string{ + "--output-file", filepath.Join(filepath.Dir(inputPath), "out.pivotal"), + recordPath, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("source revision")) + }) + }) + }) +}) diff --git a/internal/commands/carvel_upload.go b/internal/commands/carvel_upload.go new file mode 100644 index 000000000..933f39b41 --- /dev/null +++ b/internal/commands/carvel_upload.go @@ -0,0 +1,174 @@ +package commands + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + + "github.com/pivotal-cf/jhanda" + "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/carvel/models" +) + +type CarvelUpload struct { + outLogger *log.Logger + errLogger *log.Logger + Options CarvelUploadOptions +} + +type CarvelUploadOptions struct { + SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` + ArtifactoryHost string ` long:"artifactory-host" description:"Artifactory server URL" required:"true"` + ArtifactoryRepo string ` long:"artifactory-repo" description:"Artifactory repository name" required:"true"` + Username string `short:"u" long:"artifactory-username" description:"Artifactory username" required:"true"` + Password string `short:"p" long:"artifactory-password" description:"Artifactory password or API key" required:"true"` + PathTemplate string ` long:"path-template" description:"remote path template" default:"bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz"` + OutputFile string `short:"o" long:"output-file" description:"also bake the tile to this path"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` +} + +func NewCarvelUpload(outLogger, errLogger *log.Logger) CarvelUpload { + return CarvelUpload{ + outLogger: outLogger, + errLogger: errLogger, + } +} + +func (c CarvelUpload) Execute(args []string) error { + _, err := jhanda.Parse(&c.Options, args) + if err != nil { + return err + } + + sourcePath := c.Options.SourceDirectory + if sourcePath == "" { + sourcePath, err = os.Getwd() + if err != nil { + return fmt.Errorf("failed to get current directory: %w", err) + } + } else { + sourcePath, err = filepath.Abs(sourcePath) + if err != nil { + return fmt.Errorf("failed to resolve source directory: %w", err) + } + } + + baker := carvel.NewBaker() + if c.Options.Verbose { + baker.SetWriter(os.Stdout) + } + + c.outLogger.Printf("Baking Carvel tile from %s", sourcePath) + err = baker.Bake(sourcePath) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile: %w", err) + } + + tarball, err := baker.GetReleaseTarball() + if err != nil { + return fmt.Errorf("failed to locate release tarball: %w", err) + } + + ver, err := baker.GetVersion() + if err != nil { + return fmt.Errorf("failed to get tile version: %w", err) + } + + remotePath := fmt.Sprintf("bosh-releases/%s/%s-%s.tgz", baker.GetName(), baker.GetName(), ver) + + checksum, err := fileSHA256(tarball) + if err != nil { + return fmt.Errorf("failed to checksum release tarball: %w", err) + } + + c.outLogger.Printf("Uploading %s to %s/%s/%s", filepath.Base(tarball), c.Options.ArtifactoryHost, c.Options.ArtifactoryRepo, remotePath) + err = uploadToArtifactory(tarball, c.Options.ArtifactoryHost, c.Options.ArtifactoryRepo, remotePath, c.Options.Username, c.Options.Password) + if err != nil { + return fmt.Errorf("failed to upload to Artifactory: %w", err) + } + + lockfilePath := filepath.Join(sourcePath, "Kilnfile.lock") + lf := models.CarvelLockfile{ + Release: models.CarvelReleaseLock{ + Name: baker.GetName(), + Version: ver, + RemotePath: remotePath, + SHA256: checksum, + }, + } + err = lf.WriteFile(lockfilePath) + if err != nil { + return fmt.Errorf("failed to write lockfile: %w", err) + } + c.outLogger.Printf("Updated %s", lockfilePath) + + if c.Options.OutputFile != "" { + targetPath, err := filepath.Abs(c.Options.OutputFile) + if err != nil { + return fmt.Errorf("failed to resolve output file path: %w", err) + } + err = baker.KilnBake(targetPath) + if err != nil { + return fmt.Errorf("failed to bake tile: %w", err) + } + c.outLogger.Printf("Baked %s version %s to %s", baker.GetName(), ver, targetPath) + } + + return nil +} + +func (c CarvelUpload) Usage() jhanda.Usage { + return jhanda.Usage{ + Description: "Generates a BOSH release from a Carvel tile source, uploads the release tarball to Artifactory, and updates Kilnfile.lock with the remote location and checksum.", + ShortDescription: "uploads a Carvel BOSH release to Artifactory", + Flags: c.Options, + } +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func uploadToArtifactory(localPath, host, repo, remotePath, username, password string) error { + f, err := os.Open(localPath) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + uploadURL := host + "/" + repo + "/" + remotePath + + req, err := http.NewRequest(http.MethodPut, uploadURL, f) + if err != nil { + return err + } + req.SetBasicAuth(username, password) + req.Header.Set("Content-Type", "application/gzip") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} diff --git a/internal/commands/carvel_upload_test.go b/internal/commands/carvel_upload_test.go new file mode 100644 index 000000000..f1cf62eb3 --- /dev/null +++ b/internal/commands/carvel_upload_test.go @@ -0,0 +1,117 @@ +package commands_test + +import ( + "log" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/carvel/models" + "github.com/pivotal-cf/kiln/internal/commands" +) + +var _ = Describe("CarvelUpload", func() { + var ( + outLogger *log.Logger + errLogger *log.Logger + command commands.CarvelUpload + ) + + BeforeEach(func() { + outLogger = log.New(GinkgoWriter, "", 0) + errLogger = log.New(GinkgoWriter, "", 0) + command = commands.NewCarvelUpload(outLogger, errLogger) + }) + + Describe("Usage", func() { + It("returns usage information", func() { + usage := command.Usage() + Expect(usage.ShortDescription).To(Equal("uploads a Carvel BOSH release to Artifactory")) + Expect(usage.Description).To(ContainSubstring("Artifactory")) + }) + }) + + Describe("Execute", func() { + When("required arguments are missing", func() { + It("returns an error when artifactory-host is not provided", func() { + err := command.Execute([]string{ + "--artifactory-repo", "some-repo", + "--artifactory-username", "user", + "--artifactory-password", "pass", + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("artifactory-host")) + }) + }) + + When("valid arguments are provided with a mock Artifactory", func() { + var ( + inputPath string + server *httptest.Server + ) + + BeforeEach(func() { + if !boshInstalled() { + Skip("bosh CLI not installed - skipping integration test") + } + + var err error + inputPath, err = os.MkdirTemp("", "upload-test-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + cmds := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range cmds { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusCreated) + })) + }) + + AfterEach(func() { + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + if server != nil { + server.Close() + } + }) + + It("uploads the BOSH release and writes a lockfile", func() { + err := command.Execute([]string{ + "--source-directory", inputPath, + "--artifactory-host", server.URL, + "--artifactory-repo", "test-repo", + "--artifactory-username", "user", + "--artifactory-password", "pass", + "--verbose", + }) + Expect(err).NotTo(HaveOccurred()) + + lockfilePath := filepath.Join(inputPath, "Kilnfile.lock") + Expect(lockfilePath).To(BeAnExistingFile()) + + lf, err := models.ReadCarvelLockfile(lockfilePath) + Expect(err).NotTo(HaveOccurred()) + Expect(lf.Release.Name).To(Equal("k8s-tile-test")) + Expect(lf.Release.Version).To(Equal("0.1.1")) + Expect(lf.Release.SHA256).NotTo(BeEmpty()) + Expect(lf.Release.RemotePath).To(ContainSubstring("k8s-tile-test")) + }) + }) + }) +}) From 9144f4926be0f8aefb3a82e7d5ead537b1ed3f83 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Tue, 10 Mar 2026 16:06:27 -0500 Subject: [PATCH 06/18] GPP onboarding doc for Carvel based tiles --- gpp-onboarding-carvel.md | 232 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 gpp-onboarding-carvel.md diff --git a/gpp-onboarding-carvel.md b/gpp-onboarding-carvel.md new file mode 100644 index 000000000..f4bef08ae --- /dev/null +++ b/gpp-onboarding-carvel.md @@ -0,0 +1,232 @@ +This playbook provides instructions for Carvel/Kubernetes tile teams to onboard their tiles onto the Golden Path to publish workflow. + +For Carvel-based tiles, intermediary BOSH releases are generated automatically from imgpkg bundles (via ezbake) and do not need to be managed directly by your team. GPP handles BOSH release ingest and compilation behind the scenes. Your team only needs to manage final tile releases. + +The result of this work will give you a re-bakable tile in [Artifactory](https://usw1.packages.broadcom.com/ui/repos/tree/General/tas-ecosystem-generic-prod-local/tile-releases) with compiled BOSH releases that is scanned by BlackDuck. For TVS integration please notify the Slingshots team when you're ready for it along with a link to your config please. + +You may also optionally configure your Tile to generate RMT releases, and Open Source License Disclosure files from Blackduck. + +## Pre-requisites for onboarding + +### Your tile is built with Kiln + +The Golden Path does not currently support tiles built with [tile-generator](https://github.com/cf-platform-eng/tile-generator). + +Please consider using [kiln](https://github.com/pivotal-cf/kiln/blob/main/TILE_AUTHOR_GUIDE.md). Carvel tile workflows use the `kiln carvel` subcommand group (`bake`, `upload`, `publish`, `rebake`). + +### Github repo access: tiles + +Please provide write access to your private tile repositories to our bot account. Because BOSH releases are generated from the tile source (imgpkg bundles), separate BOSH release repositories are not required. + +- For github enterprise (github.gwd.broadcom.com): [tanzu-tas-ecosystem](https://github.gwd.broadcom.net/tanzu-tas-ecosystem) +- for github.com: [tas-ecosystem-bot](https://github.com/tas-ecosystem-bot) + +### TNZ team membership + +To create PRs against our configuration repo you need to be a member of the [`all`](https://github.gwd.broadcom.net/orgs/TNZ/teams/all) team in the [TNZ org](https://github.gwd.broadcom.net/orgs/TNZ). + +### Broadcom artifactory access + +Authentication is required for accessing repos and artifacts on the Broadcom Jfrog Artifactory service. To get access for your team to our artifact repos containing: bosh-releases, compiled-releases, tile-releases and tile-candidates, create a [1.Support Ticket](https://broadcomitsm.wolkenservicedesk.com/wolken-support/item_details?itemId=2422). +Specify: + +- Artifactory Server Name / URL: `https://usw1.packages.broadcom.com/ui` +- Sample Business Justification: + + ```text + Need read access to tas-ecosystem-* artifactory projects on https://usw1.packages.broadcom.com + + For the following teammates / service accounts: + - memberX + - memberY + - memberZ + - bot / service account + ``` + +#### Credentials + +Since artifactory is authenticated with Okta SSO, password authentication to the service it not allowed. Artifactory have `api_keys` and `identity_tokens` that are used as passwords. + +Once access is granted and you are able to login to the artifactory ui via SSO, an `api_key` or `identity token` needs to be created for use with Kiln + +1. Upper right click dropdown of: `Welcome, your_username` +2. Click `Edit Profile` +3. Create an `api_key` or `identity_token` here and use it as the password for `kiln` commands or the artifactory cli. + +#### Network access + +The `usw1.packages.broadcom.com` artifactory is also only available on the Broadcom network. If accessing remotely, full tunnel VPN is required. + +If you CI is on the VMware / Broadcom Network and is blocked from accessing the artifactory, reach out to Google Chat Space: [#VMW-harbor-jfrog-migration](https://chat.google.com/room/AAAAcWIWWOA?cls=7) + +## Golden Path Configuration + +Configuration for the TAS Golden Path is stored in this repo and used as inputs to generate concourse pipelines. + +For Carvel-based tiles, the onboarding is simpler than for traditional BOSH tiles because GPP manages BOSH release ingest and compilation behind the scenes. You do not need to add BOSH release config files to the `bosh/` folder. The existing [bosh-ingest](https://tpe-concourse-rock.acc.broadcom.net/teams/tas-ecosystem/pipelines/bosh-releases?group=ingest-releases) and [bosh-compile](https://tpe-concourse-rock.acc.broadcom.net/teams/tas-ecosystem/pipelines/bosh-releases?group=compile-releases) pipelines are available for inspection if needed but do not require configuration from your team. + +Overall the following steps to complete are: + +- Updating the `Kilnfile` in the git repository to use artifactory as a source for generated BOSH releases. +- (optional) Creating a branch in the git repository of your tile for a pipeline to push Kilnfile.lock updates for your review. +- Creating config files for your tile(s) in `tiles/` folder to generate pipeline that will: + - Bake tile candidates with `kiln carvel bake`. Dev builds of tiles on your main / feature branch + - [`kiln carvel rebake`](https://github.com/pivotal-cf/kiln) for versioned release tiles + - Associate the BOSH releases consumed by the tile to the Blackduck tile project + - (optional) Automatically creates RMT releases that are included in the next-available TPM managed Release Train to assist with publishing + - If RMT is enabled, then your RMT release is eligible for automatic Open Source License Notice inclusion. Please see [creating open source license disclosures](./creating_open_source_license_disclosures.md). + - (optional) TVS integration (notify the #tas-slingshots team with your tile config requesting this when ready) + +### Tile repository updates + +Set up your tile repository so that `kiln carvel` commands can fetch generated BOSH releases from Artifactory. + +1. In the main / feature branch, update the `Kilnfile` to include artifactory as the remote source for BOSH releases. + + ```yaml + release_sources: + - type: artifactory + id: artifactory_bosh_releases + artifactory_host: $(variable "artifactory_host") + repo: $(variable "artifactory_repo") + username: $(variable "artifactory_username") + password: $(variable "artifactory_password") # api_key or identity token + publishable: true # if this repo contains releases that are suitable to ship to customers + path_template: bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz + ``` + +2. Use `kiln carvel upload` to generate the BOSH release from your imgpkg bundle, upload it to Artifactory, and update the `Kilnfile.lock` with the remote location and checksum. + + Example `kiln carvel upload` command: + + ```bash + $ kiln carvel upload \ + --artifactory-host https://usw1.packages.broadcom.com \ + --artifactory-repo tas-ecosystem-generic-prod-local \ + --artifactory-username \ + --artifactory-password \ + --output-file my-tile-1.0.0-dev.pivotal + ``` + + This uploads the generated BOSH release and writes a `Kilnfile.lock` referencing the remote artifact. Commit the updated `Kilnfile.lock` to your repository. + +3. (Optional) Create a new update branch from the main / feature branch in your tile repository (eg: `autobump`). Our CI will force push commits to this branch. + While this provides an auto update functionality, you are welcome to continue using your existing auto update tools (eg: dependabot). + You can also specify your feature branch if you want our CI to push the `Kilnfile.lock` updates directly to your feature branch. + If `branch` and `update_branch` are same, force push functionality is disabled. Ensure the branch specified in `update_branch` has [push access to our bot account](#github-repo-access-tiles). + +### Tile config onboard + +1. Clone [this](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration) git repository if you have not already and create a branch locally for your changes. You should have write access to the repo and not need to create a fork to create a PR. If not, please review [this pre-requisite](#tnz-team-membership) + +2. Create a new file for each of your tiles under the [tiles](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration/tree/main/tiles) directory. Please note that `artifact_name` is especially significant because it determines the file name prefix in `artifactory`, project names prefixes in `blackduck`, and is the prefix used by the published release file in RMT / Broadcom Portal (when enabled). + Hello Tile - `hello-tile.yml` + + ```yaml + #@data/values + --- + repo: https://github.gwd.broadcom.net/TNZ/hello-tile.git + branch: main + update_branch: auto-bump + subpath: . + artifact_name: crhntr-hello #! this is the prefix for the built tiles and must be consistent with blackduck too + prerelease_format: build_increment_sha #! "sha" or "build_increment_sha" for versioning tile candidate builds. we recommend build_increment_sha + team_members: + - a@vmware.com + - b@vmware.com + team_google_chat_group: some-group #! required - google space / chat group for your team + team_slack_channel: some-channel #! If your team has slack channel + ``` + + Scheduler Tile - `p-scheduler.yml` (auto update directly on feature branch) + + ```yaml + #@data/values + --- + repo: https://github.com/pivotal-cf/p-scheduler.git + branch: master + update_branch: master + subpath: . + artifact_name: p-scheduler #! this is the prefix for the built tiles and must be consistent with blackduck too + prerelease_format: build_increment_sha #! "sha" or "build_increment_sha" for versioning tile candidate builds + team_members: + - a@vmware.com + - b@vmware.com + team_google_chat_group: some-group #! required - google space / chat group for your team + team_slack_channel: some-channel #! If your team has slack channel + ``` + +3. (optional) Add fields for automatic RMT _**draft**_-release creation + + >**Warning:** You will need to add upgrade specifiers (else Upgrade Planner will break!) and double check your release is ready to be set to GA. It defaults to a draft. + + Release tiles can be used as the basis for automatic `RMT` draft release creation. As a draft this means further steps are required prior to publishing. These include manually setting your upgrade specifiers, double checking the version, GA/EOGs dates, and release type, etc we inferred for you or read from your tile configuration's `rmt` entry. + + See [tile rmt release](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration/tree/main/docs/tile_rmt_release.md) for details. + +4. (optional) Add a field to enable automatic Black Duck tile project associations. In order to begin updating your Black Duck tile project: + >**Prerequisites:** + > Follow the [BlackDuck Onboarding section](./creating_open_source_license_disclosures.md#blackduck-onboarding) for + > your tile. BOSH release projects are managed by GPP for Carvel-based tiles. + + 1) Confirm your project exists @ https://broadcom-vmw.app.blackduck.com/ with the format `TNZ-CF--tile`, as `` is found in your tile config. + 2) If not, submit a ticket to request it ([BlackDuck Onboarding section](./creating_open_source_license_disclosures.md#blackduck-onboarding)) or rename it yourself. + > **Note:** Scanning is enabled by default. However, you may disable it by adding the following to your `./tiles/.yml` config: + ```yaml + blackduck: + enabled: false + ``` + +5. Create a PR to [this](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration) repository to add the newly created files that contain the tile information. +An extensive PR Check job will verify your change and add a comment if anything needs to be addressed. When the job passes you can merge the PR. +On merge, the respective golden path jobs will be created / updated for your tile. Please reach out to [#tas-slingshots on Google Chat](https://chat.google.com/room/AAAAZuDvKe0?cls=7) with any questions or issues getting your PR merged. + +## Updating CI for your tile and automatic RMT _draft_ releases + +### Use `kiln carvel publish --final` + +If you are using CI to create new versions of tiles, the following updates can be made to take advantage of reproducible builds via `kiln carvel rebake`. + +Update your CI to output final tile builds using `kiln carvel publish --final`. When passing the **_--final_** flag, Kiln creates a bake record file under the **_bake_records_** folder. As part of the final tile build CI job, the bake records file needs to be committed and pushed to the tile repository. + +Golden Path to publish will then use this bake record to trigger `kiln carvel rebake`, producing a final tile from our [CI](https://runway-ci-srp.eng.vmware.com/teams/tas-ecosystem/) and upload it to [artifactory](https://build-artifactory.eng.vmware.com/ui/repos/tree/General/tas-ecosystem-generic-local/) repo under the sub-path: tile-releases. + +In the case of a pre-release version the build will not result in a publish. This is useful to verify OSL triage status and test your candidate build. For example, you may create a +bake record with version `2.4.41-dev.0`, rerun the OSL generation multiple times, then finally create a `2.41.1` to trigger the full publish. + + Example `kiln carvel publish --final` command: + + ```bash + $ kiln carvel publish --final --version 2.1.41 \ + --output-file my-tile-2.1.41.pivotal \ + --source-directory . + ``` + + _Example:_ Bake record that should be committed to the tile repo that is created by `kiln carvel publish --final` as file: `bake_records/2.1.41.json`: + + ```json + { + "source_revision": "1b19d8cb80e6cfdddd7be1c7a26c8210cbd4e4c5", + "version": "2.1.41", + "kiln_version": "0.90.0", + "file_checksum": "7622143c54dc53087a6c2401f5030170515e14f466857564a980092d4c87a094", + "tile_directory": "." + } + ``` + +When executing `kiln carvel publish --final`, use the values for artifactory variables in your Kilnfile: + +- artifactory_host: `https://usw1.packages.broadcom.com` +- artifactory_repo: `tas-ecosystem-generic-prod-local` +- artifactory_username: **_your account or service account for broadcom artifactory_** +- artifactory_password: **_respective api_key or identity token_** + +**_NOTE: https://usw1.packages.broadcom.com is accessible via Broadcom VPN with full tunnel gateway and TPE concourse workers_** + +**_Commit the new bake record to the git repository of the tile_** + +**_Warning: Your bake_records directory must only contain bake records_** + +GPP will then automatically run `kiln carvel rebake` against the bake record to produce the final `.pivotal` file, validate the checksum, and upload it to Artifactory and optionally RMT. + +Please refer to [Tile RMT Release](Publish-Tiles-to-RMT) to configure publishing your tile to RMT via Golden Path to Publish. From adb5ccc8912ff42e5de07ca6dfdf9d244a4cf45d Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Tue, 17 Mar 2026 16:30:03 -0500 Subject: [PATCH 07/18] Make changes to enable reproducibility of tile builds - Git repo is assumed and not created - BOSH release is NOT uploaded on publish anymore, publish reuses the BOSH release from Kilnfile.lock --- internal/carvel/baker.go | 22 ------ internal/carvel/baker_test.go | 94 ++++++++++++++++++++++++ internal/commands/carvel_bake_test.go | 12 +++ internal/commands/carvel_publish.go | 38 +++++----- internal/commands/carvel_publish_test.go | 44 ++++++++++- 5 files changed, 168 insertions(+), 42 deletions(-) diff --git a/internal/carvel/baker.go b/internal/carvel/baker.go index df0ed56c7..e0f78b79d 100644 --- a/internal/carvel/baker.go +++ b/internal/carvel/baker.go @@ -46,10 +46,6 @@ type baker struct { } func (b *baker) KilnBake(destination string) error { - if err := b.ensureGitRepo(); err != nil { - return fmt.Errorf("failed to initialize git repo for kiln bake: %w", err) - } - b.progress("Assembling final .pivotal file...") cmd := exec.Command("kiln", "bake", @@ -67,24 +63,6 @@ func (b *baker) KilnBake(destination string) error { return nil } -// ensureGitRepo initializes a git repo with an empty commit in the -// generated tile directory so that `kiln bake` (which runs git status -// and git rev-parse HEAD) can operate on it without failing. -func (b *baker) ensureGitRepo() error { - commands := []*exec.Cmd{ - exec.Command("git", "init"), - exec.Command("git", "commit", "--allow-empty", "-m", "carvel tile build"), - } - for _, cmd := range commands { - cmd.Dir = b.destination - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("command %q failed: %s: %w", cmd.String(), string(out), err) - } - } - return nil -} - func (b *baker) Bake(source string) error { b.source = source b.destination = path.Join(source, ".carvel-tile") diff --git a/internal/carvel/baker_test.go b/internal/carvel/baker_test.go index db2744abd..2d3c0df80 100644 --- a/internal/carvel/baker_test.go +++ b/internal/carvel/baker_test.go @@ -1,6 +1,8 @@ package carvel import ( + "crypto/sha256" + "encoding/hex" "io" "os" "os/exec" @@ -29,6 +31,16 @@ func copyTestFile(src, dst string) error { return err } +func fileChecksum(path string) string { + f, err := os.Open(path) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + defer func() { _ = f.Close() }() + h := sha256.New() + _, err = io.Copy(h, f) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return hex.EncodeToString(h.Sum(nil)) +} + func boshInstalled() bool { _, err := exec.LookPath("bosh") return err == nil @@ -400,6 +412,88 @@ var _ = Describe("Carvel Baker", func() { }) }) + Context("rebake reproducibility", func() { + // Both publish and rebake now use BakeFromLockfile with the same + // cached BOSH release tarball. This test verifies the resulting + // .pivotal files are byte-for-byte identical. + It("publish and rebake produce identical tiles when using the same lockfile", func() { + if !boshInstalled() { + Skip("bosh CLI not installed") + } + if !kilnInstalled() { + Skip("kiln CLI not installed") + } + + tmpRoot, err := os.MkdirTemp("", "rebake-repro-*") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = os.RemoveAll(tmpRoot) }() + + inputPath := filepath.Join(tmpRoot, "tile") + err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + for _, cmd := range []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "git setup: "+string(out)) + } + + // Simulate `kiln carvel upload`: Bake() to produce a BOSH release, + // then cache the tarball and write a Kilnfile.lock. + uploadBaker := NewBaker() + uploadBaker.SetWriter(GinkgoWriter) + err = uploadBaker.Bake(inputPath) + Expect(err).NotTo(HaveOccurred()) + + uploadTarball, err := uploadBaker.GetReleaseTarball() + Expect(err).NotTo(HaveOccurred()) + cachedTarball := filepath.Join(tmpRoot, "cached-release.tgz") + Expect(copyTestFile(uploadTarball, cachedTarball)).To(Succeed()) + + lf := models.CarvelLockfile{ + Release: models.CarvelReleaseLock{ + Name: "k8s-tile-test", + Version: "0.1.1", + RemotePath: cachedTarball, + SHA256: "unused", + }, + } + lockfilePath := filepath.Join(tmpRoot, "Kilnfile.lock") + Expect(lf.WriteFile(lockfilePath)).To(Succeed()) + + // Simulate `kiln carvel publish --final`: BakeFromLockfile + KilnBake + publishBaker := NewBaker() + publishBaker.SetWriter(GinkgoWriter) + err = publishBaker.BakeFromLockfile(inputPath, lockfilePath) + Expect(err).NotTo(HaveOccurred()) + + publishTile := filepath.Join(tmpRoot, "publish.pivotal") + err = publishBaker.KilnBake(publishTile) + Expect(err).NotTo(HaveOccurred()) + + publishChecksum := fileChecksum(publishTile) + + // Simulate `kiln carvel rebake`: BakeFromLockfile + KilnBake + rebakeBaker := NewBaker() + rebakeBaker.SetWriter(GinkgoWriter) + err = rebakeBaker.BakeFromLockfile(inputPath, lockfilePath) + Expect(err).NotTo(HaveOccurred()) + + rebakeTile := filepath.Join(tmpRoot, "rebake.pivotal") + err = rebakeBaker.KilnBake(rebakeTile) + Expect(err).NotTo(HaveOccurred()) + + rebakeChecksum := fileChecksum(rebakeTile) + + Expect(rebakeChecksum).To(Equal(publishChecksum), + "publish and rebake should produce identical tiles when using the same cached BOSH release tarball") + }) + }) + Context("generateManifestTemplate with different entry names", func() { It("parameterizes the entry name throughout the template", func() { template := generateManifestTemplate("my-custom-pkg") diff --git a/internal/commands/carvel_bake_test.go b/internal/commands/carvel_bake_test.go index e16e78673..da293323e 100644 --- a/internal/commands/carvel_bake_test.go +++ b/internal/commands/carvel_bake_test.go @@ -1,6 +1,7 @@ package commands_test import ( + "io" "log" "os" "os/exec" @@ -21,6 +22,17 @@ func kilnInstalled() bool { return err == nil } +func copyFile(src, dst string) { + in, err := os.Open(src) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + defer func() { _ = in.Close() }() + out, err := os.Create(dst) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + defer func() { _ = out.Close() }() + _, err = io.Copy(out, in) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) +} + var _ = Describe("CarvelBake", func() { var ( outLogger *log.Logger diff --git a/internal/commands/carvel_publish.go b/internal/commands/carvel_publish.go index 8e5d06f31..28e737f28 100644 --- a/internal/commands/carvel_publish.go +++ b/internal/commands/carvel_publish.go @@ -26,7 +26,7 @@ type CarvelPublishOptions struct { SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` Version string ` long:"version" description:"tile version for the final release"` - Lockfile string `short:"l" long:"lockfile" description:"path to Kilnfile.lock for using a cached BOSH release"` + Lockfile string `short:"l" long:"lockfile" description:"path to Kilnfile.lock (auto-detected in source directory if not specified)"` Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` IsFinal bool ` long:"final" description:"create a bake record for this build"` } @@ -62,27 +62,29 @@ func (c CarvelPublish) Execute(args []string) error { return fmt.Errorf("failed to resolve output file path: %w", err) } + lockfilePath := c.Options.Lockfile + if lockfilePath == "" { + lockfilePath = filepath.Join(sourcePath, "Kilnfile.lock") + } else { + lockfilePath, err = filepath.Abs(lockfilePath) + if err != nil { + return fmt.Errorf("failed to resolve lockfile path: %w", err) + } + } + + if _, statErr := os.Stat(lockfilePath); statErr != nil { + return fmt.Errorf("Kilnfile.lock not found at %s: run 'kiln carvel upload' first to create the BOSH release and lockfile", lockfilePath) + } + b := carvel.NewBaker() if c.Options.Verbose { b.SetWriter(os.Stdout) } - if c.Options.Lockfile != "" { - lockfilePath, err := filepath.Abs(c.Options.Lockfile) - if err != nil { - return fmt.Errorf("failed to resolve lockfile path: %w", err) - } - c.outLogger.Printf("Publishing Carvel tile from %s using lockfile %s", sourcePath, lockfilePath) - err = b.BakeFromLockfile(sourcePath, lockfilePath) - if err != nil { - return fmt.Errorf("failed to prepare Carvel tile from lockfile: %w", err) - } - } else { - c.outLogger.Printf("Publishing Carvel tile from %s", sourcePath) - err = b.Bake(sourcePath) - if err != nil { - return fmt.Errorf("failed to prepare Carvel tile: %w", err) - } + c.outLogger.Printf("Publishing Carvel tile from %s using lockfile %s", sourcePath, lockfilePath) + err = b.BakeFromLockfile(sourcePath, lockfilePath) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile from lockfile: %w", err) } ver, err := b.GetVersion() @@ -142,7 +144,7 @@ func (c CarvelPublish) Execute(args []string) error { func (c CarvelPublish) Usage() jhanda.Usage { return jhanda.Usage{ - Description: "Publishes a Carvel/Kubernetes tile as a .pivotal file. When --final is specified, creates a bake record that can be used with 'kiln carvel rebake' for reproducible builds.", + Description: "Publishes a Carvel/Kubernetes tile as a .pivotal file using the cached BOSH release from Kilnfile.lock. Run 'kiln carvel upload' first to build and cache the release. When --final is specified, creates a bake record that can be used with 'kiln carvel rebake' for reproducible builds.", ShortDescription: "publishes a Carvel/Kubernetes tile", Flags: c.Options, } diff --git a/internal/commands/carvel_publish_test.go b/internal/commands/carvel_publish_test.go index 844addd17..6081a9a9d 100644 --- a/internal/commands/carvel_publish_test.go +++ b/internal/commands/carvel_publish_test.go @@ -9,6 +9,8 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/carvel/models" "github.com/pivotal-cf/kiln/internal/commands" "github.com/pivotal-cf/kiln/pkg/bake" ) @@ -43,7 +45,23 @@ var _ = Describe("CarvelPublish", func() { }) }) - When("--final flag is used", func() { + When("no Kilnfile.lock exists", func() { + It("returns an error telling the user to run upload first", func() { + tmpDir, err := os.MkdirTemp("", "publish-no-lock-*") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = os.RemoveAll(tmpDir) }() + + err = command.Execute([]string{ + "--source-directory", tmpDir, + "--output-file", filepath.Join(tmpDir, "out.pivotal"), + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Kilnfile.lock not found")) + Expect(err.Error()).To(ContainSubstring("kiln carvel upload")) + }) + }) + + When("--final flag is used with a lockfile", func() { var ( inputPath string outputPath string @@ -75,6 +93,29 @@ var _ = Describe("CarvelPublish", func() { Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) } + // Simulate `kiln carvel upload`: bake to produce a BOSH release, + // then create a Kilnfile.lock pointing to the cached tarball. + baker := carvel.NewBaker() + baker.SetWriter(GinkgoWriter) + err = baker.Bake(inputPath) + Expect(err).NotTo(HaveOccurred()) + + tarball, err := baker.GetReleaseTarball() + Expect(err).NotTo(HaveOccurred()) + + cachedTarball := filepath.Join(filepath.Dir(inputPath), "cached-release.tgz") + copyFile(tarball, cachedTarball) + + lf := models.CarvelLockfile{ + Release: models.CarvelReleaseLock{ + Name: "k8s-tile-test", + Version: "0.1.1", + RemotePath: cachedTarball, + SHA256: "test-sha", + }, + } + Expect(lf.WriteFile(filepath.Join(inputPath, "Kilnfile.lock"))).To(Succeed()) + outputPath = filepath.Join(filepath.Dir(inputPath), "output.pivotal") }) @@ -94,7 +135,6 @@ var _ = Describe("CarvelPublish", func() { Expect(err).NotTo(HaveOccurred()) Expect(outputPath).To(BeAnExistingFile()) - // Resolve symlinks (macOS /var -> /private/var) resolvedInput, resolveErr := filepath.EvalSymlinks(inputPath) if resolveErr != nil { resolvedInput = inputPath From 535041d2c515029f56dc3fe8b71b3dd80bd26fd6 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Tue, 17 Mar 2026 16:43:38 -0500 Subject: [PATCH 08/18] Use as the canonical subcommand --- internal/carvel/baker_test.go | 2 +- internal/commands/carvel.go | 15 +++++++++++++-- internal/commands/carvel_publish.go | 2 +- main.go | 1 + 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/internal/carvel/baker_test.go b/internal/carvel/baker_test.go index 2d3c0df80..dbc6c28f8 100644 --- a/internal/carvel/baker_test.go +++ b/internal/carvel/baker_test.go @@ -477,7 +477,7 @@ var _ = Describe("Carvel Baker", func() { publishChecksum := fileChecksum(publishTile) - // Simulate `kiln carvel rebake`: BakeFromLockfile + KilnBake + // Simulate `kiln carvel re-bake`: BakeFromLockfile + KilnBake rebakeBaker := NewBaker() rebakeBaker.SetWriter(GinkgoWriter) err = rebakeBaker.BakeFromLockfile(inputPath, lockfilePath) diff --git a/internal/commands/carvel.go b/internal/commands/carvel.go index 1b71b1c48..db6628443 100644 --- a/internal/commands/carvel.go +++ b/internal/commands/carvel.go @@ -14,6 +14,7 @@ type Carvel struct { outLogger *log.Logger errLogger *log.Logger commands jhanda.CommandSet + aliases map[string]bool } func NewCarvel(outLogger, errLogger *log.Logger) Carvel { @@ -21,13 +22,18 @@ func NewCarvel(outLogger, errLogger *log.Logger) Carvel { outLogger: outLogger, errLogger: errLogger, commands: jhanda.CommandSet{}, + aliases: map[string]bool{}, } // Register subcommands c.commands["bake"] = NewCarvelBake(outLogger, errLogger) c.commands["upload"] = NewCarvelUpload(outLogger, errLogger) c.commands["publish"] = NewCarvelPublish(outLogger, errLogger) - c.commands["rebake"] = NewCarvelReBake(outLogger, errLogger) + c.commands["re-bake"] = NewCarvelReBake(outLogger, errLogger) + + // Aliases (hidden from help output) + c.commands["rebake"] = c.commands["re-bake"] + c.aliases["rebake"] = true return c } @@ -59,7 +65,6 @@ func (c Carvel) Execute(args []string) error { } func (c Carvel) Usage() jhanda.Usage { - // Build subcommand list for the description var subcommandList strings.Builder subcommandList.WriteString("Commands for working with Carvel/Kubernetes tiles.\n\n") subcommandList.WriteString("Subcommands:\n") @@ -67,6 +72,9 @@ func (c Carvel) Usage() jhanda.Usage { var names []string var length int for name := range c.commands { + if c.aliases[name] { + continue + } names = append(names, name) if len(name) > length { length = len(name) @@ -95,6 +103,9 @@ func (c Carvel) printHelp() error { ) for name := range c.commands { + if c.aliases[name] { + continue + } names = append(names, name) if len(name) > length { length = len(name) diff --git a/internal/commands/carvel_publish.go b/internal/commands/carvel_publish.go index 28e737f28..735889621 100644 --- a/internal/commands/carvel_publish.go +++ b/internal/commands/carvel_publish.go @@ -144,7 +144,7 @@ func (c CarvelPublish) Execute(args []string) error { func (c CarvelPublish) Usage() jhanda.Usage { return jhanda.Usage{ - Description: "Publishes a Carvel/Kubernetes tile as a .pivotal file using the cached BOSH release from Kilnfile.lock. Run 'kiln carvel upload' first to build and cache the release. When --final is specified, creates a bake record that can be used with 'kiln carvel rebake' for reproducible builds.", + Description: "Publishes a Carvel/Kubernetes tile as a .pivotal file using the cached BOSH release from Kilnfile.lock. Run 'kiln carvel upload' first to build and cache the release. When --final is specified, creates a bake record that can be used with 'kiln carvel re-bake' for reproducible builds.", ShortDescription: "publishes a Carvel/Kubernetes tile", Flags: c.Options, } diff --git a/main.go b/main.go index 6cf14562d..339871f9f 100644 --- a/main.go +++ b/main.go @@ -76,6 +76,7 @@ func main() { bakeCommand.KilnVersion = version commandSet["bake"] = bakeCommand commandSet["re-bake"] = commands.NewReBake(bakeCommand) + commandSet["rebake"] = commandSet["re-bake"] commandSet["test"] = commands.NewTileTest() commandSet["help"] = commands.NewHelp(os.Stdout, globalFlagsUsage, commandSet) From c1ddccf09acc08c34240fac03ac55ef7c122effb Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Tue, 17 Mar 2026 16:57:32 -0500 Subject: [PATCH 09/18] Show help text when using carvel subcommands --- internal/commands/carvel.go | 5 +++-- main.go | 15 ++++++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/internal/commands/carvel.go b/internal/commands/carvel.go index db6628443..0a72b4985 100644 --- a/internal/commands/carvel.go +++ b/internal/commands/carvel.go @@ -53,8 +53,9 @@ func (c Carvel) Execute(args []string) error { return c.printHelp() } - // Check if subargs contains help flags - this handles cases like - // "kiln carvel bake --help" where --help was passed through from main + // Must intercept help flags here before delegating to CommandSet.Execute, + // which would otherwise look for a nonexistent "help" command in the + // carvel command set. for _, arg := range subargs { if arg == "-h" || arg == "--help" || arg == "help" { return c.printSubcommandHelp(subcommand) diff --git a/main.go b/main.go index 339871f9f..d4f34ab0c 100644 --- a/main.go +++ b/main.go @@ -46,7 +46,11 @@ func main() { } if global.Help { - command = "help" + if command == "carvel" && len(args) > 0 { + args = append(args, "--help") + } else { + command = "help" + } } if command == "" { @@ -102,9 +106,14 @@ func main() { log.Fatal(err) } - commandSet["carvel"] = commands.NewCarvel(outLogger, errLogger) + carvelCommand := commands.NewCarvel(outLogger, errLogger) + commandSet["carvel"] = carvelCommand - err = commandSet.Execute(command, args) + if command == "carvel" { + err = carvelCommand.Execute(args) + } else { + err = commandSet.Execute(command, args) + } if err != nil { log.Fatal(err) } From a873f96a7fae100dfbe67e2b9ff2ebde84e620b8 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Tue, 17 Mar 2026 17:22:53 -0500 Subject: [PATCH 10/18] Fix help text so it shows positional args --- internal/commands/carvel.go | 14 +++++++++++++- internal/commands/carvel_rebake.go | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/commands/carvel.go b/internal/commands/carvel.go index 0a72b4985..20d816bf2 100644 --- a/internal/commands/carvel.go +++ b/internal/commands/carvel.go @@ -15,6 +15,7 @@ type Carvel struct { errLogger *log.Logger commands jhanda.CommandSet aliases map[string]bool + synopses map[string]string } func NewCarvel(outLogger, errLogger *log.Logger) Carvel { @@ -23,6 +24,7 @@ func NewCarvel(outLogger, errLogger *log.Logger) Carvel { errLogger: errLogger, commands: jhanda.CommandSet{}, aliases: map[string]bool{}, + synopses: map[string]string{}, } // Register subcommands @@ -31,9 +33,13 @@ func NewCarvel(outLogger, errLogger *log.Logger) Carvel { c.commands["publish"] = NewCarvelPublish(outLogger, errLogger) c.commands["re-bake"] = NewCarvelReBake(outLogger, errLogger) + // Positional argument synopses for usage lines + c.synopses["re-bake"] = "" + // Aliases (hidden from help output) c.commands["rebake"] = c.commands["re-bake"] c.aliases["rebake"] = true + c.synopses["rebake"] = c.synopses["re-bake"] return c } @@ -142,7 +148,13 @@ func (c Carvel) printSubcommandHelp(subcommand string) error { fmt.Println() fmt.Println(usage.Description) fmt.Println() - fmt.Printf("Usage: kiln carvel %s []\n", subcommand) + + synopsis := c.synopses[subcommand] + if synopsis != "" { + fmt.Printf("Usage: kiln carvel %s %s []\n", subcommand, synopsis) + } else { + fmt.Printf("Usage: kiln carvel %s []\n", subcommand) + } if usage.Flags != nil { flagUsage, err := jhanda.PrintUsage(usage.Flags) diff --git a/internal/commands/carvel_rebake.go b/internal/commands/carvel_rebake.go index d948c21b8..763e039f2 100644 --- a/internal/commands/carvel_rebake.go +++ b/internal/commands/carvel_rebake.go @@ -117,7 +117,7 @@ func (c CarvelReBake) Execute(args []string) error { func (c CarvelReBake) Usage() jhanda.Usage { return jhanda.Usage{ - Description: "Re-bakes a Carvel tile from a bake record for reproducible builds. The repository must be checked out at the source_revision specified in the bake record.", + Description: "Re-bakes a Carvel tile from a bake record for reproducible builds.\nThe repository must be checked out at the source_revision specified in the bake record.\n\nThe argument is the path to a JSON bake record file produced by 'kiln carvel publish --final'.", ShortDescription: "re-bakes a Carvel tile from a bake record", Flags: c.Options, } From 3257349fe186d58daa52888f64ee3013c06da10d Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Wed, 18 Mar 2026 14:31:42 -0500 Subject: [PATCH 11/18] Add the ability to pass credentials using the same format as regular kiln --- internal/carvel/baker.go | 22 +-- internal/carvel/baker_test.go | 62 ++----- internal/carvel/testdata/sample-tile/Kilnfile | 8 + internal/commands/carvel_bake.go | 51 +++--- internal/commands/carvel_helpers.go | 152 ++++++++++++++++++ internal/commands/carvel_publish.go | 61 ++++--- internal/commands/carvel_publish_test.go | 79 +++++++-- internal/commands/carvel_rebake.go | 31 +++- internal/commands/carvel_upload.go | 100 +++++++----- internal/commands/carvel_upload_test.go | 68 +++++--- 10 files changed, 450 insertions(+), 184 deletions(-) create mode 100644 internal/carvel/testdata/sample-tile/Kilnfile create mode 100644 internal/commands/carvel_helpers.go diff --git a/internal/carvel/baker.go b/internal/carvel/baker.go index e0f78b79d..e2f5bb170 100644 --- a/internal/carvel/baker.go +++ b/internal/carvel/baker.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/pivotal-cf/kiln/internal/carvel/models" + "github.com/pivotal-cf/kiln/pkg/cargo" "github.com/hashicorp/go-version" "gopkg.in/yaml.v3" @@ -21,7 +22,7 @@ import ( // and kiln-compatible tile structure that can be baked into a .pivotal file. type Baker interface { Bake(source string) error - BakeFromLockfile(source string, lockfilePath string) error + BakeFromLockfile(source string, releaseLock cargo.BOSHReleaseTarballLock, localTarball string) error KilnBake(destination string) error GetName() string GetVersion() (string, error) @@ -111,7 +112,7 @@ func (b *baker) Bake(source string) error { return nil } -func (b *baker) BakeFromLockfile(source string, lockfilePath string) error { +func (b *baker) BakeFromLockfile(source string, releaseLock cargo.BOSHReleaseTarballLock, localTarball string) error { b.source = source b.destination = path.Join(source, ".carvel-tile") @@ -133,14 +134,8 @@ func (b *baker) BakeFromLockfile(source string, lockfilePath string) error { } b.progress(fmt.Sprintf("Tile: %s version %s (metadata_version %s)", b.metadata.Name, ver, b.metadata.MetadataVersion)) - b.progress("Reading lockfile from " + lockfilePath) - lf, err := models.ReadCarvelLockfile(lockfilePath) - if err != nil { - return fmt.Errorf("failed to read lockfile: %w", err) - } - - if lf.Release.Name != b.metadata.Name { - return fmt.Errorf("lockfile release name %q does not match tile name %q", lf.Release.Name, b.metadata.Name) + if releaseLock.Name != b.metadata.Name { + return fmt.Errorf("lockfile release name %q does not match tile name %q", releaseLock.Name, b.metadata.Name) } err = os.RemoveAll(b.destination) @@ -180,12 +175,11 @@ func (b *baker) BakeFromLockfile(source string, lockfilePath string) error { return err } - cachedTarball := lf.Release.RemotePath destTarball := path.Join(releasesDir, b.metadata.Name+"-"+ver+".tgz") - b.progress("Copying cached BOSH release from " + cachedTarball) - b.log("copying cached BOSH release from " + cachedTarball) - err = copyFileContents(cachedTarball, destTarball) + b.progress("Copying cached BOSH release from " + localTarball) + b.log("copying cached BOSH release from " + localTarball) + err = copyFileContents(localTarball, destTarball) if err != nil { return fmt.Errorf("failed to copy cached release tarball: %w", err) } diff --git a/internal/carvel/baker_test.go b/internal/carvel/baker_test.go index dbc6c28f8..9e590786c 100644 --- a/internal/carvel/baker_test.go +++ b/internal/carvel/baker_test.go @@ -13,6 +13,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/pivotal-cf/kiln/internal/carvel/models" + "github.com/pivotal-cf/kiln/pkg/cargo" "gopkg.in/yaml.v3" ) @@ -307,7 +308,7 @@ var _ = Describe("Carvel Baker", func() { }) Context("BakeFromLockfile", func() { - When("a valid lockfile references a pre-built release", func() { + When("a valid release lock references a pre-built release", func() { BeforeEach(func() { if !boshInstalled() { Skip("bosh CLI not installed - skipping integration test") @@ -334,7 +335,6 @@ var _ = Describe("Carvel Baker", func() { Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) } - // First do a normal bake to produce a real BOSH release tarball subject := NewBaker() subject.SetWriter(GinkgoWriter) err = subject.Bake(inputPath) @@ -343,28 +343,18 @@ var _ = Describe("Carvel Baker", func() { tarball, err := subject.GetReleaseTarball() Expect(err).NotTo(HaveOccurred()) - // Copy the tarball to a temp location (simulating Artifactory cache) cachedTarball := filepath.Join(filepath.Dir(inputPath), "cached-release.tgz") err = copyTestFile(tarball, cachedTarball) Expect(err).NotTo(HaveOccurred()) - // Write a lockfile pointing to the cached tarball - lf := models.CarvelLockfile{ - Release: models.CarvelReleaseLock{ - Name: "k8s-tile-test", - Version: "0.1.1", - RemotePath: cachedTarball, - SHA256: "test-sha", - }, + releaseLock := cargo.BOSHReleaseTarballLock{ + Name: "k8s-tile-test", + Version: "0.1.1", } - lockfilePath := filepath.Join(filepath.Dir(inputPath), "Kilnfile.lock") - err = lf.WriteFile(lockfilePath) - Expect(err).NotTo(HaveOccurred()) - // Now bake from lockfile subject2 := NewBaker() subject2.SetWriter(GinkgoWriter) - err = subject2.BakeFromLockfile(inputPath, lockfilePath) + err = subject2.BakeFromLockfile(inputPath, releaseLock, cachedTarball) Expect(err).NotTo(HaveOccurred()) outputPath := path.Join(inputPath, ".carvel-tile") @@ -374,7 +364,7 @@ var _ = Describe("Carvel Baker", func() { }) }) - When("the lockfile release name does not match", func() { + When("the release lock name does not match", func() { It("returns an error", func() { inputPath, err := os.MkdirTemp("", "lockfile-mismatch-*") Expect(err).NotTo(HaveOccurred()) @@ -384,18 +374,13 @@ var _ = Describe("Carvel Baker", func() { err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) Expect(err).NotTo(HaveOccurred()) - lf := models.CarvelLockfile{ - Release: models.CarvelReleaseLock{ - Name: "wrong-name", - Version: "0.1.1", - }, + releaseLock := cargo.BOSHReleaseTarballLock{ + Name: "wrong-name", + Version: "0.1.1", } - lockfilePath := filepath.Join(filepath.Dir(inputPath), "Kilnfile.lock") - err = lf.WriteFile(lockfilePath) - Expect(err).NotTo(HaveOccurred()) subject := NewBaker() - err = subject.BakeFromLockfile(inputPath, lockfilePath) + err = subject.BakeFromLockfile(inputPath, releaseLock, "/nonexistent/tarball.tgz") Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("does not match tile name")) }) @@ -413,10 +398,7 @@ var _ = Describe("Carvel Baker", func() { }) Context("rebake reproducibility", func() { - // Both publish and rebake now use BakeFromLockfile with the same - // cached BOSH release tarball. This test verifies the resulting - // .pivotal files are byte-for-byte identical. - It("publish and rebake produce identical tiles when using the same lockfile", func() { + It("publish and rebake produce identical tiles when using the same cached release", func() { if !boshInstalled() { Skip("bosh CLI not installed") } @@ -442,8 +424,6 @@ var _ = Describe("Carvel Baker", func() { Expect(err).NotTo(HaveOccurred(), "git setup: "+string(out)) } - // Simulate `kiln carvel upload`: Bake() to produce a BOSH release, - // then cache the tarball and write a Kilnfile.lock. uploadBaker := NewBaker() uploadBaker.SetWriter(GinkgoWriter) err = uploadBaker.Bake(inputPath) @@ -454,21 +434,14 @@ var _ = Describe("Carvel Baker", func() { cachedTarball := filepath.Join(tmpRoot, "cached-release.tgz") Expect(copyTestFile(uploadTarball, cachedTarball)).To(Succeed()) - lf := models.CarvelLockfile{ - Release: models.CarvelReleaseLock{ - Name: "k8s-tile-test", - Version: "0.1.1", - RemotePath: cachedTarball, - SHA256: "unused", - }, + releaseLock := cargo.BOSHReleaseTarballLock{ + Name: "k8s-tile-test", + Version: "0.1.1", } - lockfilePath := filepath.Join(tmpRoot, "Kilnfile.lock") - Expect(lf.WriteFile(lockfilePath)).To(Succeed()) - // Simulate `kiln carvel publish --final`: BakeFromLockfile + KilnBake publishBaker := NewBaker() publishBaker.SetWriter(GinkgoWriter) - err = publishBaker.BakeFromLockfile(inputPath, lockfilePath) + err = publishBaker.BakeFromLockfile(inputPath, releaseLock, cachedTarball) Expect(err).NotTo(HaveOccurred()) publishTile := filepath.Join(tmpRoot, "publish.pivotal") @@ -477,10 +450,9 @@ var _ = Describe("Carvel Baker", func() { publishChecksum := fileChecksum(publishTile) - // Simulate `kiln carvel re-bake`: BakeFromLockfile + KilnBake rebakeBaker := NewBaker() rebakeBaker.SetWriter(GinkgoWriter) - err = rebakeBaker.BakeFromLockfile(inputPath, lockfilePath) + err = rebakeBaker.BakeFromLockfile(inputPath, releaseLock, cachedTarball) Expect(err).NotTo(HaveOccurred()) rebakeTile := filepath.Join(tmpRoot, "rebake.pivotal") diff --git a/internal/carvel/testdata/sample-tile/Kilnfile b/internal/carvel/testdata/sample-tile/Kilnfile new file mode 100644 index 000000000..dd877988a --- /dev/null +++ b/internal/carvel/testdata/sample-tile/Kilnfile @@ -0,0 +1,8 @@ +--- +release_sources: + - type: artifactory + artifactory_host: http://localhost:8080 + repo: test-repo + username: test-user + password: test-pass + path_template: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz" diff --git a/internal/commands/carvel_bake.go b/internal/commands/carvel_bake.go index d1ca4c272..f60c08463 100644 --- a/internal/commands/carvel_bake.go +++ b/internal/commands/carvel_bake.go @@ -8,6 +8,7 @@ import ( "github.com/pivotal-cf/jhanda" "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/commands/flags" ) type CarvelBake struct { @@ -17,10 +18,10 @@ type CarvelBake struct { } type CarvelBakeOptions struct { + flags.Standard SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` - Lockfile string `short:"l" long:"lockfile" description:"path to Kilnfile.lock for using a cached BOSH release"` - Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` } func NewCarvelBake(outLogger, errLogger *log.Logger) CarvelBake { @@ -36,17 +37,9 @@ func (c CarvelBake) Execute(args []string) error { return err } - sourcePath := c.Options.SourceDirectory - if sourcePath == "" { - sourcePath, err = os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - } else { - sourcePath, err = filepath.Abs(sourcePath) - if err != nil { - return fmt.Errorf("failed to resolve source directory: %w", err) - } + sourcePath, err := resolveSourcePath(c.Options.SourceDirectory) + if err != nil { + return err } targetPath, err := filepath.Abs(c.Options.OutputFile) @@ -60,12 +53,32 @@ func (c CarvelBake) Execute(args []string) error { baker.SetWriter(os.Stdout) } - if c.Options.Lockfile != "" { - lockfilePath, err := filepath.Abs(c.Options.Lockfile) - if err != nil { - return fmt.Errorf("failed to resolve lockfile path: %w", err) + kilnfilePath := resolveKilnfilePath(c.Options.Kilnfile, sourcePath) + lockfilePath := kilnfilePath + ".lock" + if _, statErr := os.Stat(lockfilePath); statErr == nil { + c.Options.Kilnfile = kilnfilePath + kilnfile, kilnfileLock, loadErr := c.Options.Standard.LoadKilnfiles(nil, nil) + if loadErr != nil { + return fmt.Errorf("failed to load Kilnfiles: %w", loadErr) } - err = baker.BakeFromLockfile(sourcePath, lockfilePath) + + if len(kilnfileLock.Releases) == 0 { + return fmt.Errorf("Kilnfile.lock has no releases") + } + releaseLock := kilnfileLock.Releases[0] + + tmpDir, tmpErr := os.MkdirTemp("", "carvel-bake-*") + if tmpErr != nil { + return fmt.Errorf("failed to create temp directory: %w", tmpErr) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + localTarball, dlErr := downloadCarvelRelease(c.outLogger, kilnfile, kilnfileLock, tmpDir) + if dlErr != nil { + return fmt.Errorf("failed to download release from Artifactory: %w", dlErr) + } + + err = baker.BakeFromLockfile(sourcePath, releaseLock, localTarball) if err != nil { return fmt.Errorf("failed to prepare Carvel tile from lockfile: %w", err) } @@ -92,7 +105,7 @@ func (c CarvelBake) Execute(args []string) error { func (c CarvelBake) Usage() jhanda.Usage { return jhanda.Usage{ - Description: "Bakes a Carvel/Kubernetes tile into a .pivotal file. This command transforms a Kubernetes tile (using imgpkg bundles and Carvel packages) into a BOSH-compatible format, then bakes it into a .pivotal file that can be consumed by Operations Manager.", + Description: "Bakes a Carvel/Kubernetes tile into a .pivotal file. This command transforms a Kubernetes tile (using imgpkg bundles and Carvel packages) into a BOSH-compatible format, then bakes it into a .pivotal file that can be consumed by Operations Manager. When a Kilnfile.lock is present, it downloads the cached BOSH release from Artifactory instead of regenerating it locally.", ShortDescription: "bakes a Carvel/Kubernetes tile", Flags: c.Options, } diff --git a/internal/commands/carvel_helpers.go b/internal/commands/carvel_helpers.go new file mode 100644 index 000000000..fe626ac52 --- /dev/null +++ b/internal/commands/carvel_helpers.go @@ -0,0 +1,152 @@ +package commands + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/go-git/go-billy/v5/osfs" + "github.com/pivotal-cf/kiln/internal/baking" + "github.com/pivotal-cf/kiln/internal/commands/flags" + "github.com/pivotal-cf/kiln/internal/component" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" +) + +func loadKilnfileOnly(options flags.Standard) (cargo.Kilnfile, error) { + fs := osfs.New("") + variablesService := baking.NewTemplateVariablesService(fs) + + templateVariables, err := variablesService.FromPathsAndPairs(options.VariableFiles, options.Variables) + if err != nil { + return cargo.Kilnfile{}, fmt.Errorf("failed to parse template variables: %w", err) + } + + kilnfileFP, err := fs.Open(options.Kilnfile) + if err != nil { + return cargo.Kilnfile{}, fmt.Errorf("failed to open Kilnfile: %w", err) + } + defer func() { _ = kilnfileFP.Close() }() + + return cargo.InterpolateAndParseKilnfile(kilnfileFP, templateVariables) +} + +func findArtifactorySource(kilnfile cargo.Kilnfile) (cargo.ReleaseSourceConfig, error) { + for _, src := range kilnfile.ReleaseSources { + if src.Type == cargo.BOSHReleaseTarballSourceTypeArtifactory { + return src, nil + } + } + return cargo.ReleaseSourceConfig{}, fmt.Errorf("no artifactory release source found in Kilnfile") +} + +func downloadCarvelRelease(logger *log.Logger, kilnfile cargo.Kilnfile, lock cargo.KilnfileLock, destDir string) (string, error) { + if len(lock.Releases) == 0 { + return "", fmt.Errorf("Kilnfile.lock has no releases") + } + + releaseLock := lock.Releases[0] + sources := component.NewReleaseSourceRepo(kilnfile) + + logger.Printf("Downloading %s %s from %s", releaseLock.Name, releaseLock.Version, releaseLock.RemoteSource) + local, err := sources.DownloadRelease(destDir, releaseLock) + if err != nil { + return "", fmt.Errorf("failed to download release: %w", err) + } + + if releaseLock.SHA1 != "" && local.Lock.SHA1 != releaseLock.SHA1 { + _ = os.Remove(local.LocalPath) + return "", fmt.Errorf("downloaded release %q had incorrect SHA1 - expected %q, got %q", local.LocalPath, releaseLock.SHA1, local.Lock.SHA1) + } + + return local.LocalPath, nil +} + +func writeStandardKilnfileLock(lockfilePath string, releaseName, releaseVersion, remotePath, remoteSourceID, sha1 string) error { + lock := cargo.KilnfileLock{ + Releases: []cargo.BOSHReleaseTarballLock{ + { + Name: releaseName, + Version: releaseVersion, + RemotePath: remotePath, + RemoteSource: remoteSourceID, + SHA1: sha1, + }, + }, + Stemcell: cargo.Stemcell{ + OS: "ubuntu-jammy", + Version: "1.446", + }, + } + + data, err := yaml.Marshal(&lock) + if err != nil { + return fmt.Errorf("failed to marshal Kilnfile.lock: %w", err) + } + return os.WriteFile(lockfilePath, data, 0644) +} + +func readStandardKilnfileLock(lockfilePath string) (cargo.KilnfileLock, error) { + data, err := os.ReadFile(lockfilePath) + if err != nil { + return cargo.KilnfileLock{}, fmt.Errorf("failed to read Kilnfile.lock: %w", err) + } + var lock cargo.KilnfileLock + if err := yaml.Unmarshal(data, &lock); err != nil { + return cargo.KilnfileLock{}, fmt.Errorf("failed to parse Kilnfile.lock: %w", err) + } + return lock, nil +} + +func generateKilnfile(kilnfilePath, artifactoryHost, repo, username, password, pathTemplate string) error { + if pathTemplate == "" { + pathTemplate = "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz" + } + kf := cargo.Kilnfile{ + ReleaseSources: []cargo.ReleaseSourceConfig{ + { + Type: cargo.BOSHReleaseTarballSourceTypeArtifactory, + ArtifactoryHost: artifactoryHost, + Repo: repo, + Username: username, + Password: password, + PathTemplate: pathTemplate, + }, + }, + } + + data, err := yaml.Marshal(&kf) + if err != nil { + return fmt.Errorf("failed to marshal Kilnfile: %w", err) + } + return os.WriteFile(kilnfilePath, data, 0644) +} + +func resolveKilnfilePath(kilnfilePath, sourcePath string) string { + if kilnfilePath == "" || kilnfilePath == "Kilnfile" { + return filepath.Join(sourcePath, "Kilnfile") + } + abs, err := filepath.Abs(kilnfilePath) + if err != nil { + return kilnfilePath + } + return abs +} + +func resolveSourcePath(sourcePath string) (string, error) { + if sourcePath == "" { + var err error + sourcePath, err = os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current directory: %w", err) + } + } else { + var err error + sourcePath, err = filepath.Abs(sourcePath) + if err != nil { + return "", fmt.Errorf("failed to resolve source directory: %w", err) + } + } + return sourcePath, nil +} diff --git a/internal/commands/carvel_publish.go b/internal/commands/carvel_publish.go index 735889621..3e0c3b2b5 100644 --- a/internal/commands/carvel_publish.go +++ b/internal/commands/carvel_publish.go @@ -12,6 +12,7 @@ import ( "github.com/pivotal-cf/jhanda" "github.com/pivotal-cf/kiln/internal/builder" "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/commands/flags" "github.com/pivotal-cf/kiln/pkg/bake" ) @@ -23,12 +24,12 @@ type CarvelPublish struct { } type CarvelPublishOptions struct { + flags.Standard SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` Version string ` long:"version" description:"tile version for the final release"` - Lockfile string `short:"l" long:"lockfile" description:"path to Kilnfile.lock (auto-detected in source directory if not specified)"` - Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` IsFinal bool ` long:"final" description:"create a bake record for this build"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` } func NewCarvelPublish(outLogger, errLogger *log.Logger) CarvelPublish { @@ -44,17 +45,9 @@ func (c CarvelPublish) Execute(args []string) error { return err } - sourcePath := c.Options.SourceDirectory - if sourcePath == "" { - sourcePath, err = os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - } else { - sourcePath, err = filepath.Abs(sourcePath) - if err != nil { - return fmt.Errorf("failed to resolve source directory: %w", err) - } + sourcePath, err := resolveSourcePath(c.Options.SourceDirectory) + if err != nil { + return err } targetPath, err := filepath.Abs(c.Options.OutputFile) @@ -62,27 +55,46 @@ func (c CarvelPublish) Execute(args []string) error { return fmt.Errorf("failed to resolve output file path: %w", err) } - lockfilePath := c.Options.Lockfile - if lockfilePath == "" { - lockfilePath = filepath.Join(sourcePath, "Kilnfile.lock") - } else { - lockfilePath, err = filepath.Abs(lockfilePath) - if err != nil { - return fmt.Errorf("failed to resolve lockfile path: %w", err) - } + kilnfilePath := resolveKilnfilePath(c.Options.Kilnfile, sourcePath) + + if _, statErr := os.Stat(kilnfilePath); statErr != nil { + return fmt.Errorf("Kilnfile not found at %s: run 'kiln carvel upload' first to create the BOSH release, Kilnfile, and Kilnfile.lock", kilnfilePath) } + lockfilePath := kilnfilePath + ".lock" if _, statErr := os.Stat(lockfilePath); statErr != nil { return fmt.Errorf("Kilnfile.lock not found at %s: run 'kiln carvel upload' first to create the BOSH release and lockfile", lockfilePath) } + c.Options.Kilnfile = kilnfilePath + kilnfile, kilnfileLock, err := c.Options.Standard.LoadKilnfiles(nil, nil) + if err != nil { + return fmt.Errorf("failed to load Kilnfiles: %w", err) + } + + if len(kilnfileLock.Releases) == 0 { + return fmt.Errorf("Kilnfile.lock has no releases: run 'kiln carvel upload' first") + } + releaseLock := kilnfileLock.Releases[0] + + tmpDir, err := os.MkdirTemp("", "carvel-publish-*") + if err != nil { + return fmt.Errorf("failed to create temp directory: %w", err) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + c.outLogger.Printf("Publishing Carvel tile from %s", sourcePath) + localTarball, err := downloadCarvelRelease(c.outLogger, kilnfile, kilnfileLock, tmpDir) + if err != nil { + return fmt.Errorf("failed to download release from Artifactory: %w", err) + } + b := carvel.NewBaker() if c.Options.Verbose { b.SetWriter(os.Stdout) } - c.outLogger.Printf("Publishing Carvel tile from %s using lockfile %s", sourcePath, lockfilePath) - err = b.BakeFromLockfile(sourcePath, lockfilePath) + err = b.BakeFromLockfile(sourcePath, releaseLock, localTarball) if err != nil { return fmt.Errorf("failed to prepare Carvel tile from lockfile: %w", err) } @@ -103,7 +115,6 @@ func (c CarvelPublish) Execute(args []string) error { c.outLogger.Printf("Baked %s version %s to %s", b.GetName(), ver, targetPath) if c.Options.IsFinal { - // Resolve symlinks so git's toplevel and our absolute path match (macOS /var -> /private/var) resolvedSourcePath, err := filepath.EvalSymlinks(sourcePath) if err != nil { resolvedSourcePath = sourcePath @@ -144,7 +155,7 @@ func (c CarvelPublish) Execute(args []string) error { func (c CarvelPublish) Usage() jhanda.Usage { return jhanda.Usage{ - Description: "Publishes a Carvel/Kubernetes tile as a .pivotal file using the cached BOSH release from Kilnfile.lock. Run 'kiln carvel upload' first to build and cache the release. When --final is specified, creates a bake record that can be used with 'kiln carvel re-bake' for reproducible builds.", + Description: "Downloads the cached BOSH release from Artifactory (using credentials from the Kilnfile) and bakes a Carvel/Kubernetes tile as a .pivotal file. Run 'kiln carvel upload' first to build and cache the release. When --final is specified, creates a bake record that can be used with 'kiln carvel re-bake' for reproducible builds.", ShortDescription: "publishes a Carvel/Kubernetes tile", Flags: c.Options, } diff --git a/internal/commands/carvel_publish_test.go b/internal/commands/carvel_publish_test.go index 6081a9a9d..8f7d35c5b 100644 --- a/internal/commands/carvel_publish_test.go +++ b/internal/commands/carvel_publish_test.go @@ -3,6 +3,8 @@ package commands_test import ( "encoding/json" "log" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" @@ -10,9 +12,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/pivotal-cf/kiln/internal/carvel" - "github.com/pivotal-cf/kiln/internal/carvel/models" "github.com/pivotal-cf/kiln/internal/commands" "github.com/pivotal-cf/kiln/pkg/bake" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" ) var _ = Describe("CarvelPublish", func() { @@ -45,9 +48,9 @@ var _ = Describe("CarvelPublish", func() { }) }) - When("no Kilnfile.lock exists", func() { + When("no Kilnfile exists", func() { It("returns an error telling the user to run upload first", func() { - tmpDir, err := os.MkdirTemp("", "publish-no-lock-*") + tmpDir, err := os.MkdirTemp("", "publish-no-kilnfile-*") Expect(err).NotTo(HaveOccurred()) defer func() { _ = os.RemoveAll(tmpDir) }() @@ -56,15 +59,16 @@ var _ = Describe("CarvelPublish", func() { "--output-file", filepath.Join(tmpDir, "out.pivotal"), }) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Kilnfile.lock not found")) + Expect(err.Error()).To(ContainSubstring("Kilnfile not found")) Expect(err.Error()).To(ContainSubstring("kiln carvel upload")) }) }) - When("--final flag is used with a lockfile", func() { + When("--final flag is used with a Kilnfile and lockfile", func() { var ( inputPath string outputPath string + server *httptest.Server ) BeforeEach(func() { @@ -93,8 +97,6 @@ var _ = Describe("CarvelPublish", func() { Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) } - // Simulate `kiln carvel upload`: bake to produce a BOSH release, - // then create a Kilnfile.lock pointing to the cached tarball. baker := carvel.NewBaker() baker.SetWriter(GinkgoWriter) err = baker.Bake(inputPath) @@ -103,18 +105,60 @@ var _ = Describe("CarvelPublish", func() { tarball, err := baker.GetReleaseTarball() Expect(err).NotTo(HaveOccurred()) - cachedTarball := filepath.Join(filepath.Dir(inputPath), "cached-release.tgz") - copyFile(tarball, cachedTarball) + tarballData, err := os.ReadFile(tarball) + Expect(err).NotTo(HaveOccurred()) + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(tarballData) + })) + + kf := cargo.Kilnfile{ + ReleaseSources: []cargo.ReleaseSourceConfig{ + { + Type: "artifactory", + ArtifactoryHost: server.URL, + Repo: "test-repo", + Username: "user", + Password: "pass", + PathTemplate: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz", + }, + }, + } + kfData, err := yaml.Marshal(&kf) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(inputPath, "Kilnfile"), kfData, 0644) + Expect(err).NotTo(HaveOccurred()) - lf := models.CarvelLockfile{ - Release: models.CarvelReleaseLock{ - Name: "k8s-tile-test", - Version: "0.1.1", - RemotePath: cachedTarball, - SHA256: "test-sha", + lock := cargo.KilnfileLock{ + Releases: []cargo.BOSHReleaseTarballLock{ + { + Name: "k8s-tile-test", + Version: "0.1.1", + RemotePath: "bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz", + RemoteSource: "artifactory", + SHA1: "", + }, }, + Stemcell: cargo.Stemcell{ + OS: "ubuntu-jammy", + Version: "1.446", + }, + } + lockData, err := yaml.Marshal(&lock) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(inputPath, "Kilnfile.lock"), lockData, 0644) + Expect(err).NotTo(HaveOccurred()) + + // Re-commit with the Kilnfile and Kilnfile.lock + for _, cmd := range []*exec.Cmd{ + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "add kilnfiles"), + } { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) } - Expect(lf.WriteFile(filepath.Join(inputPath, "Kilnfile.lock"))).To(Succeed()) outputPath = filepath.Join(filepath.Dir(inputPath), "output.pivotal") }) @@ -123,6 +167,9 @@ var _ = Describe("CarvelPublish", func() { if inputPath != "" { _ = os.RemoveAll(filepath.Dir(inputPath)) } + if server != nil { + server.Close() + } }) It("bakes the tile and creates a bake record", func() { diff --git a/internal/commands/carvel_rebake.go b/internal/commands/carvel_rebake.go index 763e039f2..a0d59ec31 100644 --- a/internal/commands/carvel_rebake.go +++ b/internal/commands/carvel_rebake.go @@ -13,6 +13,7 @@ import ( "github.com/pivotal-cf/jhanda" "github.com/pivotal-cf/kiln/internal/builder" "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/commands/flags" "github.com/pivotal-cf/kiln/pkg/bake" ) @@ -23,6 +24,7 @@ type CarvelReBake struct { } type CarvelReBakeOptions struct { + flags.Standard OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` } @@ -83,10 +85,33 @@ func (c CarvelReBake) Execute(args []string) error { b.SetWriter(os.Stdout) } - lockfilePath := filepath.Join(sourcePath, "Kilnfile.lock") + kilnfilePath := resolveKilnfilePath(c.Options.Kilnfile, sourcePath) + lockfilePath := kilnfilePath + ".lock" if _, statErr := os.Stat(lockfilePath); statErr == nil { + c.Options.Kilnfile = kilnfilePath + kilnfile, kilnfileLock, loadErr := c.Options.Standard.LoadKilnfiles(nil, nil) + if loadErr != nil { + return fmt.Errorf("failed to load Kilnfiles: %w", loadErr) + } + + if len(kilnfileLock.Releases) == 0 { + return fmt.Errorf("Kilnfile.lock has no releases") + } + releaseLock := kilnfileLock.Releases[0] + + tmpDir, tmpErr := os.MkdirTemp("", "carvel-rebake-*") + if tmpErr != nil { + return fmt.Errorf("failed to create temp directory: %w", tmpErr) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + c.outLogger.Printf("Re-baking Carvel tile from %s using lockfile", sourcePath) - err = b.BakeFromLockfile(sourcePath, lockfilePath) + localTarball, dlErr := downloadCarvelRelease(c.outLogger, kilnfile, kilnfileLock, tmpDir) + if dlErr != nil { + return fmt.Errorf("failed to download release from Artifactory: %w", dlErr) + } + + err = b.BakeFromLockfile(sourcePath, releaseLock, localTarball) } else { c.outLogger.Printf("Re-baking Carvel tile from %s", sourcePath) err = b.Bake(sourcePath) @@ -117,7 +142,7 @@ func (c CarvelReBake) Execute(args []string) error { func (c CarvelReBake) Usage() jhanda.Usage { return jhanda.Usage{ - Description: "Re-bakes a Carvel tile from a bake record for reproducible builds.\nThe repository must be checked out at the source_revision specified in the bake record.\n\nThe argument is the path to a JSON bake record file produced by 'kiln carvel publish --final'.", + Description: "Re-bakes a Carvel tile from a bake record for reproducible builds.\nThe repository must be checked out at the source_revision specified in the bake record.\nWhen a Kilnfile.lock is present, downloads the cached BOSH release from Artifactory.\n\nThe argument is the path to a JSON bake record file produced by 'kiln carvel publish --final'.", ShortDescription: "re-bakes a Carvel tile from a bake record", Flags: c.Options, } diff --git a/internal/commands/carvel_upload.go b/internal/commands/carvel_upload.go index 933f39b41..66c3c36be 100644 --- a/internal/commands/carvel_upload.go +++ b/internal/commands/carvel_upload.go @@ -1,7 +1,8 @@ package commands import ( - "crypto/sha256" + "bytes" + "crypto/sha1" "encoding/hex" "fmt" "io" @@ -9,10 +10,12 @@ import ( "net/http" "os" "path/filepath" + "text/template" "github.com/pivotal-cf/jhanda" "github.com/pivotal-cf/kiln/internal/carvel" - "github.com/pivotal-cf/kiln/internal/carvel/models" + "github.com/pivotal-cf/kiln/internal/commands/flags" + "github.com/pivotal-cf/kiln/pkg/cargo" ) type CarvelUpload struct { @@ -22,14 +25,11 @@ type CarvelUpload struct { } type CarvelUploadOptions struct { - SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` - ArtifactoryHost string ` long:"artifactory-host" description:"Artifactory server URL" required:"true"` - ArtifactoryRepo string ` long:"artifactory-repo" description:"Artifactory repository name" required:"true"` - Username string `short:"u" long:"artifactory-username" description:"Artifactory username" required:"true"` - Password string `short:"p" long:"artifactory-password" description:"Artifactory password or API key" required:"true"` - PathTemplate string ` long:"path-template" description:"remote path template" default:"bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz"` - OutputFile string `short:"o" long:"output-file" description:"also bake the tile to this path"` - Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` + flags.Standard + SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` + OutputFile string `short:"o" long:"output-file" description:"also bake the tile to this path"` + PathTemplate string ` long:"path-template" description:"remote path template override" default:"bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` } func NewCarvelUpload(outLogger, errLogger *log.Logger) CarvelUpload { @@ -45,17 +45,26 @@ func (c CarvelUpload) Execute(args []string) error { return err } - sourcePath := c.Options.SourceDirectory - if sourcePath == "" { - sourcePath, err = os.Getwd() - if err != nil { - return fmt.Errorf("failed to get current directory: %w", err) - } - } else { - sourcePath, err = filepath.Abs(sourcePath) - if err != nil { - return fmt.Errorf("failed to resolve source directory: %w", err) - } + sourcePath, err := resolveSourcePath(c.Options.SourceDirectory) + if err != nil { + return err + } + + kilnfilePath := resolveKilnfilePath(c.Options.Kilnfile, sourcePath) + + if _, statErr := os.Stat(kilnfilePath); statErr != nil { + return fmt.Errorf("Kilnfile not found at %s: create a Kilnfile with an artifactory release_source", kilnfilePath) + } + + c.Options.Kilnfile = kilnfilePath + kilnfile, err := loadKilnfileOnly(c.Options.Standard) + if err != nil { + return fmt.Errorf("failed to load Kilnfile: %w", err) + } + + artConfig, err := findArtifactorySource(kilnfile) + if err != nil { + return err } baker := carvel.NewBaker() @@ -79,31 +88,31 @@ func (c CarvelUpload) Execute(args []string) error { return fmt.Errorf("failed to get tile version: %w", err) } - remotePath := fmt.Sprintf("bosh-releases/%s/%s-%s.tgz", baker.GetName(), baker.GetName(), ver) + pathTmpl := c.Options.PathTemplate + if artConfig.PathTemplate != "" { + pathTmpl = artConfig.PathTemplate + } + remotePath, err := evaluatePathTemplate(pathTmpl, baker.GetName(), ver) + if err != nil { + return fmt.Errorf("failed to evaluate path template: %w", err) + } - checksum, err := fileSHA256(tarball) + sha1sum, err := fileSHA1(tarball) if err != nil { return fmt.Errorf("failed to checksum release tarball: %w", err) } - c.outLogger.Printf("Uploading %s to %s/%s/%s", filepath.Base(tarball), c.Options.ArtifactoryHost, c.Options.ArtifactoryRepo, remotePath) - err = uploadToArtifactory(tarball, c.Options.ArtifactoryHost, c.Options.ArtifactoryRepo, remotePath, c.Options.Username, c.Options.Password) + c.outLogger.Printf("Uploading %s to %s/%s/%s", filepath.Base(tarball), artConfig.ArtifactoryHost, artConfig.Repo, remotePath) + err = uploadToArtifactory(tarball, artConfig.ArtifactoryHost, artConfig.Repo, remotePath, artConfig.Username, artConfig.Password) if err != nil { return fmt.Errorf("failed to upload to Artifactory: %w", err) } - lockfilePath := filepath.Join(sourcePath, "Kilnfile.lock") - lf := models.CarvelLockfile{ - Release: models.CarvelReleaseLock{ - Name: baker.GetName(), - Version: ver, - RemotePath: remotePath, - SHA256: checksum, - }, - } - err = lf.WriteFile(lockfilePath) + sourceID := cargo.BOSHReleaseTarballSourceID(artConfig) + lockfilePath := kilnfilePath + ".lock" + err = writeStandardKilnfileLock(lockfilePath, baker.GetName(), ver, remotePath, sourceID, sha1sum) if err != nil { - return fmt.Errorf("failed to write lockfile: %w", err) + return fmt.Errorf("failed to write Kilnfile.lock: %w", err) } c.outLogger.Printf("Updated %s", lockfilePath) @@ -124,25 +133,38 @@ func (c CarvelUpload) Execute(args []string) error { func (c CarvelUpload) Usage() jhanda.Usage { return jhanda.Usage{ - Description: "Generates a BOSH release from a Carvel tile source, uploads the release tarball to Artifactory, and updates Kilnfile.lock with the remote location and checksum.", + Description: "Generates a BOSH release from a Carvel tile source, uploads the release tarball to Artifactory, and updates Kilnfile.lock with the remote location and checksum. Artifactory credentials are read from the Kilnfile's release_sources (typically interpolated from ~/.kiln/credentials.yml).", ShortDescription: "uploads a Carvel BOSH release to Artifactory", Flags: c.Options, } } -func fileSHA256(path string) (string, error) { +func fileSHA1(path string) (string, error) { f, err := os.Open(path) if err != nil { return "", err } defer func() { _ = f.Close() }() - h := sha256.New() + h := sha1.New() if _, err := io.Copy(h, f); err != nil { return "", err } return hex.EncodeToString(h.Sum(nil)), nil } +func evaluatePathTemplate(tmpl, name, version string) (string, error) { + t, err := template.New("path").Parse(tmpl) + if err != nil { + return "", err + } + var buf bytes.Buffer + err = t.Execute(&buf, cargo.BOSHReleaseTarballSpecification{Name: name, Version: version}) + if err != nil { + return "", err + } + return buf.String(), nil +} + func uploadToArtifactory(localPath, host, repo, remotePath, username, password string) error { f, err := os.Open(localPath) if err != nil { diff --git a/internal/commands/carvel_upload_test.go b/internal/commands/carvel_upload_test.go index f1cf62eb3..91a5f8dd7 100644 --- a/internal/commands/carvel_upload_test.go +++ b/internal/commands/carvel_upload_test.go @@ -10,8 +10,9 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pivotal-cf/kiln/internal/carvel/models" "github.com/pivotal-cf/kiln/internal/commands" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" ) var _ = Describe("CarvelUpload", func() { @@ -36,19 +37,21 @@ var _ = Describe("CarvelUpload", func() { }) Describe("Execute", func() { - When("required arguments are missing", func() { - It("returns an error when artifactory-host is not provided", func() { - err := command.Execute([]string{ - "--artifactory-repo", "some-repo", - "--artifactory-username", "user", - "--artifactory-password", "pass", + When("Kilnfile is missing", func() { + It("returns an error", func() { + tmpDir, err := os.MkdirTemp("", "upload-no-kilnfile-*") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = os.RemoveAll(tmpDir) }() + + err = command.Execute([]string{ + "--source-directory", tmpDir, }) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("artifactory-host")) + Expect(err.Error()).To(ContainSubstring("Kilnfile not found")) }) }) - When("valid arguments are provided with a mock Artifactory", func() { + When("valid Kilnfile is provided with a mock Artifactory", func() { var ( inputPath string server *httptest.Server @@ -66,6 +69,27 @@ var _ = Describe("CarvelUpload", func() { err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) Expect(err).NotTo(HaveOccurred()) + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusCreated) + })) + + kf := cargo.Kilnfile{ + ReleaseSources: []cargo.ReleaseSourceConfig{ + { + Type: "artifactory", + ArtifactoryHost: server.URL, + Repo: "test-repo", + Username: "user", + Password: "pass", + PathTemplate: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz", + }, + }, + } + kfData, err := yaml.Marshal(&kf) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(inputPath, "Kilnfile"), kfData, 0644) + Expect(err).NotTo(HaveOccurred()) + cmds := []*exec.Cmd{ exec.Command("git", "init"), exec.Command("git", "add", "."), @@ -76,10 +100,6 @@ var _ = Describe("CarvelUpload", func() { out, err := cmd.CombinedOutput() Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) } - - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusCreated) - })) }) AfterEach(func() { @@ -91,13 +111,9 @@ var _ = Describe("CarvelUpload", func() { } }) - It("uploads the BOSH release and writes a lockfile", func() { + It("uploads the BOSH release and writes a standard Kilnfile.lock", func() { err := command.Execute([]string{ "--source-directory", inputPath, - "--artifactory-host", server.URL, - "--artifactory-repo", "test-repo", - "--artifactory-username", "user", - "--artifactory-password", "pass", "--verbose", }) Expect(err).NotTo(HaveOccurred()) @@ -105,12 +121,18 @@ var _ = Describe("CarvelUpload", func() { lockfilePath := filepath.Join(inputPath, "Kilnfile.lock") Expect(lockfilePath).To(BeAnExistingFile()) - lf, err := models.ReadCarvelLockfile(lockfilePath) + lockData, err := os.ReadFile(lockfilePath) + Expect(err).NotTo(HaveOccurred()) + + var lock cargo.KilnfileLock + err = yaml.Unmarshal(lockData, &lock) Expect(err).NotTo(HaveOccurred()) - Expect(lf.Release.Name).To(Equal("k8s-tile-test")) - Expect(lf.Release.Version).To(Equal("0.1.1")) - Expect(lf.Release.SHA256).NotTo(BeEmpty()) - Expect(lf.Release.RemotePath).To(ContainSubstring("k8s-tile-test")) + Expect(lock.Releases).To(HaveLen(1)) + Expect(lock.Releases[0].Name).To(Equal("k8s-tile-test")) + Expect(lock.Releases[0].Version).To(Equal("0.1.1")) + Expect(lock.Releases[0].SHA1).NotTo(BeEmpty()) + Expect(lock.Releases[0].RemotePath).To(ContainSubstring("k8s-tile-test")) + Expect(lock.Releases[0].RemoteSource).To(Equal("artifactory")) }) }) }) From bff67e1c4c24911e3d9f58aff013c93ce9b077ee Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Wed, 18 Mar 2026 15:18:22 -0500 Subject: [PATCH 12/18] Add integration tests --- .../acceptance/carvel/carvel_workflow_test.go | 410 ++++++++++++++++++ .../carvel/fixtures/sample-tile/Kilnfile | 8 + internal/commands/carvel_publish_test.go | 103 +++-- internal/commands/carvel_rebake_test.go | 152 +++++++ internal/commands/carvel_upload_test.go | 76 +++- 5 files changed, 689 insertions(+), 60 deletions(-) create mode 100644 internal/acceptance/carvel/carvel_workflow_test.go create mode 100644 internal/acceptance/carvel/fixtures/sample-tile/Kilnfile diff --git a/internal/acceptance/carvel/carvel_workflow_test.go b/internal/acceptance/carvel/carvel_workflow_test.go new file mode 100644 index 000000000..18ff3834b --- /dev/null +++ b/internal/acceptance/carvel/carvel_workflow_test.go @@ -0,0 +1,410 @@ +package acceptance_test + +import ( + "archive/zip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + + "github.com/pivotal-cf/kiln/pkg/bake" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" +) + +// mockArtifactory is a test HTTP server that faithfully simulates Artifactory's +// upload (PUT) and download (GET) behaviour, including Basic Auth verification. +// Upload stores the tarball bytes keyed by request path; download serves them +// back. The /artifactory prefix that the real download client prepends is +// handled transparently. +type mockArtifactory struct { + mu sync.Mutex + blobs map[string][]byte + server *httptest.Server + username string + password string + + putCount int + getCount int +} + +func newMockArtifactory(username, password string) *mockArtifactory { + m := &mockArtifactory{ + blobs: make(map[string][]byte), + username: username, + password: password, + } + m.server = httptest.NewServer(m) + return m +} + +func (m *mockArtifactory) ServeHTTP(w http.ResponseWriter, r *http.Request) { + u, p, ok := r.BasicAuth() + if !ok || u != m.username || p != m.password { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + // Normalize path: strip the /artifactory prefix the download client adds + key := r.URL.Path + key = strings.TrimPrefix(key, "/artifactory") + + switch r.Method { + case http.MethodPut: + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + m.mu.Lock() + m.blobs[key] = body + m.putCount++ + m.mu.Unlock() + w.WriteHeader(http.StatusCreated) + + case http.MethodGet: + m.mu.Lock() + data, found := m.blobs[key] + m.getCount++ + m.mu.Unlock() + if !found { + http.Error(w, fmt.Sprintf("not found: %s (have: %v)", key, m.storedKeys()), http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(data) + + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +func (m *mockArtifactory) storedKeys() []string { + keys := make([]string, 0, len(m.blobs)) + for k := range m.blobs { + keys = append(keys, k) + } + return keys +} + +func (m *mockArtifactory) Close() { m.server.Close() } +func (m *mockArtifactory) URL() string { return m.server.URL } + +func (m *mockArtifactory) PutCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.putCount +} + +func (m *mockArtifactory) GetCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.getCount +} + +// --------------------------------------------------------------------------- +// End-to-end acceptance test for the full Carvel tile developer workflow. +// +// This test exercises every `kiln carvel` subcommand in sequence, chaining +// their outputs exactly like a real developer and CI pipeline would: +// +// Step 1 kiln carvel bake (local bake, no Kilnfile.lock) +// Step 2 kiln carvel upload (creates BOSH release, uploads, writes lock) +// git add + commit (Kilnfile.lock) +// Step 3 kiln carvel bake (CI-style: downloads from lock) +// Step 4 kiln carvel publish --final (downloads, bakes, writes bake record) +// git add + commit (bake_records/) +// Step 5 kiln carvel rebake (re-bakes from record, verifies checksum) +// +// A mock Artifactory server stores the actual uploaded tarball and serves it +// back on download, proving the full round-trip. +// --------------------------------------------------------------------------- +var _ = Describe("carvel full workflow", Ordered, func() { + const ( + sampleTileFixture = "fixtures/sample-tile" + artUsername = "test-user" + artPassword = "test-pass" + artRepo = "test-repo" + ) + + var ( + tmpDir string + inputPath string + art *mockArtifactory + ) + + variableFlags := func() []string { + return []string{ + "--variable", "artifactory_host=" + art.URL(), + "--variable", "artifactory_repo=" + artRepo, + "--variable", "artifactory_username=" + artUsername, + "--variable", "artifactory_password=" + artPassword, + } + } + + gitInTile := func(args ...string) { + cmd := exec.Command("git", args...) + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "git %v failed: %s", args, string(out)) + } + + currentSHA := func() string { + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = inputPath + out, err := cmd.Output() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return strings.TrimSpace(string(out)) + } + + tileChecksum := func(path string) string { + f, err := os.Open(path) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + defer func() { _ = f.Close() }() + h := sha256.New() + _, err = io.Copy(h, f) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return hex.EncodeToString(h.Sum(nil)) + } + + assertValidTile := func(pivotalPath string) { + archive, err := os.Open(pivotalPath) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + defer func() { _ = archive.Close() }() + + info, err := archive.Stat() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + zr, err := zip.NewReader(archive, info.Size()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + _, err = zr.Open("metadata/metadata.yml") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "tile must contain metadata/metadata.yml") + + var hasRelease bool + for _, f := range zr.File { + if filepath.Dir(f.Name) == "releases" && filepath.Ext(f.Name) == ".tgz" { + hasRelease = true + break + } + } + ExpectWithOffset(1, hasRelease).To(BeTrue(), "tile must contain releases/*.tgz") + } + + BeforeAll(func() { + if _, err := exec.LookPath("bosh"); err != nil { + Skip("bosh CLI not installed — skipping carvel workflow acceptance tests") + } + + var err error + tmpDir, err = os.MkdirTemp("", "kiln-carvel-workflow-*") + Expect(err).NotTo(HaveOccurred()) + + inputPath = filepath.Join(tmpDir, "tile") + err = os.CopyFS(inputPath, os.DirFS(sampleTileFixture)) + Expect(err).NotTo(HaveOccurred()) + + art = newMockArtifactory(artUsername, artPassword) + + gitInTile("init") + gitInTile("config", "user.email", "test@test.com") + gitInTile("config", "user.name", "Test") + gitInTile("add", ".") + gitInTile("commit", "-m", "initial commit") + }) + + AfterAll(func() { + if art != nil { + art.Close() + } + _ = os.RemoveAll(tmpDir) + }) + + // ----------------------------------------------------------------------- + // Step 1: Local bake (no Kilnfile.lock, no Artifactory interaction) + // ----------------------------------------------------------------------- + It("Step 1: bakes a tile locally without Kilnfile.lock", func() { + outputFile := filepath.Join(tmpDir, "step1.pivotal") + + cmd := exec.Command(pathToMain, + append([]string{ + "carvel", "bake", + "--source-directory", inputPath, + "--output-file", outputFile, + "--verbose", + }, variableFlags()...)..., + ) + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "120s").Should(gexec.Exit(0)) + + assertValidTile(outputFile) + + Expect(art.PutCount()).To(Equal(0), "local bake must not upload anything") + Expect(art.GetCount()).To(Equal(0), "local bake must not download anything") + + _, err = os.Stat(filepath.Join(inputPath, "Kilnfile.lock")) + Expect(os.IsNotExist(err)).To(BeTrue(), "local bake must not create Kilnfile.lock") + }) + + // ----------------------------------------------------------------------- + // Step 2: Upload (creates BOSH release, uploads to Artifactory, writes lock) + // ----------------------------------------------------------------------- + It("Step 2: uploads the BOSH release to Artifactory and writes Kilnfile.lock", func() { + cmd := exec.Command(pathToMain, + append([]string{ + "carvel", "upload", + "--source-directory", inputPath, + "--verbose", + }, variableFlags()...)..., + ) + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "120s").Should(gexec.Exit(0)) + + Expect(art.PutCount()).To(Equal(1), "upload must PUT exactly once") + + lockfilePath := filepath.Join(inputPath, "Kilnfile.lock") + Expect(lockfilePath).To(BeAnExistingFile()) + + lockData, err := os.ReadFile(lockfilePath) + Expect(err).NotTo(HaveOccurred()) + + var lock cargo.KilnfileLock + Expect(yaml.Unmarshal(lockData, &lock)).To(Succeed()) + + Expect(lock.Releases).To(HaveLen(1)) + rel := lock.Releases[0] + Expect(rel.Name).To(Equal("k8s-tile-test")) + Expect(rel.Version).To(Equal("0.1.1")) + Expect(rel.SHA1).NotTo(BeEmpty(), "lock must contain SHA1 of uploaded tarball") + Expect(rel.RemoteSource).To(Equal("artifactory")) + Expect(rel.RemotePath).To(Equal("bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz")) + + gitInTile("add", "Kilnfile.lock") + gitInTile("commit", "-m", "add Kilnfile.lock from upload") + }) + + // ----------------------------------------------------------------------- + // Step 3: CI-style bake (downloads cached BOSH release via Kilnfile.lock) + // ----------------------------------------------------------------------- + It("Step 3: bakes a tile using Kilnfile.lock (CI path with Artifactory download)", func() { + outputFile := filepath.Join(tmpDir, "step3-ci.pivotal") + + getCountBefore := art.GetCount() + + cmd := exec.Command(pathToMain, + append([]string{ + "carvel", "bake", + "--source-directory", inputPath, + "--output-file", outputFile, + "--verbose", + }, variableFlags()...)..., + ) + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "120s").Should(gexec.Exit(0)) + + assertValidTile(outputFile) + + Expect(art.GetCount()).To(BeNumerically(">", getCountBefore), + "CI bake must download the cached BOSH release from Artifactory") + }) + + // ----------------------------------------------------------------------- + // Step 4: Publish --final (downloads, bakes, creates bake record) + // ----------------------------------------------------------------------- + var publishChecksum string + + It("Step 4: publishes a final tile and writes a bake record", func() { + outputFile := filepath.Join(tmpDir, "step4-final.pivotal") + + cmd := exec.Command(pathToMain, + append([]string{ + "carvel", "publish", + "--source-directory", inputPath, + "--output-file", outputFile, + "--final", + "--verbose", + }, variableFlags()...)..., + ) + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "120s").Should(gexec.Exit(0)) + + assertValidTile(outputFile) + publishChecksum = tileChecksum(outputFile) + + recordsDir := filepath.Join(inputPath, "bake_records") + Expect(recordsDir).To(BeADirectory()) + + recordPath := filepath.Join(recordsDir, "0.1.1.json") + Expect(recordPath).To(BeAnExistingFile()) + + recordData, err := os.ReadFile(recordPath) + Expect(err).NotTo(HaveOccurred()) + + var record bake.Record + Expect(json.Unmarshal(recordData, &record)).To(Succeed()) + + Expect(record.Version).To(Equal("0.1.1")) + Expect(record.SourceRevision).To(Equal(currentSHA()), + "bake record source_revision must match current HEAD") + Expect(record.FileChecksum).NotTo(BeEmpty()) + Expect(record.FileChecksum).To(Equal(publishChecksum), + "bake record checksum must match the actual tile file") + + // Do NOT commit bake_records yet — rebake must run at the same + // HEAD that publish captured. In a real CI pipeline the + // Concourse resource checks out the commit from the record. + }) + + // ----------------------------------------------------------------------- + // Step 5: Rebake from bake record (reproducibility verification) + // ----------------------------------------------------------------------- + It("Step 5: re-bakes from the bake record with an identical checksum", func() { + outputFile := filepath.Join(tmpDir, "step5-rebake.pivotal") + recordPath := filepath.Join(inputPath, "bake_records", "0.1.1.json") + + args := append([]string{ + "carvel", "rebake", + "--output-file", outputFile, + "--verbose", + }, variableFlags()...) + args = append(args, recordPath) + cmd := exec.Command(pathToMain, args...) + cmd.Dir = inputPath + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "120s").Should(gexec.Exit(0)) + + assertValidTile(outputFile) + + rebakeChecksum := tileChecksum(outputFile) + Expect(rebakeChecksum).To(Equal(publishChecksum), + "rebake must produce a byte-for-byte identical tile to publish") + }) + + // ----------------------------------------------------------------------- + // Meta-assertions: verify the mock was exercised correctly across the + // entire workflow. + // ----------------------------------------------------------------------- + It("exercised Artifactory correctly across the full workflow", func() { + Expect(art.PutCount()).To(Equal(1), + "exactly one upload should have occurred across the entire workflow") + Expect(art.GetCount()).To(BeNumerically(">=", 2), + "at least two downloads should have occurred (CI bake + publish)") + }) +}) diff --git a/internal/acceptance/carvel/fixtures/sample-tile/Kilnfile b/internal/acceptance/carvel/fixtures/sample-tile/Kilnfile new file mode 100644 index 000000000..3da3ae25c --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/Kilnfile @@ -0,0 +1,8 @@ +--- +release_sources: + - type: artifactory + artifactory_host: $( variable "artifactory_host" ) + repo: $( variable "artifactory_repo" ) + username: $( variable "artifactory_username" ) + password: $( variable "artifactory_password" ) + path_template: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz" diff --git a/internal/commands/carvel_publish_test.go b/internal/commands/carvel_publish_test.go index 8f7d35c5b..8955d6783 100644 --- a/internal/commands/carvel_publish_test.go +++ b/internal/commands/carvel_publish_test.go @@ -2,12 +2,15 @@ package commands_test import ( "encoding/json" + "io" "log" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" + "strings" + "sync" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -64,11 +67,14 @@ var _ = Describe("CarvelPublish", func() { }) }) - When("--final flag is used with a Kilnfile and lockfile", func() { + When("--final flag is used with a round-trip mock Artifactory", func() { var ( inputPath string outputPath string server *httptest.Server + mu sync.Mutex + blobs map[string][]byte + getCount int ) BeforeEach(func() { @@ -79,6 +85,9 @@ var _ = Describe("CarvelPublish", func() { Skip("kiln CLI not installed - skipping integration test") } + blobs = make(map[string][]byte) + getCount = 0 + var err error inputPath, err = os.MkdirTemp("", "publish-test-*") Expect(err).NotTo(HaveOccurred()) @@ -97,60 +106,68 @@ var _ = Describe("CarvelPublish", func() { Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) } - baker := carvel.NewBaker() - baker.SetWriter(GinkgoWriter) - err = baker.Bake(inputPath) - Expect(err).NotTo(HaveOccurred()) - - tarball, err := baker.GetReleaseTarball() + b := carvel.NewBaker() + b.SetWriter(GinkgoWriter) + Expect(b.Bake(inputPath)).To(Succeed()) + tarball, err := b.GetReleaseTarball() Expect(err).NotTo(HaveOccurred()) - tarballData, err := os.ReadFile(tarball) Expect(err).NotTo(HaveOccurred()) server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/gzip") - _, _ = w.Write(tarballData) + key := strings.TrimPrefix(r.URL.Path, "/artifactory") + switch r.Method { + case http.MethodPut: + body, _ := io.ReadAll(r.Body) + mu.Lock() + blobs[key] = body + mu.Unlock() + w.WriteHeader(http.StatusCreated) + case http.MethodGet: + mu.Lock() + data, found := blobs[key] + getCount++ + mu.Unlock() + if !found { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(data) + } })) + // Pre-load mock with the tarball (simulating a prior upload) + remotePath := "/test-repo/bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz" + blobs[remotePath] = tarballData + kf := cargo.Kilnfile{ - ReleaseSources: []cargo.ReleaseSourceConfig{ - { - Type: "artifactory", - ArtifactoryHost: server.URL, - Repo: "test-repo", - Username: "user", - Password: "pass", - PathTemplate: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz", - }, - }, + ReleaseSources: []cargo.ReleaseSourceConfig{{ + Type: "artifactory", + ArtifactoryHost: server.URL, + Repo: "test-repo", + Username: "user", + Password: "pass", + PathTemplate: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz", + }}, } kfData, err := yaml.Marshal(&kf) Expect(err).NotTo(HaveOccurred()) - err = os.WriteFile(filepath.Join(inputPath, "Kilnfile"), kfData, 0644) - Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(inputPath, "Kilnfile"), kfData, 0644)).To(Succeed()) lock := cargo.KilnfileLock{ - Releases: []cargo.BOSHReleaseTarballLock{ - { - Name: "k8s-tile-test", - Version: "0.1.1", - RemotePath: "bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz", - RemoteSource: "artifactory", - SHA1: "", - }, - }, - Stemcell: cargo.Stemcell{ - OS: "ubuntu-jammy", - Version: "1.446", - }, + Releases: []cargo.BOSHReleaseTarballLock{{ + Name: "k8s-tile-test", + Version: "0.1.1", + RemotePath: "bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz", + RemoteSource: "artifactory", + }}, + Stemcell: cargo.Stemcell{OS: "ubuntu-jammy", Version: "1.446"}, } lockData, err := yaml.Marshal(&lock) Expect(err).NotTo(HaveOccurred()) - err = os.WriteFile(filepath.Join(inputPath, "Kilnfile.lock"), lockData, 0644) - Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(inputPath, "Kilnfile.lock"), lockData, 0644)).To(Succeed()) - // Re-commit with the Kilnfile and Kilnfile.lock for _, cmd := range []*exec.Cmd{ exec.Command("git", "add", "."), exec.Command("git", "commit", "-m", "add kilnfiles"), @@ -172,7 +189,7 @@ var _ = Describe("CarvelPublish", func() { } }) - It("bakes the tile and creates a bake record", func() { + It("downloads the tarball, bakes the tile, and creates a bake record", func() { err := command.Execute([]string{ "--source-directory", inputPath, "--output-file", outputPath, @@ -182,6 +199,11 @@ var _ = Describe("CarvelPublish", func() { Expect(err).NotTo(HaveOccurred()) Expect(outputPath).To(BeAnExistingFile()) + By("verifying the mock received a GET (download)") + mu.Lock() + Expect(getCount).To(BeNumerically(">=", 1), "publish must download from Artifactory") + mu.Unlock() + resolvedInput, resolveErr := filepath.EvalSymlinks(inputPath) if resolveErr != nil { resolvedInput = inputPath @@ -199,8 +221,7 @@ var _ = Describe("CarvelPublish", func() { Expect(err).NotTo(HaveOccurred()) var record bake.Record - err = json.Unmarshal(recordData, &record) - Expect(err).NotTo(HaveOccurred()) + Expect(json.Unmarshal(recordData, &record)).To(Succeed()) Expect(record.Version).To(Equal("0.1.1")) Expect(record.SourceRevision).NotTo(BeEmpty()) Expect(record.FileChecksum).NotTo(BeEmpty()) diff --git a/internal/commands/carvel_rebake_test.go b/internal/commands/carvel_rebake_test.go index 3e4151642..ca6e6e280 100644 --- a/internal/commands/carvel_rebake_test.go +++ b/internal/commands/carvel_rebake_test.go @@ -2,15 +2,24 @@ package commands_test import ( "encoding/json" + "fmt" + "io" "log" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" + "strings" + "sync" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/carvel" "github.com/pivotal-cf/kiln/internal/commands" "github.com/pivotal-cf/kiln/pkg/bake" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" ) var _ = Describe("CarvelReBake", func() { @@ -115,5 +124,148 @@ var _ = Describe("CarvelReBake", func() { Expect(err.Error()).To(ContainSubstring("source revision")) }) }) + + When("a valid bake record and mock Artifactory are provided", func() { + var ( + inputPath string + outputPath string + recordPath string + server *httptest.Server + ) + + BeforeEach(func() { + if !boshInstalled() { + Skip("bosh CLI not installed — skipping rebake integration test") + } + if !kilnInstalled() { + Skip("kiln CLI not installed — skipping rebake integration test") + } + + var err error + inputPath, err = os.MkdirTemp("", "rebake-happy-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + gitCmd := func(args ...string) { + cmd := exec.Command("git", args...) + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "git %v: %s", args, out) + } + + gitCmd("init") + gitCmd("add", ".") + gitCmd("commit", "-m", "initial commit") + + b := carvel.NewBaker() + b.SetWriter(GinkgoWriter) + Expect(b.Bake(inputPath)).To(Succeed()) + tarball, err := b.GetReleaseTarball() + Expect(err).NotTo(HaveOccurred()) + tarballData, err := os.ReadFile(tarball) + Expect(err).NotTo(HaveOccurred()) + + var ( + mu sync.Mutex + blobs = make(map[string][]byte) + ) + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/artifactory") + switch r.Method { + case http.MethodPut: + body, _ := io.ReadAll(r.Body) + mu.Lock() + blobs[key] = body + mu.Unlock() + w.WriteHeader(http.StatusCreated) + case http.MethodGet: + mu.Lock() + data, ok := blobs[key] + mu.Unlock() + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(data) + } + })) + + // Pre-load the mock with the tarball at the expected path + remotePath := fmt.Sprintf("/test-repo/bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz") + blobs[remotePath] = tarballData + + kf := cargo.Kilnfile{ + ReleaseSources: []cargo.ReleaseSourceConfig{{ + Type: "artifactory", + ArtifactoryHost: server.URL, + Repo: "test-repo", + Username: "user", + Password: "pass", + PathTemplate: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz", + }}, + } + kfData, err := yaml.Marshal(&kf) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(inputPath, "Kilnfile"), kfData, 0644)).To(Succeed()) + + lock := cargo.KilnfileLock{ + Releases: []cargo.BOSHReleaseTarballLock{{ + Name: "k8s-tile-test", + Version: "0.1.1", + RemotePath: "bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz", + RemoteSource: "artifactory", + }}, + Stemcell: cargo.Stemcell{OS: "ubuntu-jammy", Version: "1.446"}, + } + lockData, err := yaml.Marshal(&lock) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(inputPath, "Kilnfile.lock"), lockData, 0644)).To(Succeed()) + + gitCmd("add", ".") + gitCmd("commit", "-m", "add kilnfiles") + + sha := strings.TrimSpace(func() string { + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = inputPath + out, _ := cmd.Output() + return string(out) + }()) + + record := bake.Record{ + SourceRevision: sha, + Version: "0.1.1", + TileDirectory: inputPath, + } + buf, err := json.Marshal(record) + Expect(err).NotTo(HaveOccurred()) + + recordPath = filepath.Join(filepath.Dir(inputPath), "record.json") + Expect(os.WriteFile(recordPath, buf, 0644)).To(Succeed()) + + outputPath = filepath.Join(filepath.Dir(inputPath), "output.pivotal") + }) + + AfterEach(func() { + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + if server != nil { + server.Close() + } + }) + + It("re-bakes successfully from the bake record", func() { + err := command.Execute([]string{ + "--output-file", outputPath, + "--verbose", + recordPath, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(outputPath).To(BeAnExistingFile()) + }) + }) }) }) diff --git a/internal/commands/carvel_upload_test.go b/internal/commands/carvel_upload_test.go index 91a5f8dd7..bcf09e1be 100644 --- a/internal/commands/carvel_upload_test.go +++ b/internal/commands/carvel_upload_test.go @@ -1,12 +1,15 @@ package commands_test import ( + "io" "log" "net/http" "net/http/httptest" "os" "os/exec" "path/filepath" + "strings" + "sync" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -51,10 +54,13 @@ var _ = Describe("CarvelUpload", func() { }) }) - When("valid Kilnfile is provided with a mock Artifactory", func() { + When("valid Kilnfile is provided with a round-trip mock Artifactory", func() { var ( inputPath string server *httptest.Server + mu sync.Mutex + blobs map[string][]byte + authOK bool ) BeforeEach(func() { @@ -62,6 +68,37 @@ var _ = Describe("CarvelUpload", func() { Skip("bosh CLI not installed - skipping integration test") } + blobs = make(map[string][]byte) + authOK = false + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u, p, ok := r.BasicAuth() + if !ok || u != "user" || p != "pass" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + key := strings.TrimPrefix(r.URL.Path, "/artifactory") + switch r.Method { + case http.MethodPut: + body, _ := io.ReadAll(r.Body) + mu.Lock() + blobs[key] = body + authOK = true + mu.Unlock() + w.WriteHeader(http.StatusCreated) + case http.MethodGet: + mu.Lock() + data, found := blobs[key] + mu.Unlock() + if !found { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(data) + } + })) + var err error inputPath, err = os.MkdirTemp("", "upload-test-*") Expect(err).NotTo(HaveOccurred()) @@ -69,26 +106,19 @@ var _ = Describe("CarvelUpload", func() { err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) Expect(err).NotTo(HaveOccurred()) - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusCreated) - })) - kf := cargo.Kilnfile{ - ReleaseSources: []cargo.ReleaseSourceConfig{ - { - Type: "artifactory", - ArtifactoryHost: server.URL, - Repo: "test-repo", - Username: "user", - Password: "pass", - PathTemplate: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz", - }, - }, + ReleaseSources: []cargo.ReleaseSourceConfig{{ + Type: "artifactory", + ArtifactoryHost: server.URL, + Repo: "test-repo", + Username: "user", + Password: "pass", + PathTemplate: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz", + }}, } kfData, err := yaml.Marshal(&kf) Expect(err).NotTo(HaveOccurred()) - err = os.WriteFile(filepath.Join(inputPath, "Kilnfile"), kfData, 0644) - Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(inputPath, "Kilnfile"), kfData, 0644)).To(Succeed()) cmds := []*exec.Cmd{ exec.Command("git", "init"), @@ -125,14 +155,22 @@ var _ = Describe("CarvelUpload", func() { Expect(err).NotTo(HaveOccurred()) var lock cargo.KilnfileLock - err = yaml.Unmarshal(lockData, &lock) - Expect(err).NotTo(HaveOccurred()) + Expect(yaml.Unmarshal(lockData, &lock)).To(Succeed()) Expect(lock.Releases).To(HaveLen(1)) Expect(lock.Releases[0].Name).To(Equal("k8s-tile-test")) Expect(lock.Releases[0].Version).To(Equal("0.1.1")) Expect(lock.Releases[0].SHA1).NotTo(BeEmpty()) Expect(lock.Releases[0].RemotePath).To(ContainSubstring("k8s-tile-test")) Expect(lock.Releases[0].RemoteSource).To(Equal("artifactory")) + + By("verifying mock Artifactory received the PUT with Basic Auth") + mu.Lock() + Expect(authOK).To(BeTrue(), "upload must authenticate with Basic Auth") + Expect(blobs).To(HaveLen(1), "exactly one blob should be stored") + for _, data := range blobs { + Expect(len(data)).To(BeNumerically(">", 0), "uploaded tarball must not be empty") + } + mu.Unlock() }) }) }) From 346c423aac2cf3157a9fd104c2a9b9e41a3c86df Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Wed, 18 Mar 2026 15:31:24 -0500 Subject: [PATCH 13/18] Update onboarding guide --- gpp-onboarding-carvel.md | 554 +++++++++++++++++++++++++++------------ 1 file changed, 392 insertions(+), 162 deletions(-) diff --git a/gpp-onboarding-carvel.md b/gpp-onboarding-carvel.md index f4bef08ae..6ef6d3eff 100644 --- a/gpp-onboarding-carvel.md +++ b/gpp-onboarding-carvel.md @@ -1,232 +1,462 @@ -This playbook provides instructions for Carvel/Kubernetes tile teams to onboard their tiles onto the Golden Path to publish workflow. +# Onboarding Carvel/Kubernetes Tiles to Golden Path to Publish -For Carvel-based tiles, intermediary BOSH releases are generated automatically from imgpkg bundles (via ezbake) and do not need to be managed directly by your team. GPP handles BOSH release ingest and compilation behind the scenes. Your team only needs to manage final tile releases. +This playbook walks Kubernetes tile teams through onboarding their tiles onto the Golden Path to Publish (GPP) workflow using **`kiln carvel`** commands. -The result of this work will give you a re-bakable tile in [Artifactory](https://usw1.packages.broadcom.com/ui/repos/tree/General/tas-ecosystem-generic-prod-local/tile-releases) with compiled BOSH releases that is scanned by BlackDuck. For TVS integration please notify the Slingshots team when you're ready for it along with a link to your config please. +> [!NOTE] +> For Carvel-based tiles, intermediary BOSH releases are generated automatically from your imgpkg bundle. You do **not** need to manage BOSH releases directly -- Kiln handles that behind the scenes. -You may also optionally configure your Tile to generate RMT releases, and Open Source License Disclosure files from Blackduck. +The result of this work will give you a **re-bakable tile** in [Artifactory](https://usw1.packages.broadcom.com/ui/repos/tree/General/tas-ecosystem-generic-prod-local/tile-releases) that is scanned by BlackDuck. You may also optionally configure RMT releases and Open Source License Disclosure files. -## Pre-requisites for onboarding +--- -### Your tile is built with Kiln +## Tile directory structure -The Golden Path does not currently support tiles built with [tile-generator](https://github.com/cf-platform-eng/tile-generator). +Your Kubernetes tile repository must contain the following structure: -Please consider using [kiln](https://github.com/pivotal-cf/kiln/blob/main/TILE_AUTHOR_GUIDE.md). Carvel tile workflows use the `kiln carvel` subcommand group (`bake`, `upload`, `publish`, `rebake`). +``` +my-tile/ +├── base.yml # Tile metadata (name, version, package_installs, etc.) +├── bundle.tar # imgpkg bundle containing Carvel packages +├── version # Tile version (e.g. "1.0.0") +├── Kilnfile # Artifactory release source config (for upload/publish/rebake) +├── packageinstalls/ # Package install definitions +│ └── .yml +├── properties/ # Property blueprints (optional) +│ └── *.yml +├── forms/ # Form definitions (optional) +│ └── *.yml +├── icon.png # Tile icon (optional but recommended) +└── .gitignore # Should ignore .boshrelease/ and .carvel-tile/ +``` -### Github repo access: tiles +> [!IMPORTANT] +> - `base.yml` must have `metadata_version >= 3.2.0` (required for Kubernetes tile support). +> - `base.yml` must include a `package_installs` array and a `compatible_kubernetes_distributions` array. -Please provide write access to your private tile repositories to our bot account. Because BOSH releases are generated from the tile source (imgpkg bundles), separate BOSH release repositories are not required. +--- -- For github enterprise (github.gwd.broadcom.com): [tanzu-tas-ecosystem](https://github.gwd.broadcom.net/tanzu-tas-ecosystem) -- for github.com: [tas-ecosystem-bot](https://github.com/tas-ecosystem-bot) +## Prerequisites + +### Required tools + +| Tool | Purpose | Install | +|------|---------|---------| +| **Kiln** | Tile building and publishing | [github.com/pivotal-cf/kiln](https://github.com/pivotal-cf/kiln) | +| **BOSH CLI** | BOSH release generation from imgpkg bundle | [bosh.io/docs/cli-v2-install](https://bosh.io/docs/cli-v2-install/) | + +### GitHub repo access + +Provide **write** access to your tile repository to our bot account. Since BOSH releases are generated from your tile source, separate BOSH release repos are not required. + +- GitHub Enterprise (`github.gwd.broadcom.com`): [tanzu-tas-ecosystem](https://github.gwd.broadcom.net/tanzu-tas-ecosystem) +- GitHub.com: [tas-ecosystem-bot](https://github.com/tas-ecosystem-bot) ### TNZ team membership -To create PRs against our configuration repo you need to be a member of the [`all`](https://github.gwd.broadcom.net/orgs/TNZ/teams/all) team in the [TNZ org](https://github.gwd.broadcom.net/orgs/TNZ). +To create PRs against the configuration repo, you need to be a member of the [`all`](https://github.gwd.broadcom.net/orgs/TNZ/teams/all) team in the [TNZ org](https://github.gwd.broadcom.net/orgs/TNZ). + +--- -### Broadcom artifactory access +## Artifactory access and credentials -Authentication is required for accessing repos and artifacts on the Broadcom Jfrog Artifactory service. To get access for your team to our artifact repos containing: bosh-releases, compiled-releases, tile-releases and tile-candidates, create a [1.Support Ticket](https://broadcomitsm.wolkenservicedesk.com/wolken-support/item_details?itemId=2422). -Specify: +### Getting access -- Artifactory Server Name / URL: `https://usw1.packages.broadcom.com/ui` -- Sample Business Justification: +Authentication is required for Broadcom JFrog Artifactory. Create a [Support Ticket](https://broadcomitsm.wolkenservicedesk.com/wolken-support/item_details?itemId=2422) with: + +- **Artifactory Server Name / URL**: `https://usw1.packages.broadcom.com/ui` +- **Business Justification**: ```text - Need read access to tas-ecosystem-* artifactory projects on https://usw1.packages.broadcom.com + Need read/write access to tas-ecosystem-* artifactory projects + on https://usw1.packages.broadcom.com For the following teammates / service accounts: - memberX - memberY - - memberZ - bot / service account ``` -#### Credentials +### Creating an API key or identity token + +Since Artifactory uses Okta SSO, password authentication is not available. You need an `api_key` or `identity_token`: + +1. Log in to the [Artifactory UI](https://usw1.packages.broadcom.com/ui) via SSO. +2. Click the dropdown **Welcome, your_username** in the upper right. +3. Click **Edit Profile**. +4. Create an `api_key` or `identity_token` -- this value is used as the password for all `kiln` commands. + +> [!WARNING] +> `usw1.packages.broadcom.com` is only accessible on the Broadcom network. If accessing remotely, **full tunnel VPN is required**. +> +> If your CI is on the VMware / Broadcom network and is blocked, reach out to [#VMW-harbor-jfrog-migration](https://chat.google.com/room/AAAAcWIWWOA?cls=7). + +--- + +## Providing credentials to Kiln + +The `kiln carvel` commands that interact with Artifactory (`upload`, `publish`, `rebake`, and `bake` with a Kilnfile.lock) need credentials. These are configured in the **Kilnfile** using variable interpolation and resolved at runtime. + +### Step 1: Set up your Kilnfile + +Create a `Kilnfile` in your tile directory with variable placeholders: + +```yaml +release_sources: + - type: artifactory + artifactory_host: $( variable "artifactory_host" ) + repo: $( variable "artifactory_repo" ) + username: $( variable "artifactory_username" ) + password: $( variable "artifactory_password" ) + path_template: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz" +``` + +### Step 2: Choose how to provide the variable values + +There are three ways to supply credentials, in order of precedence (highest first): + +#### Option A: `--variable` flags (best for CI) + +Pass each value directly on the command line: + +```bash +kiln carvel upload \ + --source-directory . \ + --variable artifactory_host=https://usw1.packages.broadcom.com \ + --variable artifactory_repo=tas-ecosystem-generic-prod-local \ + --variable artifactory_username=my-bot-account \ + --variable artifactory_password=cmVmdGtuOj... +``` + +> [!TIP] +> Use `-vr` as the short form for `--variable`. + +#### Option B: `--variables-file` flag + +Point to a YAML file containing the values: + +```bash +kiln carvel upload \ + --source-directory . \ + --variables-file path/to/credentials.yml +``` + +Where `credentials.yml` contains: + +```yaml +artifactory_host: https://usw1.packages.broadcom.com +artifactory_repo: tas-ecosystem-generic-prod-local +artifactory_username: my-bot-account +artifactory_password: cmVmdGtuOj... +``` + +> [!TIP] +> Use `-vf` as the short form for `--variables-file`. + +#### Option C: `~/.kiln/credentials.yml` (best for local development) + +When the internal `kiln bake` step runs (inside `upload`, `publish`, `rebake`, and `bake`), Kiln automatically loads `~/.kiln/credentials.yml` as a default variables file. Place your credentials there for a seamless local experience: + +```yaml +# ~/.kiln/credentials.yml +artifactory_host: https://usw1.packages.broadcom.com +artifactory_repo: tas-ecosystem-generic-prod-local +artifactory_username: your_username +artifactory_password: your_api_key_or_identity_token +``` + +> [!IMPORTANT] +> The `~/.kiln/credentials.yml` auto-loading only applies to the internal `kiln bake` step. The outer `kiln carvel` commands (which parse the Kilnfile for Artifactory config) still need credentials via `--variable` or `--variables-file` -- **unless** you hardcode the values directly in the Kilnfile (not recommended for secrets). +> +> For the simplest local workflow, use both: put credentials in `~/.kiln/credentials.yml` **and** pass `--variables-file ~/.kiln/credentials.yml` to the outer command. + +> [!CAUTION] +> Never commit credentials to your repository. Add `credentials.yml` and `~/.kiln/` to your `.gitignore`. + +### Artifactory variable values + +| Variable | Value | +|----------|-------| +| `artifactory_host` | `https://usw1.packages.broadcom.com` | +| `artifactory_repo` | `tas-ecosystem-generic-prod-local` | +| `artifactory_username` | Your account or service account | +| `artifactory_password` | Your `api_key` or `identity_token` | + +--- + +## Developer workflow + +The `kiln carvel` commands cover the full tile development lifecycle. Each step builds on the previous one: + +``` + Local dev CI integration Final release CI publish +┌──────────────┐ ┌──────────────────┐ ┌──────────────────────┐ ┌──────────────────┐ +│ carvel bake │──▶│ carvel upload │──▶│ carvel publish │──▶│ carvel rebake │ +│ (local only) │ │ (uploads + lock) │ │ --final (bake record)│ │ (reproducible) │ +└──────────────┘ └──────────────────┘ └──────────────────────┘ └──────────────────┘ + │ │ │ + git commit git commit bake record checksum verified + Kilnfile.lock .pivotal uploaded +``` + +### Step 1: Local bake (no Artifactory needed) + +Bake a tile locally to test your tile structure. No Kilnfile or credentials required. + +```bash +kiln carvel bake \ + --source-directory . \ + --output-file my-tile-0.1.0.pivotal +``` + +This generates a BOSH release from your `bundle.tar`, assembles the tile, and produces a `.pivotal` file. Nothing is uploaded; no lockfile is created. + +### Step 2: Upload to Artifactory -Since artifactory is authenticated with Okta SSO, password authentication to the service it not allowed. Artifactory have `api_keys` and `identity_tokens` that are used as passwords. +Once your local bake works, upload the generated BOSH release to Artifactory so CI can reuse it: -Once access is granted and you are able to login to the artifactory ui via SSO, an `api_key` or `identity token` needs to be created for use with Kiln +```bash +kiln carvel upload \ + --source-directory . \ + --variables-file ~/.kiln/credentials.yml +``` -1. Upper right click dropdown of: `Welcome, your_username` -2. Click `Edit Profile` -3. Create an `api_key` or `identity_token` here and use it as the password for `kiln` commands or the artifactory cli. +This command: -#### Network access +1. Generates a BOSH release from your imgpkg bundle. +2. Uploads the tarball to Artifactory using the Kilnfile's release source config. +3. Writes a `Kilnfile.lock` with the release name, version, SHA1, and remote path. -The `usw1.packages.broadcom.com` artifactory is also only available on the Broadcom network. If accessing remotely, full tunnel VPN is required. +Then commit the lockfile: -If you CI is on the VMware / Broadcom Network and is blocked from accessing the artifactory, reach out to Google Chat Space: [#VMW-harbor-jfrog-migration](https://chat.google.com/room/AAAAcWIWWOA?cls=7) +```bash +git add Kilnfile.lock +git commit -m "Add Kilnfile.lock from carvel upload" +``` -## Golden Path Configuration +> [!NOTE] +> You can also pass `--output-file my-tile.pivotal` to upload to bake a `.pivotal` in the same step. -Configuration for the TAS Golden Path is stored in this repo and used as inputs to generate concourse pipelines. +### Step 3: CI bake (automatic, via Kilnfile.lock) -For Carvel-based tiles, the onboarding is simpler than for traditional BOSH tiles because GPP manages BOSH release ingest and compilation behind the scenes. You do not need to add BOSH release config files to the `bosh/` folder. The existing [bosh-ingest](https://tpe-concourse-rock.acc.broadcom.net/teams/tas-ecosystem/pipelines/bosh-releases?group=ingest-releases) and [bosh-compile](https://tpe-concourse-rock.acc.broadcom.net/teams/tas-ecosystem/pipelines/bosh-releases?group=compile-releases) pipelines are available for inspection if needed but do not require configuration from your team. +When Kilnfile.lock is present, `kiln carvel bake` downloads the cached BOSH release from Artifactory instead of regenerating it locally. This is faster and reproducible. -Overall the following steps to complete are: +```bash +kiln carvel bake \ + --source-directory . \ + --output-file my-tile-dev.pivotal \ + --variables-file ~/.kiln/credentials.yml +``` -- Updating the `Kilnfile` in the git repository to use artifactory as a source for generated BOSH releases. -- (optional) Creating a branch in the git repository of your tile for a pipeline to push Kilnfile.lock updates for your review. -- Creating config files for your tile(s) in `tiles/` folder to generate pipeline that will: - - Bake tile candidates with `kiln carvel bake`. Dev builds of tiles on your main / feature branch - - [`kiln carvel rebake`](https://github.com/pivotal-cf/kiln) for versioned release tiles - - Associate the BOSH releases consumed by the tile to the Blackduck tile project - - (optional) Automatically creates RMT releases that are included in the next-available TPM managed Release Train to assist with publishing - - If RMT is enabled, then your RMT release is eligible for automatic Open Source License Notice inclusion. Please see [creating open source license disclosures](./creating_open_source_license_disclosures.md). - - (optional) TVS integration (notify the #tas-slingshots team with your tile config requesting this when ready) +This is what your CI pipeline should run for development/candidate tile builds. -### Tile repository updates +### Step 4: Publish a final release -Set up your tile repository so that `kiln carvel` commands can fetch generated BOSH releases from Artifactory. +Create a final, versioned tile with a bake record for reproducible builds: -1. In the main / feature branch, update the `Kilnfile` to include artifactory as the remote source for BOSH releases. +```bash +kiln carvel publish --final \ + --source-directory . \ + --output-file my-tile-1.0.0.pivotal \ + --variables-file ~/.kiln/credentials.yml +``` + +This command: + +1. Downloads the BOSH release from Artifactory (using the Kilnfile.lock). +2. Bakes the tile. +3. Computes a SHA-256 checksum of the `.pivotal` file. +4. Writes a bake record to `bake_records/.json` containing the source revision, version, and file checksum. + +Then commit the bake record: + +```bash +git add bake_records/ +git commit -m "Release version 1.0.0" +``` + +> [!TIP] +> Use `--version` to override the tile version from the `version` file. For example, `--version 2.1.41` produces `bake_records/2.1.41.json`. + +**Example bake record** (`bake_records/1.0.0.json`): + +```json +{ + "source_revision": "1b19d8cb80e6cfdddd7be1c7a26c8210cbd4e4c5", + "version": "1.0.0", + "kiln_version": "0.97.0", + "file_checksum": "7622143c54dc53087a6c2401f5030170515e14f466857564a980092d4c87a094", + "tile_directory": "." +} +``` + +> [!WARNING] +> Your `bake_records/` directory must **only** contain bake record JSON files. + +> [!NOTE] +> Pre-release versions (e.g. `2.4.41-dev.0`) will not trigger a publish. This is useful for verifying OSL triage status before creating a final version like `2.4.41`. + +### Step 5: Rebake (CI, automated by GPP) + +GPP automatically runs rebake when it detects a new bake record. The rebake command reproduces the tile from the bake record and verifies the checksum matches: + +```bash +kiln carvel rebake \ + --output-file my-tile-1.0.0.pivotal \ + --variables-file ~/.kiln/credentials.yml \ + bake_records/1.0.0.json +``` + +> [!IMPORTANT] +> The repository must be checked out at the **exact commit** recorded in `source_revision`. The rebake will fail if HEAD does not match. In CI, the Concourse resource handles this automatically. + +The rebake: + +1. Reads the bake record to determine the source revision, version, and expected checksum. +2. Downloads the BOSH release from Artifactory (if Kilnfile.lock is present). +3. Bakes the tile. +4. Verifies the output checksum matches the bake record -- **byte-for-byte reproducibility**. + +--- + +## Command reference + +| Command | Description | Requires Kilnfile? | Requires Kilnfile.lock? | Writes Kilnfile.lock? | +|---------|-------------|:-------------------:|:-----------------------:|:---------------------:| +| `kiln carvel bake` | Local bake from bundle | No | No (uses if present) | No | +| `kiln carvel upload` | Upload BOSH release to Artifactory | **Yes** | No | **Yes** | +| `kiln carvel publish --final` | Bake + create bake record | **Yes** | **Yes** | No | +| `kiln carvel rebake ` | Reproduce tile from bake record | **Yes** (if lock present) | **Yes** (if present) | No | + +### Common flags + +| Flag | Short | Description | Used by | +|------|-------|-------------|---------| +| `--source-directory` | `-s` | Path to tile source directory (defaults to `.`) | `bake`, `upload`, `publish` | +| `--output-file` | `-o` | Path for the output `.pivotal` file | `bake`, `upload` (optional), `publish`, `rebake` | +| `--variable` | `-vr` | Key-value pair for Kilnfile interpolation | All commands | +| `--variables-file` | `-vf` | Path to YAML file with variable values | All commands | +| `--kilnfile` | `-kf` | Path to Kilnfile (default: `Kilnfile` in source dir) | All commands | +| `--verbose` | `-v` | Enable verbose output | All commands | +| `--final` | | Create a bake record | `publish` only | +| `--version` | | Override tile version | `publish` only | + +--- + +## Golden Path configuration + +Configuration for the TAS Golden Path is stored in the [tas-ecosystem-configuration](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration) repo and used as inputs to generate Concourse pipelines. + +> [!NOTE] +> For Carvel-based tiles, you do **not** need to add BOSH release config files to the `bosh/` folder. GPP manages BOSH release ingest and compilation automatically. + +### Tile config onboard + +1. Clone the [tas-ecosystem-configuration](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration) repo and create a branch. + +2. Create a new file for your tile under the `tiles/` directory. + + > [!IMPORTANT] + > `artifact_name` determines file name prefixes in Artifactory, project name prefixes in BlackDuck, and the prefix for published releases in RMT. + + **Example** -- `my-k8s-tile.yml`: ```yaml - release_sources: - - type: artifactory - id: artifactory_bosh_releases - artifactory_host: $(variable "artifactory_host") - repo: $(variable "artifactory_repo") - username: $(variable "artifactory_username") - password: $(variable "artifactory_password") # api_key or identity token - publishable: true # if this repo contains releases that are suitable to ship to customers - path_template: bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz + #@data/values + --- + repo: https://github.gwd.broadcom.net/TNZ/my-k8s-tile.git + branch: main + update_branch: auto-bump + subpath: . + artifact_name: my-k8s-tile + prerelease_format: build_increment_sha + team_members: + - alice@broadcom.com + - bob@broadcom.com + team_google_chat_group: my-team-chat + team_slack_channel: my-team-slack #! optional ``` -2. Use `kiln carvel upload` to generate the BOSH release from your imgpkg bundle, upload it to Artifactory, and update the `Kilnfile.lock` with the remote location and checksum. +3. **(Optional)** Add fields for automatic RMT draft-release creation. - Example `kiln carvel upload` command: + > [!WARNING] + > You will need to add upgrade specifiers (else Upgrade Planner will break!) and verify the release is ready for GA. It defaults to a draft. - ```bash - $ kiln carvel upload \ - --artifactory-host https://usw1.packages.broadcom.com \ - --artifactory-repo tas-ecosystem-generic-prod-local \ - --artifactory-username \ - --artifactory-password \ - --output-file my-tile-1.0.0-dev.pivotal - ``` + See [tile RMT release](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration/tree/main/docs/tile_rmt_release.md) for details. - This uploads the generated BOSH release and writes a `Kilnfile.lock` referencing the remote artifact. Commit the updated `Kilnfile.lock` to your repository. +4. **(Optional)** Add BlackDuck tile project associations. -3. (Optional) Create a new update branch from the main / feature branch in your tile repository (eg: `autobump`). Our CI will force push commits to this branch. - While this provides an auto update functionality, you are welcome to continue using your existing auto update tools (eg: dependabot). - You can also specify your feature branch if you want our CI to push the `Kilnfile.lock` updates directly to your feature branch. - If `branch` and `update_branch` are same, force push functionality is disabled. Ensure the branch specified in `update_branch` has [push access to our bot account](#github-repo-access-tiles). + Confirm your project exists at https://broadcom-vmw.app.blackduck.com/ with the format `TNZ-CF--tile`. If not, submit a ticket via the [BlackDuck Onboarding section](./creating_open_source_license_disclosures.md#blackduck-onboarding). -### Tile config onboard + Scanning is enabled by default. To disable: -1. Clone [this](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration) git repository if you have not already and create a branch locally for your changes. You should have write access to the repo and not need to create a fork to create a PR. If not, please review [this pre-requisite](#tnz-team-membership) - -2. Create a new file for each of your tiles under the [tiles](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration/tree/main/tiles) directory. Please note that `artifact_name` is especially significant because it determines the file name prefix in `artifactory`, project names prefixes in `blackduck`, and is the prefix used by the published release file in RMT / Broadcom Portal (when enabled). - Hello Tile - `hello-tile.yml` - - ```yaml - #@data/values - --- - repo: https://github.gwd.broadcom.net/TNZ/hello-tile.git - branch: main - update_branch: auto-bump - subpath: . - artifact_name: crhntr-hello #! this is the prefix for the built tiles and must be consistent with blackduck too - prerelease_format: build_increment_sha #! "sha" or "build_increment_sha" for versioning tile candidate builds. we recommend build_increment_sha - team_members: - - a@vmware.com - - b@vmware.com - team_google_chat_group: some-group #! required - google space / chat group for your team - team_slack_channel: some-channel #! If your team has slack channel - ``` - - Scheduler Tile - `p-scheduler.yml` (auto update directly on feature branch) - - ```yaml - #@data/values - --- - repo: https://github.com/pivotal-cf/p-scheduler.git - branch: master - update_branch: master - subpath: . - artifact_name: p-scheduler #! this is the prefix for the built tiles and must be consistent with blackduck too - prerelease_format: build_increment_sha #! "sha" or "build_increment_sha" for versioning tile candidate builds - team_members: - - a@vmware.com - - b@vmware.com - team_google_chat_group: some-group #! required - google space / chat group for your team - team_slack_channel: some-channel #! If your team has slack channel - ``` - -3. (optional) Add fields for automatic RMT _**draft**_-release creation - - >**Warning:** You will need to add upgrade specifiers (else Upgrade Planner will break!) and double check your release is ready to be set to GA. It defaults to a draft. - - Release tiles can be used as the basis for automatic `RMT` draft release creation. As a draft this means further steps are required prior to publishing. These include manually setting your upgrade specifiers, double checking the version, GA/EOGs dates, and release type, etc we inferred for you or read from your tile configuration's `rmt` entry. - - See [tile rmt release](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration/tree/main/docs/tile_rmt_release.md) for details. - -4. (optional) Add a field to enable automatic Black Duck tile project associations. In order to begin updating your Black Duck tile project: - >**Prerequisites:** - > Follow the [BlackDuck Onboarding section](./creating_open_source_license_disclosures.md#blackduck-onboarding) for - > your tile. BOSH release projects are managed by GPP for Carvel-based tiles. - - 1) Confirm your project exists @ https://broadcom-vmw.app.blackduck.com/ with the format `TNZ-CF--tile`, as `` is found in your tile config. - 2) If not, submit a ticket to request it ([BlackDuck Onboarding section](./creating_open_source_license_disclosures.md#blackduck-onboarding)) or rename it yourself. - > **Note:** Scanning is enabled by default. However, you may disable it by adding the following to your `./tiles/.yml` config: ```yaml blackduck: - enabled: false + enabled: false ``` -5. Create a PR to [this](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration) repository to add the newly created files that contain the tile information. -An extensive PR Check job will verify your change and add a comment if anything needs to be addressed. When the job passes you can merge the PR. -On merge, the respective golden path jobs will be created / updated for your tile. Please reach out to [#tas-slingshots on Google Chat](https://chat.google.com/room/AAAAZuDvKe0?cls=7) with any questions or issues getting your PR merged. +5. Create a PR. An automated check will verify your config. On merge, GPP jobs will be created for your tile. -## Updating CI for your tile and automatic RMT _draft_ releases + Reach out to [#tas-slingshots on Google Chat](https://chat.google.com/room/AAAAZuDvKe0?cls=7) with questions. -### Use `kiln carvel publish --final` +### Optional: auto-bump branch -If you are using CI to create new versions of tiles, the following updates can be made to take advantage of reproducible builds via `kiln carvel rebake`. +Create an `update_branch` (e.g. `autobump`) for GPP to push Kilnfile.lock updates for your review. If `branch` and `update_branch` are the same, force push is disabled. Ensure the bot account has [push access](#github-repo-access) to the specified branch. -Update your CI to output final tile builds using `kiln carvel publish --final`. When passing the **_--final_** flag, Kiln creates a bake record file under the **_bake_records_** folder. As part of the final tile build CI job, the bake records file needs to be committed and pushed to the tile repository. +--- -Golden Path to publish will then use this bake record to trigger `kiln carvel rebake`, producing a final tile from our [CI](https://runway-ci-srp.eng.vmware.com/teams/tas-ecosystem/) and upload it to [artifactory](https://build-artifactory.eng.vmware.com/ui/repos/tree/General/tas-ecosystem-generic-local/) repo under the sub-path: tile-releases. +## End-to-end example -In the case of a pre-release version the build will not result in a publish. This is useful to verify OSL triage status and test your candidate build. For example, you may create a -bake record with version `2.4.41-dev.0`, rerun the OSL generation multiple times, then finally create a `2.41.1` to trigger the full publish. +Here is the complete workflow from first local bake to published tile: - Example `kiln carvel publish --final` command: +```bash +# 1. Local bake -- verify your tile structure works +kiln carvel bake -s . -o my-tile-dev.pivotal - ```bash - $ kiln carvel publish --final --version 2.1.41 \ - --output-file my-tile-2.1.41.pivotal \ - --source-directory . - ``` +# 2. Upload BOSH release to Artifactory, write lockfile +kiln carvel upload -s . -vf ~/.kiln/credentials.yml - _Example:_ Bake record that should be committed to the tile repo that is created by `kiln carvel publish --final` as file: `bake_records/2.1.41.json`: +# 3. Commit the lockfile +git add Kilnfile.lock +git commit -m "Add Kilnfile.lock" - ```json - { - "source_revision": "1b19d8cb80e6cfdddd7be1c7a26c8210cbd4e4c5", - "version": "2.1.41", - "kiln_version": "0.90.0", - "file_checksum": "7622143c54dc53087a6c2401f5030170515e14f466857564a980092d4c87a094", - "tile_directory": "." - } - ``` +# 4. CI bake (downloads cached release from Artifactory) +kiln carvel bake -s . -o my-tile-dev.pivotal -vf ~/.kiln/credentials.yml + +# 5. Final release with bake record +kiln carvel publish --final -s . -o my-tile-1.0.0.pivotal -vf ~/.kiln/credentials.yml + +# 6. Commit the bake record +git add bake_records/ +git commit -m "Release 1.0.0" +git push + +# 7. GPP automatically runs rebake, verifies checksum, and publishes to Artifactory +``` + +--- + +## Troubleshooting + +### `Kilnfile not found` + +The `upload`, `publish`, and `rebake` commands require a `Kilnfile` with an `artifactory` release source. Make sure the file exists in your tile's source directory (or pass `--kilnfile path/to/Kilnfile`). + +### `Kilnfile.lock not found` or `no releases` + +Run `kiln carvel upload` first to generate the BOSH release and create the lockfile. + +### `source revision mismatch` during rebake -When executing `kiln carvel publish --final`, use the values for artifactory variables in your Kilnfile: +The repo must be at the exact commit from the bake record's `source_revision`. Check out that commit before running rebake. -- artifactory_host: `https://usw1.packages.broadcom.com` -- artifactory_repo: `tas-ecosystem-generic-prod-local` -- artifactory_username: **_your account or service account for broadcom artifactory_** -- artifactory_password: **_respective api_key or identity token_** +### `upload failed with status 401` -**_NOTE: https://usw1.packages.broadcom.com is accessible via Broadcom VPN with full tunnel gateway and TPE concourse workers_** +Your Artifactory credentials are incorrect or expired. Regenerate your API key or identity token from the [Artifactory UI](https://usw1.packages.broadcom.com/ui). -**_Commit the new bake record to the git repository of the tile_** +### `tile checksum mismatch` during rebake -**_Warning: Your bake_records directory must only contain bake records_** +The tile produced by rebake does not match the original publish. This can happen if the source tree has been modified after the bake record was created. Ensure no uncommitted changes exist and that HEAD matches `source_revision`. -GPP will then automatically run `kiln carvel rebake` against the bake record to produce the final `.pivotal` file, validate the checksum, and upload it to Artifactory and optionally RMT. +### Network errors connecting to Artifactory -Please refer to [Tile RMT Release](Publish-Tiles-to-RMT) to configure publishing your tile to RMT via Golden Path to Publish. +`usw1.packages.broadcom.com` requires Broadcom full-tunnel VPN. Verify you are connected. From e7997f84877b33d4b90e19d230de3a060e6e0387 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Sun, 22 Mar 2026 22:23:41 -0500 Subject: [PATCH 14/18] Fix linting errors --- internal/commands/carvel.go | 2 +- internal/commands/carvel_bake.go | 4 +-- internal/commands/carvel_bake_test.go | 12 -------- internal/commands/carvel_helpers.go | 37 +------------------------ internal/commands/carvel_publish.go | 8 +++--- internal/commands/carvel_rebake.go | 4 +-- internal/commands/carvel_rebake_test.go | 3 +- internal/commands/carvel_upload.go | 2 +- 8 files changed, 12 insertions(+), 60 deletions(-) diff --git a/internal/commands/carvel.go b/internal/commands/carvel.go index 20d816bf2..b7389adb4 100644 --- a/internal/commands/carvel.go +++ b/internal/commands/carvel.go @@ -92,7 +92,7 @@ func (c Carvel) Usage() jhanda.Usage { for _, name := range names { cmd := c.commands[name] paddedName := c.pad(name, " ", length) - subcommandList.WriteString(fmt.Sprintf(" %s %s\n", paddedName, cmd.Usage().ShortDescription)) + fmt.Fprintf(&subcommandList, " %s %s\n", paddedName, cmd.Usage().ShortDescription) } subcommandList.WriteString("\nUse 'kiln carvel help ' for more information about a subcommand.") diff --git a/internal/commands/carvel_bake.go b/internal/commands/carvel_bake.go index f60c08463..72e28d188 100644 --- a/internal/commands/carvel_bake.go +++ b/internal/commands/carvel_bake.go @@ -57,13 +57,13 @@ func (c CarvelBake) Execute(args []string) error { lockfilePath := kilnfilePath + ".lock" if _, statErr := os.Stat(lockfilePath); statErr == nil { c.Options.Kilnfile = kilnfilePath - kilnfile, kilnfileLock, loadErr := c.Options.Standard.LoadKilnfiles(nil, nil) + kilnfile, kilnfileLock, loadErr := c.Options.LoadKilnfiles(nil, nil) if loadErr != nil { return fmt.Errorf("failed to load Kilnfiles: %w", loadErr) } if len(kilnfileLock.Releases) == 0 { - return fmt.Errorf("Kilnfile.lock has no releases") + return fmt.Errorf("no releases found in Kilnfile.lock") } releaseLock := kilnfileLock.Releases[0] diff --git a/internal/commands/carvel_bake_test.go b/internal/commands/carvel_bake_test.go index da293323e..e16e78673 100644 --- a/internal/commands/carvel_bake_test.go +++ b/internal/commands/carvel_bake_test.go @@ -1,7 +1,6 @@ package commands_test import ( - "io" "log" "os" "os/exec" @@ -22,17 +21,6 @@ func kilnInstalled() bool { return err == nil } -func copyFile(src, dst string) { - in, err := os.Open(src) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - defer func() { _ = in.Close() }() - out, err := os.Create(dst) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - defer func() { _ = out.Close() }() - _, err = io.Copy(out, in) - ExpectWithOffset(1, err).NotTo(HaveOccurred()) -} - var _ = Describe("CarvelBake", func() { var ( outLogger *log.Logger diff --git a/internal/commands/carvel_helpers.go b/internal/commands/carvel_helpers.go index fe626ac52..0429ddbb3 100644 --- a/internal/commands/carvel_helpers.go +++ b/internal/commands/carvel_helpers.go @@ -43,7 +43,7 @@ func findArtifactorySource(kilnfile cargo.Kilnfile) (cargo.ReleaseSourceConfig, func downloadCarvelRelease(logger *log.Logger, kilnfile cargo.Kilnfile, lock cargo.KilnfileLock, destDir string) (string, error) { if len(lock.Releases) == 0 { - return "", fmt.Errorf("Kilnfile.lock has no releases") + return "", fmt.Errorf("no releases found in Kilnfile.lock") } releaseLock := lock.Releases[0] @@ -87,41 +87,6 @@ func writeStandardKilnfileLock(lockfilePath string, releaseName, releaseVersion, return os.WriteFile(lockfilePath, data, 0644) } -func readStandardKilnfileLock(lockfilePath string) (cargo.KilnfileLock, error) { - data, err := os.ReadFile(lockfilePath) - if err != nil { - return cargo.KilnfileLock{}, fmt.Errorf("failed to read Kilnfile.lock: %w", err) - } - var lock cargo.KilnfileLock - if err := yaml.Unmarshal(data, &lock); err != nil { - return cargo.KilnfileLock{}, fmt.Errorf("failed to parse Kilnfile.lock: %w", err) - } - return lock, nil -} - -func generateKilnfile(kilnfilePath, artifactoryHost, repo, username, password, pathTemplate string) error { - if pathTemplate == "" { - pathTemplate = "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz" - } - kf := cargo.Kilnfile{ - ReleaseSources: []cargo.ReleaseSourceConfig{ - { - Type: cargo.BOSHReleaseTarballSourceTypeArtifactory, - ArtifactoryHost: artifactoryHost, - Repo: repo, - Username: username, - Password: password, - PathTemplate: pathTemplate, - }, - }, - } - - data, err := yaml.Marshal(&kf) - if err != nil { - return fmt.Errorf("failed to marshal Kilnfile: %w", err) - } - return os.WriteFile(kilnfilePath, data, 0644) -} func resolveKilnfilePath(kilnfilePath, sourcePath string) string { if kilnfilePath == "" || kilnfilePath == "Kilnfile" { diff --git a/internal/commands/carvel_publish.go b/internal/commands/carvel_publish.go index 3e0c3b2b5..8c126622d 100644 --- a/internal/commands/carvel_publish.go +++ b/internal/commands/carvel_publish.go @@ -58,22 +58,22 @@ func (c CarvelPublish) Execute(args []string) error { kilnfilePath := resolveKilnfilePath(c.Options.Kilnfile, sourcePath) if _, statErr := os.Stat(kilnfilePath); statErr != nil { - return fmt.Errorf("Kilnfile not found at %s: run 'kiln carvel upload' first to create the BOSH release, Kilnfile, and Kilnfile.lock", kilnfilePath) + return fmt.Errorf("could not find Kilnfile at %s: run 'kiln carvel upload' first to create the BOSH release, Kilnfile, and Kilnfile.lock", kilnfilePath) } lockfilePath := kilnfilePath + ".lock" if _, statErr := os.Stat(lockfilePath); statErr != nil { - return fmt.Errorf("Kilnfile.lock not found at %s: run 'kiln carvel upload' first to create the BOSH release and lockfile", lockfilePath) + return fmt.Errorf("could not find Kilnfile.lock at %s: run 'kiln carvel upload' first to create the BOSH release and lockfile", lockfilePath) } c.Options.Kilnfile = kilnfilePath - kilnfile, kilnfileLock, err := c.Options.Standard.LoadKilnfiles(nil, nil) + kilnfile, kilnfileLock, err := c.Options.LoadKilnfiles(nil, nil) if err != nil { return fmt.Errorf("failed to load Kilnfiles: %w", err) } if len(kilnfileLock.Releases) == 0 { - return fmt.Errorf("Kilnfile.lock has no releases: run 'kiln carvel upload' first") + return fmt.Errorf("no releases found in Kilnfile.lock: run 'kiln carvel upload' first") } releaseLock := kilnfileLock.Releases[0] diff --git a/internal/commands/carvel_rebake.go b/internal/commands/carvel_rebake.go index a0d59ec31..64bdf42a7 100644 --- a/internal/commands/carvel_rebake.go +++ b/internal/commands/carvel_rebake.go @@ -89,13 +89,13 @@ func (c CarvelReBake) Execute(args []string) error { lockfilePath := kilnfilePath + ".lock" if _, statErr := os.Stat(lockfilePath); statErr == nil { c.Options.Kilnfile = kilnfilePath - kilnfile, kilnfileLock, loadErr := c.Options.Standard.LoadKilnfiles(nil, nil) + kilnfile, kilnfileLock, loadErr := c.Options.LoadKilnfiles(nil, nil) if loadErr != nil { return fmt.Errorf("failed to load Kilnfiles: %w", loadErr) } if len(kilnfileLock.Releases) == 0 { - return fmt.Errorf("Kilnfile.lock has no releases") + return fmt.Errorf("no releases found in Kilnfile.lock") } releaseLock := kilnfileLock.Releases[0] diff --git a/internal/commands/carvel_rebake_test.go b/internal/commands/carvel_rebake_test.go index ca6e6e280..1f93a337d 100644 --- a/internal/commands/carvel_rebake_test.go +++ b/internal/commands/carvel_rebake_test.go @@ -2,7 +2,6 @@ package commands_test import ( "encoding/json" - "fmt" "io" "log" "net/http" @@ -194,7 +193,7 @@ var _ = Describe("CarvelReBake", func() { })) // Pre-load the mock with the tarball at the expected path - remotePath := fmt.Sprintf("/test-repo/bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz") + remotePath := "/test-repo/bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz" blobs[remotePath] = tarballData kf := cargo.Kilnfile{ diff --git a/internal/commands/carvel_upload.go b/internal/commands/carvel_upload.go index 66c3c36be..0a58e9afd 100644 --- a/internal/commands/carvel_upload.go +++ b/internal/commands/carvel_upload.go @@ -53,7 +53,7 @@ func (c CarvelUpload) Execute(args []string) error { kilnfilePath := resolveKilnfilePath(c.Options.Kilnfile, sourcePath) if _, statErr := os.Stat(kilnfilePath); statErr != nil { - return fmt.Errorf("Kilnfile not found at %s: create a Kilnfile with an artifactory release_source", kilnfilePath) + return fmt.Errorf("could not find Kilnfile at %s: create a Kilnfile with an artifactory release_source", kilnfilePath) } c.Options.Kilnfile = kilnfilePath From 28e6311979450890884d9dfb2e95f7f8d1254ecc Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Sun, 22 Mar 2026 22:28:31 -0500 Subject: [PATCH 15/18] Fix unit tests --- internal/commands/carvel_publish_test.go | 2 +- internal/commands/carvel_upload_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/commands/carvel_publish_test.go b/internal/commands/carvel_publish_test.go index 8955d6783..a064e32b7 100644 --- a/internal/commands/carvel_publish_test.go +++ b/internal/commands/carvel_publish_test.go @@ -62,7 +62,7 @@ var _ = Describe("CarvelPublish", func() { "--output-file", filepath.Join(tmpDir, "out.pivotal"), }) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Kilnfile not found")) + Expect(err.Error()).To(ContainSubstring("could not find Kilnfile")) Expect(err.Error()).To(ContainSubstring("kiln carvel upload")) }) }) diff --git a/internal/commands/carvel_upload_test.go b/internal/commands/carvel_upload_test.go index bcf09e1be..b00f7a5d8 100644 --- a/internal/commands/carvel_upload_test.go +++ b/internal/commands/carvel_upload_test.go @@ -50,7 +50,7 @@ var _ = Describe("CarvelUpload", func() { "--source-directory", tmpDir, }) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("Kilnfile not found")) + Expect(err.Error()).To(ContainSubstring("could not find Kilnfile")) }) }) From 799b17f531d000e13dd86ecdf4081e503136b0e1 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Sun, 22 Mar 2026 22:33:21 -0500 Subject: [PATCH 16/18] Add user.name and user.email in commits in tests --- internal/commands/carvel_bake_test.go | 2 +- internal/commands/carvel_rebake_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/commands/carvel_bake_test.go b/internal/commands/carvel_bake_test.go index e16e78673..c83ed6dfd 100644 --- a/internal/commands/carvel_bake_test.go +++ b/internal/commands/carvel_bake_test.go @@ -60,7 +60,7 @@ var _ = Describe("CarvelBake", func() { cmds := []*exec.Cmd{ exec.Command("git", "init"), exec.Command("git", "add", "."), - exec.Command("git", "commit", "-m", "initial commit"), + exec.Command("git", "-c", "user.name=test", "-c", "user.email=test@test.com", "commit", "-m", "initial commit"), } for _, cmd := range cmds { cmd.Dir = inputPath diff --git a/internal/commands/carvel_rebake_test.go b/internal/commands/carvel_rebake_test.go index 1f93a337d..00e822584 100644 --- a/internal/commands/carvel_rebake_test.go +++ b/internal/commands/carvel_rebake_test.go @@ -87,7 +87,7 @@ var _ = Describe("CarvelReBake", func() { cmds := []*exec.Cmd{ exec.Command("git", "init"), exec.Command("git", "add", "."), - exec.Command("git", "commit", "-m", "initial commit"), + exec.Command("git", "-c", "user.name=test", "-c", "user.email=test@test.com", "commit", "-m", "initial commit"), } for _, cmd := range cmds { cmd.Dir = inputPath From a5a1448f01f1ab0647467e0c0326210052716e14 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Sun, 22 Mar 2026 22:42:27 -0500 Subject: [PATCH 17/18] Use consistent serialization key for releases --- internal/carvel/models/lockfile.go | 2 +- internal/carvel/models/lockfile_test.go | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/internal/carvel/models/lockfile.go b/internal/carvel/models/lockfile.go index d0f4a1665..ce0cd3dbd 100644 --- a/internal/carvel/models/lockfile.go +++ b/internal/carvel/models/lockfile.go @@ -8,7 +8,7 @@ import ( ) type CarvelLockfile struct { - Release CarvelReleaseLock `yaml:"release"` + Releases []CarvelReleaseLock `yaml:"releases"` } type CarvelReleaseLock struct { diff --git a/internal/carvel/models/lockfile_test.go b/internal/carvel/models/lockfile_test.go index 242f07c54..7f5eda451 100644 --- a/internal/carvel/models/lockfile_test.go +++ b/internal/carvel/models/lockfile_test.go @@ -17,11 +17,13 @@ func TestCarvelLockfileRoundTrip(t *testing.T) { lockPath := filepath.Join(dir, "Kilnfile.lock") original := models.CarvelLockfile{ - Release: models.CarvelReleaseLock{ - Name: "my-tile", - Version: "1.2.3", - RemotePath: "bosh-releases/my-tile/my-tile-1.2.3.tgz", - SHA256: "abc123def456", + Releases: []models.CarvelReleaseLock{ + { + Name: "my-tile", + Version: "1.2.3", + RemotePath: "bosh-releases/my-tile/my-tile-1.2.3.tgz", + SHA256: "abc123def456", + }, }, } @@ -47,7 +49,7 @@ func TestReadCarvelLockfileInvalidYAML(t *testing.T) { dir := t.TempDir() lockPath := filepath.Join(dir, "Kilnfile.lock") - err := os.WriteFile(lockPath, []byte("release:\n name: [unterminated"), 0644) + err := os.WriteFile(lockPath, []byte("releases:\n - name: [unterminated"), 0644) g.Expect(err).NotTo(HaveOccurred()) _, err = models.ReadCarvelLockfile(lockPath) From 78e9363c33874c062ddf5fcfaf9bd1ba1ce9c4a9 Mon Sep 17 00:00:00 2001 From: Rizwan Reza Date: Sun, 22 Mar 2026 22:49:28 -0500 Subject: [PATCH 18/18] remove unneeded onboarding guideit it's now in tas-ecosystem repo --- gpp-onboarding-carvel.md | 462 --------------------------------------- 1 file changed, 462 deletions(-) delete mode 100644 gpp-onboarding-carvel.md diff --git a/gpp-onboarding-carvel.md b/gpp-onboarding-carvel.md deleted file mode 100644 index 6ef6d3eff..000000000 --- a/gpp-onboarding-carvel.md +++ /dev/null @@ -1,462 +0,0 @@ -# Onboarding Carvel/Kubernetes Tiles to Golden Path to Publish - -This playbook walks Kubernetes tile teams through onboarding their tiles onto the Golden Path to Publish (GPP) workflow using **`kiln carvel`** commands. - -> [!NOTE] -> For Carvel-based tiles, intermediary BOSH releases are generated automatically from your imgpkg bundle. You do **not** need to manage BOSH releases directly -- Kiln handles that behind the scenes. - -The result of this work will give you a **re-bakable tile** in [Artifactory](https://usw1.packages.broadcom.com/ui/repos/tree/General/tas-ecosystem-generic-prod-local/tile-releases) that is scanned by BlackDuck. You may also optionally configure RMT releases and Open Source License Disclosure files. - ---- - -## Tile directory structure - -Your Kubernetes tile repository must contain the following structure: - -``` -my-tile/ -├── base.yml # Tile metadata (name, version, package_installs, etc.) -├── bundle.tar # imgpkg bundle containing Carvel packages -├── version # Tile version (e.g. "1.0.0") -├── Kilnfile # Artifactory release source config (for upload/publish/rebake) -├── packageinstalls/ # Package install definitions -│ └── .yml -├── properties/ # Property blueprints (optional) -│ └── *.yml -├── forms/ # Form definitions (optional) -│ └── *.yml -├── icon.png # Tile icon (optional but recommended) -└── .gitignore # Should ignore .boshrelease/ and .carvel-tile/ -``` - -> [!IMPORTANT] -> - `base.yml` must have `metadata_version >= 3.2.0` (required for Kubernetes tile support). -> - `base.yml` must include a `package_installs` array and a `compatible_kubernetes_distributions` array. - ---- - -## Prerequisites - -### Required tools - -| Tool | Purpose | Install | -|------|---------|---------| -| **Kiln** | Tile building and publishing | [github.com/pivotal-cf/kiln](https://github.com/pivotal-cf/kiln) | -| **BOSH CLI** | BOSH release generation from imgpkg bundle | [bosh.io/docs/cli-v2-install](https://bosh.io/docs/cli-v2-install/) | - -### GitHub repo access - -Provide **write** access to your tile repository to our bot account. Since BOSH releases are generated from your tile source, separate BOSH release repos are not required. - -- GitHub Enterprise (`github.gwd.broadcom.com`): [tanzu-tas-ecosystem](https://github.gwd.broadcom.net/tanzu-tas-ecosystem) -- GitHub.com: [tas-ecosystem-bot](https://github.com/tas-ecosystem-bot) - -### TNZ team membership - -To create PRs against the configuration repo, you need to be a member of the [`all`](https://github.gwd.broadcom.net/orgs/TNZ/teams/all) team in the [TNZ org](https://github.gwd.broadcom.net/orgs/TNZ). - ---- - -## Artifactory access and credentials - -### Getting access - -Authentication is required for Broadcom JFrog Artifactory. Create a [Support Ticket](https://broadcomitsm.wolkenservicedesk.com/wolken-support/item_details?itemId=2422) with: - -- **Artifactory Server Name / URL**: `https://usw1.packages.broadcom.com/ui` -- **Business Justification**: - - ```text - Need read/write access to tas-ecosystem-* artifactory projects - on https://usw1.packages.broadcom.com - - For the following teammates / service accounts: - - memberX - - memberY - - bot / service account - ``` - -### Creating an API key or identity token - -Since Artifactory uses Okta SSO, password authentication is not available. You need an `api_key` or `identity_token`: - -1. Log in to the [Artifactory UI](https://usw1.packages.broadcom.com/ui) via SSO. -2. Click the dropdown **Welcome, your_username** in the upper right. -3. Click **Edit Profile**. -4. Create an `api_key` or `identity_token` -- this value is used as the password for all `kiln` commands. - -> [!WARNING] -> `usw1.packages.broadcom.com` is only accessible on the Broadcom network. If accessing remotely, **full tunnel VPN is required**. -> -> If your CI is on the VMware / Broadcom network and is blocked, reach out to [#VMW-harbor-jfrog-migration](https://chat.google.com/room/AAAAcWIWWOA?cls=7). - ---- - -## Providing credentials to Kiln - -The `kiln carvel` commands that interact with Artifactory (`upload`, `publish`, `rebake`, and `bake` with a Kilnfile.lock) need credentials. These are configured in the **Kilnfile** using variable interpolation and resolved at runtime. - -### Step 1: Set up your Kilnfile - -Create a `Kilnfile` in your tile directory with variable placeholders: - -```yaml -release_sources: - - type: artifactory - artifactory_host: $( variable "artifactory_host" ) - repo: $( variable "artifactory_repo" ) - username: $( variable "artifactory_username" ) - password: $( variable "artifactory_password" ) - path_template: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz" -``` - -### Step 2: Choose how to provide the variable values - -There are three ways to supply credentials, in order of precedence (highest first): - -#### Option A: `--variable` flags (best for CI) - -Pass each value directly on the command line: - -```bash -kiln carvel upload \ - --source-directory . \ - --variable artifactory_host=https://usw1.packages.broadcom.com \ - --variable artifactory_repo=tas-ecosystem-generic-prod-local \ - --variable artifactory_username=my-bot-account \ - --variable artifactory_password=cmVmdGtuOj... -``` - -> [!TIP] -> Use `-vr` as the short form for `--variable`. - -#### Option B: `--variables-file` flag - -Point to a YAML file containing the values: - -```bash -kiln carvel upload \ - --source-directory . \ - --variables-file path/to/credentials.yml -``` - -Where `credentials.yml` contains: - -```yaml -artifactory_host: https://usw1.packages.broadcom.com -artifactory_repo: tas-ecosystem-generic-prod-local -artifactory_username: my-bot-account -artifactory_password: cmVmdGtuOj... -``` - -> [!TIP] -> Use `-vf` as the short form for `--variables-file`. - -#### Option C: `~/.kiln/credentials.yml` (best for local development) - -When the internal `kiln bake` step runs (inside `upload`, `publish`, `rebake`, and `bake`), Kiln automatically loads `~/.kiln/credentials.yml` as a default variables file. Place your credentials there for a seamless local experience: - -```yaml -# ~/.kiln/credentials.yml -artifactory_host: https://usw1.packages.broadcom.com -artifactory_repo: tas-ecosystem-generic-prod-local -artifactory_username: your_username -artifactory_password: your_api_key_or_identity_token -``` - -> [!IMPORTANT] -> The `~/.kiln/credentials.yml` auto-loading only applies to the internal `kiln bake` step. The outer `kiln carvel` commands (which parse the Kilnfile for Artifactory config) still need credentials via `--variable` or `--variables-file` -- **unless** you hardcode the values directly in the Kilnfile (not recommended for secrets). -> -> For the simplest local workflow, use both: put credentials in `~/.kiln/credentials.yml` **and** pass `--variables-file ~/.kiln/credentials.yml` to the outer command. - -> [!CAUTION] -> Never commit credentials to your repository. Add `credentials.yml` and `~/.kiln/` to your `.gitignore`. - -### Artifactory variable values - -| Variable | Value | -|----------|-------| -| `artifactory_host` | `https://usw1.packages.broadcom.com` | -| `artifactory_repo` | `tas-ecosystem-generic-prod-local` | -| `artifactory_username` | Your account or service account | -| `artifactory_password` | Your `api_key` or `identity_token` | - ---- - -## Developer workflow - -The `kiln carvel` commands cover the full tile development lifecycle. Each step builds on the previous one: - -``` - Local dev CI integration Final release CI publish -┌──────────────┐ ┌──────────────────┐ ┌──────────────────────┐ ┌──────────────────┐ -│ carvel bake │──▶│ carvel upload │──▶│ carvel publish │──▶│ carvel rebake │ -│ (local only) │ │ (uploads + lock) │ │ --final (bake record)│ │ (reproducible) │ -└──────────────┘ └──────────────────┘ └──────────────────────┘ └──────────────────┘ - │ │ │ - git commit git commit bake record checksum verified - Kilnfile.lock .pivotal uploaded -``` - -### Step 1: Local bake (no Artifactory needed) - -Bake a tile locally to test your tile structure. No Kilnfile or credentials required. - -```bash -kiln carvel bake \ - --source-directory . \ - --output-file my-tile-0.1.0.pivotal -``` - -This generates a BOSH release from your `bundle.tar`, assembles the tile, and produces a `.pivotal` file. Nothing is uploaded; no lockfile is created. - -### Step 2: Upload to Artifactory - -Once your local bake works, upload the generated BOSH release to Artifactory so CI can reuse it: - -```bash -kiln carvel upload \ - --source-directory . \ - --variables-file ~/.kiln/credentials.yml -``` - -This command: - -1. Generates a BOSH release from your imgpkg bundle. -2. Uploads the tarball to Artifactory using the Kilnfile's release source config. -3. Writes a `Kilnfile.lock` with the release name, version, SHA1, and remote path. - -Then commit the lockfile: - -```bash -git add Kilnfile.lock -git commit -m "Add Kilnfile.lock from carvel upload" -``` - -> [!NOTE] -> You can also pass `--output-file my-tile.pivotal` to upload to bake a `.pivotal` in the same step. - -### Step 3: CI bake (automatic, via Kilnfile.lock) - -When Kilnfile.lock is present, `kiln carvel bake` downloads the cached BOSH release from Artifactory instead of regenerating it locally. This is faster and reproducible. - -```bash -kiln carvel bake \ - --source-directory . \ - --output-file my-tile-dev.pivotal \ - --variables-file ~/.kiln/credentials.yml -``` - -This is what your CI pipeline should run for development/candidate tile builds. - -### Step 4: Publish a final release - -Create a final, versioned tile with a bake record for reproducible builds: - -```bash -kiln carvel publish --final \ - --source-directory . \ - --output-file my-tile-1.0.0.pivotal \ - --variables-file ~/.kiln/credentials.yml -``` - -This command: - -1. Downloads the BOSH release from Artifactory (using the Kilnfile.lock). -2. Bakes the tile. -3. Computes a SHA-256 checksum of the `.pivotal` file. -4. Writes a bake record to `bake_records/.json` containing the source revision, version, and file checksum. - -Then commit the bake record: - -```bash -git add bake_records/ -git commit -m "Release version 1.0.0" -``` - -> [!TIP] -> Use `--version` to override the tile version from the `version` file. For example, `--version 2.1.41` produces `bake_records/2.1.41.json`. - -**Example bake record** (`bake_records/1.0.0.json`): - -```json -{ - "source_revision": "1b19d8cb80e6cfdddd7be1c7a26c8210cbd4e4c5", - "version": "1.0.0", - "kiln_version": "0.97.0", - "file_checksum": "7622143c54dc53087a6c2401f5030170515e14f466857564a980092d4c87a094", - "tile_directory": "." -} -``` - -> [!WARNING] -> Your `bake_records/` directory must **only** contain bake record JSON files. - -> [!NOTE] -> Pre-release versions (e.g. `2.4.41-dev.0`) will not trigger a publish. This is useful for verifying OSL triage status before creating a final version like `2.4.41`. - -### Step 5: Rebake (CI, automated by GPP) - -GPP automatically runs rebake when it detects a new bake record. The rebake command reproduces the tile from the bake record and verifies the checksum matches: - -```bash -kiln carvel rebake \ - --output-file my-tile-1.0.0.pivotal \ - --variables-file ~/.kiln/credentials.yml \ - bake_records/1.0.0.json -``` - -> [!IMPORTANT] -> The repository must be checked out at the **exact commit** recorded in `source_revision`. The rebake will fail if HEAD does not match. In CI, the Concourse resource handles this automatically. - -The rebake: - -1. Reads the bake record to determine the source revision, version, and expected checksum. -2. Downloads the BOSH release from Artifactory (if Kilnfile.lock is present). -3. Bakes the tile. -4. Verifies the output checksum matches the bake record -- **byte-for-byte reproducibility**. - ---- - -## Command reference - -| Command | Description | Requires Kilnfile? | Requires Kilnfile.lock? | Writes Kilnfile.lock? | -|---------|-------------|:-------------------:|:-----------------------:|:---------------------:| -| `kiln carvel bake` | Local bake from bundle | No | No (uses if present) | No | -| `kiln carvel upload` | Upload BOSH release to Artifactory | **Yes** | No | **Yes** | -| `kiln carvel publish --final` | Bake + create bake record | **Yes** | **Yes** | No | -| `kiln carvel rebake ` | Reproduce tile from bake record | **Yes** (if lock present) | **Yes** (if present) | No | - -### Common flags - -| Flag | Short | Description | Used by | -|------|-------|-------------|---------| -| `--source-directory` | `-s` | Path to tile source directory (defaults to `.`) | `bake`, `upload`, `publish` | -| `--output-file` | `-o` | Path for the output `.pivotal` file | `bake`, `upload` (optional), `publish`, `rebake` | -| `--variable` | `-vr` | Key-value pair for Kilnfile interpolation | All commands | -| `--variables-file` | `-vf` | Path to YAML file with variable values | All commands | -| `--kilnfile` | `-kf` | Path to Kilnfile (default: `Kilnfile` in source dir) | All commands | -| `--verbose` | `-v` | Enable verbose output | All commands | -| `--final` | | Create a bake record | `publish` only | -| `--version` | | Override tile version | `publish` only | - ---- - -## Golden Path configuration - -Configuration for the TAS Golden Path is stored in the [tas-ecosystem-configuration](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration) repo and used as inputs to generate Concourse pipelines. - -> [!NOTE] -> For Carvel-based tiles, you do **not** need to add BOSH release config files to the `bosh/` folder. GPP manages BOSH release ingest and compilation automatically. - -### Tile config onboard - -1. Clone the [tas-ecosystem-configuration](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration) repo and create a branch. - -2. Create a new file for your tile under the `tiles/` directory. - - > [!IMPORTANT] - > `artifact_name` determines file name prefixes in Artifactory, project name prefixes in BlackDuck, and the prefix for published releases in RMT. - - **Example** -- `my-k8s-tile.yml`: - - ```yaml - #@data/values - --- - repo: https://github.gwd.broadcom.net/TNZ/my-k8s-tile.git - branch: main - update_branch: auto-bump - subpath: . - artifact_name: my-k8s-tile - prerelease_format: build_increment_sha - team_members: - - alice@broadcom.com - - bob@broadcom.com - team_google_chat_group: my-team-chat - team_slack_channel: my-team-slack #! optional - ``` - -3. **(Optional)** Add fields for automatic RMT draft-release creation. - - > [!WARNING] - > You will need to add upgrade specifiers (else Upgrade Planner will break!) and verify the release is ready for GA. It defaults to a draft. - - See [tile RMT release](https://github.gwd.broadcom.net/TNZ/tas-ecosystem-configuration/tree/main/docs/tile_rmt_release.md) for details. - -4. **(Optional)** Add BlackDuck tile project associations. - - Confirm your project exists at https://broadcom-vmw.app.blackduck.com/ with the format `TNZ-CF--tile`. If not, submit a ticket via the [BlackDuck Onboarding section](./creating_open_source_license_disclosures.md#blackduck-onboarding). - - Scanning is enabled by default. To disable: - - ```yaml - blackduck: - enabled: false - ``` - -5. Create a PR. An automated check will verify your config. On merge, GPP jobs will be created for your tile. - - Reach out to [#tas-slingshots on Google Chat](https://chat.google.com/room/AAAAZuDvKe0?cls=7) with questions. - -### Optional: auto-bump branch - -Create an `update_branch` (e.g. `autobump`) for GPP to push Kilnfile.lock updates for your review. If `branch` and `update_branch` are the same, force push is disabled. Ensure the bot account has [push access](#github-repo-access) to the specified branch. - ---- - -## End-to-end example - -Here is the complete workflow from first local bake to published tile: - -```bash -# 1. Local bake -- verify your tile structure works -kiln carvel bake -s . -o my-tile-dev.pivotal - -# 2. Upload BOSH release to Artifactory, write lockfile -kiln carvel upload -s . -vf ~/.kiln/credentials.yml - -# 3. Commit the lockfile -git add Kilnfile.lock -git commit -m "Add Kilnfile.lock" - -# 4. CI bake (downloads cached release from Artifactory) -kiln carvel bake -s . -o my-tile-dev.pivotal -vf ~/.kiln/credentials.yml - -# 5. Final release with bake record -kiln carvel publish --final -s . -o my-tile-1.0.0.pivotal -vf ~/.kiln/credentials.yml - -# 6. Commit the bake record -git add bake_records/ -git commit -m "Release 1.0.0" -git push - -# 7. GPP automatically runs rebake, verifies checksum, and publishes to Artifactory -``` - ---- - -## Troubleshooting - -### `Kilnfile not found` - -The `upload`, `publish`, and `rebake` commands require a `Kilnfile` with an `artifactory` release source. Make sure the file exists in your tile's source directory (or pass `--kilnfile path/to/Kilnfile`). - -### `Kilnfile.lock not found` or `no releases` - -Run `kiln carvel upload` first to generate the BOSH release and create the lockfile. - -### `source revision mismatch` during rebake - -The repo must be at the exact commit from the bake record's `source_revision`. Check out that commit before running rebake. - -### `upload failed with status 401` - -Your Artifactory credentials are incorrect or expired. Regenerate your API key or identity token from the [Artifactory UI](https://usw1.packages.broadcom.com/ui). - -### `tile checksum mismatch` during rebake - -The tile produced by rebake does not match the original publish. This can happen if the source tree has been modified after the bake record was created. Ensure no uncommitted changes exist and that HEAD matches `source_revision`. - -### Network errors connecting to Artifactory - -`usw1.packages.broadcom.com` requires Broadcom full-tunnel VPN. Verify you are connected.