diff --git a/go.mod b/go.mod index f86e8097d..7fd93569b 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/go-git/go-git/v5 v5.16.5 github.com/google/go-github/v50 v50.2.0 github.com/jfrog/jfrog-client-go v1.52.0 + github.com/hashicorp/go-version v1.8.0 github.com/julienschmidt/httprouter v1.3.0 github.com/masterminds/sprig v2.22.0+incompatible github.com/maxbrunsfeld/counterfeiter/v6 v6.12.1 @@ -110,7 +111,6 @@ require ( github.com/hashicorp/go-immutable-radix v1.3.1 // indirect github.com/hashicorp/go-memdb v1.3.5 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect - github.com/hashicorp/go-version v1.8.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/huandu/xstrings v1.5.0 // indirect github.com/imdario/mergo v0.3.16 // indirect diff --git a/internal/acceptance/carvel/carvel_bake_test.go b/internal/acceptance/carvel/carvel_bake_test.go new file mode 100644 index 000000000..21272728b --- /dev/null +++ b/internal/acceptance/carvel/carvel_bake_test.go @@ -0,0 +1,200 @@ +package acceptance_test + +import ( + "archive/zip" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" + "github.com/onsi/gomega/gexec" +) + +var pathToMain string + +func TestCarvelAcceptance(t *testing.T) { + SetDefaultEventuallyTimeout(time.Minute) + RegisterFailHandler(Fail) + RunSpecs(t, "carvel acceptance") +} + +var _ = BeforeSuite(func() { + if _, err := exec.LookPath("bosh"); err != nil { + Skip("bosh CLI not installed - skipping carvel acceptance tests") + } + + var err error + pathToMain, err = gexec.Build("github.com/pivotal-cf/kiln") + Expect(err).NotTo(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + gexec.CleanupBuildArtifacts() +}) + +var _ = Describe("carvel bake command", func() { + var ( + outputFile string + tmpDir string + inputPath string + commandWithArgs []string + ) + + const ( + sampleTileFixture = "fixtures/sample-tile" + ) + + BeforeEach(func() { + var err error + tmpDir, err = os.MkdirTemp("", "kiln-carvel-test") + Expect(err).NotTo(HaveOccurred()) + + // Copy the sample-tile fixture to a temp directory + inputPath = filepath.Join(tmpDir, "tile") + err = os.CopyFS(inputPath, os.DirFS(sampleTileFixture)) + Expect(err).NotTo(HaveOccurred()) + + // Initialize git repo (required by kiln for metadata) + gitCommands := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "config", "user.email", "test@test.com"), + exec.Command("git", "config", "user.name", "Test"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range gitCommands { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + outputFile = filepath.Join(tmpDir, "k8s-tile-test-0.0.1.pivotal") + + commandWithArgs = []string{ + "carvel", "bake", + "--source-directory", inputPath, + "--output-file", outputFile, + } + }) + + AfterEach(func() { + _ = os.RemoveAll(tmpDir) + }) + + It("generates a tile with the correct structure", func() { + commandWithArgs = append(commandWithArgs, "--verbose") + + command := exec.Command(pathToMain, commandWithArgs...) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(session, "60s").Should(gexec.Exit(0)) + + archive, err := os.Open(outputFile) + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = archive.Close() }() + + archiveInfo, err := archive.Stat() + Expect(err).NotTo(HaveOccurred()) + + bakedTile, err := zip.NewReader(archive, archiveInfo.Size()) + Expect(err).NotTo(HaveOccurred()) + + // Verify metadata exists + _, err = bakedTile.Open("metadata/metadata.yml") + Expect(err).NotTo(HaveOccurred()) + + // Verify releases directory contains a tgz + var foundRelease bool + for _, f := range bakedTile.File { + if filepath.Dir(f.Name) == "releases" && filepath.Ext(f.Name) == ".tgz" { + foundRelease = true + break + } + } + Expect(foundRelease).To(BeTrue(), "releases/*.tgz should be in the tile") + + // Verify migrations directory exists (even if empty) + var foundMigrations bool + for _, f := range bakedTile.File { + if filepath.Dir(f.Name) == "migrations" || f.Name == "migrations/v1/" { + foundMigrations = true + break + } + } + Expect(foundMigrations).To(BeTrue(), "migrations directory should be in the tile") + + Eventually(session.Out).Should(gbytes.Say("Baked")) + Eventually(session.Out).Should(gbytes.Say("k8s-tile-test")) + }) + + It("produces a valid zip archive", func() { + command := exec.Command(pathToMain, commandWithArgs...) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(session, "60s").Should(gexec.Exit(0)) + + // Verify using unzip -t + verifyCmd := exec.Command("unzip", "-t", outputFile) + verifySession, err := gexec.Start(verifyCmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(verifySession, "10s").Should(gexec.Exit(0)) + Eventually(verifySession.Out).Should(gbytes.Say("No errors detected")) + }) + + Context("failure cases", func() { + Context("when the output-file flag is not provided", func() { + It("prints an error and exits 1", func() { + command := exec.Command(pathToMain, "carvel", "bake", + "--source-directory", inputPath, + ) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(session).Should(gexec.Exit(1)) + Eventually(session.Err).Should(gbytes.Say("output-file")) + }) + }) + + Context("when the source directory does not exist", func() { + It("prints an error and exits 1", func() { + command := exec.Command(pathToMain, "carvel", "bake", + "--source-directory", "/non/existent/path", + "--output-file", outputFile, + ) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(session).Should(gexec.Exit(1)) + }) + }) + + Context("when the source directory is missing base.yml", func() { + It("prints an error and exits 1", func() { + emptyDir, err := os.MkdirTemp(tmpDir, "empty-tile") + Expect(err).NotTo(HaveOccurred()) + + command := exec.Command(pathToMain, "carvel", "bake", + "--source-directory", emptyDir, + "--output-file", outputFile, + ) + + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + + Eventually(session).Should(gexec.Exit(1)) + Eventually(session.Err).Should(gbytes.Say("base.yml")) + }) + }) + }) +}) diff --git a/internal/acceptance/carvel/carvel_workflow_test.go b/internal/acceptance/carvel/carvel_workflow_test.go new file mode 100644 index 000000000..18ff3834b --- /dev/null +++ b/internal/acceptance/carvel/carvel_workflow_test.go @@ -0,0 +1,410 @@ +package acceptance_test + +import ( + "archive/zip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + + "github.com/pivotal-cf/kiln/pkg/bake" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" +) + +// mockArtifactory is a test HTTP server that faithfully simulates Artifactory's +// upload (PUT) and download (GET) behaviour, including Basic Auth verification. +// Upload stores the tarball bytes keyed by request path; download serves them +// back. The /artifactory prefix that the real download client prepends is +// handled transparently. +type mockArtifactory struct { + mu sync.Mutex + blobs map[string][]byte + server *httptest.Server + username string + password string + + putCount int + getCount int +} + +func newMockArtifactory(username, password string) *mockArtifactory { + m := &mockArtifactory{ + blobs: make(map[string][]byte), + username: username, + password: password, + } + m.server = httptest.NewServer(m) + return m +} + +func (m *mockArtifactory) ServeHTTP(w http.ResponseWriter, r *http.Request) { + u, p, ok := r.BasicAuth() + if !ok || u != m.username || p != m.password { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + + // Normalize path: strip the /artifactory prefix the download client adds + key := r.URL.Path + key = strings.TrimPrefix(key, "/artifactory") + + switch r.Method { + case http.MethodPut: + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + m.mu.Lock() + m.blobs[key] = body + m.putCount++ + m.mu.Unlock() + w.WriteHeader(http.StatusCreated) + + case http.MethodGet: + m.mu.Lock() + data, found := m.blobs[key] + m.getCount++ + m.mu.Unlock() + if !found { + http.Error(w, fmt.Sprintf("not found: %s (have: %v)", key, m.storedKeys()), http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(data) + + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } +} + +func (m *mockArtifactory) storedKeys() []string { + keys := make([]string, 0, len(m.blobs)) + for k := range m.blobs { + keys = append(keys, k) + } + return keys +} + +func (m *mockArtifactory) Close() { m.server.Close() } +func (m *mockArtifactory) URL() string { return m.server.URL } + +func (m *mockArtifactory) PutCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.putCount +} + +func (m *mockArtifactory) GetCount() int { + m.mu.Lock() + defer m.mu.Unlock() + return m.getCount +} + +// --------------------------------------------------------------------------- +// End-to-end acceptance test for the full Carvel tile developer workflow. +// +// This test exercises every `kiln carvel` subcommand in sequence, chaining +// their outputs exactly like a real developer and CI pipeline would: +// +// Step 1 kiln carvel bake (local bake, no Kilnfile.lock) +// Step 2 kiln carvel upload (creates BOSH release, uploads, writes lock) +// git add + commit (Kilnfile.lock) +// Step 3 kiln carvel bake (CI-style: downloads from lock) +// Step 4 kiln carvel publish --final (downloads, bakes, writes bake record) +// git add + commit (bake_records/) +// Step 5 kiln carvel rebake (re-bakes from record, verifies checksum) +// +// A mock Artifactory server stores the actual uploaded tarball and serves it +// back on download, proving the full round-trip. +// --------------------------------------------------------------------------- +var _ = Describe("carvel full workflow", Ordered, func() { + const ( + sampleTileFixture = "fixtures/sample-tile" + artUsername = "test-user" + artPassword = "test-pass" + artRepo = "test-repo" + ) + + var ( + tmpDir string + inputPath string + art *mockArtifactory + ) + + variableFlags := func() []string { + return []string{ + "--variable", "artifactory_host=" + art.URL(), + "--variable", "artifactory_repo=" + artRepo, + "--variable", "artifactory_username=" + artUsername, + "--variable", "artifactory_password=" + artPassword, + } + } + + gitInTile := func(args ...string) { + cmd := exec.Command("git", args...) + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "git %v failed: %s", args, string(out)) + } + + currentSHA := func() string { + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = inputPath + out, err := cmd.Output() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return strings.TrimSpace(string(out)) + } + + tileChecksum := func(path string) string { + f, err := os.Open(path) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + defer func() { _ = f.Close() }() + h := sha256.New() + _, err = io.Copy(h, f) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return hex.EncodeToString(h.Sum(nil)) + } + + assertValidTile := func(pivotalPath string) { + archive, err := os.Open(pivotalPath) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + defer func() { _ = archive.Close() }() + + info, err := archive.Stat() + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + zr, err := zip.NewReader(archive, info.Size()) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + + _, err = zr.Open("metadata/metadata.yml") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "tile must contain metadata/metadata.yml") + + var hasRelease bool + for _, f := range zr.File { + if filepath.Dir(f.Name) == "releases" && filepath.Ext(f.Name) == ".tgz" { + hasRelease = true + break + } + } + ExpectWithOffset(1, hasRelease).To(BeTrue(), "tile must contain releases/*.tgz") + } + + BeforeAll(func() { + if _, err := exec.LookPath("bosh"); err != nil { + Skip("bosh CLI not installed — skipping carvel workflow acceptance tests") + } + + var err error + tmpDir, err = os.MkdirTemp("", "kiln-carvel-workflow-*") + Expect(err).NotTo(HaveOccurred()) + + inputPath = filepath.Join(tmpDir, "tile") + err = os.CopyFS(inputPath, os.DirFS(sampleTileFixture)) + Expect(err).NotTo(HaveOccurred()) + + art = newMockArtifactory(artUsername, artPassword) + + gitInTile("init") + gitInTile("config", "user.email", "test@test.com") + gitInTile("config", "user.name", "Test") + gitInTile("add", ".") + gitInTile("commit", "-m", "initial commit") + }) + + AfterAll(func() { + if art != nil { + art.Close() + } + _ = os.RemoveAll(tmpDir) + }) + + // ----------------------------------------------------------------------- + // Step 1: Local bake (no Kilnfile.lock, no Artifactory interaction) + // ----------------------------------------------------------------------- + It("Step 1: bakes a tile locally without Kilnfile.lock", func() { + outputFile := filepath.Join(tmpDir, "step1.pivotal") + + cmd := exec.Command(pathToMain, + append([]string{ + "carvel", "bake", + "--source-directory", inputPath, + "--output-file", outputFile, + "--verbose", + }, variableFlags()...)..., + ) + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "120s").Should(gexec.Exit(0)) + + assertValidTile(outputFile) + + Expect(art.PutCount()).To(Equal(0), "local bake must not upload anything") + Expect(art.GetCount()).To(Equal(0), "local bake must not download anything") + + _, err = os.Stat(filepath.Join(inputPath, "Kilnfile.lock")) + Expect(os.IsNotExist(err)).To(BeTrue(), "local bake must not create Kilnfile.lock") + }) + + // ----------------------------------------------------------------------- + // Step 2: Upload (creates BOSH release, uploads to Artifactory, writes lock) + // ----------------------------------------------------------------------- + It("Step 2: uploads the BOSH release to Artifactory and writes Kilnfile.lock", func() { + cmd := exec.Command(pathToMain, + append([]string{ + "carvel", "upload", + "--source-directory", inputPath, + "--verbose", + }, variableFlags()...)..., + ) + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "120s").Should(gexec.Exit(0)) + + Expect(art.PutCount()).To(Equal(1), "upload must PUT exactly once") + + lockfilePath := filepath.Join(inputPath, "Kilnfile.lock") + Expect(lockfilePath).To(BeAnExistingFile()) + + lockData, err := os.ReadFile(lockfilePath) + Expect(err).NotTo(HaveOccurred()) + + var lock cargo.KilnfileLock + Expect(yaml.Unmarshal(lockData, &lock)).To(Succeed()) + + Expect(lock.Releases).To(HaveLen(1)) + rel := lock.Releases[0] + Expect(rel.Name).To(Equal("k8s-tile-test")) + Expect(rel.Version).To(Equal("0.1.1")) + Expect(rel.SHA1).NotTo(BeEmpty(), "lock must contain SHA1 of uploaded tarball") + Expect(rel.RemoteSource).To(Equal("artifactory")) + Expect(rel.RemotePath).To(Equal("bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz")) + + gitInTile("add", "Kilnfile.lock") + gitInTile("commit", "-m", "add Kilnfile.lock from upload") + }) + + // ----------------------------------------------------------------------- + // Step 3: CI-style bake (downloads cached BOSH release via Kilnfile.lock) + // ----------------------------------------------------------------------- + It("Step 3: bakes a tile using Kilnfile.lock (CI path with Artifactory download)", func() { + outputFile := filepath.Join(tmpDir, "step3-ci.pivotal") + + getCountBefore := art.GetCount() + + cmd := exec.Command(pathToMain, + append([]string{ + "carvel", "bake", + "--source-directory", inputPath, + "--output-file", outputFile, + "--verbose", + }, variableFlags()...)..., + ) + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "120s").Should(gexec.Exit(0)) + + assertValidTile(outputFile) + + Expect(art.GetCount()).To(BeNumerically(">", getCountBefore), + "CI bake must download the cached BOSH release from Artifactory") + }) + + // ----------------------------------------------------------------------- + // Step 4: Publish --final (downloads, bakes, creates bake record) + // ----------------------------------------------------------------------- + var publishChecksum string + + It("Step 4: publishes a final tile and writes a bake record", func() { + outputFile := filepath.Join(tmpDir, "step4-final.pivotal") + + cmd := exec.Command(pathToMain, + append([]string{ + "carvel", "publish", + "--source-directory", inputPath, + "--output-file", outputFile, + "--final", + "--verbose", + }, variableFlags()...)..., + ) + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "120s").Should(gexec.Exit(0)) + + assertValidTile(outputFile) + publishChecksum = tileChecksum(outputFile) + + recordsDir := filepath.Join(inputPath, "bake_records") + Expect(recordsDir).To(BeADirectory()) + + recordPath := filepath.Join(recordsDir, "0.1.1.json") + Expect(recordPath).To(BeAnExistingFile()) + + recordData, err := os.ReadFile(recordPath) + Expect(err).NotTo(HaveOccurred()) + + var record bake.Record + Expect(json.Unmarshal(recordData, &record)).To(Succeed()) + + Expect(record.Version).To(Equal("0.1.1")) + Expect(record.SourceRevision).To(Equal(currentSHA()), + "bake record source_revision must match current HEAD") + Expect(record.FileChecksum).NotTo(BeEmpty()) + Expect(record.FileChecksum).To(Equal(publishChecksum), + "bake record checksum must match the actual tile file") + + // Do NOT commit bake_records yet — rebake must run at the same + // HEAD that publish captured. In a real CI pipeline the + // Concourse resource checks out the commit from the record. + }) + + // ----------------------------------------------------------------------- + // Step 5: Rebake from bake record (reproducibility verification) + // ----------------------------------------------------------------------- + It("Step 5: re-bakes from the bake record with an identical checksum", func() { + outputFile := filepath.Join(tmpDir, "step5-rebake.pivotal") + recordPath := filepath.Join(inputPath, "bake_records", "0.1.1.json") + + args := append([]string{ + "carvel", "rebake", + "--output-file", outputFile, + "--verbose", + }, variableFlags()...) + args = append(args, recordPath) + cmd := exec.Command(pathToMain, args...) + cmd.Dir = inputPath + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "120s").Should(gexec.Exit(0)) + + assertValidTile(outputFile) + + rebakeChecksum := tileChecksum(outputFile) + Expect(rebakeChecksum).To(Equal(publishChecksum), + "rebake must produce a byte-for-byte identical tile to publish") + }) + + // ----------------------------------------------------------------------- + // Meta-assertions: verify the mock was exercised correctly across the + // entire workflow. + // ----------------------------------------------------------------------- + It("exercised Artifactory correctly across the full workflow", func() { + Expect(art.PutCount()).To(Equal(1), + "exactly one upload should have occurred across the entire workflow") + Expect(art.GetCount()).To(BeNumerically(">=", 2), + "at least two downloads should have occurred (CI bake + publish)") + }) +}) diff --git a/internal/acceptance/carvel/fixtures/sample-tile/.gitignore b/internal/acceptance/carvel/fixtures/sample-tile/.gitignore new file mode 100644 index 000000000..95270bce3 --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/.gitignore @@ -0,0 +1,3 @@ +.boshrelease +.carvel-tile + diff --git a/internal/acceptance/carvel/fixtures/sample-tile/Kilnfile b/internal/acceptance/carvel/fixtures/sample-tile/Kilnfile new file mode 100644 index 000000000..3da3ae25c --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/Kilnfile @@ -0,0 +1,8 @@ +--- +release_sources: + - type: artifactory + artifactory_host: $( variable "artifactory_host" ) + repo: $( variable "artifactory_repo" ) + username: $( variable "artifactory_username" ) + password: $( variable "artifactory_password" ) + path_template: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz" diff --git a/internal/acceptance/carvel/fixtures/sample-tile/README.md b/internal/acceptance/carvel/fixtures/sample-tile/README.md new file mode 100644 index 000000000..1501223a8 --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/README.md @@ -0,0 +1,4 @@ +# Sample Kubernetes Tile +This is a test "unbaked" kubernetes tile. It includes a fake imgpkg bundle tarball that doesn't actually contain a packagereop. + +It is only for integration testing purposes and cannot be deployed. diff --git a/internal/acceptance/carvel/fixtures/sample-tile/base.yml b/internal/acceptance/carvel/fixtures/sample-tile/base.yml new file mode 100644 index 000000000..d91f08bba --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/base.yml @@ -0,0 +1,19 @@ +name: k8s-tile-test +label: "test tile" +icon_image: $( icon ) +metadata_version: "3.2.0" +minimum_version_for_upgrade: 0.0.0 +product_version: $( version ) +rank: 1 +serial: false +property_blueprints: +- $( property "database_name" ) +- $( property "admin_password" ) +form_types: +- $( form "db_props" ) +variables: [] +package_installs: +- $( package "test-install" ) +compatible_kubernetes_distributions: +- name: k0s + version: '>0.0.0' diff --git a/internal/acceptance/carvel/fixtures/sample-tile/bundle.tar b/internal/acceptance/carvel/fixtures/sample-tile/bundle.tar new file mode 100644 index 000000000..6c4cdbde9 Binary files /dev/null and b/internal/acceptance/carvel/fixtures/sample-tile/bundle.tar differ diff --git a/internal/acceptance/carvel/fixtures/sample-tile/forms/.gitkeep b/internal/acceptance/carvel/fixtures/sample-tile/forms/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/acceptance/carvel/fixtures/sample-tile/forms/db_props.yml b/internal/acceptance/carvel/fixtures/sample-tile/forms/db_props.yml new file mode 100644 index 000000000..2d41e9aa7 --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/forms/db_props.yml @@ -0,0 +1,8 @@ +--- +name: "db_props" +label: "Postgres Database Properties" +description: "Postgres Database Properties" +markdown: 'Postgres Database Properties' +property_inputs: +- reference: ".properties.database_name" + label: "Database name" diff --git a/internal/acceptance/carvel/fixtures/sample-tile/icon.png b/internal/acceptance/carvel/fixtures/sample-tile/icon.png new file mode 100644 index 000000000..d556b41cc Binary files /dev/null and b/internal/acceptance/carvel/fixtures/sample-tile/icon.png differ diff --git a/internal/acceptance/carvel/fixtures/sample-tile/packageinstalls/test-install.yml b/internal/acceptance/carvel/fixtures/sample-tile/packageinstalls/test-install.yml new file mode 100644 index 000000000..eccf4450a --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/packageinstalls/test-install.yml @@ -0,0 +1,6 @@ +name: test-install +packageName: something-test.tanzu.vmware.com +packageVersion: "0.1.5" +values: + db_name: (( .properties.database_name.value )) + password: (( .properties.admin_password.value )) diff --git a/internal/acceptance/carvel/fixtures/sample-tile/properties/.gitkeep b/internal/acceptance/carvel/fixtures/sample-tile/properties/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/acceptance/carvel/fixtures/sample-tile/properties/properties.yml b/internal/acceptance/carvel/fixtures/sample-tile/properties/properties.yml new file mode 100644 index 000000000..de36e0aa6 --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/properties/properties.yml @@ -0,0 +1,9 @@ +- name: admin_password + type: secret + configurable: false + optional: false +- name: database_name + type: string + configurable: true + optional: false + default: my-db diff --git a/internal/acceptance/carvel/fixtures/sample-tile/version b/internal/acceptance/carvel/fixtures/sample-tile/version new file mode 100644 index 000000000..17e51c385 --- /dev/null +++ b/internal/acceptance/carvel/fixtures/sample-tile/version @@ -0,0 +1 @@ +0.1.1 diff --git a/internal/carvel/TODO.md b/internal/carvel/TODO.md new file mode 100644 index 000000000..6fb4cce77 --- /dev/null +++ b/internal/carvel/TODO.md @@ -0,0 +1,26 @@ +# Carvel Package TODOs + +## Refactor: Replace exec.Command("kiln bake") with internal function call + +**File:** `baker.go` (line 46) + +**Current behavior:** `KilnBake()` shells out to the system `kiln` binary via +`exec.Command("kiln", "bake", "--skip-fetch", "--output-file", destination)`. + +**Problem:** +- The inner `kiln bake` resolves to whatever binary is on PATH, which may be a + different version than the running `kiln carvel bake`. +- Integration tests must manipulate PATH to ensure the correct binary is used. +- Spawning a subprocess for logic that exists in the same codebase is unnecessary + overhead. +- Error propagation across the process boundary is lossy. + +**Desired behavior:** `KilnBake()` should call the internal bake logic directly +(e.g. instantiate and invoke `commands.Bake` or the underlying `BakeService`) +instead of shelling out. This guarantees version consistency, improves +testability, and removes the PATH dependency. + +**Complexity note:** The `Bake` command has a non-trivial setup (BakeService, +fetchers, template evaluators, checksummers). The wiring will need to be +extracted into a reusable helper or the relevant subset of bake logic factored +out for in-process use. diff --git a/internal/carvel/baker.go b/internal/carvel/baker.go new file mode 100644 index 000000000..e2f5bb170 --- /dev/null +++ b/internal/carvel/baker.go @@ -0,0 +1,713 @@ +package carvel + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strings" + + "github.com/pivotal-cf/kiln/internal/carvel/models" + "github.com/pivotal-cf/kiln/pkg/cargo" + + "github.com/hashicorp/go-version" + "gopkg.in/yaml.v3" +) + +// Baker transforms an imgpkg bundle and tile metadata into a BOSH release +// and kiln-compatible tile structure that can be baked into a .pivotal file. +type Baker interface { + Bake(source string) error + BakeFromLockfile(source string, releaseLock cargo.BOSHReleaseTarballLock, localTarball string) error + KilnBake(destination string) error + GetName() string + GetVersion() (string, error) + GetReleaseTarball() (string, error) + SetWriter(w io.Writer) + SetProgressWriter(w io.Writer) +} + +// NewBaker creates a new Baker for transforming imgpkg bundles into BOSH releases. +func NewBaker() Baker { + return &baker{ + writer: io.Discard, + progressWriter: io.Discard, + } +} + +type baker struct { + metadata models.Metadata + source, destination string + writer io.Writer + progressWriter io.Writer +} + +func (b *baker) KilnBake(destination string) error { + b.progress("Assembling final .pivotal file...") + cmd := exec.Command("kiln", + "bake", + "--skip-fetch", + "--output-file", destination, + ) + cmd.Dir = b.destination + out, err := cmd.CombinedOutput() + b.log(string(out)) + if err != nil { + b.log("failed to invoke kiln: " + string(out)) + return err + } + + return nil +} + +func (b *baker) Bake(source string) error { + b.source = source + b.destination = path.Join(source, ".carvel-tile") + + b.progress("Reading tile metadata from " + path.Join(source, "base.yml")) + yamlPath := path.Join(source, "base.yml") + yamlData, err := os.ReadFile(yamlPath) + if err != nil { + return err + } + + err = yaml.Unmarshal(yamlData, &b.metadata) + if err != nil { + return err + } + + ver, err := b.GetVersion() + if err != nil { + return err + } + b.progress(fmt.Sprintf("Tile: %s version %s (metadata_version %s)", b.metadata.Name, ver, b.metadata.MetadataVersion)) + + metadataVersion, err := version.NewVersion(b.metadata.MetadataVersion) + if err != nil { + return err + } + minVersion, _ := version.NewVersion("3.2.0") + if metadataVersion.LessThan(minVersion) { + return errors.New("tile metadata_version too old for kubernetes support (must be >=3.2.0)") + } + + b.progress("Generating BOSH release structure...") + err = b.generateBoshReleaseDir() + if err != nil { + b.log(err.Error()) + return err + } + + b.progress("Generating tile layout in " + b.destination) + err = b.generateOutputTile() + if err != nil { + b.log(err.Error()) + return err + } + + return nil +} + +func (b *baker) BakeFromLockfile(source string, releaseLock cargo.BOSHReleaseTarballLock, localTarball string) error { + b.source = source + b.destination = path.Join(source, ".carvel-tile") + + b.progress("Reading tile metadata from " + path.Join(source, "base.yml")) + yamlPath := path.Join(source, "base.yml") + yamlData, err := os.ReadFile(yamlPath) + if err != nil { + return err + } + + err = yaml.Unmarshal(yamlData, &b.metadata) + if err != nil { + return err + } + + ver, err := b.GetVersion() + if err != nil { + return err + } + b.progress(fmt.Sprintf("Tile: %s version %s (metadata_version %s)", b.metadata.Name, ver, b.metadata.MetadataVersion)) + + if releaseLock.Name != b.metadata.Name { + return fmt.Errorf("lockfile release name %q does not match tile name %q", releaseLock.Name, b.metadata.Name) + } + + err = os.RemoveAll(b.destination) + if err != nil { + return err + } + err = os.MkdirAll(b.destination, 0755) + if err != nil { + return err + } + + b.progress("Generating tile layout in " + b.destination) + err = b.generateBaseYaml() + if err != nil { + return err + } + err = b.copyFiles() + if err != nil { + return err + } + err = b.generateJobFiles() + if err != nil { + return err + } + err = b.generateInstanceGroupFiles() + if err != nil { + return err + } + err = b.generateRuntimeConfigs() + if err != nil { + return err + } + + releasesDir := path.Join(b.destination, "releases") + err = os.MkdirAll(releasesDir, 0755) + if err != nil { + return err + } + + destTarball := path.Join(releasesDir, b.metadata.Name+"-"+ver+".tgz") + + b.progress("Copying cached BOSH release from " + localTarball) + b.log("copying cached BOSH release from " + localTarball) + err = copyFileContents(localTarball, destTarball) + if err != nil { + return fmt.Errorf("failed to copy cached release tarball: %w", err) + } + + return nil +} + +func (b *baker) GetReleaseTarball() (string, error) { + ver, err := b.GetVersion() + if err != nil { + return "", err + } + tarball := path.Join(b.destination, "releases", b.metadata.Name+"-"+ver+".tgz") + if _, err := os.Stat(tarball); err != nil { + return "", fmt.Errorf("release tarball not found at %s: %w", tarball, err) + } + return tarball, nil +} + +func (b *baker) GetName() string { + return b.metadata.Name +} + +func (b *baker) GetVersion() (string, error) { + re := regexp.MustCompile(`\s+`) + + // Replace all occurrences of whitespace with an empty string + versionNoSpace := re.ReplaceAllString(b.metadata.ProductVersion, "") + if versionNoSpace != `$(version)` { + return versionNoSpace, nil + } else { + // find the version from a "version" file + version, err := os.ReadFile(path.Join(b.source, "version")) + return strings.Trim(string(version), " \t\n\r"), err + } +} + +func (b *baker) SetWriter(w io.Writer) { + b.writer = w +} + +func (b *baker) SetProgressWriter(w io.Writer) { + b.progressWriter = w +} + +func (b *baker) log(message string) { + _, _ = fmt.Fprintln(b.writer, message) +} + +func (b *baker) progress(message string) { + _, _ = fmt.Fprintln(b.progressWriter, message) +} + +func (b *baker) generateBoshReleaseDir() error { + dirName := path.Join(b.source, ".boshrelease") + err := os.RemoveAll(dirName) + if err != nil { + return err + } + + b.progress(" Initializing BOSH release") + commands := []*exec.Cmd{ + exec.Command("bosh", "init-release", "--dir="+dirName), + exec.Command("bosh", "add-blob", "--dir="+dirName, path.Join(b.source, "bundle.tar"), "imgpkg/bundle.tar"), + exec.Command("bosh", "generate-package", "--dir="+dirName, "registry-data"), + exec.Command("bosh", "generate-job", "--dir="+dirName, "registry-data"), + } + for _, cmd := range commands { + b.log("executing " + cmd.String()) + out, err := cmd.CombinedOutput() + if err != nil { + return err + } + + b.log("output: " + string(out)) + } + + // Now populate the specs for packages and jobs + fileContents := map[string]string{ + "packages/registry-data/packaging": `set -eu +mkdir -p ${BOSH_INSTALL_TARGET}/imgpkg +cp imgpkg/*.tar ${BOSH_INSTALL_TARGET}/imgpkg +`, + "packages/registry-data/spec": `--- +name: registry-data +dependencies: [] +files: +- imgpkg/bundle.tar +`, + } + for outpath, contents := range fileContents { + err = os.WriteFile(path.Join(dirName, outpath), []byte(contents), 0644) + if err != nil { + return err + } + } + + registryDataTemplates := "" + registryDataProperties := "" + + b.progress(" Configuring package installs") + for _, entry := range b.metadata.PackageInstalls { + entry = strings.Trim(entry, "$() ") + entry = strings.TrimPrefix(entry, "package") + entry = strings.Trim(entry, `"' `) + + b.progress(" - " + entry) + b.log("looking for package install: " + entry) + + // find this entry in the packageinstalls directory + matches, err := filepath.Glob(path.Join(b.source, "packageinstalls/*.yml")) + if err != nil { + return err + } + + for _, match := range matches { + yamlData, err := os.ReadFile(match) + if err != nil { + return err + } + + var pi models.PackageInstall + err = yaml.Unmarshal(yamlData, &pi) + if err != nil { + return err + } + if pi.Name != entry { + continue + } + + b.log("found " + pi.Name + " at " + match) + } + + registryDataTemplates += fmt.Sprintf(" packageinstalls/%s.yml.erb: packageinstalls/%s.yml\n", entry, entry) + + registryDataProperties += " " + entry + ":\n" + registryDataProperties += ` name: + description: "package name" + version: + description: "package version" + values: + description: "values.yml contents" +` + + if err = os.MkdirAll(path.Join(dirName, "jobs", "registry-data", "templates", "packageinstalls"), 0755); err != nil { + return err + } + + manifestTemplate := generateManifestTemplate(entry) + + err = os.WriteFile( + path.Join(dirName, "jobs", "registry-data", "templates", "packageinstalls", entry+".yml.erb"), + []byte(manifestTemplate), + 0644, + ) + if err != nil { + return err + } + } + + registryDataSpec := `--- +name: registry-data +templates: +` + registryDataTemplates + + `packages: +- registry-data +consumes: +- name: cluster + type: cluster-info + optional: true +properties: +` + registryDataProperties + + err = os.WriteFile(path.Join(dirName, "jobs", "registry-data", "spec"), []byte(registryDataSpec), 0644) + if err != nil { + return err + } + + return nil +} + +func generateManifestTemplate(entry string) string { + return `--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: <%= p("` + entry + `.name") %>-sa + namespace: <%= link("cluster").p("content-namespace") rescue "default" %> +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: <%= p("` + entry + `.name") %>-sa-cluster-role +rules: +- apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: <%= p("` + entry + `.name") %>-sa-cluster-role-binding +subjects: +- kind: ServiceAccount + name: <%= p("` + entry + `.name") %>-sa + namespace: <%= link("cluster").p("content-namespace") rescue "default" %> +roleRef: + kind: ClusterRole + name: <%= p("` + entry + `.name") %>-sa-cluster-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: v1 +kind: Secret +metadata: + name: <%= p("` + entry + `.name") %>-values + namespace: <%= link("cluster").p("content-namespace") rescue "default" %> +type: Opaque +stringData: + values.yaml: | +<% require 'yaml' %> +<% + values = p("` + entry + `.values") + values = YAML.load(values) if values.is_a?(String) + # Inject namespace from BOSH link into context + if values.is_a?(Hash) && values["context"].is_a?(Hash) + values["context"]["namespace"] = link("cluster").p("content-namespace") rescue "default" + end +%> +<%= YAML.dump(values).split("\n").map { |line| " " + line }.join("\n") %> +--- +apiVersion: packaging.carvel.dev/v1alpha1 +kind: PackageInstall +metadata: + name: <%= p("` + entry + `.name") %> + namespace: <%= link("cluster").p("content-namespace") rescue "default" %> +spec: + serviceAccountName: <%= p("` + entry + `.name") %>-sa + packageRef: + refName: <%= p("` + entry + `.name") %> + versionSelection: + constraints: <%= p("` + entry + `.version") %> + values: + - secretRef: + name: <%= p("` + entry + `.name") %>-values +` +} + +func (b *baker) generateOutputTile() error { + err := os.RemoveAll(b.destination) + if err != nil { + return err + } + + err = os.MkdirAll(b.destination, 0755) + if err != nil { + return err + } + + b.progress(" Generating base.yml") + err = b.generateBaseYaml() + if err != nil { + return err + } + + b.progress(" Copying forms, properties, and static assets") + err = b.copyFiles() + if err != nil { + return err + } + + err = b.generateJobFiles() + if err != nil { + return err + } + + err = b.generateInstanceGroupFiles() + if err != nil { + return err + } + + b.progress(" Generating runtime configs") + err = b.generateRuntimeConfigs() + if err != nil { + return err + } + + b.progress(" Creating BOSH release tarball (this may take a while)...") + err = b.createBoshRelease() + if err != nil { + return err + } + + return nil +} + +func (b *baker) generateBaseYaml() error { + meta := models.MetadataOut{} + meta.Name = b.metadata.Name + meta.Label = b.metadata.Label + meta.IconImage = b.metadata.IconImage + meta.ProductVersion = b.metadata.ProductVersion + meta.MetadataVersion = b.metadata.MetadataVersion + meta.Rank = b.metadata.Rank + meta.Serial = b.metadata.Serial + meta.CompatibleKubernetesDistributions = b.metadata.CompatibleKubernetesDistributions + meta.FormTypes = b.metadata.FormTypes + meta.PropertyBlueprints = b.metadata.PropertyBlueprints + meta.Variables = b.metadata.Variables + meta.MinimumVersionForUpgrade = b.metadata.MinimumVersionForUpgrade + meta.RequiresKubernetes = true + // stemcell criteria are dummy data that OM will ignore when the tile is folded into + // TKR, but we need them as kiln inputs. + meta.StemcellCriteria.Os = "ubuntu-jammy" + meta.StemcellCriteria.Version = "1.446" + meta.InstanceGroups = []string{} + meta.RuntimeConfigs = []string{ + `$( runtime_config "` + b.metadata.Name + `-pkgr" )`, + } + + // we will use the tile name and version as the bosh release name and version. + meta.Releases = []string{ + `$( release "` + b.metadata.Name + `" )`, + } + + yamlData, err := yaml.Marshal(&meta) + if err != nil { + return err + } + err = os.WriteFile(path.Join(b.destination, "base.yml"), yamlData, 0644) // 0644 sets file permissions + if err != nil { + return err + } + return nil +} + +// copyfiles schleps all the files that we can collect from the source directory without +// modification: +// - variables, properties and forms +// - the icon file +// - the version file +func (b *baker) copyFiles() error { + for _, subdir := range []string{"bosh_variables", "forms", "properties"} { + info, err := os.Stat(path.Join(b.source, subdir)) + if err == nil && info.IsDir() { + err = os.CopyFS(path.Join(b.destination, subdir), os.DirFS(path.Join(b.source, subdir))) + if err != nil { + return err + } + } + } + + for _, fn := range []string{"icon.png", "version"} { + info, statErr := os.Stat(path.Join(b.source, fn)) + if statErr == nil && !info.IsDir() { + if err := copyFileContents(path.Join(b.source, fn), path.Join(b.destination, fn)); err != nil { + return err + } + } + } + + return nil +} + +func (b *baker) generateRuntimeConfigs() error { + err := os.MkdirAll(path.Join(b.destination, "runtime_configs"), 0755) + if err != nil { + return err + } + + registryDataProps := map[string]models.PackageInstallProps{} + + // we need one PackageInstall for each entry in the metadata. + for _, entry := range b.metadata.PackageInstalls { + entry = strings.Trim(entry, "$() ") + entry = strings.TrimPrefix(entry, "package") + entry = strings.Trim(entry, `"' `) + + // find this entry in the packageinstalls directory + matches, err := filepath.Glob(path.Join(b.source, "packageinstalls/*.yml")) + if err != nil { + return err + } + + found := false + for _, match := range matches { + yamlData, err := os.ReadFile(match) + if err != nil { + return err + } + + var pi models.PackageInstall + err = yaml.Unmarshal(yamlData, &pi) + if err != nil { + return err + } + if pi.Name != entry { + continue + } + + found = true + registryDataProps[entry] = models.PackageInstallProps{ + Name: pi.PackageName, + Version: pi.PackageVersion, + Values: pi.Values, + } + } + if !found { + return errors.New("package install not found: " + entry) + } + } + + registryDataJob := models.Job{ + Name: "registry-data", + Release: b.metadata.Name, + Properties: registryDataProps, + } + + inner := models.RuntimeConfigInner{ + Releases: []string{ + `$( release "` + b.metadata.Name + `" )`, + }, + Addons: []models.Addon{ + { + Name: b.metadata.Name + "-pkgr", + Include: models.Inclusion{ + Deployments: []string{ + `(( ..` + b.metadata.Name + `.deployment_name ))`, + }, + Jobs: []models.Job{ + {Name: "install-package-repository", Release: "tanzu-content"}, + {Name: "install-packages", Release: "tanzu-content"}, + }, + }, + Jobs: []models.Job{ + registryDataJob, + }, + }, + }, + } + yamlData, err := yaml.Marshal(&inner) + if err != nil { + return err + } + rc := models.RuntimeConfigOuter{ + Name: b.metadata.Name + "-pkgr", + RuntimeConfig: string(yamlData), + } + + yamlData, err = yaml.Marshal(&rc) + if err != nil { + return err + } + err = os.WriteFile(path.Join(b.destination, "runtime_configs", b.metadata.Name+"-pkgr.yml"), yamlData, 0644) + if err != nil { + return err + } + + return nil +} + +// generateInstanceGroups creates an empty instance group folder +func (b *baker) generateInstanceGroupFiles() error { + err := os.MkdirAll(path.Join(b.destination, "instance_groups"), 0755) + if err != nil { + return err + } + + return nil +} + +// generateJobFiles creates an empty jobs folder +func (b *baker) generateJobFiles() error { + err := os.MkdirAll(path.Join(b.destination, "jobs"), 0755) + if err != nil { + return err + } + + return nil +} + +func (b *baker) createBoshRelease() error { + err := os.MkdirAll(path.Join(b.destination, "releases"), 0755) + if err != nil { + return err + } + + version, err := b.GetVersion() + if err != nil { + return err + } + + dirName := path.Join(b.source, ".boshrelease") + cmd := exec.Command("bosh", + "create-release", + "--dir="+dirName, + "--force", + "--name", b.metadata.Name, + "--version", version, + "--tarball", path.Join(b.destination, "releases", b.metadata.Name+"-"+version+".tgz")) + b.log("executing " + cmd.String()) + out, err := cmd.CombinedOutput() + b.log("output: " + string(out)) + if err != nil { + return err + } + + return nil +} + +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer func() { _ = in.Close() }() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} diff --git a/internal/carvel/baker_suite_test.go b/internal/carvel/baker_suite_test.go new file mode 100644 index 000000000..6ce6cf99a --- /dev/null +++ b/internal/carvel/baker_suite_test.go @@ -0,0 +1,13 @@ +package carvel + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBaker(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Carvel Baker Suite") +} diff --git a/internal/carvel/baker_test.go b/internal/carvel/baker_test.go new file mode 100644 index 000000000..9e590786c --- /dev/null +++ b/internal/carvel/baker_test.go @@ -0,0 +1,491 @@ +package carvel + +import ( + "crypto/sha256" + "encoding/hex" + "io" + "os" + "os/exec" + "path" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/carvel/models" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" +) + +func copyTestFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer func() { _ = in.Close() }() + out, err := os.Create(dst) + if err != nil { + return err + } + defer func() { _ = out.Close() }() + _, err = io.Copy(out, in) + return err +} + +func fileChecksum(path string) string { + f, err := os.Open(path) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + defer func() { _ = f.Close() }() + h := sha256.New() + _, err = io.Copy(h, f) + ExpectWithOffset(1, err).NotTo(HaveOccurred()) + return hex.EncodeToString(h.Sum(nil)) +} + +func boshInstalled() bool { + _, err := exec.LookPath("bosh") + return err == nil +} + +func kilnInstalled() bool { + _, err := exec.LookPath("kiln") + return err == nil +} + +var _ = Describe("Carvel Baker", func() { + Context("generateManifestTemplate", func() { + var template string + + BeforeEach(func() { + template = generateManifestTemplate("test-install") + }) + + It("generates a ServiceAccount", func() { + Expect(template).To(ContainSubstring("kind: ServiceAccount")) + Expect(template).To(ContainSubstring(`name: <%= p("test-install.name") %>-sa`)) + }) + + It("generates a ClusterRole (not a namespaced Role)", func() { + Expect(template).To(ContainSubstring("kind: ClusterRole")) + Expect(template).NotTo(ContainSubstring("kind: Role\n")) + Expect(template).To(ContainSubstring(`name: <%= p("test-install.name") %>-sa-cluster-role`)) + }) + + It("generates a ClusterRoleBinding (not a namespaced RoleBinding)", func() { + Expect(template).To(ContainSubstring("kind: ClusterRoleBinding")) + Expect(template).NotTo(ContainSubstring("kind: RoleBinding\n")) + Expect(template).To(ContainSubstring(`name: <%= p("test-install.name") %>-sa-cluster-role-binding`)) + }) + + It("generates a Secret for values", func() { + Expect(template).To(ContainSubstring("kind: Secret")) + Expect(template).To(ContainSubstring(`name: <%= p("test-install.name") %>-values`)) + Expect(template).To(ContainSubstring("stringData:")) + Expect(template).To(ContainSubstring("values.yaml: |")) + }) + + It("generates a PackageInstall resource", func() { + Expect(template).To(ContainSubstring("kind: PackageInstall")) + Expect(template).To(ContainSubstring("apiVersion: packaging.carvel.dev/v1alpha1")) + Expect(template).To(ContainSubstring(`name: <%= p("test-install.name") %>`)) + Expect(template).To(ContainSubstring(`serviceAccountName: <%= p("test-install.name") %>-sa`)) + }) + + It("uses BOSH link for content-namespace with fallback to default", func() { + Expect(template).To(ContainSubstring(`<%= link("cluster").p("content-namespace") rescue "default" %>`)) + }) + + It("injects content-namespace from BOSH link into values context", func() { + Expect(template).To(ContainSubstring(`values["context"]["namespace"] = link("cluster").p("content-namespace") rescue "default"`)) + }) + + It("handles YAML conversion for string values", func() { + Expect(template).To(ContainSubstring(`values = YAML.load(values) if values.is_a?(String)`)) + }) + }) + + Context("Bake", func() { + When("the input directory contains k8s tile data", func() { + BeforeEach(func() { + if !boshInstalled() { + Skip("bosh CLI not installed - skipping integration test") + } + }) + var ( + inputPath, outputPath, boshReleasePath string + subject Baker + err error + ) + BeforeEach(func() { + var err error + inputPath, err = os.MkdirTemp("", "testinput-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + outputPath = path.Join(inputPath, ".carvel-tile") + boshReleasePath = path.Join(inputPath, ".boshrelease") + err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + // create an initial git commit in the input directory + commands := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range commands { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + subject = NewBaker() + subject.SetWriter(GinkgoWriter) + }) + AfterEach(func() { + // Clean up the temp directory + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + }) + JustBeforeEach(func() { + err = subject.Bake(inputPath) + }) + When("the tile data is valid", func() { + JustBeforeEach(func() { + Expect(err).NotTo(HaveOccurred()) + }) + It("populates the output metadata", func() { + outMeta := models.MetadataOut{} + yamlPath := path.Join(outputPath, "base.yml") + yamlData, err := os.ReadFile(yamlPath) + Expect(err).NotTo(HaveOccurred()) + + err = yaml.Unmarshal(yamlData, &outMeta) + Expect(err).NotTo(HaveOccurred()) + + Expect(outMeta.Name).To(Equal("k8s-tile-test")) + Expect(outMeta.ProductVersion).To(Equal(`$( version )`)) + Expect(outMeta.MetadataVersion).To(Equal("3.2.0")) + Expect(outMeta.Rank).To(Equal(1)) + Expect(outMeta.Serial).To(BeFalse()) + Expect(outMeta.PropertyBlueprints).To(HaveLen(2)) + Expect(outMeta.FormTypes).To(HaveLen(1)) + Expect(outMeta.Variables).To(BeEmpty()) + Expect(outMeta.Releases).To(HaveLen(1)) + Expect(outMeta.Releases[0]).To(ContainSubstring("k8s-tile-test")) + Expect(outMeta.InstanceGroups).To(HaveLen(0)) + Expect(outMeta.RuntimeConfigs).To(HaveLen(1)) + Expect(outMeta.RuntimeConfigs[0]).To(Equal(`$( runtime_config "k8s-tile-test-pkgr" )`)) + Expect(outMeta.CompatibleKubernetesDistributions).To(HaveLen(1)) + Expect(outMeta.CompatibleKubernetesDistributions[0].Name).To(Equal("k0s")) + Expect(outMeta.CompatibleKubernetesDistributions[0].Version).To(Equal(">0.0.0")) + Expect(outMeta.RequiresKubernetes).To(BeTrue()) + }) + It("creates empty instance_group and jobs directories", func() { + Expect(filepath.Join(outputPath, "instance_groups")).To(BeADirectory()) + Expect(filepath.Join(outputPath, "jobs")).To(BeADirectory()) + }) + It("creates a runtime config", func() { + Expect(filepath.Join(outputPath, "runtime_configs")).To(BeADirectory()) + Expect(filepath.Join(outputPath, "runtime_configs", "k8s-tile-test-pkgr.yml")).To(BeAnExistingFile()) + }) + It("copies forms, properties, icon, version from the input", func() { + Expect(filepath.Join(outputPath, "properties", "properties.yml")).To(BeAnExistingFile()) + Expect(filepath.Join(outputPath, "forms", "db_props.yml")).To(BeAnExistingFile()) + Expect(filepath.Join(outputPath, "icon.png")).To(BeAnExistingFile()) + Expect(filepath.Join(outputPath, "version")).To(BeAnExistingFile()) + }) + It("Generates a bosh release tarball", func() { + Expect(filepath.Join(outputPath, "releases", "k8s-tile-test-0.1.1.tgz")).To(BeAnExistingFile()) + }) + It("does not generate a separate package-install job", func() { + Expect(filepath.Join(boshReleasePath, "jobs", "package-install")).NotTo(BeADirectory()) + }) + It("generates manifest templates under registry-data job", func() { + templatePath := filepath.Join(boshReleasePath, "jobs", "registry-data", "templates", "packageinstalls", "test-install.yml.erb") + Expect(templatePath).To(BeAnExistingFile()) + + contents, err := os.ReadFile(templatePath) + Expect(err).NotTo(HaveOccurred()) + templateStr := string(contents) + + Expect(templateStr).To(ContainSubstring("kind: ServiceAccount")) + Expect(templateStr).To(ContainSubstring("kind: ClusterRole")) + Expect(templateStr).To(ContainSubstring("kind: ClusterRoleBinding")) + Expect(templateStr).To(ContainSubstring("kind: Secret")) + Expect(templateStr).To(ContainSubstring("kind: PackageInstall")) + Expect(templateStr).To(ContainSubstring(`link("cluster").p("content-namespace")`)) + }) + It("generates registry-data job spec with BOSH link consumer and templates", func() { + specPath := filepath.Join(boshReleasePath, "jobs", "registry-data", "spec") + Expect(specPath).To(BeAnExistingFile()) + + contents, err := os.ReadFile(specPath) + Expect(err).NotTo(HaveOccurred()) + specStr := string(contents) + + Expect(specStr).To(ContainSubstring("name: registry-data")) + Expect(specStr).To(ContainSubstring("packageinstalls/test-install.yml.erb: packageinstalls/test-install.yml")) + Expect(specStr).To(ContainSubstring("packages:\n- registry-data")) + Expect(specStr).To(ContainSubstring("consumes:")) + Expect(specStr).To(ContainSubstring("name: cluster")) + Expect(specStr).To(ContainSubstring("type: cluster-info")) + Expect(specStr).To(ContainSubstring("optional: true")) + }) + It("generates runtime config referencing tanzu-content release", func() { + rcPath := filepath.Join(outputPath, "runtime_configs", "k8s-tile-test-pkgr.yml") + rcData, err := os.ReadFile(rcPath) + Expect(err).NotTo(HaveOccurred()) + + var rc models.RuntimeConfigOuter + err = yaml.Unmarshal(rcData, &rc) + Expect(err).NotTo(HaveOccurred()) + + var inner models.RuntimeConfigInner + err = yaml.Unmarshal([]byte(rc.RuntimeConfig), &inner) + Expect(err).NotTo(HaveOccurred()) + + Expect(inner.Addons).To(HaveLen(1)) + addon := inner.Addons[0] + + By("referencing tanzu-content release instead of registry") + Expect(addon.Include.Jobs).To(HaveLen(2)) + Expect(addon.Include.Jobs[0].Name).To(Equal("install-package-repository")) + Expect(addon.Include.Jobs[0].Release).To(Equal("tanzu-content")) + Expect(addon.Include.Jobs[1].Name).To(Equal("install-packages")) + Expect(addon.Include.Jobs[1].Release).To(Equal("tanzu-content")) + + By("having only the registry-data job (no separate package-install job)") + Expect(addon.Jobs).To(HaveLen(1)) + Expect(addon.Jobs[0].Name).To(Equal("registry-data")) + Expect(addon.Jobs[0].Release).To(Equal("k8s-tile-test")) + + By("carrying package install properties on the registry-data job") + Expect(addon.Jobs[0].Properties).To(HaveKey("test-install")) + props := addon.Jobs[0].Properties["test-install"] + Expect(props.Name).To(Equal("something-test.tanzu.vmware.com")) + Expect(props.Version).To(Equal("0.1.5")) + }) + It("can be kiln baked", func() { + if !kilnInstalled() { + Skip("kiln CLI not installed - skipping integration test") + } + err := subject.KilnBake(filepath.Join(outputPath, "my-tile.pivotal")) + Expect(err).NotTo(HaveOccurred()) + Expect(filepath.Join(outputPath, "my-tile.pivotal")).To(BeAnExistingFile()) + }) + }) + When("the tile metadata version is too old", func() { + BeforeEach(func() { + m := models.Metadata{ + Name: "k8s-tile-test", + Label: "test tile", + IconImage: "$( icon )", + MetadataVersion: "3.1.0", + MinimumVersionForUpgrade: "0.0.0", + ProductVersion: "$( version )", + Rank: 1, + Serial: false, + PropertyBlueprints: []string{ + `$( property "database_name" )`, + `$( property "admin_password" )`, + }, + FormTypes: []string{`$( form "db_props" )`}, + Variables: []string{}, + PackageInstalls: []string{`$( package "test-install" )`}, + } + yamlData, err := yaml.Marshal(&m) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(path.Join(inputPath, "base.yml"), yamlData, 0644) // 0644 sets file permissions + Expect(err).NotTo(HaveOccurred()) + }) + + It("fails to bake with an error", func() { + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("tile metadata_version too old")) + }) + }) + }) + }) + + Context("BakeFromLockfile", func() { + When("a valid release lock references a pre-built release", func() { + BeforeEach(func() { + if !boshInstalled() { + Skip("bosh CLI not installed - skipping integration test") + } + }) + + It("produces tile output without regenerating the BOSH release", func() { + inputPath, err := os.MkdirTemp("", "lockfile-test-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + defer func() { _ = os.RemoveAll(filepath.Dir(inputPath)) }() + + err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + commands := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range commands { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + subject := NewBaker() + subject.SetWriter(GinkgoWriter) + err = subject.Bake(inputPath) + Expect(err).NotTo(HaveOccurred()) + + tarball, err := subject.GetReleaseTarball() + Expect(err).NotTo(HaveOccurred()) + + cachedTarball := filepath.Join(filepath.Dir(inputPath), "cached-release.tgz") + err = copyTestFile(tarball, cachedTarball) + Expect(err).NotTo(HaveOccurred()) + + releaseLock := cargo.BOSHReleaseTarballLock{ + Name: "k8s-tile-test", + Version: "0.1.1", + } + + subject2 := NewBaker() + subject2.SetWriter(GinkgoWriter) + err = subject2.BakeFromLockfile(inputPath, releaseLock, cachedTarball) + Expect(err).NotTo(HaveOccurred()) + + outputPath := path.Join(inputPath, ".carvel-tile") + Expect(filepath.Join(outputPath, "base.yml")).To(BeAnExistingFile()) + Expect(filepath.Join(outputPath, "releases", "k8s-tile-test-0.1.1.tgz")).To(BeAnExistingFile()) + Expect(filepath.Join(outputPath, "runtime_configs")).To(BeADirectory()) + }) + }) + + When("the release lock name does not match", func() { + It("returns an error", func() { + inputPath, err := os.MkdirTemp("", "lockfile-mismatch-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + defer func() { _ = os.RemoveAll(filepath.Dir(inputPath)) }() + + err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + releaseLock := cargo.BOSHReleaseTarballLock{ + Name: "wrong-name", + Version: "0.1.1", + } + + subject := NewBaker() + err = subject.BakeFromLockfile(inputPath, releaseLock, "/nonexistent/tarball.tgz") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not match tile name")) + }) + }) + }) + + Context("GetReleaseTarball", func() { + When("called before bake", func() { + It("returns an error", func() { + subject := NewBaker() + _, err := subject.GetReleaseTarball() + Expect(err).To(HaveOccurred()) + }) + }) + }) + + Context("rebake reproducibility", func() { + It("publish and rebake produce identical tiles when using the same cached release", func() { + if !boshInstalled() { + Skip("bosh CLI not installed") + } + if !kilnInstalled() { + Skip("kiln CLI not installed") + } + + tmpRoot, err := os.MkdirTemp("", "rebake-repro-*") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = os.RemoveAll(tmpRoot) }() + + inputPath := filepath.Join(tmpRoot, "tile") + err = os.CopyFS(inputPath, os.DirFS("testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + for _, cmd := range []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "git setup: "+string(out)) + } + + uploadBaker := NewBaker() + uploadBaker.SetWriter(GinkgoWriter) + err = uploadBaker.Bake(inputPath) + Expect(err).NotTo(HaveOccurred()) + + uploadTarball, err := uploadBaker.GetReleaseTarball() + Expect(err).NotTo(HaveOccurred()) + cachedTarball := filepath.Join(tmpRoot, "cached-release.tgz") + Expect(copyTestFile(uploadTarball, cachedTarball)).To(Succeed()) + + releaseLock := cargo.BOSHReleaseTarballLock{ + Name: "k8s-tile-test", + Version: "0.1.1", + } + + publishBaker := NewBaker() + publishBaker.SetWriter(GinkgoWriter) + err = publishBaker.BakeFromLockfile(inputPath, releaseLock, cachedTarball) + Expect(err).NotTo(HaveOccurred()) + + publishTile := filepath.Join(tmpRoot, "publish.pivotal") + err = publishBaker.KilnBake(publishTile) + Expect(err).NotTo(HaveOccurred()) + + publishChecksum := fileChecksum(publishTile) + + rebakeBaker := NewBaker() + rebakeBaker.SetWriter(GinkgoWriter) + err = rebakeBaker.BakeFromLockfile(inputPath, releaseLock, cachedTarball) + Expect(err).NotTo(HaveOccurred()) + + rebakeTile := filepath.Join(tmpRoot, "rebake.pivotal") + err = rebakeBaker.KilnBake(rebakeTile) + Expect(err).NotTo(HaveOccurred()) + + rebakeChecksum := fileChecksum(rebakeTile) + + Expect(rebakeChecksum).To(Equal(publishChecksum), + "publish and rebake should produce identical tiles when using the same cached BOSH release tarball") + }) + }) + + Context("generateManifestTemplate with different entry names", func() { + It("parameterizes the entry name throughout the template", func() { + template := generateManifestTemplate("my-custom-pkg") + + Expect(template).To(ContainSubstring(`p("my-custom-pkg.name")`)) + Expect(template).To(ContainSubstring(`p("my-custom-pkg.version")`)) + Expect(template).To(ContainSubstring(`p("my-custom-pkg.values")`)) + Expect(template).NotTo(ContainSubstring("test-install")) + }) + + It("contains exactly 6 K8s resource documents", func() { + template := generateManifestTemplate("pkg") + docs := strings.Split(template, "---") + nonEmpty := 0 + for _, doc := range docs { + if strings.TrimSpace(doc) != "" { + nonEmpty++ + } + } + Expect(nonEmpty).To(Equal(5)) + }) + }) +}) diff --git a/internal/carvel/models/job.go b/internal/carvel/models/job.go new file mode 100644 index 000000000..9b8ef81b1 --- /dev/null +++ b/internal/carvel/models/job.go @@ -0,0 +1,7 @@ +package models + +type Job struct { + Name string `yaml:"name"` + Release string `yaml:"release"` + Properties map[string]PackageInstallProps `yaml:"properties,omitempty"` +} diff --git a/internal/carvel/models/lockfile.go b/internal/carvel/models/lockfile.go new file mode 100644 index 000000000..ce0cd3dbd --- /dev/null +++ b/internal/carvel/models/lockfile.go @@ -0,0 +1,39 @@ +package models + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" +) + +type CarvelLockfile struct { + Releases []CarvelReleaseLock `yaml:"releases"` +} + +type CarvelReleaseLock struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + RemotePath string `yaml:"remote_path"` + SHA256 string `yaml:"sha256"` +} + +func ReadCarvelLockfile(path string) (CarvelLockfile, error) { + data, err := os.ReadFile(path) + if err != nil { + return CarvelLockfile{}, fmt.Errorf("failed to read lockfile: %w", err) + } + var lf CarvelLockfile + if err := yaml.Unmarshal(data, &lf); err != nil { + return CarvelLockfile{}, fmt.Errorf("failed to parse lockfile: %w", err) + } + return lf, nil +} + +func (lf CarvelLockfile) WriteFile(path string) error { + data, err := yaml.Marshal(&lf) + if err != nil { + return fmt.Errorf("failed to marshal lockfile: %w", err) + } + return os.WriteFile(path, data, 0644) +} diff --git a/internal/carvel/models/lockfile_test.go b/internal/carvel/models/lockfile_test.go new file mode 100644 index 000000000..7f5eda451 --- /dev/null +++ b/internal/carvel/models/lockfile_test.go @@ -0,0 +1,58 @@ +package models_test + +import ( + "os" + "path/filepath" + "testing" + + . "github.com/onsi/gomega" + + "github.com/pivotal-cf/kiln/internal/carvel/models" +) + +func TestCarvelLockfileRoundTrip(t *testing.T) { + g := NewWithT(t) + + dir := t.TempDir() + lockPath := filepath.Join(dir, "Kilnfile.lock") + + original := models.CarvelLockfile{ + Releases: []models.CarvelReleaseLock{ + { + Name: "my-tile", + Version: "1.2.3", + RemotePath: "bosh-releases/my-tile/my-tile-1.2.3.tgz", + SHA256: "abc123def456", + }, + }, + } + + err := original.WriteFile(lockPath) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(lockPath).To(BeAnExistingFile()) + + loaded, err := models.ReadCarvelLockfile(lockPath) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(loaded).To(Equal(original)) +} + +func TestReadCarvelLockfileNotFound(t *testing.T) { + g := NewWithT(t) + + _, err := models.ReadCarvelLockfile("/nonexistent/path/Kilnfile.lock") + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("failed to read lockfile")) +} + +func TestReadCarvelLockfileInvalidYAML(t *testing.T) { + g := NewWithT(t) + + dir := t.TempDir() + lockPath := filepath.Join(dir, "Kilnfile.lock") + err := os.WriteFile(lockPath, []byte("releases:\n - name: [unterminated"), 0644) + g.Expect(err).NotTo(HaveOccurred()) + + _, err = models.ReadCarvelLockfile(lockPath) + g.Expect(err).To(HaveOccurred()) + g.Expect(err.Error()).To(ContainSubstring("failed to parse lockfile")) +} diff --git a/internal/carvel/models/metadata.go b/internal/carvel/models/metadata.go new file mode 100644 index 000000000..a22b21d3c --- /dev/null +++ b/internal/carvel/models/metadata.go @@ -0,0 +1,17 @@ +package models + +type Metadata struct { + Name string `yaml:"name"` + ProductVersion string `yaml:"product_version"` + IconImage string `yaml:"icon_image"` + Label string `yaml:"label"` + MetadataVersion string `yaml:"metadata_version"` + MinimumVersionForUpgrade string `yaml:"minimum_version_for_upgrade"` + Rank int `yaml:"rank"` + Serial bool `yaml:"serial"` + PropertyBlueprints []string `yaml:"property_blueprints"` + FormTypes []string `yaml:"form_types"` + Variables []string `yaml:"variables"` + PackageInstalls []string `yaml:"package_installs"` + CompatibleKubernetesDistributions []ProductVersion `yaml:"compatible_kubernetes_distributions,omitempty"` +} diff --git a/internal/carvel/models/metadata_out.go b/internal/carvel/models/metadata_out.go new file mode 100644 index 000000000..88b652e96 --- /dev/null +++ b/internal/carvel/models/metadata_out.go @@ -0,0 +1,31 @@ +package models + +type MetadataOut struct { + Name string `yaml:"name"` + ProductVersion string `yaml:"product_version"` + IconImage string `yaml:"icon_image"` + Label string `yaml:"label"` + MetadataVersion string `yaml:"metadata_version"` + MinimumVersionForUpgrade string `yaml:"minimum_version_for_upgrade"` + Rank int `yaml:"rank"` + Serial bool `yaml:"serial"` + PropertyBlueprints []string `yaml:"property_blueprints"` + FormTypes []string `yaml:"form_types"` + Variables []string `yaml:"variables"` + InstanceGroups []string `yaml:"job_types"` + StemcellCriteria StemcellCriteria `yaml:"stemcell_criteria"` + Releases []string `yaml:"releases"` + RuntimeConfigs []string `yaml:"runtime_configs"` + RequiresKubernetes bool `yaml:"requires_kubernetes"` + CompatibleKubernetesDistributions []ProductVersion `yaml:"compatible_kubernetes_distributions"` +} + +type StemcellCriteria struct { + Os string `yaml:"os"` + Version string `yaml:"version"` +} + +type ProductVersion struct { + Name string `yaml:"name"` + Version string `yaml:"version"` +} diff --git a/internal/carvel/models/package_install.go b/internal/carvel/models/package_install.go new file mode 100644 index 000000000..e3e9ef348 --- /dev/null +++ b/internal/carvel/models/package_install.go @@ -0,0 +1,8 @@ +package models + +type PackageInstall struct { + Name string `yaml:"name"` + PackageName string `yaml:"packageName"` + PackageVersion string `yaml:"packageVersion"` + Values interface{} `yaml:"values,omitempty"` +} diff --git a/internal/carvel/models/package_install_props.go b/internal/carvel/models/package_install_props.go new file mode 100644 index 000000000..88895fb93 --- /dev/null +++ b/internal/carvel/models/package_install_props.go @@ -0,0 +1,7 @@ +package models + +type PackageInstallProps struct { + Name string `yaml:"name"` + Version string `yaml:"version"` + Values interface{} `yaml:"values,omitempty"` +} diff --git a/internal/carvel/models/runtime_config.go b/internal/carvel/models/runtime_config.go new file mode 100644 index 000000000..c19a68fd8 --- /dev/null +++ b/internal/carvel/models/runtime_config.go @@ -0,0 +1,22 @@ +package models + +type RuntimeConfigOuter struct { + Name string `yaml:"name"` + RuntimeConfig string `yaml:"runtime_config"` +} + +type RuntimeConfigInner struct { + Releases []string `yaml:"releases"` + Addons []Addon `yaml:"addons"` +} + +type Addon struct { + Name string `yaml:"name"` + Include Inclusion `yaml:"include"` + Jobs []Job `yaml:"jobs"` +} + +type Inclusion struct { + Deployments []string `yaml:"deployments"` + Jobs []Job `yaml:"jobs"` +} diff --git a/internal/carvel/testdata/sample-tile/.gitignore b/internal/carvel/testdata/sample-tile/.gitignore new file mode 100644 index 000000000..95270bce3 --- /dev/null +++ b/internal/carvel/testdata/sample-tile/.gitignore @@ -0,0 +1,3 @@ +.boshrelease +.carvel-tile + diff --git a/internal/carvel/testdata/sample-tile/Kilnfile b/internal/carvel/testdata/sample-tile/Kilnfile new file mode 100644 index 000000000..dd877988a --- /dev/null +++ b/internal/carvel/testdata/sample-tile/Kilnfile @@ -0,0 +1,8 @@ +--- +release_sources: + - type: artifactory + artifactory_host: http://localhost:8080 + repo: test-repo + username: test-user + password: test-pass + path_template: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz" diff --git a/internal/carvel/testdata/sample-tile/README.md b/internal/carvel/testdata/sample-tile/README.md new file mode 100644 index 000000000..1501223a8 --- /dev/null +++ b/internal/carvel/testdata/sample-tile/README.md @@ -0,0 +1,4 @@ +# Sample Kubernetes Tile +This is a test "unbaked" kubernetes tile. It includes a fake imgpkg bundle tarball that doesn't actually contain a packagereop. + +It is only for integration testing purposes and cannot be deployed. diff --git a/internal/carvel/testdata/sample-tile/base.yml b/internal/carvel/testdata/sample-tile/base.yml new file mode 100644 index 000000000..d91f08bba --- /dev/null +++ b/internal/carvel/testdata/sample-tile/base.yml @@ -0,0 +1,19 @@ +name: k8s-tile-test +label: "test tile" +icon_image: $( icon ) +metadata_version: "3.2.0" +minimum_version_for_upgrade: 0.0.0 +product_version: $( version ) +rank: 1 +serial: false +property_blueprints: +- $( property "database_name" ) +- $( property "admin_password" ) +form_types: +- $( form "db_props" ) +variables: [] +package_installs: +- $( package "test-install" ) +compatible_kubernetes_distributions: +- name: k0s + version: '>0.0.0' diff --git a/internal/carvel/testdata/sample-tile/bundle.tar b/internal/carvel/testdata/sample-tile/bundle.tar new file mode 100644 index 000000000..6c4cdbde9 Binary files /dev/null and b/internal/carvel/testdata/sample-tile/bundle.tar differ diff --git a/internal/carvel/testdata/sample-tile/forms/.gitkeep b/internal/carvel/testdata/sample-tile/forms/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/carvel/testdata/sample-tile/forms/db_props.yml b/internal/carvel/testdata/sample-tile/forms/db_props.yml new file mode 100644 index 000000000..2d41e9aa7 --- /dev/null +++ b/internal/carvel/testdata/sample-tile/forms/db_props.yml @@ -0,0 +1,8 @@ +--- +name: "db_props" +label: "Postgres Database Properties" +description: "Postgres Database Properties" +markdown: 'Postgres Database Properties' +property_inputs: +- reference: ".properties.database_name" + label: "Database name" diff --git a/internal/carvel/testdata/sample-tile/icon.png b/internal/carvel/testdata/sample-tile/icon.png new file mode 100644 index 000000000..d556b41cc Binary files /dev/null and b/internal/carvel/testdata/sample-tile/icon.png differ diff --git a/internal/carvel/testdata/sample-tile/packageinstalls/test-install.yml b/internal/carvel/testdata/sample-tile/packageinstalls/test-install.yml new file mode 100644 index 000000000..eccf4450a --- /dev/null +++ b/internal/carvel/testdata/sample-tile/packageinstalls/test-install.yml @@ -0,0 +1,6 @@ +name: test-install +packageName: something-test.tanzu.vmware.com +packageVersion: "0.1.5" +values: + db_name: (( .properties.database_name.value )) + password: (( .properties.admin_password.value )) diff --git a/internal/carvel/testdata/sample-tile/properties/.gitkeep b/internal/carvel/testdata/sample-tile/properties/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/carvel/testdata/sample-tile/properties/properties.yml b/internal/carvel/testdata/sample-tile/properties/properties.yml new file mode 100644 index 000000000..de36e0aa6 --- /dev/null +++ b/internal/carvel/testdata/sample-tile/properties/properties.yml @@ -0,0 +1,9 @@ +- name: admin_password + type: secret + configurable: false + optional: false +- name: database_name + type: string + configurable: true + optional: false + default: my-db diff --git a/internal/carvel/testdata/sample-tile/version b/internal/carvel/testdata/sample-tile/version new file mode 100644 index 000000000..17e51c385 --- /dev/null +++ b/internal/carvel/testdata/sample-tile/version @@ -0,0 +1 @@ +0.1.1 diff --git a/internal/commands/carvel.go b/internal/commands/carvel.go new file mode 100644 index 000000000..b7389adb4 --- /dev/null +++ b/internal/commands/carvel.go @@ -0,0 +1,187 @@ +package commands + +import ( + "fmt" + "log" + "sort" + "strings" + + "github.com/pivotal-cf/jhanda" +) + +// Carvel is a command group for Carvel/Kubernetes tile operations +type Carvel struct { + outLogger *log.Logger + errLogger *log.Logger + commands jhanda.CommandSet + aliases map[string]bool + synopses map[string]string +} + +func NewCarvel(outLogger, errLogger *log.Logger) Carvel { + c := Carvel{ + outLogger: outLogger, + errLogger: errLogger, + commands: jhanda.CommandSet{}, + aliases: map[string]bool{}, + synopses: map[string]string{}, + } + + // Register subcommands + c.commands["bake"] = NewCarvelBake(outLogger, errLogger) + c.commands["upload"] = NewCarvelUpload(outLogger, errLogger) + c.commands["publish"] = NewCarvelPublish(outLogger, errLogger) + c.commands["re-bake"] = NewCarvelReBake(outLogger, errLogger) + + // Positional argument synopses for usage lines + c.synopses["re-bake"] = "" + + // Aliases (hidden from help output) + c.commands["rebake"] = c.commands["re-bake"] + c.aliases["rebake"] = true + c.synopses["rebake"] = c.synopses["re-bake"] + + return c +} + +func (c Carvel) Execute(args []string) error { + if len(args) == 0 { + return c.printHelp() + } + + subcommand := args[0] + subargs := args[1:] + + if subcommand == "help" || subcommand == "-h" || subcommand == "--help" { + if len(subargs) > 0 { + return c.printSubcommandHelp(subargs[0]) + } + return c.printHelp() + } + + // Must intercept help flags here before delegating to CommandSet.Execute, + // which would otherwise look for a nonexistent "help" command in the + // carvel command set. + for _, arg := range subargs { + if arg == "-h" || arg == "--help" || arg == "help" { + return c.printSubcommandHelp(subcommand) + } + } + + return c.commands.Execute(subcommand, subargs) +} + +func (c Carvel) Usage() jhanda.Usage { + var subcommandList strings.Builder + subcommandList.WriteString("Commands for working with Carvel/Kubernetes tiles.\n\n") + subcommandList.WriteString("Subcommands:\n") + + var names []string + var length int + for name := range c.commands { + if c.aliases[name] { + continue + } + names = append(names, name) + if len(name) > length { + length = len(name) + } + } + sort.Strings(names) + + for _, name := range names { + cmd := c.commands[name] + paddedName := c.pad(name, " ", length) + fmt.Fprintf(&subcommandList, " %s %s\n", paddedName, cmd.Usage().ShortDescription) + } + subcommandList.WriteString("\nUse 'kiln carvel help ' for more information about a subcommand.") + + return jhanda.Usage{ + Description: subcommandList.String(), + ShortDescription: "commands for Carvel/Kubernetes tiles", + Flags: nil, + } +} + +func (c Carvel) printHelp() error { + var ( + length int + names []string + ) + + for name := range c.commands { + if c.aliases[name] { + continue + } + names = append(names, name) + if len(name) > length { + length = len(name) + } + } + + sort.Strings(names) + + fmt.Println("kiln carvel - commands for Carvel/Kubernetes tiles") + fmt.Println() + fmt.Println("Usage: kiln carvel []") + fmt.Println() + fmt.Println("Subcommands:") + for _, name := range names { + cmd := c.commands[name] + paddedName := c.pad(name, " ", length) + fmt.Printf(" %s %s\n", paddedName, cmd.Usage().ShortDescription) + } + fmt.Println() + fmt.Println("Use 'kiln carvel help ' for more information about a subcommand.") + + return nil +} + +func (c Carvel) printSubcommandHelp(subcommand string) error { + cmd, ok := c.commands[subcommand] + if !ok { + return fmt.Errorf("unknown subcommand: %s", subcommand) + } + + usage := cmd.Usage() + fmt.Printf("kiln carvel %s - %s\n", subcommand, usage.ShortDescription) + fmt.Println() + fmt.Println(usage.Description) + fmt.Println() + + synopsis := c.synopses[subcommand] + if synopsis != "" { + fmt.Printf("Usage: kiln carvel %s %s []\n", subcommand, synopsis) + } else { + fmt.Printf("Usage: kiln carvel %s []\n", subcommand) + } + + if usage.Flags != nil { + flagUsage, err := jhanda.PrintUsage(usage.Flags) + if err != nil { + return err + } + + flagList := strings.Split(flagUsage, "\n") + if len(flagList) > 0 { + fmt.Println() + fmt.Println("Arguments:") + for _, flag := range flagList { + if flag != "" { + fmt.Printf(" %s\n", flag) + } + } + } + } + + return nil +} + +func (c Carvel) pad(str, pad string, length int) string { + for { + str += pad + if len(str) > length { + return str[0:length] + } + } +} diff --git a/internal/commands/carvel_bake.go b/internal/commands/carvel_bake.go new file mode 100644 index 000000000..72e28d188 --- /dev/null +++ b/internal/commands/carvel_bake.go @@ -0,0 +1,112 @@ +package commands + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/pivotal-cf/jhanda" + "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/commands/flags" +) + +type CarvelBake struct { + outLogger *log.Logger + errLogger *log.Logger + Options CarvelBakeOptions +} + +type CarvelBakeOptions struct { + flags.Standard + SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` + OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` +} + +func NewCarvelBake(outLogger, errLogger *log.Logger) CarvelBake { + return CarvelBake{ + outLogger: outLogger, + errLogger: errLogger, + } +} + +func (c CarvelBake) Execute(args []string) error { + _, err := jhanda.Parse(&c.Options, args) + if err != nil { + return err + } + + sourcePath, err := resolveSourcePath(c.Options.SourceDirectory) + if err != nil { + return err + } + + targetPath, err := filepath.Abs(c.Options.OutputFile) + if err != nil { + return fmt.Errorf("failed to resolve output file path: %w", err) + } + + baker := carvel.NewBaker() + baker.SetProgressWriter(os.Stdout) + if c.Options.Verbose { + baker.SetWriter(os.Stdout) + } + + kilnfilePath := resolveKilnfilePath(c.Options.Kilnfile, sourcePath) + lockfilePath := kilnfilePath + ".lock" + if _, statErr := os.Stat(lockfilePath); statErr == nil { + c.Options.Kilnfile = kilnfilePath + kilnfile, kilnfileLock, loadErr := c.Options.LoadKilnfiles(nil, nil) + if loadErr != nil { + return fmt.Errorf("failed to load Kilnfiles: %w", loadErr) + } + + if len(kilnfileLock.Releases) == 0 { + return fmt.Errorf("no releases found in Kilnfile.lock") + } + releaseLock := kilnfileLock.Releases[0] + + tmpDir, tmpErr := os.MkdirTemp("", "carvel-bake-*") + if tmpErr != nil { + return fmt.Errorf("failed to create temp directory: %w", tmpErr) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + localTarball, dlErr := downloadCarvelRelease(c.outLogger, kilnfile, kilnfileLock, tmpDir) + if dlErr != nil { + return fmt.Errorf("failed to download release from Artifactory: %w", dlErr) + } + + err = baker.BakeFromLockfile(sourcePath, releaseLock, localTarball) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile from lockfile: %w", err) + } + } else { + err = baker.Bake(sourcePath) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile: %w", err) + } + } + + v, err := baker.GetVersion() + if err != nil { + return fmt.Errorf("failed to get tile version: %w", err) + } + + err = baker.KilnBake(targetPath) + if err != nil { + return fmt.Errorf("failed to bake tile: %w", err) + } + + c.outLogger.Printf("Done! Baked %s version %s to %s", baker.GetName(), v, targetPath) + return nil +} + +func (c CarvelBake) Usage() jhanda.Usage { + return jhanda.Usage{ + Description: "Bakes a Carvel/Kubernetes tile into a .pivotal file. This command transforms a Kubernetes tile (using imgpkg bundles and Carvel packages) into a BOSH-compatible format, then bakes it into a .pivotal file that can be consumed by Operations Manager. When a Kilnfile.lock is present, it downloads the cached BOSH release from Artifactory instead of regenerating it locally.", + ShortDescription: "bakes a Carvel/Kubernetes tile", + Flags: c.Options, + } +} diff --git a/internal/commands/carvel_bake_test.go b/internal/commands/carvel_bake_test.go new file mode 100644 index 000000000..c83ed6dfd --- /dev/null +++ b/internal/commands/carvel_bake_test.go @@ -0,0 +1,106 @@ +package commands_test + +import ( + "log" + "os" + "os/exec" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/commands" +) + +func boshInstalled() bool { + _, err := exec.LookPath("bosh") + return err == nil +} + +func kilnInstalled() bool { + _, err := exec.LookPath("kiln") + return err == nil +} + +var _ = Describe("CarvelBake", func() { + var ( + outLogger *log.Logger + errLogger *log.Logger + command commands.CarvelBake + ) + + BeforeEach(func() { + outLogger = log.New(GinkgoWriter, "", 0) + errLogger = log.New(GinkgoWriter, "", 0) + command = commands.NewCarvelBake(outLogger, errLogger) + }) + + Describe("Usage", func() { + It("returns usage information", func() { + usage := command.Usage() + Expect(usage.ShortDescription).To(Equal("bakes a Carvel/Kubernetes tile")) + Expect(usage.Description).To(ContainSubstring("Carvel/Kubernetes tile")) + }) + }) + + Describe("Execute", func() { + var ( + inputPath string + outputPath string + ) + + BeforeEach(func() { + var err error + inputPath, err = os.MkdirTemp("", "testinput-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + // create an initial git commit in the input directory + cmds := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "-c", "user.name=test", "-c", "user.email=test@test.com", "commit", "-m", "initial commit"), + } + for _, cmd := range cmds { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + outputPath = filepath.Join(inputPath, "output.pivotal") + }) + + AfterEach(func() { + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + }) + + When("required arguments are missing", func() { + It("returns an error when output-file is not provided", func() { + err := command.Execute([]string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("output-file")) + }) + }) + + When("valid arguments are provided", func() { + It("successfully bakes a tile", func() { + if !boshInstalled() { + Skip("bosh CLI not installed - skipping integration test") + } + if !kilnInstalled() { + Skip("kiln CLI not installed - skipping integration test") + } + err := command.Execute([]string{ + "--source-directory", inputPath, + "--output-file", outputPath, + "--verbose", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(outputPath).To(BeAnExistingFile()) + }) + }) + }) +}) diff --git a/internal/commands/carvel_helpers.go b/internal/commands/carvel_helpers.go new file mode 100644 index 000000000..0429ddbb3 --- /dev/null +++ b/internal/commands/carvel_helpers.go @@ -0,0 +1,117 @@ +package commands + +import ( + "fmt" + "log" + "os" + "path/filepath" + + "github.com/go-git/go-billy/v5/osfs" + "github.com/pivotal-cf/kiln/internal/baking" + "github.com/pivotal-cf/kiln/internal/commands/flags" + "github.com/pivotal-cf/kiln/internal/component" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" +) + +func loadKilnfileOnly(options flags.Standard) (cargo.Kilnfile, error) { + fs := osfs.New("") + variablesService := baking.NewTemplateVariablesService(fs) + + templateVariables, err := variablesService.FromPathsAndPairs(options.VariableFiles, options.Variables) + if err != nil { + return cargo.Kilnfile{}, fmt.Errorf("failed to parse template variables: %w", err) + } + + kilnfileFP, err := fs.Open(options.Kilnfile) + if err != nil { + return cargo.Kilnfile{}, fmt.Errorf("failed to open Kilnfile: %w", err) + } + defer func() { _ = kilnfileFP.Close() }() + + return cargo.InterpolateAndParseKilnfile(kilnfileFP, templateVariables) +} + +func findArtifactorySource(kilnfile cargo.Kilnfile) (cargo.ReleaseSourceConfig, error) { + for _, src := range kilnfile.ReleaseSources { + if src.Type == cargo.BOSHReleaseTarballSourceTypeArtifactory { + return src, nil + } + } + return cargo.ReleaseSourceConfig{}, fmt.Errorf("no artifactory release source found in Kilnfile") +} + +func downloadCarvelRelease(logger *log.Logger, kilnfile cargo.Kilnfile, lock cargo.KilnfileLock, destDir string) (string, error) { + if len(lock.Releases) == 0 { + return "", fmt.Errorf("no releases found in Kilnfile.lock") + } + + releaseLock := lock.Releases[0] + sources := component.NewReleaseSourceRepo(kilnfile) + + logger.Printf("Downloading %s %s from %s", releaseLock.Name, releaseLock.Version, releaseLock.RemoteSource) + local, err := sources.DownloadRelease(destDir, releaseLock) + if err != nil { + return "", fmt.Errorf("failed to download release: %w", err) + } + + if releaseLock.SHA1 != "" && local.Lock.SHA1 != releaseLock.SHA1 { + _ = os.Remove(local.LocalPath) + return "", fmt.Errorf("downloaded release %q had incorrect SHA1 - expected %q, got %q", local.LocalPath, releaseLock.SHA1, local.Lock.SHA1) + } + + return local.LocalPath, nil +} + +func writeStandardKilnfileLock(lockfilePath string, releaseName, releaseVersion, remotePath, remoteSourceID, sha1 string) error { + lock := cargo.KilnfileLock{ + Releases: []cargo.BOSHReleaseTarballLock{ + { + Name: releaseName, + Version: releaseVersion, + RemotePath: remotePath, + RemoteSource: remoteSourceID, + SHA1: sha1, + }, + }, + Stemcell: cargo.Stemcell{ + OS: "ubuntu-jammy", + Version: "1.446", + }, + } + + data, err := yaml.Marshal(&lock) + if err != nil { + return fmt.Errorf("failed to marshal Kilnfile.lock: %w", err) + } + return os.WriteFile(lockfilePath, data, 0644) +} + + +func resolveKilnfilePath(kilnfilePath, sourcePath string) string { + if kilnfilePath == "" || kilnfilePath == "Kilnfile" { + return filepath.Join(sourcePath, "Kilnfile") + } + abs, err := filepath.Abs(kilnfilePath) + if err != nil { + return kilnfilePath + } + return abs +} + +func resolveSourcePath(sourcePath string) (string, error) { + if sourcePath == "" { + var err error + sourcePath, err = os.Getwd() + if err != nil { + return "", fmt.Errorf("failed to get current directory: %w", err) + } + } else { + var err error + sourcePath, err = filepath.Abs(sourcePath) + if err != nil { + return "", fmt.Errorf("failed to resolve source directory: %w", err) + } + } + return sourcePath, nil +} diff --git a/internal/commands/carvel_publish.go b/internal/commands/carvel_publish.go new file mode 100644 index 000000000..8c126622d --- /dev/null +++ b/internal/commands/carvel_publish.go @@ -0,0 +1,175 @@ +package commands + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "log" + "os" + "path/filepath" + + "github.com/pivotal-cf/jhanda" + "github.com/pivotal-cf/kiln/internal/builder" + "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/commands/flags" + "github.com/pivotal-cf/kiln/pkg/bake" +) + +type CarvelPublish struct { + outLogger *log.Logger + errLogger *log.Logger + KilnVersion string + Options CarvelPublishOptions +} + +type CarvelPublishOptions struct { + flags.Standard + SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` + OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` + Version string ` long:"version" description:"tile version for the final release"` + IsFinal bool ` long:"final" description:"create a bake record for this build"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` +} + +func NewCarvelPublish(outLogger, errLogger *log.Logger) CarvelPublish { + return CarvelPublish{ + outLogger: outLogger, + errLogger: errLogger, + } +} + +func (c CarvelPublish) Execute(args []string) error { + _, err := jhanda.Parse(&c.Options, args) + if err != nil { + return err + } + + sourcePath, err := resolveSourcePath(c.Options.SourceDirectory) + if err != nil { + return err + } + + targetPath, err := filepath.Abs(c.Options.OutputFile) + if err != nil { + return fmt.Errorf("failed to resolve output file path: %w", err) + } + + kilnfilePath := resolveKilnfilePath(c.Options.Kilnfile, sourcePath) + + if _, statErr := os.Stat(kilnfilePath); statErr != nil { + return fmt.Errorf("could not find Kilnfile at %s: run 'kiln carvel upload' first to create the BOSH release, Kilnfile, and Kilnfile.lock", kilnfilePath) + } + + lockfilePath := kilnfilePath + ".lock" + if _, statErr := os.Stat(lockfilePath); statErr != nil { + return fmt.Errorf("could not find Kilnfile.lock at %s: run 'kiln carvel upload' first to create the BOSH release and lockfile", lockfilePath) + } + + c.Options.Kilnfile = kilnfilePath + kilnfile, kilnfileLock, err := c.Options.LoadKilnfiles(nil, nil) + if err != nil { + return fmt.Errorf("failed to load Kilnfiles: %w", err) + } + + if len(kilnfileLock.Releases) == 0 { + return fmt.Errorf("no releases found in Kilnfile.lock: run 'kiln carvel upload' first") + } + releaseLock := kilnfileLock.Releases[0] + + tmpDir, err := os.MkdirTemp("", "carvel-publish-*") + if err != nil { + return fmt.Errorf("failed to create temp directory: %w", err) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + c.outLogger.Printf("Publishing Carvel tile from %s", sourcePath) + localTarball, err := downloadCarvelRelease(c.outLogger, kilnfile, kilnfileLock, tmpDir) + if err != nil { + return fmt.Errorf("failed to download release from Artifactory: %w", err) + } + + b := carvel.NewBaker() + if c.Options.Verbose { + b.SetWriter(os.Stdout) + } + + err = b.BakeFromLockfile(sourcePath, releaseLock, localTarball) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile from lockfile: %w", err) + } + + ver, err := b.GetVersion() + if err != nil { + return fmt.Errorf("failed to get tile version: %w", err) + } + if c.Options.Version != "" { + ver = c.Options.Version + } + + err = b.KilnBake(targetPath) + if err != nil { + return fmt.Errorf("failed to bake tile: %w", err) + } + + c.outLogger.Printf("Baked %s version %s to %s", b.GetName(), ver, targetPath) + + if c.Options.IsFinal { + resolvedSourcePath, err := filepath.EvalSymlinks(sourcePath) + if err != nil { + resolvedSourcePath = sourcePath + } + + sha, err := builder.GitMetadataSHA(resolvedSourcePath, false) + if err != nil { + return fmt.Errorf("failed to get git SHA: %w", err) + } + + checksum, err := tileFileChecksum(targetPath) + if err != nil { + return fmt.Errorf("failed to checksum tile: %w", err) + } + + record := bake.Record{ + SourceRevision: sha, + Version: ver, + KilnVersion: c.KilnVersion, + FileChecksum: checksum, + } + + record, err = record.SetTileDirectory(resolvedSourcePath) + if err != nil { + return fmt.Errorf("failed to set tile directory on bake record: %w", err) + } + + err = record.WriteFile(resolvedSourcePath) + if err != nil { + return fmt.Errorf("failed to write bake record: %w", err) + } + + c.outLogger.Printf("Wrote bake record for version %s", ver) + } + + return nil +} + +func (c CarvelPublish) Usage() jhanda.Usage { + return jhanda.Usage{ + Description: "Downloads the cached BOSH release from Artifactory (using credentials from the Kilnfile) and bakes a Carvel/Kubernetes tile as a .pivotal file. Run 'kiln carvel upload' first to build and cache the release. When --final is specified, creates a bake record that can be used with 'kiln carvel re-bake' for reproducible builds.", + ShortDescription: "publishes a Carvel/Kubernetes tile", + Flags: c.Options, + } +} + +func tileFileChecksum(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/commands/carvel_publish_test.go b/internal/commands/carvel_publish_test.go new file mode 100644 index 000000000..a064e32b7 --- /dev/null +++ b/internal/commands/carvel_publish_test.go @@ -0,0 +1,231 @@ +package commands_test + +import ( + "encoding/json" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/commands" + "github.com/pivotal-cf/kiln/pkg/bake" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" +) + +var _ = Describe("CarvelPublish", func() { + var ( + outLogger *log.Logger + errLogger *log.Logger + command commands.CarvelPublish + ) + + BeforeEach(func() { + outLogger = log.New(GinkgoWriter, "", 0) + errLogger = log.New(GinkgoWriter, "", 0) + command = commands.NewCarvelPublish(outLogger, errLogger) + }) + + Describe("Usage", func() { + It("returns usage information", func() { + usage := command.Usage() + Expect(usage.ShortDescription).To(Equal("publishes a Carvel/Kubernetes tile")) + Expect(usage.Description).To(ContainSubstring("bake record")) + }) + }) + + Describe("Execute", func() { + When("required arguments are missing", func() { + It("returns an error when output-file is not provided", func() { + err := command.Execute([]string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("output-file")) + }) + }) + + When("no Kilnfile exists", func() { + It("returns an error telling the user to run upload first", func() { + tmpDir, err := os.MkdirTemp("", "publish-no-kilnfile-*") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = os.RemoveAll(tmpDir) }() + + err = command.Execute([]string{ + "--source-directory", tmpDir, + "--output-file", filepath.Join(tmpDir, "out.pivotal"), + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("could not find Kilnfile")) + Expect(err.Error()).To(ContainSubstring("kiln carvel upload")) + }) + }) + + When("--final flag is used with a round-trip mock Artifactory", func() { + var ( + inputPath string + outputPath string + server *httptest.Server + mu sync.Mutex + blobs map[string][]byte + getCount int + ) + + BeforeEach(func() { + if !boshInstalled() { + Skip("bosh CLI not installed - skipping integration test") + } + if !kilnInstalled() { + Skip("kiln CLI not installed - skipping integration test") + } + + blobs = make(map[string][]byte) + getCount = 0 + + var err error + inputPath, err = os.MkdirTemp("", "publish-test-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + cmds := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range cmds { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + b := carvel.NewBaker() + b.SetWriter(GinkgoWriter) + Expect(b.Bake(inputPath)).To(Succeed()) + tarball, err := b.GetReleaseTarball() + Expect(err).NotTo(HaveOccurred()) + tarballData, err := os.ReadFile(tarball) + Expect(err).NotTo(HaveOccurred()) + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/artifactory") + switch r.Method { + case http.MethodPut: + body, _ := io.ReadAll(r.Body) + mu.Lock() + blobs[key] = body + mu.Unlock() + w.WriteHeader(http.StatusCreated) + case http.MethodGet: + mu.Lock() + data, found := blobs[key] + getCount++ + mu.Unlock() + if !found { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(data) + } + })) + + // Pre-load mock with the tarball (simulating a prior upload) + remotePath := "/test-repo/bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz" + blobs[remotePath] = tarballData + + kf := cargo.Kilnfile{ + ReleaseSources: []cargo.ReleaseSourceConfig{{ + Type: "artifactory", + ArtifactoryHost: server.URL, + Repo: "test-repo", + Username: "user", + Password: "pass", + PathTemplate: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz", + }}, + } + kfData, err := yaml.Marshal(&kf) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(inputPath, "Kilnfile"), kfData, 0644)).To(Succeed()) + + lock := cargo.KilnfileLock{ + Releases: []cargo.BOSHReleaseTarballLock{{ + Name: "k8s-tile-test", + Version: "0.1.1", + RemotePath: "bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz", + RemoteSource: "artifactory", + }}, + Stemcell: cargo.Stemcell{OS: "ubuntu-jammy", Version: "1.446"}, + } + lockData, err := yaml.Marshal(&lock) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(inputPath, "Kilnfile.lock"), lockData, 0644)).To(Succeed()) + + for _, cmd := range []*exec.Cmd{ + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "add kilnfiles"), + } { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + outputPath = filepath.Join(filepath.Dir(inputPath), "output.pivotal") + }) + + AfterEach(func() { + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + if server != nil { + server.Close() + } + }) + + It("downloads the tarball, bakes the tile, and creates a bake record", func() { + err := command.Execute([]string{ + "--source-directory", inputPath, + "--output-file", outputPath, + "--final", + "--verbose", + }) + Expect(err).NotTo(HaveOccurred()) + Expect(outputPath).To(BeAnExistingFile()) + + By("verifying the mock received a GET (download)") + mu.Lock() + Expect(getCount).To(BeNumerically(">=", 1), "publish must download from Artifactory") + mu.Unlock() + + resolvedInput, resolveErr := filepath.EvalSymlinks(inputPath) + if resolveErr != nil { + resolvedInput = inputPath + } + + recordsDir := filepath.Join(resolvedInput, "bake_records") + Expect(recordsDir).To(BeADirectory()) + + entries, err := os.ReadDir(recordsDir) + Expect(err).NotTo(HaveOccurred()) + Expect(entries).To(HaveLen(1)) + Expect(entries[0].Name()).To(Equal("0.1.1.json")) + + recordData, err := os.ReadFile(filepath.Join(recordsDir, "0.1.1.json")) + Expect(err).NotTo(HaveOccurred()) + + var record bake.Record + Expect(json.Unmarshal(recordData, &record)).To(Succeed()) + Expect(record.Version).To(Equal("0.1.1")) + Expect(record.SourceRevision).NotTo(BeEmpty()) + Expect(record.FileChecksum).NotTo(BeEmpty()) + }) + }) + }) +}) diff --git a/internal/commands/carvel_rebake.go b/internal/commands/carvel_rebake.go new file mode 100644 index 000000000..64bdf42a7 --- /dev/null +++ b/internal/commands/carvel_rebake.go @@ -0,0 +1,162 @@ +package commands + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "os" + "path/filepath" + + "github.com/pivotal-cf/jhanda" + "github.com/pivotal-cf/kiln/internal/builder" + "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/commands/flags" + "github.com/pivotal-cf/kiln/pkg/bake" +) + +type CarvelReBake struct { + outLogger *log.Logger + errLogger *log.Logger + Options CarvelReBakeOptions +} + +type CarvelReBakeOptions struct { + flags.Standard + OutputFile string `short:"o" long:"output-file" description:"path to where the tile will be output" required:"true"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` +} + +func NewCarvelReBake(outLogger, errLogger *log.Logger) CarvelReBake { + return CarvelReBake{ + outLogger: outLogger, + errLogger: errLogger, + } +} + +func (c CarvelReBake) Execute(args []string) error { + remaining, err := jhanda.Parse(&c.Options, args) + if err != nil { + return err + } + if len(remaining) != 1 { + return fmt.Errorf("exactly one bake record argument is required, got %d", len(remaining)) + } + + recordPath := remaining[0] + recordBuf, err := os.ReadFile(recordPath) + if err != nil { + return fmt.Errorf("failed to read bake record file: %w", err) + } + + var record bake.Record + if err := json.Unmarshal(recordBuf, &record); err != nil { + return fmt.Errorf("failed to parse bake record: %w", err) + } + + tileDir := filepath.FromSlash(record.TileDirectory) + if tileDir == "" { + tileDir = "." + } + + sourcePath, err := filepath.Abs(tileDir) + if err != nil { + return fmt.Errorf("failed to resolve tile directory: %w", err) + } + + workingDirectorySHA, err := builder.GitMetadataSHA(sourcePath, false) + if err != nil { + return err + } + + if got, exp := workingDirectorySHA, record.SourceRevision; got != exp { + return fmt.Errorf("expected worktree at source revision %s but current HEAD is %s", exp, got) + } + + targetPath, err := filepath.Abs(c.Options.OutputFile) + if err != nil { + return fmt.Errorf("failed to resolve output file path: %w", err) + } + + b := carvel.NewBaker() + if c.Options.Verbose { + b.SetWriter(os.Stdout) + } + + kilnfilePath := resolveKilnfilePath(c.Options.Kilnfile, sourcePath) + lockfilePath := kilnfilePath + ".lock" + if _, statErr := os.Stat(lockfilePath); statErr == nil { + c.Options.Kilnfile = kilnfilePath + kilnfile, kilnfileLock, loadErr := c.Options.LoadKilnfiles(nil, nil) + if loadErr != nil { + return fmt.Errorf("failed to load Kilnfiles: %w", loadErr) + } + + if len(kilnfileLock.Releases) == 0 { + return fmt.Errorf("no releases found in Kilnfile.lock") + } + releaseLock := kilnfileLock.Releases[0] + + tmpDir, tmpErr := os.MkdirTemp("", "carvel-rebake-*") + if tmpErr != nil { + return fmt.Errorf("failed to create temp directory: %w", tmpErr) + } + defer func() { _ = os.RemoveAll(tmpDir) }() + + c.outLogger.Printf("Re-baking Carvel tile from %s using lockfile", sourcePath) + localTarball, dlErr := downloadCarvelRelease(c.outLogger, kilnfile, kilnfileLock, tmpDir) + if dlErr != nil { + return fmt.Errorf("failed to download release from Artifactory: %w", dlErr) + } + + err = b.BakeFromLockfile(sourcePath, releaseLock, localTarball) + } else { + c.outLogger.Printf("Re-baking Carvel tile from %s", sourcePath) + err = b.Bake(sourcePath) + } + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile: %w", err) + } + + err = b.KilnBake(targetPath) + if err != nil { + return fmt.Errorf("failed to bake tile: %w", err) + } + + checksum, err := rebakeFileChecksum(targetPath) + if err != nil { + return fmt.Errorf("failed to checksum tile: %w", err) + } + + if record.FileChecksum != "" && record.FileChecksum != checksum { + return fmt.Errorf("tile checksum mismatch: record has %s, rebake produced %s", record.FileChecksum, checksum) + } + + ver, _ := b.GetVersion() + c.outLogger.Printf("Re-baked %s version %s to %s", b.GetName(), ver, targetPath) + + return nil +} + +func (c CarvelReBake) Usage() jhanda.Usage { + return jhanda.Usage{ + Description: "Re-bakes a Carvel tile from a bake record for reproducible builds.\nThe repository must be checked out at the source_revision specified in the bake record.\nWhen a Kilnfile.lock is present, downloads the cached BOSH release from Artifactory.\n\nThe argument is the path to a JSON bake record file produced by 'kiln carvel publish --final'.", + ShortDescription: "re-bakes a Carvel tile from a bake record", + Flags: c.Options, + } +} + +func rebakeFileChecksum(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} diff --git a/internal/commands/carvel_rebake_test.go b/internal/commands/carvel_rebake_test.go new file mode 100644 index 000000000..00e822584 --- /dev/null +++ b/internal/commands/carvel_rebake_test.go @@ -0,0 +1,270 @@ +package commands_test + +import ( + "encoding/json" + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/commands" + "github.com/pivotal-cf/kiln/pkg/bake" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" +) + +var _ = Describe("CarvelReBake", func() { + var ( + outLogger *log.Logger + errLogger *log.Logger + command commands.CarvelReBake + ) + + BeforeEach(func() { + outLogger = log.New(GinkgoWriter, "", 0) + errLogger = log.New(GinkgoWriter, "", 0) + command = commands.NewCarvelReBake(outLogger, errLogger) + }) + + Describe("Usage", func() { + It("returns usage information", func() { + usage := command.Usage() + Expect(usage.ShortDescription).To(Equal("re-bakes a Carvel tile from a bake record")) + Expect(usage.Description).To(ContainSubstring("bake record")) + }) + }) + + Describe("Execute", func() { + When("required arguments are missing", func() { + It("returns an error when no bake record is provided", func() { + err := command.Execute([]string{ + "--output-file", "/tmp/out.pivotal", + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("exactly one bake record argument")) + }) + + It("returns an error when output-file is not provided", func() { + err := command.Execute([]string{"some-record.json"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("output-file")) + }) + }) + + When("the bake record file does not exist", func() { + It("returns an error", func() { + err := command.Execute([]string{ + "--output-file", "/tmp/out.pivotal", + "/nonexistent/record.json", + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to read bake record")) + }) + }) + + When("the bake record has a mismatched source revision", func() { + var ( + inputPath string + recordPath string + ) + + BeforeEach(func() { + var err error + inputPath, err = os.MkdirTemp("", "rebake-test-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + cmds := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "-c", "user.name=test", "-c", "user.email=test@test.com", "commit", "-m", "initial commit"), + } + for _, cmd := range cmds { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + + record := bake.Record{ + SourceRevision: "0000000000000000000000000000000000000000", + Version: "0.1.1", + TileDirectory: inputPath, + } + buf, err := json.Marshal(record) + Expect(err).NotTo(HaveOccurred()) + + recordPath = filepath.Join(filepath.Dir(inputPath), "record.json") + err = os.WriteFile(recordPath, buf, 0644) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + }) + + It("returns a source revision mismatch error", func() { + err := command.Execute([]string{ + "--output-file", filepath.Join(filepath.Dir(inputPath), "out.pivotal"), + recordPath, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("source revision")) + }) + }) + + When("a valid bake record and mock Artifactory are provided", func() { + var ( + inputPath string + outputPath string + recordPath string + server *httptest.Server + ) + + BeforeEach(func() { + if !boshInstalled() { + Skip("bosh CLI not installed — skipping rebake integration test") + } + if !kilnInstalled() { + Skip("kiln CLI not installed — skipping rebake integration test") + } + + var err error + inputPath, err = os.MkdirTemp("", "rebake-happy-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + gitCmd := func(args ...string) { + cmd := exec.Command("git", args...) + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "git %v: %s", args, out) + } + + gitCmd("init") + gitCmd("add", ".") + gitCmd("commit", "-m", "initial commit") + + b := carvel.NewBaker() + b.SetWriter(GinkgoWriter) + Expect(b.Bake(inputPath)).To(Succeed()) + tarball, err := b.GetReleaseTarball() + Expect(err).NotTo(HaveOccurred()) + tarballData, err := os.ReadFile(tarball) + Expect(err).NotTo(HaveOccurred()) + + var ( + mu sync.Mutex + blobs = make(map[string][]byte) + ) + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + key := strings.TrimPrefix(r.URL.Path, "/artifactory") + switch r.Method { + case http.MethodPut: + body, _ := io.ReadAll(r.Body) + mu.Lock() + blobs[key] = body + mu.Unlock() + w.WriteHeader(http.StatusCreated) + case http.MethodGet: + mu.Lock() + data, ok := blobs[key] + mu.Unlock() + if !ok { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(data) + } + })) + + // Pre-load the mock with the tarball at the expected path + remotePath := "/test-repo/bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz" + blobs[remotePath] = tarballData + + kf := cargo.Kilnfile{ + ReleaseSources: []cargo.ReleaseSourceConfig{{ + Type: "artifactory", + ArtifactoryHost: server.URL, + Repo: "test-repo", + Username: "user", + Password: "pass", + PathTemplate: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz", + }}, + } + kfData, err := yaml.Marshal(&kf) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(inputPath, "Kilnfile"), kfData, 0644)).To(Succeed()) + + lock := cargo.KilnfileLock{ + Releases: []cargo.BOSHReleaseTarballLock{{ + Name: "k8s-tile-test", + Version: "0.1.1", + RemotePath: "bosh-releases/k8s-tile-test/k8s-tile-test-0.1.1.tgz", + RemoteSource: "artifactory", + }}, + Stemcell: cargo.Stemcell{OS: "ubuntu-jammy", Version: "1.446"}, + } + lockData, err := yaml.Marshal(&lock) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(inputPath, "Kilnfile.lock"), lockData, 0644)).To(Succeed()) + + gitCmd("add", ".") + gitCmd("commit", "-m", "add kilnfiles") + + sha := strings.TrimSpace(func() string { + cmd := exec.Command("git", "rev-parse", "HEAD") + cmd.Dir = inputPath + out, _ := cmd.Output() + return string(out) + }()) + + record := bake.Record{ + SourceRevision: sha, + Version: "0.1.1", + TileDirectory: inputPath, + } + buf, err := json.Marshal(record) + Expect(err).NotTo(HaveOccurred()) + + recordPath = filepath.Join(filepath.Dir(inputPath), "record.json") + Expect(os.WriteFile(recordPath, buf, 0644)).To(Succeed()) + + outputPath = filepath.Join(filepath.Dir(inputPath), "output.pivotal") + }) + + AfterEach(func() { + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + if server != nil { + server.Close() + } + }) + + It("re-bakes successfully from the bake record", func() { + err := command.Execute([]string{ + "--output-file", outputPath, + "--verbose", + recordPath, + }) + Expect(err).NotTo(HaveOccurred()) + Expect(outputPath).To(BeAnExistingFile()) + }) + }) + }) +}) diff --git a/internal/commands/carvel_upload.go b/internal/commands/carvel_upload.go new file mode 100644 index 000000000..0a58e9afd --- /dev/null +++ b/internal/commands/carvel_upload.go @@ -0,0 +1,196 @@ +package commands + +import ( + "bytes" + "crypto/sha1" + "encoding/hex" + "fmt" + "io" + "log" + "net/http" + "os" + "path/filepath" + "text/template" + + "github.com/pivotal-cf/jhanda" + "github.com/pivotal-cf/kiln/internal/carvel" + "github.com/pivotal-cf/kiln/internal/commands/flags" + "github.com/pivotal-cf/kiln/pkg/cargo" +) + +type CarvelUpload struct { + outLogger *log.Logger + errLogger *log.Logger + Options CarvelUploadOptions +} + +type CarvelUploadOptions struct { + flags.Standard + SourceDirectory string `short:"s" long:"source-directory" description:"path to the Carvel tile source directory (defaults to current directory)"` + OutputFile string `short:"o" long:"output-file" description:"also bake the tile to this path"` + PathTemplate string ` long:"path-template" description:"remote path template override" default:"bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz"` + Verbose bool `short:"v" long:"verbose" description:"enable verbose output"` +} + +func NewCarvelUpload(outLogger, errLogger *log.Logger) CarvelUpload { + return CarvelUpload{ + outLogger: outLogger, + errLogger: errLogger, + } +} + +func (c CarvelUpload) Execute(args []string) error { + _, err := jhanda.Parse(&c.Options, args) + if err != nil { + return err + } + + sourcePath, err := resolveSourcePath(c.Options.SourceDirectory) + if err != nil { + return err + } + + kilnfilePath := resolveKilnfilePath(c.Options.Kilnfile, sourcePath) + + if _, statErr := os.Stat(kilnfilePath); statErr != nil { + return fmt.Errorf("could not find Kilnfile at %s: create a Kilnfile with an artifactory release_source", kilnfilePath) + } + + c.Options.Kilnfile = kilnfilePath + kilnfile, err := loadKilnfileOnly(c.Options.Standard) + if err != nil { + return fmt.Errorf("failed to load Kilnfile: %w", err) + } + + artConfig, err := findArtifactorySource(kilnfile) + if err != nil { + return err + } + + baker := carvel.NewBaker() + if c.Options.Verbose { + baker.SetWriter(os.Stdout) + } + + c.outLogger.Printf("Baking Carvel tile from %s", sourcePath) + err = baker.Bake(sourcePath) + if err != nil { + return fmt.Errorf("failed to prepare Carvel tile: %w", err) + } + + tarball, err := baker.GetReleaseTarball() + if err != nil { + return fmt.Errorf("failed to locate release tarball: %w", err) + } + + ver, err := baker.GetVersion() + if err != nil { + return fmt.Errorf("failed to get tile version: %w", err) + } + + pathTmpl := c.Options.PathTemplate + if artConfig.PathTemplate != "" { + pathTmpl = artConfig.PathTemplate + } + remotePath, err := evaluatePathTemplate(pathTmpl, baker.GetName(), ver) + if err != nil { + return fmt.Errorf("failed to evaluate path template: %w", err) + } + + sha1sum, err := fileSHA1(tarball) + if err != nil { + return fmt.Errorf("failed to checksum release tarball: %w", err) + } + + c.outLogger.Printf("Uploading %s to %s/%s/%s", filepath.Base(tarball), artConfig.ArtifactoryHost, artConfig.Repo, remotePath) + err = uploadToArtifactory(tarball, artConfig.ArtifactoryHost, artConfig.Repo, remotePath, artConfig.Username, artConfig.Password) + if err != nil { + return fmt.Errorf("failed to upload to Artifactory: %w", err) + } + + sourceID := cargo.BOSHReleaseTarballSourceID(artConfig) + lockfilePath := kilnfilePath + ".lock" + err = writeStandardKilnfileLock(lockfilePath, baker.GetName(), ver, remotePath, sourceID, sha1sum) + if err != nil { + return fmt.Errorf("failed to write Kilnfile.lock: %w", err) + } + c.outLogger.Printf("Updated %s", lockfilePath) + + if c.Options.OutputFile != "" { + targetPath, err := filepath.Abs(c.Options.OutputFile) + if err != nil { + return fmt.Errorf("failed to resolve output file path: %w", err) + } + err = baker.KilnBake(targetPath) + if err != nil { + return fmt.Errorf("failed to bake tile: %w", err) + } + c.outLogger.Printf("Baked %s version %s to %s", baker.GetName(), ver, targetPath) + } + + return nil +} + +func (c CarvelUpload) Usage() jhanda.Usage { + return jhanda.Usage{ + Description: "Generates a BOSH release from a Carvel tile source, uploads the release tarball to Artifactory, and updates Kilnfile.lock with the remote location and checksum. Artifactory credentials are read from the Kilnfile's release_sources (typically interpolated from ~/.kiln/credentials.yml).", + ShortDescription: "uploads a Carvel BOSH release to Artifactory", + Flags: c.Options, + } +} + +func fileSHA1(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + h := sha1.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil +} + +func evaluatePathTemplate(tmpl, name, version string) (string, error) { + t, err := template.New("path").Parse(tmpl) + if err != nil { + return "", err + } + var buf bytes.Buffer + err = t.Execute(&buf, cargo.BOSHReleaseTarballSpecification{Name: name, Version: version}) + if err != nil { + return "", err + } + return buf.String(), nil +} + +func uploadToArtifactory(localPath, host, repo, remotePath, username, password string) error { + f, err := os.Open(localPath) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + + uploadURL := host + "/" + repo + "/" + remotePath + + req, err := http.NewRequest(http.MethodPut, uploadURL, f) + if err != nil { + return err + } + req.SetBasicAuth(username, password) + req.Header.Set("Content-Type", "application/gzip") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return fmt.Errorf("upload failed with status %d: %s", resp.StatusCode, string(body)) + } + + return nil +} diff --git a/internal/commands/carvel_upload_test.go b/internal/commands/carvel_upload_test.go new file mode 100644 index 000000000..b00f7a5d8 --- /dev/null +++ b/internal/commands/carvel_upload_test.go @@ -0,0 +1,177 @@ +package commands_test + +import ( + "io" + "log" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/pivotal-cf/kiln/internal/commands" + "github.com/pivotal-cf/kiln/pkg/cargo" + "gopkg.in/yaml.v3" +) + +var _ = Describe("CarvelUpload", func() { + var ( + outLogger *log.Logger + errLogger *log.Logger + command commands.CarvelUpload + ) + + BeforeEach(func() { + outLogger = log.New(GinkgoWriter, "", 0) + errLogger = log.New(GinkgoWriter, "", 0) + command = commands.NewCarvelUpload(outLogger, errLogger) + }) + + Describe("Usage", func() { + It("returns usage information", func() { + usage := command.Usage() + Expect(usage.ShortDescription).To(Equal("uploads a Carvel BOSH release to Artifactory")) + Expect(usage.Description).To(ContainSubstring("Artifactory")) + }) + }) + + Describe("Execute", func() { + When("Kilnfile is missing", func() { + It("returns an error", func() { + tmpDir, err := os.MkdirTemp("", "upload-no-kilnfile-*") + Expect(err).NotTo(HaveOccurred()) + defer func() { _ = os.RemoveAll(tmpDir) }() + + err = command.Execute([]string{ + "--source-directory", tmpDir, + }) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("could not find Kilnfile")) + }) + }) + + When("valid Kilnfile is provided with a round-trip mock Artifactory", func() { + var ( + inputPath string + server *httptest.Server + mu sync.Mutex + blobs map[string][]byte + authOK bool + ) + + BeforeEach(func() { + if !boshInstalled() { + Skip("bosh CLI not installed - skipping integration test") + } + + blobs = make(map[string][]byte) + authOK = false + + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u, p, ok := r.BasicAuth() + if !ok || u != "user" || p != "pass" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + key := strings.TrimPrefix(r.URL.Path, "/artifactory") + switch r.Method { + case http.MethodPut: + body, _ := io.ReadAll(r.Body) + mu.Lock() + blobs[key] = body + authOK = true + mu.Unlock() + w.WriteHeader(http.StatusCreated) + case http.MethodGet: + mu.Lock() + data, found := blobs[key] + mu.Unlock() + if !found { + http.Error(w, "not found", http.StatusNotFound) + return + } + w.Header().Set("Content-Type", "application/gzip") + _, _ = w.Write(data) + } + })) + + var err error + inputPath, err = os.MkdirTemp("", "upload-test-*") + Expect(err).NotTo(HaveOccurred()) + inputPath += "/tile" + err = os.CopyFS(inputPath, os.DirFS("../carvel/testdata/sample-tile")) + Expect(err).NotTo(HaveOccurred()) + + kf := cargo.Kilnfile{ + ReleaseSources: []cargo.ReleaseSourceConfig{{ + Type: "artifactory", + ArtifactoryHost: server.URL, + Repo: "test-repo", + Username: "user", + Password: "pass", + PathTemplate: "bosh-releases/{{.Name}}/{{.Name}}-{{.Version}}.tgz", + }}, + } + kfData, err := yaml.Marshal(&kf) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(inputPath, "Kilnfile"), kfData, 0644)).To(Succeed()) + + cmds := []*exec.Cmd{ + exec.Command("git", "init"), + exec.Command("git", "add", "."), + exec.Command("git", "commit", "-m", "initial commit"), + } + for _, cmd := range cmds { + cmd.Dir = inputPath + out, err := cmd.CombinedOutput() + Expect(err).NotTo(HaveOccurred(), "error invoking git: "+string(out)) + } + }) + + AfterEach(func() { + if inputPath != "" { + _ = os.RemoveAll(filepath.Dir(inputPath)) + } + if server != nil { + server.Close() + } + }) + + It("uploads the BOSH release and writes a standard Kilnfile.lock", func() { + err := command.Execute([]string{ + "--source-directory", inputPath, + "--verbose", + }) + Expect(err).NotTo(HaveOccurred()) + + lockfilePath := filepath.Join(inputPath, "Kilnfile.lock") + Expect(lockfilePath).To(BeAnExistingFile()) + + lockData, err := os.ReadFile(lockfilePath) + Expect(err).NotTo(HaveOccurred()) + + var lock cargo.KilnfileLock + Expect(yaml.Unmarshal(lockData, &lock)).To(Succeed()) + Expect(lock.Releases).To(HaveLen(1)) + Expect(lock.Releases[0].Name).To(Equal("k8s-tile-test")) + Expect(lock.Releases[0].Version).To(Equal("0.1.1")) + Expect(lock.Releases[0].SHA1).NotTo(BeEmpty()) + Expect(lock.Releases[0].RemotePath).To(ContainSubstring("k8s-tile-test")) + Expect(lock.Releases[0].RemoteSource).To(Equal("artifactory")) + + By("verifying mock Artifactory received the PUT with Basic Auth") + mu.Lock() + Expect(authOK).To(BeTrue(), "upload must authenticate with Basic Auth") + Expect(blobs).To(HaveLen(1), "exactly one blob should be stored") + for _, data := range blobs { + Expect(len(data)).To(BeNumerically(">", 0), "uploaded tarball must not be empty") + } + mu.Unlock() + }) + }) + }) +}) diff --git a/main.go b/main.go index fb6c5fe58..d4f34ab0c 100644 --- a/main.go +++ b/main.go @@ -46,7 +46,11 @@ func main() { } if global.Help { - command = "help" + if command == "carvel" && len(args) > 0 { + args = append(args, "--help") + } else { + command = "help" + } } if command == "" { @@ -76,6 +80,7 @@ func main() { bakeCommand.KilnVersion = version commandSet["bake"] = bakeCommand commandSet["re-bake"] = commands.NewReBake(bakeCommand) + commandSet["rebake"] = commandSet["re-bake"] commandSet["test"] = commands.NewTileTest() commandSet["help"] = commands.NewHelp(os.Stdout, globalFlagsUsage, commandSet) @@ -101,7 +106,14 @@ func main() { log.Fatal(err) } - err = commandSet.Execute(command, args) + carvelCommand := commands.NewCarvel(outLogger, errLogger) + commandSet["carvel"] = carvelCommand + + if command == "carvel" { + err = carvelCommand.Execute(args) + } else { + err = commandSet.Execute(command, args) + } if err != nil { log.Fatal(err) }