diff --git a/docs/md/melange.md b/docs/md/melange.md index 80bec630c..344f2f8b2 100644 --- a/docs/md/melange.md +++ b/docs/md/melange.md @@ -33,6 +33,7 @@ toc: true * [melange scan](/docs/md/melange_scan.md) - Scan an existing APK to regenerate .PKGINFO * [melange sign](/docs/md/melange_sign.md) - Sign an APK package * [melange sign-index](/docs/md/melange_sign-index.md) - Sign an APK index +* [melange source](/docs/md/melange_source.md) - Manage melange source code * [melange test](/docs/md/melange_test.md) - Test a package with a YAML configuration file * [melange update-cache](/docs/md/melange_update-cache.md) - Update a source artifact cache * [melange version](/docs/md/melange_version.md) - Prints the version diff --git a/docs/md/melange_source.md b/docs/md/melange_source.md new file mode 100644 index 000000000..07caf1265 --- /dev/null +++ b/docs/md/melange_source.md @@ -0,0 +1,37 @@ +--- +title: "melange source" +slug: melange_source +url: /docs/md/melange_source.md +draft: false +images: [] +type: "article" +toc: true +--- +## melange source + +Manage melange source code + +### Synopsis + +Commands for managing source code from melange configurations. + +### Options + +``` + -h, --help help for source + -o, --output string output directory for extracted source (default "./source") + --source-dir string directory where patches and other sources are located (defaults to ./package-name/) +``` + +### Options inherited from parent commands + +``` + --log-level string log level (e.g. debug, info, warn, error) (default "INFO") +``` + +### SEE ALSO + +* [melange](/docs/md/melange.md) - +* [melange source get](/docs/md/melange_source_get.md) - Extract source code from melange configuration +* [melange source pop](/docs/md/melange_source_pop.md) - Generate patches from modified source and update melange configuration + diff --git a/docs/md/melange_source_get.md b/docs/md/melange_source_get.md new file mode 100644 index 000000000..8464fc2c5 --- /dev/null +++ b/docs/md/melange_source_get.md @@ -0,0 +1,49 @@ +--- +title: "melange source get" +slug: melange_source_get +url: /docs/md/melange_source_get.md +draft: false +images: [] +type: "article" +toc: true +--- +## melange source get + +Extract source code from melange configuration + +### Synopsis + +Extract source code by cloning git repositories from melange configuration. + +This command parses a melange configuration file and extracts sources to the given directory +Currently only supports git-checkout. + + +``` +melange source get [config.yaml] [flags] +``` + +### Examples + +``` + melange source get vim.yaml -o ./src +``` + +### Options + +``` + -h, --help help for get +``` + +### Options inherited from parent commands + +``` + --log-level string log level (e.g. debug, info, warn, error) (default "INFO") + -o, --output string output directory for extracted source (default "./source") + --source-dir string directory where patches and other sources are located (defaults to ./package-name/) +``` + +### SEE ALSO + +* [melange source](/docs/md/melange_source.md) - Manage melange source code + diff --git a/docs/md/melange_source_pop.md b/docs/md/melange_source_pop.md new file mode 100644 index 000000000..1cb04a687 --- /dev/null +++ b/docs/md/melange_source_pop.md @@ -0,0 +1,53 @@ +--- +title: "melange source pop" +slug: melange_source_pop +url: /docs/md/melange_source_pop.md +draft: false +images: [] +type: "article" +toc: true +--- +## melange source pop + +Generate patches from modified source and update melange configuration + +### Synopsis + +Generate git format-patch patches from commits made on top of the expected-commit +and update the melange configuration to use git-am pipeline instead of patch pipeline. + +This command: +1. Reads the expected-commit from git-checkout pipeline +2. Generates patches from expected-commit..HEAD in the cloned source +3. Writes patches to the source directory +4. Updates the YAML to replace 'patch' with 'git-am' pipeline + + +``` +melange source pop [config.yaml] [flags] +``` + +### Examples + +``` + melange source pop apk-tools.yaml +``` + +### Options + +``` + -h, --help help for pop +``` + +### Options inherited from parent commands + +``` + --log-level string log level (e.g. debug, info, warn, error) (default "INFO") + -o, --output string output directory for extracted source (default "./source") + --source-dir string directory where patches and other sources are located (defaults to ./package-name/) +``` + +### SEE ALSO + +* [melange source](/docs/md/melange_source.md) - Manage melange source code + diff --git a/pkg/build/pipelines/README.md b/pkg/build/pipelines/README.md index d90c71e27..5038c64a8 100644 --- a/pkg/build/pipelines/README.md +++ b/pkg/build/pipelines/README.md @@ -8,6 +8,7 @@ new built-in pipelines, consult [Creating a new built-in pipeline](/docs/PIPELIN - [fetch](#fetch) +- [git-am](#git-am) - [git-checkout](#git-checkout) - [patch](#patch) - [strip](#strip) @@ -33,6 +34,16 @@ Fetch and extract external object into workspace | timeout | false | The timeout (in seconds) to use for connecting and reading. The fetch will fail if the timeout is hit. | 5 | | uri | true | The URI to fetch as an artifact. | | +## git-am + +Apply patches with git am + +### Inputs + +| Name | Required | Description | Default | +| ---- | -------- | ----------- | ------- | +| patches | true | A list of patches to apply with git am, as a whitespace delimited string. | | + ## git-checkout Check out sources from git diff --git a/pkg/build/pipelines/git-am.yaml b/pkg/build/pipelines/git-am.yaml new file mode 100644 index 000000000..449886d59 --- /dev/null +++ b/pkg/build/pipelines/git-am.yaml @@ -0,0 +1,15 @@ +name: Apply patches with git am + +needs: + packages: + - git + +inputs: + patches: + description: | + A list of patches to apply with git am, as a whitespace delimited string. + required: true + +pipeline: + - runs: | + git am ${{inputs.patches}} diff --git a/pkg/cli/commands.go b/pkg/cli/commands.go index ed50807a4..3f7e79c76 100644 --- a/pkg/cli/commands.go +++ b/pkg/cli/commands.go @@ -64,6 +64,7 @@ func New() *cobra.Command { cmd.AddCommand(scan()) cmd.AddCommand(signCmd()) cmd.AddCommand(signIndex()) + cmd.AddCommand(sourceCmd()) cmd.AddCommand(test()) cmd.AddCommand(updateCache()) cmd.AddCommand(version.Version()) diff --git a/pkg/cli/source.go b/pkg/cli/source.go new file mode 100644 index 000000000..f0a458617 --- /dev/null +++ b/pkg/cli/source.go @@ -0,0 +1,347 @@ +// Copyright 2025 Chainguard, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/chainguard-dev/clog" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + + "chainguard.dev/melange/pkg/config" + "chainguard.dev/melange/pkg/source" +) + +func sourceCmd() *cobra.Command { + var outputDir string + var sourceDir string + + cmd := &cobra.Command{ + Use: "source", + Short: "Manage melange source code", + Long: `Commands for managing source code from melange configurations.`, + } + + // Shared flags for all source subcommands + cmd.PersistentFlags().StringVarP(&outputDir, "output", "o", "./source", "output directory for extracted source") + cmd.PersistentFlags().StringVar(&sourceDir, "source-dir", "", "directory where patches and other sources are located (defaults to ./package-name/)") + + // Add subcommands + cmd.AddCommand(sourceGetCmd(&outputDir, &sourceDir)) + cmd.AddCommand(sourcePopCmd(&outputDir, &sourceDir)) + + return cmd +} + +func sourceGetCmd(outputDir *string, sourceDir *string) *cobra.Command { + cmd := &cobra.Command{ + Use: "get [config.yaml]", + Short: "Extract source code from melange configuration", + Long: `Extract source code by cloning git repositories from melange configuration. + +This command parses a melange configuration file and extracts sources to the given directory +Currently only supports git-checkout. +`, + Example: ` melange source get vim.yaml -o ./src`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + log := clog.FromContext(ctx) + + buildConfigPath := args[0] + + cfg, err := config.ParseConfiguration(ctx, buildConfigPath) + if err != nil { + return fmt.Errorf("failed to parse melange config: %w", err) + } + + // Look for git-checkout and patch pipelines + gitCheckoutIndex := -1 + var patches string + + // First pass: find git-checkout step + for i, step := range cfg.Pipeline { + if step.Uses == "git-checkout" { + gitCheckoutIndex = i + break + } + } + + if gitCheckoutIndex == -1 { + return fmt.Errorf("no git-checkout pipeline found in configuration") + } + + // Second pass: find patch steps that come after git-checkout + for i := gitCheckoutIndex + 1; i < len(cfg.Pipeline); i++ { + step := cfg.Pipeline[i] + if step.Uses == "patch" { + if patchList := step.With["patches"]; patchList != "" { + patches = patchList + break // Only process first patch step + } + } + } + + // Now perform the git checkout with patches + step := cfg.Pipeline[gitCheckoutIndex] + log.Infof("Found git-checkout step") + + // Construct destination: outputDir/packageName + destination := fmt.Sprintf("%s/%s", *outputDir, cfg.Package.Name) + + // Default sourceDir to package-name subdirectory in config file's directory + // This matches melange build behavior: --source-dir ./package-name/ + srcDir := *sourceDir + if srcDir == "" { + srcDir = filepath.Join(filepath.Dir(buildConfigPath), cfg.Package.Name) + } + + // Make sourceDir absolute since git commands will run from the cloned repo + absSourceDir, err := filepath.Abs(srcDir) + if err != nil { + return fmt.Errorf("failed to get absolute path for source-dir: %w", err) + } + + opts := &source.GitCheckoutOptions{ + Repository: step.With["repository"], + Destination: destination, + ExpectedCommit: step.With["expected-commit"], + CherryPicks: step.With["cherry-picks"], + Patches: patches, + WorkspaceDir: absSourceDir, + } + + if err := source.GitCheckout(ctx, opts); err != nil { + return fmt.Errorf("failed to checkout source: %w", err) + } + + log.Infof("Successfully extracted source to %s", *outputDir) + return nil + }, + } + + return cmd +} + +func sourcePopCmd(outputDir *string, sourceDir *string) *cobra.Command { + cmd := &cobra.Command{ + Use: "pop [config.yaml]", + Short: "Generate patches from modified source and update melange configuration", + Long: `Generate git format-patch patches from commits made on top of the expected-commit +and update the melange configuration to use git-am pipeline instead of patch pipeline. + +This command: +1. Reads the expected-commit from git-checkout pipeline +2. Generates patches from expected-commit..HEAD in the cloned source +3. Writes patches to the source directory +4. Updates the YAML to replace 'patch' with 'git-am' pipeline +`, + Example: ` melange source pop apk-tools.yaml`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + log := clog.FromContext(ctx) + + buildConfigPath := args[0] + + cfg, err := config.ParseConfiguration(ctx, buildConfigPath) + if err != nil { + return fmt.Errorf("failed to parse melange config: %w", err) + } + + // Find git-checkout step to get expected-commit + var expectedCommit string + for _, step := range cfg.Pipeline { + if step.Uses == "git-checkout" { + expectedCommit = step.With["expected-commit"] + break + } + } + + if expectedCommit == "" { + return fmt.Errorf("no expected-commit found in git-checkout pipeline") + } + + // Default sourceDir to package-name subdirectory in config file's directory + srcDir := *sourceDir + if srcDir == "" { + srcDir = filepath.Join(filepath.Dir(buildConfigPath), cfg.Package.Name) + } + + // Make sourceDir absolute + absSrcDir, err := filepath.Abs(srcDir) + if err != nil { + return fmt.Errorf("failed to get absolute path for source-dir: %w", err) + } + + // Cloned source location + clonedSource := filepath.Join(*outputDir, cfg.Package.Name) + absClonedSource, err := filepath.Abs(clonedSource) + if err != nil { + return fmt.Errorf("failed to get absolute path for cloned source: %w", err) + } + + log.Infof("Generating patches from %s in %s", expectedCommit, absClonedSource) + + // Generate patches using git format-patch + formatPatchCmd := exec.CommandContext(ctx, "git", "format-patch", "-o", absSrcDir, expectedCommit+"..HEAD") + formatPatchCmd.Dir = absClonedSource + output, err := formatPatchCmd.Output() + if err != nil { + return fmt.Errorf("failed to generate patches: %w", err) + } + + // Parse the patch filenames from git format-patch output + patchLines := strings.Split(strings.TrimSpace(string(output)), "\n") + var patchFiles []string + for _, line := range patchLines { + if line != "" { + // Extract just the filename + patchFiles = append(patchFiles, filepath.Base(line)) + } + } + + if len(patchFiles) == 0 { + return fmt.Errorf("no patches generated - no commits found after %s", expectedCommit) + } + + log.Infof("Generated %d patches: %v", len(patchFiles), patchFiles) + + // Read the original YAML file + yamlData, err := os.ReadFile(buildConfigPath) + if err != nil { + return fmt.Errorf("failed to read YAML file: %w", err) + } + + // Parse as generic YAML to preserve structure and comments + var doc yaml.Node + if err := yaml.Unmarshal(yamlData, &doc); err != nil { + return fmt.Errorf("failed to parse YAML: %w", err) + } + + // Update the pipeline: remove 'patch' steps and add 'git-am' step + if err := updatePipelineWithGitAm(&doc, patchFiles); err != nil { + return fmt.Errorf("failed to update pipeline: %w", err) + } + + // Write back the updated YAML + updatedYaml, err := yaml.Marshal(&doc) + if err != nil { + return fmt.Errorf("failed to marshal YAML: %w", err) + } + + // #nosec G306 these are melange yaml files + if err := os.WriteFile(buildConfigPath, updatedYaml, 0o644); err != nil { + return fmt.Errorf("failed to write updated YAML: %w", err) + } + + // Try to run yam to fix formatting + yamCmd := exec.CommandContext(ctx, "yam", buildConfigPath) + if err := yamCmd.Run(); err != nil { + log.Warnf("Failed to run yam for formatting (continuing anyway): %v", err) + } else { + log.Infof("Formatted YAML with yam") + } + + log.Infof("Updated %s with git-am pipeline using %d patches", buildConfigPath, len(patchFiles)) + return nil + }, + } + + return cmd +} + +// updatePipelineWithGitAm finds the pipeline array in the YAML node tree, +// and replaces any 'patch' pipeline step with a 'git-am' step with the given patches. +// If no patch step exists, inserts git-am after git-checkout. +func updatePipelineWithGitAm(doc *yaml.Node, patchFiles []string) error { + // Navigate to the pipeline array + // doc.Content[0] is the document node + // doc.Content[0].Content contains key-value pairs of the root map + + if len(doc.Content) == 0 || len(doc.Content[0].Content) == 0 { + return fmt.Errorf("invalid YAML structure") + } + + rootMap := doc.Content[0] + var pipelineNode *yaml.Node + + // Find the 'pipeline' key + for i := 0; i < len(rootMap.Content); i += 2 { + if rootMap.Content[i].Value == "pipeline" { + pipelineNode = rootMap.Content[i+1] + break + } + } + + if pipelineNode == nil { + return fmt.Errorf("no pipeline found in YAML") + } + + // Create git-am step + gitAmStep := &yaml.Node{ + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "uses"}, + {Kind: yaml.ScalarNode, Value: "git-am"}, + {Kind: yaml.ScalarNode, Value: "with"}, + { + Kind: yaml.MappingNode, + Content: []*yaml.Node{ + {Kind: yaml.ScalarNode, Value: "patches"}, + {Kind: yaml.ScalarNode, Value: strings.Join(patchFiles, " ")}, + }, + }, + }, + } + + // Try to replace 'uses: patch' step with 'uses: git-am' step in place + replacedAny := false + for i, step := range pipelineNode.Content { + // Check if this step has 'uses: patch' + for j := 0; j < len(step.Content); j += 2 { + if step.Content[j].Value == "uses" && step.Content[j+1].Value == "patch" { + // Replace this step with git-am step + pipelineNode.Content[i] = gitAmStep + replacedAny = true + break + } + } + } + + // If no patch step found, insert git-am after git-checkout + if !replacedAny { + var newContent []*yaml.Node + for _, step := range pipelineNode.Content { + newContent = append(newContent, step) + // Check if this is git-checkout step + for j := 0; j < len(step.Content); j += 2 { + if step.Content[j].Value == "uses" && step.Content[j+1].Value == "git-checkout" { + // Insert git-am step right after + newContent = append(newContent, gitAmStep) + break + } + } + } + pipelineNode.Content = newContent + } + + return nil +} diff --git a/pkg/source/git.go b/pkg/source/git.go new file mode 100644 index 000000000..c148c7f46 --- /dev/null +++ b/pkg/source/git.go @@ -0,0 +1,231 @@ +// Copyright 2025 Chainguard, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package source + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "strings" + + "github.com/chainguard-dev/clog" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" +) + +type GitCheckoutOptions struct { + Repository string + Destination string + ExpectedCommit string + CherryPicks string + Patches string + WorkspaceDir string // Directory where patch files are located (usually config dir) +} + +func GitCheckout(ctx context.Context, opts *GitCheckoutOptions) error { + log := clog.FromContext(ctx) + + if opts.Repository == "" { + return fmt.Errorf("repository is required") + } + + if opts.Destination == "" { + return fmt.Errorf("destination is required") + } + + log.Infof("Cloning %s to %s", opts.Repository, opts.Destination) + + cloneOpts := &git.CloneOptions{ + URL: opts.Repository, + Progress: os.Stdout, + } + + repo, err := git.PlainClone(opts.Destination, false, cloneOpts) + if err != nil { + return fmt.Errorf("failed to clone repository: %w", err) + } + + if opts.ExpectedCommit != "" { + log.Infof("Checking out commit %s", opts.ExpectedCommit) + + wt, err := repo.Worktree() + if err != nil { + return fmt.Errorf("failed to get worktree: %w", err) + } + + err = wt.Checkout(&git.CheckoutOptions{ + Hash: plumbing.NewHash(opts.ExpectedCommit), + }) + if err != nil { + return fmt.Errorf("failed to checkout commit %s: %w", opts.ExpectedCommit, err) + } + } + + // Show what we checked out + head, err := repo.Head() + if err != nil { + return fmt.Errorf("failed to get HEAD: %w", err) + } + + log.Infof("Checked out commit %s", head.Hash().String()) + + // Apply cherry-picks if specified + if opts.CherryPicks != "" { + log.Infof("Applying cherry-picks") + picks, err := parseCherryPicks(opts.CherryPicks) + if err != nil { + return fmt.Errorf("failed to parse cherry-picks: %w", err) + } + + if err := applyCherryPicks(ctx, opts.Destination, picks); err != nil { + return fmt.Errorf("failed to apply cherry-picks: %w", err) + } + } + + // Apply patches if specified + if opts.Patches != "" { + log.Infof("Applying patches") + patches := parsePatchList(opts.Patches) + + if err := applyPatches(ctx, opts.Destination, opts.WorkspaceDir, patches); err != nil { + return fmt.Errorf("failed to apply patches: %w", err) + } + } + + return nil +} + +func parseCherryPicks(input string) ([]string, error) { + commits := make([]string, 0) + + for line := range strings.SplitSeq(input, "\n") { + // Trim whitespace + line = strings.TrimSpace(line) + + // Skip empty lines and comments + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + // Parse format: [branch/]commit: comment + // We only care about the commit hash + parts := strings.SplitN(line, ":", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid cherry-pick format (expected '[branch/]commit: comment'): %s", line) + } + + pickSpec := strings.TrimSpace(parts[0]) + + // Strip optional branch prefix (we don't need it with full clone) + commit := path.Base(pickSpec) + + commits = append(commits, commit) + } + + return commits, nil +} + +func applyCherryPicks(ctx context.Context, repoPath string, commits []string) error { + log := clog.FromContext(ctx) + + for _, commit := range commits { + log.Infof("Cherry-picking %s", commit) + + cmd := exec.CommandContext(ctx, "git", "cherry-pick", "-x", commit) + cmd.Dir = repoPath + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to cherry-pick %s: %w", commit, err) + } + } + + return nil +} + +func parsePatchList(input string) []string { + // Parse whitespace-delimited patch list + return strings.Fields(input) +} + +func applyPatches(ctx context.Context, repoPath string, workspaceDir string, patches []string) error { + log := clog.FromContext(ctx) + + for _, patch := range patches { + // Resolve patch path relative to workspace directory + patchPath := patch + if workspaceDir != "" && !strings.HasPrefix(patch, "/") { + patchPath = fmt.Sprintf("%s/%s", workspaceDir, patch) + } + + log.Infof("Applying patch %s", patchPath) + + // Try git am first (preserves commit metadata if present) + amCmd := exec.CommandContext(ctx, "git", "am", patchPath) + amCmd.Dir = repoPath + amCmd.Stdout = os.Stdout + amCmd.Stderr = os.Stderr + + if err := amCmd.Run(); err != nil { + // git am failed, abort to clean up + log.Infof("git am failed, aborting to clean up") + abortCmd := exec.CommandContext(ctx, "git", "am", "--abort") + abortCmd.Dir = repoPath + _ = abortCmd.Run() // Ignore error, may not be in am session + + // Try git apply --check to see if patch is valid + log.Infof("Checking if patch can be applied with git apply") + checkCmd := exec.CommandContext(ctx, "git", "apply", "--check", patchPath) + checkCmd.Dir = repoPath + checkCmd.Stderr = os.Stderr + + if err := checkCmd.Run(); err != nil { + return fmt.Errorf("patch %s cannot be applied: %w", patchPath, err) + } + + // Apply the patch + applyCmd := exec.CommandContext(ctx, "git", "apply", patchPath) + applyCmd.Dir = repoPath + applyCmd.Stdout = os.Stdout + applyCmd.Stderr = os.Stderr + if err := applyCmd.Run(); err != nil { + return fmt.Errorf("failed to apply patch %s: %w", patchPath, err) + } + + // Stage all changes + addCmd := exec.CommandContext(ctx, "git", "add", "-A") + addCmd.Dir = repoPath + if err := addCmd.Run(); err != nil { + return fmt.Errorf("failed to stage changes for patch %s: %w", patchPath, err) + } + + // Commit with patch filename + commitMsg := fmt.Sprintf("Apply patch: %s", patch) + commitCmd := exec.CommandContext(ctx, "git", "commit", "-m", commitMsg) + commitCmd.Dir = repoPath + commitCmd.Stdout = os.Stdout + commitCmd.Stderr = os.Stderr + if err := commitCmd.Run(); err != nil { + return fmt.Errorf("failed to commit patch %s: %w", patchPath, err) + } + + log.Infof("Applied patch %s using git apply + commit", patchPath) + } + } + + return nil +}