feat: support sparse + shallow git clones

pkg/envctx/env_context.go

@@ -34,6 +34,10 @@ type EnvironmentContext struct {
 
 	// GitRepository the current name  of the current git repository we are in
 	GitRepository string
+
+	// SparseCheckout when true the dev environment clone used to load the version stream uses a
+	// sparse checkout of the versionStream/ directory rather than cloning the whole repository
+	SparseCheckout bool
 }
 
 // TeamSettings returns the team settings for the current environment

pkg/envctx/load.go

@@ -91,7 +91,12 @@ func (e *EnvironmentContext) LazyLoad(gclient gitclient.Interface, jxClient vers
 			return fmt.Errorf("failed to add user and token to git url %s: %w", url, err)
 		}
 
-		cloneDir, err := gitclient.CloneToDir(gitter, gitCloneURL, "")
+		var cloneDir string
+		if e.SparseCheckout {
+			cloneDir, err = gitclient.SparseCloneToDir(gitter, gitCloneURL, "", true, "/versionStream/")
+		} else {
+			cloneDir, err = gitclient.CloneToDir(gitter, gitCloneURL, "")
+		}
 		if err != nil {
 			return fmt.Errorf("failed to clone URL %s: %w", gitCloneURL, err)
 		}

pkg/environments/gitops.go

@@ -22,6 +22,11 @@ const (
 	LabelUpdatebot = "updatebot"
 )
 
+// DefaultSparseCheckoutPatterns is the default git sparse-checkout pattern set used when sparse
+// checkout is enabled without explicit patterns. It covers the helmfile and helm promotion rules:
+// the root helmfile, nested helmfiles, the promote config under .jx/ and the helm chart under env/.
+var DefaultSparseCheckoutPatterns = []string{"/helmfile.yaml", "/helmfiles/", "/.jx/", "/env/"}
+
 // Create a pull request against the environment repository for env.
 // The EnvironmentPullRequestOptions are used to provide a Gitter client for performing git operations,
 // a GitProvider client for talking to the git provider,
@@ -68,16 +73,32 @@ func (o *EnvironmentPullRequestOptions) Create(gitURL, prDir string, labels []st
 	}
 
 	var dir string
-	if len(o.SparseCheckoutPatterns) > 0 {
-		dir, err = gitclient.SparseCloneToDir(o.Gitter, cloneGitURL, "", true, o.SparseCheckoutPatterns...)
+	sparse := o.SparseCheckout || len(o.SparseCheckoutPatterns) > 0
+	if sparse {
+		patterns := o.SparseCheckoutPatterns
+		if len(patterns) == 0 {
+			patterns = DefaultSparseCheckoutPatterns
+		}
+		// a shallow (--depth=1) clone implies --single-branch, so it only fetches the default
+		// branch. That is unsafe when: (a) this is a fork, which rebases against the full upstream
+		// history; or (b) we need to check out a non-default base branch below, whose ref would not
+		// be fetched. In those cases fall back to a full-history clone.
+		shallow := !o.Fork && o.BaseBranchName == ""
+		dir, err = gitclient.SparseCloneToDir(o.Gitter, cloneGitURL, "", shallow, patterns...)
+		if err != nil {
+			// sparse-checkout is not supported by every git server
+			// fall-back to a full clone if that fails
+			log.Logger().Warnf("sparse clone of %s failed (%s); falling back to a full clone", cloneGitURLSafe, err)
+			dir, err = gitclient.CloneToDir(o.Gitter, cloneGitURL, "")
+		}
 	} else {
 		dir, err = gitclient.CloneToDir(o.Gitter, cloneGitURL, "")
-		if o.BaseBranchName != "" {
-			log.Logger().Infof("checking out remote base branch %s from %s", o.BaseBranchName, gitURL)
-			err = gitclient.CheckoutRemoteBranch(o.Gitter, dir, o.BaseBranchName)
-			if err != nil {
-				return nil, fmt.Errorf("failed to checkout remote branch %s from %s: %w", o.BaseBranchName, gitURL, err)
-			}
+	}
+	if err == nil && o.BaseBranchName != "" {
+		log.Logger().Infof("checking out remote base branch %s from %s", o.BaseBranchName, gitURL)
+		err = gitclient.CheckoutRemoteBranch(o.Gitter, dir, o.BaseBranchName)
+		if err != nil {
+			return nil, fmt.Errorf("failed to checkout remote branch %s from %s: %w", o.BaseBranchName, gitURL, err)
 		}
 	}
 	if err != nil {

pkg/environments/types.go

@@ -58,6 +58,7 @@ type EnvironmentPullRequestOptions struct {
 	UseGitHubOAuth         bool
 	Fork                   bool
 	ReusePullRequest       bool
+	SparseCheckout         bool
 	SparseCheckoutPatterns []string
 	Application            string
 }

pkg/promote/pr.go

@@ -3,19 +3,59 @@ package promote
 import (
 	"fmt"
 	"os"
+	"path"
+	"strings"
 
 	"github.com/jenkins-x-plugins/jx-promote/pkg/environments"
 	"github.com/jenkins-x/jx-helpers/v3/pkg/requirements"
 
 	jxcore "github.com/jenkins-x/jx-api/v4/pkg/apis/core/v4beta1"
 
+	"github.com/jenkins-x-plugins/jx-promote/pkg/apis/promote/v1alpha1"
 	"github.com/jenkins-x-plugins/jx-promote/pkg/promoteconfig"
 	"github.com/jenkins-x-plugins/jx-promote/pkg/rules"
 	"github.com/jenkins-x-plugins/jx-promote/pkg/rules/factory"
 	"github.com/jenkins-x/jx-helpers/v3/pkg/gitclient"
 	"github.com/jenkins-x/jx-helpers/v3/pkg/gitclient/gitconfig"
+	"github.com/jenkins-x/jx-logging/v3/pkg/log"
 )
 
+// sparsePatternForRule derives the git sparse-checkout pattern (gitignore syntax, root anchored)
+// needed to materialize the target of a file/kpt promote rule so it works under a sparse clone.
+// A FileRule modifies exactly its Path; a KptRule operates within Path/<app>. Returns "" if the
+// target cannot be derived (in which case the caller should fail rather than clone incompletely).
+func sparsePatternForRule(spec v1alpha1.PromoteSpec, appName string) string {
+	if spec.FileRule != nil {
+		p := strings.Trim(spec.FileRule.Path, "/")
+		if p == "" {
+			return ""
+		}
+		return "/" + p
+	}
+	if spec.KptRule != nil {
+		target := strings.Trim(path.Join(spec.KptRule.Path, appName), "/")
+		if target == "" {
+			return ""
+		}
+		return "/" + target + "/"
+	}
+	return ""
+}
+
+// isSparseCheckout reports whether the git working tree in dir is in sparse-checkout
+// mode. SparseCloneToDir enables it via `git sparse-checkout set`, which sets
+// core.sparseCheckout=true; the full-clone fallback in Create() leaves it unset. We
+// key the sparse expansion off this actual repo state rather than the request flags,
+// so a fallback full clone (which already contains every path) is not treated as
+// sparse.
+func isSparseCheckout(gitter gitclient.Interface, dir string) bool {
+	out, err := gitter.Command(dir, "config", "--get", "core.sparseCheckout")
+	if err != nil {
+		return false // key unset -> git exits non-zero -> not sparse
+	}
+	return strings.TrimSpace(out) == "true"
+}
+
 func (o *Options) PromoteViaPullRequest(envs []*jxcore.EnvironmentConfig, releaseInfo *ReleaseInfo, draftPR bool) error {
 	version := o.Version
 	versionName := version
@@ -95,6 +135,24 @@ func (o *Options) PromoteViaPullRequest(envs []*jxcore.EnvironmentConfig, releas
 
 			// lets check if we need the apps git URL
 			if promoteConfig.Spec.FileRule != nil || promoteConfig.Spec.KptRule != nil {
+				// file/kpt rules write to paths not covered by the default sparse-checkout pattern
+				// set. The promote config itself is available (the default patterns include .jx/), so
+				// we can derive the rule's target path and expand the sparse checkout to include it
+				// rather than failing. Gate on the repo's actual sparse-checkout state, not the
+				// request flags: Create() may have fallen back to a full clone when sparse checkout
+				// was unsupported, and that clone already contains the rule's target path.
+				if isSparseCheckout(o.Gitter, dir) {
+					pattern := sparsePatternForRule(promoteConfig.Spec, o.Application)
+					if pattern == "" {
+						return fmt.Errorf("promote config in dir %s uses a file/kpt rule whose target path cannot be derived; please pass explicit --sparse-checkout-pattern values (or omit --sparse-checkout)", dir)
+					}
+					// git sparse-checkout add appends the pattern and re-applies it to the working
+					// tree, lazily fetching the now-included blobs from the (blobless) clone
+					if _, err := o.Gitter.Command(dir, "sparse-checkout", "add", pattern); err != nil {
+						return fmt.Errorf("failed to expand sparse checkout with file/kpt rule path %q in dir %s: %w", pattern, dir, err)
+					}
+					log.Logger().Infof("expanded sparse checkout to include file/kpt rule path %s", pattern)
+				}
 				if o.AppGitURL == "" {
 					_, gitConf, err := gitclient.FindGitConfigDir("")
 					if err != nil {

pkg/promote/pr_internal_test.go

@@ -0,0 +1,94 @@
+//go:build unit
+// +build unit
+
+package promote
+
+import (
+	"testing"
+
+	"github.com/jenkins-x-plugins/jx-promote/pkg/apis/promote/v1alpha1"
+	"github.com/jenkins-x/jx-helpers/v3/pkg/cmdrunner"
+	"github.com/jenkins-x/jx-helpers/v3/pkg/gitclient/cli"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSparsePatternForRule(t *testing.T) {
+	testCases := []struct {
+		name     string
+		spec     v1alpha1.PromoteSpec
+		appName  string
+		expected string
+	}{
+		{
+			name:     "file rule modifies a single file",
+			spec:     v1alpha1.PromoteSpec{FileRule: &v1alpha1.FileRule{Path: "Makefile"}},
+			appName:  "myapp",
+			expected: "/Makefile",
+		},
+		{
+			name:     "file rule with a nested path",
+			spec:     v1alpha1.PromoteSpec{FileRule: &v1alpha1.FileRule{Path: "config/values.yaml"}},
+			appName:  "myapp",
+			expected: "/config/values.yaml",
+		},
+		{
+			name:     "file rule with a leading slash is normalised",
+			spec:     v1alpha1.PromoteSpec{FileRule: &v1alpha1.FileRule{Path: "/Makefile"}},
+			appName:  "myapp",
+			expected: "/Makefile",
+		},
+		{
+			name:     "file rule without a path cannot be derived",
+			spec:     v1alpha1.PromoteSpec{FileRule: &v1alpha1.FileRule{}},
+			appName:  "myapp",
+			expected: "",
+		},
+		{
+			name:     "kpt rule covers the namespace/app subtree",
+			spec:     v1alpha1.PromoteSpec{KptRule: &v1alpha1.KptRule{Path: "namespaces/jx-staging"}},
+			appName:  "myapp",
+			expected: "/namespaces/jx-staging/myapp/",
+		},
+		{
+			name:     "kpt rule with an empty path falls back to the app dir at root",
+			spec:     v1alpha1.PromoteSpec{KptRule: &v1alpha1.KptRule{}},
+			appName:  "myapp",
+			expected: "/myapp/",
+		},
+		{
+			name:     "kpt rule with neither path nor app cannot be derived",
+			spec:     v1alpha1.PromoteSpec{KptRule: &v1alpha1.KptRule{}},
+			appName:  "",
+			expected: "",
+		},
+		{
+			name:     "helm/helmfile rules need no extra pattern",
+			spec:     v1alpha1.PromoteSpec{HelmfileRule: &v1alpha1.HelmfileRule{Path: "helmfile.yaml"}},
+			appName:  "myapp",
+			expected: "",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.expected, sparsePatternForRule(tc.spec, tc.appName), tc.name)
+		})
+	}
+}
+
+func TestIsSparseCheckout(t *testing.T) {
+	dir := t.TempDir()
+	gitter := cli.NewCLIClient("", cmdrunner.QuietCommandRunner)
+
+	_, err := gitter.Command(dir, "init")
+	require.NoError(t, err, "git init")
+
+	// a plain clone/init is not in sparse-checkout mode (mirrors Create()'s full-clone fallback)
+	assert.False(t, isSparseCheckout(gitter, dir), "fresh repo should not be sparse")
+
+	// enabling sparse checkout is exactly what SparseCloneToDir does via `sparse-checkout set`
+	_, err = gitter.Command(dir, "sparse-checkout", "set", "--no-cone", "/x")
+	require.NoError(t, err, "git sparse-checkout set")
+	assert.True(t, isSparseCheckout(gitter, dir), "repo should be sparse after sparse-checkout set")
+}

pkg/promote/promote.go

@@ -197,6 +197,8 @@ func (o *Options) AddOptions(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&o.PullRequestPollTime, optionPullRequestPollTime, "", "20s", "Poll time when waiting for a Pull Request to merge")
 	cmd.Flags().StringVarP(&o.DevEnvContext.GitUsername, "git-user", "", "", "Git username used to clone the development environment. If not specified its loaded from the git credentials file")
 	cmd.Flags().StringVarP(&o.DevEnvContext.GitToken, "git-token", "", "", "Git token used to clone the development environment. If not specified its loaded from the git credentials file")
+	cmd.Flags().BoolVarP(&o.SparseCheckout, "sparse-checkout", "", false, "Enables sparse git checkout of the environment and dev (version stream) repositories to reduce the clone footprint. Uses a default pattern set (helmfile.yaml, helmfiles/, .jx/, env/) when no --sparse-checkout-pattern is given. NOTE: file/kpt promote rules touch arbitrary paths and need explicit --sparse-checkout-pattern values")
+	cmd.Flags().StringArrayVarP(&o.SparseCheckoutPatterns, "sparse-checkout-pattern", "", nil, "Explicit git sparse-checkout patterns (gitignore syntax). Implies --sparse-checkout and overrides the default pattern set")
 
 	cmd.Flags().BoolVarP(&o.NoHelmUpdate, "no-helm-update", "", false, "Allows the 'helm repo update' command if you are sure your local helm cache is up to date with the version you wish to promote")
 	cmd.Flags().BoolVarP(&o.NoMergePullRequest, "no-merge", "", false, "Disables automatic merge of promote Pull Requests")
@@ -384,6 +386,9 @@ func (o *Options) Run() error {
 		o.GitClient = cli.NewCLIClient("", o.CommandRunner)
 	}
 
+	// file/kpt promote rules touch arbitrary paths so default sparse patterns are insufficient
+	o.DevEnvContext.SparseCheckout = o.SparseCheckout || len(o.SparseCheckoutPatterns) > 0
+
 	err = o.DevEnvContext.LazyLoad(o.GitClient, o.JXClient, o.Namespace, o.Git(), o.Dir)
 	if err != nil {
 		return fmt.Errorf("failed to lazy load the EnvironmentContext: %w", err)