Skip to content

Commit bf39a7e

Browse files
authored
Merge pull request #3725 from buildkite/ming/a-970
A-970: Skip git fetch for already-present commits during checkout
2 parents c71221f + 2f17a00 commit bf39a7e

File tree

6 files changed

+113
-73
lines changed

6 files changed

+113
-73
lines changed

agent/agent_configuration.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ type AgentConfiguration struct {
2525
GitFetchFlags string
2626
GitSubmodules bool
2727
SkipCheckout bool
28+
GitSkipFetchExistingCommits bool
2829
AllowedRepositories []*regexp.Regexp
2930
AllowedPlugins []*regexp.Regexp
3031
AllowedEnvironmentVariables []*regexp.Regexp

agent/job_runner.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -568,6 +568,9 @@ BUILDKITE_AGENT_JWKS_KEY_ID`
568568
if r.conf.AgentConfiguration.SkipCheckout {
569569
setEnv("BUILDKITE_SKIP_CHECKOUT", "true")
570570
}
571+
if r.conf.AgentConfiguration.GitSkipFetchExistingCommits {
572+
setEnv("BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS", "true")
573+
}
571574
setEnv("BUILDKITE_COMMAND_EVAL", fmt.Sprint(r.conf.AgentConfiguration.CommandEval))
572575
setEnv("BUILDKITE_PLUGINS_ENABLED", fmt.Sprint(r.conf.AgentConfiguration.PluginsEnabled))
573576
// Allow BUILDKITE_PLUGINS_ALWAYS_CLONE_FRESH to be enabled either by config

clicommand/agent_start.go

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -141,16 +141,17 @@ type AgentStartConfig struct {
141141
WaitForECSMetaDataTimeout string `cli:"wait-for-ecs-meta-data-timeout"`
142142
WaitForGCPLabelsTimeout string `cli:"wait-for-gcp-labels-timeout"`
143143

144-
GitCheckoutFlags string `cli:"git-checkout-flags"`
145-
GitCloneFlags string `cli:"git-clone-flags"`
146-
GitCloneMirrorFlags string `cli:"git-clone-mirror-flags"`
147-
GitCleanFlags string `cli:"git-clean-flags"`
148-
GitFetchFlags string `cli:"git-fetch-flags"`
149-
GitMirrorsPath string `cli:"git-mirrors-path" normalize:"filepath"`
150-
GitMirrorsLockTimeout int `cli:"git-mirrors-lock-timeout"`
151-
GitMirrorsSkipUpdate bool `cli:"git-mirrors-skip-update"`
152-
NoGitSubmodules bool `cli:"no-git-submodules"`
153-
SkipCheckout bool `cli:"skip-checkout"`
144+
GitCheckoutFlags string `cli:"git-checkout-flags"`
145+
GitCloneFlags string `cli:"git-clone-flags"`
146+
GitCloneMirrorFlags string `cli:"git-clone-mirror-flags"`
147+
GitCleanFlags string `cli:"git-clean-flags"`
148+
GitFetchFlags string `cli:"git-fetch-flags"`
149+
GitMirrorsPath string `cli:"git-mirrors-path" normalize:"filepath"`
150+
GitMirrorsLockTimeout int `cli:"git-mirrors-lock-timeout"`
151+
GitMirrorsSkipUpdate bool `cli:"git-mirrors-skip-update"`
152+
NoGitSubmodules bool `cli:"no-git-submodules"`
153+
SkipCheckout bool `cli:"skip-checkout"`
154+
GitSkipFetchExistingCommits bool `cli:"git-skip-fetch-existing-commits"`
154155

155156
NoSSHKeyscan bool `cli:"no-ssh-keyscan"`
156157
NoCommandEval bool `cli:"no-command-eval"`
@@ -630,6 +631,11 @@ var AgentStartCommand = cli.Command{
630631
Usage: "Skip the git checkout phase entirely",
631632
EnvVar: "BUILDKITE_SKIP_CHECKOUT",
632633
},
634+
cli.BoolFlag{
635+
Name: "git-skip-fetch-existing-commits",
636+
Usage: "Skip git fetch if the commit already exists in the local git directory (default: false)",
637+
EnvVar: "BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS",
638+
},
633639
cli.BoolFlag{
634640
Name: "no-feature-reporting",
635641
Usage: "Disables sending a list of enabled features back to the Buildkite mothership. We use this information to measure feature usage, but if you're not comfortable sharing that information then that's totally okay :) (default: false)",
@@ -1047,6 +1053,7 @@ var AgentStartCommand = cli.Command{
10471053
GitFetchFlags: cfg.GitFetchFlags,
10481054
GitSubmodules: !cfg.NoGitSubmodules,
10491055
SkipCheckout: cfg.SkipCheckout,
1056+
GitSkipFetchExistingCommits: cfg.GitSkipFetchExistingCommits,
10501057
SSHKeyscan: !cfg.NoSSHKeyscan,
10511058
CommandEval: !cfg.NoCommandEval,
10521059
PluginsEnabled: !cfg.NoPlugins,

clicommand/bootstrap.go

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ type BootstrapConfig struct {
6969
ArtifactUploadDestination string `cli:"artifact-upload-destination"`
7070
CleanCheckout bool `cli:"clean-checkout"`
7171
SkipCheckout bool `cli:"skip-checkout"`
72+
GitSkipFetchExistingCommits bool `cli:"git-skip-fetch-existing-commits"`
7273
GitCheckoutFlags string `cli:"git-checkout-flags"`
7374
GitCloneFlags string `cli:"git-clone-flags"`
7475
GitFetchFlags string `cli:"git-fetch-flags"`
@@ -233,6 +234,11 @@ var BootstrapCommand = cli.Command{
233234
Usage: "Skip the git checkout phase entirely",
234235
EnvVar: "BUILDKITE_SKIP_CHECKOUT",
235236
},
237+
cli.BoolFlag{
238+
Name: "git-skip-fetch-existing-commits",
239+
Usage: "Skip git fetch if the commit already exists in the local git directory",
240+
EnvVar: "BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS",
241+
},
236242
cli.StringFlag{
237243
Name: "git-checkout-flags",
238244
Value: "-f",
@@ -475,6 +481,7 @@ var BootstrapCommand = cli.Command{
475481
SignalGracePeriod: signalGracePeriod,
476482
CleanCheckout: cfg.CleanCheckout,
477483
SkipCheckout: cfg.SkipCheckout,
484+
GitSkipFetchExistingCommits: cfg.GitSkipFetchExistingCommits,
478485
Command: cfg.Command,
479486
CommandEval: cfg.CommandEval,
480487
Commit: cfg.Commit,

internal/job/checkout.go

Lines changed: 82 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -587,69 +587,16 @@ func (e *Executor) getOrUpdateMirrorDir(ctx context.Context, repository string)
587587
return e.updateGitMirror(ctx, repository)
588588
}
589589

590-
// defaultCheckoutPhase is called by the CheckoutPhase if no global or plugin checkout
591-
// hook exists. It performs the default checkout on the Repository provided in the config
592-
func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
593-
span, _ := tracetools.StartSpanFromContext(ctx, "repo-checkout", e.TracingBackend)
594-
span.AddAttributes(map[string]string{
595-
"checkout.repo_name": e.Repository,
596-
"checkout.refspec": e.RefSpec,
597-
"checkout.commit": e.Commit,
598-
})
599-
var err error
600-
defer func() { span.FinishWithError(err) }()
601-
602-
if e.SSHKeyscan {
603-
addRepositoryHostToSSHKnownHosts(ctx, e.shell, e.Repository)
604-
}
605-
606-
var mirrorDir string
607-
608-
// If we can, get a mirror of the git repository to use for reference later
609-
if e.GitMirrorsPath != "" && e.Repository != "" {
610-
span.AddAttributes(map[string]string{"checkout.is_using_git_mirrors": "true"})
611-
mirrorDir, err = e.getOrUpdateMirrorDir(ctx, e.Repository)
612-
if err != nil {
613-
return fmt.Errorf("getting/updating git mirror: %w", err)
614-
}
615-
616-
e.shell.Env.Set("BUILDKITE_REPO_MIRROR", mirrorDir)
617-
}
618-
619-
// Make sure the build directory exists and that we change directory into it
620-
if err := e.createCheckoutDir(); err != nil {
621-
return fmt.Errorf("creating checkout dir: %w", err)
622-
}
623-
624-
gitCloneFlags := e.GitCloneFlags
625-
if mirrorDir != "" {
626-
gitCloneFlags += fmt.Sprintf(" --reference %q", mirrorDir)
627-
}
628-
629-
// Does the git directory exist?
630-
existingGitDir := filepath.Join(e.shell.Getwd(), ".git")
631-
if osutil.FileExists(existingGitDir) {
632-
// Update the origin of the repository so we can gracefully handle
633-
// repository renames
634-
if _, err := e.updateRemoteURL(ctx, "", e.Repository); err != nil {
635-
return fmt.Errorf("setting origin: %w", err)
636-
}
637-
} else {
638-
if err := gitClone(ctx, e.shell, gitCloneFlags, e.Repository, "."); err != nil {
639-
return fmt.Errorf("cloning git repository: %w", err)
640-
}
641-
}
642-
643-
// Git clean prior to checkout, we do this even if submodules have been
644-
// disabled to ensure previous submodules are cleaned up
645-
if hasGitSubmodules(e.shell) {
646-
if err := gitCleanSubmodules(ctx, e.shell, e.GitCleanFlags); err != nil {
647-
return fmt.Errorf("cleaning git submodules: %w", err)
648-
}
649-
}
650-
651-
if err := gitClean(ctx, e.shell, e.GitCleanFlags); err != nil {
652-
return fmt.Errorf("cleaning git repository: %w", err)
590+
// fetchSource fetches the git source for the job. If GitSkipFetchExistingCommits is
591+
// enabled and the commit already exists locally, the fetch is skipped entirely.
592+
func (e *Executor) fetchSource(ctx context.Context) error {
593+
// If configured, skip the fetch when the commit already exists locally.
594+
// This is useful when a pre-populated git mirror is used with --reference,
595+
// as the commit objects are already reachable and fetching is redundant.
596+
if e.GitSkipFetchExistingCommits && e.Commit != "HEAD" &&
597+
hasGitCommit(ctx, e.shell, ".git", e.Commit) {
598+
e.shell.Commentf("Commit %q already exists locally, skipping fetch", e.Commit)
599+
return nil
653600
}
654601

655602
gitFetchFlags := e.GitFetchFlags
@@ -738,6 +685,78 @@ func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
738685
}
739686
}
740687

688+
return nil
689+
}
690+
691+
// defaultCheckoutPhase is called by the CheckoutPhase if no global or plugin checkout
692+
// hook exists. It performs the default checkout on the Repository provided in the config
693+
func (e *Executor) defaultCheckoutPhase(ctx context.Context) error {
694+
span, _ := tracetools.StartSpanFromContext(ctx, "repo-checkout", e.TracingBackend)
695+
span.AddAttributes(map[string]string{
696+
"checkout.repo_name": e.Repository,
697+
"checkout.refspec": e.RefSpec,
698+
"checkout.commit": e.Commit,
699+
})
700+
var err error
701+
defer func() { span.FinishWithError(err) }()
702+
703+
if e.SSHKeyscan {
704+
addRepositoryHostToSSHKnownHosts(ctx, e.shell, e.Repository)
705+
}
706+
707+
var mirrorDir string
708+
709+
// If we can, get a mirror of the git repository to use for reference later
710+
if e.GitMirrorsPath != "" && e.Repository != "" {
711+
span.AddAttributes(map[string]string{"checkout.is_using_git_mirrors": "true"})
712+
mirrorDir, err = e.getOrUpdateMirrorDir(ctx, e.Repository)
713+
if err != nil {
714+
return fmt.Errorf("getting/updating git mirror: %w", err)
715+
}
716+
717+
e.shell.Env.Set("BUILDKITE_REPO_MIRROR", mirrorDir)
718+
}
719+
720+
// Make sure the build directory exists and that we change directory into it
721+
if err := e.createCheckoutDir(); err != nil {
722+
return fmt.Errorf("creating checkout dir: %w", err)
723+
}
724+
725+
gitCloneFlags := e.GitCloneFlags
726+
if mirrorDir != "" {
727+
gitCloneFlags += fmt.Sprintf(" --reference %q", mirrorDir)
728+
}
729+
730+
// Does the git directory exist?
731+
existingGitDir := filepath.Join(e.shell.Getwd(), ".git")
732+
if osutil.FileExists(existingGitDir) {
733+
// Update the origin of the repository so we can gracefully handle
734+
// repository renames
735+
if _, err := e.updateRemoteURL(ctx, "", e.Repository); err != nil {
736+
return fmt.Errorf("setting origin: %w", err)
737+
}
738+
} else {
739+
if err := gitClone(ctx, e.shell, gitCloneFlags, e.Repository, "."); err != nil {
740+
return fmt.Errorf("cloning git repository: %w", err)
741+
}
742+
}
743+
744+
// Git clean prior to checkout, we do this even if submodules have been
745+
// disabled to ensure previous submodules are cleaned up
746+
if hasGitSubmodules(e.shell) {
747+
if err := gitCleanSubmodules(ctx, e.shell, e.GitCleanFlags); err != nil {
748+
return fmt.Errorf("cleaning git submodules: %w", err)
749+
}
750+
}
751+
752+
if err := gitClean(ctx, e.shell, e.GitCleanFlags); err != nil {
753+
return fmt.Errorf("cleaning git repository: %w", err)
754+
}
755+
756+
if err := e.fetchSource(ctx); err != nil {
757+
return err
758+
}
759+
741760
gitCheckoutFlags := e.GitCheckoutFlags
742761

743762
if e.Commit == "HEAD" {

internal/job/config.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,9 @@ type ExecutorConfig struct {
7878
// Skip the checkout phase entirely
7979
SkipCheckout bool `env:"BUILDKITE_SKIP_CHECKOUT"`
8080

81+
// Skip git fetch if the commit already exists locally
82+
GitSkipFetchExistingCommits bool `env:"BUILDKITE_GIT_SKIP_FETCH_EXISTING_COMMITS"`
83+
8184
// Flags to pass to "git checkout" command
8285
GitCheckoutFlags string `env:"BUILDKITE_GIT_CHECKOUT_FLAGS"`
8386

0 commit comments

Comments
 (0)