internal/task: add GenerateAutoSubmitChange method to CloudBuildClient

I originally planned to use the existing CloudBuildClient.RunScript API
for this task, similarly to how it's used for monthly x/ repo tagging.
One key limitation of the RunScript API is that it requires the caller
to pass the exact files that will be copied out to Google Cloud Storage.
That works fine if go.mod and go.sum files were the only ones to change,
but 'go fix ./...' may update arbitrary other files too.

It's not too hard to expand RunScript to be able to copy directories
recursively, but that means it'd be copying the entire x/ repo content
even if only a few files changed. It's possible to try to detect which
files changed, and copy only those, or to create a .patch file.

However, we're already relying on the Cloud Build to clone the repo and
run commands to generate files, so it can work well for the Cloud Build
invocation to also take on the responsibility of mailing the auto-submit
CL, avoiding the need to copy anything to temporary storage only so that
relui can do the mailing itself. Also reuse git-generate as a building
block, pinning a known version for stability.

The result is an API that's very easy to use, and has the benefit of
including the script that describes the change in the commit message.

For golang/go#69095.

Change-Id: I78bb69567044e334fea693bf32a02de86696e7c9
Reviewed-on: https://go-review.googlesource.com/c/build/+/651336
Auto-Submit: Dmitri Shuralyov <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
Reviewed-by: Cherry Mui <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: dmitshur

internal/task/cloudbuild.go

@@ -5,22 +5,40 @@
 package task
 
 import (
+	"bytes"
 	"context"
+	cryptorand "crypto/rand"
 	"fmt"
 	"io/fs"
 	"math/rand"
+	"regexp"
 	"strings"
+	"time"
 
 	cloudbuild "cloud.google.com/go/cloudbuild/apiv1/v2"
 	"cloud.google.com/go/cloudbuild/apiv1/v2/cloudbuildpb"
 	"cloud.google.com/go/storage"
+	"golang.org/x/build/gerrit"
 	"golang.org/x/build/internal/gcsfs"
+	"golang.org/x/build/internal/secret"
+	wf "golang.org/x/build/internal/workflow"
 )
 
+const gitGenerateVersion = "v0.0.0-20240603191855-5c202b9c66be"
+
 type CloudBuildClient interface {
 	// RunBuildTrigger runs an existing trigger in project with the given
 	// substitutions.
 	RunBuildTrigger(ctx context.Context, project, trigger string, substitutions map[string]string) (CloudBuild, error)
+
+	// GenerateAutoSubmitChange generates a change with the given metadata and
+	// contents generated via the [git-generate] script that must be in the commit message,
+	// starts trybots with auto-submit enabled, and returns its change ID.
+	// If the requested contents match the state of the repository, no change
+	// is created and the returned change ID will be empty.
+	// Reviewers is the username part of a golang.org or google.com email address.
+	GenerateAutoSubmitChange(ctx *wf.TaskContext, input gerrit.ChangeInput, reviewers []string) (changeID string, _ error)
+
 	// RunScript runs the given script under bash -eux -o pipefail in
 	// ScriptProject. Outputs are collected into the build's ResultURL,
 	// readable with ResultFS. The script will have the latest version of Go
@@ -35,10 +53,12 @@ type CloudBuildClient interface {
 	// Prefer RunScript for simpler scenarios.
 	// Reference: https://cloud.google.com/build/docs/build-config-file-schema
 	RunCustomSteps(ctx context.Context, steps func(resultURL string) []*cloudbuildpb.BuildStep, opts *CloudBuildOptions) (CloudBuild, error)
+
 	// Completed reports whether a build has finished, returning an error if
 	// it's failed. It's suitable for use with AwaitCondition.
 	Completed(ctx context.Context, build CloudBuild) (detail string, completed bool, _ error)
 	// ResultFS returns an FS that contains the results of the given build.
+	// The build must've been created by RunScript or RunCustomSteps.
 	ResultFS(ctx context.Context, build CloudBuild) (fs.FS, error)
 }
 
@@ -139,6 +159,172 @@ func (c *RealCloudBuildClient) RunScript(ctx context.Context, script string, ger
 	return c.RunCustomSteps(ctx, steps, nil)
 }
 
+func (c *RealCloudBuildClient) GenerateAutoSubmitChange(ctx *wf.TaskContext, input gerrit.ChangeInput, reviewers []string) (changeID string, _ error) {
+	if input.Project == "" {
+		return "", fmt.Errorf("input.Project must be specified")
+	} else if input.Branch == "" {
+		return "", fmt.Errorf("input.Branch must be specified")
+	} else if !strings.Contains(input.Subject, "\n[git-generate]\n") {
+		return "", fmt.Errorf("a commit message with a [git-generate] script must be provided")
+	}
+
+	// Add a Change-Id trailer to the commit message if it's not already present.
+	var changeIDTrailers int
+	if strings.HasPrefix(input.Subject, "Change-Id: ") {
+		changeIDTrailers++
+	}
+	changeIDTrailers += strings.Count(input.Subject, "\nChange-Id: ")
+	if changeIDTrailers > 1 {
+		return "", fmt.Errorf("multiple Change-Id lines")
+	}
+	if changeIDTrailers == 0 {
+		// randomBytes returns 20 random bytes suitable for use in a Change-Id line.
+		randomBytes := func() []byte { var id [20]byte; cryptorand.Read(id[:]); return id[:] }
+
+		// endsWithMetadataLine reports whether the given commit message ends with a
+		// metadata line such as "Bug: #42" or "Signed-off-by: Al <[email protected]>".
+		metadataLineRE := regexp.MustCompile(`^[a-zA-Z0-9-]+: `)
+		endsWithMetadataLine := func(msg string) bool {
+			i := strings.LastIndexByte(msg, '\n')
+			return i >= 0 && metadataLineRE.MatchString(msg[i+1:])
+		}
+
+		msg := strings.TrimRight(input.Subject, "\n")
+		sep := "\n\n"
+		if endsWithMetadataLine(msg) {
+			sep = "\n"
+		}
+		input.Subject += fmt.Sprintf("%sChange-Id: I%x", sep, randomBytes())
+	}
+
+	refspec := fmt.Sprintf("HEAD:refs/for/%s%%l=Auto-Submit,l=Commit-Queue+1", input.Branch)
+	reviewerEmails, err := coordinatorEmails(reviewers)
+	if err != nil {
+		return "", err
+	}
+	for _, r := range reviewerEmails {
+		refspec += ",r=" + r
+	}
+
+	// Create a Cloud Build that will generate and mail the CL.
+	//
+	// To remove the possibility of mailing multiple CLs due to
+	// automated retries, allow only manual retries from this point.
+	ctx.DisableRetries()
+	op, err := c.BuildClient.CreateBuild(ctx, &cloudbuildpb.CreateBuildRequest{
+		ProjectId: c.ScriptProject,
+		Build: &cloudbuildpb.Build{
+			Steps: []*cloudbuildpb.BuildStep{
+				{
+					Name: "bash", Script: cloudBuildClientDownloadGoScript,
+				},
+				{
+					Name: "gcr.io/cloud-builders/git",
+					Args: []string{"clone", "--branch=" + input.Branch, "--depth=1", "--",
+						"https://go.googlesource.com/" + input.Project, "checkout"},
+				},
+				{
+					Name: "gcr.io/cloud-builders/git",
+					Args: []string{"-c", "user.name=Gopher Robot", "-c", "[email protected]",
+						"commit", "--allow-empty", "-m", input.Subject},
+					Dir: "checkout",
+				},
+				{
+					Name:       "gcr.io/cloud-builders/git",
+					Entrypoint: "/workspace/released_go/bin/go",
+					Args:       []string{"run", "rsc.io/rf/git-generate@" + gitGenerateVersion},
+					Dir:        "checkout",
+				},
+				{
+					Name: "gcr.io/cloud-builders/git",
+					Args: []string{"-c", "user.name=Gopher Robot", "-c", "[email protected]",
+						"commit", "--amend", "--no-edit"},
+					Dir: "checkout",
+				},
+				{
+					Name: "gcr.io/cloud-builders/git",
+					Args: []string{"show", "HEAD"},
+					Dir:  "checkout",
+				},
+				{
+					Name: "bash", Args: []string{"-c", `touch .gitcookies && chmod 0600 .gitcookies && printf ".googlesource.com\tTRUE\t/\tTRUE\t2147483647\to\tgit-gobot.golang.org=$$GOBOT_TOKEN\n" >> .gitcookies`},
+					SecretEnv: []string{"GOBOT_TOKEN"},
+				},
+				{
+					Name:       "gcr.io/cloud-builders/git",
+					Entrypoint: "bash",
+					Args:       []string{"-c", `git -c http.cookieFile=../.gitcookies push origin ` + refspec + ` 2>&1 | tee "$$BUILDER_OUTPUT/output"`},
+					Dir:        "checkout",
+				},
+				{
+					Name: "bash", Args: []string{"-c", "rm .gitcookies"},
+				},
+			},
+			Options: &cloudbuildpb.BuildOptions{
+				MachineType: cloudbuildpb.BuildOptions_E2_HIGHCPU_8,
+				Logging:     cloudbuildpb.BuildOptions_CLOUD_LOGGING_ONLY,
+			},
+			ServiceAccount: c.ScriptAccount,
+			AvailableSecrets: &cloudbuildpb.Secrets{
+				SecretManager: []*cloudbuildpb.SecretManagerSecret{
+					{
+						VersionName: "projects/" + c.ScriptProject + "/secrets/" + secret.NameGobotPassword + "/versions/latest",
+						Env:         "GOBOT_TOKEN",
+					},
+				},
+			},
+		},
+	})
+	if err != nil {
+		return "", fmt.Errorf("creating build: %w", err)
+	}
+	if _, err = op.Poll(ctx); err != nil {
+		return "", fmt.Errorf("polling: %w", err)
+	}
+	meta, err := op.Metadata()
+	if err != nil {
+		return "", fmt.Errorf("reading metadata: %w", err)
+	}
+	build := CloudBuild{Project: c.ScriptProject, ID: meta.Build.Id}
+
+	// Await the Cloud Build and extract the ID of the CL that was mailed.
+	ctx.Printf("Awaiting completion of build %q in %s.", build.ID, build.Project)
+	return AwaitCondition(ctx, 30*time.Second, func() (changeID string, completed bool, _ error) {
+		return c.completedGeneratingCL(ctx, build)
+	})
+}
+
+// completedGeneratingCL reports whether a build has finished,
+// returning the change ID that the given build generated.
+// The build must've been created by GenerateAutoSubmitChange.
+// It's suitable for use with AwaitCondition.
+func (c *RealCloudBuildClient) completedGeneratingCL(ctx context.Context, build CloudBuild) (changeID string, completed bool, _ error) {
+	b, err := c.BuildClient.GetBuild(ctx, &cloudbuildpb.GetBuildRequest{
+		ProjectId: build.Project,
+		Id:        build.ID,
+	})
+	if err != nil {
+		return "", false, err
+	}
+	if b.FinishTime == nil {
+		return "", false, nil
+	}
+	if b.Status != cloudbuildpb.Build_SUCCESS {
+		return "", false, fmt.Errorf("build %q failed, see %v: %v", build.ID, b.LogUrl, b.FailureInfo)
+	}
+
+	// Extract the CL number from the output using a simple regexp.
+	re := regexp.MustCompile(`https:\/\/go-review\.googlesource\.com\/c\/([a-zA-Z0-9_\-]+)\/\+\/(\d+)`)
+	gitPushOutput := bytes.Join(b.GetResults().GetBuildStepOutputs(), nil)
+	if matches := re.FindSubmatch(gitPushOutput); len(matches) == 3 {
+		changeID = fmt.Sprintf("%s~%s", matches[1], matches[2])
+	} else {
+		return "", false, fmt.Errorf("no match for successful mail of generated CL in git push output:\n%s", gitPushOutput)
+	}
+
+	return changeID, true, nil
+}
+
 func (c *RealCloudBuildClient) RunCustomSteps(ctx context.Context, steps func(resultURL string) []*cloudbuildpb.BuildStep, options *CloudBuildOptions) (CloudBuild, error) {
 	resultURL := fmt.Sprintf("%v/script-build-%v", c.ScratchURL, rand.Int63())
 	build := &cloudbuildpb.Build{

internal/task/fakes.go

@@ -350,6 +350,17 @@ func (g *FakeGerrit) CreateAutoSubmitChange(_ *wf.TaskContext, input gerrit.Chan
 	return changeID, nil
 }
 
+func (g *FakeGerrit) considerCommitSubmitted(repo *FakeRepo, commit string) (changeID string) {
+	if g.repos[repo.name] != repo {
+		repo.t.Fatalf("FakeGerrit.createSubmittedChange: provided repo %q isn't a part of this FakeGerrit instance", repo.name)
+	}
+	g.changesMu.Lock()
+	changeID = fmt.Sprintf("%s~%d", repo.name, len(g.changes)+1)
+	g.changes[changeID] = commit
+	g.changesMu.Unlock()
+	return changeID
+}
+
 func (g *FakeGerrit) Submitted(ctx context.Context, changeID, baseCommit string) (string, bool, error) {
 	g.changesMu.Lock()
 	commit, ok := g.changes[changeID]
@@ -924,6 +935,45 @@ func (cb *FakeCloudBuild) RunScript(ctx context.Context, script string, gerritPr
 	return CloudBuild{Project: cb.project, ID: id, ResultURL: "file://" + resultDir}, nil
 }
 
+func (cb *FakeCloudBuild) GenerateAutoSubmitChange(ctx *wf.TaskContext, input gerrit.ChangeInput, reviewers []string) (changeID string, _ error) {
+	if input.Project == "" {
+		return "", fmt.Errorf("input.Project must be specified")
+	} else if input.Branch == "" {
+		return "", fmt.Errorf("input.Branch must be specified")
+	} else if !strings.Contains(input.Subject, "\n[git-generate]\n") {
+		return "", fmt.Errorf("a commit message with a [git-generate] script must be provided")
+	}
+
+	r, err := cb.gerrit.repo(input.Project)
+	if err != nil {
+		return "", err
+	}
+	if input.Branch != "master" {
+		return "", fmt.Errorf("FakeCloudBuild.GenerateAutoSubmitChange: not implemented for branch %q", input.Branch)
+	}
+
+	// Create an empty commit with the git-generate script in commit message.
+	r.runGit("commit", "--allow-empty", "-m", input.Subject)
+
+	// Run git-generate.
+	cmd := exec.CommandContext(ctx, "go", "run", "rsc.io/rf/git-generate@"+gitGenerateVersion)
+	cmd.Dir = r.dir.dir
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("git-generate failed: %v output:\n%s", err, out)
+	}
+
+	// Update the empty commit with generated content.
+	r.runGit("commit", "--amend", "--no-edit")
+
+	if testing.Verbose() {
+		cb.t.Logf("FakeCloudBuild.GenerateAutoSubmitChange: generated commit content:\n%s", r.runGit("show", "HEAD"))
+	}
+
+	commit := strings.TrimSpace(string(r.runGit("rev-parse", "HEAD")))
+	return cb.gerrit.considerCommitSubmitted(r, commit), nil
+}
+
 func (cb *FakeCloudBuild) RunCustomSteps(ctx context.Context, steps func(resultURL string) []*cloudbuildpb.BuildStep, _ *CloudBuildOptions) (CloudBuild, error) {
 	var gerritProject, fullScript string
 	resultURL := "file://" + cb.t.TempDir()

internal/task/privx.go

@@ -195,10 +195,10 @@ func (x *PrivXPatch) NewDefinition(tagx *TagXReposTasks) *wf.Definition {
 			return nil, fmt.Errorf("git push failed: %v, stdout: %q stderr: %q", err, stdout.String(), stderr.String())
 		}
 
-		// Extract the CL number from the output using a quick and dirty regex.
-		re, err := regexp.Compile(fmt.Sprintf(`https:\/\/go-review.googlesource.com\/c\/%s\/\+\/(\d+)`, regexp.QuoteMeta(repoName)))
+		// Extract the CL number from the output using a simple regexp.
+		re, err := regexp.Compile(fmt.Sprintf(`https:\/\/go-review\.googlesource\.com\/c\/%s\/\+\/(\d+)`, regexp.QuoteMeta(repoName)))
 		if err != nil {
-			return nil, fmt.Errorf("failed to compile regex: %s", err)
+			return nil, fmt.Errorf("failed to compile regexp: %s", err)
 		}
 		matches := re.FindSubmatch(stderr.Bytes())
 		if len(matches) != 2 {

internal/workflow/workflow.go

@@ -534,6 +534,14 @@ func (c *TaskContext) Printf(format string, v ...interface{}) {
 	c.Logger.Printf(format, v...)
 }
 
+// DisableRetries disables automatic retries for the remaining
+// task execution. It's intended to be used when interacting with
+// external systems, where it's preferable to leave the decision
+// of whether to retry to the human release coordinator instead of
+// automatically retrying a number of times.
+//
+// Once the current task completes, the following task runs as usual
+// with automatic reties enabled once again.
 func (c *TaskContext) DisableRetries() {
 	c.disableRetries = true
 }