cmd/watchflakes: detect and report broken bots

This change automates issue creation for bots requiring human
intervention. When a bot enters a failed state needing manual
resolution, it checks for an existing GitHub issue. If none exists,
it creates a new issue in the 'Broken Bot' project and tags the bot's
port owners.

Fixes golang/go#68790

Change-Id: I3a64d3a06f80e948fc895a2b15d89ea8d6f37a03
Reviewed-on: https://go-review.googlesource.com/c/build/+/646838
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Auto-Submit: Carlos Amedee <carlos@golang.org>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>

Author: cagedmantis

cmd/watchflakes/github.go

@@ -40,6 +40,7 @@ var (
 	repo       *github.Repo
 	labels     map[string]*github.Label
 	testFlakes *github.Project
+	brokenBots *github.Project
 )
 
 // readIssues reads the GitHub issues in the Test Flakes project.
@@ -109,6 +110,71 @@ func readIssues(old []*Issue) ([]*Issue, error) {
 	return issues, nil
 }
 
+// readBuilderIssues reads the GitHub issues in the Broken Bots project.
+// It also sets up the repo, labels, and testFlakes variables for
+// use by other functions below.
+func readBuilderIssues() ([]*Issue, error) {
+	// Find repo.
+	r, err := gh.Repo("golang", "go")
+	if err != nil {
+		return nil, err
+	}
+	repo = r
+
+	var builderLabel *github.Label
+
+	// Find labels.
+	list, err := gh.SearchLabels("golang", "go", "")
+	if err != nil {
+		return nil, err
+	}
+	for _, label := range list {
+		if label.Name == "Builders" {
+			builderLabel = label
+			break
+		}
+	}
+	if builderLabel == nil {
+		return nil, fmt.Errorf("cannot find builder label")
+	}
+
+	labels = make(map[string]*github.Label)
+	for _, label := range list {
+		labels[label.Name] = label
+	}
+
+	// Find Test Flakes project.
+	ps, err := gh.Projects("golang", "")
+	if err != nil {
+		return nil, err
+	}
+	for _, p := range ps {
+		if p.Title == "Broken Bots" {
+			brokenBots = p
+			break
+		}
+	}
+	if brokenBots == nil {
+		return nil, fmt.Errorf("cannot find Broken Bots project")
+	}
+
+	// Read all issues in Test Flakes.
+	var issues []*Issue
+	items, err := gh.ProjectItems(brokenBots)
+	if err != nil {
+		return nil, err
+	}
+	for _, item := range items {
+		if item.Issue != nil {
+			issues = append(issues, &Issue{Issue: item.Issue, NewBody: true, Stale: true})
+		}
+	}
+	sort.Slice(issues, func(i, j int) bool {
+		return issues[i].Number < issues[j].Number
+	})
+	return issues, nil
+}
+
 // findScripts finds the scripts in the issues,
 // initializing issue.Script and .ScriptText or else .Error
 // in each issue.
@@ -329,7 +395,7 @@ func readComments(issue *Issue) {
 }
 
 // postNew creates a new issue with the given title and body,
-// setting the NeedsInvestigation label and placing the issue int
+// setting the NeedsInvestigation label and placing the issue in
 // the Test Flakes project.
 // It automatically adds signature to the body.
 func postNew(title, body string) *github.Issue {
@@ -346,6 +412,24 @@ func postNew(title, body string) *github.Issue {
 	return issue
 }
 
+// postNewBrokenBot creates a new issue with the given title and body,
+// setting the NeedsInvestigation label and placing the issue in
+// the Broken Bots project.
+// It automatically adds signature to the body.
+func postNewBrokenBot(title, body string) (*github.Issue, error) {
+	var args []any
+	if lab := labels["NeedsInvestigation"]; lab != nil {
+		args = append(args, lab)
+	}
+	if lab := labels["Builders"]; lab != nil {
+		args = append(args, lab)
+	}
+
+	args = append(args, brokenBots)
+	issue, err := gh.CreateIssue(repo, title, body+signature, args...)
+	return issue, err
+}
+
 // postComment posts a new comment on the issue.
 // It automatically adds signature to the comment.
 func postComment(issue *Issue, body string) error {

cmd/watchflakes/luci.go

@@ -23,6 +23,7 @@ import (
 	"go.chromium.org/luci/grpc/prpc"
 	rdbpb "go.chromium.org/luci/resultdb/proto/v1"
 	spb "go.chromium.org/luci/swarming/proto/api_v2"
+	goluci "golang.org/x/build/internal/luci"
 	"golang.org/x/sync/errgroup"
 	"google.golang.org/protobuf/types/known/fieldmaskpb"
 	"google.golang.org/protobuf/types/known/timestamppb"
@@ -147,6 +148,8 @@ type Failure struct {
 
 type Bot struct {
 	ID          string
+	Goos        string
+	Goarch      string
 	Dead        bool
 	Quarantined bool
 }
@@ -686,7 +689,8 @@ func (c *LUCIClient) fetchLogsForBuild(r *BuildResult) {
 // from the list of returned bots.
 type filter func(bot *spb.BotInfo) bool
 
-// filterOutDarwin filters out darwin machines which no longer exist.
+// filterOutDarwin filters out darwin bots which no longer exist but are still
+// listed as valid bots.
 func filterOutDarwin(bot *spb.BotInfo) bool {
 	return strings.HasPrefix(bot.BotId, "darwin-")
 }
@@ -724,7 +728,12 @@ nextCursor:
 			continue
 		}
 		if bot.IsDead || bot.Quarantined {
-			brokenBots = append(brokenBots, Bot{ID: bot.BotId, Dead: bot.IsDead, Quarantined: bot.Quarantined})
+			goos, goarch, err := platform(bot)
+			if err != nil {
+				fmt.Printf("failed to determine platform for %s: %s\n", bot.GetBotId(), err)
+				continue
+			}
+			brokenBots = append(brokenBots, Bot{ID: bot.BotId, Dead: bot.IsDead, Quarantined: bot.Quarantined, Goos: goos, Goarch: goarch})
 		}
 	}
 	if resp.GetCursor() != "" {
@@ -734,6 +743,40 @@ nextCursor:
 	return brokenBots, nil
 }
 
+// platform determines the platform that the bot is running.
+func platform(bot *spb.BotInfo) (string, string, error) {
+	var goos, goarch, targetGoos, targetGoarch string
+	var err error
+
+	for _, d := range bot.GetDimensions() {
+		key := d.GetKey()
+		switch key {
+		case "cipd_platform":
+			val := d.GetValue()
+			if len(val) == 0 {
+				return "", "", fmt.Errorf("invalid cipd_platform value: %+v", val)
+			}
+			goos, goarch, err = goluci.PlatformToGoValues(val[0])
+			if err != nil {
+				return "", "", fmt.Errorf("unable to parse cipd_platform value %q: %w", val[0], err)
+			}
+		case "target_goarch":
+			if val := d.GetValue(); len(val) == 1 {
+				targetGoarch = val[0]
+			}
+		case "target_goos":
+			if val := d.GetValue(); len(val) == 1 {
+				targetGoos = val[0]
+			}
+		default:
+		}
+	}
+	if targetGoarch != "" && targetGoos != "" {
+		goos, goarch = targetGoos, targetGoarch
+	}
+	return goos, goarch, nil
+}
+
 func fetchURL(url string) string {
 	resp, err := http.Get(url)
 	if err != nil {

cmd/watchflakes/main.go

@@ -7,11 +7,16 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"flag"
 	"fmt"
+	"io"
 	"log"
+	"net/http"
 	"os"
+	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
@@ -21,6 +26,7 @@ import (
 	rdbpb "go.chromium.org/luci/resultdb/proto/v1"
 	"golang.org/x/build/buildenv"
 	"golang.org/x/build/cmd/watchflakes/internal/script"
+	"golang.org/x/build/devapp/owners"
 	"golang.org/x/build/internal/secret"
 	"rsc.io/github"
 )
@@ -106,6 +112,7 @@ func main() {
 Repeat:
 	startTime := time.Now()
 	ctx, cancel := context.WithTimeout(context.Background(), timeout)
+	reportBrokenBots(ctx, c)
 	var boards []*Dashboard
 	if *build == "" {
 		// fetch the dashboard
@@ -302,6 +309,132 @@ Repeat:
 	}
 }
 
+func reportBrokenBots(ctx context.Context, c *LUCIClient) {
+	// query for broken bots
+	brokenBots, err := c.ListBrokenBots(ctx, filterOutDarwin)
+	if err != nil {
+		log.Printf("failed to query for bots: %s", err)
+		return
+	}
+	// query for existing broken bot issues
+	existingIssues, err := readBuilderIssues()
+	if err != nil {
+		log.Printf("failed querying for existing builder issues: %s", err)
+		return
+	}
+	// map used as set to check for existing issues for a bot ID.
+	// botID -> issue number
+	botIssues := make(map[string]int)
+	for _, issue := range existingIssues {
+		if botID, ok := botIDFromIssueBody(issue.Body); ok {
+			fmt.Printf("found existing issue: %+v for %s\n", issue, botID)
+			botIssues[botID] = issue.Number
+		}
+	}
+	po, err := getPlatformOwners()
+	if err != nil {
+		log.Printf("failed to query for platform owners: %s", err)
+	}
+	// for each broken bot, is there an existing open issue?
+	for _, bot := range brokenBots {
+		if issueID, ok := botIssues[bot.ID]; ok {
+			fmt.Printf("issue #%d found for broken bot %s\n", issueID, bot.ID)
+			continue
+		}
+		title := brokenBotIssueTitle(bot.ID)
+		var botOwners []string
+		if v, ok := po[bot.Goos]; ok {
+			for _, bo := range v {
+				botOwners = append(botOwners, "@"+bo)
+			}
+		}
+		if v, ok := po[bot.Goarch]; ok {
+			for _, bo := range v {
+				botOwners = append(botOwners, "@"+bo)
+			}
+		}
+		if !*post {
+			fmt.Printf("dry-run: skipped posting a new broken bot issue for %s\n", bot.ID)
+			continue
+		}
+		i, err := postNewBrokenBot(title, brokenBotIssueBody(bot, botOwners))
+		if err != nil {
+			log.Printf("failed to post broken bot issue: %s", err)
+			continue
+		}
+		fmt.Printf("Posted new broken bot issue for %s, issue: %s\n", bot.ID, i.ID)
+	}
+}
+
+func getPlatformOwners() (map[string][]string, error) {
+	url := "https://dev.golang.org/owners"
+	var o owners.Request
+	o.Payload.Platform = true
+
+	body, err := json.Marshal(o)
+	if err != nil {
+		return nil, fmt.Errorf("unable to marshal json: %s", err)
+	}
+	r, err := http.Post(url, "application/json", bytes.NewBuffer(body))
+	if err != nil {
+		return nil, fmt.Errorf("failed to query for platform owners: %s", err)
+	}
+	defer r.Body.Close()
+	if r.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("failed to query for platform owners, response code=%d", r.StatusCode)
+	}
+	b, err := io.ReadAll(r.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read http response body: %w", err)
+	}
+	var response owners.Response
+	err = json.Unmarshal(b, &response)
+	if err != nil {
+		return nil, fmt.Errorf("unable to unmarshal owners response: %s", err)
+	}
+	m := map[string][]string{}
+	for k, v := range response.Payload.Platforms {
+		if len(v.Primary) == 0 {
+			continue
+		}
+		var primaries []string
+		for _, p := range v.Primary {
+			primaries = append(primaries, p.GitHubUsername)
+		}
+		m[k] = primaries
+	}
+	return m, nil
+}
+
+var issueFooter = regexp.MustCompile(`<!-- DO NOT EDIT: (.*?) -->`)
+
+func botIDFromIssueBody(body string) (string, bool) {
+	matches := issueFooter.FindStringSubmatch(body)
+	if len(matches) != 2 {
+		return "", false
+	}
+	return matches[1], true
+}
+
+func brokenBotIssueTitle(botID string) string {
+	return fmt.Sprintf("x/build: bot %s reported as broken", botID)
+}
+
+func brokenBotIssueBody(bot Bot, owners []string) string {
+	var state string
+	if bot.Dead {
+		state = "dead"
+	} else if bot.Quarantined {
+		state = "quarantined"
+	} else {
+		state = "unknown"
+	}
+	botURL := fmt.Sprintf("https://chromium-swarm.appspot.com/bot?id=%s", bot.ID)
+	body := "The bot [%s](%s) has been reported as broken. It is currently in %q state. Please work to resolve the issue.\n\n%s\n%s"
+	footer := fmt.Sprintf("<!-- DO NOT EDIT: %s -->", bot.ID)
+	return fmt.Sprintf(body, bot.ID, botURL, state, strings.Join(owners, "\n"), footer)
+}
+
 const SKIP = bbpb.Status_STATUS_UNSPECIFIED // for smashing the status to skip a non-flake failure
 
 // skipBrokenCommits identifies broken commits,