VAULT-33074: add github sub-command to pipeline (#29403)

* VAULT-33074: add `github` sub-command to `pipeline`

Investigating test workflow failures is common task that engineers on the
sustaining rotation perform. This task often requires quite a bit of
manual labor by manually inspecting all failed/cancelled workflows in
the Github UI on per repo/branch/workflow basis and performing root cause
analysis.

As we work to improve our pipeline discoverability this PR adds a new `github`
sub-command to the `pipeline` utility that allows querying for such workflows
and returning either machine readable or human readable summaries in a single
place. Eventually we plan to automate sending a summary of this data to
an OTEL collector automatically but for now sustaining engineers can
utilize it to query for workflows with lots of various criteria.

A common pattern for investigating build/enos test failure workflows would be:
```shell
export GITHUB_TOKEN="YOUR_TOKEN"
go run -race ./tools/pipeline/... github list-workflow-runs -o hashicorp -r vault -d '2025-01-13..2025-01-23' --branch main --status failure build
```

This will list `build` workflow runs in `hashicorp/vault` repo for the
`main` branch with the `status` or `conclusion` of `failure` within the date
range of `2025-01-13..2025-01-23`.

A sustaining engineer will likely do this for both `vault` and
`vault-enterprise` repositories along with `enos-release-testing-oss` and
`enos-release-testing-ent` workflows in addition to `build` in order to
get a full picture of the last weeks failures.

You can also use this utility to summarize workflows based on other
statuses, branches, HEAD SHA's, event triggers, github actors, etc. For
a full list of filter arguments you can pass `-h` to the sub-command.

> [!CAUTION]
> Be careful not to run this without setting strict filter arguments.
> Failing to do so could result in trying to summarize way too many
> workflows resulting in your API token being disabled for an hour.

Signed-off-by: Ryan Cragun <[email protected]>

Author: ryancragun

.gitignore

@@ -133,4 +133,6 @@ website/components/node_modules
 tools/godoctests/.bin
 tools/gonilnilfunctions/.bin
 tools/codechecker/.bin
-.ci-bootstrap
+tools/pipeline/.bin
+tools/pipeline/pipeline
+.ci-bootstrap

tools/pipeline/go.mod

@@ -4,10 +4,11 @@ go 1.23.2
 
 require (
 	github.com/Masterminds/semver v1.5.0
-	github.com/hashicorp/hcl/v2 v2.22.0
+	github.com/google/go-github/v68 v68.0.0
+	github.com/hashicorp/hcl/v2 v2.23.0
 	github.com/hashicorp/releases-api v0.1.23
 	github.com/spf13/cobra v1.8.1
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
 	github.com/veqryn/slog-context v0.7.0
 )
 
@@ -17,29 +18,30 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/docker/go-units v0.5.0 // indirect
-	github.com/fatih/color v1.16.0 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/fatih/color v1.18.0 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/analysis v0.23.0 // indirect
 	github.com/go-openapi/errors v0.22.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/loads v0.22.0 // indirect
-	github.com/go-openapi/runtime v0.26.0 // indirect
+	github.com/go-openapi/runtime v0.28.0 // indirect
 	github.com/go-openapi/spec v0.21.0 // indirect
 	github.com/go-openapi/strfmt v0.23.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-openapi/validate v0.24.0 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/go-hclog v1.6.3 // indirect
 	github.com/hashicorp/go-version v1.7.0 // indirect
-	github.com/imdario/mergo v0.3.15 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jessevdk/go-flags v1.6.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mailru/easyjson v0.9.0 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/hashstructure v1.1.0 // indirect
@@ -49,17 +51,17 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/zclconf/go-cty v1.15.0 // indirect
-	go.mongodb.org/mongo-driver v1.14.0 // indirect
-	go.opentelemetry.io/otel v1.20.0 // indirect
-	go.opentelemetry.io/otel/metric v1.20.0 // indirect
-	go.opentelemetry.io/otel/trace v1.20.0 // indirect
-	golang.org/x/mod v0.21.0 // indirect
-	golang.org/x/net v0.30.0 // indirect
-	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.26.0 // indirect
-	golang.org/x/text v0.19.0 // indirect
-	golang.org/x/tools v0.26.0 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	github.com/zclconf/go-cty v1.16.2 // indirect
+	go.mongodb.org/mongo-driver v1.17.2 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/otel v1.34.0 // indirect
+	go.opentelemetry.io/otel/metric v1.34.0 // indirect
+	go.opentelemetry.io/otel/trace v1.34.0 // indirect
+	golang.org/x/mod v0.22.0 // indirect
+	golang.org/x/net v0.34.0 // indirect
+	golang.org/x/sync v0.10.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/tools v0.29.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

tools/pipeline/go.sum

@@ -0,0 +1,26 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package cmd
+
+import "github.com/spf13/cobra"
+
+type githubCommandFlags struct {
+	Format string `json:"format,omitempty"`
+}
+
+var githubCmdFlags = &githubCommandFlags{}
+
+func newGithubCmd() *cobra.Command {
+	github := &cobra.Command{
+		Use:   "github",
+		Short: "Github commands",
+		Long:  "Github commands",
+	}
+
+	github.PersistentFlags().StringVarP(&githubCmdFlags.Format, "format", "f", "table", "The output format. Can be 'json' or 'table'")
+
+	github.AddCommand(newGithubListRunCmd())
+
+	return github
+}

tools/pipeline/internal/cmd/github.go

@@ -0,0 +1,89 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	ghclient "github.com/google/go-github/v68/github"
+	"github.com/spf13/cobra"
+
+	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/github"
+)
+
+var listGithubWorkflowRuns = &github.ListWorkflowRunsReq{}
+
+func newGithubListRunCmd() *cobra.Command {
+	listRuns := &cobra.Command{
+		Use:   "list-workflow-runs [WORKFLOW_NAME]",
+		Short: "List workflow runs",
+		Long:  "List Github Actions workflow runs for a given workflow. Be sure to use filter arguments to reduce the search, otherwise you'll likely hit your API limit.",
+		RunE:  runListGithubWorkflowsCmd,
+		Args: func(cmd *cobra.Command, args []string) error {
+			switch len(args) {
+			case 1:
+				listGithubWorkflowRuns.WorkflowName = args[0]
+				return nil
+			case 0:
+				return errors.New("no workflow name argument has been provided")
+			default:
+				return fmt.Errorf("expected a single workflow name as an argument, received (%d): %v", len(args), args)
+			}
+		},
+	}
+
+	listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.Actor, "actor", "a", "", "Filter using a specific Github actor")
+	listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.Branch, "branch", "b", "", "Filter using a specific Github branch")
+	listRuns.PersistentFlags().Int64VarP(&listGithubWorkflowRuns.CheckSuiteID, "check-suite-id", "c", 0, "Filter using a specific Github check suite")
+	listRuns.PersistentFlags().BoolVar(&listGithubWorkflowRuns.Compact, "compact", true, "When given a status filter, only fetch data for workflows, jobs, checks, and annotations that match our status and/or conclusion. Disabling compact mode with a large query range might result in Github throttling the requests.")
+	listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.DateQuery, "date-query", "d", fmt.Sprintf("%s..*", time.Now().Add(-168*time.Hour).Format(time.DateOnly)), "Filter using a date range query. It supports the Github ISO8601-ish date range query format. Default is newer than one week ago")
+	listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.Event, "event", "e", "", "Filter using a workflow triggered by an event type. E.g. push, pull_request, issue")
+	listRuns.PersistentFlags().BoolVarP(&listGithubWorkflowRuns.IncludePRs, "include-prs", "p", false, "Include workflow runs triggered via pull requests")
+	listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.Owner, "owner", "o", "hashicorp", "The Github organization")
+	listRuns.PersistentFlags().StringVarP(&listGithubWorkflowRuns.Repo, "repo", "r", "vault", "The Github repository. Private repositories require auth via a GITHUB_TOKEN env var")
+	listRuns.PersistentFlags().StringVar(&listGithubWorkflowRuns.Sha, "sha", "", "Filter based on the HEAD SHA associated with the workflow run")
+	listRuns.PersistentFlags().StringVar(&listGithubWorkflowRuns.Status, "status", "", "Filter by a given run status. For example: completed, cancelled, failure, skipped, success, in_progress")
+
+	return listRuns
+}
+
+func runListGithubWorkflowsCmd(cmd *cobra.Command, args []string) error {
+	cmd.SilenceUsage = true // Don't spam the usage on failure
+
+	client := ghclient.NewClient(nil)
+	if token, set := os.LookupEnv("GITHUB_TOKEN"); set {
+		client = client.WithAuthToken(token)
+	} else {
+		fmt.Println("\x1b[1;33;49mWARNING\x1b[0m: GITHUB_TOKEN has not been set. While not required for public repositories you're likely to get throttled without it")
+	}
+
+	res, err := listGithubWorkflowRuns.Run(context.TODO(), client)
+	if err != nil {
+		return fmt.Errorf("listing github workflow failures: %w", err)
+	}
+
+	switch githubCmdFlags.Format {
+	case "json":
+		b, err := json.Marshal(res)
+		if err != nil {
+			return fmt.Errorf("marshaling response to JSON: %w", err)
+		}
+		fmt.Println(string(b))
+	default:
+		for _, run := range res.Runs {
+			summary, err := run.Summary()
+			if err != nil {
+				return fmt.Errorf("generating workflow run response summary: %w", err)
+			}
+			fmt.Println(summary)
+		}
+	}
+
+	return err
+}

tools/pipeline/internal/cmd/github_list_runs.go

@@ -28,6 +28,7 @@ func newRootCmd() *cobra.Command {
 	rootCmd.PersistentFlags().StringVar(&rootCfg.logLevel, "log", "warn", "Set the log level. One of 'debug', 'info', 'warn', 'error'")
 
 	rootCmd.AddCommand(newGenerateCmd())
+	rootCmd.AddCommand(newGithubCmd())
 	rootCmd.AddCommand(newReleasesCmd())
 
 	rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error {

tools/pipeline/internal/cmd/root.go

@@ -10,8 +10,9 @@ import (
 	"slices"
 	"testing"
 
-	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases"
 	"github.com/stretchr/testify/require"
+
+	"github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases"
 )
 
 var testAPIVersions = []string{
@@ -230,21 +231,29 @@ func Test_EnosDynamicConfigReq_Run(t *testing.T) {
 							AWSRegion:             []string{"us-east-1", "us-west-2"},
 							DistroVersionAmzn:     []string{"2023"},
 							DistroVersionLeap:     []string{"15.6"},
-							DistroVersionRhel:     []string{"8.10, 9.4"},
+							DistroVersionRhel:     []string{"8.10", "9.4"},
 							DistroVersionSles:     []string{"15.6"},
 							DistroVersionUbuntu:   []string{"20.04", "24.04"},
 							UpgradeInitialVersion: versions,
 						},
 					},
 				}
 			},
-			hcl: []byte(`
+			hcl: []byte(`# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+# Code generated by pipeline generate enos-dynamic-config DO NOT EDIT.
+
+# This file is overwritten in CI as it contains branch specific and sometimes ever-changing values.
+# It's checked in here so that enos samples and scenarios can be performed, just be aware that this
+# might change out from under you.
+
 globals {
   sample_attributes = {
     aws_region              = ["us-east-1", "us-west-2"]
     distro_version_amzn     = ["2023"]
     distro_version_leap     = ["15.6"]
-    distro_version_rhel     = ["8.10, 9.4"]
+    distro_version_rhel     = ["8.10", "9.4"]
     distro_version_sles     = ["15.6"]
     distro_version_ubuntu   = ["20.04", "24.04"]
     upgrade_initial_version = ["1.16.6", "1.16.7", "1.16.8", "1.16.9", "1.16.10", "1.17.3", "1.17.4", "1.17.6", "1.18.0-rc1"]