Create issues on failures of integration tests

Author: nfx

.github/workflows/build.sh

@@ -3,8 +3,8 @@
 app=$1
 
 # Define platforms and architectures
-platforms=("windows" "linux" "darwin")
-architectures=("amd64" "arm64")
+platforms=("linux")
+architectures=("amd64")
 mkdir -p dist
 
 # Build binaries for each platform and architecture combination

.github/workflows/nightly-go-libs.yml

@@ -0,0 +1,36 @@
+name: acceptance
+
+on:
+  workflow_dispatch:
+    # ...
+
+permissions:
+  id-token: write
+  contents: read
+  pull-requests: write
+  issues: write
+
+jobs:
+  acceptance:
+    environment: tool
+    runs-on: larger
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: 1.21
+
+      - name: Acceptance
+        uses: databrickslabs/sandbox/acceptance@acceptance/v0.2.0
+        with:
+          directory: go-libs
+          vault_uri: ${{ secrets.VAULT_URI }}
+          create_issues: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          ARM_CLIENT_ID: ${{ secrets.ARM_CLIENT_ID }}
+          ARM_TENANT_ID: ${{ secrets.ARM_TENANT_ID }}
+        

acceptance/action.yml

@@ -20,6 +20,10 @@ inputs:
     description: 'Maximum suite execution time. Defaults to 1h'
     required: false
     default: 1h
+  create_issues:
+    description: 'Create issues in the repository for failed tests'
+    required: false
+    default: false
 outputs:
   sample:
     description: 'Sample output'

acceptance/boilerplate/actions.go

@@ -92,6 +92,40 @@ func (a *boilerplate) RunURL(ctx context.Context) (string, error) {
 	return "", fmt.Errorf("id not found for current run: %s", a.context.Job)
 }
 
+func (a *boilerplate) CreateIssueIfNotOpen(ctx context.Context, newIssue github.NewIssue) error {
+	org, repo := a.context.Repo()
+	it := a.GitHub.ListRepositoryIssues(ctx, org, repo, &github.ListIssues{
+		State: "open",
+	})
+	created := map[string]bool{}
+	for it.HasNext(ctx) {
+		issue, err := it.Next(ctx)
+		if err != nil {
+			return fmt.Errorf("issue: %w", err)
+		}
+		created[issue.Title] = true
+	}
+	if created[newIssue.Title] {
+		return nil
+	}
+	body, err := a.taggedComment(ctx, newIssue.Body)
+	if err != nil {
+		return fmt.Errorf("tagged comment: %w", err)
+	}
+	// with the tagged comment, which has the workflow ref, we can link to the run
+	issue, err := a.GitHub.CreateIssue(ctx, org, repo, github.NewIssue{
+		Title:     newIssue.Title,
+		Assignees: newIssue.Assignees,
+		Labels:    newIssue.Labels,
+		Body:      body,
+	})
+	if err != nil {
+		return fmt.Errorf("new issue: %w", err)
+	}
+	logger.Infof(ctx, "Created issue: https://github.com/%s/%s/issues/%d", issue.Number)
+	return nil
+}
+
 func (a *boilerplate) tag() string {
 	// The ref path to the workflow. For example,
 	// octocat/hello-world/.github/workflows/my-workflow.yml@refs/heads/my_branch.

acceptance/ecosystem/report.go

@@ -25,6 +25,19 @@ func (tr TestResult) Duration() time.Duration {
 	return time.Duration(tr.Elapsed * float64(time.Second))
 }
 
+func (tr TestResult) Failed() bool {
+	return !tr.Pass && !tr.Skip
+}
+
+func (tr TestResult) Summary() string {
+	res := []string{}
+	res = append(res, "<details>")
+	res = append(res, fmt.Sprintf("<summary>%s</summary>", tr))
+	res = append(res, fmt.Sprintf("\n```\n%s\n```\n", tr.Output))
+	res = append(res, "</details>")
+	return strings.Join(res, "\n")
+}
+
 func (tr TestResult) String() string {
 	summary := ""
 	if !tr.Pass {
@@ -138,13 +151,10 @@ func (r TestReport) String() string {
 func (r TestReport) StepSummary() string {
 	res := []string{r.String()}
 	for _, v := range r {
-		if v.Pass || v.Skip {
+		if !v.Failed() {
 			continue
 		}
-		res = append(res, "<details>")
-		res = append(res, fmt.Sprintf("<summary>%s</summary>", v))
-		res = append(res, fmt.Sprintf("\n```\n%s\n```\n", v.Output))
-		res = append(res, "</details>")
+		res = append(res, v.Summary())
 	}
 	if r.Flaky() {
 		res = append(res, "\nFlaky tests:\n")

acceptance/main.go

@@ -5,13 +5,15 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/databrickslabs/sandbox/acceptance/boilerplate"
 	"github.com/databrickslabs/sandbox/acceptance/ecosystem"
 	"github.com/databrickslabs/sandbox/acceptance/notify"
 	"github.com/databrickslabs/sandbox/acceptance/testenv"
 	"github.com/databrickslabs/sandbox/go-libs/env"
+	"github.com/databrickslabs/sandbox/go-libs/github"
 	"github.com/sethvargo/go-githubactions"
 )
 
@@ -79,21 +81,43 @@ func run(ctx context.Context, opts ...githubactions.Option) error {
 		return fmt.Errorf("upload artifact: %w", err)
 	}
 	slackWebhook := b.Action.GetInput("slack_webhook")
-	if !report.Pass() && slackWebhook != "" {
+	createIssues := strings.ToLower(b.Action.GetInput("create_issues"))
+	needsSlack := slackWebhook != ""
+	needsIssues := createIssues == "true" || createIssues == "yes"
+	needsNotification := needsSlack || needsIssues
+	if !report.Pass() && needsNotification {
 		runUrl, err := b.RunURL(ctx)
 		if err != nil {
 			return fmt.Errorf("run url: %w", err)
 		}
-		err = notify.Notification{
+		alert := notify.Notification{
 			Project: project,
 			Report:  report,
 			Cloud:   loaded.Cloud(),
 			RunName: b.WorkflowRunName(),
 			WebHook: slackWebhook,
 			RunURL:  runUrl,
-		}.ToSlack()
-		if err != nil {
-			return fmt.Errorf("slack: %w", err)
+		}
+		if needsSlack {
+			err = alert.ToSlack()
+			if err != nil {
+				return fmt.Errorf("slack: %w", err)
+			}
+		}
+		if needsIssues {
+			for _, v := range report {
+				if !v.Failed() {
+					continue
+				}
+				err = b.CreateIssueIfNotOpen(ctx, github.NewIssue{
+					Title: fmt.Sprintf("Test failure: `%s`", v.Name),
+					Body:  v.Summary(),
+					Labels: []string{"bug"},
+				})
+				if err != nil {
+					return fmt.Errorf("create issue: %w", err)
+				}
+			}
 		}
 	}
 	return report.Failed()

acceptance/shim.js

@@ -1,4 +1,4 @@
-const version = 'v0.1.4';
+const version = 'v0.2.0';
 
 const { createWriteStream, chmodSync } = require('fs');
 const { createGunzip } = require('zlib');

go-libs/github/github.go

@@ -292,6 +292,18 @@ func (c *GitHubClient) GetIssueComments(ctx context.Context, org, repo string, n
 	})
 }
 
+func (c *GitHubClient) CreateIssue(ctx context.Context, org, repo string, body NewIssue) (*Issue, error) {
+	path := fmt.Sprintf("%s/repos/%s/%s/issues", gitHubAPI, org, repo)
+	var res Issue
+	err := c.api.Do(ctx, "POST", path,
+		httpclient.WithRequestData(body),
+		httpclient.WithResponseUnmarshal(&res))
+	if err != nil {
+		return nil, err
+	}
+	return &res, nil
+}
+
 func (c *GitHubClient) CreateIssueComment(ctx context.Context, org, repo string, number int, body string) (*IssueComment, error) {
 	path := fmt.Sprintf("%s/repos/%s/%s/issues/%d/comments", gitHubAPI, org, repo, number)
 	var res IssueComment

go-libs/github/issues.go

@@ -15,6 +15,13 @@ type ListIssues struct {
 	PageOptions
 }
 
+type NewIssue struct {
+	Title     string   `json:"title,omitempty"`
+	Body      string   `json:"body,omitempty"`
+	Assignees []string `json:"assignees,omitempty"`
+	Labels    []string `json:"labels,omitempty"`
+}
+
 type Issue struct {
 	ID                int64      `json:"id,omitempty"`
 	Number            int        `json:"number,omitempty"`

go-libs/sqlexec/sqlexec_test.go

@@ -82,7 +82,7 @@ func TestAccQueryTypes(t *testing.T) {
 	require.NoError(t, result.Err())
 
 	require.Equal(t, []int{1, 2}, arrayType)
-	require.Equal(t, []byte("a"), binaryType)
+	require.Equal(t, []byte("a..."), binaryType)
 	require.Equal(t, true, booleanType)
 	require.Equal(t, time.Now().UTC().Truncate(24*time.Hour), dateType)
 	require.InDelta(t, 3, decimalType, 0.01)

go.work.sum

@@ -445,6 +445,7 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -615,6 +616,7 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240205150955-31a09d347014/go.
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20231127180814-3a041ad873d4/go.mod h1:o8b+u5ZiOSKuCwaZNjqXDJtJ0CmB9NtUPgCfO4rbakw=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:ZSvZ8l+AWJwXw91DoTjWjaVLpWU6o0eZ4YLYpH8aLeQ=
 google.golang.org/genproto/googleapis/bytestream v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:om8Bj876Z0v9ei+RD1LnEWig7vpHQ371PUqsgjmLQEA=
+google.golang.org/genproto/googleapis/bytestream v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:vh/N7795ftP0AkN1w8XKqN4w1OdUKXW5Eummda+ofv8=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc=