Allow larger logs from acceptance tests and implement experimental OIDC refresh (#261)

Author: nfx

acceptance/Makefile

@@ -6,7 +6,7 @@ fmt: lint
 	@echo "✓ Formatting source code with gofmt ..."
 	@gofmt -w $(shell find . -type f -name '*.go' -not -path "./vendor/*")
 
-lint: vendor
+lint:
 	@echo "✓ Linting source code with https://staticcheck.io/ ..."
 	@staticcheck ./...
 

acceptance/README.md

@@ -30,6 +30,32 @@ Executes tests, comments on PR, links to worflow run, uploads artifacts for late
 
 ![Alt text](docs/comments.png)
 
+```mermaid
+sequenceDiagram
+    acceptance->>+ACTIONS_ID_TOKEN_REQUEST_URL: (1) ACTIONS_ID_TOKEN_REQUEST_TOKEN
+    ACTIONS_ID_TOKEN_REQUEST_URL->>-acceptance: (2) JWT assertion
+    acceptance->>+Microsoft Entra: (3) JWT assertion + client ID + resource ID
+    Microsoft Entra->>-acceptance: (4) Access Token for Azure Key Vault
+    acceptance->>+Azure Key Vault: (5) request environment variables
+    Azure Key Vault->>-acceptance: (6) test environment
+    acceptance->>+Metadata Server: (7) start auth token proxy in a thread
+    Metadata Server->>-acceptance: (8) http://localhost:<random-port>/<random-prefix>
+    acceptance->>+Test Runner: (9) start test runner subprocess with relevant environment
+    Test Runner->>+test: (10) start test execution
+    test->>+SDK: (11) call API
+    SDK->>+Metadata Server: (12) call localhost:<random-port>/<random-prefix>
+    Metadata Server->>+ACTIONS_ID_TOKEN_REQUEST_URL: (13) ACTIONS_ID_TOKEN_REQUEST_TOKEN
+    ACTIONS_ID_TOKEN_REQUEST_URL->>-Metadata Server: (14) JWT assertion to request token for Databricks
+    Metadata Server->>+Microsoft Entra: (15) JWT assertion + client ID + resource ID
+    Microsoft Entra->>-Metadata Server: (16) Access token for Databricks
+    Metadata Server->>-SDK: (17) Access token for Databricks
+    SDK->>+Databricks API: (18) call API
+    Databricks API->>-SDK: (19) deserialized result
+    SDK->>-test: (20) deserialized result
+    test->>-Test Runner: (21) success or failure
+    Test Runner->>-acceptance: (22) success or failure + redacted logs
+```
+
 ## Usage
 
 Add to your `.github/workflows` folder:

acceptance/action.yml

@@ -17,9 +17,9 @@ inputs:
     description: 'Slack Webhook'
     required: false
   timeout:
-    description: 'Maximum suite execution time. Defaults to 1h'
+    description: 'Maximum suite execution time. Defaults to 2h'
     required: false
-    default: 1h
+    default: 2h
   create_issues:
     description: 'Create issues in the repository for failed tests'
     required: false

acceptance/ecosystem/python.go

@@ -21,10 +21,10 @@ import (
 
 	"github.com/databricks/databricks-sdk-go/logger"
 	"github.com/databrickslabs/sandbox/acceptance/redaction"
-	"github.com/databrickslabs/sandbox/go-libs/toolchain"
 	"github.com/databrickslabs/sandbox/go-libs/env"
 	"github.com/databrickslabs/sandbox/go-libs/fileset"
 	"github.com/databrickslabs/sandbox/go-libs/process"
+	"github.com/databrickslabs/sandbox/go-libs/toolchain"
 	"github.com/nxadm/tail"
 )
 

acceptance/ecosystem/report.go

@@ -29,11 +29,16 @@ func (tr TestResult) Failed() bool {
 	return !tr.Pass && !tr.Skip
 }
 
-func (tr TestResult) Summary() string {
+func (tr TestResult) Summary(cap int) string {
+	out, padding := tr.Output, 512
+	diff := len(out) + padding - cap
+	if diff > 0 {
+		out = fmt.Sprintf("... (skipped %d bytes)\n%s", diff, out[diff:])
+	}
 	res := []string{}
 	res = append(res, "<details>")
 	res = append(res, fmt.Sprintf("<summary>%s</summary>", tr))
-	res = append(res, fmt.Sprintf("\n```\n%s\n```\n", tr.Output))
+	res = append(res, fmt.Sprintf("\n```\n%s\n```\n", out))
 	res = append(res, "</details>")
 	return strings.Join(res, "\n")
 }
@@ -104,7 +109,7 @@ func (r TestReport) Failed() error {
 	if r.Pass() {
 		return nil
 	}
-	return fmt.Errorf(r.String())
+	return fmt.Errorf("failed: %s", r.String())
 }
 
 func (r TestReport) String() string {
@@ -148,13 +153,24 @@ func (r TestReport) String() string {
 	return fmt.Sprintf("%s %s", emoji, strings.Join(parts, ", "))
 }
 
+const CommentMaxSize = 65536
+
 func (r TestReport) StepSummary() string {
-	res := []string{r.String()}
+	res, failures, maybeOutput, padding := []string{r.String()}, []TestResult{}, 0, 1024
 	for _, v := range r {
 		if !v.Failed() {
 			continue
 		}
-		res = append(res, v.Summary())
+		failures = append(failures, v)
+		maybeOutput += len(v.Summary(CommentMaxSize))
+	}
+	summaryCap := CommentMaxSize - len(strings.Join(res, "\n")) - padding
+	if maybeOutput > (CommentMaxSize - padding) {
+		// if the output is too large, truncate the summaries up to a fraction of the total size
+		summaryCap /= len(failures)
+	}
+	for _, v := range failures {
+		res = append(res, v.Summary(summaryCap))
 	}
 	if r.Flaky() {
 		res = append(res, "\nFlaky tests:\n")

acceptance/go.mod

@@ -8,11 +8,11 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 // MIT
 	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.3.0 // MIT
 	github.com/databricks/databricks-sdk-go v0.40.0 // Apache 2.0
-	github.com/databrickslabs/sandbox/go-libs v0.1.0 // Databricks License
+	github.com/databrickslabs/sandbox/go-libs v0.4.0 // Databricks License
 	github.com/nxadm/tail v1.4.11 // MIT
 	github.com/sethvargo/go-githubactions v1.2.0 // Apache 2.0
-	github.com/stretchr/testify v1.8.4 // MIT
-	golang.org/x/oauth2 v0.17.0 // BSD
+	github.com/stretchr/testify v1.9.0 // MIT
+	golang.org/x/oauth2 v0.19.0 // BSD
 )
 
 require (
@@ -38,23 +38,21 @@ require (
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.48.0 // indirect
-	go.opentelemetry.io/otel v1.23.1 // indirect
-	go.opentelemetry.io/otel/metric v1.23.1 // indirect
-	go.opentelemetry.io/otel/trace v1.23.1 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect
-	golang.org/x/mod v0.15.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
+	golang.org/x/mod v0.16.0 // indirect
+	golang.org/x/net v0.24.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/api v0.166.0 // indirect
-	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c // indirect
+	google.golang.org/api v0.169.0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78 // indirect
 	google.golang.org/grpc v1.62.0 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect