ci: add github-actions-extended-ci workflow and maybe_stress job

This is a replacement for the TeamCity job `maybe_stress` (part of Bazel
Extended CI) that runs on EngFlow instead of being `stress`'ed inside of
the local TeamCity agent. We run each affected test 10 times and limit
arbitrarily to 10 tests per package and 10 different packages (for now,
we may decide to change this later).

The code for `ci-stress` is adapted from `github-pull-request-make`, but
rewritten substantially to make the code simpler. We don't simply use
the existing code as a helper function so we can modify the new code
without changing the existing code.

Tests are stressed on EngFlow at a lower priority so as to not
drastically impact the main CI jobs which must succeed to merge. Having
said that, we still must monitor CI performance over the next few weeks
to determine if we are safe to proceed. We also hope to add a
`maybe_stressrace` build.

Part of: DEVINF-1640
Epic: DEVINF-1582
Release note: none

Author: rickystewart

.github/workflows/github-actions-extended-ci.yml

@@ -0,0 +1,54 @@
+name: GitHub Actions Extended CI
+on:
+  pull_request:
+    types: [opened, reopened, synchronize]
+    branches:
+      - "master"
+      - "release-*"
+      - "staging-*"
+      - "!release-1.0*"
+      - "!release-1.1*"
+      - "!release-2.0*"
+      - "!release-2.1*"
+      - "!release-19.1*"
+      - "!release-19.2*"
+      - "!release-20.1*"
+      - "!release-20.2*"
+      - "!release-21.1*"
+      - "!release-21.2*"
+      - "!release-22.1*"
+      - "!release-22.2*"
+      - "!release-23.1*"
+      - "!release-23.2*"
+      - "!staging-v22.2*"
+      - "!staging-v23.1*"
+      - "!staging-v23.2*"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+jobs:
+  maybe_stress:
+    if: !github.event.pull_request.draft
+    runs-on: [self-hosted, ubuntu_2004]
+    timeout-minutes: 180
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.ref }}
+      - name: Fetch the base branch (base ref)
+        run: git fetch origin ${{ github.event.pull_request.base.ref }}:${{ github.event.pull_request.base.ref }}
+
+      - name: compute metadata
+        run: echo GITHUB_ACTIONS_BRANCH=${{ github.event.pull_request.number || github.ref_name}} >> "$GITHUB_ENV"
+      - run: ./build/github/get-engflow-keys.sh
+      - run: ./build/github/prepare-summarize-build.sh
+      - name: run tests
+        run: ./build/github/maybe-stress.sh ${{ github.event.pull_request.base.ref }}
+      - name: upload build results
+        run: ./build/github/summarize-build.sh bes.bin
+        if: always()
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: clean up
+        run: ./build/github/cleanup-engflow-keys.sh
+        if: always()

build/github/maybe-stress.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# Copyright 2025 The Cockroach Authors.
+#
+# Use of this software is governed by the CockroachDB Software License
+# included in the /LICENSE file.
+
+set -euxo pipefail
+
+# Usage: must provide a base branch as argument
+
+if [ -z "$1" ]
+then
+    echo 'Usage: build.sh BASEBRANCH'
+    exit 1
+fi
+
+BASEBRANCH="$1"
+
+bazel build --config crosslinux //pkg/cmd/ci-stress \
+    --jobs 50 $(./build/github/engflow-args.sh)
+
+# NB: These jobs will run at a lower priority to ensure that the Essential CI
+# jobs (required to merge) will be minimally impacted.
+$(bazel info bazel-bin --config=crosslinux)/pkg/cmd/ci-stress/ci-stress_/ci-stress \
+    $BASEBRANCH --config crosslinux --jobs 100 \
+    --remote_execution_priority=-1 \
+    --remote_download_minimal \
+    --bes_keywords ci-stress --config=use_ci_timeouts \
+    --build_event_binary_file=bes.bin \
+    $(./build/github/engflow-args.sh)

pkg/BUILD.bazel

@@ -134,6 +134,7 @@ ALL_TESTS = [
     "//pkg/cmd/bazci/githubpost:githubpost_test",
     "//pkg/cmd/bazci/testfilter:testfilter_test",
     "//pkg/cmd/bazci:bazci_lib_disallowed_imports_test",
+    "//pkg/cmd/ci-stress:ci-stress_test",
     "//pkg/cmd/cmpconn:cmpconn_test",
     "//pkg/cmd/cockroach:cockroach_lib_disallowed_imports_test",
     "//pkg/cmd/dev:dev_lib_disallowed_imports_test",
@@ -1154,6 +1155,9 @@ GO_TARGETS = [
     "//pkg/cmd/bazci/testfilter:testfilter_test",
     "//pkg/cmd/bazci:bazci",
     "//pkg/cmd/bazci:bazci_lib",
+    "//pkg/cmd/ci-stress:ci-stress",
+    "//pkg/cmd/ci-stress:ci-stress_lib",
+    "//pkg/cmd/ci-stress:ci-stress_test",
     "//pkg/cmd/cmdutil:cmdutil",
     "//pkg/cmd/cmp-protocol/pgconnect:pgconnect",
     "//pkg/cmd/cmp-protocol:cmp-protocol",

pkg/cmd/ci-stress/BUILD.bazel

@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library", "go_test")
+
+filegroup(
+    name = "testdata",
+    srcs = glob(["testdata/**"]),
+    visibility = ["//pkg/cmd/github-pull-request-make:__pkg__"],
+)
+
+go_library(
+    name = "ci-stress_lib",
+    srcs = ["main.go"],
+    importpath = "github.com/cockroachdb/cockroach/pkg/cmd/ci-stress",
+    visibility = ["//visibility:private"],
+    deps = ["//pkg/util/buildutil"],
+)
+
+go_binary(
+    name = "ci-stress",
+    embed = [":ci-stress_lib"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "ci-stress_test",
+    srcs = ["main_test.go"],
+    data = [":testdata"],  # keep
+    embed = [":ci-stress_lib"],
+    deps = ["//pkg/testutils/datapathutils"],
+)

pkg/cmd/ci-stress/main.go

@@ -0,0 +1,167 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"math/rand/v2"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"slices"
+	"strings"
+
+	"github.com/cockroachdb/cockroach/pkg/util/buildutil"
+)
+
+// It is a Test if there is a character after Test that is not a lower-case letter.
+const goTestStr = `func (Test[^a-z]\w*)\(.*\*testing\.TB?\) {$`
+
+var currentGoTestRE = regexp.MustCompile(`.*` + goTestStr)
+
+// getDiff returns the output of `git diff` from the given baseRef to the
+// current `HEAD`.
+func getDiff(ctx context.Context, baseRef string) (string, error) {
+	cmd := exec.CommandContext(ctx, "git", "diff", "--no-ext-diff", fmt.Sprintf("%s..HEAD", baseRef))
+	outputBytes, err := cmd.Output()
+	if err != nil {
+		return "", fmt.Errorf("unable to get diff: git diff %s..HEAD [...]: %w", baseRef, err)
+	}
+	return strings.TrimSpace(string(outputBytes)), nil
+}
+
+// getPkgToTests parses a git-style diff and returns a mapping from directories
+// to affected tests in those directories in the given diff.
+func getPkgToTests(diff string) map[string][]string {
+	const newFilePrefix = "+++ b/"
+
+	ret := make(map[string][]string)
+
+	var curPkgName string
+
+	for _, line := range strings.Split(diff, "\n") {
+		if strings.HasPrefix(line, newFilePrefix) {
+			if strings.HasSuffix(line, ".go") {
+				curPkgName = filepath.Dir(strings.TrimPrefix(line, newFilePrefix))
+			} else {
+				curPkgName = ""
+			}
+		} else if currentGoTestRE.MatchString(line) && curPkgName != "" {
+			curTestName := ""
+			if !strings.HasPrefix(line, "-") {
+				curTestName = currentGoTestRE.ReplaceAllString(line, "$1")
+			}
+			if curPkgName != "" && curTestName != "" {
+				ret[curPkgName] = append(ret[curPkgName], curTestName)
+			}
+		}
+	}
+
+	// Sanity-check: Make sure there is a `BUILD.bazel` file in each pkg,
+	// or else it's not a real Go test. (Could be testdata for a different
+	// package, etc.) Don't do this in test builds however, as we would
+	// never find those files.
+	//
+	// We also take the opportunity to limit stressing to a constant
+	// number of packages in case this PR changed a ton of packages
+	// (find-and-replace, bulk changes, etc.)
+	if !buildutil.CrdbTestBuild {
+		taken := 0
+		for pkg := range ret {
+			_, err := os.Stat(filepath.Join(pkg, "BUILD.bazel"))
+			if taken >= 10 {
+				delete(ret, pkg)
+				continue
+			}
+			if err != nil && errors.Is(err, os.ErrNotExist) {
+				fmt.Printf("skipping testing package %s as we could not find a BUILD.bazel file in that directory\n", pkg)
+				delete(ret, pkg)
+				continue
+			} else if err != nil {
+				panic(err)
+			}
+			taken += 1
+		}
+	}
+
+	for _, tests := range ret {
+		slices.Sort(tests)
+	}
+	// De-duplicate.
+	for pkg := range ret {
+		ret[pkg] = slices.Compact(ret[pkg])
+	}
+
+	// We arbitrarily limit the number of tests per package. We randomize
+	// the tests selected.
+	for pkg := range ret {
+		const maxTests = 10
+		tests := ret[pkg]
+		if len(tests) > maxTests {
+			rand.Shuffle(len(tests), func(i, j int) {
+				tests[i], tests[j] = tests[j], tests[i]
+			})
+			tests = tests[:maxTests]
+			slices.Sort(tests)
+			ret[pkg] = tests
+		}
+	}
+
+	return ret
+}
+
+func runTests(ctx context.Context, pkgToTests map[string][]string, extraBazelArgs []string) error {
+	var testPackages []string
+	for pkg := range pkgToTests {
+		testPackages = append(testPackages, fmt.Sprintf("//%s:%s_test", pkg, filepath.Base(pkg)))
+	}
+	allTests := make(map[string]struct{})
+	for _, tests := range pkgToTests {
+		for _, test := range tests {
+			allTests[test] = struct{}{}
+		}
+	}
+	allTestsSlice := make([]string, 0, len(allTests))
+	for test := range allTests {
+		allTestsSlice = append(allTestsSlice, test)
+	}
+	slices.Sort(allTestsSlice)
+	testFilter := strings.Join(allTestsSlice, "|")
+	testFilter = "^" + testFilter + "$"
+	// Run each test multiple times.
+	bazelArgs := []string{"test", "--test_filter", testFilter, "--runs_per_test", "10"}
+	bazelArgs = append(bazelArgs, testPackages...)
+	bazelArgs = append(bazelArgs, extraBazelArgs...)
+	fmt.Printf("running `bazel` with args %+v\n", bazelArgs)
+	cmd := exec.CommandContext(ctx, "bazel", bazelArgs...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func mainImpl(baseRef string, bazelArgs []string) error {
+	ctx := context.Background()
+	diff, err := getDiff(ctx, baseRef)
+	if err != nil {
+		return err
+	}
+	pkgToTests := getPkgToTests(diff)
+	return runTests(ctx, pkgToTests, bazelArgs)
+}
+
+func main() {
+	if len(os.Args) == 1 {
+		panic("expected at least one argument (the ref of the base brach)")
+	}
+	baseRef := os.Args[1]
+	bazelArgs := os.Args[2:]
+	if err := mainImpl(baseRef, bazelArgs); err != nil {
+		panic(err)
+	}
+}

pkg/cmd/ci-stress/main_test.go

@@ -0,0 +1,62 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package main
+
+import (
+	"os"
+	"reflect"
+	"testing"
+
+	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
+)
+
+func TestPkgToTests(t *testing.T) {
+	for filename, expPkgs := range map[string]map[string][]string{
+		datapathutils.TestDataPath(t, "skip.diff"): {
+			"pkg/ccl/storageccl": []string{"TestPutS3"},
+		},
+		datapathutils.TestDataPath(t, "modified.diff"): {
+			"pkg/crosscluster/physical": []string{"TestStreamingAutoReplan"},
+		},
+		datapathutils.TestDataPath(t, "removed.diff"): {},
+		datapathutils.TestDataPath(t, "not_go.diff"):  {},
+		datapathutils.TestDataPath(t, "new_test.diff"): {
+			"pkg/crosscluster/streamclient": []string{
+				"TestExternalConnectionClient",
+				"TestGetFirstActiveClientEmpty",
+			},
+		},
+		datapathutils.TestDataPath(t, "27595.diff"): {
+			"pkg/storage/closedts/container": []string{
+				"TestTwoNodes",
+			},
+			"pkg/storage/closedts/minprop": []string{
+				"TestTrackerConcurrentUse",
+			},
+			"pkg/storage/closedts/storage": []string{
+				"TestConcurrent",
+			},
+			"pkg/storage/closedts/transport": []string{
+				"TestTransportClientReceivesEntries",
+				"TestTransportConnectOnRequest",
+			},
+		},
+	} {
+		t.Run(filename, func(t *testing.T) {
+			content, err := os.ReadFile(filename)
+			if err != nil {
+				t.Fatal(err)
+			}
+			pkgToTests := getPkgToTests(string(content))
+			if err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(pkgToTests, expPkgs) {
+				t.Errorf("expected %s, got %s", expPkgs, pkgToTests)
+			}
+		})
+	}
+}