Skip to content

Commit c0de95e

Browse files
abhinavabhinav
authored
ci: Shard script tests (#747)
Running all script tests in one job takes quite a while. Try sharding them across multiple jobs to speed up CI.
1 parent 8f86e17 commit c0de95e

File tree

8 files changed

+266
-23
lines changed

8 files changed

+266
-23
lines changed

‎.github/actions/go-cache/action.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
# uses: ./.github/actions/go-cache
77

88
name: Cache Go
9+
description: Caches Go build artifacts to speed up subsequent builds.
910

1011
runs:
1112
using: "composite"

‎.github/actions/install-git/action.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
# Must run after checkout and mise-action.
1111

1212
name: Install Git
13+
description: Installs a specific version of git from source.
1314

1415
inputs:
1516
version:

‎.github/workflows/ci.yml‎

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -24,19 +24,42 @@ jobs:
2424
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2525
- run: mise run lint
2626

27-
test:
28-
runs-on: ${{ matrix.os }}
29-
name: Test (${{ matrix.os}}, Git ${{ matrix.git-version }})
27+
test-matrix:
28+
name: Generate test matrix
29+
runs-on: ubuntu-latest
30+
steps:
31+
- uses: actions/checkout@v4
32+
name: Check out repository
33+
- uses: jdx/mise-action@v2
34+
env:
35+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
36+
- name: Go cache
37+
uses: ./.github/actions/go-cache
38+
- name: Generate test matrix
39+
id: generate
40+
run: echo "matrix=$(go run ./tools/ci/test-matrix)" >> "$GITHUB_OUTPUT"
41+
outputs:
42+
matrix: ${{ steps.generate.outputs.matrix }}
3043

44+
test:
45+
needs: [test-matrix]
3146
strategy:
32-
matrix:
33-
os: ["ubuntu-latest", "windows-latest"]
34-
git-version: ["system"]
35-
include:
36-
# On Linux, also test against specific versions built from source.
37-
- {os: ubuntu-latest, git-version: "2.38.0"}
38-
# On Windows, run without coverage.
39-
- {os: windows-latest, no-cover: true}
47+
fail-fast: true
48+
matrix: ${{ fromJson(needs.test-matrix.outputs.matrix) }}
49+
# Schema of matrix:
50+
# name: string
51+
# os: ubuntu-latest | windows-latest
52+
# git-version: system | 2.38.0 | ...
53+
# suite: default | script
54+
# race: bool
55+
# cover: bool
56+
#
57+
# If suite is 'script', then:
58+
# shard-index: int
59+
# shard-count: int
60+
61+
runs-on: ${{ matrix.os }}
62+
name: Test / ${{ matrix.name }}
4063

4164
steps:
4265
- uses: actions/checkout@v4
@@ -56,21 +79,35 @@ jobs:
5679
run:
5780
git --version
5881

82+
# TODO: can probably generate the exact test command in the matrix
5983
- name: Test
84+
if: ${{ matrix.suite == 'default' }}
6085
run: >- # join lines with spaces
6186
mise run
62-
${{ (matrix.no-cover == true) && 'test' || 'cover' }}
63-
${{ (matrix.os != 'windows-latest') && '--race' || '' }}
87+
${{ matrix.cover && 'cover:default' || 'test:default' }}
88+
${{ matrix.race && '--race' || '' }}
6489
# NB:
6590
# Windows tests are already slow.
6691
# Run them without race detection to avoid slowing them further.
6792
shell: bash
6893
env:
6994
GOTESTSUM_FORMAT: github-actions
7095

96+
- name: Script tests
97+
if: ${{ matrix.suite == 'script' }}
98+
run: |
99+
mise run \
100+
${{ matrix.cover && 'cover:script' || 'test:script' }} \
101+
${{ matrix.race && '--race' || '' }} \
102+
--shard-index "${{ matrix.shard-index || '0' }}" \
103+
--shard-count "${{ matrix.shard-count || '1' }}"
104+
shell: bash
105+
env:
106+
GOTESTSUM_FORMAT: github-actions
107+
71108
- name: Upload coverage
72109
uses: codecov/codecov-action@v5.4.3
73-
if: ${{ matrix.no-cover != true }}
110+
if: ${{ matrix.os != 'windows-latest' }}
74111
with:
75112
files: ./cover.out
76113
token: ${{ secrets.CODECOV_TOKEN }}

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/bin
22
/dist
3+
/cover
34
cover.out
45
cover.html
56
/.idea

‎mise.toml‎

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -41,17 +41,54 @@ run = [
4141
]
4242

4343
[tasks.test]
44+
depends = ["test:*"]
45+
46+
[tasks."test:default"]
4447
wait_for = ["generate"]
45-
description = "Run tests"
46-
run = "gotestsum -- -race={{flag(name='race')}} ./..."
48+
description = "Run default tests"
49+
run = "gotestsum -- ./... -race={{flag(name='race')}}"
4750

48-
[tasks.cover]
51+
[tasks."test:script"]
4952
wait_for = ["generate"]
50-
description = "Run tests with coverage"
51-
run = [
52-
"gotestsum -- '-coverprofile=cover.out' '-coverpkg=./...' -race={{flag(name='race')}} ./...",
53-
"go tool cover '-html=cover.out' -o cover.html",
54-
]
53+
description = "Run script tests"
54+
run = """
55+
gotestsum -- {# -#}
56+
-tags=script {# enable script tests -#}
57+
-run '^TestScript$' {# run only script tests -#}
58+
-race={{flag(name='race')}} {# race detection -#}
59+
-shard-index={{option(name='shard-index', default='0')}} {# shard index -#}
60+
-shard-count={{option(name='shard-count', default='1')}} {# shard count -#}
61+
"""
62+
63+
[tasks."_cover_report"]
64+
run = "go tool cover -html=cover.out -o cover.html"
65+
66+
[tasks."cover:default"]
67+
wait_for = ["generate"]
68+
depends_post = ["_cover_report"]
69+
description = "Run default tests with coverage"
70+
run = """
71+
gotestsum -- {# -#}
72+
./... {# run all tests -#}
73+
-race={{flag(name='race')}} {# race detection -#}
74+
-coverpkg=./... {# cover all packages -#}
75+
-coverprofile=cover.out {# -#}
76+
"""
77+
78+
[tasks."cover:script"]
79+
wait_for = ["generate"]
80+
depends_post = ["_cover_report"]
81+
description = "Run script tests with coverage"
82+
run = """
83+
gotestsum -- {# -#}
84+
-tags=script {# enable script tests -#}
85+
-run '^TestScript$' {# run only script tests -#}
86+
-race={{flag(name='race')}} {# race detection -#}
87+
-coverpkg=./... {# cover all packages -#}
88+
-coverprofile=cover.out {# -#}
89+
-shard-index={{option(name='shard-index', default='0')}} {# shard index -#}
90+
-shard-count={{option(name='shard-count', default='1')}} {# shard count -#}
91+
"""
5592

5693
[tasks.tidy]
5794
run = "go mod tidy"

‎script_test.go‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
//go:build script
2+
13
package main
24

35
import (
@@ -30,6 +32,9 @@ import (
3032
var (
3133
_update = flag.Bool("update", false, "update golden files")
3234
_debug = flag.Bool("debug", false, "enable debug logging")
35+
36+
_shardIndex = flag.Int("shard-index", 0, "index of the test shard to run")
37+
_shardCount = flag.Int("shard-count", 1, "total number of test shards")
3338
)
3439

3540
func TestMain(m *testing.M) {
@@ -105,8 +110,36 @@ func TestScript(t *testing.T) {
105110

106111
var shamhubCmd shamhub.Cmd
107112

113+
scriptDir := filepath.Join("testdata", "script")
114+
if *_shardCount > 1 {
115+
// If we're running in sharded mode,
116+
// copy a subset of the test scripts
117+
// into a temporary directory based on the shard index.
118+
119+
scripts, err := filepath.Glob(filepath.Join(scriptDir, "*.txt"))
120+
require.NoError(t, err)
121+
122+
shardScriptDir := t.TempDir()
123+
for i, script := range scripts {
124+
if i%*_shardCount != *_shardIndex {
125+
// This script does not belong to this shard.
126+
continue
127+
}
128+
129+
t.Logf("Selected script: %s", script)
130+
131+
bs, err := os.ReadFile(script)
132+
require.NoError(t, err)
133+
dst := filepath.Join(shardScriptDir, filepath.Base(script))
134+
require.NoError(t, os.WriteFile(dst, bs, 0o644))
135+
}
136+
137+
t.Logf("Using shard script directory: %s", shardScriptDir)
138+
scriptDir = shardScriptDir
139+
}
140+
108141
testscript.Run(t, testscript.Params{
109-
Dir: filepath.Join("testdata", "script"),
142+
Dir: scriptDir,
110143
UpdateScripts: *_update,
111144
RequireUniqueNames: true,
112145
Setup: func(e *testscript.Env) error {

‎tools/ci/test-matrix/config.go‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package main
2+
3+
var (
4+
_linuxConfig = testConfig{
5+
OS: "ubuntu-latest",
6+
GitVersions: []string{"system", "2.38.0"},
7+
ScriptShards: 4,
8+
Race: true,
9+
Cover: true,
10+
}
11+
12+
_windowsConfig = testConfig{
13+
OS: "windows-latest",
14+
// We won't compile Git from source on Windows,
15+
GitVersions: []string{"system"},
16+
17+
// Windows workers are limited so don't take too many up.
18+
ScriptShards: 2,
19+
20+
// Windows tests are quite slow, so we don't enable
21+
// race detection or coverage tracking for them.
22+
}
23+
24+
_configs = []testConfig{
25+
_linuxConfig,
26+
_windowsConfig,
27+
}
28+
)

‎tools/ci/test-matrix/main.go‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
// test-matrix generates the matrix for the "test" job in the CI pipeline.
2+
package main
3+
4+
import (
5+
"cmp"
6+
"encoding/json"
7+
"flag"
8+
"fmt"
9+
"log"
10+
"os"
11+
"strings"
12+
)
13+
14+
var _indent = flag.Bool("indent", false, "indent JSON output")
15+
16+
type testConfig struct {
17+
// GitVersion is the list of Git versions
18+
// against which this test job should run.
19+
OS string
20+
21+
// GitVersions is the list of Git versions
22+
// against which this test job should run.
23+
GitVersions []string
24+
25+
// ScriptShards is the number of shards to use for script tests.
26+
ScriptShards int
27+
28+
// Whether to run with race detector and coverage.
29+
Race, Cover bool
30+
}
31+
32+
type testSuite string
33+
34+
const (
35+
defaultTests testSuite = "default"
36+
scriptTests testSuite = "script"
37+
)
38+
39+
type matrixEntry struct {
40+
Name string `json:"name"`
41+
OS string `json:"os"`
42+
43+
GitVersion string `json:"git-version"`
44+
Suite testSuite `json:"suite"`
45+
46+
Race bool `json:"race"`
47+
Cover bool `json:"cover"`
48+
49+
// ShardIndex and ShardCount are set only for script tests.
50+
ShardIndex int `json:"shard-index"`
51+
ShardCount int `json:"shard-count"`
52+
}
53+
54+
func main() {
55+
log.SetFlags(0)
56+
flag.Parse()
57+
58+
var entries []matrixEntry
59+
for _, cfg := range _configs {
60+
for _, gitVersion := range cfg.GitVersions {
61+
var name strings.Builder
62+
_, _ = fmt.Fprintf(&name, "os=%s git=%s", cfg.OS, gitVersion)
63+
64+
// Always run the default tests.
65+
entries = append(entries, matrixEntry{
66+
Name: name.String() + " suite=" + string(defaultTests),
67+
OS: cfg.OS,
68+
GitVersion: gitVersion,
69+
Suite: defaultTests,
70+
Race: cfg.Race,
71+
Cover: cfg.Cover,
72+
})
73+
74+
scriptShards := cmp.Or(cfg.ScriptShards, 1)
75+
name.WriteString(" suite=" + string(scriptTests))
76+
for shardIndex := range scriptShards {
77+
name := name.String()
78+
name += fmt.Sprintf(" shard=[%d/%d]", shardIndex+1, scriptShards)
79+
80+
entries = append(entries, matrixEntry{
81+
Name: name,
82+
OS: cfg.OS,
83+
GitVersion: gitVersion,
84+
Suite: scriptTests,
85+
Race: cfg.Race,
86+
Cover: cfg.Cover,
87+
ShardIndex: shardIndex,
88+
ShardCount: scriptShards,
89+
})
90+
}
91+
}
92+
}
93+
94+
var output struct {
95+
Include []matrixEntry `json:"include"`
96+
}
97+
output.Include = entries
98+
enc := json.NewEncoder(os.Stdout)
99+
if *_indent {
100+
enc.SetIndent("", " ")
101+
}
102+
if err := enc.Encode(output); err != nil {
103+
log.Fatalf("failed to encode JSON: %v", err)
104+
}
105+
}

0 commit comments

Comments
 (0)