diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 12fb3c9a..baacfa64 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,6 +29,20 @@ jobs: packages: "./... ./examples/..." race: true + bench-smoke: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: "1.26" + - name: Bench (examples, single iteration) + uses: ./ + with: + packages: "./examples/..." + bench: "true" + flags: "-benchtime=1x" + test: runs-on: ubuntu-latest strategy: diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index d206b3f1..f420713a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -244,6 +244,40 @@ No `TestMain` is generated — both `package foo` and `package foo_test` can define fixture-bound suites without conflict. Teardown runs via `t.Cleanup` (reverse-wavefront: leaves first, roots last). +For suites with `Benchmark*` methods, one `Benchmark` wrapper is +generated (standalone or fixture-bound, same shape as above), with one +`b.Run` per benchmark method and the timer fenced around each: + +```go +func BenchmarkFooTestSuite(b *testing.B) { + ƒ_setupFixtures(b) // only if fixture-bound + s := &ƒƒ_GOTEST_FooTestSuite{...} + b.Cleanup(func() { s.AfterAll(lifecycleT) }) + s.BeforeAll(lifecycleT) + + b.Run("BenchmarkParse", func(b *testing.B) { + b.StopTimer() + s.BeforeEach(eachT) // outside timing + b.StartTimer() + b.ResetTimer() + s.BenchmarkParse(gotest.NewB(b)) // user's b.Loop() bounds measurement + b.StopTimer() + s.AfterEach(eachT) // outside timing + }) +} +``` + +A suite must be named `*TestSuite` for its `Benchmark*` methods to be +collected at all — Pass 1 discovery matches on that suffix regardless of +whether the struct has `Test*` methods. A bench-only struct without the +suffix is invisible to the collector; its methods are silently dropped. +`ValidateContextConsistency` (Pass 4) additionally rejects a suite that +mixes `Benchmark*` methods with a returning `BeforeEach` (its context type +can't thread through `*gotest.B`) or with any stdlib `*testing.T` lifecycle +hook. The resolver rejects a fixture with `BeforeEach`/`AfterEach` bound to +a suite with `Benchmark*` methods — per-method fixture hooks aren't +supported for benchmarks. + The overlay filesystem (`-overlay=path/overlay.json`) injects generated files without modifying source. Go's compiler reads virtual paths from the overlay. @@ -425,6 +459,23 @@ The system has **four levels of parallelism**, each with distinct mechanisms: └─────────────────────────────────────────────────────────────────────┘ ``` +### Bench mode: serial, non-streaming + +`gotest bench` runs the same pipeline with `PipelineConfig.Bench: true`, and +that flag changes two things: + +- `resolveMaxParallel` short-circuits to `maxParallel = 1` — Level 3 above + collapses to one suite subprocess at a time, `--parallel`/`-test.parallel` + are ignored for scheduling. Running benchmarks concurrently would make + their timing numbers meaningless. +- `Streaming: false` — compilation and execution are not overlapped for + bench runs; `runBatch` compiles every package first, then runs. `go test -c` + never competes with a running benchmark for CPU. + +Process-per-suite isolation (Level 3's existing design) is also a +methodological benefit for benchmarks: GC pressure from one benchmark +cannot pollute another's numbers, without any extra bench-specific code. + ### Streaming Execution (Compile-Execute Overlap) `RunPipeline` with `Streaming: true` is the primary execution path. It overlaps @@ -644,6 +695,46 @@ Or without test2json: -test.run=^TestFooTestSuite$ [flags] ``` +### Bench Target Construction + +`BuildBenchTargets` is the bench-mode sibling of `BuildSuiteTargets` — it +walks `overlay.BenchesByPkg` (suites with `Benchmark*` methods) instead of +`overlay.SuitesByPkg`, and filters on two independent regexes matched +against the same `Benchmark` function name: + +``` +BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, runFlags, userRunFilter, userBenchFilter) + │ + for each package: + for each bench-suite name (e.g., "FooTestSuite"): + │ + ├─ benchFuncName = "Benchmark" + "FooTestSuite" = "BenchmarkFooTestSuite" + │ + ├─ userRunFilter set (from -run) → must match benchFuncName, else skip + ├─ userBenchFilter set (from -bench) → must match benchFuncName, else skip + │ (both apply — AND semantics; a -run Test value matches nothing + │ here since the wrapper is named Benchmark, not Test) + │ + └─ SuiteTarget{ ..., Bench: true } +``` + +A user-supplied `-bench` value is extracted out of `RunFlags` by +`ExtractBenchFilter`/`StripBenchFilter` before target construction — it must +not reach `buildSuiteCmd` as a raw `-test.bench` flag, since that would be +appended after the generated one and silently win, defeating per-suite +scoping. `buildSuiteCmd` builds the actual test binary invocation for a +bench target as: + +``` + -test.run=^$ -test.bench=^BenchmarkFooTestSuite$ -test.benchmem [flags] +``` + +`-test.run=^$` disables ordinary tests for the run; `-test.benchmem` is +appended unless already present in the forwarded flags. +`resolveMaxParallel` returns `1` whenever `PipelineConfig.Bench` is set, so +`RunSuites` dispatches these targets one at a time regardless of +`--parallel`. + --- ## 8. Full Timeline (Happy Path) diff --git a/README.md b/README.md index 6f196802..6b6f18be 100644 --- a/README.md +++ b/README.md @@ -571,6 +571,73 @@ Or when running `go test` directly: GOTEST_UPDATE_SNAPSHOTS=1 go test ./... ``` +## Benchmarking + +Benchmark methods live on the same suites as tests — same struct, same lifecycle, own subcommand. + +### Authoring + +```go +type ParserTestSuite struct { + p *Parser +} + +func (s *ParserTestSuite) BeforeEach(t *gotest.T) { s.p = NewParser() } + +func (s *ParserTestSuite) BenchmarkParse(b *gotest.B) { + doc := loadTestdata("large.json") + for b.Loop() { + s.p.Parse(doc) + } +} +``` + +`Benchmark*` methods take `*gotest.B` (or `*testing.B`) and honor the same `F_`/`X_` focus/exclude prefixes as test methods. +`BeforeEach`/`AfterEach` run once per benchmark method, outside the timing window — the generated wrapper stops the timer before `BeforeEach`, starts and resets it right before your method runs, and stops it again before `AfterEach`. +Use `b.Loop()` (Go 1.24+) rather than `b.N` — it excludes setup before the loop from timing automatically. + +The suite must be named `*TestSuite`, even if it only has benchmarks — a struct without that suffix is never discovered, so its `Benchmark*` methods are silently dropped. +Benchmarks can't coexist with a returning `BeforeEach` (its context type can't thread through `*gotest.B`), and every lifecycle hook on a benchmark suite must take `*gotest.T`, not `*testing.T`. +Fixture-bound suites work — fixtures hydrate before benchmarks run — but a fixture with `BeforeEach`/`AfterEach` bound to a suite that has `Benchmark*` methods is rejected at generation time: per-method fixture hooks aren't supported for benchmarks. + +### Running + +```bash +gotest bench ./... # all benchmarks +gotest bench ./pkg/parser -run Parse # filter by suite name +gotest bench ./pkg/parser -bench Parse # filter by benchmark name +``` + +`gotest ./...` never runs benchmarks — use `gotest bench` explicitly. + +Benchmark suites always run serially, one suite process at a time, regardless of `--parallel`/`-test.parallel`. +Running benchmarks concurrently makes their timing numbers meaningless — the runner disables streaming and compiles everything up front so `go test -c` never competes with a running benchmark for CPU. +A fresh process per suite is also a methodological win, not just an implementation detail: GC pressure from one benchmark can't pollute another's numbers. + +`-test.benchmem` is on by default — every benchmark line reports `B/op`/`allocs/op` alongside `ns/op`. +`-benchtime` and `-count` are forwarded to `go test` unchanged. +`-coverprofile` works; `--min` (coverage gate) isn't available in bench mode. +`-run` and `-bench` both filter which suites run, both matched against each suite's `Benchmark` wrapper function — `-run` the same way it scopes suites for `gotest ./...`, `-bench` by benchmark function name. +Given both, a suite must match both to run (AND semantics). +`-run Test`-style values match nothing in bench mode — filter by the suite name itself, not the `Test` prefix. + +With no matching benchmarks, `gotest bench` prints `no benchmarks found` and exits 0. + +### Spec view + +```bash +gotest bench --spec ./examples/notification -benchtime=10x +``` + +``` +BenchmarkNotificationDispatchBench + ✓ Dispatch 810.6 ns/op · 596 B/op · 2 allocs/op + +1 suites, 1 benchmarks: +``` + +Each line reports `ns/op`, `B/op`, and `allocs/op` — the same numbers `go test -bench` prints, rendered as a spec. + ## Configuration Every fixture and suite runs with sensible defaults — 2-minute fixture timeout, 30-second per-test timeout. @@ -742,12 +809,42 @@ Use the official action for CI pipelines with failure summaries, inline PR annot By default (`version: gomod`), the action resolves `gotest` from your `go.mod` — no version drift between CI and local development. Set `version: latest` or a specific tag to install a standalone binary instead. -The action emits `::error` annotations that appear inline on PR diffs and writes a markdown summary to the GitHub step summary panel. See the [reference](https://mvrahden.github.io/go-test/reference/#ci-integration) for the full inputs/outputs table. +The action emits `::error` annotations that appear inline on PR diffs and writes a markdown summary to the GitHub step summary panel. + +With `bench: true`, a benchmark step runs after the tests (`gotest bench --spec --json`): the spec view plus delta table land in the step summary, the versioned JSON report lands in a temp file exposed as the `bench-report` output, and a breached gate fails the step with the offending keys in `bench-breached-keys`. + +### Inputs + +The tables below are the canonical action surface — a drift guard test keeps them in sync with `action.yml`. + +| Input | Description | +|---|---| +| `packages` | Package patterns to test (default `./...`) | +| `race` | Enable the race detector (default `false`) | +| `coverage` | Enable coverage profiling and reporting (default `false`) | +| `min-coverage` | Minimum coverage percentage (0-100, fails if below) | +| `flags` | Additional gotest flags (`--double-dash` style; also forwarded to the bench step) | +| `go-test-flags` | Additional go test flags (`-single-dash` style) | +| `bench` | Run benchmarks after tests via `gotest bench --spec --json` (default `false`) | +| `bench-baseline` | Baseline JSON file to compare benchmarks against (`--against`) | +| `bench-gate` | Fail if any benchmark regresses by more than this percent (`--gate`) | +| `bench-save` | Save the run as a JSON baseline at this path (`--save`); an explicit empty string saves to `bench.baseline` from `.gotest.yml`; default `false` saves nothing | +| `version` | `gomod` (default) resolves from go.mod; a tag (e.g. `v1.0.0`, `latest`) installs globally | + +### Outputs + +| Output | Description | +|---|---| +| `exit-code` | Test process exit code | +| `coverage` | Coverage percentage (empty if coverage not enabled) | +| `bench-report` | Path to the `gotest bench --json` report file (empty if bench not enabled) | +| `bench-breached-keys` | Comma-joined benchmark keys that breached the gate (empty if none or no gate) | ## Commands ```bash gotest ./... -v -race # generate overlays and run tests (default) +gotest bench ./... # run BenchmarkX suite methods, serially gotest spec ./... # behavioral specification view gotest summary ./... # failure-focused summary for CI gotest watch ./... -v # watch mode with auto-rerun diff --git a/action.yml b/action.yml index 263759c8..2371028f 100644 --- a/action.yml +++ b/action.yml @@ -24,6 +24,18 @@ inputs: go-test-flags: description: "Additional go test flags (-single-dash style)" required: false + bench: + description: "Run benchmarks after tests (gotest bench --spec)" + default: "false" + bench-baseline: + description: "Baseline JSON file to compare benchmarks against (gotest bench --against)" + required: false + bench-gate: + description: "Fail the run if any benchmark regresses by more than this percent (gotest bench --gate)" + required: false + bench-save: + description: "Save the benchmark run as a JSON baseline at this path (gotest bench --save); an explicit empty string saves to bench.baseline from .gotest.yml" + default: "false" version: description: "gotest version: 'gomod' (default) resolves from go.mod, or a version tag (e.g. v1.0.0, latest) to install globally." default: "gomod" @@ -35,6 +47,12 @@ outputs: coverage: description: "Coverage percentage (empty if coverage not enabled)" value: ${{ steps.test.outputs.coverage }} + bench-report: + description: "Path to the gotest bench --json report file (empty if bench not enabled)" + value: ${{ steps.bench.outputs.bench-report }} + bench-breached-keys: + description: "Comma-joined benchmark keys that breached the gate (empty if none or no gate)" + value: ${{ steps.bench.outputs.bench-breached-keys }} runs: using: "composite" @@ -109,3 +127,69 @@ runs: fi exit "$exit_code" + + - name: Run benchmarks + id: bench + if: ${{ inputs.bench == 'true' }} + shell: bash + env: + INPUT_PACKAGES: ${{ inputs.packages }} + INPUT_BENCH_BASELINE: ${{ inputs.bench-baseline }} + INPUT_BENCH_GATE: ${{ inputs.bench-gate }} + INPUT_BENCH_SAVE: ${{ inputs.bench-save }} + INPUT_FLAGS: ${{ inputs.flags }} + INPUT_VERSION: ${{ inputs.version }} + run: | + if [ "$INPUT_VERSION" != "gomod" ]; then + cmd=(gotest) + else + cmd=(go run github.com/mvrahden/go-test/cmd/gotest) + fi + + # gotest bench's own --spec/--github wiring writes the markdown + # delta table to $GITHUB_STEP_SUMMARY automatically (mirroring how + # `gotest summary --github` does it above). Under --json the human + # rendering moves entirely into the step summary: stdout carries + # the versioned report document, captured to a file for consumers. + report="$(mktemp -t gotest-bench-XXXXXX.json)" + args=(bench --spec --json) + + if [ -n "$INPUT_BENCH_BASELINE" ]; then + args+=("--against=$INPUT_BENCH_BASELINE") + fi + + if [ -n "$INPUT_BENCH_GATE" ]; then + args+=("--gate=$INPUT_BENCH_GATE") + fi + + # bench-save: "false" (the default) skips saving; an explicit empty + # string becomes bare --save= (the bench.baseline config fallback); + # anything else is the baseline path. + if [ "$INPUT_BENCH_SAVE" != "false" ]; then + args+=("--save=$INPUT_BENCH_SAVE") + fi + + if [ -n "$INPUT_FLAGS" ]; then + read -ra extra <<< "$INPUT_FLAGS" + args+=("${extra[@]}") + fi + + read -ra pkgs <<< "$INPUT_PACKAGES" + args+=("${pkgs[@]}") + + set +e + "${cmd[@]}" "${args[@]}" > "$report" + exit_code=$? + set -e + + echo "bench-report=$report" >> "$GITHUB_OUTPUT" + + # Comma-join gate.breachedKeys from the report without requiring + # jq on the runner (python3 ships on all hosted images). + breached="" + if [ -s "$report" ]; then + breached=$(python3 -c 'import json, sys; r = json.load(open(sys.argv[1])); g = r.get("gate") or {}; print(",".join(g.get("breachedKeys") or []), end="")' "$report" 2>/dev/null || true) + fi + echo "bench-breached-keys=$breached" >> "$GITHUB_OUTPUT" + + exit "$exit_code" diff --git a/cmd/gotest/args.go b/cmd/gotest/args.go index 6dfe3732..1c61a4fd 100644 --- a/cmd/gotest/args.go +++ b/cmd/gotest/args.go @@ -66,6 +66,7 @@ type ExecConfig struct { var knownSubcommands = map[string]bool{ "discover": true, "prepare": true, + "bench": true, "generate": true, "scaffold": true, "migrate": true, diff --git a/cmd/gotest/bench.go b/cmd/gotest/bench.go new file mode 100644 index 00000000..8d8dab3f --- /dev/null +++ b/cmd/gotest/bench.go @@ -0,0 +1,315 @@ +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "os/signal" + "strconv" + + "github.com/mvrahden/go-test/internal/gotestbench" + "github.com/mvrahden/go-test/internal/gotestgen" + "github.com/mvrahden/go-test/internal/gotestrunner" + "github.com/mvrahden/go-test/internal/gotestspec" +) + +// runBench runs BenchmarkX wrapper functions for suites containing +// BenchmarkX methods, always dispatching them serially (never concurrently) +// so timing results stay meaningful. It mirrors runTest's compact flow +// (SplitArgs -> ClassifyGoTestArgs -> LoadPackages -> GenerateOverlay -> +// RunPipeline) with Bench: true. +func runBench(inv Invocation) int { //nolint:gocritic // hugeParam: stable API + ownArgs, goTestArgs, err := SplitArgs(inv.DefaultArgs(), benchAllowed) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) + return 2 + } + + specRequested := hasFlag(ownArgs, "--spec") + noColor := hasFlag(ownArgs, "--no-color") + jsonRequested := hasFlag(ownArgs, "--json") + + // -v is forced further down for benchmark result visibility; capture + // whether the caller actually asked for it first, since that's also our + // signal for showing every delta row (vs. only significant ones) below. + verboseRequested := gotestrunner.HasVerboseFlag(goTestArgs) + + saveTarget := extractStringFlag(ownArgs, "--save", "") + if saveTarget == "" && hasFlag(ownArgs, "--save") { + // `--save=` with no value asks for the configured baseline path — + // the same fallback --against already has. Config resolution stays + // in the CLI so tooling never parses .gotest.yml itself. + saveTarget = inv.Config.Bench.Baseline + if saveTarget == "" { + fmt.Fprintln(os.Stderr, "FAIL: --save needs a path (or bench.baseline in .gotest.yml)") + return 2 + } + } + + againstPath := extractStringFlag(ownArgs, "--against", "") + if againstPath == "" { + againstPath = inv.Config.Bench.Baseline + } + + gateGiven := hasFlag(ownArgs, "--gate") + gatePct := inv.Config.Bench.Gate + if gateGiven { + raw := extractStringFlag(ownArgs, "--gate", "") + v, perr := strconv.ParseFloat(raw, 64) + if perr != nil { + fmt.Fprintf(os.Stderr, "FAIL: invalid --gate value %q: %s\n", raw, perr) + return 2 + } + gatePct = v + } + gateActive := gateGiven || inv.Config.Bench.Gate > 0 + if gateActive && againstPath == "" { + fmt.Fprintln(os.Stderr, "FAIL: --gate requires --against (or bench.baseline in .gotest.yml)") + return 2 + } + + // --json needs the harvested results even without --save/--against, so it + // rides the same capture path. + benchAnalysisRequested := saveTarget != "" || againstPath != "" || jsonRequested + + // Unlike ordinary tests, benchmark results are the point of running + // gotest bench at all: stdlib `go test -bench=.` always prints result + // lines regardless of -v. Our batch collector, however, suppresses a + // suite's stdout unless verbose or failed (mirroring go test's PASS + // suppression for ordinary tests). Force verbose so ns/op lines always + // show, matching go test's actual benchmark UX. + if !verboseRequested { + goTestArgs = append(goTestArgs, "-v") + } + + cfg, err := parseExecFlags(ownArgs, goTestArgs, &inv.Config) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) + return 2 + } + + classified := gotestrunner.ClassifyGoTestArgs(goTestArgs) + loadFlags := gotestrunner.StripCoverBuildFlags(classified.BuildFlags) + loaded, broken, err := gotestgen.LoadPackages(cfg.PackagePatterns, loadFlags) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) + return 2 + } + // Timing numbers from a partially built tree are meaningless: fail fast + // like generate/prepare rather than book-and-continue like run. + if reportBrokenPackages(broken) { + return 2 + } + + overlay, cleanup, err := gotestrunner.GenerateOverlay(loaded, broken, cfg.Debug, cfg.NoCache) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) + return 2 + } + defer cleanup() + + if len(overlay.BenchesByPkg) == 0 { + if jsonRequested { + // Machine consumers get a valid empty document, not prose. + return emitBenchReport(gotestbench.FromPackages(nil), nil, nil) + } + fmt.Println("no benchmarks found") + return 0 + } + + ctx, stop := signal.NotifyContext(context.Background(), shutdownSignals...) + defer stop() + + if cfg.GlobalTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, cfg.GlobalTimeout) + defer cancel() + } + + mode := gotestrunner.RunBatchText + if specRequested || benchAnalysisRequested { + mode = gotestrunner.RunCaptureJSON + } + + result, err := gotestrunner.RunPipeline(ctx, gotestrunner.PipelineConfig{ + GoTestArgs: cfg.GoTestArgs, + SetupTimeout: cfg.SetupTimeout, + UpdateSnapshots: cfg.UpdateSnapshots, + CI: cfg.CI, + Parallel: cfg.Parallel, + CompileParallel: cfg.CompileParallel, + Streaming: false, + OutputMode: mode, + Bench: true, + BenchesByPkg: overlay.BenchesByPkg, + }, overlay) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) + return 2 + } + + code := result.ExitCode + if cfg.GlobalTimeout > 0 && ctx.Err() == context.DeadlineExceeded { + fmt.Fprintf(os.Stderr, "FAIL: global --timeout exceeded after %v\n", cfg.GlobalTimeout) + if code == 0 { + code = 1 + } + } + + var tree []*gotestspec.Package + if mode == gotestrunner.RunCaptureJSON { + events, err := gotestspec.ParseEvents(bytes.NewReader(result.CapturedJSON)) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL: parsing test events: %s\n", err) + return 2 + } + tree = gotestspec.BuildTree(events) + } + + var newBaseline gotestbench.Baseline + if benchAnalysisRequested { + newBaseline = gotestbench.FromPackages(tree) + } + + if saveTarget != "" { + if err := gotestbench.Save(saveTarget, newBaseline); err != nil { + fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) + return 2 + } + } + + var deltas []gotestbench.Delta + var specDeltas []gotestspec.BenchDelta + + if againstPath != "" { + oldBaseline, err := gotestbench.Load(againstPath) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) + return 2 + } + + deltas = gotestbench.Compare(oldBaseline, newBaseline) + specDeltas = toSpecDeltas(filterDeltas(deltas, verboseRequested)) + } + + // Render the spec view exactly once. --spec/--save always want the full + // tree; --against alone still needs it too, otherwise a bare + // `gotest bench --against=X` shows nothing but a delta table (or, if no + // delta is significant and -v wasn't passed, nothing at all) with zero + // evidence any benchmark actually ran. The delta table (if a comparison + // ran) renders as part of this same call, via WithBenchDeltas, rather + // than a second stacked summary trailer. Under --json, stdout belongs to + // the report document alone: the human rendering is suppressed entirely + // (--spec included). + if !jsonRequested && (specRequested || saveTarget != "" || againstPath != "") { + var renderOpts []gotestspec.RenderOption + if noColor { + renderOpts = append(renderOpts, gotestspec.WithNoColor()) + } + if againstPath != "" { + renderOpts = append(renderOpts, gotestspec.WithBenchDeltas(specDeltas)) + } + gotestspec.RenderTerminal(os.Stdout, tree, renderOpts...) + } + + var gateVerdict *gotestbench.Gate + if againstPath != "" && gateActive { + verdict := gotestbench.GateVerdict(deltas, gatePct) + gateVerdict = &verdict + if verdict.Breached { + fmt.Fprintf(os.Stderr, "bench gate: %s +%.1f%% exceeds %g%% gate\n", verdict.WorstKey, verdict.WorstPct, gatePct) + if code == 0 { + code = 1 + } + } + } + + if jsonRequested { + if jsonCode := emitBenchReport(newBaseline, deltasForReport(deltas, againstPath), gateVerdict); jsonCode != 0 { + return jsonCode + } + } + + // Mirror gotest summary --github's own $GITHUB_STEP_SUMMARY wiring: + // under GitHub Actions, append a markdown rendering (with the delta + // table, when --against ran) so bench results show up in the job + // summary alongside the annotations/summary the "summary" subcommand + // already writes there. + if tree != nil && os.Getenv("GITHUB_ACTIONS") == "true" { + if summaryPath := os.Getenv("GITHUB_STEP_SUMMARY"); summaryPath != "" { + sf, err := os.OpenFile(summaryPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err == nil { + var mdOpts []gotestspec.RenderOption + if againstPath != "" { + mdOpts = append(mdOpts, gotestspec.WithBenchDeltas(specDeltas)) + } + gotestspec.RenderMarkdownSummary(sf, tree, mdOpts...) + sf.Close() + } + } + } + + return code +} + +// filterDeltas returns deltas as-is when showAll (set by -v) is true; +// otherwise it returns only the significant rows. Used to decide what to +// display before converting to gotestspec.BenchDelta, since the gate check +// (WorstRegression / worstRegressionKey) always needs the full, unfiltered +// deltas regardless of what's shown. +func filterDeltas(deltas []gotestbench.Delta, showAll bool) []gotestbench.Delta { + if showAll { + return deltas + } + var out []gotestbench.Delta + for _, d := range deltas { + if d.Significant { + out = append(out, d) + } + } + return out +} + +// toSpecDeltas converts gotestbench.Delta rows to gotestspec.BenchDelta, +// the local mirror gotestspec renders via WithBenchDeltas (gotestspec must +// not import gotestbench, so this conversion lives at the call site). +func toSpecDeltas(deltas []gotestbench.Delta) []gotestspec.BenchDelta { + out := make([]gotestspec.BenchDelta, len(deltas)) + for i, d := range deltas { + out[i] = gotestspec.BenchDelta{ + Key: d.Key, + OldNs: d.OldNs, + NewNs: d.NewNs, + PercentChange: d.PercentChange, + Significant: d.Significant, + } + } + return out +} + +// emitBenchReport writes the versioned --json document to stdout. It returns +// a non-zero exit code only when the document itself cannot be produced. +func emitBenchReport(b gotestbench.Baseline, deltas []gotestbench.Delta, gate *gotestbench.Gate) int { //nolint:gocritic // hugeParam: stable API + data, err := gotestbench.MarshalReport(gotestbench.NewReport(b, deltas, gate)) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) + return 2 + } + fmt.Println(string(data)) + return 0 +} + +// deltasForReport keeps the report's deltas field absent (nil) when no +// comparison ran, and present-but-complete when one did: the document always +// carries every delta, significant or not — what to display is the +// consumer's call, significance is theirs to respect. +func deltasForReport(deltas []gotestbench.Delta, againstPath string) []gotestbench.Delta { + if againstPath == "" { + return nil + } + if deltas == nil { + deltas = []gotestbench.Delta{} + } + return deltas +} diff --git a/cmd/gotest/cli.go b/cmd/gotest/cli.go index 6aa2a03c..c46535d0 100644 --- a/cmd/gotest/cli.go +++ b/cmd/gotest/cli.go @@ -50,6 +50,8 @@ func main() { os.Exit(runSummary(inv)) case "watch": os.Exit(runWatch(inv)) + case "bench": + os.Exit(runBench(inv)) case "refactor": os.Exit(runRefactor(inv)) case "lint": diff --git a/cmd/gotest/discover.go b/cmd/gotest/discover.go index 43ba89ee..cfefd80a 100644 --- a/cmd/gotest/discover.go +++ b/cmd/gotest/discover.go @@ -42,17 +42,18 @@ type discoverPackage struct { } type discoverSuite struct { - Name string `json:"name"` - Parallel bool `json:"parallel"` - Focused bool `json:"focused"` - Excluded bool `json:"excluded"` - Guarded bool `json:"guarded"` - File string `json:"file"` - Line int `json:"line"` - Col int `json:"col"` - Lifecycle []string `json:"lifecycle"` - Fixtures []string `json:"fixtures"` - Methods []discoverMethod `json:"methods"` + Name string `json:"name"` + Parallel bool `json:"parallel"` + Focused bool `json:"focused"` + Excluded bool `json:"excluded"` + Guarded bool `json:"guarded"` + File string `json:"file"` + Line int `json:"line"` + Col int `json:"col"` + Lifecycle []string `json:"lifecycle"` + Fixtures []string `json:"fixtures"` + Methods []discoverMethod `json:"methods"` + Benchmarks []discoverMethod `json:"benchmarks"` } type discoverMethod struct { @@ -211,6 +212,25 @@ func buildDiscoverSuite(suite *gotestast.TestSuiteSpec) discoverSuite { } ds.Methods = methods + // Benchmarks (Benchmark* methods) + var benchmarks []discoverMethod + for _, bm := range suite.Benchmarks() { + bPos := fset.Position(bm.Pos()) + benchmarks = append(benchmarks, discoverMethod{ + Name: bm.Identifier(), + Parallel: false, + Focused: bm.IsFocused(), + Excluded: bm.IsExcluded(), + File: filepath.Base(bPos.Filename), + Line: bPos.Line, + Col: bPos.Column, + }) + } + if benchmarks == nil { + benchmarks = []discoverMethod{} + } + ds.Benchmarks = benchmarks + return ds } diff --git a/cmd/gotest/export_test.go b/cmd/gotest/export_test.go index ec80277c..e083c205 100644 --- a/cmd/gotest/export_test.go +++ b/cmd/gotest/export_test.go @@ -38,3 +38,4 @@ var ExportGotestFlags = gotestFlags var ExportTestAllowed = testAllowed var ExportSpecAllowed = specAllowed var ExportWatchAllowed = watchAllowed +var ExportBenchDeltaLines = benchDeltaLines diff --git a/cmd/gotest/flags.go b/cmd/gotest/flags.go index e92e83c9..d441a0ea 100644 --- a/cmd/gotest/flags.go +++ b/cmd/gotest/flags.go @@ -28,6 +28,11 @@ var gotestFlags = map[string]FlagKind{ "--parallel": ValueFlag, "--compile-parallel": ValueFlag, "--timeout": ValueFlag, + "--save": ValueFlag, + "--against": ValueFlag, + "--gate": ValueFlag, + "--bench": BoolFlag, + "--json": BoolFlag, } var testAllowed = flagSet( @@ -51,6 +56,12 @@ var summaryAllowed = flagSet( var watchAllowed = flagSet( "--debug", "--ci", "--update-snapshots", "--no-cache", "--spec", "--setup-timeout", "--timeout", "--debounce", "--parallel", "--compile-parallel", + "--bench", +) + +var benchAllowed = flagSet( + "--timeout", "--setup-timeout", "--no-cache", "--debug", "--spec", "--no-color", + "--save", "--against", "--gate", "--json", ) func flagSet(names ...string) map[string]bool { diff --git a/cmd/gotest/gotest_suite_test.go b/cmd/gotest/gotest_suite_test.go index ec239019..5d38e89c 100644 --- a/cmd/gotest/gotest_suite_test.go +++ b/cmd/gotest/gotest_suite_test.go @@ -4,17 +4,21 @@ import ( "bytes" "context" "encoding/json" + "errors" "os" "os/exec" "path/filepath" "regexp" + "runtime" "strings" "time" + "go.yaml.in/yaml/v3" "golang.org/x/tools/go/packages" . "github.com/mvrahden/go-test/cmd/gotest" "github.com/mvrahden/go-test/internal/config" + "github.com/mvrahden/go-test/internal/gotestbench" "github.com/mvrahden/go-test/internal/gotestgen" "github.com/mvrahden/go-test/internal/gotestrunner" "github.com/mvrahden/go-test/internal/gotestspec" @@ -24,12 +28,56 @@ import ( // CmdGotestTestSuite tests CLI argument parsing, subcommands, // discovery, spec rendering, and code generation. -type CmdGotestTestSuite struct{} +// +//nolint:lifecycle-pair // BeforeAll's binary lives under t.TempDir(), which the framework removes automatically +type CmdGotestTestSuite struct { + binary string + repoRoot string +} func (s *CmdGotestTestSuite) SuiteConfig() gotest.SuiteConfig { return gotest.SuiteConfig{Parallel: true} } +func (s *CmdGotestTestSuite) BeforeAll(t *gotest.T) { + absRoot, err := filepath.Abs("../..") + gotest.NoError(t, err) + s.repoRoot = absRoot + + binDir := t.TempDir() + binaryName := "gotest" + if runtime.GOOS == "windows" { + binaryName += ".exe" + } + s.binary = filepath.Join(binDir, binaryName) + cmd := exec.Command("go", "build", "-o", s.binary, "./cmd/gotest") //nolint:gosec // G204: go tool with controlled arguments + cmd.Dir = absRoot + out, err := cmd.CombinedOutput() + gotest.NoError(t, err, "build gotest binary: %s", string(out)) +} + +// runCLI runs the built gotest binary from the repo root and returns its +// combined stdout+stderr output. +func (s *CmdGotestTestSuite) runCLI(t *gotest.T, args ...string) string { + out, _ := s.runCLIExit(t, args...) + return out +} + +// runCLIExit runs the built gotest binary from the repo root and returns its +// combined stdout+stderr output along with its exit code. +func (s *CmdGotestTestSuite) runCLIExit(t *gotest.T, args ...string) (string, int) { + cmd := exec.Command(s.binary, args...) //nolint:gosec // G204: controlled binary with fixed args + cmd.Dir = s.repoRoot + out, err := cmd.CombinedOutput() + var exitErr *exec.ExitError + gotest.True(t, err == nil || errors.As(err, &exitErr), "running gotest binary: %v\n%s", err, out) + code := 0 + if cmd.ProcessState != nil { + code = cmd.ProcessState.ExitCode() + } + return string(out), code +} + func (s *CmdGotestTestSuite) TestDefaultArgs(t *gotest.T) { t.When("CLI absent", func(w *gotest.T) { for sub, tc := range gotest.Each(w, []struct { //nolint:gocritic // rangeValCopy: intentional @@ -250,6 +298,55 @@ func (s *CmdGotestTestSuite) TestCLISurfaceMatchesSpec(t *gotest.T) { }) } +// TestActionSurfaceMatchesSpec is a drift guard: README.md's GitHub Actions +// inputs/outputs tables are the canonical documented action surface and must +// stay in sync with action.yml. +func (s *CmdGotestTestSuite) TestActionSurfaceMatchesSpec(t *gotest.T) { + actionRaw, err := os.ReadFile(filepath.Join("..", "..", "action.yml")) + gotest.NoError(t, err) + var action struct { + Inputs map[string]any `yaml:"inputs"` + Outputs map[string]any `yaml:"outputs"` + } + gotest.NoError(t, yaml.Unmarshal(actionRaw, &action)) + + readme, err := os.ReadFile(filepath.Join("..", "..", "README.md")) + gotest.NoError(t, err) + doc := string(readme) + + t.When("comparing the Inputs table", func(w *gotest.T) { + documented := specTableEntries(doc, "### Inputs") + w.It("documents every action input", func(it *gotest.T) { + for name := range action.Inputs { + gotest.True(it, documented[name], "action input %q missing from README.md", name) + } + }) + w.It("documents no phantom inputs", func(it *gotest.T) { + for name := range documented { + _, known := action.Inputs[name] + gotest.True(it, known, "README.md documents unknown action input %q", name) + } + }) + }) + + t.When("comparing the Outputs table", func(w *gotest.T) { + // The Outputs table is the last subsection of its ## section, so it + // ends at the next ## heading, not at a ### one. + documented := specTableEntriesUntil(doc, "### Outputs", "\n## ") + w.It("documents every action output", func(it *gotest.T) { + for name := range action.Outputs { + gotest.True(it, documented[name], "action output %q missing from README.md", name) + } + }) + w.It("documents no phantom outputs", func(it *gotest.T) { + for name := range documented { + _, known := action.Outputs[name] + gotest.True(it, known, "README.md documents unknown action output %q", name) + } + }) + }) +} + // CmdEnvTestSuite is deliberately sequential: Setenv is illegal in parallel tests. type CmdEnvTestSuite struct{} @@ -679,6 +776,52 @@ func (s *CmdGotestTestSuite) TestRunDiscover_SimpleSuite(t *gotest.T) { }) } +func (s *CmdGotestTestSuite) TestRunDiscover_Benchmarks(t *gotest.T) { + t.It("includes benchmark methods in discover JSON, marking exclusions", func(it *gotest.T) { + srcPath := filepath.Join( + s.repoRoot, "internal", "gotestgen", "testdata", "sources", + "TestCollector_BenchmarkMethod", "test.go", + ) + src, err := os.ReadFile(srcPath) + gotest.NoError(it, err) + + // The Task 3 testdata source isn't its own module, so stage it as a + // throwaway package inside the examples module (already `use`d by + // go.work) rather than fighting GOWORK for an out-of-workspace dir. + fixtureDir, err := os.MkdirTemp(filepath.Join(s.repoRoot, "examples"), "discoverbench-") + gotest.NoError(it, err) + defer os.RemoveAll(fixtureDir) + gotest.NoError(it, os.WriteFile(filepath.Join(fixtureDir, "bench_fixture.go"), src, 0600)) + + // gotestgen.LoadPackages requires Tests:true's "[pkg.test]" variant, + // which only exists for packages with _test.go files; this fixture + // (copied verbatim from the Task 3 testdata, filename "test.go") has + // none, so load it as a plain package instead — CollectSuiteSpecs + // only needs Syntax/Types, not a test-binary variant. + pkgs, err := packages.Load(&packages.Config{ + Mode: packages.NeedModule | packages.NeedSyntax | packages.NeedName | + packages.NeedTypes | packages.NeedTypesInfo | packages.NeedImports | packages.NeedDeps, + }, fixtureDir) + gotest.NoError(it, err) + gotest.Len(it, pkgs, 1) + gotest.Empty(it, pkgs[0].Errors, "expected no package load errors, got: %v", pkgs[0].Errors) + + c := gotestgen.NewCollector() + result := c.CollectSuiteSpecs(pkgs[0]) + gotest.Empty(it, result.Errs, "expected no collector errors, got: %v", result.Errs) + gotest.Len(it, result.Suites, 1) + + ds := ExportBuildDiscoverSuite(result.Suites[0]) + data, err := json.Marshal(ds) + gotest.NoError(it, err) + payload := string(data) + + gotest.Contains(it, payload, `"benchmarks":[{"name":"BenchmarkParse"`) + gotest.Contains(it, payload, `"X_BenchmarkOld"`) + gotest.Contains(it, payload, `"excluded":true`) + }) +} + func (s *CmdGotestTestSuite) TestFocusViolation_String(t *gotest.T) { for sub, tc := range gotest.Each(t, []struct { Desc string @@ -1067,3 +1210,149 @@ func (s *CmdGotestTestSuite) TestWatchHelpers(t *gotest.T) { } }) } + +func (s *CmdGotestTestSuite) TestBenchDeltaLines(t *gotest.T) { + t.When("first run", func(w *gotest.T) { + w.It("prints no deltas but records ns/op", func(it *gotest.T) { + results := []gotestbench.Result{ + {Package: "p", Suite: "Foo", Name: "BenchmarkBar", Samples: []gotestbench.Sample{{NsPerOp: 100}}}, + } + lines, next := ExportBenchDeltaLines(results, nil) + gotest.Empty(it, lines) + gotest.Equal(it, 100.0, next["p\x00Foo\x00BenchmarkBar"]) + }) + }) + + t.When("a benchmark regresses", func(w *gotest.T) { + w.It("reports a positive delta", func(it *gotest.T) { + results := []gotestbench.Result{ + {Package: "p", Suite: "Foo", Name: "BenchmarkBar", Samples: []gotestbench.Sample{{NsPerOp: 200}}}, + } + prev := map[string]float64{"p\x00Foo\x00BenchmarkBar": 100} + + lines, next := ExportBenchDeltaLines(results, prev) + + gotest.Len(it, lines, 1) + gotest.Contains(it, lines[0], "BenchmarkBar") + gotest.Contains(it, lines[0], "200.00 ns/op") + gotest.Contains(it, lines[0], "+100.0%") + gotest.Equal(it, 200.0, next["p\x00Foo\x00BenchmarkBar"]) + }) + }) + + t.When("a benchmark improves", func(w *gotest.T) { + w.It("reports a negative delta", func(it *gotest.T) { + results := []gotestbench.Result{ + {Package: "p", Suite: "Foo", Name: "BenchmarkBar", Samples: []gotestbench.Sample{{NsPerOp: 50}}}, + } + prev := map[string]float64{"p\x00Foo\x00BenchmarkBar": 100} + + lines, _ := ExportBenchDeltaLines(results, prev) + + gotest.Len(it, lines, 1) + gotest.Contains(it, lines[0], "-50.0%") + }) + }) + + t.When("a benchmark has multiple samples", func(w *gotest.T) { + w.It("averages ns/op across samples", func(it *gotest.T) { + results := []gotestbench.Result{ + {Package: "p", Suite: "", Name: "BenchmarkBaz", Samples: []gotestbench.Sample{{NsPerOp: 100}, {NsPerOp: 200}}}, + } + _, next := ExportBenchDeltaLines(results, nil) + gotest.Equal(it, 150.0, next["p\x00\x00BenchmarkBaz"]) + }) + }) + + t.When("a benchmark is new this run", func(w *gotest.T) { + w.It("prints no delta for it", func(it *gotest.T) { + results := []gotestbench.Result{ + {Package: "p", Suite: "Foo", Name: "BenchmarkNew", Samples: []gotestbench.Sample{{NsPerOp: 100}}}, + } + prev := map[string]float64{"p\x00Foo\x00BenchmarkOther": 50} + + lines, next := ExportBenchDeltaLines(results, prev) + + gotest.Empty(it, lines) + gotest.Equal(it, 100.0, next["p\x00Foo\x00BenchmarkNew"]) + }) + }) +} + +func (s *CmdGotestTestSuite) TestBenchSubcommand(t *gotest.T) { + t.It("runs suite benchmarks serially and prints ns/op lines", func(it *gotest.T) { + out := s.runCLI(it, "bench", "./examples/notification", "-benchtime=10x") + gotest.Contains(it, out, "BenchmarkNotificationDispatchBenchTestSuite") + gotest.Contains(it, out, "ns/op") + }) + t.It("reports when no benchmarks exist", func(it *gotest.T) { + out := s.runCLI(it, "bench", "./internal/protocol") + gotest.Contains(it, out, "no benchmarks found") + }) +} + +func (s *CmdGotestTestSuite) TestBenchSaveAgainstGate(t *gotest.T) { + t.It("saves a baseline with one Sample per -count repetition", func(it *gotest.T) { + dir := it.TempDir() + baselinePath := filepath.Join(dir, "baseline.json") + + out, code := s.runCLIExit(it, "bench", "./examples/notification", "-benchtime=10x", "-count=4", "--save="+baselinePath) + gotest.Equal(it, 0, code) + // --save implies spec rendering (Task 7's spec view), which trims + // the suite wrapper's "TestSuite" suffix, so it shows up as + // "BenchmarkNotificationDispatchBench" rather than the raw + // "BenchmarkNotificationDispatchBenchTestSuite" wrapper name. + gotest.Contains(it, out, "NotificationDispatchBench") + gotest.Contains(it, out, "ns/op") + + data, err := os.ReadFile(baselinePath) + gotest.NoError(it, err) + var b gotestbench.Baseline + gotest.NoError(it, json.Unmarshal(data, &b)) + gotest.NotEmpty(it, b.Results) + for _, r := range b.Results { + gotest.Len(it, r.Samples, 4) + } + }) + + t.It("compares two saved baselines and passes an impossible-to-trip gate", func(it *gotest.T) { + dir := it.TempDir() + firstPath := filepath.Join(dir, "first.json") + + _, code := s.runCLIExit(it, "bench", "./examples/notification", "-benchtime=10x", "-count=6", "--save="+firstPath) + gotest.Equal(it, 0, code) + + out, code := s.runCLIExit(it, "bench", "./examples/notification", "-benchtime=10x", "-count=6", "--against="+firstPath, "--gate=500") + gotest.Equal(it, 0, code) + // A bare --against (no --spec/--save) must still show the benchmark + // actually ran, not just a delta table header: the tree's own + // ns/op result line has to render alongside the comparison. + gotest.Contains(it, out, "ns/op") + gotest.Contains(it, out, "BENCHMARK") + gotest.Contains(it, out, "OLD ns/op") + gotest.Contains(it, out, "NEW ns/op") + }) + + t.It("errors when --gate is given without a baseline source", func(it *gotest.T) { + out, code := s.runCLIExit(it, "bench", "./examples/notification", "-benchtime=10x", "--gate=10") + gotest.NotEqual(it, 0, code) + gotest.Contains(it, out, "--gate requires --against") + }) + + t.It("renders exactly one summary trailer for --spec --against", func(it *gotest.T) { + dir := it.TempDir() + firstPath := filepath.Join(dir, "first.json") + + _, code := s.runCLIExit(it, "bench", "./examples/notification", "-benchtime=10x", "-count=6", "--save="+firstPath) + gotest.Equal(it, 0, code) + + out, code := s.runCLIExit(it, "bench", "./examples/notification", "-benchtime=10x", "-count=6", "--spec", "--against="+firstPath) + gotest.Equal(it, 0, code) + // The spec tree's own trailing counts line ("N suites, N + // benchmarks: ...") must appear exactly once, with no second, + // stacked "N tests passed (...)" trailer from a separate + // RenderSummary call. + gotest.Contains(it, out, "benchmarks:") + gotest.NotContains(it, out, "tests passed (") + }) +} diff --git a/cmd/gotest/help.go b/cmd/gotest/help.go index a7e68853..90d2b968 100644 --- a/cmd/gotest/help.go +++ b/cmd/gotest/help.go @@ -31,6 +31,8 @@ func showHelp(topic string) { printSummaryHelp() case "watch": printWatchHelp() + case "bench": + printBenchHelp() case "discover": printDiscoverHelp() case "scaffold": @@ -65,6 +67,7 @@ Usage: gotest [flags] [packages...] Subcommands: + bench Run BenchmarkX suite methods serially spec Render behavioral specification from test output summary Show failure-focused test summary for CI watch Watch for file changes and re-run tests @@ -262,6 +265,89 @@ Examples: `) } +func printBenchHelp() { + fmt.Print(`gotest bench — run BenchmarkX suite methods + +Usage: + gotest bench [flags] [--] [go-test-flags] [packages...] + +Discovers suites containing BenchmarkX methods and runs each suite's +generated Benchmark wrapper via "go test -bench". Benchmark +suites always dispatch serially, one at a time, regardless of --parallel +or -test.parallel — running benchmarks concurrently would make their +timing results meaningless. --min is not available in bench mode; +-coverprofile is still supported and profiles are merged as usual. + +The standard go test flag -test.benchmem is enabled by default, so every +benchmark line reports allocations (B/op, allocs/op) alongside ns/op. + +Flags: + --spec Render spec view instead of raw benchmark output + --no-color Disable ANSI color codes (with --spec) + --debug Keep generated overlay for inspection + --no-cache Disable overlay cache, force fresh generation + --setup-timeout= Total budget for shared fixture setup (default: 2m, 0 to disable) + --timeout= Global pipeline deadline (default: 15m, 0 to disable) + --save= Save this run's results as a JSON baseline (forces + capture mode; the spec view is still rendered so + results remain visible). A bare --save= saves to + bench.baseline from .gotest.yml + --against= Compare this run against a saved baseline and print + a delta table (defaults to bench.baseline in + .gotest.yml when omitted) + --gate= Fail (exit 1) if the worst significant regression + against --against exceeds pct percent (e.g. 10 for + 10%); requires --against or bench.baseline in + .gotest.yml (defaults to bench.gate in + .gotest.yml, 0 disables) + --json Emit one versioned JSON document to stdout instead + of human output: the run's results (baseline + shape), every delta when a comparison ran + (significant or not), and the gate verdict when a + gate is active. Suppresses --spec rendering; + intended for tooling (the VS Code extension). + +All standard go test flags (-single-dash) are forwarded automatically. +Use a bare "--" to pass unrecognized flags without validation. + +Filtering: + -bench= Select which benchmark suites run, matched against + each suite's Benchmark wrapper name. + Sub-benchmark segments scope to single methods: + -bench='^BenchmarkFooTestSuite$/^BenchmarkParse$' + runs only that method, since the generated wrapper + runs each method under b.Run with its method name + -benchtime= Iterations or duration per benchmark (e.g. 100x, 2s) + -count= Run each benchmark n times + +Note: -run and -bench both filter which suites run in bench mode, matched +against each suite's Benchmark wrapper name — -run the same +way it scopes suites for "gotest test", -bench by its pattern's first +slash segment (later segments select sub-benchmarks inside the wrapper). +When both are given, a suite must match both to run (e.g. +"gotest bench ./pkg/parser -run Parse" filters by suite). + +If no packages contain any BenchmarkX methods, prints "no benchmarks +found" and exits 0 without invoking go test. + +--against prints a delta table (BENCHMARK / OLD ns/op / NEW ns/op / Δ) +comparing each benchmark's new mean ns/op against the saved baseline's. +Only statistically significant deltas are shown by default; pass -v to +show every row. Significant regressions are marked with a trailing "⚠". +Deltas alone never change the exit code — only --gate does. + +Examples: + gotest bench ./... Run all benchmark suites + gotest bench ./pkg/auth/... -benchtime=2s Longer per-benchmark budget + gotest bench -bench=Cache ./... Only suites matching "Cache" + gotest bench --spec ./... Spec-style rendering + gotest bench --save=bench.json ./... Save results as a baseline + gotest bench --against=bench.json ./... Compare against a baseline + gotest bench --against=bench.json --gate=10 ./... + Fail if any benchmark regresses >10% +`) +} + func printDiscoverHelp() { fmt.Print(`gotest discover — discover test suites and output JSON metadata @@ -489,6 +575,9 @@ Fields: debounce: Watch mode re-run delay (e.g., "500ms", default: 200ms) lint: skip: [, ...] Lint rules to disable globally + bench: + baseline: Default --against baseline path for "gotest bench" + gate: Default --gate regression percentage (0 disables) Skippable lint rules (non-integrity only): assertion-redundant, assertion-simplify, fail-guard, stdlib-test, t-escape, testify @@ -501,5 +590,8 @@ Example .gotest.yml: lint: skip: - testify + bench: + baseline: bench-baseline.json + gate: 10 `) } diff --git a/cmd/gotest/watch.go b/cmd/gotest/watch.go index cc224b4a..78be23ae 100644 --- a/cmd/gotest/watch.go +++ b/cmd/gotest/watch.go @@ -12,6 +12,7 @@ import ( "time" "github.com/fsnotify/fsnotify" + "github.com/mvrahden/go-test/internal/gotestbench" "github.com/mvrahden/go-test/internal/gotestgen" "github.com/mvrahden/go-test/internal/gotestrunner" "github.com/mvrahden/go-test/internal/gotestspec" @@ -49,6 +50,7 @@ func runWatch(inv Invocation) int { //nolint:gocritic // hugeParam: stable API fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) return 2 } + bench := hasFlag(ownArgs, "--bench") jsonMode, goTestArgs := stripJSONFlag(goTestArgs) specMode := hasFlag(ownArgs, "--spec") if specMode && jsonMode { @@ -76,7 +78,8 @@ func runWatch(inv Invocation) int { //nolint:gocritic // hugeParam: stable API if !jsonMode { fmt.Printf("\033[2m running tests...\033[0m\n") } - watchRunOnce(ctx, cfg, jsonMode, specMode) + var benchNs map[string]float64 + _, benchNs = watchRunOnce(ctx, cfg, jsonMode, specMode, bench, benchNs) if !jsonMode { fmt.Printf("\n\033[2m watching for changes...\033[0m\n") } @@ -128,7 +131,7 @@ func runWatch(inv Invocation) int { //nolint:gocritic // hugeParam: stable API changedCfg := cfg changedCfg.GoTestArgs = pkgArgs changedCfg.PackagePatterns = pkgPatterns - watchRunOnce(ctx, changedCfg, jsonMode, specMode) + _, benchNs = watchRunOnce(ctx, changedCfg, jsonMode, specMode, bench, benchNs) changedDirs = nil if !jsonMode { fmt.Printf("\n\033[2m watching for changes...\033[0m\n") @@ -143,7 +146,11 @@ func runWatch(inv Invocation) int { //nolint:gocritic // hugeParam: stable API } } -func watchRunOnce(ctx context.Context, cfg ExecConfig, jsonMode, specMode bool) int { //nolint:gocritic // hugeParam: stable API +// watchRunOnce runs one watch iteration. benchNs carries the previous +// iteration's per-benchmark mean ns/op (nil on the first run) and the +// returned map becomes the caller's benchNs for the next iteration; it is +// only ever populated/consulted when bench is true. +func watchRunOnce(ctx context.Context, cfg ExecConfig, jsonMode, specMode, bench bool, benchNs map[string]float64) (int, map[string]float64) { //nolint:gocritic // hugeParam: stable API classified := gotestrunner.ClassifyGoTestArgs(cfg.GoTestArgs) loadFlags := gotestrunner.StripCoverBuildFlags(classified.BuildFlags) loaded, broken, err := gotestgen.LoadPackages(cfg.PackagePatterns, loadFlags) @@ -153,7 +160,7 @@ func watchRunOnce(ctx context.Context, cfg ExecConfig, jsonMode, specMode bool) } else { fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) } - return 2 + return 2, benchNs } if cfg.CI { @@ -163,9 +170,9 @@ func watchRunOnce(ctx context.Context, cfg ExecConfig, jsonMode, specMode bool) } else { fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) } - return 2 + return 2, benchNs } else if code != 0 { - return code + return code, benchNs } } @@ -176,7 +183,7 @@ func watchRunOnce(ctx context.Context, cfg ExecConfig, jsonMode, specMode bool) } else { fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) } - return 2 + return 2, benchNs } defer cleanup() @@ -185,11 +192,23 @@ func watchRunOnce(ctx context.Context, cfg ExecConfig, jsonMode, specMode bool) cfg.JSON = true } + // Bench mode needs -v injected the same way runBench forces it (see + // cmd/gotest/bench.go): our batch collector otherwise suppresses a + // passing suite's stdout, hiding the ns/op lines go test would + // normally always print for -bench. + if bench && !gotestrunner.HasVerboseFlag(cfg.GoTestArgs) { + cfg.GoTestArgs = append(cfg.GoTestArgs, "-v") + } + mode := gotestrunner.RunBatchText if cfg.JSON { mode = gotestrunner.RunStreamJSON } - if specMode { + if specMode || bench { + // Both renders below need the result tree, which only RunCaptureJSON + // produces (RunBatchText prints go test's own text output directly; + // RunStreamJSON streams JSON live instead of returning it). This also + // matches runBench's own capture path. mode = gotestrunner.RunCaptureJSON } @@ -209,26 +228,107 @@ func watchRunOnce(ctx context.Context, cfg ExecConfig, jsonMode, specMode bool) CompileParallel: cfg.CompileParallel, Streaming: false, OutputMode: mode, + Bench: bench, + BenchesByPkg: overlay.BenchesByPkg, }, overlay) if err != nil { fmt.Fprintf(os.Stderr, "FAIL: %s\n", err) - return 2 + return 2, benchNs + } + + if bench { + benchNs = renderBenchRun(result.CapturedJSON, jsonMode, benchNs) } + if cfg.GlobalTimeout > 0 && runCtx.Err() == context.DeadlineExceeded { fmt.Fprintf(os.Stderr, "FAIL: global --timeout exceeded after %v\n", cfg.GlobalTimeout) if result.ExitCode == 0 { - return 1 + return 1, benchNs } } - if specMode { + if bench { + // Bench rendering already draws the full tree, so it subsumes --spec. + benchNs = renderBenchRun(result.CapturedJSON, jsonMode, benchNs) + } else if specMode { events, perr := gotestspec.ParseEvents(bytes.NewReader(result.CapturedJSON)) if perr != nil { fmt.Fprintf(os.Stderr, "FAIL: parsing test events: %s\n", perr) - return 2 + return 2, benchNs } gotestspec.RenderTerminal(os.Stdout, gotestspec.BuildTree(events)) } - return result.ExitCode + return result.ExitCode, benchNs +} + +// renderBenchRun parses a bench run's captured test2json output into a +// gotestspec tree, prints the results (in text mode) followed by any +// per-benchmark "Δ" delta lines vs prevNs, and returns the updated ns/op +// map for the next watch iteration. In JSON mode the raw captured events +// are written through as-is instead (mirroring what RunStreamJSON would +// have streamed live) and no delta lines are printed, since JSON consumers +// expect only test2json-shaped lines on stdout. +func renderBenchRun(capturedJSON []byte, jsonMode bool, prevNs map[string]float64) map[string]float64 { + if jsonMode { + os.Stdout.Write(capturedJSON) //nolint:errcheck // best-effort watch output + return prevNs + } + + events, err := gotestspec.ParseEvents(bytes.NewReader(capturedJSON)) + if err != nil { + fmt.Fprintf(os.Stderr, "FAIL: parsing bench events: %s\n", err) + return prevNs + } + tree := gotestspec.BuildTree(events) + gotestspec.RenderTerminal(os.Stdout, tree) + + baseline := gotestbench.FromPackages(tree) + lines, nextNs := benchDeltaLines(baseline.Results, prevNs) + for _, line := range lines { + fmt.Println(line) + } + return nextNs +} + +// benchDeltaKey uniquely identifies a benchmark result across watch runs, +// scoped by package and suite so same-named benchmarks in different +// packages/suites don't collide. +func benchDeltaKey(r gotestbench.Result) string { + return r.Package + "\x00" + r.Suite + "\x00" + r.Name +} + +// meanNsPerOp returns the mean ns/op across r's samples, or 0 if it has +// none. +func meanNsPerOp(r gotestbench.Result) float64 { + if len(r.Samples) == 0 { + return 0 + } + var sum float64 + for _, s := range r.Samples { + sum += s.NsPerOp + } + return sum / float64(len(r.Samples)) +} + +// benchDeltaLines computes the current run's mean ns/op for each of +// results and, for every benchmark that also appeared in prevNs (the +// previous run's ns/op, keyed by benchDeltaKey), formats a "Δ" delta line +// comparing the two. Benchmarks with no prior entry — the watcher's first +// bench run, or a benchmark that's new this run — produce no line, since +// there's nothing yet to compare against. The returned map becomes prevNs +// on the caller's next invocation, so deltas always compare against the +// immediately preceding in-memory run, never a persisted baseline. +func benchDeltaLines(results []gotestbench.Result, prevNs map[string]float64) (lines []string, nextNs map[string]float64) { + nextNs = make(map[string]float64, len(results)) + for _, r := range results { + mean := meanNsPerOp(r) + key := benchDeltaKey(r) + if old, ok := prevNs[key]; ok && old != 0 { + pct := (mean - old) / old * 100 + lines = append(lines, fmt.Sprintf("%s %.2f ns/op (Δ %+.1f%%)", r.Name, mean, pct)) + } + nextNs[key] = mean + } + return lines, nextNs } func addWatchDirs(w *fsnotify.Watcher, pattern string) { diff --git a/docs/design/spec.md b/docs/design/spec.md index af8ac766..ea51e333 100644 --- a/docs/design/spec.md +++ b/docs/design/spec.md @@ -119,6 +119,7 @@ gotest [subcommand] [packages...] [go-test-flags...] [--gotest-flags...] | `migrate` | Convert testify/suite tests to go-test suites | | `spec` | Run tests and render behavioral specification | | `summary` | Run tests and render a failure-focused summary (CI mode) | +| `bench` | Run `BenchmarkX` suite methods serially via `go test -bench` | | `lint` | Run gotest-specific linter checks | | `refactor` | Toggle focus prefixes: `refactor toggle-focus ` | | `discover` | Discover test suites and output JSON metadata | @@ -148,6 +149,11 @@ gotest [subcommand] [packages...] [go-test-flags...] [--gotest-flags...] | `--github` | Emit GitHub annotations and step summary (auto-enabled in GitHub Actions) | | `--coverage=` | Coverage profile path for `summary` subcommand | | `--render-only` | With `--input`, exit 0 on a failing stream: the code reports whether rendering succeeded, not whether the tests passed (requires `--input`) | +| `--bench` | Benchmark mode for `watch`: re-run benchmarks on change with ns/op deltas | +| `--save=` | Save a benchmark run as a JSON baseline (`bench`; a bare `--save=` uses `bench.baseline` from `.gotest.yml`) | +| `--against=` | Compare a benchmark run against a saved baseline and print the delta table (`bench`; defaults to `bench.baseline`) | +| `--gate=` | Fail (exit 1) if the worst significant benchmark regression exceeds the threshold (`bench`) | +| `--json` | Emit one versioned JSON report to stdout — results, deltas, gate verdict — instead of human output (`bench`; for tooling) | ### Disambiguation @@ -1298,6 +1304,7 @@ Rules are grouped into three tiers by what breaks when a finding is ignored; the | `assertion-type-guard` | `Nil`/`Empty` on types their runtime guards would reject | | `generated-file` | `gotest_p(x)suite_test.go` files present in source control | | `shared-fixture-undeclared` | Suite-method reads of a `*SharedFixture` value the suite never declared as a pointer field (directly or through the fixture DAG) — window scheduling starts only declared fixtures, so the value may be absent; locally-constructed fixtures (fixture self-tests) are exempt | +| `bench-loop` | `Benchmark*` suite methods that never touch `b.Loop()`/`b.N` — nothing iterates, so the numbers lie | **Expressiveness** — the test is correct but says it worse. Suppressible per line or project-wide via `lint.skip`. @@ -1307,6 +1314,8 @@ Rules are grouped into three tiers by what breaks when a finding is ignored; the | `assertion-redundant` | An assertion made redundant by the next one on the same argument | | `fail-guard` | `if cond { gotest.Fail(…) }` guards (also halting `Fatal`/`Fatalf`/`FailNow` bodies) — the assertion expresses the check directly; `\|\|` conditions and `else if` chains decompose into sequential assertions, non-halting `Errorf` bodies and init-scoped guards report without a fix; fires only in files that import gotest | | `t-escape` | Unnecessary `t.T()` convenience escapes: `Errorf`/`FailNow`/`Skipf`/`Setenv`/`TempDir` (available on `gotest.T`), `Skip`/`SkipNow` (use `Skipf`), `Helper` (degrades call-site reporting), `Log`/`Fatal`/`Fatalf` (use assertions and their message args) | +| `bench-fixture-io` | `Benchmark*` methods reading fixture-backed state inside the measured loop — times whatever backs the fixture, not the code under test (heuristic; hoist the read above the loop) | +| `bench-wait` | `time.Sleep`/`gotest.Eventually`/`gotest.Consistently` inside the measured loop — times the wait, not the code | **Migration** — legitimate coexistence, nudged. Suppressible per line or project-wide via `lint.skip`. diff --git a/examples/benchmarking/README.md b/examples/benchmarking/README.md new file mode 100644 index 00000000..f13cee8e --- /dev/null +++ b/examples/benchmarking/README.md @@ -0,0 +1,196 @@ +# benchmarking — LRU Cache Hot Path + +A fixed-capacity in-process LRU cache sitting in front of a slow store — the +kind of thing almost every service ends up writing, and the one place "is +this allocation-free?" is a question people actually ask about a hot path. + +## Structure + +- **cache.go** — `Cache`, a hand-rolled intrusive doubly linked list LRU (not + `container/list`, whose `Element` boxes its payload into `any` and would + cost an allocation on every promotion) +- **fixtures_test.go** — `KeyCorpusFixture`, a `BeforeAll`-only shared fixture +- **suite_test.go** — `CacheTestSuite`: one correctness test plus four + benchmarks + +## The benchmarks + +| Benchmark | Measures | +|---|---| +| `BenchmarkGetHit` | Fetching a key already in the cache — the hot path. Zero allocations. | +| `BenchmarkGetMiss` | Fetching a key that was never cached. | +| `BenchmarkPutEviction` | Inserting into a cache already at capacity — every `Put` evicts the current tail before inserting. | +| `BenchmarkFillFromEmpty` | Building a cache from scratch: one op fills a fresh, exactly-sized cache end to end, so eviction is impossible by construction. | + +`BenchmarkFillFromEmpty`'s "op" is a full fill of `fillSize` (4096) entries, +not a single `Put` — its ns/op and allocs/op measure that whole fill, so +don't read it side by side with the single-`Put` numbers from the other +three rows. An earlier version tried to get the same "never evicts" +property from a single cache with headroom (`New(1<<20)`) instead of an +exact-fit fresh cache per op; at default `-benchtime` it ran millions of +iterations, filled that headroom about a fifth of the way through, and then +quietly evicted for the remaining 77% of the run — measuring +`BenchmarkPutEviction`'s path under `BenchmarkPutCold`'s name. Filling a +cache whose capacity exactly equals what gets put into it makes eviction +impossible by construction, independent of how many times `b.Loop()` runs. + +## The teaching point: `BeforeEach` is outside the timer + +Every benchmark here needs a freshly warmed cache — reusing one left over +from a previous iteration would measure the wrong thing. `CacheTestSuite`'s +`BeforeEach` rebuilds `s.cache` from the corpus before *every* benchmark +method runs, and gotest's generated wrapper fences the timer around that +call: it stops the timer before `BeforeEach`, starts it fresh right before +your `Benchmark*` method body runs, and stops it again before `AfterEach`. +The rebuild is structurally excluded from the measurement. + +A hand-written Go benchmark doing the same warm-up needs an explicit +`b.StopTimer()` before rebuilding the cache and `b.StartTimer()` after — +easy to forget, and silent when you do (the benchmark still runs, it just +quietly measures your setup code too). Here there is no timer to remember +to stop; see the comment on `BeforeEach` in `suite_test.go`. + +## Why the fixture is `BeforeAll`-only + +`KeyCorpusFixture` builds a deterministic 4096-key/value corpus once for the +whole package — every benchmark reads the same keys, which is what makes +`--against` comparisons meaningful run over run. The same corpus warms the +shared `s.cache` used by `BenchmarkGetHit`/`BenchmarkGetMiss`/ +`BenchmarkPutEviction`, and its full size is what `BenchmarkFillFromEmpty` +fills per op. `KeyCorpusFixture` defines `BeforeAll` and nothing else. + +That's not a style choice: gotest rejects a fixture with `BeforeEach`/ +`AfterEach` bound to a suite that has `Benchmark*` methods, at generation +time. Per-method fixture hooks would run *inside* the timed method's +lifecycle — there is no way to fence them out the way the suite's own +`BeforeEach` is fenced — so the generator refuses to build the wrapper +rather than produce a benchmark that silently times someone else's setup. +`BeforeAll`/`AfterAll` run once, outside any benchmark's timing window +entirely, which is the only shape that's safe. + +## Running + +``` +$ go run ./cmd/gotest bench ./examples/benchmarking +goos: linux +goarch: amd64 +pkg: github.com/mvrahden/go-test/examples/benchmarking +cpu: AMD Ryzen 9 7950X3D 16-Core Processor +BenchmarkCacheTestSuite +BenchmarkCacheTestSuite/BenchmarkGetHit +BenchmarkCacheTestSuite/BenchmarkGetHit-6 98496046 13.22 ns/op 0 B/op 0 allocs/op +BenchmarkCacheTestSuite/BenchmarkGetMiss +BenchmarkCacheTestSuite/BenchmarkGetMiss-6 160750837 7.446 ns/op 0 B/op 0 allocs/op +BenchmarkCacheTestSuite/BenchmarkPutEviction +BenchmarkCacheTestSuite/BenchmarkPutEviction-6 7545878 150.4 ns/op 55 B/op 2 allocs/op +BenchmarkCacheTestSuite/BenchmarkFillFromEmpty +BenchmarkCacheTestSuite/BenchmarkFillFromEmpty-6 3754 318682 ns/op 415094 B/op 4114 allocs/op +PASS +ok github.com/mvrahden/go-test/examples/benchmarking 4.842s +``` + +`BenchmarkGetHit` really is 0 B/op, 0 allocs/op — a map lookup plus a +handful of pointer swaps to move the entry to the front of the list, never +a heap allocation. + +The two write benchmarks allocate for different reasons, not the same one. +`BenchmarkPutEviction` puts a key that has never existed before on every +call (an ever-incrementing counter through `strconv.Itoa`), so it allocates +a new `*entry` on every call, and a new key string on nearly every call too +(`strconv.Itoa` only avoids allocating for the first 100 integers, out of +the millions this benchmark runs through) — that's 55 B/op here. Its +reported allocs/op flips between 1 and 2 from run to run at that same 55 +B/op; that's not the code behaving differently, it's how the metric is +computed. `testing.BenchmarkResult.AllocsPerOp()` is `int64(MemAllocs) / +int64(N)` — integer division, so it can only ever report a whole number. +The true per-call average sits close to 2 (essentially every call allocates +twice) but drifts either side of that boundary run to run, from ordinary +timing and allocation-count variance across a run's iterations — and +integer truncation turns "close to 2" into a clean "1" or "2" depending on +which side it lands on, never a fraction. `BenchmarkFillFromEmpty` reuses +the corpus's pre-built key/value strings, so it never allocates a key; its +4114 allocs/op is consistently just under one `*entry` per `fillSize` +(4096) entries inserted, plus a handful of allocations from the destination +cache's map growing to size as it fills. Both land far from +`BenchmarkGetHit`'s zero — that's the contrast this example exists to show. + +### Spec view + +``` +$ go run ./cmd/gotest bench --spec --no-color ./examples/benchmarking +BenchmarkCache + ✓ GetHit 9.1 ns/op · 0 B/op · 0 allocs/op + ✓ GetMiss 7.2 ns/op · 0 B/op · 0 allocs/op + ✓ PutEviction 141.7 ns/op · 55 B/op · 1 allocs/op + ✓ FillFromEmpty 291740 ns/op · 415091 B/op · 4114 allocs/op + +1 suites, 4 benchmarks: +``` + +### Baseline, compare, gate + +``` +$ go run ./cmd/gotest bench ./examples/benchmarking --save=/tmp/cache-baseline.json -count=6 +BenchmarkCache + ✓ GetHit 9.3 ns/op · 0 B/op · 0 allocs/op + ✓ GetMiss 7.7 ns/op · 0 B/op · 0 allocs/op + ✓ PutEviction 146.1 ns/op · 55 B/op · 1 allocs/op + ✓ FillFromEmpty 328712 ns/op · 415093 B/op · 4114 allocs/op + +1 suites, 4 benchmarks: +``` + +``` +$ go run ./cmd/gotest bench ./examples/benchmarking --against=/tmp/cache-baseline.json +BenchmarkCache + ✓ GetHit 9.5 ns/op · 0 B/op · 0 allocs/op + ✓ GetMiss 7.4 ns/op · 0 B/op · 0 allocs/op + ✓ PutEviction 144.5 ns/op · 55 B/op · 1 allocs/op + ✓ FillFromEmpty 312995 ns/op · 415093 B/op · 4114 allocs/op + +BENCHMARK OLD ns/op NEW ns/op Δ + +1 suites, 4 benchmarks: +``` + +``` +$ go run ./cmd/gotest bench ./examples/benchmarking --against=/tmp/cache-baseline.json --gate=10 +BenchmarkCache + ✓ GetHit 9.0 ns/op · 0 B/op · 0 allocs/op + ✓ GetMiss 7.0 ns/op · 0 B/op · 0 allocs/op + ✓ PutEviction 142.4 ns/op · 55 B/op · 2 allocs/op + ✓ FillFromEmpty 300361 ns/op · 415091 B/op · 4114 allocs/op + +BENCHMARK OLD ns/op NEW ns/op Δ + +1 suites, 4 benchmarks: +``` + +The command above exits `0`. + +None of the four benchmarks appear in the delta table in this pair of +runs — every old-vs-new difference here is small enough that it didn't +clear the statistical significance test, so it's correctly reported as +noise rather than a real change, and there's nothing for `--gate=10` to +act on. Had a row cleared significance in the slow direction by more than +10%, it would appear with a trailing `⚠` and the command would exit 1; +`gotest bench`'s own `--against` docs (`gotest help bench`) describe the +same delta table appearing with rows in it when that happens. Deltas alone +never change the exit code — only `--gate` does. + +## Why these numbers are trustworthy + +- **Serial execution.** `gotest bench` runs benchmark suites one at a time, + regardless of `--parallel`. Two benchmarks racing for the same CPU + cores would make both of their timings meaningless. +- **Process-per-suite isolation.** Each suite's benchmarks run in their own + compiled test binary. GC pressure or heap growth from one suite's + benchmarks can't leak into another's numbers. +- **`BeforeEach` outside the timer.** Every benchmark above measures only + the operation named — never the corpus lookup, cache rebuild, or fixture + hydration that gets it there. See "The teaching point" above. +- **Properties true by construction, not by luck.** `BenchmarkFillFromEmpty` + never evicts because its cache's capacity exactly equals what gets put + into it, for every op, regardless of iteration count — not because a + headroom number happens to outrun a given `-benchtime`. See "The + benchmarks" above. diff --git a/examples/benchmarking/cache.go b/examples/benchmarking/cache.go new file mode 100644 index 00000000..04b189ff --- /dev/null +++ b/examples/benchmarking/cache.go @@ -0,0 +1,112 @@ +// Package benchmarking is a fixed-capacity LRU cache sitting in front of a +// slow store — the kind of thing almost every service ends up writing, and +// the one place "is this allocation-free?" is a question people actually +// ask about their hot path. +package benchmarking + +// entry is a node in an intrusive doubly linked list that tracks recency. +// Hand-rolling this (rather than reaching for container/list, whose Element +// stores its payload as `any` and boxes it on every insertion) is what keeps +// Get's hot path allocation-free: promoting an entry to the front is just a +// handful of pointer writes, never a heap allocation. +type entry struct { + key, value string + prev, next *entry +} + +// Cache is a fixed-capacity, in-process LRU cache. It is not safe for +// concurrent use. +type Cache struct { + capacity int + items map[string]*entry + + // head is the most recently used entry, tail the least recently used. + head, tail *entry +} + +// New creates a Cache holding at most capacity entries. A non-positive +// capacity is treated as 1. +func New(capacity int) *Cache { + if capacity < 1 { + capacity = 1 + } + return &Cache{ + capacity: capacity, + items: make(map[string]*entry, capacity), + } +} + +// Get looks up key and, on a hit, promotes it to most-recently-used. +func (c *Cache) Get(key string) (string, bool) { + e, ok := c.items[key] + if !ok { + return "", false + } + c.moveToFront(e) + return e.value, true +} + +// Put inserts or updates key. Updating an existing key also promotes it to +// most-recently-used. Inserting past capacity evicts the least recently +// used entry first. +func (c *Cache) Put(key, value string) { + if e, ok := c.items[key]; ok { + e.value = value + c.moveToFront(e) + return + } + if len(c.items) >= c.capacity { + c.evictTail() + } + e := &entry{key: key, value: value} + c.items[key] = e + c.pushFront(e) +} + +// Len reports the number of entries currently cached. +func (c *Cache) Len() int { + return len(c.items) +} + +func (c *Cache) pushFront(e *entry) { + e.prev = nil + e.next = c.head + if c.head != nil { + c.head.prev = e + } + c.head = e + if c.tail == nil { + c.tail = e + } +} + +func (c *Cache) unlink(e *entry) { + if e.prev != nil { + e.prev.next = e.next + } else { + c.head = e.next + } + if e.next != nil { + e.next.prev = e.prev + } else { + c.tail = e.prev + } + e.prev, e.next = nil, nil +} + +func (c *Cache) moveToFront(e *entry) { + if c.head == e { + return + } + c.unlink(e) + c.pushFront(e) +} + +func (c *Cache) evictTail() { + e := c.tail + if e == nil { + return + } + c.unlink(e) + delete(c.items, e.key) +} diff --git a/examples/benchmarking/fixtures_test.go b/examples/benchmarking/fixtures_test.go new file mode 100644 index 00000000..2a720d2d --- /dev/null +++ b/examples/benchmarking/fixtures_test.go @@ -0,0 +1,39 @@ +package benchmarking + +import ( + "context" + "fmt" +) + +// corpusSize is the number of key/value pairs KeyCorpusFixture generates. It +// serves two purposes: it's the capacity CacheTestSuite's BeforeEach gives +// the warmed cache (the corpus exactly fills it), and it's the fill size +// BenchmarkFillFromEmpty uses to build a cache from scratch one op at a +// time (see suite_test.go's fillSize). +const corpusSize = 4096 + +// KeyCorpusFixture builds a deterministic key/value corpus once for the +// whole package, so no benchmark pays for generating its own inputs, and +// every run reads from the same data — which is what makes --against +// comparisons meaningful. +// +// Fixtures bound to a benchmark suite may define BeforeAll/AfterAll only: +// gotest rejects per-method fixture hooks (BeforeEach/AfterEach on the +// fixture itself) for benchmark suites at generation time, because they +// would run inside the timed method's lifecycle instead of around it — +// exactly the leak the suite's own BeforeEach fencing (see suite_test.go) +// is designed to avoid. +type KeyCorpusFixture struct { + Keys []string + Values []string +} + +func (f *KeyCorpusFixture) BeforeAll(ctx context.Context) error { + f.Keys = make([]string, corpusSize) + f.Values = make([]string, corpusSize) + for i := range corpusSize { + f.Keys[i] = fmt.Sprintf("key-%04d", i) + f.Values[i] = fmt.Sprintf("value-%04d", i) + } + return nil +} diff --git a/examples/benchmarking/suite_test.go b/examples/benchmarking/suite_test.go new file mode 100644 index 00000000..0da66540 --- /dev/null +++ b/examples/benchmarking/suite_test.go @@ -0,0 +1,158 @@ +package benchmarking + +import ( + "strconv" + + "github.com/mvrahden/go-test/pkg/gotest" +) + +// CacheTestSuite exercises Cache both for correctness and for performance. +// Corpus is a named field bound to KeyCorpusFixture (see fixtures_test.go), +// built once for the whole suite; cache is rebuilt by BeforeEach below. +type CacheTestSuite struct { + Corpus *KeyCorpusFixture + cache *Cache +} + +// BeforeEach rebuilds a cache warmed from the corpus before every test and +// every BenchmarkX method. This is the whole point of the example: gotest's +// generated wrapper fences the timer around a benchmark method's BeforeEach +// call, so this rebuild is never part of the measurement. A hand-written Go +// benchmark reusing this same warm-cache-per-iteration setup would need an +// explicit b.StopTimer() before the rebuild and b.StartTimer() after it — +// easy to forget, and silent when you do. Here it's structural: there is no +// timer to remember to stop. +func (s *CacheTestSuite) BeforeEach(t *gotest.T) { + s.cache = New(len(s.Corpus.Keys)) + for i, key := range s.Corpus.Keys { + s.cache.Put(key, s.Corpus.Values[i]) + } +} + +func (s *CacheTestSuite) TestEvictsLeastRecentlyUsed(t *gotest.T) { + t.When("a cache at capacity receives a new key", func(t *gotest.T) { + cache := New(2) + cache.Put("a", "1") + cache.Put("b", "2") + cache.Get("a") // touch "a" so "b" becomes the least recently used + + cache.Put("c", "3") + + t.It("evicts the least recently used entry", func(t *gotest.T) { + _, ok := cache.Get("b") + gotest.False(t, ok) + }) + + t.It("keeps the entries that were used", func(t *gotest.T) { + v, ok := cache.Get("a") + gotest.True(t, ok) + gotest.Equal(t, "1", v) + + v, ok = cache.Get("c") + gotest.True(t, ok) + gotest.Equal(t, "3", v) + }) + + t.It("stays within capacity", func(t *gotest.T) { + gotest.Equal(t, 2, cache.Len()) + }) + }) + + t.When("an existing key is put again", func(t *gotest.T) { + cache := New(2) + cache.Put("a", "1") + cache.Put("b", "2") + + cache.Put("a", "1-updated") // update should also promote "a" + cache.Put("c", "3") // cache is full again; "b" is now least recently used + + t.It("updates the value", func(t *gotest.T) { + v, ok := cache.Get("a") + gotest.True(t, ok) + gotest.Equal(t, "1-updated", v) + }) + + t.It("promotes the updated key instead of evicting it", func(t *gotest.T) { + _, ok := cache.Get("b") + gotest.False(t, ok) + + v, ok := cache.Get("c") + gotest.True(t, ok) + gotest.Equal(t, "3", v) + }) + }) +} + +// BenchmarkGetHit is the hot path: fetch a key already in the cache. Nothing +// above "for b.Loop()" — including BeforeEach's rebuild of s.cache — is part +// of the measurement; see the comment on BeforeEach for why that matters. +// This is the benchmark to watch for 0 allocs/op. +// +// key is read from s.Corpus (a fixture field) above the loop, deliberately: +// the bench-fixture-io rule flags a fixture read that happens *inside* +// "for b.Loop()", because that's the one thing the timer fence can't save +// you from — if the fixture were backed by a database or a network service, +// an in-loop read would time that I/O instead of Cache.Get. +func (s *CacheTestSuite) BenchmarkGetHit(b *gotest.B) { + key := s.Corpus.Keys[0] + for b.Loop() { + s.cache.Get(key) + } +} + +// BenchmarkGetMiss looks up a key that was never in the cache. +func (s *CacheTestSuite) BenchmarkGetMiss(b *gotest.B) { + for b.Loop() { + s.cache.Get("not-a-key") + } +} + +// BenchmarkPutEviction measures steady-state inserts into a cache that is +// already at capacity: BeforeEach warms s.cache from the full corpus, so +// every Put below evicts the current tail before inserting a fresh key. +// +// Its reported allocs/op flips between 1 and 2 run to run at a constant 55 +// B/op. That's the metric, not the code: AllocsPerOp() is an integer +// division (total allocs / N), so a true average sitting close to 2 (an +// *entry plus a key string on nearly every call) gets truncated to whichever +// whole number it happens to land nearest to that run — never a fraction. +func (s *CacheTestSuite) BenchmarkPutEviction(b *gotest.B) { + i := 0 + for b.Loop() { + s.cache.Put(strconv.Itoa(i), "v") + i++ + } +} + +// fillSize is the number of entries BenchmarkFillFromEmpty inserts per op — +// the fixture's entire corpus, so a full fill uses every generated key once. +const fillSize = corpusSize + +// BenchmarkFillFromEmpty measures building a cache from scratch. One op is a +// full fill of fillSize entries into a fresh cache whose capacity is exactly +// fillSize, so no entry can ever be evicted — the contrast against +// BenchmarkPutEviction is structural (a cache that can never be full enough +// to evict), not a matter of how many iterations b.Loop() happens to run. +// A capacity-headroom version of this benchmark was tried first and +// rejected: at default -benchtime it ran millions of iterations against a +// cache with "only" 1<<20 headroom, filled that headroom about a fifth of +// the way through the run, and then evicted for the remaining 77% of +// iterations — silently measuring BenchmarkPutEviction's path instead of +// its own. This shape makes that impossible regardless of iteration count. +// +// Every Put here allocates a new *entry, and one op also allocates the +// destination cache's backing map — so its allocation profile isn't "N +// individual Puts" so much as "one cache's worth of Puts plus one map +// alloc," and its ns/op is the cost of fillSize inserts, not one insert: +// don't compare it directly to BenchmarkPutEviction's or BenchmarkGetHit's +// per-op numbers. +func (s *CacheTestSuite) BenchmarkFillFromEmpty(b *gotest.B) { + keys := s.Corpus.Keys[:fillSize] + vals := s.Corpus.Values[:fillSize] + for b.Loop() { + c := New(fillSize) + for i := range keys { + c.Put(keys[i], vals[i]) + } + } +} diff --git a/examples/notification/bench_suite_test.go b/examples/notification/bench_suite_test.go new file mode 100644 index 00000000..40fa59d2 --- /dev/null +++ b/examples/notification/bench_suite_test.go @@ -0,0 +1,22 @@ +package notification + +import ( + "github.com/mvrahden/go-test/pkg/gotest" +) + +type NotificationDispatchBenchTestSuite struct { + dispatcher *dispatcher +} + +func (s *NotificationDispatchBenchTestSuite) BeforeEach(t *gotest.T) { + s.dispatcher = newDispatcher() +} + +func (s *NotificationDispatchBenchTestSuite) BenchmarkDispatch(b *gotest.B) { + for b.Loop() { + s.dispatcher.Send(Notification{ + To: "bench@example.com", + Subject: "Benchmark", + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 984fc7e5..7136a8d2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -37,6 +37,8 @@ type ProjectConfig struct { Debounce *Duration `yaml:"debounce"` // Lint holds lint-specific configuration. Lint LintConfig `yaml:"lint"` + // Bench holds benchmark baseline/gate configuration. + Bench BenchConfig `yaml:"bench"` } // LintConfig controls which lint rules are disabled project-wide. @@ -45,6 +47,17 @@ type LintConfig struct { Skip []string `yaml:"skip"` } +// BenchConfig controls default baseline comparison settings for +// "gotest bench". CLI flags (--against, --gate) take precedence over these. +type BenchConfig struct { + // Baseline is the default path used for --against when not given on the + // CLI. + Baseline string `yaml:"baseline"` + // Gate is the default regression gate percentage used for --gate when + // not given on the CLI. Zero disables the gate. + Gate float64 `yaml:"gate"` +} + // Duration wraps time.Duration with human-readable YAML unmarshaling. type Duration time.Duration diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 90e0d7cd..ba956cc6 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -22,6 +22,9 @@ lint: skip: - stdlib-test - testify +bench: + baseline: bench-baseline.json + gate: 10.5 `) cfg, err := Load(dir) @@ -37,6 +40,8 @@ lint: assertEqual(t, "compile-parallel", cfg.CompileParallel, 2) assertDuration(t, "debounce", cfg.Debounce, 500*time.Millisecond) assertSliceEqual(t, "lint.skip", cfg.Lint.Skip, []string{"stdlib-test", "testify"}) + assertEqual(t, "bench.baseline", cfg.Bench.Baseline, "bench-baseline.json") + assertEqual(t, "bench.gate", cfg.Bench.Gate, 10.5) } func TestLoad_NoFile_ReturnsZero(t *testing.T) { @@ -58,6 +63,8 @@ func TestLoad_NoFile_ReturnsZero(t *testing.T) { if len(cfg.Lint.Skip) != 0 { t.Errorf("lint.skip: got %v, want empty", cfg.Lint.Skip) } + assertEqual(t, "bench.baseline", cfg.Bench.Baseline, "") + assertEqual(t, "bench.gate", cfg.Bench.Gate, 0.0) } func TestLoad_PartialConfig(t *testing.T) { diff --git a/internal/gotestast/spec.go b/internal/gotestast/spec.go index fc6327fe..87821fc2 100644 --- a/internal/gotestast/spec.go +++ b/internal/gotestast/spec.go @@ -12,6 +12,7 @@ import ( "github.com/dlclark/regexp2" "github.com/mvrahden/go-test/internal/about" + "github.com/mvrahden/go-test/internal/protocol" "github.com/mvrahden/go-test/internal/x/slices" "golang.org/x/tools/go/packages" ) @@ -28,13 +29,14 @@ func (r *regexpW) MatchString(s string) bool { var ( GEN_TESTSUITE_FILE = regexp.MustCompile(`^// Code generated by "gotest \(github\.com/mvrahden/go-test\)"; DO NOT EDIT\.(?:$|\n)`) IS_TEST_SUITE = ®expW{regexp2.MustCompile(`^(?!ƒƒ_GOTEST_|_)(?:X_|F_)?.+TestSuite$`, regexp2.ECMAScript)} - IS_TEST_SUITE_METHOD = ®expW{regexp2.MustCompile(`^(?:BeforeAll|AfterAll|BeforeEach|AfterEach|SuiteConfig|SuiteGuard|(?:X_|F_)?Test.+)$`, regexp2.ECMAScript)} + IS_TEST_SUITE_METHOD = ®expW{regexp2.MustCompile(fmt.Sprintf(`^(?:BeforeAll|AfterAll|BeforeEach|AfterEach|SuiteConfig|SuiteGuard|(?:X_|F_)?Test.+|(?:X_|F_)?%s.+)$`, protocol.PrefixBenchmark), regexp2.ECMAScript)} IS_BEFORE_ALL = regexp.MustCompile(`^BeforeAll$`) IS_AFTER_ALL = regexp.MustCompile(`^AfterAll$`) IS_BEFORE_EACH = regexp.MustCompile(`^BeforeEach$`) IS_AFTER_EACH = regexp.MustCompile(`^AfterEach$`) - IS_TEST_CASE = ®expW{regexp2.MustCompile(`^(?:X_|F_)?Test.+$`, regexp2.ECMAScript)} // matches all test cases - IS_TEST_CASE_ASYNC = regexp.MustCompile(`^(?:X_|F_)?Test.+Async$`) // matches all test cases with async suffix + IS_TEST_CASE = ®expW{regexp2.MustCompile(`^(?:X_|F_)?Test.+$`, regexp2.ECMAScript)} // matches all test cases + IS_TEST_CASE_ASYNC = regexp.MustCompile(`^(?:X_|F_)?Test.+Async$`) // matches all test cases with async suffix + IS_BENCHMARK = ®expW{regexp2.MustCompile(fmt.Sprintf(`^(?:X_|F_)?%s.+$`, protocol.PrefixBenchmark), regexp2.ECMAScript)} // matches all benchmark methods ) type TestSuiteSpecSet []*TestSuiteSpec @@ -93,6 +95,29 @@ func (ts TestSuiteSpecSet) ReduceToEffectiveSet() (TestSuiteSpecSet, SkippedTest }) } + { // reduce BENCHMARK methods + slices.Range(effectiveSet, func(v *TestSuiteSpec, _ int) { + // split focused from unfocused + focused, unfocused := slices.SplitBy(v.th.Benchmarks, func(v *TestSuiteMethod, _ int) bool { + return v.IsFocused() + }) + v.th.Benchmarks = unfocused + if len(focused) > 0 { + v.th.Benchmarks = focused + skippedTestCases[v] = append(skippedTestCases[v], unfocused...) + } + + // split excluded from included + excluded, included := slices.SplitBy(v.th.Benchmarks, func(v *TestSuiteMethod, _ int) bool { + return v.IsExcluded() + }) + v.th.Benchmarks = included + if len(excluded) > 0 { + skippedTestCases[v] = append(skippedTestCases[v], excluded...) + } + }) + } + return effectiveSet, skippedTestSuites, skippedTestCases } @@ -190,6 +215,13 @@ func (ts *TestSuiteSpec) TestCases() []*TestSuiteMethod { return ts.th.TestCases } +// Benchmarks returns the benchmark methods slice. +// +// FOR RENDERING +func (ts *TestSuiteSpec) Benchmarks() []*TestSuiteMethod { + return ts.th.Benchmarks +} + // HasConfig returns true when the suite defines a SuiteConfig() marker method. // // FOR RENDERING @@ -288,6 +320,7 @@ type TestSuiteHarness struct { AfterAll *TestSuiteMethod AfterEach *TestSuiteMethod TestCases []*TestSuiteMethod + Benchmarks []*TestSuiteMethod Config *types.Func // SuiteConfig() method, may be nil Guard *types.Func // SuiteGuard() method, may be nil ConfigParallel bool @@ -531,6 +564,34 @@ func DetermineTestSuite(n ast.Node, pkg *packages.Package) (*TestSuiteSpec, toke return &TestSuiteSpec{pkg: pkg, n: n, ts: ts, typ: typ, underlyingTypeName: underlyingTypeName, th: &TestSuiteHarness{}}, -1, nil } +// detectParamB reports whether sig's sole parameter is *testing.B or *gotest.B. +// usesStdlibB is true for *testing.B; ok is false for any other signature shape. +func detectParamB(sig *types.Signature) (usesStdlibB bool, ok bool) { + if sig.Params().Len() != 1 { + return false, false + } + ptr, isPtr := sig.Params().At(0).Type().(*types.Pointer) + if !isPtr { + return false, false + } + named, isNamed := ptr.Elem().(*types.Named) + if !isNamed { + return false, false + } + obj := named.Obj() + if obj.Name() != "B" { + return false, false + } + pkg := obj.Pkg() + if pkg == nil { + return false, false + } + if pkg.Path() == "testing" { + return true, true + } + return false, strings.HasSuffix(pkg.Path(), "/pkg/gotest") +} + func DetermineTestSuiteHarness(n ast.Node, pkg *packages.Package, s *TestSuiteSpec) (token.Pos, error) { decl, ok := n.(*ast.FuncDecl) if !ok { @@ -617,6 +678,7 @@ func DetermineTestSuiteHarness(n ast.Node, pkg *packages.Package, s *TestSuiteSp tm := &TestSuiteMethod{n: n, m: m, sig: sig} isTestCase := IS_TEST_CASE.MatchString(m.Name()) + isBenchmark := IS_BENCHMARK.MatchString(m.Name()) switch { case IS_BEFORE_ALL.MatchString(m.Name()): s.th.BeforeAll = tm @@ -626,6 +688,14 @@ func DetermineTestSuiteHarness(n ast.Node, pkg *packages.Package, s *TestSuiteSp s.th.BeforeEach = tm case IS_AFTER_EACH.MatchString(m.Name()): s.th.AfterEach = tm + case isBenchmark: + usesStdlibB, ok := detectParamB(sig) + if !ok { + return m.Pos(), fmt.Errorf("benchmark method %s must accept exactly one parameter of type *gotest.B or *testing.B", methodID) + } + tm.usesStdlibT = usesStdlibB + s.th.Benchmarks = append(s.th.Benchmarks, tm) + return -1, nil case isTestCase: // pass-through: these will be handled further down the line default: @@ -740,6 +810,28 @@ func ValidateContextConsistency(ts *TestSuiteSpec) error { return fmt.Errorf("test suite %q must be exported — go test never runs its generated Test function", suiteName) } + // Bench Rule 1: benchmarks cannot coexist with a returning BeforeEach — a + // returning-BeforeEach context type can't thread through *testing.B/*gotest.B. + if len(ts.th.Benchmarks) > 0 && be != nil && be.HasReturn() { + return fmt.Errorf("suite %s has benchmark methods but a returning BeforeEach — move benchmarks to a dedicated suite", suiteName) + } + + // Bench Rule 2: bench lifecycles require *gotest.T hooks (not *testing.T). + if len(ts.th.Benchmarks) > 0 { + if ts.th.BeforeAll != nil && ts.th.BeforeAll.UsesStdlibT() { + return fmt.Errorf("suite %s has benchmark methods but %s uses *testing.T — bench lifecycles require *gotest.T hooks", suiteName, "BeforeAll") + } + if ts.th.AfterAll != nil && ts.th.AfterAll.UsesStdlibT() { + return fmt.Errorf("suite %s has benchmark methods but %s uses *testing.T — bench lifecycles require *gotest.T hooks", suiteName, "AfterAll") + } + if be != nil && be.UsesStdlibT() { + return fmt.Errorf("suite %s has benchmark methods but %s uses *testing.T — bench lifecycles require *gotest.T hooks", suiteName, "BeforeEach") + } + if ae != nil && ae.UsesStdlibT() { + return fmt.Errorf("suite %s has benchmark methods but %s uses *testing.T — bench lifecycles require *gotest.T hooks", suiteName, "AfterEach") + } + } + // Rule 1/7: Parallel requires returning BeforeEach (void BeforeEach forbidden) if ts.th.ConfigParallel && be != nil && !be.HasReturn() { return fmt.Errorf("%s: SuiteConfig has Parallel: true, but BeforeEach has no return value. Parallel methods require per-test isolation — move per-test fields to a context struct and return it from BeforeEach", suiteName) diff --git a/internal/gotestbench/baseline.go b/internal/gotestbench/baseline.go new file mode 100644 index 00000000..fc04bf1e --- /dev/null +++ b/internal/gotestbench/baseline.go @@ -0,0 +1,264 @@ +// Package gotestbench provides a JSON baseline format for benchmark +// results extracted from gotestspec trees, and a Welch's t-test based +// significance comparison between two baselines. +package gotestbench + +import ( + "encoding/json" + "fmt" + "os" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "time" + + "github.com/mvrahden/go-test/internal/gotestspec" +) + +// schemaVersion is the only Baseline.SchemaVersion this package writes and +// accepts on Load. +const schemaVersion = 1 + +// Baseline is a persisted snapshot of benchmark results, keyed by package, +// suite, and benchmark name. +type Baseline struct { + SchemaVersion int `json:"schemaVersion"` // 1 + CreatedAt time.Time `json:"createdAt"` + GoVersion string `json:"goVersion"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + Results []Result `json:"results"` +} + +// Result is one benchmark's samples, identified by its package path, its +// enclosing suite name (empty for bare top-level benchmarks with no suite +// wrapper), and its own method name. +type Result struct { + Package string `json:"package"` + Suite string `json:"suite"` + Name string `json:"name"` + Samples []Sample `json:"samples"` // one per -count repetition (see FromPackages) +} + +// Sample mirrors the fields go test prints per benchmark run: +// N ns/op B/op allocs/op. +type Sample struct { + Iterations int `json:"iterations"` + NsPerOp float64 `json:"nsPerOp"` + BytesPerOp int64 `json:"bytesPerOp"` + AllocsPerOp int64 `json:"allocsPerOp"` +} + +// FromPackages walks the KindBenchmark leaves of pkgs and produces a +// Baseline. +// +// Key concept: Package is the Go package path, Suite is the bench wrapper's +// suite name (the top-level node name with its "Benchmark" prefix +// stripped), and Name is the leaf benchmark method's own node name. Bare +// top-level benchmarks with no enclosing suite wrapper get an empty Suite. +// +// Sample harvesting for -count=N: go test's own -json encoder only tags the +// first repetition's output line with the benchmark's Test field, so +// gotestspec.BuildTree attributes only that first repetition to the tree +// node. Repetitions 2..N arrive as untagged package-level output lines +// (routed by BuildTree into Package.Output) and are invisible to the tree +// walk above. FromPackages recovers them in a second pass: it scans each +// package's Output for lines matching the bench-line shape and matches +// them by their full benchmark name (e.g. "BenchmarkSuite/BenchmarkParse") +// to the Result a tree leaf already produced, appending one Sample per +// matched line. A `-count=4` run therefore produces `len(Samples)==4` per +// benchmark. Lines that don't match any known Result (e.g. stray output) +// are ignored. +func FromPackages(pkgs []*gotestspec.Package) Baseline { + results := make([]Result, 0) + index := make(map[string]int) + + for _, pkg := range pkgs { + for _, top := range pkg.Nodes { + collectBenchLeaves(pkg.Path, top, top, &results, index) + } + } + + for _, pkg := range pkgs { + harvestPackageOutputSamples(pkg.Path, pkg.Output, &results, index) + } + + sort.Slice(results, func(i, j int) bool { + a, b := results[i], results[j] + if a.Package != b.Package { + return a.Package < b.Package + } + if a.Suite != b.Suite { + return a.Suite < b.Suite + } + return a.Name < b.Name + }) + + return Baseline{ + SchemaVersion: schemaVersion, + CreatedAt: time.Now().UTC(), + GoVersion: runtime.Version(), + GOOS: runtime.GOOS, + GOARCH: runtime.GOARCH, + Results: results, + } +} + +// collectBenchLeaves recurses through n's subtree, recording a Sample for +// every KindBenchmark leaf (a benchmark node with no children). top is the +// package-level ancestor node used to derive Suite; when n is itself top +// (a bare top-level benchmark with no suite wrapper), Suite is left empty. +func collectBenchLeaves(pkgPath string, top, n *gotestspec.Node, results *[]Result, index map[string]int) { + if n.Kind == gotestspec.KindBenchmark && len(n.Children) == 0 { + // Defense in depth: a leaf can reach here with Iterations==0 if its + // bench output line was never successfully parsed (e.g. split + // across output events in a way even the joined scan in tree.go + // missed, or a malformed line). Recording it would poison Welch + // mean/variance comparisons downstream with a zero sample, so skip + // it rather than sampling a benchmark that never actually ran. + if n.Iterations == 0 { + return + } + + suite := "" + if n != top { + suite = strings.TrimPrefix(top.Name, "Benchmark") + } + + sample := Sample{ + Iterations: n.Iterations, + NsPerOp: n.NsPerOp, + BytesPerOp: n.BytesPerOp, + AllocsPerOp: n.AllocsPerOp, + } + + key := pkgPath + "\x00" + suite + "\x00" + n.Name + if idx, ok := index[key]; ok { + (*results)[idx].Samples = append((*results)[idx].Samples, sample) + return + } + index[key] = len(*results) + *results = append(*results, Result{ + Package: pkgPath, + Suite: suite, + Name: n.Name, + Samples: []Sample{sample}, + }) + return + } + + for _, c := range n.Children { + collectBenchLeaves(pkgPath, top, c, results, index) + } +} + +// packageBenchLineRe mirrors gotestspec's own benchLineRe (tree.go), except +// it additionally captures the benchmark's full name (including any +// "Suite/Method" nesting and the trailing "-" suffix it's +// stripped from), since package-level output isn't already scoped to a +// known Test the way tagged output is. +var packageBenchLineRe = regexp.MustCompile(`^(Benchmark\S+?)(?:-\d+)?\s+(\d+)\s+([\d.]+) ns/op(?:\s+(\d+) B/op)?(?:\s+(\d+) allocs/op)?`) + +// harvestPackageOutputSamples scans a package's untagged Output lines for +// -count=N repetitions 2..N (see FromPackages) and appends a Sample to the +// matching Result in results/index for each one found. Lines that don't +// match the bench-line shape, or that match a name with no corresponding +// Result (e.g. incidental package output), are skipped. +// +// Package.Output entries are individual test2json "output" event payloads, +// not guaranteed to be line-aligned: under real subprocess I/O timing a +// benchmark result line can arrive split across two consecutive events. +// Joining them back into one stream and re-splitting on "\n" reconstructs +// complete lines before matching, regardless of how test2json chunked them. +func harvestPackageOutputSamples(pkgPath string, output []string, results *[]Result, index map[string]int) { + lines := strings.Split(strings.Join(output, ""), "\n") + for _, line := range lines { + fullName, iters, nsPerOp, bPerOp, allocsPerOp, ok := parsePackageBenchLine(line) + if !ok { + continue + } + suite, name := splitBenchFullName(fullName) + key := pkgPath + "\x00" + suite + "\x00" + name + idx, ok := index[key] + if !ok { + continue + } + (*results)[idx].Samples = append((*results)[idx].Samples, Sample{ + Iterations: iters, + NsPerOp: nsPerOp, + BytesPerOp: bPerOp, + AllocsPerOp: allocsPerOp, + }) + } +} + +// parsePackageBenchLine parses a go test benchmark result line the same way +// gotestspec's parseBenchOutput does, additionally returning the +// benchmark's full name (e.g. "BenchmarkFooTestSuite/BenchmarkParse-8", +// GOMAXPROCS suffix included) captured from the line itself. +func parsePackageBenchLine(line string) (fullName string, iters int, nsPerOp float64, bPerOp, allocsPerOp int64, ok bool) { + m := packageBenchLineRe.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + return "", 0, 0, 0, 0, false + } + iters, err := strconv.Atoi(m[2]) + if err != nil { + return "", 0, 0, 0, 0, false + } + nsPerOp, err = strconv.ParseFloat(m[3], 64) + if err != nil { + return "", 0, 0, 0, 0, false + } + if m[4] != "" { + bPerOp, _ = strconv.ParseInt(m[4], 10, 64) + } + if m[5] != "" { + allocsPerOp, _ = strconv.ParseInt(m[5], 10, 64) + } + return m[1], iters, nsPerOp, bPerOp, allocsPerOp, true +} + +// splitBenchFullName splits a benchmark's full name (as captured from a +// result line, e.g. "BenchmarkFooTestSuite/BenchmarkParse") into the Suite +// and Name fields used to key a Result, mirroring collectBenchLeaves: Suite +// is the top-level segment with its "Benchmark" prefix stripped, and Name +// is the leaf's own (last) segment. A name with no "/" is a bare top-level +// benchmark, reported with an empty Suite. +func splitBenchFullName(full string) (suite, name string) { + parts := strings.Split(full, "/") + if len(parts) == 1 { + return "", parts[0] + } + return strings.TrimPrefix(parts[0], "Benchmark"), parts[len(parts)-1] +} + +// Save writes b to path as indented JSON (0644). +func Save(path string, b Baseline) error { //nolint:gocritic // hugeParam: stable API + data, err := json.MarshalIndent(b, "", " ") + if err != nil { + return fmt.Errorf("gotestbench: marshal baseline: %w", err) + } + if err := os.WriteFile(path, data, 0o600); err != nil { + return fmt.Errorf("gotestbench: write baseline: %w", err) + } + return nil +} + +// Load reads and parses a Baseline from path. It rejects any +// SchemaVersion other than the one this package writes. +func Load(path string) (Baseline, error) { + data, err := os.ReadFile(path) + if err != nil { + return Baseline{}, fmt.Errorf("gotestbench: read baseline: %w", err) + } + var b Baseline + if err := json.Unmarshal(data, &b); err != nil { + return Baseline{}, fmt.Errorf("gotestbench: parse baseline: %w", err) + } + if b.SchemaVersion != schemaVersion { + return Baseline{}, fmt.Errorf("gotestbench: unsupported schema version %d (want %d)", b.SchemaVersion, schemaVersion) + } + return b, nil +} diff --git a/internal/gotestbench/baseline_suite_test.go b/internal/gotestbench/baseline_suite_test.go new file mode 100644 index 00000000..de12611f --- /dev/null +++ b/internal/gotestbench/baseline_suite_test.go @@ -0,0 +1,197 @@ +package gotestbench_test + +import ( + "bytes" + "path/filepath" + "strings" + "time" + + "github.com/mvrahden/go-test/internal/gotestbench" + "github.com/mvrahden/go-test/internal/gotestspec" + "github.com/mvrahden/go-test/pkg/gotest" +) + +// BaselineTestSuite tests baseline extraction from spec trees and its +// JSON persistence (Save/Load round trip). +type BaselineTestSuite struct{} + +func (s *BaselineTestSuite) SuiteConfig() gotest.SuiteConfig { + return gotest.SuiteConfig{Parallel: true} +} + +// buildTree parses a go test -json stream (one JSON object per line) into a +// spec tree, mirroring how cmd/gotest builds trees from captured JSON. +func buildTree(it *gotest.T, ndjson string) []*gotestspec.Package { + events, err := gotestspec.ParseEvents(bytes.NewReader([]byte(strings.TrimSpace(ndjson)))) + gotest.NoError(it, err) + return gotestspec.BuildTree(events) +} + +func (s *BaselineTestSuite) TestFromPackages(t *gotest.T) { + t.When("a benchmark suite wrapper contains a leaf benchmark", func(w *gotest.T) { + w.It("derives Package/Suite/Name and one Sample from the leaf's fields", func(it *gotest.T) { + pkgs := buildTree(it, ` +{"Action":"run","Package":"example.com/pkg","Test":"BenchmarkFooTestSuite"} +{"Action":"run","Package":"example.com/pkg","Test":"BenchmarkFooTestSuite/BenchmarkParse"} +{"Action":"output","Package":"example.com/pkg","Test":"BenchmarkFooTestSuite/BenchmarkParse","Output":"BenchmarkFooTestSuite/BenchmarkParse-8 \t 1201 \t 985.2 ns/op \t 24 B/op \t 3 allocs/op\n"} +{"Action":"pass","Package":"example.com/pkg","Test":"BenchmarkFooTestSuite/BenchmarkParse","Elapsed":0.01} +{"Action":"pass","Package":"example.com/pkg","Test":"BenchmarkFooTestSuite","Elapsed":0.01} +`) + + b := gotestbench.FromPackages(pkgs) + + gotest.Equal(it, 1, b.SchemaVersion) + gotest.Len(it, b.Results, 1) + + r := b.Results[0] + gotest.Equal(it, "example.com/pkg", r.Package) + gotest.Equal(it, "FooTestSuite", r.Suite) + gotest.Equal(it, "BenchmarkParse", r.Name) + gotest.Len(it, r.Samples, 1) + + sample := r.Samples[0] + gotest.Equal(it, 1201, sample.Iterations) + gotest.InDelta(it, 985.2, sample.NsPerOp, 0.01) + gotest.Equal(it, int64(24), sample.BytesPerOp) + gotest.Equal(it, int64(3), sample.AllocsPerOp) + }) + }) + + t.When("a bare top-level benchmark has no enclosing suite wrapper", func(w *gotest.T) { + w.It("reports an empty Suite", func(it *gotest.T) { + pkgs := buildTree(it, ` +{"Action":"run","Package":"example.com/pkg","Test":"BenchmarkStandalone"} +{"Action":"output","Package":"example.com/pkg","Test":"BenchmarkStandalone","Output":"BenchmarkStandalone-8 \t 500 \t 100.0 ns/op\n"} +{"Action":"pass","Package":"example.com/pkg","Test":"BenchmarkStandalone","Elapsed":0.01} +`) + + b := gotestbench.FromPackages(pkgs) + + gotest.Len(it, b.Results, 1) + r := b.Results[0] + gotest.Empty(it, r.Suite) + gotest.Equal(it, "BenchmarkStandalone", r.Name) + gotest.Len(it, r.Samples, 1) + }) + }) + + t.When("no benchmark leaves are present", func(w *gotest.T) { + w.It("returns an empty Results slice", func(it *gotest.T) { + b := gotestbench.FromPackages(nil) + gotest.Empty(it, b.Results) + }) + }) + + t.When("a KindBenchmark leaf never recorded metrics (Iterations==0)", func(w *gotest.T) { + w.It("is skipped rather than poisoning the baseline with a zero sample", func(it *gotest.T) { + // A leaf can reach FromPackages with Iterations==0 when its bench + // output line was never successfully parsed (e.g. a malformed or + // truncated line). Recording it would corrupt Welch mean/variance + // comparisons downstream, so it must be dropped, not sampled. + pkgs := []*gotestspec.Package{ + { + Path: "example.com/pkg", + Nodes: []*gotestspec.Node{ + { + Name: "BenchmarkFoo", + Kind: gotestspec.KindBenchmark, + }, + }, + }, + } + + b := gotestbench.FromPackages(pkgs) + + gotest.Empty(it, b.Results) + }) + }) + + t.When("a -count=4 run leaves repetitions 2..N as package-level output", func(w *gotest.T) { + w.It("harvests all 4 repetitions into Samples on the matching Result", func(it *gotest.T) { + // Mirrors real go test -json -count=4 output: only the first + // repetition's line is tagged with the benchmark's Test field + // (and gets attributed to the tree node); repetitions 2..N + // arrive as untagged package-level "output" events carrying the + // same "Benchmark/Benchmark-" prefix. + pkgs := buildTree(it, ` +{"Action":"run","Package":"example.com/pkg","Test":"BenchmarkFooTestSuite"} +{"Action":"run","Package":"example.com/pkg","Test":"BenchmarkFooTestSuite/BenchmarkParse"} +{"Action":"output","Package":"example.com/pkg","Test":"BenchmarkFooTestSuite/BenchmarkParse","Output":"BenchmarkFooTestSuite/BenchmarkParse-8 \t 1200\t 980.0 ns/op\t 24 B/op\t 3 allocs/op\n"} +{"Action":"output","Package":"example.com/pkg","Output":"BenchmarkFooTestSuite/BenchmarkParse-8 \t 1190\t 990.0 ns/op\t 24 B/op\t 3 allocs/op\n"} +{"Action":"output","Package":"example.com/pkg","Output":"BenchmarkFooTestSuite/BenchmarkParse-8 \t 1210\t 970.0 ns/op\t 24 B/op\t 3 allocs/op\n"} +{"Action":"output","Package":"example.com/pkg","Output":"BenchmarkFooTestSuite/BenchmarkParse-8 \t 1180\t 1000.0 ns/op\t 24 B/op\t 3 allocs/op\n"} +{"Action":"output","Package":"example.com/pkg","Output":"PASS\n"} +{"Action":"pass","Package":"example.com/pkg","Elapsed":0.02} +`) + + b := gotestbench.FromPackages(pkgs) + + gotest.Len(it, b.Results, 1) + r := b.Results[0] + gotest.Equal(it, "FooTestSuite", r.Suite) + gotest.Equal(it, "BenchmarkParse", r.Name) + gotest.Len(it, r.Samples, 4) + + gotest.Equal(it, 1200, r.Samples[0].Iterations) + gotest.InDelta(it, 980.0, r.Samples[0].NsPerOp, 0.01) + gotest.Equal(it, 1190, r.Samples[1].Iterations) + gotest.InDelta(it, 990.0, r.Samples[1].NsPerOp, 0.01) + gotest.Equal(it, 1210, r.Samples[2].Iterations) + gotest.InDelta(it, 970.0, r.Samples[2].NsPerOp, 0.01) + gotest.Equal(it, 1180, r.Samples[3].Iterations) + gotest.InDelta(it, 1000.0, r.Samples[3].NsPerOp, 0.01) + }) + }) +} + +func (s *BaselineTestSuite) TestSaveLoadRoundTrip(t *gotest.T) { + t.When("a baseline is saved and reloaded", func(w *gotest.T) { + w.It("preserves the JSON structure exactly", func(it *gotest.T) { + dir := it.TempDir() + path := filepath.Join(dir, "baseline.json") + + original := gotestbench.Baseline{ + SchemaVersion: 1, + CreatedAt: time.Date(2026, 7, 26, 12, 0, 0, 0, time.UTC), + GoVersion: "go1.24.0", + GOOS: "linux", + GOARCH: "amd64", + Results: []gotestbench.Result{ + { + Package: "example.com/pkg", + Suite: "FooTestSuite", + Name: "BenchmarkParse", + Samples: []gotestbench.Sample{ + {Iterations: 1201, NsPerOp: 985.2, BytesPerOp: 24, AllocsPerOp: 3}, + {Iterations: 1180, NsPerOp: 990.1, BytesPerOp: 24, AllocsPerOp: 3}, + }, + }, + }, + } + + err := gotestbench.Save(path, original) + gotest.NoError(it, err) + + loaded, err := gotestbench.Load(path) + gotest.NoError(it, err) + + gotest.Equal(it, original.SchemaVersion, loaded.SchemaVersion) + gotest.True(it, original.CreatedAt.Equal(loaded.CreatedAt)) + gotest.Equal(it, original.GoVersion, loaded.GoVersion) + gotest.Equal(it, original.GOOS, loaded.GOOS) + gotest.Equal(it, original.GOARCH, loaded.GOARCH) + gotest.JSONEq(it, original.Results, loaded.Results) + }) + + w.It("rejects an unknown schema version", func(it *gotest.T) { + dir := it.TempDir() + path := filepath.Join(dir, "baseline.json") + + err := gotestbench.Save(path, gotestbench.Baseline{SchemaVersion: 2}) + gotest.NoError(it, err) + + _, err = gotestbench.Load(path) + gotest.Error(it, err) + }) + }) +} diff --git a/internal/gotestbench/compare.go b/internal/gotestbench/compare.go new file mode 100644 index 00000000..8ca4312a --- /dev/null +++ b/internal/gotestbench/compare.go @@ -0,0 +1,254 @@ +package gotestbench + +import ( + "fmt" + "math" + "sort" +) + +// minSamplesForTTest is the minimum sample count required on both sides of +// a comparison before Welch's t-test is trusted; below that, Compare falls +// back to a plain percent-change heuristic (see Delta.InsufficientSample). +const minSamplesForTTest = 4 + +// significanceLevel is the two-tailed p-value threshold below which a +// Welch's t-test result is considered significant. +const significanceLevel = 0.05 + +// insufficientSampleThresholdPct is the documented fallback heuristic used +// when either side has fewer than minSamplesForTTest samples: a benchmark +// is flagged as significant if its mean changed by at least this many +// percent, since a proper significance test isn't trustworthy with so few +// samples. +const insufficientSampleThresholdPct = 20.0 + +// Delta is the comparison result for one benchmark that exists in both +// baselines. The JSON tags are part of the versioned Report document (see +// report.go); renaming a tag is a report schema change. +type Delta struct { + Key string `json:"key"` // "pkg Suite/Name" + OldNs float64 `json:"oldNs"` // means + NewNs float64 `json:"newNs"` + PercentChange float64 `json:"percentChange"` + Significant bool `json:"significant"` // Welch's t-test, p < 0.05; requires >=4 samples each side + InsufficientSample bool `json:"insufficientSample"` +} + +// Compare matches results in old and new by (Package, Suite, Name) and +// returns one Delta per benchmark present in both baselines. Benchmarks +// present in only one baseline are omitted. The returned slice is sorted +// by Key for deterministic output. +func Compare(old, new Baseline) []Delta { //nolint:gocritic // hugeParam: stable API + oldIndex := make(map[string]Result, len(old.Results)) + for _, r := range old.Results { + oldIndex[resultKey(r)] = r + } + + deltas := make([]Delta, 0, len(new.Results)) + for _, nr := range new.Results { + key := resultKey(nr) + or, ok := oldIndex[key] + if !ok { + continue + } + deltas = append(deltas, compareResult(key, or, nr)) + } + + sort.Slice(deltas, func(i, j int) bool { return deltas[i].Key < deltas[j].Key }) + return deltas +} + +func resultKey(r Result) string { + return fmt.Sprintf("%s %s/%s", r.Package, r.Suite, r.Name) +} + +func compareResult(key string, old, new Result) Delta { + oldNs := nsSamples(old.Samples) + newNs := nsSamples(new.Samples) + + oldMean, _ := meanVariance(oldNs) + newMean, _ := meanVariance(newNs) + + var pctChange float64 + if oldMean != 0 { + pctChange = (newMean - oldMean) / oldMean * 100 + } + + d := Delta{ + Key: key, + OldNs: oldMean, + NewNs: newMean, + PercentChange: pctChange, + } + + if len(oldNs) < minSamplesForTTest || len(newNs) < minSamplesForTTest { + d.InsufficientSample = true + d.Significant = math.Abs(pctChange) >= insufficientSampleThresholdPct + return d + } + + p := welchPValue(oldNs, newNs) + d.Significant = p < significanceLevel + return d +} + +// WorstRegression returns the largest significant positive PercentChange +// across deltas, or 0 if none of the significant deltas are regressions +// (positive change, i.e. slower). +func WorstRegression(deltas []Delta) float64 { + worst := 0.0 + for _, d := range deltas { + if d.Significant && d.PercentChange > worst { + worst = d.PercentChange + } + } + return worst +} + +func nsSamples(samples []Sample) []float64 { + ns := make([]float64, len(samples)) + for i, s := range samples { + ns[i] = s.NsPerOp + } + return ns +} + +// meanVariance returns the sample mean and unbiased (n-1) sample variance +// of xs. Callers with len(xs) < 2 get a variance of 0. +func meanVariance(xs []float64) (mean, variance float64) { + n := float64(len(xs)) + if n == 0 { + return 0, 0 + } + var sum float64 + for _, x := range xs { + sum += x + } + mean = sum / n + if n < 2 { + return mean, 0 + } + var ss float64 + for _, x := range xs { + d := x - mean + ss += d * d + } + variance = ss / (n - 1) + return mean, variance +} + +// welchPValue computes the two-tailed p-value of Welch's t-test comparing +// the means of a and b, allowing unequal variances and sample sizes. +// +// - t statistic: (mean(a) - mean(b)) / sqrt(var(a)/n_a + var(b)/n_b) +// - degrees of freedom: Welch–Satterthwaite equation +// - p-value: the regularized incomplete beta function evaluated via +// I_x(df/2, 1/2) with x = df/(df+t^2), the standard closed form for the +// two-tailed Student's t CDF (Numerical Recipes in C, 3rd ed., §6.4). +func welchPValue(a, b []float64) float64 { + na, nb := float64(len(a)), float64(len(b)) + meanA, varA := meanVariance(a) + meanB, varB := meanVariance(b) + + seA := varA / na + seB := varB / nb + se2 := seA + seB + if se2 <= 0 { + if meanA == meanB { + return 1 + } + return 0 + } + + t := (meanA - meanB) / math.Sqrt(se2) + df := (se2 * se2) / (seA*seA/(na-1) + seB*seB/(nb-1)) + if df <= 0 || math.IsNaN(df) { + return 1 + } + + x := df / (df + t*t) + return incompleteBeta(df/2, 0.5, x) +} + +// incompleteBeta returns the regularized incomplete beta function I_x(a, b) +// via the continued-fraction expansion, following Numerical Recipes in C, +// 3rd ed., §6.4 ("Incomplete Beta Function") equations 6.4.1-6.4.6: the +// continued fraction converges quickly for x < (a+1)/(a+b+2), and the +// symmetry relation I_x(a,b) = 1 - I_{1-x}(b,a) is used otherwise. +func incompleteBeta(a, b, x float64) float64 { + if x <= 0 { + return 0 + } + if x >= 1 { + return 1 + } + + lnBeta, _ := math.Lgamma(a + b) + lnA, _ := math.Lgamma(a) + lnB, _ := math.Lgamma(b) + lnBt := lnBeta - lnA - lnB + a*math.Log(x) + b*math.Log(1-x) + bt := math.Exp(lnBt) + + if x < (a+1)/(a+b+2) { + return bt * betaContinuedFraction(a, b, x) / a + } + return 1 - bt*betaContinuedFraction(b, a, 1-x)/b +} + +// betaContinuedFraction evaluates the continued fraction in the incomplete +// beta function via the modified Lentz algorithm (Numerical Recipes in C, +// 3rd ed., §6.4, function betacf). +func betaContinuedFraction(a, b, x float64) float64 { + const ( + maxIterations = 200 + epsilon = 3e-7 + tiny = 1e-30 + ) + + qab := a + b + qap := a + 1 + qam := a - 1 + + c := 1.0 + d := 1 - qab*x/qap + if math.Abs(d) < tiny { + d = tiny + } + d = 1 / d + h := d + + for m := 1; m <= maxIterations; m++ { + m2 := float64(2 * m) + + aa := float64(m) * (b - float64(m)) * x / ((qam + m2) * (a + m2)) + d = 1 + aa*d + if math.Abs(d) < tiny { + d = tiny + } + c = 1 + aa/c + if math.Abs(c) < tiny { + c = tiny + } + d = 1 / d + h *= d * c + + aa = -(a + float64(m)) * (qab + float64(m)) * x / ((a + m2) * (qap + m2)) + d = 1 + aa*d + if math.Abs(d) < tiny { + d = tiny + } + c = 1 + aa/c + if math.Abs(c) < tiny { + c = tiny + } + d = 1 / d + del := d * c + h *= del + + if math.Abs(del-1) < epsilon { + break + } + } + + return h +} diff --git a/internal/gotestbench/compare_suite_test.go b/internal/gotestbench/compare_suite_test.go new file mode 100644 index 00000000..867fe6fb --- /dev/null +++ b/internal/gotestbench/compare_suite_test.go @@ -0,0 +1,122 @@ +package gotestbench_test + +import ( + "github.com/mvrahden/go-test/internal/gotestbench" + "github.com/mvrahden/go-test/pkg/gotest" +) + +// CompareTestSuite tests Welch's t-test significance comparison between two +// baselines. +type CompareTestSuite struct{} + +func (s *CompareTestSuite) SuiteConfig() gotest.SuiteConfig { + return gotest.SuiteConfig{Parallel: true} +} + +func mkResult(pkg, suite, name string, ns []float64) gotestbench.Result { + samples := make([]gotestbench.Sample, len(ns)) + for i, v := range ns { + samples[i] = gotestbench.Sample{Iterations: 1000, NsPerOp: v} + } + return gotestbench.Result{Package: pkg, Suite: suite, Name: name, Samples: samples} +} + +func (s *CompareTestSuite) TestCompare(t *gotest.T) { + t.When("both sides have >=4 samples from the same distribution", func(w *gotest.T) { + w.It("is not significant", func(it *gotest.T) { + old := gotestbench.Baseline{Results: []gotestbench.Result{ + mkResult("pkg", "FooTestSuite", "BenchmarkParse", []float64{98, 102, 99, 101, 100, 103}), + }} + new_ := gotestbench.Baseline{Results: []gotestbench.Result{ + mkResult("pkg", "FooTestSuite", "BenchmarkParse", []float64{101, 97, 100, 102, 99, 100}), + }} + + deltas := gotestbench.Compare(old, new_) + gotest.Len(it, deltas, 1) + d := deltas[0] + gotest.Equal(it, "pkg FooTestSuite/BenchmarkParse", d.Key) + gotest.False(it, d.InsufficientSample) + gotest.False(it, d.Significant) + }) + }) + + t.When("the new side is shifted +30% with low variance across 6 samples", func(w *gotest.T) { + w.It("is significant with PercentChange near 30", func(it *gotest.T) { + old := gotestbench.Baseline{Results: []gotestbench.Result{ + mkResult("pkg", "FooTestSuite", "BenchmarkParse", []float64{99, 101, 100, 98, 102, 100}), + }} + new_ := gotestbench.Baseline{Results: []gotestbench.Result{ + mkResult("pkg", "FooTestSuite", "BenchmarkParse", []float64{128.7, 131.3, 130.0, 127.4, 132.6, 130.0}), + }} + + deltas := gotestbench.Compare(old, new_) + gotest.Len(it, deltas, 1) + d := deltas[0] + gotest.False(it, d.InsufficientSample) + gotest.True(it, d.Significant) + gotest.InDelta(it, 30.0, d.PercentChange, 2.0) + }) + }) + + t.When("either side has fewer than 4 samples", func(w *gotest.T) { + w.It("falls back to the 20%% heuristic and marks InsufficientSample", func(it *gotest.T) { + old := gotestbench.Baseline{Results: []gotestbench.Result{ + mkResult("pkg", "FooTestSuite", "BenchmarkParse", []float64{100}), + }} + bigShift := gotestbench.Baseline{Results: []gotestbench.Result{ + mkResult("pkg", "FooTestSuite", "BenchmarkParse", []float64{131}), + }} + smallShift := gotestbench.Baseline{Results: []gotestbench.Result{ + mkResult("pkg", "FooTestSuite", "BenchmarkParse", []float64{105}), + }} + + dBig := gotestbench.Compare(old, bigShift) + gotest.Len(it, dBig, 1) + gotest.True(it, dBig[0].InsufficientSample) + gotest.True(it, dBig[0].Significant) + gotest.InDelta(it, 31.0, dBig[0].PercentChange, 0.5) + + dSmall := gotestbench.Compare(old, smallShift) + gotest.Len(it, dSmall, 1) + gotest.True(it, dSmall[0].InsufficientSample) + gotest.False(it, dSmall[0].Significant) + }) + }) + + t.When("a benchmark exists only in one baseline", func(w *gotest.T) { + w.It("is omitted from the deltas", func(it *gotest.T) { + old := gotestbench.Baseline{Results: []gotestbench.Result{ + mkResult("pkg", "FooTestSuite", "BenchmarkOld", []float64{100, 100, 100, 100}), + }} + new_ := gotestbench.Baseline{Results: []gotestbench.Result{ + mkResult("pkg", "FooTestSuite", "BenchmarkNew", []float64{100, 100, 100, 100}), + }} + + deltas := gotestbench.Compare(old, new_) + gotest.Empty(it, deltas) + }) + }) +} + +func (s *CompareTestSuite) TestWorstRegression(t *gotest.T) { + t.When("multiple deltas include significant and insignificant regressions", func(w *gotest.T) { + w.It("returns the max significant positive PercentChange", func(it *gotest.T) { + deltas := []gotestbench.Delta{ + {Key: "a", PercentChange: 5, Significant: false}, + {Key: "b", PercentChange: 40, Significant: true}, + {Key: "c", PercentChange: 15, Significant: true}, + {Key: "d", PercentChange: -50, Significant: true}, + } + gotest.InDelta(it, 40.0, gotestbench.WorstRegression(deltas), 0.001) + }) + }) + + t.When("no deltas are significant", func(w *gotest.T) { + w.It("returns 0", func(it *gotest.T) { + deltas := []gotestbench.Delta{ + {Key: "a", PercentChange: 90, Significant: false}, + } + gotest.Equal(it, 0.0, gotestbench.WorstRegression(deltas)) + }) + }) +} diff --git a/internal/gotestbench/report.go b/internal/gotestbench/report.go new file mode 100644 index 00000000..8b10980b --- /dev/null +++ b/internal/gotestbench/report.go @@ -0,0 +1,74 @@ +package gotestbench + +import "encoding/json" + +// reportSchemaVersion is the version stamped on every Report this package +// writes. It is independent of the Baseline schemaVersion: the report is the +// machine-readable output of one `gotest bench --json` run, the baseline is +// the persisted comparison anchor. +const reportSchemaVersion = 1 + +// Report is the versioned JSON document `gotest bench --json` emits: the +// run's results in baseline shape, plus — when a comparison ran — its deltas +// and the gate verdict. Consumers (the VS Code extension) read this instead +// of scraping text, so every field is contract: additions are fine, renames +// and removals need a schema bump. +type Report struct { + SchemaVersion int `json:"schemaVersion"` + Baseline Baseline `json:"baseline"` + Deltas []Delta `json:"deltas,omitempty"` + Gate *Gate `json:"gate,omitempty"` +} + +// Gate is the report's gate verdict: the configured threshold, the worst +// significant regression the comparison found, and whether it breached. +// WorstKey is empty when no significant regression exists. BreachedKeys +// lists every delta the gate rule condemns (significant AND above the +// threshold) so consumers can mark each offender without re-deriving the +// rule — the rule lives here, nowhere else. +type Gate struct { + ThresholdPct float64 `json:"thresholdPct"` + WorstPct float64 `json:"worstPct"` + WorstKey string `json:"worstKey,omitempty"` + Breached bool `json:"breached"` + BreachedKeys []string `json:"breachedKeys,omitempty"` +} + +// NewReport assembles the document for one run. deltas may be nil when no +// comparison ran; gate may be nil when no gate is active. +func NewReport(b Baseline, deltas []Delta, gate *Gate) Report { //nolint:gocritic // hugeParam: stable API + return Report{ + SchemaVersion: reportSchemaVersion, + Baseline: b, + Deltas: deltas, + Gate: gate, + } +} + +// GateVerdict evaluates deltas against thresholdPct: the worst significant +// positive PercentChange, the Key it belongs to, and whether it exceeds the +// threshold. This is the single source for both the report's gate object and +// the CLI's failure message. +func GateVerdict(deltas []Delta, thresholdPct float64) Gate { + g := Gate{ThresholdPct: thresholdPct} + for _, d := range deltas { + if !d.Significant { + continue + } + if d.PercentChange > g.WorstPct { + g.WorstPct = d.PercentChange + g.WorstKey = d.Key + } + if d.PercentChange > thresholdPct { + g.BreachedKeys = append(g.BreachedKeys, d.Key) + } + } + g.Breached = g.WorstPct > thresholdPct + return g +} + +// MarshalReport renders r as indented JSON, the exact bytes `--json` writes +// to stdout. +func MarshalReport(r Report) ([]byte, error) { //nolint:gocritic // hugeParam: stable API + return json.MarshalIndent(r, "", " ") +} diff --git a/internal/gotestbench/report_suite_test.go b/internal/gotestbench/report_suite_test.go new file mode 100644 index 00000000..cbe7411e --- /dev/null +++ b/internal/gotestbench/report_suite_test.go @@ -0,0 +1,107 @@ +package gotestbench_test + +import ( + "encoding/json" + + "github.com/mvrahden/go-test/internal/gotestbench" + "github.com/mvrahden/go-test/pkg/gotest" +) + +// ReportTestSuite tests the versioned `gotest bench --json` document: its +// schema stamp, its delta/gate presence rules, and the gate verdict math. +type ReportTestSuite struct{} + +func (s *ReportTestSuite) SuiteConfig() gotest.SuiteConfig { + return gotest.SuiteConfig{Parallel: true} +} + +func (s *ReportTestSuite) TestNewReport(t *gotest.T) { + base := gotestbench.Baseline{Results: []gotestbench.Result{ + mkResult("pkg", "CacheTestSuite", "BenchmarkGetHit", []float64{100}), + }} + + t.When("no comparison ran", func(w *gotest.T) { + report := gotestbench.NewReport(base, nil, nil) + + w.It("stamps schema version 1", func(it *gotest.T) { + gotest.Equal(it, 1, report.SchemaVersion) + }) + + w.It("omits deltas and gate from the JSON entirely", func(it *gotest.T) { + data, err := gotestbench.MarshalReport(report) + gotest.NoError(it, err) + gotest.NotContains(it, string(data), `"deltas"`) + gotest.NotContains(it, string(data), `"gate"`) + gotest.Contains(it, string(data), `"baseline"`) + }) + }) + + t.When("a comparison and gate ran", func(w *gotest.T) { + deltas := []gotestbench.Delta{ + {Key: "pkg CacheTestSuite/BenchmarkGetHit", OldNs: 100, NewNs: 112.3, PercentChange: 12.3, Significant: true}, + } + gate := gotestbench.GateVerdict(deltas, 5) + report := gotestbench.NewReport(base, deltas, &gate) + + w.It("round-trips deltas with their contract field names", func(it *gotest.T) { + data, err := gotestbench.MarshalReport(report) + gotest.NoError(it, err) + gotest.Contains(it, string(data), `"percentChange": 12.3`) + gotest.Contains(it, string(data), `"significant": true`) + + var parsed gotestbench.Report + gotest.NoError(it, json.Unmarshal(data, &parsed)) + gotest.Equal(it, report.Deltas, parsed.Deltas) + gotest.Equal(it, *report.Gate, *parsed.Gate) + }) + }) +} + +func (s *ReportTestSuite) TestGateVerdict(t *gotest.T) { + t.When("a significant regression exceeds the threshold", func(w *gotest.T) { + verdict := gotestbench.GateVerdict([]gotestbench.Delta{ + {Key: "pkg A/B", PercentChange: 3, Significant: true}, + {Key: "pkg C/D", PercentChange: 12.3, Significant: true}, + {Key: "pkg E/F", PercentChange: 40, Significant: false}, + }, 5) + + w.It("reports the worst significant regression and breaches", func(it *gotest.T) { + gotest.Equal(it, 12.3, verdict.WorstPct) + gotest.Equal(it, "pkg C/D", verdict.WorstKey) + gotest.True(it, verdict.Breached) + }) + + w.It("lists exactly the significant deltas above the threshold as breached", func(it *gotest.T) { + gotest.Equal(it, []string{"pkg C/D"}, verdict.BreachedKeys) + }) + + w.It("never lets an insignificant delta drive the verdict", func(it *gotest.T) { + gotest.NotEqual(it, "pkg E/F", verdict.WorstKey) + gotest.NotContains(it, verdict.BreachedKeys, "pkg E/F") + }) + }) + + t.When("regressions stay under the threshold", func(w *gotest.T) { + verdict := gotestbench.GateVerdict([]gotestbench.Delta{ + {Key: "pkg A/B", PercentChange: 3, Significant: true}, + }, 5) + + w.It("does not breach", func(it *gotest.T) { + gotest.False(it, verdict.Breached) + gotest.Equal(it, 3.0, verdict.WorstPct) + gotest.Empty(it, verdict.BreachedKeys) + }) + }) + + t.When("only improvements exist", func(w *gotest.T) { + verdict := gotestbench.GateVerdict([]gotestbench.Delta{ + {Key: "pkg A/B", PercentChange: -8.1, Significant: true}, + }, 5) + + w.It("reports a zero worst regression with no key", func(it *gotest.T) { + gotest.Zero(it, verdict.WorstPct) + gotest.Zero(it, verdict.WorstKey) + gotest.False(it, verdict.Breached) + }) + }) +} diff --git a/internal/gotestgen/collector_suite_test.go b/internal/gotestgen/collector_suite_test.go index bafd1f6b..0d4adab0 100644 --- a/internal/gotestgen/collector_suite_test.go +++ b/internal/gotestgen/collector_suite_test.go @@ -801,6 +801,44 @@ func (s *CollectorTestSuite) TestValidation(t *gotest.T) { }) } +func (s *CollectorTestSuite) TestBenchmarkMethod(t *gotest.T) { + t.When("suite has benchmark methods", func(w *gotest.T) { + w.It("classifies benchmark methods and applies X_ exclusion", func(it *gotest.T) { + pkg := gotestgen.ExportMustTestPkg(it.T(), "TestCollector_BenchmarkMethod") + c := gotestgen.NewCollector() + result := c.CollectSuiteSpecs(pkg) + gotest.Empty(it, result.Errs, "expected no errors, got: %v", result.Errs) + + spec := gotest.Must(c.ApplyTestSuiteSpecs(result)) + gotest.Len(it, spec.EffectiveTestSuites, 1) + + suite := spec.EffectiveTestSuites[0] + gotest.Len(it, suite.Benchmarks(), 1) + gotest.Equal(it, "BenchmarkParse", suite.Benchmarks()[0].Identifier()) + }) + }) + + t.When("benchmark method has an unsupported signature", func(w *gotest.T) { + w.It("rejects benchmark methods without *gotest.B/*testing.B", func(it *gotest.T) { + pkg := gotestgen.ExportMustTestPkg(it.T(), "TestCollector_BenchmarkMethod_BadSignature") + c := gotestgen.NewCollector() + result := c.CollectSuiteSpecs(pkg) + gotest.NotEmpty(it, result.Errs, "expected error for unsupported benchmark param type") + gotest.ErrorContains(it, result.Errs[0].Err, "must accept exactly one parameter of type *gotest.B or *testing.B") + }) + }) + + t.When("suite has benchmarks and a returning BeforeEach", func(w *gotest.T) { + w.It("rejects benchmarks on returning-BeforeEach suites", func(it *gotest.T) { + pkg := gotestgen.ExportMustTestPkg(it.T(), "TestCollector_BenchmarkMethod_ReturningBeforeEach") + c := gotestgen.NewCollector() + result := c.CollectSuiteSpecs(pkg) + gotest.NotEmpty(it, result.Errs, "expected error: benchmarks with returning BeforeEach") + gotest.ErrorContains(it, result.Errs[0].Err, "move benchmarks to a dedicated suite") + }) + }) +} + func (s *CollectorTestSuite) TestApplyTestSuiteSpecs(t *gotest.T) { t.When("valid result with fixtures only", func(w *gotest.T) { w.It("returns no suites", func(it *gotest.T) { diff --git a/internal/gotestgen/generator.go b/internal/gotestgen/generator.go index f8661045..12115c39 100644 --- a/internal/gotestgen/generator.go +++ b/internal/gotestgen/generator.go @@ -20,6 +20,7 @@ type GenerateResult struct { PTest []byte // generated internal test source PXTest []byte // generated external test source SuiteNames []string // suite struct identifiers (e.g. "FooTestSuite") + BenchSuiteNames []string // suite struct identifiers with >=1 effective benchmark method SkippedSuiteNames []string // identifiers of suites excluded by focus/X_ rules ExclusiveSuiteNames []string // identifiers of suites with SuiteConfig{Exclusive: true} — dispatched alone, after the parallel bulk FixtureDepSuites []string // test function names that depend on shared fixtures (e.g. "TestFooSuite") @@ -316,6 +317,29 @@ func generateFromLoaded(loadResults []*LoadResult) (GenerateResults, []SharedFix } } + benchSeen := map[string]bool{} + var benchNames []string + for _, s := range ptestSpec.EffectiveTestSuites { + if len(s.Benchmarks()) == 0 { + continue + } + id := s.Identifier() + if !benchSeen[id] { + benchSeen[id] = true + benchNames = append(benchNames, id) + } + } + for _, s := range pxtestSpec.EffectiveTestSuites { + if len(s.Benchmarks()) == 0 { + continue + } + id := s.Identifier() + if !benchSeen[id] { + benchSeen[id] = true + benchNames = append(benchNames, id) + } + } + var skippedNames []string for _, s := range ptestSpec.SkippedTestSuites { id := s.Identifier() @@ -346,6 +370,7 @@ func generateFromLoaded(loadResults []*LoadResult) (GenerateResults, []SharedFix PTest: ptestBuf, PXTest: pxtestBuf, SuiteNames: suiteNames, + BenchSuiteNames: benchNames, SkippedSuiteNames: skippedNames, ExclusiveSuiteNames: exclusiveNames, FixtureDepSuites: append(ptestFixtureDeps, pxtestFixtureDeps...), diff --git a/internal/gotestgen/generator_suite_test.go b/internal/gotestgen/generator_suite_test.go index 49c794ed..4676c64b 100644 --- a/internal/gotestgen/generator_suite_test.go +++ b/internal/gotestgen/generator_suite_test.go @@ -58,6 +58,35 @@ func (s *GeneratorTestSuite) TestE2ECLI(t *gotest.T) { }) } +func (s *GeneratorTestSuite) TestGenerateFromLoaded_BenchSuiteNames(t *gotest.T) { + t.When("a suite has effective benchmark methods", func(w *gotest.T) { + w.It("includes the suite identifier in BenchSuiteNames", func(it *gotest.T) { + pkg := gotestgen.ExportMustTestPkg(it.T(), "TestCollector_BenchmarkMethod") + loaded := []*gotestgen.LoadResult{ + {PkgPath: pkg.PkgPath, PkgDir: "/fake/dir", Ptest: pkg}, + } + results, _, err := gotestgen.GenerateFromLoaded(loaded) + gotest.NoError(it, err) + gotest.Len(it, results, 1) + gotest.Contains(it, results[0].SuiteNames, "BenchTestSuite") + gotest.Contains(it, results[0].BenchSuiteNames, "BenchTestSuite") + }) + }) + + t.When("a suite has no benchmark methods", func(w *gotest.T) { + w.It("excludes the suite identifier from BenchSuiteNames", func(it *gotest.T) { + pkg := gotestgen.ExportMustTestPkg(it.T(), "TestCollector_SuiteGuard_Detected") + loaded := []*gotestgen.LoadResult{ + {PkgPath: pkg.PkgPath, PkgDir: "/fake/dir", Ptest: pkg}, + } + results, _, err := gotestgen.GenerateFromLoaded(loaded) + gotest.NoError(it, err) + gotest.Len(it, results, 1) + gotest.Empty(it, results[0].BenchSuiteNames) + }) + }) +} + func (s *GeneratorTestSuite) TestE2ENoTestSuites(t *gotest.T) { t.When("packages without test suites", func(w *gotest.T) { for sub, tC := range gotest.Each(w, []struct { diff --git a/internal/gotestgen/renderer.go b/internal/gotestgen/renderer.go index 2bde07fc..b98e3c46 100644 --- a/internal/gotestgen/renderer.go +++ b/internal/gotestgen/renderer.go @@ -107,6 +107,10 @@ func (r renderer) RenderTestSuiteSpec(pkg *packages.Package, spec SpecOutcome, r } } + if err := r.renderBenchSuites(buf, spec, resolved.SuiteSharedFixtures, allFixtures, resolved.SuiteFixtureFields); err != nil { + return nil, fmt.Errorf("failed rendering benchmark suites. err: %w", err) + } + return r.formatOutput(buf) } @@ -189,6 +193,21 @@ func (r *renderer) renderTestSuites(buf *bytes.Buffer, spec SpecOutcome, suiteSh }) } +func (r *renderer) renderBenchSuites(buf *bytes.Buffer, spec SpecOutcome, suiteSharedFixtures map[string][]SharedFixtureRef, allFixtures []*ResolvedFixture, suiteFixtureFields map[string][]FixtureFieldBinding) error { //nolint:gocritic // hugeParam: stable API + // Reuse the exact same fixture-bound view model gotest.fixture.tpl renders + // Test from, reshaped as a map for O(1) per-suite template lookup + // (mirroring how SuiteSharedFixtures is already passed as a lookup map). + suiteFixtures := make(map[string]*FlatFixtureSuite) + for _, fs := range flattenSuitesDAG(allFixtures, suiteFixtureFields) { + suiteFixtures[fs.Suite.Identifier()] = &fs + } + return gotestTpl.ExecuteTemplate(buf, "gotest.bench.tpl", map[string]any{ + "Spec": spec, + "SuiteSharedFixtures": suiteSharedFixtures, + "SuiteFixtures": suiteFixtures, + }) +} + func (r *renderer) renderFixtures(buf *bytes.Buffer, fixtureBound []*gotestast.TestSuiteSpec, allFixtures []*ResolvedFixture, suiteFixtureFields map[string][]FixtureFieldBinding, sfNodes []*SharedFixtureNodeVM, fixtureTestNames []string) error { if len(allFixtures) == 0 && len(sfNodes) == 0 { return nil diff --git a/internal/gotestgen/renderer_suite_test.go b/internal/gotestgen/renderer_suite_test.go index 29a6e22f..45d2118d 100644 --- a/internal/gotestgen/renderer_suite_test.go +++ b/internal/gotestgen/renderer_suite_test.go @@ -488,3 +488,53 @@ func (s *RendererTestSuite) TestDeterministicOutput(t *gotest.T) { } }) } + +// --- Benchmark wrapper rendering tests --- + +func (s *RendererTestSuite) TestRenderer_BenchmarkWrapper(t *gotest.T) { + t.It("emits Benchmark with lifecycle fencing", func(it *gotest.T) { + pkg := gotestgen.ExportMustTestPkg(it.T(), "TestCollector_BenchmarkMethod") + out, _ := renderTestPkg(it.T(), pkg) + + gotest.Contains(it, out, "func BenchmarkBenchTestSuite(b *testing.B)") + gotest.Contains(it, out, `b.Run("BenchmarkParse"`) + gotest.Contains(it, out, "b.StopTimer()") + gotest.Contains(it, out, "s.BeforeEach(ƒeachT)") + gotest.NotContains(it, out, "X_BenchmarkOld") + }) + + t.It("never emits NewTWithDeadline — benchmarks are bounded by -benchtime, not deadlines", func(it *gotest.T) { + pkg := gotestgen.ExportMustTestPkg(it.T(), "TestCollector_BenchmarkMethod") + out, _ := renderTestPkg(it.T(), pkg) + + idx := strings.Index(out, "func BenchmarkBenchTestSuite") + gotest.GreaterOrEqual(it, idx, 0, "bench wrapper missing from output") + benchFn := out[idx:] + gotest.NotContains(it, benchFn, "NewTWithDeadline", "bench wrapper must not apply a suite-config deadline") + }) + + t.When("benchmark method takes *testing.B directly", func(w *gotest.T) { + w.It("dispatches with the raw *testing.B instead of gotest.NewB(b)", func(it *gotest.T) { + pkg := gotestgen.ExportMustTestPkg(it.T(), "TestRenderer_Benchmark_StdlibB") + out, _ := renderTestPkg(it.T(), pkg) + + gotest.Contains(it, out, "func BenchmarkStdlibBenchTestSuite(b *testing.B)") + gotest.Contains(it, out, "s.BenchmarkRaw(b)") + gotest.NotContains(it, out, "s.BenchmarkRaw(gotest.NewB(b))") + }) + }) + + t.When("benchmark suite is bound to a package fixture", func(w *gotest.T) { + w.It("calls ƒ_setupFixtures and constructs the suite with fixture fields populated", func(it *gotest.T) { + pkg := gotestgen.ExportMustTestPkg(it.T(), "TestRenderer_FixtureBoundBenchmark") + out, _ := renderTestPkg(it.T(), pkg) + + idx := strings.Index(out, "func BenchmarkParserTestSuite") + gotest.GreaterOrEqual(it, idx, 0, "bench wrapper missing from output") + benchFn := out[idx:] + gotest.Contains(it, benchFn, "ƒ_setupFixtures(b)") + gotest.Contains(it, benchFn, "ParserTestSuite: ParserTestSuite{") + gotest.Contains(it, benchFn, "PoolFixture: ƒ_PoolFixture") + }) + }) +} diff --git a/internal/gotestgen/resolver.go b/internal/gotestgen/resolver.go index 626cc2fd..81f230b6 100644 --- a/internal/gotestgen/resolver.go +++ b/internal/gotestgen/resolver.go @@ -119,6 +119,14 @@ func Resolve(targetPkg *packages.Package, suites []*gotestast.TestSuiteSpec, loc return nil, err } if len(fixtures) > 0 { + if len(suite.Benchmarks()) > 0 { + for _, fm := range fixtures { + if bad := findHookedFixture(fm.resolved); bad != nil { + return nil, fmt.Errorf("suite %s has benchmark methods but fixture %s defines BeforeEach/AfterEach — per-method fixture hooks are not supported for benchmarks", suite.Identifier(), bad.Identifier) + } + } + } + if result.SuiteFixtureFields == nil { result.SuiteFixtureFields = make(map[string][]FixtureFieldBinding) } @@ -232,6 +240,27 @@ func Resolve(targetPkg *packages.Package, suites []*gotestast.TestSuiteSpec, loc return result, nil } +// findHookedFixture walks a fixture and its transitive parents looking for one +// that defines BeforeEach/AfterEach. Per-method fixture hooks assume a fresh +// invocation per test case; benchmarks run their body in a tight b.Loop(), so +// wiring fixture BeforeEach/AfterEach around each benchmark method is out of +// scope for now (see docs/design/bench-fuzz.md Part 1) — reject it at +// resolve-time instead of generating code that silently ignores the hooks. +func findHookedFixture(rf *ResolvedFixture) *ResolvedFixture { + if rf == nil { + return nil + } + if rf.BeforeEach || rf.AfterEach { + return rf + } + for _, p := range rf.Parents { + if bad := findHookedFixture(p); bad != nil { + return bad + } + } + return nil +} + func hasChildSuitesRecursive(rf *ResolvedFixture) bool { if len(rf.ChildSuites) > 0 { return true diff --git a/internal/gotestgen/resolver_suite_test.go b/internal/gotestgen/resolver_suite_test.go index 41570f92..ef923ebc 100644 --- a/internal/gotestgen/resolver_suite_test.go +++ b/internal/gotestgen/resolver_suite_test.go @@ -449,6 +449,21 @@ func (s *ResolverTestSuite) TestResolutionErrors(t *gotest.T) { gotest.ErrorContains(it, err, "channel") }) }) + + t.When("benchmark suite is bound to a fixture that defines BeforeEach", func(w *gotest.T) { + w.It("rejects with a per-method fixture hooks error", func(it *gotest.T) { + pkg := gotestgen.ExportMustTestPkg(it.T(), "TestResolve_Benchmark_FixtureBeforeEachRejected") + c := gotestgen.NewCollector() + result := c.CollectSuiteSpecs(pkg) + gotest.Empty(it, result.Errs) + + spec, err := c.ApplyTestSuiteSpecs(result) + gotest.NoError(it, err) + + _, err = gotestgen.Resolve(pkg, spec.EffectiveTestSuites, result.Fixtures) + gotest.ErrorContains(it, err, "WorkerTestSuite has benchmark methods but fixture HookedFixture defines BeforeEach/AfterEach") + }) + }) } func (s *ResolverTestSuite) TestMixedFieldStylesSameFixture(t *gotest.T) { diff --git a/internal/gotestgen/static/gotest.bench.tpl b/internal/gotestgen/static/gotest.bench.tpl new file mode 100644 index 00000000..dd2c5441 --- /dev/null +++ b/internal/gotestgen/static/gotest.bench.tpl @@ -0,0 +1,48 @@ +{{ range $i, $ts := .Spec.EffectiveTestSuites }} +{{- if $ts.Benchmarks }} +func Benchmark{{ $ts.Identifier }}(b *testing.B) { +{{- $fx := index $.SuiteFixtures $ts.Identifier }} +{{- $sfRefs := index $.SuiteSharedFixtures $ts.Identifier }} +{{- if or $fx $sfRefs }} + ƒ_setupFixtures(b) +{{- end }} +{{- if $fx }} + s := &ƒƒ_GOTEST_{{ $ts.Identifier }}{ + {{ $ts.Identifier }}: {{ $ts.Identifier }}{ +{{- range $id, $field := $fx.FixtureFields }} + {{ $field }}: ƒ_{{ $id }}, +{{- end }} + }, + } +{{- else }} + s := &ƒƒ_GOTEST_{{ $ts.Identifier }}{} +{{- end }} +{{- if $ts.HasGuard }} + if ƒreason := s.{{ $ts.Identifier }}.SuiteGuard(); ƒreason != "" { + b.Skipf("suite guard: %s", ƒreason) + return + } +{{- end }} +{{- if not $fx }} +{{- range $sf := $sfRefs }} + s.{{ $sf.FieldName }} = ƒ_sf_{{ $sf.Identifier }} +{{- end }} +{{- end }} + ƒlifecycleT := gotest.NewTFromTB(b) + b.Cleanup(func() { s.AfterAll(gotest.NewTFromTB(b)) }) + s.BeforeAll(ƒlifecycleT) +{{ range $bm := $ts.Benchmarks }} + b.Run("{{ $bm.Identifier }}", func(b *testing.B) { + b.StopTimer() + ƒeachT := gotest.NewTFromTB(b) + s.BeforeEach(ƒeachT) + b.StartTimer() + b.ResetTimer() + s.{{ $bm.Identifier }}({{ if $bm.UsesStdlibT }}b{{ else }}gotest.NewB(b){{ end }}) + b.StopTimer() + s.AfterEach(ƒeachT) + }) +{{ end }} +} +{{- end }} +{{- end }} diff --git a/internal/gotestgen/static/gotest.fixture.tpl b/internal/gotestgen/static/gotest.fixture.tpl index c9a71097..be5290d9 100644 --- a/internal/gotestgen/static/gotest.fixture.tpl +++ b/internal/gotestgen/static/gotest.fixture.tpl @@ -41,7 +41,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { {{- /* Each config is derived exactly once, but inside ƒ_fixtureOnce.Do rather than at diff --git a/internal/gotestgen/testdata/__snapshots__/TestGeneratorTestSuite_ext.snap b/internal/gotestgen/testdata/__snapshots__/TestGeneratorTestSuite_ext.snap index 8eecb55e..b0b2917e 100644 --- a/internal/gotestgen/testdata/__snapshots__/TestGeneratorTestSuite_ext.snap +++ b/internal/gotestgen/testdata/__snapshots__/TestGeneratorTestSuite_ext.snap @@ -34,7 +34,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒcfg_AppFixture = (&AppFixture{}).FixtureConfig() ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) @@ -189,7 +189,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -618,7 +618,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration diff --git a/internal/gotestgen/testdata/__snapshots__/TestRendererTestSuite_ext.snap b/internal/gotestgen/testdata/__snapshots__/TestRendererTestSuite_ext.snap index 3a0391a6..51bd49b8 100644 --- a/internal/gotestgen/testdata/__snapshots__/TestRendererTestSuite_ext.snap +++ b/internal/gotestgen/testdata/__snapshots__/TestRendererTestSuite_ext.snap @@ -37,7 +37,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -108,6 +108,77 @@ func TestQueryTestSuite(t *testing.T) { } +=== SNAP TestBeforeEachRendering/returning_BeforeEach_parallel/renders_parallel_markers_and_WaitGroup === +// Code generated by "gotest (github.com/mvrahden/go-test)"; DO NOT EDIT. + +package testpkg + +import ( + "github.com/mvrahden/go-test/pkg/gotest" + "sync" + "sync/atomic" + "testing" +) + +//go:noinline +func ƒƒ_GOTEST_exec(fn gotest.TestCase, t *gotest.T) { fn(t) } + +type ƒƒ_GOTEST_OrderTestSuite struct { + OrderTestSuite +} + +func (ts *ƒƒ_GOTEST_OrderTestSuite) BeforeAll(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_OrderTestSuite) AfterAll(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_OrderTestSuite) BeforeEach(it *gotest.T) *myCtx { + return ts.OrderTestSuite.BeforeEach(it) +} +func (ts *ƒƒ_GOTEST_OrderTestSuite) AfterEach(it *gotest.T, ctx *myCtx) { + ts.OrderTestSuite.AfterEach(it, ctx) +} + +func TestOrderTestSuite(t *testing.T) { + s := &ƒƒ_GOTEST_OrderTestSuite{} + ƒcfg := s.OrderTestSuite.SuiteConfig() + wg := &sync.WaitGroup{} + ƒfailed := &atomic.Bool{} + + ƒsetupT := gotest.NewT(t) + if ƒcfg.SetupTimeout > 0 { + ƒsetupT = gotest.NewTWithDeadline(t, ƒcfg.SetupTimeout) + } + t.Cleanup(func() { + wg.Wait() + ƒteardownT := gotest.NewT(t) + if ƒcfg.SetupTimeout > 0 { + ƒteardownT = gotest.NewTWithDeadline(t, ƒcfg.SetupTimeout) + } + s.AfterAll(ƒteardownT) + }) + s.BeforeAll(ƒsetupT) + + t.Run("TestOne", func(it *testing.T) { + wg.Add(1) + it.Parallel() + defer wg.Done() + if ƒcfg.FailFast && ƒfailed.Load() { + it.Skip("FailFast: earlier test failed") + } + defer func() { + if it.Failed() { + ƒfailed.Store(true) + } + }() + ttt := gotest.NewT(it) + if ƒcfg.Timeout > 0 { + ttt = gotest.NewTWithDeadline(it, ƒcfg.Timeout) + } + ctx := s.BeforeEach(ttt) + defer s.AfterEach(ttt, ctx) + s.TestOne(ttt, ctx) + }) + +} + === SNAP TestBeforeEachRendering/returning_BeforeEach_parallel/renders_parallel_markers_without_a_WaitGroup === // Code generated by "gotest (github.com/mvrahden/go-test)"; DO NOT EDIT. @@ -277,6 +348,127 @@ func TestOrderTestSuite(t *testing.T) { } +=== SNAP TestFixtureConfig/fixture_with_config/renders_config_overlay_in_fixture_node === +// Code generated by "gotest (github.com/mvrahden/go-test)"; DO NOT EDIT. + +package testpkg + +import ( + "context" + "github.com/mvrahden/go-test/pkg/gotest" + "github.com/mvrahden/go-test/pkg/gotestruntime" + "sync/atomic" + "testing" + "time" +) + +//go:noinline +func ƒƒ_GOTEST_exec(fn gotest.TestCase, t *gotest.T) { fn(t) } + +type ƒƒ_GOTEST_CFGTestSuite struct { + CFGTestSuite +} + +func (ts *ƒƒ_GOTEST_CFGTestSuite) BeforeAll(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_CFGTestSuite) AfterAll(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_CFGTestSuite) BeforeEach(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_CFGTestSuite) AfterEach(it *gotest.T) {} + +var ƒ_CFGFixture *CFGFixture + +var ƒ_fixtureOnce gotestruntime.FixtureOnce +var ƒ_fixtureDAG *gotestruntime.FixtureDAG +var ƒ_fixtureTestNames = []string{ + "CFGTestSuite", +} +var ƒ_pending atomic.Int32 + +func ƒ_setupFixtures(t testing.TB) { + if err := ƒ_fixtureOnce.Do(func() error { + ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) + var ƒmaxSuiteSetup time.Duration + + { + ƒscfg := gotest.DefaultSuiteConfig() + if ƒscfg.SetupTimeout > ƒmaxSuiteSetup { + ƒmaxSuiteSetup = ƒscfg.SetupTimeout + } + } + + var err error + ƒ_fixtureDAG, err = gotestruntime.SetupFixtureDAG(context.Background(), gotestruntime.MainConfig{ + Fixtures: []*gotestruntime.FixtureNode{ + { + Name: "CFGFixture", + Config: func() gotest.FixtureConfig { + cfg := gotest.DefaultFixtureConfig() + gotest.OverlayFixtureConfig(&cfg, (&CFGFixture{}).FixtureConfig()) + return cfg + }(), + Init: func() { + ƒ_CFGFixture = &CFGFixture{} + }, + BeforeAll: func(ctx context.Context) error { + return ƒ_CFGFixture.BeforeAll(ctx) + }, + AfterAll: func(ctx context.Context) error { + return ƒ_CFGFixture.AfterAll(ctx) + }, + }, + }, + MaxSuiteSetupTimeout: ƒmaxSuiteSetup, + }) + return err + }); err != nil { + t.Fatalf("fixture setup: %v", err) + } + t.Cleanup(func() { + if ƒ_pending.Add(-1) == 0 { + if ƒ_fixtureDAG.Teardown() { + t.Errorf("fixture teardown failed") + } + } + }) +} + +func TestCFGTestSuite(t *testing.T) { + ƒ_setupFixtures(t) + + s := &ƒƒ_GOTEST_CFGTestSuite{ + CFGTestSuite: CFGTestSuite{ + CFGFixture: ƒ_CFGFixture, + }, + } + ƒcfg := gotest.DefaultSuiteConfig() + + ƒsetupT := gotest.NewT(t) + if ƒcfg.SetupTimeout > 0 { + ƒsetupT = gotest.NewTWithDeadline(t, ƒcfg.SetupTimeout) + } + t.Cleanup(func() { + ƒteardownT := gotest.NewT(t) + if ƒcfg.SetupTimeout > 0 { + ƒteardownT = gotest.NewTWithDeadline(t, ƒcfg.SetupTimeout) + } + s.AfterAll(ƒteardownT) + }) + s.BeforeAll(ƒsetupT) + + t.Run("TestOne", func(it *testing.T) { + ttt := gotest.NewT(it) + if ƒcfg.Timeout > 0 { + ttt = gotest.NewTWithDeadline(it, ƒcfg.Timeout) + } + defer s.AfterEach(ttt) + s.BeforeEach(ttt) + ƒƒ_GOTEST_exec(s.TestOne, ttt) + }) + if ƒcfg.FailFast && t.Failed() { + return + } + +} + === SNAP TestFixtureConfig/fixture_with_config/uses_the_marker's_config_verbatim_in_the_fixture_node === // Code generated by "gotest (github.com/mvrahden/go-test)"; DO NOT EDIT. @@ -313,7 +505,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒcfg_CFGFixture = (&CFGFixture{}).FixtureConfig() ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) @@ -424,7 +616,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -495,6 +687,120 @@ func TestPlainTestSuite(t *testing.T) { } +=== SNAP TestFixtureConfig/fixture_without_config/uses_default_config_without_overlay === +// Code generated by "gotest (github.com/mvrahden/go-test)"; DO NOT EDIT. + +package testpkg + +import ( + "context" + "github.com/mvrahden/go-test/pkg/gotest" + "github.com/mvrahden/go-test/pkg/gotestruntime" + "sync/atomic" + "testing" + "time" +) + +//go:noinline +func ƒƒ_GOTEST_exec(fn gotest.TestCase, t *gotest.T) { fn(t) } + +type ƒƒ_GOTEST_PlainTestSuite struct { + PlainTestSuite +} + +func (ts *ƒƒ_GOTEST_PlainTestSuite) BeforeAll(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_PlainTestSuite) AfterAll(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_PlainTestSuite) BeforeEach(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_PlainTestSuite) AfterEach(it *gotest.T) {} + +var ƒ_PlainFixture *PlainFixture + +var ƒ_fixtureOnce gotestruntime.FixtureOnce +var ƒ_fixtureDAG *gotestruntime.FixtureDAG +var ƒ_fixtureTestNames = []string{ + "PlainTestSuite", +} +var ƒ_pending atomic.Int32 + +func ƒ_setupFixtures(t testing.TB) { + if err := ƒ_fixtureOnce.Do(func() error { + ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) + var ƒmaxSuiteSetup time.Duration + + { + ƒscfg := gotest.DefaultSuiteConfig() + if ƒscfg.SetupTimeout > ƒmaxSuiteSetup { + ƒmaxSuiteSetup = ƒscfg.SetupTimeout + } + } + + var err error + ƒ_fixtureDAG, err = gotestruntime.SetupFixtureDAG(context.Background(), gotestruntime.MainConfig{ + Fixtures: []*gotestruntime.FixtureNode{ + { + Name: "PlainFixture", + Config: gotest.DefaultFixtureConfig(), + Init: func() { + ƒ_PlainFixture = &PlainFixture{} + }, + BeforeAll: func(ctx context.Context) error { + return ƒ_PlainFixture.BeforeAll(ctx) + }, + }, + }, + MaxSuiteSetupTimeout: ƒmaxSuiteSetup, + }) + return err + }); err != nil { + t.Fatalf("fixture setup: %v", err) + } + t.Cleanup(func() { + if ƒ_pending.Add(-1) == 0 { + if ƒ_fixtureDAG.Teardown() { + t.Errorf("fixture teardown failed") + } + } + }) +} + +func TestPlainTestSuite(t *testing.T) { + ƒ_setupFixtures(t) + + s := &ƒƒ_GOTEST_PlainTestSuite{ + PlainTestSuite: PlainTestSuite{ + PlainFixture: ƒ_PlainFixture, + }, + } + ƒcfg := gotest.DefaultSuiteConfig() + + ƒsetupT := gotest.NewT(t) + if ƒcfg.SetupTimeout > 0 { + ƒsetupT = gotest.NewTWithDeadline(t, ƒcfg.SetupTimeout) + } + t.Cleanup(func() { + ƒteardownT := gotest.NewT(t) + if ƒcfg.SetupTimeout > 0 { + ƒteardownT = gotest.NewTWithDeadline(t, ƒcfg.SetupTimeout) + } + s.AfterAll(ƒteardownT) + }) + s.BeforeAll(ƒsetupT) + + t.Run("TestOne", func(it *testing.T) { + ttt := gotest.NewT(it) + if ƒcfg.Timeout > 0 { + ttt = gotest.NewTWithDeadline(it, ƒcfg.Timeout) + } + defer s.AfterEach(ttt) + s.BeforeEach(ttt) + ƒƒ_GOTEST_exec(s.TestOne, ttt) + }) + if ƒcfg.FailFast && t.Failed() { + return + } + +} + === SNAP TestFixtureRendering/fixture_with_BeforeEach/AfterEach/renders_lifecycle_methods_with_proper_ordering === // Code generated by "gotest (github.com/mvrahden/go-test)"; DO NOT EDIT. @@ -530,7 +836,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -647,7 +953,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -768,7 +1074,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -874,7 +1180,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -980,7 +1286,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -1121,7 +1427,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -1268,7 +1574,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -1423,7 +1729,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -1583,7 +1889,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -1702,7 +2008,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -1802,7 +2108,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -1921,7 +2227,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -2045,7 +2351,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -2164,7 +2470,7 @@ var ƒ_fixtureTestNames = []string{ } var ƒ_pending atomic.Int32 -func ƒ_setupFixtures(t *testing.T) { +func ƒ_setupFixtures(t testing.TB) { if err := ƒ_fixtureOnce.Do(func() error { ƒ_pending.Store(int32(gotestruntime.CountMatchingTests(ƒ_fixtureTestNames))) var ƒmaxSuiteSetup time.Duration @@ -2353,6 +2659,61 @@ func TestPlainTestSuite(t *testing.T) { } +=== SNAP TestSuiteConfig/suite_with_config/renders_config_overlay_and_deadline === +// Code generated by "gotest (github.com/mvrahden/go-test)"; DO NOT EDIT. + +package testpkg + +import ( + "github.com/mvrahden/go-test/pkg/gotest" + "testing" +) + +//go:noinline +func ƒƒ_GOTEST_exec(fn gotest.TestCase, t *gotest.T) { fn(t) } + +type ƒƒ_GOTEST_ConfiguredTestSuite struct { + ConfiguredTestSuite +} + +func (ts *ƒƒ_GOTEST_ConfiguredTestSuite) BeforeAll(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_ConfiguredTestSuite) AfterAll(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_ConfiguredTestSuite) BeforeEach(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_ConfiguredTestSuite) AfterEach(it *gotest.T) {} + +func TestConfiguredTestSuite(t *testing.T) { + s := &ƒƒ_GOTEST_ConfiguredTestSuite{} + ƒcfg := gotest.DefaultSuiteConfig() + gotest.OverlaySuiteConfig(&ƒcfg, s.ConfiguredTestSuite.SuiteConfig()) + + ƒsetupT := gotest.NewT(t) + if ƒcfg.SetupTimeout > 0 { + ƒsetupT = gotest.NewTWithDeadline(t, ƒcfg.SetupTimeout) + } + t.Cleanup(func() { + ƒteardownT := gotest.NewT(t) + if ƒcfg.SetupTimeout > 0 { + ƒteardownT = gotest.NewTWithDeadline(t, ƒcfg.SetupTimeout) + } + s.AfterAll(ƒteardownT) + }) + s.BeforeAll(ƒsetupT) + + t.Run("TestOne", func(it *testing.T) { + ttt := gotest.NewT(it) + if ƒcfg.Timeout > 0 { + ttt = gotest.NewTWithDeadline(it, ƒcfg.Timeout) + } + defer s.AfterEach(ttt) + s.BeforeEach(ttt) + ƒƒ_GOTEST_exec(s.TestOne, ttt) + }) + if ƒcfg.FailFast && t.Failed() { + return + } + +} + === SNAP TestSuiteConfig/suite_with_config/uses_the_marker's_config_verbatim_and_renders_the_deadline === // Code generated by "gotest (github.com/mvrahden/go-test)"; DO NOT EDIT. @@ -2447,3 +2808,57 @@ func TestPlainTestSuite(t *testing.T) { } +=== SNAP TestSuiteConfig/suite_without_config/uses_default_config_without_overlay === +// Code generated by "gotest (github.com/mvrahden/go-test)"; DO NOT EDIT. + +package testpkg + +import ( + "github.com/mvrahden/go-test/pkg/gotest" + "testing" +) + +//go:noinline +func ƒƒ_GOTEST_exec(fn gotest.TestCase, t *gotest.T) { fn(t) } + +type ƒƒ_GOTEST_PlainTestSuite struct { + PlainTestSuite +} + +func (ts *ƒƒ_GOTEST_PlainTestSuite) BeforeAll(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_PlainTestSuite) AfterAll(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_PlainTestSuite) BeforeEach(it *gotest.T) {} +func (ts *ƒƒ_GOTEST_PlainTestSuite) AfterEach(it *gotest.T) {} + +func TestPlainTestSuite(t *testing.T) { + s := &ƒƒ_GOTEST_PlainTestSuite{} + ƒcfg := gotest.DefaultSuiteConfig() + + ƒsetupT := gotest.NewT(t) + if ƒcfg.SetupTimeout > 0 { + ƒsetupT = gotest.NewTWithDeadline(t, ƒcfg.SetupTimeout) + } + t.Cleanup(func() { + ƒteardownT := gotest.NewT(t) + if ƒcfg.SetupTimeout > 0 { + ƒteardownT = gotest.NewTWithDeadline(t, ƒcfg.SetupTimeout) + } + s.AfterAll(ƒteardownT) + }) + s.BeforeAll(ƒsetupT) + + t.Run("TestOne", func(it *testing.T) { + ttt := gotest.NewT(it) + if ƒcfg.Timeout > 0 { + ttt = gotest.NewTWithDeadline(it, ƒcfg.Timeout) + } + defer s.AfterEach(ttt) + s.BeforeEach(ttt) + ƒƒ_GOTEST_exec(s.TestOne, ttt) + }) + if ƒcfg.FailFast && t.Failed() { + return + } + +} + diff --git a/internal/gotestgen/testdata/sources/TestCollector_BenchmarkMethod/test.go b/internal/gotestgen/testdata/sources/TestCollector_BenchmarkMethod/test.go new file mode 100644 index 00000000..5585be0c --- /dev/null +++ b/internal/gotestgen/testdata/sources/TestCollector_BenchmarkMethod/test.go @@ -0,0 +1,16 @@ +package testpkg + +import "github.com/mvrahden/go-test/pkg/gotest" + +type BenchTestSuite struct{} + +func (s *BenchTestSuite) BeforeEach(t *gotest.T) {} +func (s *BenchTestSuite) TestOne(t *gotest.T) {} +func (s *BenchTestSuite) BenchmarkParse(b *gotest.B) { + for b.Loop() { + } +} +func (s *BenchTestSuite) X_BenchmarkOld(b *gotest.B) { + for b.Loop() { + } +} diff --git a/internal/gotestgen/testdata/sources/TestCollector_BenchmarkMethod_BadSignature/test.go b/internal/gotestgen/testdata/sources/TestCollector_BenchmarkMethod_BadSignature/test.go new file mode 100644 index 00000000..818c60bf --- /dev/null +++ b/internal/gotestgen/testdata/sources/TestCollector_BenchmarkMethod_BadSignature/test.go @@ -0,0 +1,7 @@ +package testpkg + +import "github.com/mvrahden/go-test/pkg/gotest" + +type BadBenchTestSuite struct{} + +func (s *BadBenchTestSuite) BenchmarkBad(t *gotest.T) {} diff --git a/internal/gotestgen/testdata/sources/TestCollector_BenchmarkMethod_ReturningBeforeEach/test.go b/internal/gotestgen/testdata/sources/TestCollector_BenchmarkMethod_ReturningBeforeEach/test.go new file mode 100644 index 00000000..4d9ed41e --- /dev/null +++ b/internal/gotestgen/testdata/sources/TestCollector_BenchmarkMethod_ReturningBeforeEach/test.go @@ -0,0 +1,14 @@ +package testpkg + +import "github.com/mvrahden/go-test/pkg/gotest" + +type benchCtx struct{ val string } + +type ReturningBenchTestSuite struct{} + +func (s *ReturningBenchTestSuite) BeforeEach(t *gotest.T) *benchCtx { return &benchCtx{} } +func (s *ReturningBenchTestSuite) TestOne(t *gotest.T, ctx *benchCtx) {} +func (s *ReturningBenchTestSuite) BenchmarkParse(b *gotest.B) { + for b.Loop() { + } +} diff --git a/internal/gotestgen/testdata/sources/TestRenderer_Benchmark_StdlibB/test.go b/internal/gotestgen/testdata/sources/TestRenderer_Benchmark_StdlibB/test.go new file mode 100644 index 00000000..ec48797c --- /dev/null +++ b/internal/gotestgen/testdata/sources/TestRenderer_Benchmark_StdlibB/test.go @@ -0,0 +1,10 @@ +package testpkg + +import "testing" + +type StdlibBenchTestSuite struct{} + +func (s *StdlibBenchTestSuite) BenchmarkRaw(b *testing.B) { + for b.Loop() { + } +} diff --git a/internal/gotestgen/testdata/sources/TestRenderer_FixtureBoundBenchmark/test.go b/internal/gotestgen/testdata/sources/TestRenderer_FixtureBoundBenchmark/test.go new file mode 100644 index 00000000..ec8ba755 --- /dev/null +++ b/internal/gotestgen/testdata/sources/TestRenderer_FixtureBoundBenchmark/test.go @@ -0,0 +1,23 @@ +package testpkg + +import ( + "context" + + "github.com/mvrahden/go-test/pkg/gotest" +) + +type PoolFixture struct { + Pool string +} + +func (f *PoolFixture) BeforeAll(ctx context.Context) error { return nil } +func (f *PoolFixture) AfterAll(ctx context.Context) error { return nil } + +type ParserTestSuite struct { + *PoolFixture +} + +func (s *ParserTestSuite) BenchmarkParse(b *gotest.B) { + for b.Loop() { + } +} diff --git a/internal/gotestgen/testdata/sources/TestResolve_Benchmark_FixtureBeforeEachRejected/test.go b/internal/gotestgen/testdata/sources/TestResolve_Benchmark_FixtureBeforeEachRejected/test.go new file mode 100644 index 00000000..b551518a --- /dev/null +++ b/internal/gotestgen/testdata/sources/TestResolve_Benchmark_FixtureBeforeEachRejected/test.go @@ -0,0 +1,22 @@ +package testpkg + +import ( + "context" + + "github.com/mvrahden/go-test/pkg/gotest" +) + +type HookedFixture struct{} + +func (f *HookedFixture) BeforeAll(ctx context.Context) error { return nil } +func (f *HookedFixture) AfterAll(ctx context.Context) error { return nil } +func (f *HookedFixture) BeforeEach(ctx context.Context) error { return nil } + +type WorkerTestSuite struct { + *HookedFixture +} + +func (s *WorkerTestSuite) BenchmarkWork(b *gotest.B) { + for b.Loop() { + } +} diff --git a/internal/gotestrunner/args.go b/internal/gotestrunner/args.go index 5b79dc9e..cea2fbd8 100644 --- a/internal/gotestrunner/args.go +++ b/internal/gotestrunner/args.go @@ -249,6 +249,12 @@ func ExtractRunFilter(runFlags []string) string { return extractFlag(runFlags, " // StripRunFilter removes -run and its value from run flags. func StripRunFilter(runFlags []string) []string { return stripFlag(runFlags, "-run") } +// ExtractBenchFilter returns the value of -bench from run flags, if present. +func ExtractBenchFilter(runFlags []string) string { return extractFlag(runFlags, "-bench") } + +// StripBenchFilter removes -bench and its value from run flags. +func StripBenchFilter(runFlags []string) []string { return stripFlag(runFlags, "-bench") } + // ExtractCoverProfile returns the value of -coverprofile from run flags, if present. func ExtractCoverProfile(runFlags []string) string { return extractFlag(runFlags, "-coverprofile") } diff --git a/internal/gotestrunner/export_test.go b/internal/gotestrunner/export_test.go index b4852bdd..a0fd4219 100644 --- a/internal/gotestrunner/export_test.go +++ b/internal/gotestrunner/export_test.go @@ -28,6 +28,7 @@ var ExportOverlayContentHash = overlayContentHash var ExportCacheRoot = cacheRoot var ExportFilterPackageLevelEvents = filterPackageLevelEvents var ExportIsPackageSummaryLine = protocol.IsPackageSummaryLine +var ResolveBenchParallelismForTest = resolveMaxParallel // ExportProcessPID and ExportProcessDone let the teardown tests observe the // shared fixture subprocess directly: whether it is still alive, and when it is @@ -46,7 +47,7 @@ func ExportSetTeardownTimeout(p *SharedFixtureProcess, d time.Duration) { p.teardownTimeout = d } -func ExportAutoDetectCI(cfg PipelineConfig) PipelineConfig { +func ExportAutoDetectCI(cfg PipelineConfig) PipelineConfig { //nolint:gocritic // hugeParam: stable API if !cfg.CI && os.Getenv(protocol.EnvCI) == "" && os.Getenv("CI") != "" { cfg.CI = true } @@ -70,9 +71,10 @@ func ExportNewSharedFixtureProcess(sharedDir string, state map[string]json.RawMe type ExportFixtureWindows = fixtureWindows var ExportPlanFixtureWindows = planFixtureWindows +var ExportPlanBenchFixtureWindows = planBenchFixtureWindows +var ExportBenchSlotPlan = benchSlotPlan var ExportPlanSuitePhases = planSuitePhases var ExportAliveFixtureKeys = aliveFixtureKeys var ExportSortTargetIndices = sortTargetIndices var ExportLogSlowBuild = logSlowBuild -var ExportComputeDispatchConcurrency = computeDispatchConcurrency diff --git a/internal/gotestrunner/fixturewindow.go b/internal/gotestrunner/fixturewindow.go index f20b6f71..5c417cb0 100644 --- a/internal/gotestrunner/fixturewindow.go +++ b/internal/gotestrunner/fixturewindow.go @@ -146,3 +146,97 @@ func aliveFixtureKeys(phaseSuites map[string][]string, reqKeys map[string]map[st func sharedFixtureKey(sf *gotestgen.SharedFixtureInfo) string { return sf.PkgPath + "." + sf.Identifier } + +// Bench windows: bench dispatch is strictly serial, so every slot is its own +// phase. A fixture is resident from the first slot that needs it until the +// last — opened by StartKeys just before the first, released by TeardownKeys +// right after the last, never restarted in between. + +// planBenchFixtureWindows computes the residency plan for a bench run. The +// selection mirrors BuildBenchTargets minus compile results: -run and -bench +// are matched against Benchmark, not the test function names the +// default plan uses. Bulk is the first planned slot's needs — started +// up-front, concurrent with compile; every other fixture rides deferred and +// opens with its slot. +func planBenchFixtureWindows(overlay *OverlayResult, userRunFilter, userBenchFilter string) fixtureWindows { + type slot struct { + pkg string + suite string + } + var slots []slot + for pkg, suites := range overlay.BenchesByPkg { + for _, suiteName := range suites { + benchFuncName := "Benchmark" + suiteName + if userRunFilter != "" && !matchesSuiteFunc(userRunFilter, benchFuncName) { + continue + } + if userBenchFilter != "" && !anyBranchMatchesSuiteFunc(userBenchFilter, benchFuncName) { + continue + } + slots = append(slots, slot{pkg: pkg, suite: suiteName}) + } + } + // Dispatch order: the same deterministic (Package, SuiteName) order the + // serial dispatcher applies. + sort.Slice(slots, func(a, b int) bool { + if slots[a].pkg != slots[b].pkg { + return slots[a].pkg < slots[b].pkg + } + return slots[a].suite < slots[b].suite + }) + + all := map[string][]string{} + for _, sl := range slots { + all[sl.pkg] = append(all[sl.pkg], "Test"+sl.suite) + } + w := fixtureWindows{ + Bulk: map[string]bool{}, + Tail: map[string]bool{}, + } + if len(slots) > 0 { + first := map[string][]string{slots[0].pkg: {"Test" + slots[0].suite}} + w.Bulk = aliveFixtureKeys(first, overlay.SuiteRequiredSharedFixtureKeys, overlay.SharedFixtures) + } + alive := aliveFixtureKeys(all, overlay.SuiteRequiredSharedFixtureKeys, overlay.SharedFixtures) + for i := range overlay.SharedFixtures { + key := sharedFixtureKey(&overlay.SharedFixtures[i]) + switch { + case w.Bulk[key]: + w.Fixtures = append(w.Fixtures, overlay.SharedFixtures[i]) + case alive[key]: + sf := overlay.SharedFixtures[i] + sf.Deferred = true + w.Fixtures = append(w.Fixtures, sf) + default: + w.Skipped++ + } + } + return w +} + +// benchSlotPlan computes each slot's needed keys and the union of every later +// slot's needs, for bench targets already in dispatch order. laterNeeds has +// len(targets)+1 entries; the final one is empty — after the last slot, +// nothing is needed. +func benchSlotPlan(targets []SuiteTarget, reqKeys map[string]map[string][]string, fixtures []gotestgen.SharedFixtureInfo) (needs, laterNeeds []map[string]bool) { + needs = make([]map[string]bool, len(targets)) + for i := range targets { + // Bench targets carry the bare suite identifier; required keys are + // keyed by test function name. + phase := map[string][]string{targets[i].Package: {"Test" + targets[i].SuiteName}} + needs[i] = aliveFixtureKeys(phase, reqKeys, fixtures) + } + laterNeeds = make([]map[string]bool, len(targets)+1) + laterNeeds[len(targets)] = map[string]bool{} + for i := len(targets) - 1; i >= 0; i-- { + later := make(map[string]bool, len(laterNeeds[i+1])+len(needs[i])) + for k := range laterNeeds[i+1] { + later[k] = true + } + for k := range needs[i] { + later[k] = true + } + laterNeeds[i] = later + } + return needs, laterNeeds +} diff --git a/internal/gotestrunner/fixturewindow_suite_test.go b/internal/gotestrunner/fixturewindow_suite_test.go index ca4e11ee..d5cf2c10 100644 --- a/internal/gotestrunner/fixturewindow_suite_test.go +++ b/internal/gotestrunner/fixturewindow_suite_test.go @@ -186,3 +186,88 @@ func (s *FixtureWindowTestSuite) TestRealOverlayFiltering(t *gotest.T) { }) }) } + +// benchOverlay extends the synthetic overlay with bench suites: +// +// pkg/a: AlphaSuite (bench, needs Alpha) MultiSuite (bench, needs Alpha+Beta) +// pkg/b: ChainSuite (bench, needs Chain → Alpha) +// +// Orphan stays required by nobody. +func benchOverlay() *gotestrunner.OverlayResult { + overlay := windowOverlay() + overlay.BenchesByPkg = map[string][]string{ + "pkg/a": {"AlphaSuite", "MultiSuite"}, + "pkg/b": {"ChainSuite"}, + } + return overlay +} + +func (s *FixtureWindowTestSuite) TestBenchWindowPlanning(t *gotest.T) { + overlay := benchOverlay() + + t.When("no filters are set", func(w *gotest.T) { + win := gotestrunner.ExportPlanBenchFixtureWindows(overlay, "", "") + + w.It("starts only the first slot's fixtures up-front and defers the rest", func(it *gotest.T) { + // Dispatch order: pkg/a AlphaSuite, pkg/a MultiSuite, pkg/b ChainSuite. + gotest.Equal(it, map[string]bool{winKey("Alpha"): true}, win.Bulk) + deferred := map[string]bool{} + for i := range win.Fixtures { + deferred[win.Fixtures[i].Identifier] = win.Fixtures[i].Deferred + } + gotest.Equal(it, map[string]bool{"Alpha": false, "Beta": true, "Chain": true}, deferred) + }) + + w.It("never starts the fixture no bench suite requires", func(it *gotest.T) { + gotest.Equal(it, 1, win.Skipped, "Orphan is not in the plan") + }) + }) + + t.When("-bench selects only the chain suite", func(w *gotest.T) { + win := gotestrunner.ExportPlanBenchFixtureWindows(overlay, "", "BenchmarkChainSuite") + + w.It("keeps its DAG-closed needs and nothing else, all up-front", func(it *gotest.T) { + gotest.Equal(it, map[string]bool{winKey("Chain"): true, winKey("Alpha"): true}, win.Bulk, + "the one slot is the first slot: its closure starts with compile") + gotest.ElementsMatch(it, []string{"Alpha", "Chain"}, fixtureIdentifiers(win.Fixtures)) + gotest.Equal(it, 2, win.Skipped) + }) + }) + + t.When("-run matches Benchmark names, not test names", func(w *gotest.T) { + win := gotestrunner.ExportPlanBenchFixtureWindows(overlay, "^BenchmarkMultiSuite$", "") + + w.It("plans for the bench suites the filter selects", func(it *gotest.T) { + gotest.Equal(it, map[string]bool{winKey("Alpha"): true, winKey("Beta"): true}, win.Bulk) + gotest.Equal(it, 2, win.Skipped) + }) + }) +} + +func (s *FixtureWindowTestSuite) TestBenchSlotPlan(t *gotest.T) { + overlay := benchOverlay() + targets := []gotestrunner.SuiteTarget{ + {SuiteSpec: gotestrunner.SuiteSpec{Package: "pkg/a", SuiteName: "AlphaSuite"}}, + {SuiteSpec: gotestrunner.SuiteSpec{Package: "pkg/a", SuiteName: "MultiSuite"}}, + {SuiteSpec: gotestrunner.SuiteSpec{Package: "pkg/b", SuiteName: "ChainSuite"}}, + } + + t.When("computing per-slot windows", func(w *gotest.T) { + needs, laterNeeds := gotestrunner.ExportBenchSlotPlan(targets, overlay.SuiteRequiredSharedFixtureKeys, overlay.SharedFixtures) + + w.It("resolves each slot's DAG-closed needs", func(it *gotest.T) { + gotest.Equal(it, map[string]bool{winKey("Alpha"): true}, needs[0]) + gotest.Equal(it, map[string]bool{winKey("Alpha"): true, winKey("Beta"): true}, needs[1]) + gotest.Equal(it, map[string]bool{winKey("Chain"): true, winKey("Alpha"): true}, needs[2]) + }) + + w.It("keeps a fixture resident until its last slot, then releases it", func(it *gotest.T) { + // Alpha is needed by every slot: it must survive slot 1 even + // though slot 1 also needs Beta — resident through the whole run. + gotest.True(it, laterNeeds[1][winKey("Alpha")]) + gotest.True(it, laterNeeds[2][winKey("Alpha")], "Chain's closure keeps Alpha alive through the last slot") + gotest.False(it, laterNeeds[2][winKey("Beta")], "Beta's window closes after its only slot") + gotest.Empty(it, laterNeeds[3], "after the final slot nothing is needed") + }) + }) +} diff --git a/internal/gotestrunner/gotestrunner_suite_test.go b/internal/gotestrunner/gotestrunner_suite_test.go index 6afa785f..97b72b6f 100644 --- a/internal/gotestrunner/gotestrunner_suite_test.go +++ b/internal/gotestrunner/gotestrunner_suite_test.go @@ -787,9 +787,177 @@ func (s *GotestrunnerTestSuite) TestBuildSuiteCmd(t *gotest.T) { gotest.Equal(it, goPath, cmd.Path) }) }) + + t.When("bench mode", func(w *gotest.T) { + ctx := context.Background() + + w.It("targets Benchmark and disables tests", func(it *gotest.T) { + target := gotestrunner.SuiteTarget{ + SuiteSpec: gotestrunner.SuiteSpec{SuiteName: "BenchTestSuite", Package: "example.com/p", Dir: it.TempDir()}, + BinaryPath: "/tmp/bin.test", + Bench: true, + } + cmd := gotestrunner.ExportBuildSuiteCmd(ctx, target, nil, false) + gotest.Contains(it, cmd.Args, "-test.run=^$") + gotest.Contains(it, cmd.Args, "-test.bench=^BenchmarkBenchTestSuite$") + gotest.Contains(it, cmd.Args, "-test.benchmem") + }) + + w.It("quotes regex-special characters in the suite name", func(it *gotest.T) { + target := gotestrunner.SuiteTarget{ + SuiteSpec: gotestrunner.SuiteSpec{SuiteName: "Foo.Bar+Baz", Package: "example.com/p"}, + BinaryPath: "/tmp/bin.test", + Bench: true, + } + cmd := gotestrunner.ExportBuildSuiteCmd(ctx, target, nil, false) + gotest.Contains(it, cmd.Args, `-test.bench=^BenchmarkFoo\.Bar\+Baz$`) + }) + + w.It("does not append -test.benchmem when the user already passed one", func(it *gotest.T) { + target := gotestrunner.SuiteTarget{ + SuiteSpec: gotestrunner.SuiteSpec{SuiteName: "BenchTestSuite", Package: "example.com/p"}, + BinaryPath: "/tmp/bin.test", + Bench: true, + RunFlags: []string{"-test.benchmem", "-test.benchtime=2x"}, + } + cmd := gotestrunner.ExportBuildSuiteCmd(ctx, target, nil, false) + count := 0 + for _, a := range cmd.Args { + if a == "-test.benchmem" { + count++ + } + } + gotest.Equal(it, 1, count, "expected exactly one -test.benchmem, got args: %v", cmd.Args) + gotest.Contains(it, cmd.Args, "-test.benchtime=2x") + }) + + w.It("ignores RunFilter when Bench is set", func(it *gotest.T) { + target := gotestrunner.SuiteTarget{ + SuiteSpec: gotestrunner.SuiteSpec{SuiteName: "BenchTestSuite", Package: "example.com/p", RunFilter: "^TestFoo$/^Bar$"}, + BinaryPath: "/tmp/bin.test", + Bench: true, + } + cmd := gotestrunner.ExportBuildSuiteCmd(ctx, target, nil, false) + gotest.Contains(it, cmd.Args, "-test.run=^$") + gotest.NotContains(it, cmd.Args, "-test.run=^TestFoo$/^Bar$") + }) + }) } -// --- OutputCollector tests --- +func (s *GotestrunnerTestSuite) TestBuildBenchTargets(t *gotest.T) { + compiled := []gotestrunner.CompileResult{ + {Package: "example.com/pkg", BinaryPath: "/tmp/pkg.test"}, + } + dirsByPkg := map[string]string{"example.com/pkg": "/src/pkg"} + + t.When("building targets from benchesByPkg", func(w *gotest.T) { + w.It("builds a Bench target per suite with the bare suite name", func(it *gotest.T) { + benchesByPkg := map[string][]string{"example.com/pkg": {"BenchTestSuite"}} + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, nil, "", "") + gotest.Len(it, targets, 1) + gotest.True(it, targets[0].Bench) + gotest.Equal(it, "BenchTestSuite", targets[0].SuiteName) + gotest.Equal(it, "example.com/pkg", targets[0].Package) + gotest.Equal(it, "/src/pkg", targets[0].Dir) + gotest.Equal(it, "/tmp/pkg.test", targets[0].BinaryPath) + }) + + w.It("skips packages with no compiled binary", func(it *gotest.T) { + benchesByPkg := map[string][]string{"example.com/other": {"BenchTestSuite"}} + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, nil, "", "") + gotest.Empty(it, targets) + }) + + w.It("filters by userRunFilter (-run) matching Benchmark", func(it *gotest.T) { + benchesByPkg := map[string][]string{"example.com/pkg": {"BenchTestSuite", "OtherSuite"}} + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, nil, "^BenchmarkBenchTestSuite$", "") + gotest.Len(it, targets, 1) + gotest.Equal(it, "BenchTestSuite", targets[0].SuiteName) + }) + + w.It("filters by userBenchFilter (-bench) matching Benchmark", func(it *gotest.T) { + benchesByPkg := map[string][]string{"example.com/pkg": {"BenchTestSuite", "OtherSuite"}} + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, nil, "", "OtherSuite") + gotest.Len(it, targets, 1) + gotest.Equal(it, "OtherSuite", targets[0].SuiteName) + }) + + w.It("AND-composes -run and -bench when both are set", func(it *gotest.T) { + benchesByPkg := map[string][]string{"example.com/pkg": {"Foo", "Bar", "Baz"}} + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, nil, "^Benchmark(Foo|Bar)$", "Foo") + gotest.Len(it, targets, 1) + gotest.Equal(it, "Foo", targets[0].SuiteName) + }) + + w.It("includes all targets when neither filter is set", func(it *gotest.T) { + benchesByPkg := map[string][]string{"example.com/pkg": {"Foo", "Bar", "Baz"}} + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, nil, "", "") + gotest.Len(it, targets, 3) + }) + + w.It("translates run flags to -test. prefixed form", func(it *gotest.T) { + benchesByPkg := map[string][]string{"example.com/pkg": {"BenchTestSuite"}} + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, []string{"-benchtime=2x"}, "", "") + gotest.Len(it, targets, 1) + gotest.Contains(it, targets[0].RunFlags, "-test.benchtime=2x") + }) + }) + + t.When("the -bench pattern carries sub-benchmark segments", func(w *gotest.T) { + w.It("selects the suite by the first segment and carries the full pattern as BenchFilter", func(it *gotest.T) { + benchesByPkg := map[string][]string{"example.com/pkg": {"CacheTestSuite", "OtherSuite"}} + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, nil, "", "^BenchmarkCacheTestSuite$/^BenchmarkGetHit$") + gotest.Len(it, targets, 1) + gotest.Equal(it, "CacheTestSuite", targets[0].SuiteName) + gotest.Equal(it, "^BenchmarkCacheTestSuite$/^BenchmarkGetHit$", targets[0].BenchFilter) + }) + + w.It("keeps only the alternation branches that match each suite's wrapper", func(it *gotest.T) { + benchesByPkg := map[string][]string{"example.com/pkg": {"Foo", "Bar"}} + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, nil, "", "^BenchmarkFoo$/^BenchmarkA$|^BenchmarkBar$/^BenchmarkB$") + gotest.Len(it, targets, 2) + byName := map[string]string{} + for i := range targets { + byName[targets[i].SuiteName] = targets[i].BenchFilter + } + gotest.Equal(it, "^BenchmarkFoo$/^BenchmarkA$", byName["Foo"]) + gotest.Equal(it, "^BenchmarkBar$/^BenchmarkB$", byName["Bar"]) + }) + + w.It("leaves BenchFilter empty for a suite-only pattern", func(it *gotest.T) { + benchesByPkg := map[string][]string{"example.com/pkg": {"CacheTestSuite"}} + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, nil, "", "^BenchmarkCacheTestSuite$") + gotest.Len(it, targets, 1) + gotest.Zero(it, targets[0].BenchFilter) + }) + }) + + t.When("building the bench subprocess command", func(w *gotest.T) { + ctx := context.Background() + + w.It("forces the exact wrapper pattern without a BenchFilter", func(it *gotest.T) { + target := gotestrunner.SuiteTarget{ + SuiteSpec: gotestrunner.SuiteSpec{Package: "example.com/pkg", SuiteName: "CacheTestSuite"}, + BinaryPath: "/tmp/pkg.test", + Bench: true, + } + cmd := gotestrunner.ExportBuildSuiteCmd(ctx, target, nil, false) + gotest.Contains(it, cmd.Args, `-test.bench=^BenchmarkCacheTestSuite$`) + }) + + w.It("hands a BenchFilter to the binary verbatim so go test scopes sub-benchmarks", func(it *gotest.T) { + target := gotestrunner.SuiteTarget{ + SuiteSpec: gotestrunner.SuiteSpec{Package: "example.com/pkg", SuiteName: "CacheTestSuite"}, + BinaryPath: "/tmp/pkg.test", + Bench: true, + BenchFilter: "^BenchmarkCacheTestSuite$/^BenchmarkGetHit$", + } + cmd := gotestrunner.ExportBuildSuiteCmd(ctx, target, nil, false) + gotest.Contains(it, cmd.Args, `-test.bench=^BenchmarkCacheTestSuite$/^BenchmarkGetHit$`) + gotest.NotContains(it, cmd.Args, `-test.bench=^BenchmarkCacheTestSuite$`) + }) + }) +} func (s *GotestrunnerTestSuite) TestExclusiveDispatch(t *gotest.T) { t.When("BuildSuiteTargets sees a suite marked exclusive", func(w *gotest.T) { @@ -822,6 +990,8 @@ func (s *GotestrunnerTestSuite) TestExclusiveDispatch(t *gotest.T) { }) } +// --- OutputCollector tests --- + func (s *GotestrunnerTestSuite) TestOutputCollector(t *gotest.T) { pass := func(d time.Duration) gotestrunner.SuiteResult { return gotestrunner.SuiteResult{Stdout: []byte("PASS\n"), ExitCode: 0, Duration: d} @@ -1492,30 +1662,70 @@ func (s *GotestrunnerTestSuite) TestInjectParallel(t *gotest.T) { } } -var jsonTimestampRe = regexp.MustCompile(`\d+\.\d+s`) +func (s *GotestrunnerTestSuite) TestExtractBenchFilter(t *gotest.T) { + for sub, tc := range gotest.Each(t, []struct { + Name string + flags []string + expect string + }{ + {"empty", nil, ""}, + {"not present", []string{"-v", "-timeout=10m"}, ""}, + {"equals form", []string{"-bench=Foo"}, "Foo"}, + {"space form", []string{"-bench", "Foo", "-v"}, "Foo"}, + {"stops at -args", []string{"-args", "-bench=Foo"}, ""}, + }) { + got := gotestrunner.ExtractBenchFilter(tc.flags) + gotest.Equal(sub, tc.expect, got) + } +} -func normalizeJSON(raw string) string { - var lines []string - for line := range strings.SplitSeq(strings.TrimRight(raw, "\n"), "\n") { - if line == "" { - continue - } - var ev map[string]any - if json.Unmarshal([]byte(line), &ev) != nil { - lines = append(lines, line) - continue - } - ev["Time"] = "«TIME»" - if _, ok := ev["Elapsed"]; ok { - ev["Elapsed"] = "«ELAPSED»" - } - if output, ok := ev["Output"].(string); ok { - ev["Output"] = jsonTimestampRe.ReplaceAllString(output, "«TS»") - } - normalized := gotest.Must(json.Marshal(ev)) - lines = append(lines, string(normalized)) +func (s *GotestrunnerTestSuite) TestStripBenchFilter(t *gotest.T) { + for sub, tc := range gotest.Each(t, []struct { + Name string + flags []string + expect []string + }{ + {"empty", nil, nil}, + {"equals form", []string{"-bench=Foo", "-v"}, []string{"-v"}}, + {"space form", []string{"-bench", "Foo", "-v"}, []string{"-v"}}, + {"leaves other flags untouched", []string{"-benchtime=2x", "-count=1"}, []string{"-benchtime=2x", "-count=1"}}, + {"does not strip past -args", []string{"-v", "-args", "-bench=Foo"}, []string{"-v", "-args", "-bench=Foo"}}, + }) { + got := gotestrunner.StripBenchFilter(tc.flags) + gotest.Equal(sub, tc.expect, got) } - return strings.Join(lines, "\n") + "\n" +} + +func (s *GotestrunnerTestSuite) TestResolveBenchParallelism(t *gotest.T) { + t.When("resolving dispatch concurrency", func(w *gotest.T) { + w.It("forces serial dispatch (1) in bench mode regardless of budget", func(it *gotest.T) { + cfg := gotestrunner.PipelineConfig{Bench: true, Parallel: 8} + runFlags := []string{} + got := gotestrunner.ResolveBenchParallelismForTest(cfg, &runFlags, 5, false) + gotest.Equal(it, 1, got) + }) + + w.It("does not inject -parallel into run flags in bench mode", func(it *gotest.T) { + cfg := gotestrunner.PipelineConfig{Bench: true} + runFlags := []string{"-v"} + gotestrunner.ResolveBenchParallelismForTest(cfg, &runFlags, 5, false) + gotest.Equal(it, []string{"-v"}, runFlags) + }) + + w.It("falls back to computeDispatchConcurrency (with intra-injection) outside bench mode", func(it *gotest.T) { + cfg := gotestrunner.PipelineConfig{Bench: false} + runFlags := []string{} + got := gotestrunner.ResolveBenchParallelismForTest(cfg, &runFlags, 4, false) + gotest.Greater(it, got, 0) + found := false + for _, f := range runFlags { + if strings.HasPrefix(f, "-parallel") { + found = true + } + } + gotest.True(it, found, "expected -parallel to be injected outside bench mode, got %v", runFlags) + }) + }) } func (s *GotestrunnerTestSuite) TestLogSlowBuild(t *gotest.T) { @@ -1548,16 +1758,18 @@ func (s *GotestrunnerTestSuite) TestSanitizerAwareDispatch(t *gotest.T) { t.When("an instrumentation build flag is active with default parallelism", func(w *gotest.T) { w.It("halves the process cap so instrumented suites keep scheduling headroom", func(it *gotest.T) { + cfg := gotestrunner.PipelineConfig{} runFlags := []string{} - got := gotestrunner.ExportComputeDispatchConcurrency(&runFlags, 0, 64, true) + got := gotestrunner.ResolveBenchParallelismForTest(cfg, &runFlags, 64, true) gotest.Equal(it, max(1, procs/2), got) }) w.It("keeps an explicit --parallel budget untouched — the user's number wins", func(it *gotest.T) { + cfg := gotestrunner.PipelineConfig{Parallel: 10} runFlags := []string{} - withRace := gotestrunner.ExportComputeDispatchConcurrency(&runFlags, 10, 64, true) + withRace := gotestrunner.ResolveBenchParallelismForTest(cfg, &runFlags, 64, true) runFlags = []string{} - without := gotestrunner.ExportComputeDispatchConcurrency(&runFlags, 10, 64, false) + without := gotestrunner.ResolveBenchParallelismForTest(cfg, &runFlags, 64, false) gotest.Equal(it, without, withRace) }) }) @@ -1573,6 +1785,59 @@ func (s *GotestrunnerTestSuite) TestSanitizerAwareDispatch(t *gotest.T) { }) } +func (s *GotestrunnerTestSuite) TestBenchFilterNeverReachesTargetRunFlags(t *gotest.T) { + t.When("a user -bench filter is present in raw run flags", func(w *gotest.T) { + w.It("is extracted and stripped before BuildBenchTargets so it never leaks into RunFlags", func(it *gotest.T) { + rawRunFlags := []string{"-bench=Foo", "-benchtime=2x"} + userBenchFilter := gotestrunner.ExtractBenchFilter(rawRunFlags) + runFlags := gotestrunner.StripBenchFilter(rawRunFlags) + + gotest.Equal(it, "Foo", userBenchFilter) + gotest.NotContains(it, runFlags, "-bench=Foo") + + compiled := []gotestrunner.CompileResult{ + {Package: "example.com/pkg", BinaryPath: "/tmp/pkg.test"}, + } + benchesByPkg := map[string][]string{"example.com/pkg": {"Foo"}} + dirsByPkg := map[string]string{"example.com/pkg": "/src/pkg"} + + targets := gotestrunner.BuildBenchTargets(compiled, benchesByPkg, dirsByPkg, runFlags, "", userBenchFilter) + gotest.Len(it, targets, 1) + for _, f := range targets[0].RunFlags { + leaked := f == "-test.bench" || strings.HasPrefix(f, "-test.bench=") + gotest.False(it, leaked, "user -bench leaked into target RunFlags: %v", targets[0].RunFlags) + } + gotest.Contains(it, targets[0].RunFlags, "-test.benchtime=2x") + }) + }) +} + +var jsonTimestampRe = regexp.MustCompile(`\d+\.\d+s`) + +func normalizeJSON(raw string) string { + var lines []string + for line := range strings.SplitSeq(strings.TrimRight(raw, "\n"), "\n") { + if line == "" { + continue + } + var ev map[string]any + if json.Unmarshal([]byte(line), &ev) != nil { + lines = append(lines, line) + continue + } + ev["Time"] = "«TIME»" + if _, ok := ev["Elapsed"]; ok { + ev["Elapsed"] = "«ELAPSED»" + } + if output, ok := ev["Output"].(string); ok { + ev["Output"] = jsonTimestampRe.ReplaceAllString(output, "«TS»") + } + normalized := gotest.Must(json.Marshal(ev)) + lines = append(lines, string(normalized)) + } + return strings.Join(lines, "\n") + "\n" +} + func (s *GotestrunnerTestSuite) TestOutputGolden(t *gotest.T) { t.When("text non-verbose", func(w *gotest.T) { w.It("single passing package", func(it *gotest.T) { diff --git a/internal/gotestrunner/overlay.go b/internal/gotestrunner/overlay.go index 4ee844d9..a46e3d60 100644 --- a/internal/gotestrunner/overlay.go +++ b/internal/gotestrunner/overlay.go @@ -30,6 +30,7 @@ type OverlayResult struct { StdlibTestsByPkg map[string]int // stdlib func TestX counts per package — gotest reports but does not run them SuitesByPkg map[string][]string ExclusiveSuitesByPkg map[string]map[string]bool + BenchesByPkg map[string][]string DirsByPkg map[string]string SkippedSuitesByPkg map[string][]string FixtureDepSuites map[string]map[string]bool @@ -72,6 +73,7 @@ func GenerateOverlay(loaded []*gotestgen.LoadResult, broken []gotestgen.BrokenPa var noSuitePkgs []string stdlibByPkg := map[string]int{} suitesByPkg := map[string][]string{} + benchesByPkg := map[string][]string{} dirsByPkg := map[string]string{} skippedSuitesByPkg := map[string][]string{} exclusiveSuitesByPkg := map[string]map[string]bool{} @@ -89,6 +91,9 @@ func GenerateOverlay(loaded []*gotestgen.LoadResult, broken []gotestgen.BrokenPa if len(r.SuiteNames) > 0 { suitesByPkg[r.PkgPath] = r.SuiteNames } + if len(r.BenchSuiteNames) > 0 { + benchesByPkg[r.PkgPath] = r.BenchSuiteNames + } if r.AbsPath != "" { dirsByPkg[r.PkgPath] = r.AbsPath } @@ -125,6 +130,7 @@ func GenerateOverlay(loaded []*gotestgen.LoadResult, broken []gotestgen.BrokenPa StdlibTestsByPkg: stdlibByPkg, SuitesByPkg: suitesByPkg, ExclusiveSuitesByPkg: exclusiveSuitesByPkg, + BenchesByPkg: benchesByPkg, DirsByPkg: dirsByPkg, SkippedSuitesByPkg: skippedSuitesByPkg, FixtureDepSuites: fixtureDepSuites, diff --git a/internal/gotestrunner/pipeline.go b/internal/gotestrunner/pipeline.go index a2ffa304..5f65a06b 100644 --- a/internal/gotestrunner/pipeline.go +++ b/internal/gotestrunner/pipeline.go @@ -61,6 +61,18 @@ func computeDispatchConcurrency(runFlags *[]string, budget, totalSuites int, san return inter } +// resolveMaxParallel computes the dispatch concurrency for a batch run. In +// bench mode, dispatch is always serial (1) — benchmarks must not run +// concurrently with each other for meaningful timing — and this entirely +// skips computeDispatchConcurrency's -parallel intra-injection, since +// benchmark suites must not be told to run their methods in parallel either. +func resolveMaxParallel(cfg PipelineConfig, runFlags *[]string, totalSuites int, sanitized bool) int { //nolint:gocritic // hugeParam: stable API + if cfg.Bench { + return 1 + } + return computeDispatchConcurrency(runFlags, cfg.Parallel, totalSuites, sanitized) +} + type PipelineConfig struct { GoTestArgs []string SetupTimeout time.Duration @@ -70,6 +82,8 @@ type PipelineConfig struct { CompileParallel int Streaming bool OutputMode RunMode + Bench bool + BenchesByPkg map[string][]string } type PipelineResult struct { @@ -177,7 +191,7 @@ func appendRunFailureEvents(stream []byte, pkg, msg string) []byte { return stream } -func RunPipeline(ctx context.Context, cfg PipelineConfig, overlay *OverlayResult) (PipelineResult, error) { +func RunPipeline(ctx context.Context, cfg PipelineConfig, overlay *OverlayResult) (PipelineResult, error) { //nolint:gocritic // hugeParam: stable API if !cfg.CI && os.Getenv(protocol.EnvCI) == "" { if v := os.Getenv("CI"); v != "" && v != "0" && v != "false" { cfg.CI = true @@ -185,13 +199,17 @@ func RunPipeline(ctx context.Context, cfg PipelineConfig, overlay *OverlayResult } pf := ParseExecFlags(cfg.GoTestArgs) + if cfg.Bench { + cfg.Streaming = false + } + if cfg.Streaming { return runStreaming(ctx, cfg, overlay, pf) } return runBatch(ctx, cfg, overlay, pf) } -func buildExtraEnv(cfg PipelineConfig, proc *SharedFixtureProcess) map[string]string { +func buildExtraEnv(cfg PipelineConfig, proc *SharedFixtureProcess) map[string]string { //nolint:gocritic // hugeParam: stable API env := make(map[string]string) if cfg.UpdateSnapshots { env[protocol.EnvUpdateSnapshots] = "1" @@ -205,7 +223,7 @@ func buildExtraEnv(cfg PipelineConfig, proc *SharedFixtureProcess) map[string]st return env } -func buildBaseEnv(cfg PipelineConfig) []string { +func buildBaseEnv(cfg PipelineConfig) []string { //nolint:gocritic // hugeParam: stable API env := os.Environ() if cfg.UpdateSnapshots { env = append(env, protocol.EnvUpdateSnapshots+"=1") @@ -307,7 +325,15 @@ func setupCoverage(targets []SuiteTarget, overlay *OverlayResult, userCoverProfi } func runBatch(ctx context.Context, cfg PipelineConfig, overlay *OverlayResult, pf ParsedFlags) (result PipelineResult, err error) { //nolint:gocritic // hugeParam: stable API - win := planFixtureWindows(overlay, pf.UserRunFilter) + // Bench runs dispatch bench targets, not test suites, and match -run and + // -bench against Benchmark: their residency plan must come from + // the same selection, or fixtures would follow the wrong schedule. + var win fixtureWindows + if cfg.Bench { + win = planBenchFixtureWindows(overlay, pf.UserRunFilter, ExtractBenchFilter(pf.RunFlags)) + } else { + win = planFixtureWindows(overlay, pf.UserRunFilter) + } win.reportSkipped() compiled, compileFailures, setupProc, cancelPrepare, err := prepareTestRun(ctx, overlay, win.Fixtures, pf.BuildFlags, cfg.SetupTimeout, cfg.CompileParallel) if err != nil { @@ -337,8 +363,28 @@ func runBatch(ctx context.Context, cfg PipelineConfig, overlay *OverlayResult, p if cfg.OutputMode == RunCaptureJSON { runFlags = append(append([]string(nil), runFlags...), "-v") } - maxParallel := computeDispatchConcurrency(&runFlags, cfg.Parallel, totalSuites, SanitizerActive(pf.BuildFlags)) - targets := BuildSuiteTargets(compiled, overlay.SuitesByPkg, overlay.DirsByPkg, overlay.ExclusiveSuitesByPkg, runFlags, pf.UserRunFilter) + + var targets []SuiteTarget + var maxParallel int + if cfg.Bench { + // A user-supplied -bench value arrives in RunFlags via normal flag + // classification. It must not reach buildSuiteCmd as a raw + // -test.bench flag — that would be appended after the generated + // -test.bench=^Benchmark$ and silently win, defeating + // per-suite scoping. Extract and strip it here, then feed it to + // BuildBenchTargets as a bench-name filter, matched against + // Benchmark. -run (pf.UserRunFilter) is passed through + // alongside it as an independent suite filter — both compose with + // AND semantics, e.g. `gotest bench ./pkg/parser -run Parse` + // filters by suite while -bench filters by benchmark name. + userBenchFilter := ExtractBenchFilter(runFlags) + runFlags = StripBenchFilter(runFlags) + targets = BuildBenchTargets(compiled, cfg.BenchesByPkg, overlay.DirsByPkg, runFlags, pf.UserRunFilter, userBenchFilter) + maxParallel = resolveMaxParallel(cfg, &runFlags, totalSuites, SanitizerActive(pf.BuildFlags)) + } else { + maxParallel = resolveMaxParallel(cfg, &runFlags, totalSuites, SanitizerActive(pf.BuildFlags)) + targets = BuildSuiteTargets(compiled, overlay.SuitesByPkg, overlay.DirsByPkg, overlay.ExclusiveSuitesByPkg, runFlags, pf.UserRunFilter) + } collector := NewOutputCollector(cfg.OutputMode, pf.Verbose) collector.StdlibTestsByPkg = overlay.StdlibTestsByPkg @@ -361,25 +407,78 @@ func runBatch(ctx context.Context, cfg PipelineConfig, overlay *OverlayResult, p defer mergeCoverProfiles(targets, pf.UserCoverProfile) } - // The barrier re-windows shared fixtures between the parallel bulk and - // the exclusive tail. Alive(tail) comes from the actual exclusive targets - // — the plan narrowed by compile results — so a fixture whose only tail - // suite never became runnable is released, not started. - var tailTargets []SuiteTarget - for i := range targets { - if targets[i].Exclusive { - tailTargets = append(tailTargets, targets[i]) + if cfg.Bench { + // Serial dispatch, one window per slot: StartKeys(needed ∖ alive) + // before each bench suite, TeardownKeys(alive ∖ needed-by-any-later- + // target) after it. A fixture is resident exactly from the first slot + // that needs it through the last. + SortTargetsSerial(targets) + needs, laterNeeds := benchSlotPlan(targets, overlay.SuiteRequiredSharedFixtureKeys, win.Fixtures) + alive := make(map[string]bool, len(win.Bulk)) + for k := range win.Bulk { + alive[k] = true } - } - barrier := func() { - if len(tailTargets) == 0 { - return // nothing dispatches after the bulk; run-end teardown owns the rest + var windowErrs []error + beforeSlot := func(i int) { + if setupProc == nil || ctx.Err() != nil { + return + } + start := diffKeys(needs[i], alive) + if len(start) == 0 { + return + } + // Marked alive on failure too: the subprocess counts a failed + // fixture as started, so retrying on a later slot would only + // re-report the same failure. + for _, k := range start { + alive[k] = true + } + err := setupProc.StartKeys(start, resolveSetupTimeout(cfg.SetupTimeout)) + if err == nil { + err = setupProc.RefreshStateFile() + } + if err != nil { + windowErrs = append(windowErrs, fmt.Errorf("bench fixture window open (%s): %w", targets[i].SuiteName, err)) + } + } + afterSlot := func(i int) { + if setupProc == nil || ctx.Err() != nil { + return + } + release := diffKeys(alive, laterNeeds[i+1]) + if len(release) == 0 { + return + } + for _, k := range release { + delete(alive, k) + } + if err := setupProc.TeardownKeys(release, setupProc.teardownBudget()); err != nil { + windowErrs = append(windowErrs, fmt.Errorf("bench fixture window close (%s): %w", targets[i].SuiteName, err)) + } + } + RunBenchSuites(ctx, targets, extraEnv, collector, beforeSlot, afterSlot) + barrierErr = errors.Join(windowErrs...) + } else { + // The barrier re-windows shared fixtures between the parallel bulk and + // the exclusive tail. Alive(tail) comes from the actual exclusive targets + // — the plan narrowed by compile results — so a fixture whose only tail + // suite never became runnable is released, not started. + var tailTargets []SuiteTarget + for i := range targets { + if targets[i].Exclusive { + tailTargets = append(tailTargets, targets[i]) + } + } + barrier := func() { + if len(tailTargets) == 0 { + return // nothing dispatches after the bulk; run-end teardown owns the rest + } + tailAlive := aliveFromTargets(tailTargets, overlay.SuiteRequiredSharedFixtureKeys, win.Fixtures) + barrierErr = fixtureBarrier(ctx, setupProc, win.Bulk, tailAlive, resolveSetupTimeout(cfg.SetupTimeout), true) } - tailAlive := aliveFromTargets(tailTargets, overlay.SuiteRequiredSharedFixtureKeys, win.Fixtures) - barrierErr = fixtureBarrier(ctx, setupProc, win.Bulk, tailAlive, resolveSetupTimeout(cfg.SetupTimeout), true) - } - RunSuites(ctx, targets, extraEnv, maxParallel, collector, barrier) + RunSuites(ctx, targets, extraEnv, maxParallel, collector, barrier) + } collector.Finalize(overlay.NoSuitePackages) return PipelineResult{ diff --git a/internal/gotestrunner/suiterun.go b/internal/gotestrunner/suiterun.go index d95aebc5..69641d6a 100644 --- a/internal/gotestrunner/suiterun.go +++ b/internal/gotestrunner/suiterun.go @@ -8,6 +8,7 @@ import ( "os/exec" "regexp" "runtime" + "slices" "sort" "strings" "sync" @@ -33,6 +34,8 @@ type SuiteTarget struct { RunFlags []string // test binary flags (with -test. prefix) CoverProfile string // per-suite cover profile path (empty if no -coverprofile) BudgetFile string // sidecar path for teardown budget (empty = use default) + Bench bool // when true, run the Benchmark wrapper instead of the suite's tests + BenchFilter string // raw -test.bench value carrying the user's sub-benchmark segments (empty = the exact Benchmark wrapper) Exclusive bool // SuiteConfig{Exclusive: true}: dispatched strictly alone, after every non-exclusive suite } @@ -137,16 +140,82 @@ func RunSuites(ctx context.Context, targets []SuiteTarget, extraEnv map[string]s } } -func buildSuiteCmd(ctx context.Context, target SuiteTarget, env []string, test2json bool) *exec.Cmd { //nolint:gocritic // hugeParam: stable API - var runArg string - if target.RunFilter != "" { - runArg = "-test.run=" + target.RunFilter - } else { - runArg = fmt.Sprintf("-test.run=^%s$", regexp.QuoteMeta(target.SuiteName)) +// SortTargetsSerial orders targets in place by (Package, SuiteName) — the +// deterministic dispatch order RunBenchSuites executes them in. +func SortTargetsSerial(targets []SuiteTarget) { + sort.Slice(targets, func(a, b int) bool { + if targets[a].Package != targets[b].Package { + return targets[a].Package < targets[b].Package + } + return targets[a].SuiteName < targets[b].SuiteName + }) //nolint:gocritic // mirror of sortTargetIndices over the targets themselves +} + +// RunBenchSuites executes bench targets strictly one at a time, in slice +// order (see SortTargetsSerial): benchmarks own the machine, and their +// verdicts are wall-clock measurements no concurrent suite may corrupt. +// beforeSlot/afterSlot (nil-safe) bracket each slot with its index, so the +// caller can open and close per-slot fixture windows. +func RunBenchSuites(ctx context.Context, targets []SuiteTarget, extraEnv map[string]string, collector *OutputCollector, beforeSlot, afterSlot func(i int)) { + pkgCount := map[string]int{} + var pkgOrder []string + localIdx := make([]int, len(targets)) + for i := range targets { + if _, seen := pkgCount[targets[i].Package]; !seen { + pkgOrder = append(pkgOrder, targets[i].Package) + } + localIdx[i] = pkgCount[targets[i].Package] + pkgCount[targets[i].Package]++ + } + for _, pkg := range pkgOrder { + collector.Register(pkg, pkgCount[pkg]) } + useTest2JSON := collector.UsesTest2JSON() + env := os.Environ() + for k, v := range extraEnv { + env = append(env, k+"="+v) + } + + for i := range targets { + if ctx.Err() != nil { + return + } + if beforeSlot != nil { + beforeSlot(i) + } + r := RunSingleSuite(ctx, targets[i], env, useTest2JSON) + collector.RecordResult(targets[i].Package, localIdx[i], r) + if afterSlot != nil { + afterSlot(i) + } + } +} + +func buildSuiteCmd(ctx context.Context, target SuiteTarget, env []string, test2json bool) *exec.Cmd { //nolint:gocritic // hugeParam: stable API var testArgs []string - testArgs = append(testArgs, runArg) + switch { + case target.Bench: + // BenchFilter carries a user -bench pattern with sub-benchmark + // segments ("^BenchmarkSuite$/^BenchmarkMethod$"): the generated + // wrapper runs each method under b.Run with its method name, so go + // test's own slash matching scopes the run to single methods. The + // suite was already selected by the pattern's first segment in + // BuildBenchTargets; without segments the exact wrapper name runs + // every method, as before. + benchArg := fmt.Sprintf("-test.bench=^Benchmark%s$", regexp.QuoteMeta(target.SuiteName)) + if target.BenchFilter != "" { + benchArg = "-test.bench=" + target.BenchFilter + } + testArgs = append(testArgs, "-test.run=^$", benchArg) + if !slices.Contains(target.RunFlags, "-test.benchmem") { + testArgs = append(testArgs, "-test.benchmem") + } + case target.RunFilter != "": + testArgs = append(testArgs, "-test.run="+target.RunFilter) + default: + testArgs = append(testArgs, fmt.Sprintf("-test.run=^%s$", regexp.QuoteMeta(target.SuiteName))) + } if test2json { testArgs = append(testArgs, "-test.v=test2json") @@ -315,6 +384,66 @@ func BuildSuiteTargets(compiled []CompileResult, suitesByPkg map[string][]string return targets } +// BuildBenchTargets constructs SuiteTarget entries for benchmark suites from +// compiled binaries and bench-eligible suite names. benchesByPkg maps import +// path to a list of suite struct names (e.g., "FooTestSuite") that have at +// least one effective benchmark. The generated benchmark wrapper function +// name is "Benchmark" + suite struct name. +// +// userRunFilter (from -run) and userBenchFilter (from -bench) are applied +// independently, each matched against the benchmark wrapper name exactly as +// matchesSuiteFunc does elsewhere (first slash segment only). When both are +// non-empty, a suite must satisfy both (AND semantics) to be included; +// either one alone filters on its own, and when both are empty all suites +// are included. +// +// A userBenchFilter with sub-benchmark segments additionally scopes what +// runs inside the wrapper: the per-suite portion of the pattern (extracted +// with suiteRunFilter, the same helper -run uses for subtest scoping) is +// carried on the target as BenchFilter and becomes the binary's -test.bench +// value, where go test's slash matching selects the b.Run sub-benchmarks. +func BuildBenchTargets(compiled []CompileResult, benchesByPkg map[string][]string, dirsByPkg map[string]string, runFlags []string, userRunFilter, userBenchFilter string) []SuiteTarget { + binByPkg := make(map[string]string, len(compiled)) + for _, cr := range compiled { + binByPkg[cr.Package] = cr.BinaryPath + } + + translatedFlags := TranslateToTestBinaryFlags(runFlags) + + var targets []SuiteTarget + for pkg, suites := range benchesByPkg { + bin, ok := binByPkg[pkg] + if !ok { + continue + } + + pkgDir := dirsByPkg[pkg] + + for _, suiteName := range suites { + benchFuncName := "Benchmark" + suiteName + if userRunFilter != "" && !matchesSuiteFunc(userRunFilter, benchFuncName) { + continue + } + if userBenchFilter != "" && !anyBranchMatchesSuiteFunc(userBenchFilter, benchFuncName) { + continue + } + target := SuiteTarget{ + SuiteSpec: SuiteSpec{ + Package: pkg, + Dir: pkgDir, + SuiteName: suiteName, + }, + BinaryPath: bin, + RunFlags: translatedFlags, + Bench: true, + BenchFilter: suiteRunFilter(userBenchFilter, benchFuncName), + } + targets = append(targets, target) + } + } + return targets +} + // sortTargetIndices orders target indices by (Package, SuiteName) so // exclusive dispatch is deterministic run over run. func sortTargetIndices(targets []SuiteTarget, idx []int) { @@ -330,6 +459,20 @@ func sortTargetIndices(targets []SuiteTarget, idx []int) { // matchesSuiteFunc checks if the user's -run regex could match a given // test function name. The first segment (before /) of the regex is tested // against the function name. +// anyBranchMatchesSuiteFunc reports whether any top-level alternation branch +// of pattern selects funcName by its first slash segment. matchesSuiteFunc +// alone takes the first segment of the whole pattern, which drops every +// suite but the first from "^BenchmarkFoo$/^A$|^BenchmarkBar$/^B$"-shaped +// filters; sub-benchmark scoping needs each branch judged on its own. +func anyBranchMatchesSuiteFunc(pattern, funcName string) bool { + for _, alt := range splitTopLevelOr(pattern) { + if matchesSuiteFunc(alt, funcName) { + return true + } + } + return false +} + func matchesSuiteFunc(runRegex string, funcName string) bool { parts := strings.SplitN(runRegex, "/", 2) topLevel := parts[0] diff --git a/internal/gotestspec/event.go b/internal/gotestspec/event.go index 43cb5422..df1f4764 100644 --- a/internal/gotestspec/event.go +++ b/internal/gotestspec/event.go @@ -15,6 +15,13 @@ const ( ActionPass Action = "pass" ActionFail Action = "fail" ActionSkip Action = "skip" + // ActionBench is not emitted by go test's own -json encoder (real + // benchmark runs only ever produce "run"/"output", plus "fail" on + // failure — there is no terminal "pass"-equivalent event for a + // benchmark). It is accepted here so BuildTree can also finalize a + // benchmark node's status when fed synthetic or hand-authored event + // streams (e.g. via `gotest spec --input`) that choose to emit one. + ActionBench Action = "bench" ) type TestEvent struct { diff --git a/internal/gotestspec/json.go b/internal/gotestspec/json.go index 37901276..8e6e5d75 100644 --- a/internal/gotestspec/json.go +++ b/internal/gotestspec/json.go @@ -19,26 +19,31 @@ type jsonPackage struct { } type jsonNode struct { - Name string `json:"name"` - Display string `json:"display"` - Kind string `json:"kind"` - Status string `json:"status"` - Duration float64 `json:"duration"` - Focused bool `json:"focused"` - Excluded bool `json:"excluded"` - External bool `json:"external"` - Variant int `json:"variant,omitempty"` - Output []string `json:"output"` - Children []jsonNode `json:"children"` + Name string `json:"name"` + Display string `json:"display"` + Kind string `json:"kind"` + Status string `json:"status"` + Duration float64 `json:"duration"` + Focused bool `json:"focused"` + Excluded bool `json:"excluded"` + External bool `json:"external"` + Variant int `json:"variant,omitempty"` + Output []string `json:"output"` + Children []jsonNode `json:"children"` + Iterations int `json:"iterations,omitempty"` + NsPerOp float64 `json:"ns_per_op,omitempty"` + BytesPerOp int64 `json:"bytes_per_op,omitempty"` + AllocsPerOp int64 `json:"allocs_per_op,omitempty"` } type jsonStats struct { - Suites int `json:"suites"` - Behaviors int `json:"behaviors"` - Tests int `json:"tests"` - Passed int `json:"passed"` - Failed int `json:"failed"` - Skipped int `json:"skipped"` + Suites int `json:"suites"` + Behaviors int `json:"behaviors"` + Tests int `json:"tests"` + Benchmarks int `json:"benchmarks"` + Passed int `json:"passed"` + Failed int `json:"failed"` + Skipped int `json:"skipped"` // FailedPackages carries package-level verdicts (build failures, deaths // outside any test); the packages array shows which via status "fail". FailedPackages int `json:"failedPackages,omitempty"` @@ -53,6 +58,7 @@ func RenderJSON(w io.Writer, packages []*Package) { Suites: stats.Suites, Behaviors: stats.Behaviors, Tests: stats.Tests, + Benchmarks: stats.Benchmarks, Passed: stats.Passed, Failed: stats.Failed, Skipped: stats.Skipped, @@ -86,17 +92,21 @@ func convertNodes(nodes []*Node) []jsonNode { result := make([]jsonNode, len(nodes)) for i, n := range nodes { result[i] = jsonNode{ - Name: n.Name, - Display: n.Display, - Kind: kindString(n.Kind), - Status: statusString(n.Status), - Duration: n.Duration.Seconds(), - Focused: n.Focused, - Excluded: n.Excluded, - External: n.External, - Variant: n.Variant, - Output: n.Output, - Children: convertNodes(n.Children), + Name: n.Name, + Display: n.Display, + Kind: kindString(n.Kind), + Status: statusString(n.Status), + Duration: n.Duration.Seconds(), + Focused: n.Focused, + Excluded: n.Excluded, + External: n.External, + Variant: n.Variant, + Output: n.Output, + Children: convertNodes(n.Children), + Iterations: n.Iterations, + NsPerOp: n.NsPerOp, + BytesPerOp: n.BytesPerOp, + AllocsPerOp: n.AllocsPerOp, } if result[i].Output == nil { result[i].Output = []string{} @@ -130,6 +140,8 @@ func kindString(k NodeKind) string { return "block" case KindTest: return "test" + case KindBenchmark: + return "benchmark" default: return "unknown" } diff --git a/internal/gotestspec/json_test.go b/internal/gotestspec/json_test.go index 5cf987cd..0db7931b 100644 --- a/internal/gotestspec/json_test.go +++ b/internal/gotestspec/json_test.go @@ -174,6 +174,100 @@ func TestRenderJSON_ErrorOutput(t *testing.T) { } } +func TestRenderJSON_BenchmarkNode(t *testing.T) { + packages := []*Package{{ + Path: "p", + Nodes: []*Node{{ + Kind: KindSuite, + Display: "Foo", + Children: []*Node{ + { + Kind: KindBenchmark, + Display: "Parse", + Status: StatusPass, + Iterations: 1201, + NsPerOp: 985.2, + BytesPerOp: 24, + AllocsPerOp: 3, + }, + {Kind: KindMethod, Display: "Regular", Status: StatusPass, Duration: time.Millisecond}, + }, + }}, + }} + + var buf bytes.Buffer + RenderJSON(&buf, packages) + + var raw map[string]any + if err := json.Unmarshal(buf.Bytes(), &raw); err != nil { + t.Fatalf("invalid JSON: %s", err) + } + + pkgs, _ := raw["packages"].([]any) + nodes, _ := pkgs[0].(map[string]any)["nodes"].([]any) + suite, _ := nodes[0].(map[string]any) + children, _ := suite["children"].([]any) + bench, _ := children[0].(map[string]any) + regular, _ := children[1].(map[string]any) + + if bench["kind"] != "benchmark" { + t.Errorf("kind = %v, want benchmark", bench["kind"]) + } + + benchKeys := []string{"ns_per_op", "bytes_per_op", "allocs_per_op", "iterations"} + for _, key := range benchKeys { + if _, ok := bench[key]; !ok { + t.Errorf("bench node missing key %q, got: %v", key, bench) + } + } + if bench["ns_per_op"] != 985.2 { + t.Errorf("ns_per_op = %v, want 985.2", bench["ns_per_op"]) + } + if bench["bytes_per_op"] != float64(24) { + t.Errorf("bytes_per_op = %v, want 24", bench["bytes_per_op"]) + } + if bench["allocs_per_op"] != float64(3) { + t.Errorf("allocs_per_op = %v, want 3", bench["allocs_per_op"]) + } + if bench["iterations"] != float64(1201) { + t.Errorf("iterations = %v, want 1201", bench["iterations"]) + } + + for _, key := range benchKeys { + if _, ok := regular[key]; ok { + t.Errorf("non-bench node should omit key %q, got present: %v", key, regular) + } + } +} + +func TestRenderJSON_IncludesBenchmarkStats(t *testing.T) { + packages := []*Package{{ + Path: "p", + Nodes: []*Node{{ + Kind: KindSuite, + Display: "Foo", + Children: []*Node{ + {Kind: KindBenchmark, Display: "Parse", Status: StatusPass, Iterations: 10, NsPerOp: 1.0}, + }, + }}, + }} + + var buf bytes.Buffer + RenderJSON(&buf, packages) + + var result jsonRoot + if err := json.Unmarshal(buf.Bytes(), &result); err != nil { + t.Fatalf("invalid JSON: %s", err) + } + + if result.Stats.Benchmarks != 1 { + t.Errorf("benchmarks = %d, want 1", result.Stats.Benchmarks) + } + if result.Stats.Behaviors != 0 { + t.Errorf("behaviors = %d, want 0", result.Stats.Behaviors) + } +} + func TestRenderJSON_IncludesPackageOutput(t *testing.T) { packages := []*Package{{ Path: "p", diff --git a/internal/gotestspec/markdown.go b/internal/gotestspec/markdown.go index 8b8d1cf2..9cf5692c 100644 --- a/internal/gotestspec/markdown.go +++ b/internal/gotestspec/markdown.go @@ -21,6 +21,9 @@ func RenderMarkdown(w io.Writer, packages []*Package) { if stats.Tests > 0 { counts = append(counts, fmt.Sprintf("%d stdlib tests", stats.Tests)) } + if stats.Benchmarks > 0 { + counts = append(counts, fmt.Sprintf("%d benchmarks", stats.Benchmarks)) + } fmt.Fprintf(w, "%s: %d passed, %d failed, %d skipped.\n", strings.Join(counts, ", "), stats.Passed, stats.Failed, stats.Skipped) fmt.Fprintln(w) @@ -54,11 +57,14 @@ func renderMarkdownNode(w io.Writer, n *Node, headingLevel int) { } fmt.Fprintf(w, "%s %s\n\n", heading, label) - var leafChildren, nestedChildren []*Node + var leafChildren, benchChildren, nestedChildren []*Node for _, c := range n.Children { - if len(c.Children) == 0 { + switch { + case c.Kind == KindBenchmark && len(c.Children) == 0: + benchChildren = append(benchChildren, c) + case len(c.Children) == 0: leafChildren = append(leafChildren, c) - } else { + default: nestedChildren = append(nestedChildren, c) } } @@ -73,6 +79,10 @@ func renderMarkdownNode(w io.Writer, n *Node, headingLevel int) { fmt.Fprintln(w) } + if len(benchChildren) > 0 { + renderMarkdownBenchTable(w, benchChildren) + } + for _, c := range nestedChildren { renderMarkdownNode(w, c, headingLevel+1) } @@ -88,6 +98,15 @@ func renderMarkdownNode(w io.Writer, n *Node, headingLevel int) { renderMarkdownTable(w, n.Children, 0) fmt.Fprintln(w) + case KindBenchmark: + if len(n.Children) != 0 { + return + } + // A bare top-level benchmark node with no enclosing suite. + heading := strings.Repeat("#", headingLevel) + fmt.Fprintf(w, "%s %s\n\n", heading, n.Display) + renderMarkdownBenchTable(w, []*Node{n}) + default: if len(n.Children) == 0 { fmt.Fprintf(w, "- %s %s (%s)\n", statusText(n.Status), n.Display, formatDuration(n.Duration)) @@ -95,6 +114,16 @@ func renderMarkdownNode(w io.Writer, n *Node, headingLevel int) { } } +func renderMarkdownBenchTable(w io.Writer, nodes []*Node) { + fmt.Fprintln(w, "| Benchmark | ns/op | B/op | allocs/op |") + fmt.Fprintln(w, "|-----------|-------|------|-----------|") + for _, c := range nodes { + fmt.Fprintf(w, "| %s | %s | %d | %d |\n", + c.Display, formatNs(c.NsPerOp), c.BytesPerOp, c.AllocsPerOp) + } + fmt.Fprintln(w) +} + func renderMarkdownTable(w io.Writer, nodes []*Node, depth int) { indent := strings.Repeat("  ", depth) for _, n := range nodes { diff --git a/internal/gotestspec/render_test.go b/internal/gotestspec/render_test.go index b7cfb438..da5cf4dc 100644 --- a/internal/gotestspec/render_test.go +++ b/internal/gotestspec/render_test.go @@ -135,6 +135,115 @@ func TestRenderTerminal_SummaryLine(t *testing.T) { } } +func TestRenderTerminal_BenchmarkLeaf(t *testing.T) { + packages := []*Package{{ + Path: "p", + Nodes: []*Node{{ + Kind: KindSuite, + Display: "Foo", + Children: []*Node{{ + Kind: KindBenchmark, + Display: "Parse", + Status: StatusPass, + Iterations: 1201, + NsPerOp: 985.2, + BytesPerOp: 24, + AllocsPerOp: 3, + }}, + }}, + }} + + var buf bytes.Buffer + RenderTerminal(&buf, packages) + out := stripANSI(buf.String()) + + for _, want := range []string{"Parse", "985.2 ns/op", "24 B/op", "3 allocs/op", "1 benchmarks"} { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } +} + +func TestRenderTerminal_WithBenchDeltas(t *testing.T) { + packages := []*Package{{ + Path: "p", + Nodes: []*Node{{ + Kind: KindBenchmark, + Display: "BenchmarkFoo", + Status: StatusPass, + Iterations: 100, + NsPerOp: 120, + }}, + }} + + deltas := []BenchDelta{ + {Key: "p Foo/BenchmarkFoo", OldNs: 100, NewNs: 150, PercentChange: 50, Significant: true}, + } + + var buf bytes.Buffer + RenderTerminal(&buf, packages, WithBenchDeltas(deltas)) + out := stripANSI(buf.String()) + + if !strings.Contains(out, "ns/op") { + t.Errorf("expected the benchmark result line to render, got:\n%s", out) + } + if !strings.Contains(out, "BENCHMARK OLD ns/op NEW ns/op Δ") { + t.Errorf("expected delta table header, got:\n%s", out) + } + if !strings.Contains(out, "p Foo/BenchmarkFoo 100.0 150.0 +50.0% ⚠") { + t.Errorf("expected regression row, got:\n%s", out) + } + if !strings.Contains(out, "1 benchmarks:") { + t.Errorf("expected the trailing counts line, got:\n%s", out) + } + if strings.Contains(out, "tests passed (") { + t.Errorf("expected a single summary trailer, not a stacked RenderSummary one, got:\n%s", out) + } +} + +func TestRenderTerminal_WithBenchDeltas_FilteredToEmptyStillPrintsHeader(t *testing.T) { + packages := []*Package{{ + Path: "p", + Nodes: []*Node{{ + Kind: KindBenchmark, + Display: "BenchmarkFoo", + Status: StatusPass, + Iterations: 100, + NsPerOp: 120, + }}, + }} + + // An empty-but-non-nil slice models what the CLI passes when a + // comparison ran but every delta was filtered out (no significant + // regression, -v not passed): the header should still prove a + // comparison happened, alongside the tree's own ns/op line. + var buf bytes.Buffer + RenderTerminal(&buf, packages, WithBenchDeltas([]BenchDelta{})) + out := stripANSI(buf.String()) + + if !strings.Contains(out, "ns/op") { + t.Errorf("expected the benchmark result line to render, got:\n%s", out) + } + if !strings.Contains(out, "BENCHMARK OLD ns/op NEW ns/op Δ") { + t.Errorf("expected delta table header even with zero rows, got:\n%s", out) + } +} + +func TestRenderTerminal_NoBenchDeltas(t *testing.T) { + packages := []*Package{{ + Path: "p", + Nodes: []*Node{{Kind: KindTest, Display: "Foo", Status: StatusPass, Duration: time.Millisecond}}, + }} + + var buf bytes.Buffer + RenderTerminal(&buf, packages) + out := stripANSI(buf.String()) + + if strings.Contains(out, "BENCHMARK") { + t.Errorf("expected no delta table when WithBenchDeltas wasn't given, got:\n%s", out) + } +} + func TestRenderMarkdown_SuiteHierarchy(t *testing.T) { packages := []*Package{{ Path: "example.com/pkg", @@ -170,6 +279,38 @@ func TestRenderMarkdown_SuiteHierarchy(t *testing.T) { } } +func TestRenderMarkdown_BenchmarkTable(t *testing.T) { + packages := []*Package{{ + Path: "p", + Nodes: []*Node{{ + Kind: KindSuite, + Display: "Foo", + Children: []*Node{{ + Kind: KindBenchmark, + Display: "Parse", + Status: StatusPass, + Iterations: 1201, + NsPerOp: 985.2, + BytesPerOp: 24, + AllocsPerOp: 3, + }}, + }}, + }} + + var buf bytes.Buffer + RenderMarkdown(&buf, packages) + out := buf.String() + + for _, want := range []string{ + "| Benchmark | ns/op | B/op | allocs/op |", + "| Parse | 985.2 | 24 | 3 |", + } { + if !strings.Contains(out, want) { + t.Errorf("output missing %q:\n%s", want, out) + } + } +} + func TestRenderMarkdown_SkippedSuite(t *testing.T) { packages := []*Package{{ Path: "p", diff --git a/internal/gotestspec/summary.go b/internal/gotestspec/summary.go index 41958c6f..0c1a29d7 100644 --- a/internal/gotestspec/summary.go +++ b/internal/gotestspec/summary.go @@ -100,6 +100,7 @@ func RenderSummary(w io.Writer, packages []*Package, opts ...RenderOption) { if cfg.coverage != nil { fmt.Fprintf(w, "%sCoverage: %.1f%%%s\n", c.dim, cfg.coverage.Total, c.reset) } + renderBenchDeltaTable(w, cfg.benchDeltas, c) return } @@ -142,6 +143,7 @@ func RenderSummary(w io.Writer, packages []*Package, opts ...RenderOption) { if cfg.coverage != nil { fmt.Fprintf(w, "%sCoverage: %.1f%%%s\n", c.dim, cfg.coverage.Total, c.reset) } + renderBenchDeltaTable(w, cfg.benchDeltas, c) renderSummary(w, stats, c) } @@ -161,6 +163,7 @@ func RenderMarkdownSummary(w io.Writer, packages []*Package, opts ...RenderOptio if cfg.coverage != nil { renderMarkdownCoverage(w, cfg.coverage) } + renderMarkdownBenchDeltaTable(w, cfg.benchDeltas) return } @@ -205,6 +208,8 @@ func RenderMarkdownSummary(w io.Writer, packages []*Package, opts ...RenderOptio renderMarkdownCoverage(w, cfg.coverage) } + renderMarkdownBenchDeltaTable(w, cfg.benchDeltas) + fmt.Fprint(w, "---\n") var parts []string if stats.Suites > 0 { @@ -226,6 +231,33 @@ func RenderMarkdownSummary(w io.Writer, packages []*Package, opts ...RenderOptio fmt.Fprintf(w, "%s: %s\n", strings.Join(parts, ", "), trailer) } +// renderMarkdownBenchDeltaTable renders deltas as a markdown table mirroring +// renderBenchDeltaTable's terminal columns. deltas is rendered as given — +// filtering significant-only vs. every row (-v) is the caller's +// responsibility (see WithBenchDeltas). A nil slice (WithBenchDeltas never +// called) no-ops; an empty-but-non-nil slice still prints the header (see +// renderBenchDeltaTable for why). +func renderMarkdownBenchDeltaTable(w io.Writer, deltas []BenchDelta) { + if deltas == nil { + return + } + fmt.Fprintln(w, "| Benchmark | old ns/op | new ns/op | Δ |") + fmt.Fprintln(w, "|---|---|---|---|") + for _, d := range deltas { + sign := "" + if d.PercentChange >= 0 { + sign = "+" + } + warn := "" + if d.Significant && d.PercentChange > 0 { + warn = " ⚠" + } + fmt.Fprintf(w, "| %s | %.1f | %.1f | %s%.1f%%%s |\n", + d.Key, d.OldNs, d.NewNs, sign, d.PercentChange, warn) + } + fmt.Fprintln(w) +} + func renderMarkdownCoverage(w io.Writer, report *CoverageReport) { fmt.Fprintf(w, "### Coverage: %.1f%%\n\n", report.Total) if len(report.Packages) > 1 { diff --git a/internal/gotestspec/summary_test.go b/internal/gotestspec/summary_test.go index 262da711..76c308be 100644 --- a/internal/gotestspec/summary_test.go +++ b/internal/gotestspec/summary_test.go @@ -302,6 +302,111 @@ func TestRenderMarkdownSummary_PackageDiagnostic(t *testing.T) { } } +func TestRenderSummary_WithBenchDeltas(t *testing.T) { + packages := []*Package{{ + Path: "p", + Nodes: []*Node{ + {Kind: KindTest, Display: "Foo", Status: StatusPass, Duration: time.Millisecond}, + }, + }} + + deltas := []BenchDelta{ + {Key: "p Foo/BenchmarkFoo", OldNs: 100, NewNs: 150, PercentChange: 50, Significant: true}, + {Key: "p Foo/BenchmarkBar", OldNs: 100, NewNs: 105, PercentChange: 5, Significant: false}, + } + + var buf bytes.Buffer + RenderSummary(&buf, packages, WithNoColor(), WithBenchDeltas(deltas)) + out := buf.String() + + if !strings.Contains(out, "BENCHMARK OLD ns/op NEW ns/op Δ") { + t.Errorf("expected delta table header, got:\n%s", out) + } + if !strings.Contains(out, "p Foo/BenchmarkFoo 100.0 150.0 +50.0% ⚠") { + t.Errorf("expected regression row with warning, got:\n%s", out) + } + if !strings.Contains(out, "p Foo/BenchmarkBar 100.0 105.0 +5.0%\n") { + t.Errorf("expected insignificant row without warning, got:\n%s", out) + } + if strings.Contains(out, "p Foo/BenchmarkBar 100.0 105.0 +5.0% ⚠") { + t.Errorf("insignificant row should not carry a warning, got:\n%s", out) + } +} + +func TestRenderSummary_NoBenchDeltas(t *testing.T) { + packages := []*Package{{ + Path: "p", + Nodes: []*Node{ + {Kind: KindTest, Display: "Foo", Status: StatusPass, Duration: time.Millisecond}, + }, + }} + + var buf bytes.Buffer + RenderSummary(&buf, packages, WithNoColor()) + out := buf.String() + + if strings.Contains(out, "BENCHMARK") { + t.Errorf("expected no delta table when no deltas were given, got:\n%s", out) + } +} + +func TestRenderMarkdownSummary_WithBenchDeltas(t *testing.T) { + packages := []*Package{{ + Path: "pkg/foo", + Nodes: []*Node{ + { + Kind: KindTest, + Display: "Bad", + Status: StatusFail, + Duration: 10 * time.Millisecond, + Output: []string{" foo_test.go:1: boom\n"}, + }, + }, + }} + + deltas := []BenchDelta{ + {Key: "p Foo/BenchmarkFoo", OldNs: 100, NewNs: 200, PercentChange: 100, Significant: true}, + {Key: "p Foo/BenchmarkBar", OldNs: 100, NewNs: 102, PercentChange: 2, Significant: false}, + } + + var buf bytes.Buffer + RenderMarkdownSummary(&buf, packages, WithBenchDeltas(deltas)) + out := buf.String() + + if !strings.Contains(out, "| Benchmark | old ns/op | new ns/op | Δ |") { + t.Errorf("expected markdown delta table header, got:\n%s", out) + } + if !strings.Contains(out, "| p Foo/BenchmarkFoo | 100.0 | 200.0 | +100.0% ⚠ |") { + t.Errorf("expected regression row, got:\n%s", out) + } + if !strings.Contains(out, "| p Foo/BenchmarkBar | 100.0 | 102.0 | +2.0% |") { + t.Errorf("expected insignificant row, got:\n%s", out) + } + + tableIdx := strings.Index(out, "| Benchmark |") + ruleIdx := strings.Index(out, "---\n") + if tableIdx == -1 || ruleIdx == -1 || tableIdx > ruleIdx { + t.Errorf("expected delta table before the trailing ---, got:\n%s", out) + } +} + +func TestRenderMarkdownSummary_NoBenchDeltas(t *testing.T) { + packages := []*Package{{ + Path: "p", + Nodes: []*Node{ + {Kind: KindTest, Display: "Foo", Status: StatusPass, Duration: time.Millisecond}, + }, + }} + + var buf bytes.Buffer + RenderMarkdownSummary(&buf, packages) + out := buf.String() + + if strings.Contains(out, "| Benchmark |") { + t.Errorf("expected no delta table when no deltas were given, got:\n%s", out) + } +} + func TestRenderSummary_BothTestFailureAndPackageDiagnostic(t *testing.T) { packages := []*Package{{ Path: "p", diff --git a/internal/gotestspec/terminal.go b/internal/gotestspec/terminal.go index f88b2dab..238ec119 100644 --- a/internal/gotestspec/terminal.go +++ b/internal/gotestspec/terminal.go @@ -3,6 +3,7 @@ package gotestspec import ( "fmt" "io" + "math" "strings" "time" ) @@ -23,9 +24,10 @@ var ansiColors = colors{ var noColors = colors{} type renderConfig struct { - color bool - coverage *CoverageReport - elapsed time.Duration + color bool + coverage *CoverageReport + elapsed time.Duration + benchDeltas []BenchDelta } type RenderOption func(*renderConfig) @@ -42,6 +44,28 @@ func WithElapsed(d time.Duration) RenderOption { return func(c *renderConfig) { c.elapsed = d } } +// BenchDelta is one benchmark's old-vs-new comparison, as rendered by +// RenderSummary and RenderMarkdownSummary via WithBenchDeltas. It mirrors +// gotestbench.Delta's shape but lives here (rather than being consumed +// directly) so that gotestspec never has to import gotestbench — callers +// (cmd/gotest/bench.go) convert []gotestbench.Delta to []BenchDelta at the +// call site. Filtering (e.g. significant-only vs. every row for -v) is +// also the caller's responsibility: the renderers show exactly the rows +// they're given. +type BenchDelta struct { + Key string // "pkg Suite/Name", matching gotestbench.Delta.Key + OldNs, NewNs float64 + PercentChange float64 + Significant bool +} + +// WithBenchDeltas attaches a benchmark old-vs-new comparison table to be +// rendered by RenderTerminal, RenderSummary, and RenderMarkdownSummary. A +// nil or empty slice renders no table. +func WithBenchDeltas(deltas []BenchDelta) RenderOption { + return func(c *renderConfig) { c.benchDeltas = deltas } +} + func RenderTerminal(w io.Writer, packages []*Package, opts ...RenderOption) { cfg := renderConfig{color: true} for _, o := range opts { @@ -79,6 +103,7 @@ func RenderTerminal(w io.Writer, packages []*Package, opts ...RenderOption) { } } + renderBenchDeltaTable(w, cfg.benchDeltas, c) fmt.Fprintln(w) stats := CollectStats(packages) renderSummary(w, stats, c) @@ -90,6 +115,18 @@ func renderNode(w io.Writer, n *Node, depth int, c *colors) { if isLeaf { icon, clr := statusIcon(n.Status, c) + + if n.Kind == KindBenchmark && n.Iterations > 0 { + fmt.Fprintf(w, "%s%s%s%s %s %s ns/op · %d B/op · %d allocs/op%s\n", + indent, clr, icon, c.reset, + n.Display, formatNs(n.NsPerOp), n.BytesPerOp, n.AllocsPerOp, c.reset) + + if n.Status == StatusFail { + renderErrorOutput(w, n.Output, depth+2, c) + } + return + } + dur := formatDuration(n.Duration) suffix := "" @@ -109,7 +146,7 @@ func renderNode(w io.Writer, n *Node, depth int, c *colors) { } label := n.Display - if n.Kind == KindSuite || n.Kind == KindFixture || n.Kind == KindMethod || n.Kind == KindTest { + if n.Kind == KindSuite || n.Kind == KindFixture || n.Kind == KindMethod || n.Kind == KindTest || n.Kind == KindBenchmark { label = c.bold + label + c.reset } @@ -160,6 +197,16 @@ func statusIcon(s Status, c *colors) (string, string) { } } +// formatNs renders a ns/op value the way Go's own benchmark output does: +// no trailing decimal for whole numbers ("1243"), one decimal place +// otherwise ("985.2"). +func formatNs(ns float64) string { + if ns == math.Trunc(ns) { + return fmt.Sprintf("%.0f", ns) + } + return fmt.Sprintf("%.1f", ns) +} + func formatDuration(d time.Duration) string { ms := d.Milliseconds() if ms < 1 { @@ -213,6 +260,38 @@ func filterOutput(output []string) []string { return filtered } +// renderBenchDeltaTable renders the "old vs new ns/op" comparison table in +// the same column format `gotest bench --against` established +// (cmd/gotest/bench.go's printBenchDeltaTable), so the table looks +// identical whether it's driven directly by the bench command or via a +// spec/summary render pass. deltas is rendered as given — filtering +// significant-only vs. every row (-v) is the caller's responsibility (see +// WithBenchDeltas). No-ops when deltas is empty. +func renderBenchDeltaTable(w io.Writer, deltas []BenchDelta, c colors) { //nolint:gocritic // hugeParam: stable API + // nil (WithBenchDeltas never called) means "no comparison happened at + // all" -> no-op. An empty-but-non-nil slice (a comparison ran but every + // row was filtered out, e.g. no significant deltas without -v) still + // prints the header, so the reader can see a comparison was attempted + // even when it found nothing worth flagging. + if deltas == nil { + return + } + fmt.Fprintln(w) + fmt.Fprintf(w, "%sBENCHMARK OLD ns/op NEW ns/op Δ%s\n", c.bold, c.reset) + for _, d := range deltas { + sign := "" + if d.PercentChange >= 0 { + sign = "+" + } + warn, clr, reset := "", "", "" + if d.Significant && d.PercentChange > 0 { + warn, clr, reset = " ⚠", c.red, c.reset + } + fmt.Fprintf(w, "%s%s %.1f %.1f %s%.1f%%%s%s\n", + clr, d.Key, d.OldNs, d.NewNs, sign, d.PercentChange, warn, reset) + } +} + func renderSummary(w io.Writer, stats Stats, c colors) { //nolint:gocritic // hugeParam: stable API var parts []string if stats.Passed > 0 { @@ -238,6 +317,9 @@ func renderSummary(w io.Writer, stats Stats, c colors) { //nolint:gocritic // hu if stats.Tests > 0 { counts = append(counts, fmt.Sprintf("%d stdlib tests", stats.Tests)) } + if stats.Benchmarks > 0 { + counts = append(counts, fmt.Sprintf("%d benchmarks", stats.Benchmarks)) + } if len(counts) == 0 { counts = append(counts, "0 suites") } diff --git a/internal/gotestspec/tree.go b/internal/gotestspec/tree.go index 9ed2b8fd..e53f99c7 100644 --- a/internal/gotestspec/tree.go +++ b/internal/gotestspec/tree.go @@ -1,9 +1,13 @@ package gotestspec import ( + "regexp" "sort" + "strconv" "strings" "time" + "unicode" + "unicode/utf8" "github.com/mvrahden/go-test/internal/protocol" ) @@ -26,21 +30,28 @@ const ( KindMethod KindBlock KindTest + // KindBenchmark is appended at the end of the iota block to keep the + // numeric values of the existing kinds stable across serialized state. + KindBenchmark ) type Node struct { - Name string - Display string - Kind NodeKind - Status Status - Duration time.Duration - Output []string - Children []*Node - Focused bool - Excluded bool - External bool - Variant int - duplicate bool + Name string + Display string + Kind NodeKind + Status Status + Duration time.Duration + Output []string + Children []*Node + Focused bool + Excluded bool + External bool + Variant int + duplicate bool + Iterations int + NsPerOp float64 + BytesPerOp int64 + AllocsPerOp int64 } type Package struct { @@ -52,12 +63,13 @@ type Package struct { } type Stats struct { - Suites int - Behaviors int - Tests int - Passed int - Failed int - Skipped int + Suites int + Behaviors int + Tests int + Benchmarks int + Passed int + Failed int + Skipped int // FailedPackages counts packages whose verdict sits on the package itself // — a build failure, a TestMain os.Exit, a crash outside any test. These // carry no failing behavior, so folding them into Failed would break the @@ -148,6 +160,33 @@ func BuildTree(events []TestEvent) []*Package { switch ev.Action { case ActionOutput: node.Output = append(node.Output, ev.Output) + lastSegment := resolvedSegments[len(resolvedSegments)-1] + if isBenchmarkName(lastSegment) { + // test2json "output" events are not guaranteed line-aligned; + // under real subprocess pipe timing a bench result line can + // arrive split mid-token across two consecutive events. Scan + // the node's joined output (not just this single event) so a + // line only completed by a later event is still found. This + // mirrors the defense harvestPackageOutputSamples applies to + // the untagged package-output path (see baseline.go). + if iters, nsPerOp, bPerOp, allocsPerOp, ok := scanBenchOutput(node.Output); ok { + node.Iterations = iters + node.NsPerOp = nsPerOp + node.BytesPerOp = bPerOp + node.AllocsPerOp = allocsPerOp + // go test's own -json encoder never emits a "pass" + // event for a benchmark (see ActionBench doc comment); + // reaching a parsed ns/op line is the only success + // signal a real benchmark run ever produces. + if node.Status == StatusNone { + node.Status = StatusPass + } + } + } + case ActionBench: + if node.Status == StatusNone { + node.Status = StatusPass + } case ActionPass, ActionFail, ActionSkip: node.Status = statusFrom(ev.Action) node.Duration = elapsed(ev.Elapsed) @@ -239,6 +278,66 @@ func stripDuplicateSuffix(s string) string { return s[:idx] } +// isBenchmarkName reports whether name is a Go benchmark identifier: +// "Benchmark" followed by nothing or a non-lowercase character — mirroring +// the stdlib's TestXxx/BenchmarkXxx convention. This is boundary-aware so +// that names like "Benchmarking_the_new_endpoint" (lowercase continuation) +// are not mistaken for benchmark identifiers. +func isBenchmarkName(name string) bool { + rest, ok := strings.CutPrefix(name, protocol.PrefixBenchmark) + if !ok { + return false + } + if rest == "" { + return true + } + r, _ := utf8.DecodeRuneInString(rest) + return !unicode.IsLower(r) +} + +var benchLineRe = regexp.MustCompile(`^Benchmark\S+?(?:-\d+)?\s+(\d+)\s+([\d.]+) ns/op(?:\s+(\d+) B/op)?(?:\s+(\d+) allocs/op)?`) + +// parseBenchOutput parses a go test benchmark result line, e.g.: +// +// BenchmarkFoo-8 1201 985.2 ns/op 24 B/op 3 allocs/op +// +// B/op and allocs/op are optional (absent unless -benchmem is set). +func parseBenchOutput(line string) (iters int, nsPerOp float64, bPerOp, allocsPerOp int64, ok bool) { + m := benchLineRe.FindStringSubmatch(strings.TrimSpace(line)) + if m == nil { + return 0, 0, 0, 0, false + } + iters, err := strconv.Atoi(m[1]) + if err != nil { + return 0, 0, 0, 0, false + } + nsPerOp, err = strconv.ParseFloat(m[2], 64) + if err != nil { + return 0, 0, 0, 0, false + } + if m[3] != "" { + bPerOp, _ = strconv.ParseInt(m[3], 10, 64) + } + if m[4] != "" { + allocsPerOp, _ = strconv.ParseInt(m[4], 10, 64) + } + return iters, nsPerOp, bPerOp, allocsPerOp, true +} + +// scanBenchOutput joins a node's accumulated output events back into a +// single stream and re-splits it on "\n" before scanning for a bench result +// line, so a line split mid-token across two output events (as test2json +// may produce under real pipe timing) is still parsed correctly. +func scanBenchOutput(output []string) (iters int, nsPerOp float64, bPerOp, allocsPerOp int64, ok bool) { + joined := strings.Join(output, "") + for _, line := range strings.Split(joined, "\n") { + if iters, nsPerOp, bPerOp, allocsPerOp, ok = parseBenchOutput(line); ok { + return + } + } + return 0, 0, 0, 0, false +} + func CollectStats(packages []*Package) Stats { var s Stats for _, pkg := range packages { @@ -270,6 +369,10 @@ func PkgFailedOnItsOwn(pkg *Package) bool { } func collectStats(n *Node, s *Stats, inStdlib bool) { + if n.Kind == KindBenchmark && len(n.Children) == 0 { + s.Benchmarks++ + return + } if n.Kind == KindSuite { s.Suites++ } @@ -308,6 +411,7 @@ func classify(n *Node, topLevel bool) { name := n.Name if topLevel { + hasTestPrefix := strings.HasPrefix(name, "Test") raw := strings.TrimPrefix(name, "Test") if strings.HasPrefix(raw, protocol.PrefixFocused) { @@ -325,6 +429,14 @@ func classify(n *Node, topLevel bool) { case strings.HasSuffix(raw, protocol.SuffixTestSuite): n.Kind = KindSuite n.Display = strings.TrimSuffix(raw, protocol.SuffixTestSuite) + case !hasTestPrefix && isBenchmarkName(raw): + // A bare top-level Benchmark* node (no enclosing TestSuite), + // e.g. from a plain go test -bench=. -json stream fed via + // `gotest spec --input`. Names that started with "Test" (e.g. + // "TestBenchmarkFoo", a legitimate stdlib test) must never + // reach this branch — hasTestPrefix guards against that. + n.Kind = KindBenchmark + n.Display = strings.TrimPrefix(raw, protocol.PrefixBenchmark) default: n.Kind = KindTest n.Display = strings.TrimPrefix(raw, "_") @@ -339,6 +451,9 @@ func classify(n *Node, topLevel bool) { } switch { + case isBenchmarkName(name): + n.Kind = KindBenchmark + n.Display = strings.TrimPrefix(name, protocol.PrefixBenchmark) case strings.HasPrefix(name, "Test"): n.Kind = KindMethod n.Display = strings.TrimPrefix(name, "Test") diff --git a/internal/gotestspec/tree_test.go b/internal/gotestspec/tree_test.go index ccca26bc..38ce89c0 100644 --- a/internal/gotestspec/tree_test.go +++ b/internal/gotestspec/tree_test.go @@ -3,6 +3,8 @@ package gotestspec //nolint:stdlib-test import ( "strings" "testing" + + "github.com/mvrahden/go-test/pkg/gotest" ) func TestBuildTree_SuiteHierarchy(t *testing.T) { @@ -17,65 +19,33 @@ func TestBuildTree_SuiteHierarchy(t *testing.T) { {"Action":"pass","Package":"example.com/pkg","Elapsed":0.5}` events, err := ParseEvents(strings.NewReader(input)) - if err != nil { - t.Fatal(err) - } + gotest.NoError(t, err) tree := BuildTree(events) - if len(tree) != 1 { - t.Fatalf("expected 1 package, got %d", len(tree)) - } + gotest.Len(t, tree, 1, "packages") pkg := tree[0] - if pkg.Path != "example.com/pkg" { - t.Errorf("package path = %q", pkg.Path) - } - if len(pkg.Nodes) != 1 { - t.Fatalf("expected 1 root node, got %d", len(pkg.Nodes)) - } + gotest.Equal(t, "example.com/pkg", pkg.Path) + gotest.Len(t, pkg.Nodes, 1, "root nodes") suite := pkg.Nodes[0] - if suite.Kind != KindSuite { - t.Errorf("root kind = %d, want KindSuite", suite.Kind) - } - if suite.Display != "UserService" { - t.Errorf("suite display = %q, want UserService", suite.Display) - } + gotest.Equal(t, KindSuite, suite.Kind) + gotest.Equal(t, "UserService", suite.Display) - if len(suite.Children) != 1 { - t.Fatalf("expected 1 method, got %d", len(suite.Children)) - } + gotest.Len(t, suite.Children, 1, "methods") method := suite.Children[0] - if method.Kind != KindMethod { - t.Errorf("method kind = %d, want KindMethod", method.Kind) - } - if method.Display != "Create" { - t.Errorf("method display = %q, want Create", method.Display) - } + gotest.Equal(t, KindMethod, method.Kind) + gotest.Equal(t, "Create", method.Display) - if len(method.Children) != 1 { - t.Fatalf("expected 1 when block, got %d", len(method.Children)) - } + gotest.Len(t, method.Children, 1, "when blocks") when := method.Children[0] - if when.Kind != KindBlock { - t.Errorf("when kind = %d, want KindBlock", when.Kind) - } - if when.Display != "when email is valid" { - t.Errorf("when display = %q", when.Display) - } + gotest.Equal(t, KindBlock, when.Kind) + gotest.Equal(t, "when email is valid", when.Display) - if len(when.Children) != 1 { - t.Fatalf("expected 1 it block, got %d", len(when.Children)) - } + gotest.Len(t, when.Children, 1, "it blocks") it := when.Children[0] - if it.Kind != KindBlock { - t.Errorf("it kind = %d, want KindBlock", it.Kind) - } - if it.Display != "creates the user" { - t.Errorf("it display = %q", it.Display) - } - if it.Status != StatusPass { - t.Errorf("it status = %d, want StatusPass", it.Status) - } + gotest.Equal(t, KindBlock, it.Kind) + gotest.Equal(t, "creates the user", it.Display) + gotest.Equal(t, StatusPass, it.Status) } func TestBuildTree_FixtureHierarchy(t *testing.T) { @@ -90,43 +60,25 @@ func TestBuildTree_FixtureHierarchy(t *testing.T) { {"Action":"pass","Package":"example.com/e2e","Elapsed":0.1}` events, err := ParseEvents(strings.NewReader(input)) - if err != nil { - t.Fatal(err) - } + gotest.NoError(t, err) tree := BuildTree(events) pkg := tree[0] fixture := pkg.Nodes[0] - if fixture.Kind != KindFixture { - t.Errorf("root kind = %d, want KindFixture", fixture.Kind) - } - if fixture.Display != "Infra" { - t.Errorf("fixture display = %q, want Infra", fixture.Display) - } + gotest.Equal(t, KindFixture, fixture.Kind) + gotest.Equal(t, "Infra", fixture.Display) child := fixture.Children[0] - if child.Kind != KindFixture { - t.Errorf("child kind = %d, want KindFixture", child.Kind) - } - if child.Display != "API" { - t.Errorf("child display = %q, want API", child.Display) - } + gotest.Equal(t, KindFixture, child.Kind) + gotest.Equal(t, "API", child.Display) suite := child.Children[0] - if suite.Kind != KindSuite { - t.Errorf("suite kind = %d, want KindSuite", suite.Kind) - } - if suite.Display != "Batch" { - t.Errorf("suite display = %q, want Batch", suite.Display) - } + gotest.Equal(t, KindSuite, suite.Kind) + gotest.Equal(t, "Batch", suite.Display) method := suite.Children[0] - if method.Kind != KindMethod { - t.Errorf("method kind = %d, want KindMethod", method.Kind) - } - if method.Display != "Dispatch" { - t.Errorf("method display = %q, want Dispatch", method.Display) - } + gotest.Equal(t, KindMethod, method.Kind) + gotest.Equal(t, "Dispatch", method.Display) } func TestBuildTree_FocusedSuite(t *testing.T) { @@ -140,12 +92,8 @@ func TestBuildTree_FocusedSuite(t *testing.T) { tree := BuildTree(events) suite := tree[0].Nodes[0] - if !suite.Focused { - t.Error("expected suite to be focused") - } - if suite.Display != "PaymentService" { - t.Errorf("display = %q, want PaymentService", suite.Display) - } + gotest.True(t, suite.Focused, "expected suite to be focused") + gotest.Equal(t, "PaymentService", suite.Display) } func TestBuildTree_ExcludedSuite(t *testing.T) { @@ -157,15 +105,9 @@ func TestBuildTree_ExcludedSuite(t *testing.T) { tree := BuildTree(events) suite := tree[0].Nodes[0] - if !suite.Excluded { - t.Error("expected suite to be excluded") - } - if suite.Display != "Broken" { - t.Errorf("display = %q, want Broken", suite.Display) - } - if suite.Status != StatusSkip { - t.Errorf("status = %d, want StatusSkip", suite.Status) - } + gotest.True(t, suite.Excluded, "expected suite to be excluded") + gotest.Equal(t, "Broken", suite.Display) + gotest.Equal(t, StatusSkip, suite.Status) } func TestCollectStats(t *testing.T) { @@ -185,24 +127,12 @@ func TestCollectStats(t *testing.T) { tree := BuildTree(events) stats := CollectStats(tree) - if stats.Suites != 2 { - t.Errorf("suites = %d, want 2", stats.Suites) - } - if stats.Behaviors != 3 { - t.Errorf("behaviors = %d, want 3", stats.Behaviors) - } - if stats.Tests != 0 { - t.Errorf("tests = %d, want 0", stats.Tests) - } - if stats.Passed != 1 { - t.Errorf("passed = %d, want 1", stats.Passed) - } - if stats.Failed != 1 { - t.Errorf("failed = %d, want 1", stats.Failed) - } - if stats.Skipped != 1 { - t.Errorf("skipped = %d, want 1", stats.Skipped) - } + gotest.Equal(t, 2, stats.Suites, "suites") + gotest.Equal(t, 3, stats.Behaviors, "behaviors") + gotest.Equal(t, 0, stats.Tests, "tests") + gotest.Equal(t, 1, stats.Passed, "passed") + gotest.Equal(t, 1, stats.Failed, "failed") + gotest.Equal(t, 1, stats.Skipped, "skipped") } func TestBuildTree_StdlibTest(t *testing.T) { @@ -215,32 +145,18 @@ func TestBuildTree_StdlibTest(t *testing.T) { {"Action":"pass","Package":"example.com/pkg","Elapsed":0.01}` events, err := ParseEvents(strings.NewReader(input)) - if err != nil { - t.Fatal(err) - } + gotest.NoError(t, err) tree := BuildTree(events) pkg := tree[0] - if len(pkg.Nodes) != 1 { - t.Fatalf("expected 1 root node, got %d", len(pkg.Nodes)) - } + gotest.Len(t, pkg.Nodes, 1, "root nodes") test := pkg.Nodes[0] - if test.Kind != KindTest { - t.Errorf("root kind = %d, want KindTest", test.Kind) - } - if test.Display != "CreateUser" { - t.Errorf("display = %q, want CreateUser", test.Display) - } - if len(test.Children) != 2 { - t.Fatalf("expected 2 subtests, got %d", len(test.Children)) - } - if test.Children[0].Kind != KindBlock { - t.Errorf("subtest kind = %d, want KindBlock", test.Children[0].Kind) - } - if test.Children[0].Display != "valid email" { - t.Errorf("subtest display = %q, want 'valid email'", test.Children[0].Display) - } + gotest.Equal(t, KindTest, test.Kind) + gotest.Equal(t, "CreateUser", test.Display) + gotest.Len(t, test.Children, 2, "subtests") + gotest.Equal(t, KindBlock, test.Children[0].Kind) + gotest.Equal(t, "valid email", test.Children[0].Display) } func TestCollectStats_Mixed(t *testing.T) { @@ -260,18 +176,10 @@ func TestCollectStats_Mixed(t *testing.T) { tree := BuildTree(events) stats := CollectStats(tree) - if stats.Suites != 1 { - t.Errorf("suites = %d, want 1", stats.Suites) - } - if stats.Behaviors != 1 { - t.Errorf("behaviors = %d, want 1", stats.Behaviors) - } - if stats.Tests != 2 { - t.Errorf("tests = %d, want 2", stats.Tests) - } - if stats.Passed != 3 { - t.Errorf("passed = %d, want 3", stats.Passed) - } + gotest.Equal(t, 1, stats.Suites, "suites") + gotest.Equal(t, 1, stats.Behaviors, "behaviors") + gotest.Equal(t, 2, stats.Tests, "tests") + gotest.Equal(t, 3, stats.Passed, "passed") } func TestCollectStats_StdlibOnly(t *testing.T) { @@ -285,18 +193,10 @@ func TestCollectStats_StdlibOnly(t *testing.T) { tree := BuildTree(events) stats := CollectStats(tree) - if stats.Suites != 0 { - t.Errorf("suites = %d, want 0", stats.Suites) - } - if stats.Behaviors != 0 { - t.Errorf("behaviors = %d, want 0", stats.Behaviors) - } - if stats.Tests != 2 { - t.Errorf("tests = %d, want 2", stats.Tests) - } - if stats.Passed != 2 { - t.Errorf("passed = %d, want 2", stats.Passed) - } + gotest.Equal(t, 0, stats.Suites, "suites") + gotest.Equal(t, 0, stats.Behaviors, "behaviors") + gotest.Equal(t, 2, stats.Tests, "tests") + gotest.Equal(t, 2, stats.Passed, "passed") } func TestSplitTestPath(t *testing.T) { @@ -320,13 +220,9 @@ func TestSplitTestPath(t *testing.T) { if len(got) == 0 && len(tt.want) == 0 { return } - if len(got) != len(tt.want) { - t.Fatalf("splitTestPath(%q) = %v, want %v", tt.path, got, tt.want) - } + gotest.Len(t, got, len(tt.want), "splitTestPath(%q) = %v, want %v", tt.path, got, tt.want) for i := range got { - if got[i] != tt.want[i] { - t.Errorf("splitTestPath(%q)[%d] = %q, want %q", tt.path, i, got[i], tt.want[i]) - } + gotest.Equal(t, tt.want[i], got[i], "splitTestPath(%q)[%d]", tt.path, i) } }) } @@ -350,73 +246,41 @@ func TestBuildTree_DuplicateSuite_PtestPxtest(t *testing.T) { {"Action":"pass","Package":"example.com/stdlib","Elapsed":0.05}` events, err := ParseEvents(strings.NewReader(input)) - if err != nil { - t.Fatal(err) - } + gotest.NoError(t, err) tree := BuildTree(events) - if len(tree) != 1 { - t.Fatalf("expected 1 package, got %d", len(tree)) - } + gotest.Len(t, tree, 1, "packages") pkg := tree[0] // Should produce 2 separate suite nodes, not 1 merged one. - if len(pkg.Nodes) != 2 { - t.Fatalf("expected 2 root nodes, got %d", len(pkg.Nodes)) - } + gotest.Len(t, pkg.Nodes, 2, "root nodes") suite1 := pkg.Nodes[0] suite2 := pkg.Nodes[1] - if suite1.Kind != KindSuite { - t.Errorf("suite1 kind = %d, want KindSuite", suite1.Kind) - } - if suite2.Kind != KindSuite { - t.Errorf("suite2 kind = %d, want KindSuite", suite2.Kind) - } - if suite1.Display != "Unit" { - t.Errorf("suite1 display = %q, want Unit", suite1.Display) - } - if suite2.Display != "Unit" { - t.Errorf("suite2 display = %q, want Unit", suite2.Display) - } + gotest.Equal(t, KindSuite, suite1.Kind, "suite1 kind") + gotest.Equal(t, KindSuite, suite2.Kind, "suite2 kind") + gotest.Equal(t, "Unit", suite1.Display, "suite1 display") + gotest.Equal(t, "Unit", suite2.Display, "suite2 display") // Each suite should have 2 methods (not 4 merged). - if len(suite1.Children) != 2 { - t.Fatalf("suite1 expected 2 children, got %d", len(suite1.Children)) - } - if len(suite2.Children) != 2 { - t.Fatalf("suite2 expected 2 children, got %d", len(suite2.Children)) - } + gotest.Len(t, suite1.Children, 2, "suite1 children") + gotest.Len(t, suite2.Children, 2, "suite2 children") // Children of suite2 should NOT have #01 suffix. for _, c := range suite2.Children { - if strings.Contains(c.Name, "#") { - t.Errorf("suite2 child %q still has # suffix", c.Name) - } - if strings.Contains(c.Display, "#") { - t.Errorf("suite2 child display %q still has # suffix", c.Display) - } + gotest.NotContains(t, c.Name, "#", "suite2 child name still has # suffix") + gotest.NotContains(t, c.Display, "#", "suite2 child display still has # suffix") } // suite2 should be marked as variant 2 and external. - if suite2.Variant != 2 { - t.Errorf("suite2 variant = %d, want 2", suite2.Variant) - } - if !suite2.External { - t.Error("expected suite2 to be external") - } - if suite1.External { - t.Error("expected suite1 to not be external") - } + gotest.Equal(t, 2, suite2.Variant, "suite2 variant") + gotest.True(t, suite2.External, "expected suite2 to be external") + gotest.False(t, suite1.External, "expected suite1 to not be external") // Both should have pass status. - if suite1.Status != StatusPass { - t.Errorf("suite1 status = %d, want StatusPass", suite1.Status) - } - if suite2.Status != StatusPass { - t.Errorf("suite2 status = %d, want StatusPass", suite2.Status) - } + gotest.Equal(t, StatusPass, suite1.Status, "suite1 status") + gotest.Equal(t, StatusPass, suite2.Status, "suite2 status") } func TestClassify_ParallelMethod(t *testing.T) { @@ -430,12 +294,100 @@ func TestClassify_ParallelMethod(t *testing.T) { tree := BuildTree(events) method := tree[0].Nodes[0].Children[0] - if method.Kind != KindMethod { - t.Errorf("kind = %d, want KindMethod", method.Kind) - } - if method.Display != "ParallelCreate" { - t.Errorf("display = %q, want ParallelCreate", method.Display) - } + gotest.Equal(t, KindMethod, method.Kind) + gotest.Equal(t, "ParallelCreate", method.Display) +} + +func TestBuildTree_BenchmarkEvents(t *testing.T) { + events := []TestEvent{ + {Action: ActionOutput, Package: "p", Test: "BenchmarkFooTestSuite/BenchmarkParse", + Output: "BenchmarkFooTestSuite/BenchmarkParse-8 \t 1201 \t 985.2 ns/op \t 24 B/op \t 3 allocs/op\n"}, + {Action: ActionBench, Package: "p", Test: "BenchmarkFooTestSuite/BenchmarkParse"}, + } + pkgs := BuildTree(events) + leaf := pkgs[0].Nodes[0].Children[0] + gotest.Equal(t, KindBenchmark, leaf.Kind) + gotest.Equal(t, StatusPass, leaf.Status) + gotest.InDelta(t, 985.2, leaf.NsPerOp, 0.001) + gotest.Equal(t, int64(24), leaf.BytesPerOp) + gotest.Equal(t, int64(3), leaf.AllocsPerOp) + stats := CollectStats(pkgs) + gotest.Equal(t, 1, stats.Benchmarks) + gotest.Equal(t, 0, stats.Behaviors) +} + +func TestBuildTree_BenchmarkOutputSplitAcrossEvents(t *testing.T) { + // test2json "output" events are not guaranteed line-aligned; under real + // subprocess pipe timing a bench result line can arrive split mid-token + // across two consecutive events for the same tagged Test. Metrics must + // still be recovered by scanning the node's joined output, not just the + // single event that happens to complete the line. + events := []TestEvent{ + {Action: ActionOutput, Package: "p", Test: "BenchmarkFoo", + Output: "BenchmarkFoo-8 \t 12"}, + {Action: ActionOutput, Package: "p", Test: "BenchmarkFoo", + Output: "01 \t 985.2 ns/op\n"}, + {Action: ActionBench, Package: "p", Test: "BenchmarkFoo"}, + } + pkgs := BuildTree(events) + leaf := pkgs[0].Nodes[0] + gotest.Equal(t, KindBenchmark, leaf.Kind) + gotest.Equal(t, StatusPass, leaf.Status) + gotest.Equal(t, 1201, leaf.Iterations) + gotest.InDelta(t, 985.2, leaf.NsPerOp, 0.001) +} + +func TestClassify_TopLevelTestNamedBenchmarkIsNotABenchmark(t *testing.T) { + // "TestBenchmarkFoo" is a legitimate stdlib test whose name merely + // starts with "Benchmark" after the "Test" prefix is trimmed. It must + // resolve through the ordinary test classification, never the bench + // branch. + input := `{"Action":"run","Package":"p","Test":"TestBenchmarkFoo"} +{"Action":"pass","Package":"p","Test":"TestBenchmarkFoo","Elapsed":0.01} +{"Action":"pass","Package":"p","Elapsed":0.01}` + + events, err := ParseEvents(strings.NewReader(input)) + gotest.NoError(t, err) + tree := BuildTree(events) + + node := tree[0].Nodes[0] + gotest.Equal(t, KindTest, node.Kind) + gotest.Equal(t, "BenchmarkFoo", node.Display) +} + +func TestClassify_NestedBenchmarkingPrefixIsNotABenchmark(t *testing.T) { + // "Benchmarking_the_new_endpoint" starts with "Benchmark" but continues + // with a lowercase letter ("ing..."), so it is an ordinary "when/it" + // style block name, not a Go benchmark identifier. + input := `{"Action":"run","Package":"p","Test":"TestFooTestSuite"} +{"Action":"run","Package":"p","Test":"TestFooTestSuite/TestBar"} +{"Action":"run","Package":"p","Test":"TestFooTestSuite/TestBar/Benchmarking_the_new_endpoint"} +{"Action":"pass","Package":"p","Test":"TestFooTestSuite/TestBar/Benchmarking_the_new_endpoint","Elapsed":0.01} +{"Action":"pass","Package":"p","Test":"TestFooTestSuite/TestBar","Elapsed":0.01} +{"Action":"pass","Package":"p","Test":"TestFooTestSuite","Elapsed":0.01} +{"Action":"pass","Package":"p","Elapsed":0.01}` + + events, err := ParseEvents(strings.NewReader(input)) + gotest.NoError(t, err) + tree := BuildTree(events) + + block := tree[0].Nodes[0].Children[0].Children[0] + gotest.Equal(t, KindBlock, block.Kind) + gotest.Equal(t, "Benchmarking the new endpoint", block.Display) +} + +func TestClassify_NestedBenchmarkName(t *testing.T) { + input := `{"Action":"run","Package":"p","Test":"TestFooTestSuite"} +{"Action":"run","Package":"p","Test":"TestFooTestSuite/BenchmarkParse"} +{"Action":"output","Package":"p","Test":"TestFooTestSuite/BenchmarkParse","Output":"BenchmarkParse-8 \t 100 \t 10.0 ns/op\n"}` + + events, err := ParseEvents(strings.NewReader(input)) + gotest.NoError(t, err) + tree := BuildTree(events) + + leaf := tree[0].Nodes[0].Children[0] + gotest.Equal(t, KindBenchmark, leaf.Kind) + gotest.Equal(t, "Parse", leaf.Display) } func TestBuildTree_PackageLevelOutput(t *testing.T) { @@ -452,44 +404,26 @@ func TestBuildTree_PackageLevelOutput(t *testing.T) { {"Action":"fail","Package":"p","Elapsed":1.0}` events, err := ParseEvents(strings.NewReader(input)) - if err != nil { - t.Fatal(err) - } + gotest.NoError(t, err) tree := BuildTree(events) - if len(tree) != 1 { - t.Fatalf("expected 1 package, got %d", len(tree)) - } + gotest.Len(t, tree, 1, "packages") pkg := tree[0] // Test node should be present and passed - if len(pkg.Nodes) != 1 { - t.Fatalf("expected 1 node, got %d", len(pkg.Nodes)) - } - if pkg.Nodes[0].Status != StatusPass { - t.Errorf("test status = %d, want StatusPass", pkg.Nodes[0].Status) - } + gotest.Len(t, pkg.Nodes, 1, "nodes") + gotest.Equal(t, StatusPass, pkg.Nodes[0].Status) // Package should have failed - if pkg.Status != StatusFail { - t.Errorf("package status = %d, want StatusFail", pkg.Status) - } + gotest.Equal(t, StatusFail, pkg.Status) // Package-level diagnostic output should be collected - if len(pkg.Output) == 0 { - t.Fatal("expected package-level output, got none") - } + gotest.NotEmpty(t, pkg.Output, "expected package-level output") combined := strings.Join(pkg.Output, "") - if !strings.Contains(combined, "WARNING: DATA RACE") { - t.Errorf("package output should contain race warning, got:\n%s", combined) - } - if !strings.Contains(combined, "Found 1 data race(s)") { - t.Errorf("package output should contain race count, got:\n%s", combined) - } + gotest.Contains(t, combined, "WARNING: DATA RACE") + gotest.Contains(t, combined, "Found 1 data race(s)") // Summary lines should NOT be in package output - if strings.Contains(combined, "FAIL\tp\t") { - t.Errorf("package output should not contain summary line, got:\n%s", combined) - } + gotest.NotContains(t, combined, "FAIL\tp\t", "package output should not contain summary line") } diff --git a/internal/lint/bench.go b/internal/lint/bench.go new file mode 100644 index 00000000..736305ff --- /dev/null +++ b/internal/lint/bench.go @@ -0,0 +1,313 @@ +package lint + +import ( + "go/ast" + "go/token" + "go/types" + "strings" + + "github.com/mvrahden/go-test/internal/protocol" + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/ast/inspector" +) + +// checkBenchLoop flags a suite Benchmark* method whose body never touches +// b.Loop() or b.N — i.e. it never actually iterates, so nothing is +// measured. Only pointer-receiver methods on a discovered suite are +// considered; a non-pointer-receiver suite method is already flagged by +// the Receiver rule and would never be dispatched as a benchmark anyway. +func checkBenchLoop(pass *analysis.Pass, insp *inspector.Inspector, suites map[string]*suiteInfo) { + insp.Preorder([]ast.Node{(*ast.FuncDecl)(nil)}, func(n ast.Node) { + fd := n.(*ast.FuncDecl) + if fd.Body == nil || !isPointerReceiver(fd.Recv) { + return + } + + recvName := receiverTypeName(fd.Recv) + if _, ok := suites[recvName]; !ok { + return + } + + methodName := fd.Name.Name + if !isBenchmarkMethodName(methodName) { + return + } + + param := benchParamName(fd) + if param == "" || benchBodyUsesLoopOrN(fd.Body, param) { + return + } + + report(pass, BenchLoop, fd.Pos(), + "benchmark %s never calls b.Loop() — nothing is measured", recvName+"."+methodName) + }) +} + +// checkBenchFixtureIO flags a suite Benchmark* method that reads a +// fixture-typed field from inside its measured loop. gotest's generated +// wrapper fences the timer around BeforeEach/AfterEach (b.StopTimer() ... +// BeforeEach ... b.StartTimer(); b.ResetTimer(); method), so fixture setup +// itself is never measured — that structural guarantee is what the docs +// promise. What it cannot save you from is the benchmark method's own body +// reading fixture-backed state *inside* the loop: if the fixture is backed +// by a database or a network service, that read times whatever backs the +// fixture, not the code under test. +func checkBenchFixtureIO(pass *analysis.Pass, insp *inspector.Inspector, suites map[string]*suiteInfo) { + insp.Preorder([]ast.Node{(*ast.FuncDecl)(nil)}, func(n ast.Node) { + fd := n.(*ast.FuncDecl) + if fd.Body == nil || !isPointerReceiver(fd.Recv) { + return + } + + recvName := receiverTypeName(fd.Recv) + suite, ok := suites[recvName] + if !ok || len(suite.fixtureFields) == 0 { + return + } + + methodName := fd.Name.Name + if !isBenchmarkMethodName(methodName) { + return + } + + recvIdent := receiverIdentName(fd.Recv) + if recvIdent == "" || recvIdent == "_" { + return + } + + param := benchParamName(fd) + if param == "" || param == "_" { + return + } + + region := benchMeasuredRegion(fd.Body, param) + if region == nil { + // No b.Loop()/b.N loop found — the bench-loop rule already + // covers that case; nothing to report here. + return + } + + field := findFixtureRead(region, recvIdent, suite.fixtureFields) + if field == "" { + return + } + + report(pass, BenchFixtureIO, fd.Pos(), + "benchmark %s reads fixture-backed state %s inside the measured loop — hoist the read above the loop, or you are timing whatever backs the fixture", + recvName+"."+methodName, recvIdent+"."+field) + }) +} + +// checkBenchWait flags waiting primitives inside the measured loop of a +// Benchmark* suite method: time.Sleep and gotest's Eventually/Consistently +// pollers. The loop then times the wait, not the code — a result that says +// nothing about the operation being benchmarked. Settling belongs above +// the loop; a property that needs polling belongs in a test, not a +// benchmark. +func checkBenchWait(pass *analysis.Pass, insp *inspector.Inspector, suites map[string]*suiteInfo) { + insp.Preorder([]ast.Node{(*ast.FuncDecl)(nil)}, func(n ast.Node) { + fd := n.(*ast.FuncDecl) + if fd.Body == nil || !isPointerReceiver(fd.Recv) { + return + } + + recvName := receiverTypeName(fd.Recv) + if _, ok := suites[recvName]; !ok { + return + } + + methodName := fd.Name.Name + if !isBenchmarkMethodName(methodName) { + return + } + + param := benchParamName(fd) + if param == "" { + return + } + region := benchMeasuredRegion(fd.Body, param) + if region == nil { + return + } + + ast.Inspect(region, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + if name := waitingCallName(pass, call); name != "" { + report(pass, BenchWait, call.Pos(), + "benchmark %s calls %s inside the measured loop — this times the wait, not the code; move it outside the loop", + recvName+"."+methodName, name) + } + return true + }) + }) +} + +// waitingCallName resolves call to a known waiting primitive — time.Sleep +// (matched by import path, so aliased imports stay covered) or gotest's +// Eventually/Consistently — and returns its display name, or "". +func waitingCallName(pass *analysis.Pass, call *ast.CallExpr) string { + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return "" + } + if id, ok := sel.X.(*ast.Ident); ok && sel.Sel.Name == "Sleep" { + if pkgName, ok := pass.TypesInfo.Uses[id].(*types.PkgName); ok && pkgName.Imported().Path() == "time" { + return "time.Sleep" + } + } + if fn, ok := pass.TypesInfo.Uses[sel.Sel].(*types.Func); ok && fn.Pkg() != nil && fn.Pkg().Path() == gotestImportPath { + if fn.Name() == "Eventually" || fn.Name() == "Consistently" { + return "gotest." + fn.Name() + } + } + return "" +} + +// receiverIdentName returns the receiver's identifier name (e.g. "s" in +// "func (s *Suite) Method()"), or "" if the receiver has no name. +func receiverIdentName(recv *ast.FieldList) string { + if recv == nil || len(recv.List) == 0 || len(recv.List[0].Names) == 0 { + return "" + } + return recv.List[0].Names[0].Name +} + +// benchMeasuredRegion returns the body of the benchmark method's timed +// loop: either `for .Loop() { ... }` or the classic +// `for i := 0; i < .N; i++ { ... }` form. It searches the whole +// method body (not just its top level), and returns nil if no such loop is +// found — the bench-loop rule already flags that shape. +// +// Only the first such loop is returned. A benchmark with two of them is not +// merely unusual: a second b.Loop() panics at run time with "B.Loop called +// with timer stopped", so the shape cannot survive execution long enough for +// a missed diagnostic to matter. +func benchMeasuredRegion(body *ast.BlockStmt, param string) *ast.BlockStmt { + var region *ast.BlockStmt + ast.Inspect(body, func(n ast.Node) bool { + if region != nil { + return false + } + forStmt, ok := n.(*ast.ForStmt) + if !ok { + return true + } + if isBenchLoopCond(forStmt.Cond, param) || isBenchNCond(forStmt.Cond, param) { + region = forStmt.Body + return false + } + return true + }) + return region +} + +// isBenchLoopCond reports whether cond is a call to .Loop(). +func isBenchLoopCond(cond ast.Expr, param string) bool { + call, ok := cond.(*ast.CallExpr) + if !ok { + return false + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return false + } + id, ok := sel.X.(*ast.Ident) + return ok && id.Name == param && sel.Sel.Name == "Loop" +} + +// isBenchNCond reports whether cond is the classic `i < .N` form. +func isBenchNCond(cond ast.Expr, param string) bool { + bin, ok := cond.(*ast.BinaryExpr) + if !ok || bin.Op != token.LSS { + return false + } + sel, ok := bin.Y.(*ast.SelectorExpr) + if !ok { + return false + } + id, ok := sel.X.(*ast.Ident) + return ok && id.Name == param && sel.Sel.Name == "N" +} + +// findFixtureRead walks region — including nested loops and closures — +// for the first selector expression of the form . where field +// is a known fixture-typed field, returning that field's name ("" if none +// is found). +func findFixtureRead(region *ast.BlockStmt, recv string, fields map[string]bool) string { + found := "" + ast.Inspect(region, func(n ast.Node) bool { + if found != "" { + return false + } + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return true + } + id, ok := sel.X.(*ast.Ident) + if !ok || id.Name != recv { + return true + } + if fields[sel.Sel.Name] { + found = sel.Sel.Name + return false + } + return true + }) + return found +} + +// isBenchmarkMethodName mirrors gotestast's IS_BENCHMARK classification +// (^(?:X_|F_)?Benchmark.+$): an optional X_/F_ marker prefix, then +// "Benchmark" followed by at least one more character. +func isBenchmarkMethodName(name string) bool { + stripped := strings.TrimPrefix(strings.TrimPrefix(name, protocol.PrefixFocused), protocol.PrefixExcluded) + rest, ok := strings.CutPrefix(stripped, protocol.PrefixBenchmark) + return ok && rest != "" +} + +// benchParamName returns the name of a benchmark method's sole parameter +// (its *gotest.B, by convention), or "" if it has none/is unnamed. +func benchParamName(fd *ast.FuncDecl) string { + if fd.Type.Params == nil || len(fd.Type.Params.List) == 0 { + return "" + } + field := fd.Type.Params.List[0] + if len(field.Names) == 0 { + return "" + } + return field.Names[0].Name +} + +// benchBodyUsesLoopOrN reports whether body references param.Loop or +// param.N anywhere — a plain selector check, not requiring Loop to +// actually be called, matching the shape used elsewhere in this package +// for similar heuristics (e.g. isTMethodCall). +func benchBodyUsesLoopOrN(body *ast.BlockStmt, param string) bool { + found := false + ast.Inspect(body, func(n ast.Node) bool { + if found { + return false + } + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return true + } + id, ok := sel.X.(*ast.Ident) + if !ok || id.Name != param { + return true + } + if sel.Sel.Name == "Loop" || sel.Sel.Name == "N" { + found = true + return false + } + return true + }) + return found +} + +// Fixture-typed field discovery lives in fixturefields.go, shared with the +// shared-fixture-undeclared rule: discoverSuites feeds suiteInfo.fixtureFields +// through structFixtureFieldNames there. diff --git a/internal/lint/lint.go b/internal/lint/lint.go index bcaaf372..9b41709b 100644 --- a/internal/lint/lint.go +++ b/internal/lint/lint.go @@ -38,6 +38,9 @@ const ( AssertionRedundant Rule = "assertion-redundant" TEscape Rule = "t-escape" SuiteLifecycle Rule = "suite-lifecycle" + BenchLoop Rule = "bench-loop" + BenchFixtureIO Rule = "bench-fixture-io" + BenchWait Rule = "bench-wait" // SharedFixtureUndeclared is integrity: window scheduling starts only // the fixtures scheduled suites declare, so an undeclared read may hit // a fixture that never started or is already released. @@ -87,6 +90,15 @@ var ruleMeta = map[Rule]struct { TEscape: {TierExpressiveness, ScopeSuites}, SuiteLifecycle: {TierIntegrity, ScopeSuites}, FailGuard: {TierExpressiveness, ScopeGotestFiles}, + // A benchmark that never iterates measures nothing — its numbers lie, + // so bench-loop is integrity. bench-fixture-io is a heuristic about + // what the timed loop includes; legitimate setups exist, so it stays + // skippable. + BenchLoop: {TierIntegrity, ScopeSuites}, + BenchFixtureIO: {TierExpressiveness, ScopeSuites}, + // bench-wait sits with bench-fixture-io: a deliberate settle inside + // the loop can be legitimate, so it stays skippable. + BenchWait: {TierExpressiveness, ScopeSuites}, SharedFixtureUndeclared: {TierIntegrity, ScopeSuites}, } @@ -176,6 +188,9 @@ func run(pass *analysis.Pass) (any, error) { checkAssertionSimplify(pass, insp, cl) checkFailGuard(pass, insp, cl) checkRedundantAssertion(pass, insp, cl) + checkBenchLoop(pass, insp, suites) + checkBenchFixtureIO(pass, insp, suites) + checkBenchWait(pass, insp, suites) return nil, nil } @@ -346,6 +361,7 @@ type suiteInfo struct { pos token.Pos methods map[string]token.Pos recvTypePositions []token.Pos + fixtureFields map[string]bool // names of *...Fixture / *...SharedFixture fields (see bench-fixture-io) } func discoverSuites(insp *inspector.Inspector) map[string]*suiteInfo { @@ -365,9 +381,10 @@ func discoverSuites(insp *inspector.Inspector) map[string]*suiteInfo { stripped := strings.TrimPrefix(strings.TrimPrefix(name, protocol.PrefixFocused), protocol.PrefixExcluded) if strings.HasSuffix(stripped, protocol.SuffixTestSuite) { suites[name] = &suiteInfo{ - name: name, - pos: ts.Pos(), - methods: make(map[string]token.Pos), + name: name, + pos: ts.Pos(), + methods: make(map[string]token.Pos), + fixtureFields: structFixtureFieldNames(ts.Type), } } } diff --git a/internal/lint/lint_suite_test.go b/internal/lint/lint_suite_test.go index 809e5e73..6768c2d6 100644 --- a/internal/lint/lint_suite_test.go +++ b/internal/lint/lint_suite_test.go @@ -93,6 +93,12 @@ func (s *LintTestSuite) TestAnalyzer(t *gotest.T) { analysistest.Run(it.T(), testdata, lint.Analyzer, "withsharedfixture") }) }) + + t.When("benchmark methods", func(w *gotest.T) { + w.It("detects bench-loop, bench-fixture-io and bench-wait violations", func(it *gotest.T) { + analysistest.Run(it.T(), testdata, lint.Analyzer, "bench") + }) + }) } func (s *LintTestSuite) TestDisableNolintFlag(t *gotest.T) { @@ -108,11 +114,17 @@ func (s *LintTestSuite) TestDisableNolintFlag(t *gotest.T) { func (s *LintTestSuite) TestTierPolicy(t *gotest.T) { t.When("tier-derived skip surface", func(w *gotest.T) { w.It("registers a skip flag for every non-integrity rule and none for integrity rules", func(it *gotest.T) { - for _, rule := range []lint.Rule{lint.StdlibTest, lint.Testify, lint.AssertionSimplify, lint.AssertionRedundant, lint.FailGuard, lint.TEscape} { + for _, rule := range []lint.Rule{ + lint.StdlibTest, lint.Testify, lint.AssertionSimplify, lint.AssertionRedundant, lint.FailGuard, lint.TEscape, + lint.BenchFixtureIO, lint.BenchWait, + } { gotest.NotZero(it, lint.Analyzer.Flags.Lookup("skip-"+string(rule)), "missing skip flag for %s", rule) gotest.True(it, lint.SkippableRules[rule], "rule %s should be skippable", rule) } - for _, rule := range []lint.Rule{lint.Focus, lint.PollScope, lint.TestSignature, lint.SuiteLifecycle, lint.SharedFixtureUndeclared} { + for _, rule := range []lint.Rule{ + lint.Focus, lint.PollScope, lint.TestSignature, lint.SuiteLifecycle, + lint.BenchLoop, lint.SharedFixtureUndeclared, + } { gotest.Zero(it, lint.Analyzer.Flags.Lookup("skip-"+string(rule)), "unexpected skip flag for %s", rule) gotest.False(it, lint.SkippableRules[rule], "integrity rule %s must not be skippable", rule) } diff --git a/internal/lint/testdata/src/bench/bench_test.go b/internal/lint/testdata/src/bench/bench_test.go new file mode 100644 index 00000000..120c8629 --- /dev/null +++ b/internal/lint/testdata/src/bench/bench_test.go @@ -0,0 +1,157 @@ +package bench + +import ( + "testing" + "time" + + "github.com/mvrahden/go-test/pkg/gotest" +) + +// CacheFixture is a fixture-typed field (ends in "Fixture") for the +// bench-fixture-io testdata below. +type CacheFixture struct{} + +type ParserBenchTestSuite struct{} + +func (s *ParserBenchTestSuite) BenchmarkNoLoop(b *gotest.B) { // want `benchmark ParserBenchTestSuite.BenchmarkNoLoop never calls b.Loop\(\) — nothing is measured` + _ = 1 +} + +func (s *ParserBenchTestSuite) BenchmarkWithLoop(b *gotest.B) { + for b.Loop() { + _ = 1 + } +} + +// BenchmarkWithN accepts a stdlib *testing.B directly (also a valid +// benchmark signature) and iterates via the classic b.N idiom instead of +// b.Loop() — still "measures something", so bench-loop must not fire. +func (s *ParserBenchTestSuite) BenchmarkWithN(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = i + } +} + +func (s *ParserBenchTestSuite) BenchmarkSuppressed(b *gotest.B) { //nolint:bench-loop + _ = 1 +} + +// CacheBenchTestSuite holds a fixture-typed field and a BeforeEach that +// rebuilds per-test/per-benchmark state from it — the shape the framework +// recommends (see examples/benchmarking). bench-fixture-io does not care +// about BeforeEach at all: gotest's generated wrapper fences the timer +// around it, so it can never pollute a measurement. What it does care about +// is whether a benchmark method's own body reads the fixture field *inside* +// its measured loop. +type CacheBenchTestSuite struct { + fixture *CacheFixture + cache int +} + +func (s *CacheBenchTestSuite) BeforeEach(t *gotest.T) { + s.fixture = &CacheFixture{} +} + +// BenchmarkQueryInsideLoop reads the fixture field from inside b.Loop() — +// that read is timed, so it fires. +func (s *CacheBenchTestSuite) BenchmarkQueryInsideLoop(b *gotest.B) { // want `benchmark CacheBenchTestSuite.BenchmarkQueryInsideLoop reads fixture-backed state s.fixture inside the measured loop — hoist the read above the loop, or you are timing whatever backs the fixture` + for b.Loop() { + _ = s.fixture + } +} + +// BenchmarkQueryHoisted reads the same fixture field, but above the loop — +// this is the pattern the framework recommends (see +// examples/benchmarking), and it used to false-positive under the old +// structural rule. It must stay clean. +func (s *CacheBenchTestSuite) BenchmarkQueryHoisted(b *gotest.B) { + f := s.fixture + for b.Loop() { + _ = f + } +} + +// BenchmarkNestedRead buries the fixture read in a loop nested inside the +// measured loop. It must still fire: the rule walks the whole measured +// region, not just its direct children, and an implementation that only +// scanned the loop body's top level would silently miss this. +func (s *CacheBenchTestSuite) BenchmarkNestedRead(b *gotest.B) { // want `benchmark CacheBenchTestSuite.BenchmarkNestedRead reads fixture-backed state s.fixture inside the measured loop — hoist the read above the loop, or you are timing whatever backs the fixture` + for b.Loop() { + for i := 0; i < 4; i++ { + _ = s.fixture + } + } +} + +// BenchmarkClosureRead reads the fixture from a closure defined inside the +// measured loop — the same descent requirement as BenchmarkNestedRead. +func (s *CacheBenchTestSuite) BenchmarkClosureRead(b *gotest.B) { // want `benchmark CacheBenchTestSuite.BenchmarkClosureRead reads fixture-backed state s.fixture inside the measured loop — hoist the read above the loop, or you are timing whatever backs the fixture` + for b.Loop() { + func() { _ = s.fixture }() + } +} + +// BenchmarkNoFixtureRead has a fixture field and a BeforeEach (the old +// trigger shape) but never reads the fixture at all — clean. +func (s *CacheBenchTestSuite) BenchmarkNoFixtureRead(b *gotest.B) { + for b.Loop() { + s.cache++ + } +} + +// BenchmarkNInsideLoop uses the classic b.N form with a fixture read +// inside the loop body — must fire the same as the b.Loop() case. +func (s *CacheBenchTestSuite) BenchmarkNInsideLoop(b *testing.B) { // want `benchmark CacheBenchTestSuite.BenchmarkNInsideLoop reads fixture-backed state s.fixture inside the measured loop — hoist the read above the loop, or you are timing whatever backs the fixture` + for i := 0; i < b.N; i++ { + _ = s.fixture + } +} + +// BenchmarkDocSuppressed reads the fixture inside the loop but is +// suppressed via a nolint directive that lives in its doc comment (last +// line of the block) rather than on the same line as the declaration — the +// doc-comment suppression path. +// +//nolint:bench-fixture-io +func (s *CacheBenchTestSuite) BenchmarkDocSuppressed(b *gotest.B) { + for b.Loop() { + _ = s.fixture + } +} + +// WaitBenchTestSuite exercises bench-wait: waiting primitives inside the +// measured loop time the wait, not the code. +type WaitBenchTestSuite struct{} + +func (s *WaitBenchTestSuite) BenchmarkSleepInLoop(b *gotest.B) { + for b.Loop() { + time.Sleep(time.Millisecond) // want `benchmark WaitBenchTestSuite.BenchmarkSleepInLoop calls time.Sleep inside the measured loop — this times the wait, not the code; move it outside the loop` + } +} + +func (s *WaitBenchTestSuite) BenchmarkEventuallyInLoop(b *gotest.B) { + for b.Loop() { + gotest.Eventually(b, time.Second, time.Millisecond, func(poll *gotest.R) {}) // want `benchmark WaitBenchTestSuite.BenchmarkEventuallyInLoop calls gotest.Eventually inside the measured loop — this times the wait, not the code; move it outside the loop` + } +} + +func (s *WaitBenchTestSuite) BenchmarkConsistentlyInNLoop(b *testing.B) { + for i := 0; i < b.N; i++ { + gotest.Consistently(b, time.Second, time.Millisecond, func(poll *gotest.R) {}) // want `benchmark WaitBenchTestSuite.BenchmarkConsistentlyInNLoop calls gotest.Consistently inside the measured loop — this times the wait, not the code; move it outside the loop` + } +} + +// BenchmarkSleepAboveLoop settles before the measured region — clean. +func (s *WaitBenchTestSuite) BenchmarkSleepAboveLoop(b *gotest.B) { + time.Sleep(time.Millisecond) + for b.Loop() { + _ = 1 + } +} + +// BenchmarkSleepSuppressed opts out per line — a deliberate settle. +func (s *WaitBenchTestSuite) BenchmarkSleepSuppressed(b *gotest.B) { + for b.Loop() { + time.Sleep(time.Millisecond) //nolint:bench-wait + } +} diff --git a/internal/lint/testdata/src/github.com/mvrahden/go-test/pkg/gotest/gotest.go b/internal/lint/testdata/src/github.com/mvrahden/go-test/pkg/gotest/gotest.go index 694d416d..79353ee2 100644 --- a/internal/lint/testdata/src/github.com/mvrahden/go-test/pkg/gotest/gotest.go +++ b/internal/lint/testdata/src/github.com/mvrahden/go-test/pkg/gotest/gotest.go @@ -24,6 +24,14 @@ func (t *T) T() *testing.T { return nil } func (t *T) It(string, func(*T)) {} func (t *T) When(string, func(*T)) {} +type B struct{} + +func (b *B) Loop() bool { return false } +func (b *B) B() *testing.B { return nil } +func (b *B) ReportAllocs() {} +func (b *B) Errorf(format string, args ...any) {} +func (b *B) FailNow() {} + type testingT interface { Errorf(format string, args ...any) FailNow() diff --git a/skills/writing-gotest-tests/reference/ci.md b/skills/writing-gotest-tests/reference/ci.md index db9e889f..439e0fcb 100644 --- a/skills/writing-gotest-tests/reference/ci.md +++ b/skills/writing-gotest-tests/reference/ci.md @@ -38,6 +38,15 @@ jobs: `min-coverage`, `flags` (`--double-dash` style), `go-test-flags` (`-single-dash` style). The action adds a failure-focused summary, GitHub annotations, and coverage reporting on top of the plain CLI run. +- **v1.27+ bench inputs:** `bench: true` runs `gotest bench --spec --json` + after the tests (the `flags` input is forwarded to it — the place for + `-benchtime=1x` smoke runs); `bench-baseline` compares (`--against`), + `bench-gate` fails on regressions above the percentage, and `bench-save` + writes a baseline (a path; an explicit empty string saves to + `bench.baseline` from `.gotest.yml`; the `false` default saves nothing). + Outputs: `bench-report` (path to the `--json` report file) and + `bench-breached-keys` (comma-joined gate offenders). README.md's inputs/ + outputs tables are canonical and drift-guarded. - CI environments auto-arm `--ci` (any non-falsy `CI`/`GOTEST_CI` value): committed `F_` focus prefixes FAIL the run, and snapshots become read-only (`--update-snapshots` will not write). Opt out with diff --git a/skills/writing-gotest-tests/reference/cli.md b/skills/writing-gotest-tests/reference/cli.md index b04861d1..72b6dc56 100644 --- a/skills/writing-gotest-tests/reference/cli.md +++ b/skills/writing-gotest-tests/reference/cli.md @@ -37,6 +37,41 @@ baselines in CI-detected environments. `--update-snapshots` rewrites `MatchSnapshot` baselines (outside CI). `--spec` renders the spec view instead of default output. +## Benchmarks — `gotest bench` (v1.27+) + +`go tool gotest bench ./...` runs `Benchmark*` suite methods (signature +`func (s *X) BenchmarkParse(b *gotest.B)`, or stdlib `*testing.B`) through +the generated wrappers — always serially, ignoring `--parallel`, because +concurrent benchmarks time contention instead of code. `-test.benchmem` +is on by default. Flags: + +- `--spec` — render the spec view; under GitHub Actions the markdown + (with delta table) lands in the step summary automatically. +- `--save=` — write this run as a JSON baseline. Bare `--save=` + (empty value) falls back to `bench.baseline` from `.gotest.yml` and + errors when neither names a path. +- `--against=` — compare against a saved baseline and render the + delta table (significant rows only unless `-v`; defaults to + `bench.baseline`). Significance is Welch-tested, so run with `-count` + high enough to give it samples. +- `--gate=` — exit 1 when the worst significant regression exceeds + the threshold (needs `--against` or `bench.baseline`). +- `--json` — emit ONE versioned report document to stdout instead of + human output: `schemaVersion` 1, the run's results in baseline shape, + `deltas` when a comparison ran, and `gate` with `breachedKeys` (every + significant delta above the threshold) when a gate was active. Consume + this, never scrape text. +- Scoping: `-bench` matches the generated `Benchmark` wrapper by + its first slash segment; later segments select methods — + `-bench='^BenchmarkFooTestSuite$/^BenchmarkParse$'` runs one method. + `-benchtime=100x|2s` and `-count=` pass through. + +Baseline workflow: `bench --save=` on the trunk build; `bench +--against= --gate=10` (or `bench.gate` in `.gotest.yml`) on branches; +promote a new baseline by re-running `--save=` after accepting a change. +In CI, prefer the action's `bench`/`bench-baseline`/`bench-gate`/ +`bench-save` inputs (see `ci.md`). + Machine-readable capture, verified end-to-end: ```sh diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go index 570440ec..ec033d27 100644 --- a/tests/e2e/e2e_suite_test.go +++ b/tests/e2e/e2e_suite_test.go @@ -237,6 +237,84 @@ func (s *E2ETestSuite) TestOutputFormatGolden(t *gotest.T) { }) } +func (s *E2ETestSuite) TestBenchJSONReport(t *gotest.T) { + baselinePath := filepath.Join(t.TempDir(), "baseline.json") + + runBench := func(it *gotest.T, extra ...string) []byte { + args := append([]string{"bench", "github.com/mvrahden/go-test/examples/benchmarking", + "-bench=^BenchmarkCacheTestSuite$/^BenchmarkGetHit$", "-benchtime=10x"}, extra...) + cmd := exec.Command(s.binary, args...) //nolint:gosec // G204: controlled binary with fixed args + cmd.Dir = filepath.Join(s.workDir, "examples") + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + gotest.NoError(it, cmd.Run(), "bench run failed:\nstdout: %s\nstderr: %s", stdout.String(), stderr.String()) + return stdout.Bytes() + } + + type reportDoc struct { + SchemaVersion int `json:"schemaVersion"` + Baseline struct { + SchemaVersion int `json:"schemaVersion"` + GOOS string `json:"goos"` + Results []struct { + Suite string `json:"suite"` + Name string `json:"name"` + Samples []struct { + Iterations int `json:"iterations"` + NsPerOp float64 `json:"nsPerOp"` + } `json:"samples"` + } `json:"results"` + } `json:"baseline"` + Deltas []struct { + Key string `json:"key"` + Significant bool `json:"significant"` + } `json:"deltas"` + Gate *struct { + ThresholdPct float64 `json:"thresholdPct"` + Breached bool `json:"breached"` + } `json:"gate"` + } + + // Two CLI invocations total, not three: the save run doubles as the + // plain-report probe. Every bench invocation compiles the example with + // -race, and this suite runs concurrently with the timing-sensitive + // budget harnesses — the gate has no headroom for redundant load. + t.When("--json runs a slash-scoped single method and saves a baseline", func(w *gotest.T) { + out := runBench(w, "--save="+baselinePath, "--json") + var report reportDoc + gotest.NoError(w, json.Unmarshal(out, &report), "stdout must be one JSON document:\n%s", out) + + w.It("emits the versioned report with exactly the scoped method", func(it *gotest.T) { + gotest.Equal(it, 1, report.SchemaVersion) + gotest.Equal(it, 1, report.Baseline.SchemaVersion) + gotest.Len(it, report.Baseline.Results, 1) + gotest.Equal(it, "CacheTestSuite", report.Baseline.Results[0].Suite) + gotest.Equal(it, "BenchmarkGetHit", report.Baseline.Results[0].Name) + gotest.Equal(it, 10, report.Baseline.Results[0].Samples[0].Iterations) + }) + + w.It("omits deltas and gate when no comparison ran", func(it *gotest.T) { + gotest.Empty(it, report.Deltas) + gotest.Zero(it, report.Gate) + }) + }) + + t.When("--json compares against the saved baseline with a gate", func(w *gotest.T) { + out := runBench(w, "--against="+baselinePath, "--gate=1000", "--json") + var report reportDoc + gotest.NoError(w, json.Unmarshal(out, &report), "stdout must be one JSON document:\n%s", out) + + w.It("carries one delta per matched benchmark and the gate verdict", func(it *gotest.T) { + gotest.Len(it, report.Deltas, 1) + gotest.Contains(it, report.Deltas[0].Key, "CacheTestSuite/BenchmarkGetHit") + gotest.NotZero(it, report.Gate) + gotest.Equal(it, 1000.0, report.Gate.ThresholdPct) + gotest.False(it, report.Gate.Breached) + }) + }) +} + func normalizeOutput(raw string, workDir string) string { s := strings.ReplaceAll(raw, workDir, "") s = strings.ReplaceAll(s, "\r\n", "\n") diff --git a/tests/sharedfixture/benching/suite_test.go b/tests/sharedfixture/benching/suite_test.go new file mode 100644 index 00000000..f558f1f3 --- /dev/null +++ b/tests/sharedfixture/benching/suite_test.go @@ -0,0 +1,39 @@ +package benching + +import ( + "strings" + + "github.com/mvrahden/go-test/pkg/gotest" + "github.com/mvrahden/go-test/tests/sharedfixture/fixtures" +) + +// The two suites need disjoint shared fixtures, so a serial bench run must +// close Beta's window after the first slot and open Delta's just before the +// second: `gotest bench ./tests/sharedfixture/...` passing drives the +// per-slot JIT window scheduling end to end. + +type BetaBenchTestSuite struct { + Beta *fixtures.BetaSharedFixture +} + +func (s *BetaBenchTestSuite) BenchmarkLabel(b *gotest.B) { + label := s.Beta.Label // hoisted: the fixture is the setup, not the measurement + for b.Loop() { + if !strings.HasPrefix(label, "beta") { + b.Errorf("Beta not hydrated: %q", label) + } + } +} + +type DeltaBenchTestSuite struct { + Delta *fixtures.DeltaSharedFixture +} + +func (s *DeltaBenchTestSuite) BenchmarkStamp(b *gotest.B) { + stamp := s.Delta.Stamp // hoisted: the fixture is the setup, not the measurement + for b.Loop() { + if stamp != "delta-shared" { + b.Errorf("Delta not hydrated: %q", stamp) + } + } +} diff --git a/vscode-gotest/README.md b/vscode-gotest/README.md index a37f8e28..86e37ced 100644 --- a/vscode-gotest/README.md +++ b/vscode-gotest/README.md @@ -67,6 +67,7 @@ Test results persist across sessions, so you see pass/fail state immediately aft **Run** and **Debug** buttons appear inline above every suite and test method in `_test.go` files. Click to execute immediately. +Benchmark methods get **Bench**, **5×**, and a persistent result annotation (see [Benchmarks](#benchmarks)). Package-level and file-level actions appear on the `package` declaration line: @@ -110,6 +111,40 @@ Place your cursor on a suite or method definition and use the **Quick Fix** menu A status bar warning and inline diagnostics alert you when focused tests exist, preventing CI failures from `gotest --ci`. +### Benchmarks + +A benchmark answers with a number, and a number is meaningless in isolation. +Every surface here exists to close the loop between "I changed this code" and +"what happened to the number" — powered by `gotest bench --json`; all +statistics (Welch's t-test significance, deltas, the gate rule) are computed +by the CLI, never re-derived in the extension: + +- **▶ Bench** CodeLens on every `Benchmark*` method — runs exactly that + method (go test's sub-benchmark scoping). **▶ Bench Suite** on the suite + runs them all; **5×** runs with `-count=5` for a trustworthy mean ± spread. +- **Result annotations** — the last measured numbers render right above the + method (`1.24µs/op · 480 B/op · 3 allocs/op — 2m ago`) and survive editor + reloads. Results are keyed per goos/goarch: a number measured on another + platform is a different number and is never shown on this host. +- **Trend on hover** — hovering a benchmark shows its run-over-run sparkline + with the endpoints spelled out, from a bounded local history (last 50 runs). +- **Baselines** — *Go Test: Save Bench Baseline* and *Go Test: Compare vs + Baseline* wrap `--save`/`--against`, defaulting to `bench.baseline` from + `.gotest.yml`. After a compare, significant deltas render inline + (`+12.3% vs baseline`); deltas the CLI did not mark significant display + nothing — the UI never dresses up noise. +- **Gate warnings** — when `bench.gate` is configured and a run breaches it, + the offending methods get warning diagnostics at their definitions. +- **Test Explorer** — benchmarks appear under their suites with a dedicated + **Bench** run profile. They never run as part of a normal test run: timing + numbers taken while tests hammer the machine are noise. +- **Profiling** — *Go Test: Profile Benchmark (CPU/Mem)* runs one benchmark + with `-cpuprofile`/`-memprofile` and opens `go tool pprof -http` on the + result. + +Benchmarks are deliberate acts: there is no bench-on-save and watch mode +never benchmarks. + ### Scaffold Generate test suite skeletons from existing code: @@ -140,6 +175,11 @@ Projects using `go.work` are also supported. | Go Test: Show Spec View | Open the BDD spec output panel | | Go Test: Start Watch | Start continuous testing for a package scope | | Go Test: Stop Watch | Stop all active watch processes | +| Go Test: Run Benchmark | Run a suite's or method's benchmarks via `gotest bench` | +| Go Test: Bench (stable, 5×) | Run a benchmark with `-count=5` for mean ± spread | +| Go Test: Save Bench Baseline | Save the workspace's benchmark results as a baseline | +| Go Test: Compare vs Baseline | Compare current numbers against a saved baseline | +| Go Test: Profile Benchmark (CPU/Mem) | Profile one benchmark and open `go tool pprof` | | Go Test: Scaffold Suite | Generate a test suite from a target | | Go Test: Scaffold Target | Generate a test suite for a specific target | | Go Test: Copy Coverage Summary | Copy coverage table to clipboard | diff --git a/vscode-gotest/package.json b/vscode-gotest/package.json index 482c1899..a5283c97 100644 --- a/vscode-gotest/package.json +++ b/vscode-gotest/package.json @@ -48,6 +48,26 @@ "command": "gotest.runFile", "title": "Go Test: Run File" }, + { + "command": "gotest.runBench", + "title": "Go Test: Run Benchmark" + }, + { + "command": "gotest.runBenchStable", + "title": "Go Test: Bench (stable, 5×)" + }, + { + "command": "gotest.profileBench", + "title": "Go Test: Profile Benchmark (CPU/Mem)" + }, + { + "command": "gotest.saveBenchBaseline", + "title": "Go Test: Save Bench Baseline" + }, + { + "command": "gotest.compareBenchBaseline", + "title": "Go Test: Compare vs Baseline" + }, { "command": "gotest.refreshTests", "title": "Go Test: Refresh" diff --git a/vscode-gotest/src/benchDiagnostics.test.ts b/vscode-gotest/src/benchDiagnostics.test.ts new file mode 100644 index 00000000..063dbe5b --- /dev/null +++ b/vscode-gotest/src/benchDiagnostics.test.ts @@ -0,0 +1,140 @@ +import { describe, it, expect, vi } from "vitest"; + +const collections: MockCollection[] = []; + +class MockCollection { + entries = new Map(); + cleared = 0; + set(uri: { fsPath: string }, diagnostics: unknown[]) { + this.entries.set(uri.fsPath, diagnostics); + } + clear() { + this.cleared++; + this.entries.clear(); + } + dispose() {} +} + +vi.mock("vscode", () => ({ + languages: { + createDiagnosticCollection: () => { + const c = new MockCollection(); + collections.push(c); + return c; + }, + }, + Uri: { file: (p: string) => ({ fsPath: p }) }, + Range: class { + constructor( + public a: number, + public b: number, + public c: number, + public d: number, + ) {} + }, + Diagnostic: class { + source?: string; + constructor( + public range: unknown, + public message: string, + public severity: number, + ) {} + }, + DiagnosticSeverity: { Warning: 1 }, +})); + +import { BenchGateDiagnostics, parseDeltaKey } from "./benchDiagnostics.js"; +import type { BenchReport } from "./benchReport.js"; +import type { DiscoveryCache } from "./discovery.js"; + +describe("parseDeltaKey", () => { + it("splits the CLI's 'pkg Suite/Name' shape", () => { + expect( + parseDeltaKey("example.com/pkg CacheTestSuite/BenchmarkGetHit"), + ).toEqual({ + importPath: "example.com/pkg", + suiteName: "CacheTestSuite", + methodName: "BenchmarkGetHit", + }); + }); + + it("refuses malformed keys", () => { + expect(parseDeltaKey("no-space")).toBeUndefined(); + expect(parseDeltaKey("pkg noslash")).toBeUndefined(); + }); +}); + +function mockCache(): DiscoveryCache { + return { + getPackage: (ip: string) => + ip === "example.com/pkg" + ? { + importPath: ip, + dir: "/ws/pkg", + suites: [ + { + name: "CacheTestSuite", + benchmarks: [ + { + name: "BenchmarkGetHit", + file: "cache_test.go", + line: 20, + col: 9, + }, + ], + }, + ], + } + : undefined, + } as unknown as DiscoveryCache; +} + +function gateReport(breachedKeys: string[]): BenchReport { + return { + schemaVersion: 1, + baseline: { schemaVersion: 1, goos: "linux", goarch: "amd64", results: [] }, + deltas: [ + { + key: "example.com/pkg CacheTestSuite/BenchmarkGetHit", + oldNs: 100, + newNs: 112.3, + percentChange: 12.3, + significant: true, + insufficientSample: false, + }, + ], + gate: { + thresholdPct: 5, + worstPct: 12.3, + worstKey: "example.com/pkg CacheTestSuite/BenchmarkGetHit", + breached: breachedKeys.length > 0, + breachedKeys, + }, + }; +} + +describe("BenchGateDiagnostics", () => { + it("places a warning on each breached benchmark method", () => { + const diags = new BenchGateDiagnostics(mockCache()); + const collection = collections[collections.length - 1]; + + diags.apply(gateReport(["example.com/pkg CacheTestSuite/BenchmarkGetHit"])); + + const fileDiags = collection.entries.get("/ws/pkg/cache_test.go") as Array<{ + message: string; + }>; + expect(fileDiags).toHaveLength(1); + expect(fileDiags[0].message).toContain("BenchmarkGetHit regressed +12.3%"); + expect(fileDiags[0].message).toContain("5% bench gate"); + }); + + it("clears previous verdicts when a report carries no breaches", () => { + const diags = new BenchGateDiagnostics(mockCache()); + const collection = collections[collections.length - 1]; + + diags.apply(gateReport(["example.com/pkg CacheTestSuite/BenchmarkGetHit"])); + diags.apply(gateReport([])); + + expect(collection.entries.size).toBe(0); + }); +}); diff --git a/vscode-gotest/src/benchDiagnostics.ts b/vscode-gotest/src/benchDiagnostics.ts new file mode 100644 index 00000000..6a49c746 --- /dev/null +++ b/vscode-gotest/src/benchDiagnostics.ts @@ -0,0 +1,94 @@ +// benchDiagnostics raises warning squiggles on benchmark methods the CLI's +// gate condemned. Which methods breach is decided in Go (gate.breachedKeys) +// — this module only places the verdicts at their source positions, the +// same mechanism the focus warnings use. + +import * as vscode from "vscode"; +import * as path from "node:path"; +import type { DiscoveryCache } from "./discovery.js"; +import type { BenchReport } from "./benchReport.js"; + +/** parseDeltaKey splits the CLI's "pkg Suite/Name" delta key. */ +export function parseDeltaKey( + key: string, +): { importPath: string; suiteName: string; methodName: string } | undefined { + const space = key.indexOf(" "); + if (space <= 0) return undefined; + const importPath = key.slice(0, space); + const rest = key.slice(space + 1); + const slash = rest.indexOf("/"); + if (slash <= 0 || slash === rest.length - 1) return undefined; + return { + importPath, + suiteName: rest.slice(0, slash), + methodName: rest.slice(slash + 1), + }; +} + +export class BenchGateDiagnostics implements vscode.Disposable { + private readonly collection: vscode.DiagnosticCollection; + + constructor(private readonly cache: DiscoveryCache) { + this.collection = + vscode.languages.createDiagnosticCollection("gotest-bench-gate"); + } + + /** + * apply replaces all gate diagnostics with the ones this report carries. + * A report without a gate — or without breaches — clears the board: the + * verdict belongs to the latest run, not to history. + */ + apply(report: BenchReport): void { + this.collection.clear(); + const breached = report.gate?.breachedKeys ?? []; + if (breached.length === 0) return; + + const byFile = new Map(); + const deltaByKey = new Map( + (report.deltas ?? []).map((d) => [d.key, d] as const), + ); + + for (const key of breached) { + const parsed = parseDeltaKey(key); + if (!parsed) continue; + const pkg = this.cache.getPackage(parsed.importPath); + if (!pkg) continue; + const suite = pkg.suites.find((s) => s.name === parsed.suiteName); + const method = suite?.benchmarks.find( + (b) => b.name === parsed.methodName, + ); + if (!suite || !method) continue; + + const delta = deltaByKey.get(key); + const pct = delta ? `+${Math.abs(delta.percentChange).toFixed(1)}%` : ""; + const threshold = report.gate?.thresholdPct ?? 0; + const message = `${parsed.methodName} regressed ${pct} vs baseline — exceeds the ${threshold}% bench gate`; + + const line = method.line - 1; + const col = method.col - 1; + const diagnostic = new vscode.Diagnostic( + new vscode.Range(line, col, line, col + parsed.methodName.length), + message, + vscode.DiagnosticSeverity.Warning, + ); + diagnostic.source = "gotest"; + + const file = path.join(pkg.dir, method.file); + const list = byFile.get(file) ?? []; + list.push(diagnostic); + byFile.set(file, list); + } + + for (const [file, diagnostics] of byFile) { + this.collection.set(vscode.Uri.file(file), diagnostics); + } + } + + clear(): void { + this.collection.clear(); + } + + dispose(): void { + this.collection.dispose(); + } +} diff --git a/vscode-gotest/src/benchHover.test.ts b/vscode-gotest/src/benchHover.test.ts new file mode 100644 index 00000000..c76666c8 --- /dev/null +++ b/vscode-gotest/src/benchHover.test.ts @@ -0,0 +1,89 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("vscode", () => ({ + Hover: class { + constructor(public contents: unknown) {} + }, + MarkdownString: class { + constructor(public value: string) {} + }, +})); + +import { sparkline, buildBenchHoverMarkdown } from "./benchHover.js"; +import type { BenchEntry } from "./benchResultStore.js"; + +function entry(overrides: Partial = {}): BenchEntry { + return { + nsPerOp: 100, + bytesPerOp: 480, + allocsPerOp: 3, + iterations: 10, + sampleCount: 1, + minNsPerOp: 100, + maxNsPerOp: 100, + recordedAt: Date.parse("2026-08-14T11:58:00Z"), + goos: "linux", + goarch: "amd64", + ...overrides, + }; +} + +describe("sparkline", () => { + it("scales values into the block range, oldest first", () => { + const line = sparkline([0, 50, 100]); + expect(line).toHaveLength(3); + expect(line[0]).toBe("▁"); + expect(line[2]).toBe("█"); + }); + + it("renders a flat line for identical values", () => { + expect(sparkline([5, 5, 5])).toBe("▁▁▁"); + }); + + it("renders nothing for no values", () => { + expect(sparkline([])).toBe(""); + }); +}); + +describe("buildBenchHoverMarkdown", () => { + const now = Date.parse("2026-08-14T12:00:00Z"); + + it("returns nothing when the method has never been measured here", () => { + expect(buildBenchHoverMarkdown("BenchmarkX", undefined, [], now)).toBe( + undefined, + ); + }); + + it("shows the latest numbers, platform, and age", () => { + const md = buildBenchHoverMarkdown("BenchmarkGetHit", entry(), [], now)!; + expect(md).toContain("**BenchmarkGetHit** — 100 ns/op · 2m ago"); + expect(md).toContain("480 B/op · 3 allocs/op · linux/amd64"); + expect(md).not.toContain("Trend"); + }); + + it("shows mean ± spread for a stable multi-count run", () => { + const md = buildBenchHoverMarkdown( + "BenchmarkGetHit", + entry({ sampleCount: 5, minNsPerOp: 95, maxNsPerOp: 105 }), + [], + now, + )!; + expect(md).toContain("100 ns/op ±5.0% (mean of 5×)"); + }); + + it("draws the trend for repeated runs, endpoints spelled out", () => { + const history = [ + { nsPerOp: 100, recordedAt: 1, sampleCount: 1 }, + { nsPerOp: 150, recordedAt: 2, sampleCount: 1 }, + { nsPerOp: 80, recordedAt: 3, sampleCount: 1 }, + ]; + const md = buildBenchHoverMarkdown( + "BenchmarkGetHit", + entry(), + history, + now, + )!; + expect(md).toContain("Trend (last 3 runs)"); + expect(md).toContain("100 ns/op → 80 ns/op"); + }); +}); diff --git a/vscode-gotest/src/benchHover.ts b/vscode-gotest/src/benchHover.ts new file mode 100644 index 00000000..8c7a2bf6 --- /dev/null +++ b/vscode-gotest/src/benchHover.ts @@ -0,0 +1,115 @@ +// benchHover renders a benchmark method's run-over-run trend on hover: the +// stored history for this host's platform, drawn as a unicode sparkline with +// the endpoints spelled out. Pure display — every number shown was measured +// by the CLI and recorded verbatim. + +import * as vscode from "vscode"; +import * as path from "node:path"; +import type { DiscoveryCache } from "./discovery.js"; +import { + hostPlatform, + type BenchResultStore, + type BenchEntry, + type BenchHistoryPoint, +} from "./benchResultStore.js"; +import { formatNsPerOp, formatAge } from "./benchReport.js"; + +const SPARK_BLOCKS = "▁▂▃▄▅▆▇█"; +const SPARK_POINTS = 20; + +/** sparkline draws values (oldest first) scaled to their own min..max. */ +export function sparkline(values: number[]): string { + if (values.length === 0) return ""; + const min = Math.min(...values); + const max = Math.max(...values); + const span = max - min; + return values + .map((v) => { + const idx = + span === 0 + ? 0 + : Math.min( + SPARK_BLOCKS.length - 1, + Math.floor(((v - min) / span) * SPARK_BLOCKS.length), + ); + return SPARK_BLOCKS[idx]; + }) + .join(""); +} + +/** + * buildBenchHoverMarkdown renders the hover body, or undefined when there is + * nothing recorded for this method on this platform. + */ +export function buildBenchHoverMarkdown( + methodName: string, + entry: BenchEntry | undefined, + history: BenchHistoryPoint[], + now: number = Date.now(), +): string | undefined { + if (!entry) return undefined; + + const lines: string[] = []; + const spread = + entry.sampleCount > 1 && entry.nsPerOp > 0 + ? ` ±${((((entry.maxNsPerOp - entry.minNsPerOp) / 2) * 100) / entry.nsPerOp).toFixed(1)}%` + : ""; + lines.push( + `**${methodName}** — ${formatNsPerOp(entry.nsPerOp)}${spread}` + + (entry.sampleCount > 1 ? ` (mean of ${entry.sampleCount}×)` : "") + + ` · ${formatAge(entry.recordedAt, now)}`, + ); + lines.push(""); + lines.push( + `${entry.bytesPerOp} B/op · ${entry.allocsPerOp} allocs/op · ${entry.goos}/${entry.goarch}`, + ); + + if (history.length > 1) { + const recent = history.slice(-SPARK_POINTS); + const values = recent.map((p) => p.nsPerOp); + lines.push(""); + lines.push(`Trend (last ${recent.length} runs): \`${sparkline(values)}\``); + lines.push( + `${formatNsPerOp(values[0])} → ${formatNsPerOp(values[values.length - 1])}`, + ); + } + + return lines.join("\n"); +} + +export class BenchHoverProvider implements vscode.HoverProvider { + constructor( + private readonly cache: DiscoveryCache, + private readonly store: BenchResultStore, + ) {} + + provideHover( + document: vscode.TextDocument, + position: vscode.Position, + ): vscode.Hover | undefined { + const importPath = this.cache.resolveFileToPackage(document.fileName); + if (!importPath) return undefined; + const pkg = this.cache.getPackage(importPath); + if (!pkg) return undefined; + + const platform = hostPlatform(); + for (const suite of pkg.suites) { + for (const bench of suite.benchmarks ?? []) { + if ( + path.join(pkg.dir, bench.file) !== document.fileName || + bench.line - 1 !== position.line + ) { + continue; + } + const markdown = buildBenchHoverMarkdown( + bench.name, + this.store.getLatest(importPath, suite.name, bench.name, platform), + this.store.getHistory(importPath, suite.name, bench.name, platform), + ); + if (!markdown) return undefined; + return new vscode.Hover(new vscode.MarkdownString(markdown)); + } + } + return undefined; + } +} diff --git a/vscode-gotest/src/benchReport.test.ts b/vscode-gotest/src/benchReport.test.ts new file mode 100644 index 00000000..6734c3df --- /dev/null +++ b/vscode-gotest/src/benchReport.test.ts @@ -0,0 +1,160 @@ +import { describe, it, expect } from "vitest"; +import { + parseBenchReport, + formatNsPerOp, + formatBenchAnnotation, + formatAge, +} from "./benchReport.js"; + +const validReport = JSON.stringify({ + schemaVersion: 1, + baseline: { + schemaVersion: 1, + createdAt: "2026-08-14T10:00:00Z", + goVersion: "go1.26.5", + goos: "linux", + goarch: "amd64", + results: [ + { + package: "example.com/pkg", + suite: "CacheTestSuite", + name: "BenchmarkGetHit", + samples: [ + { iterations: 100, nsPerOp: 56.1, bytesPerOp: 0, allocsPerOp: 0 }, + ], + }, + ], + }, +}); + +describe("parseBenchReport", () => { + it("parses a valid schema-1 report", () => { + const report = parseBenchReport(validReport); + expect(report.baseline.goos).toBe("linux"); + expect(report.baseline.results).toHaveLength(1); + expect(report.baseline.results[0].name).toBe("BenchmarkGetHit"); + expect(report.deltas).toBeUndefined(); + expect(report.gate).toBeUndefined(); + }); + + it("rejects an unknown schema version instead of guessing", () => { + const doc = JSON.stringify({ schemaVersion: 2, baseline: {} }); + expect(() => parseBenchReport(doc)).toThrow(/schema version 2/); + }); + + it("rejects non-JSON output with the head of the text for context", () => { + expect(() => parseBenchReport("FAIL: something broke")).toThrow( + /not a bench report/, + ); + }); + + it("carries deltas and gate through untouched", () => { + const doc = JSON.stringify({ + schemaVersion: 1, + baseline: { + schemaVersion: 1, + goos: "linux", + goarch: "amd64", + results: [], + }, + deltas: [ + { + key: "example.com/pkg CacheTestSuite/BenchmarkGetHit", + oldNs: 100, + newNs: 112.3, + percentChange: 12.3, + significant: true, + insufficientSample: false, + }, + ], + gate: { thresholdPct: 5, worstPct: 12.3, worstKey: "k", breached: true }, + }); + const report = parseBenchReport(doc); + expect(report.deltas).toHaveLength(1); + expect(report.deltas?.[0].significant).toBe(true); + expect(report.gate?.breached).toBe(true); + }); +}); + +describe("formatNsPerOp", () => { + it("scales through ns, µs, ms and s", () => { + expect(formatNsPerOp(56.1)).toBe("56 ns/op"); + expect(formatNsPerOp(999)).toBe("999 ns/op"); + expect(formatNsPerOp(1240)).toBe("1.24µs/op"); + expect(formatNsPerOp(359245)).toBe("359.25µs/op"); + expect(formatNsPerOp(2_500_000)).toBe("2.50ms/op"); + expect(formatNsPerOp(1_200_000_000)).toBe("1.20s/op"); + }); +}); + +describe("formatAge", () => { + it("buckets into now/seconds/minutes/hours/days", () => { + const now = Date.parse("2026-08-14T12:00:00Z"); + expect(formatAge(now - 2_000, now)).toBe("just now"); + expect(formatAge(now - 42_000, now)).toBe("42s ago"); + expect(formatAge(now - 2 * 60_000, now)).toBe("2m ago"); + expect(formatAge(now - 3 * 3_600_000, now)).toBe("3h ago"); + expect(formatAge(now - 2 * 86_400_000, now)).toBe("2d ago"); + }); +}); + +describe("formatBenchAnnotation", () => { + it("renders the ns/op · B/op · allocs/op — age line", () => { + const now = Date.parse("2026-08-14T12:00:00Z"); + const line = formatBenchAnnotation( + { nsPerOp: 1240, bytesPerOp: 480, allocsPerOp: 3, iterations: 100 }, + now - 2 * 60_000, + now, + ); + expect(line).toBe("1.24µs/op · 480 B/op · 3 allocs/op — 2m ago"); + }); + + it("omits the alloc fields for allocation-free benchmarks", () => { + const now = Date.parse("2026-08-14T12:00:00Z"); + const line = formatBenchAnnotation( + { nsPerOp: 56.1, bytesPerOp: 0, allocsPerOp: 0, iterations: 100 }, + now - 42_000, + now, + ); + expect(line).toBe("56 ns/op · 0 allocs/op — 42s ago"); + }); +}); + +describe("formatBenchAnnotation deltas", () => { + const now = Date.parse("2026-08-14T12:00:00Z"); + const numbers = { + nsPerOp: 1240, + bytesPerOp: 480, + allocsPerOp: 3, + iterations: 100, + }; + + it("appends a significant regression as +N.N% vs baseline", () => { + const line = formatBenchAnnotation(numbers, now - 120_000, now, { + percentChange: 12.34, + significant: true, + insufficientSample: false, + }); + expect(line).toBe( + "1.24µs/op · 480 B/op · 3 allocs/op — 2m ago · +12.3% vs baseline", + ); + }); + + it("appends a significant improvement with a minus sign", () => { + const line = formatBenchAnnotation(numbers, now - 120_000, now, { + percentChange: -8.1, + significant: true, + insufficientSample: false, + }); + expect(line).toContain("· −8.1% vs baseline"); + }); + + it("stays neutral for a delta the CLI did not mark significant", () => { + const line = formatBenchAnnotation(numbers, now - 120_000, now, { + percentChange: 40, + significant: false, + insufficientSample: false, + }); + expect(line).toBe("1.24µs/op · 480 B/op · 3 allocs/op — 2m ago"); + }); +}); diff --git a/vscode-gotest/src/benchReport.ts b/vscode-gotest/src/benchReport.ts new file mode 100644 index 00000000..9fcae1e4 --- /dev/null +++ b/vscode-gotest/src/benchReport.ts @@ -0,0 +1,145 @@ +// benchReport parses and formats the versioned JSON document emitted by +// `gotest bench --json`. All statistics live in the CLI (internal/gotestbench) +// — this module only decodes and displays what the CLI computed. If the +// extension ever needs more data, the CLI grows a field; nothing is derived +// here beyond unit scaling and relative timestamps. + +/** One benchmark's samples, mirroring gotestbench.Result. */ +export interface BenchResult { + package: string; + suite: string; + name: string; + samples: BenchSample[]; +} + +/** Mirrors gotestbench.Sample: what go test prints per benchmark run. */ +export interface BenchSample { + iterations: number; + nsPerOp: number; + bytesPerOp: number; + allocsPerOp: number; +} + +/** Mirrors gotestbench.Delta. Significance is the CLI's verdict, never ours. */ +export interface BenchDelta { + key: string; + oldNs: number; + newNs: number; + percentChange: number; + significant: boolean; + insufficientSample: boolean; +} + +/** Mirrors gotestbench.Gate. */ +export interface BenchGate { + thresholdPct: number; + worstPct: number; + worstKey?: string; + breached: boolean; + /** Every delta the CLI's gate rule condemned — the rule lives in Go. */ + breachedKeys?: string[]; +} + +export interface BenchBaseline { + schemaVersion: number; + createdAt?: string; + goVersion?: string; + goos: string; + goarch: string; + results: BenchResult[]; +} + +export interface BenchReport { + schemaVersion: number; + baseline: BenchBaseline; + deltas?: BenchDelta[]; + gate?: BenchGate; +} + +const SUPPORTED_SCHEMA_VERSION = 1; + +/** + * parseBenchReport decodes one `gotest bench --json` stdout document. + * Unknown schema versions are refused outright: silently misreading a future + * format is worse than asking the user to update the extension. + */ +export function parseBenchReport(stdout: string): BenchReport { + let doc: unknown; + try { + doc = JSON.parse(stdout); + } catch { + const head = stdout.trimStart().slice(0, 120); + throw new Error(`not a bench report (expected --json output): ${head}`); + } + const report = doc as BenchReport; + if (report.schemaVersion !== SUPPORTED_SCHEMA_VERSION) { + throw new Error( + `unsupported bench report schema version ${report.schemaVersion} (extension understands ${SUPPORTED_SCHEMA_VERSION})`, + ); + } + if (!report.baseline || !Array.isArray(report.baseline.results)) { + throw new Error("bench report carries no baseline results"); + } + return report; +} + +/** formatNsPerOp scales a ns/op mean into the customary go bench units. */ +export function formatNsPerOp(ns: number): string { + if (ns < 1000) return `${Math.round(ns)} ns/op`; + if (ns < 1_000_000) return `${(ns / 1000).toFixed(2)}µs/op`; + if (ns < 1_000_000_000) return `${(ns / 1_000_000).toFixed(2)}ms/op`; + return `${(ns / 1_000_000_000).toFixed(2)}s/op`; +} + +/** formatAge renders a relative timestamp for annotations. */ +export function formatAge(then: number, now: number = Date.now()): string { + const ms = Math.max(0, now - then); + if (ms < 10_000) return "just now"; + if (ms < 60_000) return `${Math.round(ms / 1000)}s ago`; + if (ms < 3_600_000) return `${Math.round(ms / 60_000)}m ago`; + if (ms < 86_400_000) return `${Math.round(ms / 3_600_000)}h ago`; + return `${Math.round(ms / 86_400_000)}d ago`; +} + +/** The per-run summary an annotation displays (last sample's numbers). */ +export interface BenchNumbers { + iterations: number; + nsPerOp: number; + bytesPerOp: number; + allocsPerOp: number; +} + +/** The comparison verdict attached to an entry (see BenchEntryDelta). */ +export interface AnnotationDelta { + percentChange: number; + significant: boolean; + insufficientSample: boolean; +} + +/** + * formatBenchAnnotation renders the CodeLens line above a benchmark method: + * "1.24µs/op · 480 B/op · 3 allocs/op — 2m ago". Allocation-free benchmarks + * skip the B/op term rather than shouting "0 B/op". + * + * A delta is appended ONLY when the CLI marked it significant — a UI that + * confidently displays noise is worse than no UI. Everything else renders + * exactly as if no comparison had happened. + */ +export function formatBenchAnnotation( + numbers: BenchNumbers, + recordedAt: number, + now: number = Date.now(), + delta?: AnnotationDelta, +): string { + const parts = [formatNsPerOp(numbers.nsPerOp)]; + if (numbers.bytesPerOp > 0) { + parts.push(`${numbers.bytesPerOp} B/op`); + } + parts.push(`${numbers.allocsPerOp} allocs/op`); + let line = `${parts.join(" · ")} — ${formatAge(recordedAt, now)}`; + if (delta?.significant) { + const sign = delta.percentChange >= 0 ? "+" : "−"; + line += ` · ${sign}${Math.abs(delta.percentChange).toFixed(1)}% vs baseline`; + } + return line; +} diff --git a/vscode-gotest/src/benchResultStore.test.ts b/vscode-gotest/src/benchResultStore.test.ts new file mode 100644 index 00000000..0e944735 --- /dev/null +++ b/vscode-gotest/src/benchResultStore.test.ts @@ -0,0 +1,272 @@ +import { describe, it, expect } from "vitest"; +import { + BenchResultStore, + benchKey, + hostPlatform, + type MementoLike, +} from "./benchResultStore.js"; +import type { BenchReport } from "./benchReport.js"; + +function fakeMemento(): MementoLike & { data: Map } { + const data = new Map(); + return { + data, + get(key: string, defaultValue: T): T { + return (data.has(key) ? data.get(key) : defaultValue) as T; + }, + update(key: string, value: unknown) { + data.set(key, value); + return Promise.resolve(); + }, + }; +} + +function report(goos: string, goarch: string, nsPerOp: number): BenchReport { + return { + schemaVersion: 1, + baseline: { + schemaVersion: 1, + goos, + goarch, + results: [ + { + package: "example.com/pkg", + suite: "CacheTestSuite", + name: "BenchmarkGetHit", + samples: [ + { iterations: 100, nsPerOp, bytesPerOp: 480, allocsPerOp: 3 }, + ], + }, + ], + }, + }; +} + +describe("benchKey", () => { + it("keys by package, suite, method, and platform", () => { + expect( + benchKey( + "example.com/pkg", + "CacheTestSuite", + "BenchmarkGetHit", + "linux", + "amd64", + ), + ).toBe("example.com/pkg/CacheTestSuite/BenchmarkGetHit@linux/amd64"); + }); +}); + +describe("BenchResultStore", () => { + it("records a report's results keyed by the report's own platform stamps", () => { + const store = new BenchResultStore(fakeMemento()); + store.recordReport( + report("linux", "amd64", 56.1), + Date.parse("2026-08-14T12:00:00Z"), + ); + + const entry = store.getLatest( + "example.com/pkg", + "CacheTestSuite", + "BenchmarkGetHit", + { goos: "linux", goarch: "amd64" }, + ); + expect(entry?.nsPerOp).toBe(56.1); + expect(entry?.bytesPerOp).toBe(480); + expect(entry?.recordedAt).toBe(Date.parse("2026-08-14T12:00:00Z")); + }); + + it("never serves a result recorded on a different platform", () => { + const store = new BenchResultStore(fakeMemento()); + store.recordReport(report("darwin", "arm64", 42), Date.now()); + + const entry = store.getLatest( + "example.com/pkg", + "CacheTestSuite", + "BenchmarkGetHit", + { goos: "linux", goarch: "amd64" }, + ); + expect(entry).toBeUndefined(); + }); + + it("uses the mean of a multi-sample run for the displayed numbers", () => { + const store = new BenchResultStore(fakeMemento()); + const multi = report("linux", "amd64", 0); + multi.baseline.results[0].samples = [ + { iterations: 100, nsPerOp: 100, bytesPerOp: 480, allocsPerOp: 3 }, + { iterations: 100, nsPerOp: 120, bytesPerOp: 480, allocsPerOp: 3 }, + ]; + store.recordReport(multi, Date.now()); + + const entry = store.getLatest( + "example.com/pkg", + "CacheTestSuite", + "BenchmarkGetHit", + { goos: "linux", goarch: "amd64" }, + ); + expect(entry?.nsPerOp).toBe(110); + expect(entry?.sampleCount).toBe(2); + }); + + it("survives a reload through the memento", () => { + const memento = fakeMemento(); + const store = new BenchResultStore(memento); + store.recordReport(report("linux", "amd64", 56.1), Date.now()); + + const reloaded = new BenchResultStore(memento); + const entry = reloaded.getLatest( + "example.com/pkg", + "CacheTestSuite", + "BenchmarkGetHit", + { goos: "linux", goarch: "amd64" }, + ); + expect(entry?.nsPerOp).toBe(56.1); + }); + + it("ignores stored data with an unknown version", () => { + const memento = fakeMemento(); + memento.data.set("gotest.benchResults", { version: 99, entries: {} }); + const store = new BenchResultStore(memento); + expect( + store.getLatest("example.com/pkg", "CacheTestSuite", "BenchmarkGetHit", { + goos: "linux", + goarch: "amd64", + }), + ).toBeUndefined(); + }); + + it("notifies listeners when a report lands", () => { + const store = new BenchResultStore(fakeMemento()); + let fired = 0; + store.onDidUpdate(() => fired++); + store.recordReport(report("linux", "amd64", 56.1), Date.now()); + expect(fired).toBe(1); + }); +}); + +describe("hostPlatform", () => { + it("maps the node process platform/arch to GOOS/GOARCH names", () => { + expect(hostPlatform({ platform: "linux", arch: "x64" })).toEqual({ + goos: "linux", + goarch: "amd64", + }); + expect(hostPlatform({ platform: "darwin", arch: "arm64" })).toEqual({ + goos: "darwin", + goarch: "arm64", + }); + expect(hostPlatform({ platform: "win32", arch: "x64" })).toEqual({ + goos: "windows", + goarch: "amd64", + }); + }); +}); + +describe("BenchResultStore deltas", () => { + function compareReport(pct: number, significant: boolean): BenchReport { + const r = report("linux", "amd64", 112.3); + r.deltas = [ + { + key: "example.com/pkg CacheTestSuite/BenchmarkGetHit", + oldNs: 100, + newNs: 112.3, + percentChange: pct, + significant, + insufficientSample: false, + }, + ]; + return r; + } + + it("attaches the CLI's delta verdict to the matching entry", () => { + const store = new BenchResultStore(fakeMemento()); + store.recordReport(compareReport(12.3, true), Date.now()); + + const entry = store.getLatest( + "example.com/pkg", + "CacheTestSuite", + "BenchmarkGetHit", + { goos: "linux", goarch: "amd64" }, + ); + expect(entry?.delta).toEqual({ + percentChange: 12.3, + significant: true, + insufficientSample: false, + }); + }); + + it("clears a stale delta when a later run has no comparison", () => { + const store = new BenchResultStore(fakeMemento()); + store.recordReport(compareReport(12.3, true), Date.now()); + store.recordReport(report("linux", "amd64", 60), Date.now()); + + const entry = store.getLatest( + "example.com/pkg", + "CacheTestSuite", + "BenchmarkGetHit", + { goos: "linux", goarch: "amd64" }, + ); + expect(entry?.delta).toBeUndefined(); + }); +}); + +describe("BenchResultStore history", () => { + it("appends one history point per run, newest last", () => { + const store = new BenchResultStore(fakeMemento()); + store.recordReport(report("linux", "amd64", 100), 1000); + store.recordReport(report("linux", "amd64", 120), 2000); + + const history = store.getHistory( + "example.com/pkg", + "CacheTestSuite", + "BenchmarkGetHit", + { goos: "linux", goarch: "amd64" }, + ); + expect(history.map((h) => h.nsPerOp)).toEqual([100, 120]); + expect(history.map((h) => h.recordedAt)).toEqual([1000, 2000]); + }); + + it("caps history at 50 runs per key, dropping the oldest", () => { + const store = new BenchResultStore(fakeMemento()); + for (let i = 0; i < 55; i++) { + store.recordReport(report("linux", "amd64", i), i); + } + const history = store.getHistory( + "example.com/pkg", + "CacheTestSuite", + "BenchmarkGetHit", + { goos: "linux", goarch: "amd64" }, + ); + expect(history).toHaveLength(50); + expect(history[0].nsPerOp).toBe(5); + expect(history[49].nsPerOp).toBe(54); + }); + + it("keeps history platform-scoped like everything else", () => { + const store = new BenchResultStore(fakeMemento()); + store.recordReport(report("darwin", "arm64", 100), 1000); + expect( + store.getHistory("example.com/pkg", "CacheTestSuite", "BenchmarkGetHit", { + goos: "linux", + goarch: "amd64", + }), + ).toEqual([]); + }); + + it("records the sample spread of a multi-count run", () => { + const store = new BenchResultStore(fakeMemento()); + const multi = report("linux", "amd64", 0); + multi.baseline.results[0].samples = [ + { iterations: 10, nsPerOp: 100, bytesPerOp: 0, allocsPerOp: 0 }, + { iterations: 10, nsPerOp: 130, bytesPerOp: 0, allocsPerOp: 0 }, + { iterations: 10, nsPerOp: 110, bytesPerOp: 0, allocsPerOp: 0 }, + ]; + store.recordReport(multi, 1000); + const entry = store.getLatest( + "example.com/pkg", + "CacheTestSuite", + "BenchmarkGetHit", + { goos: "linux", goarch: "amd64" }, + ); + expect(entry?.minNsPerOp).toBe(100); + expect(entry?.maxNsPerOp).toBe(130); + }); +}); diff --git a/vscode-gotest/src/benchResultStore.ts b/vscode-gotest/src/benchResultStore.ts new file mode 100644 index 00000000..d3c4255c --- /dev/null +++ b/vscode-gotest/src/benchResultStore.ts @@ -0,0 +1,239 @@ +// benchResultStore keeps the last benchmark numbers per method so CodeLens +// annotations survive editor reloads. Entries are keyed by package, suite, +// method, AND goos/goarch — a number measured on another platform is a +// different number, and the store refuses to serve it for this host, the +// same way the CLI refuses cross-platform baseline comparisons. +// +// Persistence is the workspace Memento (workspaceState): benchmark numbers +// are workspace-scoped working state, not artifacts. The store depends only +// on the minimal MementoLike surface so tests run without a vscode mock. + +import type { BenchReport } from "./benchReport.js"; + +export interface MementoLike { + get(key: string, defaultValue: T): T; + update(key: string, value: unknown): Thenable; +} + +export interface PlatformKey { + goos: string; + goarch: string; +} + +/** The stored summary of one benchmark's most recent run. */ +/** + * The CLI's comparison verdict for one entry, copied verbatim from the + * report. `significant` is Welch's t-test speaking — the UI never overrides + * it, and an insignificant delta renders as nothing at all. + */ +export interface BenchEntryDelta { + percentChange: number; + significant: boolean; + insufficientSample: boolean; +} + +export interface BenchEntry { + nsPerOp: number; + bytesPerOp: number; + allocsPerOp: number; + iterations: number; + /** Number of samples behind the mean (>1 under -count=N). */ + sampleCount: number; + /** Fastest/slowest rep of a multi-sample run (equal to nsPerOp for 1×). */ + minNsPerOp: number; + maxNsPerOp: number; + recordedAt: number; + goos: string; + goarch: string; + /** Present only when the recording run compared against a baseline. */ + delta?: BenchEntryDelta; +} + +/** One point of a benchmark's run-over-run trend. */ +export interface BenchHistoryPoint { + nsPerOp: number; + recordedAt: number; + sampleCount: number; +} + +/** Bounded run-over-run history per key; the trend a hover shows. */ +const MAX_HISTORY = 50; + +interface StoredData { + version: 1; + entries: Record; + history?: Record; +} + +const STORAGE_KEY = "gotest.benchResults"; + +export function benchKey( + importPath: string, + suiteName: string, + methodName: string, + goos: string, + goarch: string, +): string { + return `${importPath}/${suiteName}/${methodName}@${goos}/${goarch}`; +} + +/** hostPlatform maps Node's platform/arch names onto GOOS/GOARCH. */ +export function hostPlatform( + proc: { platform: string; arch: string } = process, +): PlatformKey { + const goos = + proc.platform === "win32" + ? "windows" + : proc.platform === "sunos" + ? "solaris" + : proc.platform; + const archMap: Record = { + x64: "amd64", + ia32: "386", + arm64: "arm64", + arm: "arm", + }; + return { goos, goarch: archMap[proc.arch] ?? proc.arch }; +} + +export class BenchResultStore { + private entries = new Map(); + private history = new Map(); + private listeners: Array<() => void> = []; + + constructor(private readonly memento: MementoLike) { + const stored = this.memento.get( + STORAGE_KEY, + undefined, + ); + if (stored && stored.version === 1) { + for (const [key, entry] of Object.entries(stored.entries)) { + this.entries.set(key, entry); + } + for (const [key, points] of Object.entries(stored.history ?? {})) { + this.history.set(key, points); + } + } + } + + onDidUpdate(listener: () => void): { dispose(): void } { + this.listeners.push(listener); + return { + dispose: () => { + this.listeners = this.listeners.filter((l) => l !== listener); + }, + }; + } + + /** + * recordReport stores one entry per result in the report, stamped with the + * report's own goos/goarch. Multi-sample runs (-count=N) store the mean — + * the CLI's comparison logic consumes the full samples, the annotation + * only needs one honest number. + */ + recordReport(report: BenchReport, recordedAt: number): void { + const { goos, goarch } = report.baseline; + + // Delta rows are keyed "pkg Suite/Name" by the CLI; import paths never + // contain spaces and suite/method names never contain slashes. + const deltaByKey = new Map(); + for (const d of report.deltas ?? []) { + deltaByKey.set(d.key, { + percentChange: d.percentChange, + significant: d.significant, + insufficientSample: d.insufficientSample, + }); + } + + for (const result of report.baseline.results) { + const n = result.samples.length; + if (n === 0) continue; + const mean = (pick: (s: (typeof result.samples)[number]) => number) => + result.samples.reduce((sum, s) => sum + pick(s), 0) / n; + const nsValues = result.samples.map((s) => s.nsPerOp); + const meanNs = mean((s) => s.nsPerOp); + + const key = benchKey( + result.package, + result.suite, + result.name, + goos, + goarch, + ); + this.entries.set(key, { + nsPerOp: meanNs, + bytesPerOp: Math.round(mean((s) => s.bytesPerOp)), + allocsPerOp: Math.round(mean((s) => s.allocsPerOp)), + iterations: result.samples[0].iterations, + sampleCount: n, + minNsPerOp: Math.min(...nsValues), + maxNsPerOp: Math.max(...nsValues), + recordedAt, + goos, + goarch, + // A run without a comparison clears any stale delta: the verdict + // belonged to the numbers it was computed against. + delta: deltaByKey.get( + `${result.package} ${result.suite}/${result.name}`, + ), + }); + + const points = this.history.get(key) ?? []; + points.push({ nsPerOp: meanNs, recordedAt, sampleCount: n }); + if (points.length > MAX_HISTORY) { + points.splice(0, points.length - MAX_HISTORY); + } + this.history.set(key, points); + } + void this.persist(); + for (const listener of this.listeners) { + listener(); + } + } + + getLatest( + importPath: string, + suiteName: string, + methodName: string, + platform: PlatformKey, + ): BenchEntry | undefined { + return this.entries.get( + benchKey( + importPath, + suiteName, + methodName, + platform.goos, + platform.goarch, + ), + ); + } + + /** getHistory returns the trend for one key, oldest first. */ + getHistory( + importPath: string, + suiteName: string, + methodName: string, + platform: PlatformKey, + ): BenchHistoryPoint[] { + return ( + this.history.get( + benchKey( + importPath, + suiteName, + methodName, + platform.goos, + platform.goarch, + ), + ) ?? [] + ); + } + + private persist(): Thenable { + const data: StoredData = { + version: 1, + entries: Object.fromEntries(this.entries), + history: Object.fromEntries(this.history), + }; + return this.memento.update(STORAGE_KEY, data); + } +} diff --git a/vscode-gotest/src/benchRunner.test.ts b/vscode-gotest/src/benchRunner.test.ts new file mode 100644 index 00000000..d2af538a --- /dev/null +++ b/vscode-gotest/src/benchRunner.test.ts @@ -0,0 +1,94 @@ +import { describe, it, expect, vi } from "vitest"; + +vi.mock("vscode", () => ({ + EventEmitter: class {}, + window: { showErrorMessage: vi.fn() }, +})); + +import { + planBenchInvocations, + benchTargetFromItemId, + buildProfileArgs, +} from "./benchRunner.js"; + +describe("planBenchInvocations", () => { + it("keeps distinct method targets in selection order", () => { + const plan = planBenchInvocations([ + { importPath: "a/b", suiteName: "S", methodName: "BenchmarkX" }, + { importPath: "a/b", suiteName: "S", methodName: "BenchmarkY" }, + ]); + expect(plan).toHaveLength(2); + expect(plan[0].methodName).toBe("BenchmarkX"); + }); + + it("collapses duplicates", () => { + const plan = planBenchInvocations([ + { importPath: "a/b", suiteName: "S", methodName: "BenchmarkX" }, + { importPath: "a/b", suiteName: "S", methodName: "BenchmarkX" }, + ]); + expect(plan).toHaveLength(1); + }); + + it("lets a suite-level target subsume its method targets", () => { + const plan = planBenchInvocations([ + { importPath: "a/b", suiteName: "S", methodName: "BenchmarkX" }, + { importPath: "a/b", suiteName: "S" }, + { importPath: "a/b", suiteName: "Other", methodName: "BenchmarkZ" }, + ]); + expect(plan).toEqual([ + { importPath: "a/b", suiteName: "S" }, + { importPath: "a/b", suiteName: "Other", methodName: "BenchmarkZ" }, + ]); + }); +}); + +describe("benchTargetFromItemId", () => { + it("splits import path, suite, and method from the right", () => { + expect( + benchTargetFromItemId( + "github.com/x/y/pkg/CacheTestSuite/BenchmarkGetHit", + ), + ).toEqual({ + importPath: "github.com/x/y/pkg", + suiteName: "CacheTestSuite", + methodName: "BenchmarkGetHit", + }); + }); + + it("refuses ids whose leaf is not a benchmark method", () => { + expect( + benchTargetFromItemId("github.com/x/y/pkg/CacheTestSuite/TestGet"), + ).toBeUndefined(); + }); + + it("refuses ids that are too short to carry a method", () => { + expect(benchTargetFromItemId("pkg/Suite")).toBeUndefined(); + }); +}); + +describe("buildProfileArgs", () => { + it("adds the go test profile flag with an absolute output path", () => { + expect( + buildProfileArgs( + { importPath: "a/b", suiteName: "S", methodName: "BenchmarkX" }, + "cpu", + "/tmp/prof", + ), + ).toEqual([ + "bench", + "a/b", + "-bench=^BenchmarkS$/^BenchmarkX$", + "-cpuprofile=/tmp/prof/cpu.pprof", + "--json", + ]); + }); + + it("uses -memprofile for the memory kind", () => { + const args = buildProfileArgs( + { importPath: "a/b", suiteName: "S" }, + "mem", + "/tmp/prof", + ); + expect(args).toContain("-memprofile=/tmp/prof/mem.pprof"); + }); +}); diff --git a/vscode-gotest/src/benchRunner.ts b/vscode-gotest/src/benchRunner.ts new file mode 100644 index 00000000..c16446be --- /dev/null +++ b/vscode-gotest/src/benchRunner.ts @@ -0,0 +1,490 @@ +// benchRunner drives `gotest bench --json` for the Bench run profile and the +// bench CodeLenses. Benchmarks are deliberate acts: nothing here hooks into +// save events or watch mode, and invocations run strictly serially — timing +// numbers taken in parallel are noise. + +import * as vscode from "vscode"; +import { spawn } from "node:child_process"; +import { mkdtemp } from "node:fs/promises"; +import * as os from "node:os"; +import * as path from "node:path"; +import type { GoTestController } from "./testController.js"; +import type { DiscoveryCache } from "./discovery.js"; +import { + buildCliCommand, + buildBenchArgs, + formatCliCommand, + type CliCommand, +} from "./cli.js"; +import { + parseBenchReport, + formatBenchAnnotation, + type BenchReport, +} from "./benchReport.js"; +import type { BenchResultStore } from "./benchResultStore.js"; + +export interface BenchTarget { + importPath: string; + suiteName: string; + /** Absent = every benchmark method in the suite. */ + methodName?: string; + /** + * go test -count: repetitions per benchmark. The CLI harvests one sample + * per rep, and its significance machinery consumes them natively — this + * is how a "stable" run gets an honest mean ± spread with zero TS stats. + */ + count?: number; +} + +/** + * planBenchInvocations dedups a selection into the runs actually needed: a + * suite-level target subsumes its method-level targets, and duplicates + * collapse. Order is preserved (first occurrence wins) because benchmarks + * run serially and the user watches them land one by one. + */ +export function planBenchInvocations(targets: BenchTarget[]): BenchTarget[] { + const suiteLevel = new Set(); + for (const t of targets) { + if (!t.methodName) suiteLevel.add(`${t.importPath}/${t.suiteName}`); + } + const seen = new Set(); + const plan: BenchTarget[] = []; + for (const t of targets) { + const suiteKey = `${t.importPath}/${t.suiteName}`; + if (t.methodName && suiteLevel.has(suiteKey)) continue; + const key = t.methodName ? `${suiteKey}/${t.methodName}` : suiteKey; + if (seen.has(key)) continue; + seen.add(key); + plan.push(t); + } + return plan; +} + +/** + * benchTargetFromItemId decodes a benchmark TestItem id + * ("//"). Import paths contain slashes, + * suite and method names never do, so splitting from the right is exact. + */ +export function benchTargetFromItemId(id: string): BenchTarget | undefined { + const parts = id.split("/"); + if (parts.length < 3) return undefined; + const methodName = parts[parts.length - 1]; + const suiteName = parts[parts.length - 2]; + const importPath = parts.slice(0, -2).join("/"); + if (!methodName.startsWith("Benchmark")) return undefined; + return { importPath, suiteName, methodName }; +} + +export type ProfileKind = "cpu" | "mem"; + +/** + * buildProfileArgs is the profiling variant of a bench invocation: the same + * scoped run plus go test's own -cpuprofile/-memprofile, written into a + * caller-owned directory (absolute path — the suite subprocess runs in the + * package dir, and profiles must never land in the source tree). + */ +export function buildProfileArgs( + target: BenchTarget, + kind: ProfileKind, + outDir: string, +): string[] { + const args = buildBenchArgs( + target.importPath, + target.suiteName, + target.methodName, + ); + args.push(`-${kind}profile=${outDir}/${kind}.pprof`); + args.push("--json"); + return args; +} + +export class BenchRunner { + private active: vscode.CancellationTokenSource | undefined; + + constructor( + private readonly controller: GoTestController, + private readonly cache: DiscoveryCache, + private readonly store: BenchResultStore, + private readonly outputChannel: vscode.LogOutputChannel, + /** Called with every parsed report — gate diagnostics hang off this. */ + private readonly onReport?: (report: BenchReport) => void, + ) {} + + dispose(): void { + this.active?.cancel(); + this.active = undefined; + } + + /** Handler for the tag-scoped "Bench" run profile. */ + async runProfile( + request: vscode.TestRunRequest, + token: vscode.CancellationToken, + ): Promise { + const items = (request.include ?? []).filter((i) => + i.tags.some((t) => t.id === "benchmark"), + ); + if (items.length === 0) return; + + const targets: BenchTarget[] = []; + const byKey = new Map(); + for (const item of items) { + const target = benchTargetFromItemId(item.id); + if (!target) continue; + targets.push(target); + byKey.set(item.id, item); + } + + const run = this.controller.createTestRun(request, "Go Bench Run"); + for (const item of items) run.started(item); + try { + await this.execute(planBenchInvocations(targets), token, { + onResult: (importPath, suiteName, methodName, ok, message) => { + const item = byKey.get(`${importPath}/${suiteName}/${methodName}`); + if (!item) return; + if (ok) { + run.passed(item); + } else { + run.failed(item, new vscode.TestMessage(message ?? "bench failed")); + } + }, + onError: (target, message) => { + for (const item of items) { + const t = benchTargetFromItemId(item.id); + if ( + t && + t.importPath === target.importPath && + t.suiteName === target.suiteName && + (!target.methodName || t.methodName === target.methodName) + ) { + run.errored(item, new vscode.TestMessage(message)); + } + } + }, + }); + } finally { + run.end(); + } + } + + /** Entry point for the bench CodeLenses and commands. */ + async runTarget(target: BenchTarget): Promise { + const cts = new vscode.CancellationTokenSource(); + this.active?.cancel(); + this.active = cts; + try { + await this.execute([target], cts.token, {}); + } finally { + if (this.active === cts) this.active = undefined; + cts.dispose(); + } + } + + /** + * profileTarget runs one scoped benchmark with go test's own profiler and + * opens `go tool pprof -http` on the result. The profile lands in a fresh + * temp directory, never in the source tree; the run's numbers are recorded + * like any other bench run. + */ + async profileTarget(target: BenchTarget, kind: ProfileKind): Promise { + const workspaceDir = this.cache.getWorkspaceDir(target.importPath); + if (!workspaceDir) { + vscode.window.showErrorMessage( + `gotest bench: no workspace dir for ${target.importPath}`, + ); + return; + } + + const outDir = await mkdtemp(path.join(os.tmpdir(), "gotest-bench-prof-")); + const cmd = await buildCliCommand( + buildProfileArgs(target, kind, outDir), + workspaceDir, + this.outputChannel, + ); + this.outputChannel.info(`[bench] ${formatCliCommand(cmd)}`); + + const cts = new vscode.CancellationTokenSource(); + this.active?.cancel(); + this.active = cts; + try { + const { stdout, stderr, code } = await this.spawnBench( + cmd, + workspaceDir, + cts.token, + ); + if (stderr.trim()) this.outputChannel.warn(stderr.trimEnd()); + if (!stdout.trim()) { + throw new Error(`gotest bench exited with code ${code}`); + } + this.recordAndLog(parseBenchReport(stdout)); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + vscode.window.showErrorMessage(`gotest bench failed: ${message}`); + return; + } finally { + if (this.active === cts) this.active = undefined; + cts.dispose(); + } + + const profile = path.join(outDir, `${kind}.pprof`); + // pprof owns its lifetime: it picks a free port and opens the browser. + // Detached on purpose — closing the editor must not kill the analysis. + spawn("go", ["tool", "pprof", "-http=127.0.0.1:0", profile], { + cwd: workspaceDir, + detached: true, + stdio: "ignore", + }).unref(); + this.outputChannel.info(`[bench] pprof UI launched for ${profile}`); + } + + /** + * saveBaseline runs every benchmark in the workspace and saves a baseline. + * The path comes from bench.baseline in .gotest.yml — resolved by the CLI, + * never parsed here — with a save dialog as the fallback when the project + * has no configured baseline. + */ + async saveBaseline(workspaceDir: string): Promise { + const first = await this.runWorkspace(workspaceDir, ["--save="]); + if (first.ok) { + vscode.window.showInformationMessage("Bench baseline saved."); + return; + } + if (!/--save needs a path/.test(first.error)) { + vscode.window.showErrorMessage(`gotest bench failed: ${first.error}`); + return; + } + const picked = await vscode.window.showSaveDialog({ + title: "Save Bench Baseline", + filters: { "Bench baseline": ["json"] }, + }); + if (!picked) return; + const second = await this.runWorkspace(workspaceDir, [ + `--save=${picked.fsPath}`, + ]); + if (second.ok) { + vscode.window.showInformationMessage( + `Bench baseline saved to ${picked.fsPath}.`, + ); + } else { + vscode.window.showErrorMessage(`gotest bench failed: ${second.error}`); + } + } + + /** + * compareBaseline runs every benchmark in the workspace and compares. The + * CLI compares against bench.baseline automatically when configured; when + * the run comes back without deltas, the user picks a baseline file and + * the comparison reruns explicitly. + */ + async compareBaseline(workspaceDir: string): Promise { + let outcome = await this.runWorkspace(workspaceDir, []); + if (outcome.ok && !outcome.report.deltas) { + const picked = await vscode.window.showOpenDialog({ + title: "Compare vs Bench Baseline", + canSelectMany: false, + filters: { "Bench baseline": ["json"] }, + }); + if (!picked || picked.length === 0) return; + outcome = await this.runWorkspace(workspaceDir, [ + `--against=${picked[0].fsPath}`, + ]); + } + if (!outcome.ok) { + vscode.window.showErrorMessage(`gotest bench failed: ${outcome.error}`); + return; + } + + const deltas = outcome.report.deltas ?? []; + const significant = deltas.filter((d) => d.significant).length; + const gate = outcome.report.gate; + if (gate?.breached) { + vscode.window.showWarningMessage( + `Bench gate breached: ${gate.worstKey} +${gate.worstPct.toFixed(1)}% exceeds ${gate.thresholdPct}%.`, + ); + } else { + vscode.window.showInformationMessage( + `Compared ${deltas.length} benchmark${deltas.length === 1 ? "" : "s"}: ${ + significant === 0 + ? "no significant change" + : `${significant} significant change${significant === 1 ? "" : "s"}` + }.`, + ); + } + } + + /** + * runWorkspace runs `gotest bench ./... --json` with extra flags in one + * workspace, records the report, and surfaces the outcome to the caller + * instead of the UI — the baseline commands own their own messaging. + */ + private async runWorkspace( + workspaceDir: string, + extra: string[], + ): Promise<{ ok: true; report: BenchReport } | { ok: false; error: string }> { + const cmd = await buildCliCommand( + ["bench", "./...", ...extra, "--json"], + workspaceDir, + this.outputChannel, + ); + this.outputChannel.info(`[bench] ${formatCliCommand(cmd)}`); + + const cts = new vscode.CancellationTokenSource(); + this.active?.cancel(); + this.active = cts; + try { + const { stdout, stderr, code } = await this.spawnBench( + cmd, + workspaceDir, + cts.token, + ); + if (stderr.trim()) this.outputChannel.warn(stderr.trimEnd()); + if (!stdout.trim()) { + return { + ok: false, + error: stderr.trim() || `gotest bench exited with code ${code}`, + }; + } + const report = parseBenchReport(stdout); + this.recordAndLog(report); + return { ok: true, report }; + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + this.outputChannel.error(`[bench] failed: ${message}`); + return { ok: false, error: message }; + } finally { + if (this.active === cts) this.active = undefined; + cts.dispose(); + } + } + + private async execute( + plan: BenchTarget[], + token: vscode.CancellationToken, + hooks: { + onResult?: ( + importPath: string, + suiteName: string, + methodName: string, + ok: boolean, + message?: string, + ) => void; + onError?: (target: BenchTarget, message: string) => void; + }, + ): Promise { + for (const target of plan) { + if (token.isCancellationRequested) return; + + const workspaceDir = this.cache.getWorkspaceDir(target.importPath); + if (!workspaceDir) { + const msg = `no workspace dir for ${target.importPath}`; + this.outputChannel.error(`[bench] ${msg}`); + hooks.onError?.(target, msg); + continue; + } + + const args = [ + ...buildBenchArgs( + target.importPath, + target.suiteName, + target.methodName, + ), + ]; + if (target.count && target.count > 1) { + args.push(`-count=${target.count}`); + } + args.push("--json"); + const cmd = await buildCliCommand(args, workspaceDir, this.outputChannel); + this.outputChannel.info(`[bench] ${formatCliCommand(cmd)}`); + + let report: BenchReport; + try { + const { stdout, stderr, code } = await this.spawnBench( + cmd, + workspaceDir, + token, + ); + if (stderr.trim()) this.outputChannel.warn(stderr.trimEnd()); + if (code !== 0 && !stdout.trim()) { + throw new Error(`gotest bench exited with code ${code}`); + } + report = parseBenchReport(stdout); + } catch (err: unknown) { + const message = err instanceof Error ? err.message : String(err); + this.outputChannel.error(`[bench] failed: ${message}`); + hooks.onError?.(target, message); + if (!hooks.onError) { + vscode.window.showErrorMessage(`gotest bench failed: ${message}`); + } + continue; + } + + this.recordAndLog(report); + for (const result of report.baseline.results) { + hooks.onResult?.(result.package, result.suite, result.name, true); + } + } + } + + /** + * recordAndLog persists the report and writes the supplementary log: one + * line per method with the same numbers the annotation shows, so the + * channel remains a readable record of every run. + */ + private recordAndLog(report: BenchReport): void { + const now = Date.now(); + this.store.recordReport(report, now); + this.onReport?.(report); + + for (const result of report.baseline.results) { + const n = result.samples.length; + const nsValues = result.samples.map((s) => s.nsPerOp); + const mean = nsValues.reduce((sum, v) => sum + v, 0) / Math.max(1, n); + const last = result.samples[n - 1]; + let line = formatBenchAnnotation( + { + nsPerOp: mean, + bytesPerOp: last?.bytesPerOp ?? 0, + allocsPerOp: last?.allocsPerOp ?? 0, + iterations: last?.iterations ?? 0, + }, + now, + now, + ); + if (n > 1 && mean > 0) { + const halfSpreadPct = + (((Math.max(...nsValues) - Math.min(...nsValues)) / 2) * 100) / mean; + line += ` (mean of ${n}×, ±${halfSpreadPct.toFixed(1)}%)`; + } + this.outputChannel.info( + `[bench] ${result.suite}/${result.name}: ${line}`, + ); + } + } + + private spawnBench( + cmd: CliCommand, + cwd: string, + token: vscode.CancellationToken, + ): Promise<{ stdout: string; stderr: string; code: number | null }> { + return new Promise((resolve, reject) => { + const child = spawn(cmd.bin, cmd.args, { cwd }); + const sub = token.onCancellationRequested(() => child.kill("SIGTERM")); + let stdout = ""; + let stderr = ""; + + child.stdout.on("data", (data: Buffer) => { + stdout += data.toString(); + }); + child.stderr.on("data", (data: Buffer) => { + stderr += data.toString(); + }); + child.on("close", (code) => { + sub.dispose(); + resolve({ stdout, stderr, code }); + }); + child.on("error", (err: Error) => { + sub.dispose(); + reject(err); + }); + }); + } +} diff --git a/vscode-gotest/src/cli.test.ts b/vscode-gotest/src/cli.test.ts index 3a4738a1..690211e8 100644 --- a/vscode-gotest/src/cli.test.ts +++ b/vscode-gotest/src/cli.test.ts @@ -11,7 +11,12 @@ vi.mock("vscode", () => ({ window: { showWarningMessage: vi.fn(), showErrorMessage: vi.fn() }, })); -import { compareVersions, escapeRegExp, formatCliCommand } from "./cli.js"; +import { + compareVersions, + escapeRegExp, + formatCliCommand, + buildBenchArgs, +} from "./cli.js"; describe("compareVersions", () => { it("returns 0 for equal versions", () => { @@ -54,6 +59,35 @@ describe("escapeRegExp", () => { }); }); +describe("buildBenchArgs", () => { + it("scopes to the suite via -bench=, matching the generated wrapper name", () => { + expect(buildBenchArgs("example.com/pkg", "FooTestSuite")).toEqual([ + "bench", + "example.com/pkg", + "-bench=^BenchmarkFooTestSuite$", + ]); + }); + + it("scopes to a single method via go test's sub-benchmark slash pattern", () => { + expect( + buildBenchArgs("example.com/pkg", "FooTestSuite", "BenchmarkParse"), + ).toEqual([ + "bench", + "example.com/pkg", + "-bench=^BenchmarkFooTestSuite$/^BenchmarkParse$", + ]); + }); + + it("does not pass -run, since it matches the same wrapper name and AND-composes with -bench", () => { + const args = buildBenchArgs( + "example.com/pkg", + "FooTestSuite", + "BenchmarkParse", + ); + expect(args).not.toContain("-run"); + }); +}); + describe("formatCliCommand", () => { it("joins bin and args", () => { expect(formatCliCommand({ bin: "gotest", args: ["run", "./..."] })).toBe( diff --git a/vscode-gotest/src/cli.ts b/vscode-gotest/src/cli.ts index f4795b5f..98acde9e 100644 --- a/vscode-gotest/src/cli.ts +++ b/vscode-gotest/src/cli.ts @@ -292,6 +292,25 @@ export function formatCliCommand(cmd: CliCommand): string { return `${cmd.bin} ${cmd.args.join(" ")}`; } +// buildBenchArgs constructs the `gotest bench` subcommand arguments for a +// single suite. The generated wrapper is named "Benchmark" and runs +// each method under b.Run with its method name, so go test's slash matching +// scopes runs: "-bench=^Benchmark$" runs the whole suite, and +// "-bench=^Benchmark$/^$" a single method. Always the = +// form: the CLI pairs space-separated values too, but = keeps the argv +// unambiguous. +export function buildBenchArgs( + importPath: string, + suiteName: string, + methodName?: string, +): string[] { + const pattern = methodName + ? `^Benchmark${suiteName}$/^${methodName}$` + : `^Benchmark${suiteName}$`; + return ["bench", importPath, `-bench=${pattern}`]; +} + + export function scopedConfig( workspaceDir?: string, ): vscode.WorkspaceConfiguration { diff --git a/vscode-gotest/src/codeLens.ts b/vscode-gotest/src/codeLens.ts index 1b833512..cad869ea 100644 --- a/vscode-gotest/src/codeLens.ts +++ b/vscode-gotest/src/codeLens.ts @@ -1,6 +1,8 @@ import * as vscode from "vscode"; import * as path from "node:path"; import type { DiscoveryCache } from "./discovery.js"; +import { hostPlatform, type BenchResultStore } from "./benchResultStore.js"; +import { formatBenchAnnotation } from "./benchReport.js"; export class GoTestCodeLensProvider implements vscode.CodeLensProvider, vscode.Disposable @@ -9,12 +11,20 @@ export class GoTestCodeLensProvider readonly onDidChangeCodeLenses: vscode.Event = this._onDidChangeCodeLenses.event; - private subscription: vscode.Disposable; + private subscriptions: vscode.Disposable[] = []; - constructor(private readonly cache: DiscoveryCache) { - this.subscription = cache.onDidUpdate(() => - this._onDidChangeCodeLenses.fire(), + constructor( + private readonly cache: DiscoveryCache, + private readonly benchStore?: BenchResultStore, + ) { + this.subscriptions.push( + cache.onDidUpdate(() => this._onDidChangeCodeLenses.fire()), ); + if (benchStore) { + this.subscriptions.push( + benchStore.onDidUpdate(() => this._onDidChangeCodeLenses.fire()), + ); + } } provideCodeLenses( @@ -140,6 +150,64 @@ export class GoTestCodeLensProvider } } + const fileBenchmarks = suite.benchmarks.filter( + (m) => path.join(pkg.dir, m.file) === docPath, + ); + + if (suiteInFile && fileBenchmarks.length > 1) { + const range = new vscode.Range(suite.line - 1, 0, suite.line - 1, 0); + lenses.push( + new vscode.CodeLens(range, { + title: "▶ Bench Suite", + command: "gotest.runBench", + arguments: [importPath, suite.name], + }), + ); + } + + const platform = hostPlatform(); + for (const method of fileBenchmarks) { + const range = new vscode.Range(method.line - 1, 0, method.line - 1, 0); + + lenses.push( + new vscode.CodeLens(range, { + title: "▶ Bench", + command: "gotest.runBench", + arguments: [importPath, suite.name, method.name], + }), + // Five repetitions give the CLI enough samples for a trustworthy + // Welch comparison and an honest ± spread on the annotation. + new vscode.CodeLens(range, { + title: "5×", + command: "gotest.runBenchStable", + arguments: [importPath, suite.name, method.name], + }), + ); + + // The last measured numbers, right where the code is — but only + // numbers taken on this host's goos/goarch: a result from another + // platform is a different number and never shown here. + const latest = this.benchStore?.getLatest( + importPath, + suite.name, + method.name, + platform, + ); + if (latest) { + lenses.push( + new vscode.CodeLens(range, { + title: formatBenchAnnotation( + latest, + latest.recordedAt, + Date.now(), + latest.delta, + ), + command: "", + }), + ); + } + } + if (suiteInFile && suiteHasSnapshots) { const range = new vscode.Range(suite.line - 1, 0, suite.line - 1, 0); const testPath = `${importPath}/${suite.name}`; @@ -157,7 +225,10 @@ export class GoTestCodeLensProvider } dispose(): void { - this.subscription.dispose(); + for (const sub of this.subscriptions) { + sub.dispose(); + } + this.subscriptions = []; this._onDidChangeCodeLenses.dispose(); } } diff --git a/vscode-gotest/src/discovery.test.ts b/vscode-gotest/src/discovery.test.ts index 3ba8cb49..bbd19ca0 100644 --- a/vscode-gotest/src/discovery.test.ts +++ b/vscode-gotest/src/discovery.test.ts @@ -230,6 +230,55 @@ describe("DiscoveryService", () => { }); }); + describe("when a suite has benchmarks", () => { + it("carries the benchmarks array through into the cache", async () => { + mockExecFileAsync.mockResolvedValueOnce({ + stdout: JSON.stringify({ + packages: [ + { + importPath: "example.com/pkg", + dir: "/ws/pkg", + suites: [ + { + name: "FooTestSuite", + parallel: false, + focused: false, + excluded: false, + guarded: false, + file: "foo_test.go", + line: 1, + col: 1, + lifecycle: [], + fixtures: [], + methods: [], + benchmarks: [ + { + name: "BenchmarkParse", + parallel: false, + focused: false, + excluded: false, + file: "foo_test.go", + line: 8, + col: 1, + }, + ], + }, + ], + }, + ], + }), + stderr: "", + }); + + await service.discover("/ws", ["./..."]); + + const pkg = cache.getPackage("example.com/pkg"); + expect(pkg?.suites[0].benchmarks).toEqual([ + expect.objectContaining({ name: "BenchmarkParse", excluded: false }), + ]); + }); + }); + describe("when discovery recovers after a previous total failure", () => { it("re-enables the warning toast for future failures", async () => { mockExecFileAsync.mockRejectedValue(new Error("fail")); @@ -293,6 +342,7 @@ describe("DiscoveryCache broken packages", () => { lifecycle: [], fixtures: [], methods: [], + benchmarks: [], }; } diff --git a/vscode-gotest/src/extension.ts b/vscode-gotest/src/extension.ts index 1de144a3..a682d0c4 100644 --- a/vscode-gotest/src/extension.ts +++ b/vscode-gotest/src/extension.ts @@ -2,6 +2,10 @@ import * as vscode from "vscode"; import { DiscoveryCache, DiscoveryService } from "./discovery.js"; import { GoTestController } from "./testController.js"; import { TestRunner } from "./runner.js"; +import { BenchRunner } from "./benchRunner.js"; +import { BenchResultStore } from "./benchResultStore.js"; +import { BenchGateDiagnostics } from "./benchDiagnostics.js"; +import { BenchHoverProvider } from "./benchHover.js"; import { GoTestCodeLensProvider } from "./codeLens.js"; import { DebugLauncher } from "./debug.js"; import { FocusExcludeProvider } from "./focusExclude.js"; @@ -53,6 +57,9 @@ export function activate(context: vscode.ExtensionContext): void { let runner!: TestRunner; let coverageRunner!: CoverageRunner; + let benchRunner!: BenchRunner; + + const benchResultStore = new BenchResultStore(context.workspaceState); const controller = new GoTestController( cache, @@ -71,6 +78,7 @@ export function activate(context: vscode.ExtensionContext): void { ), (request, token) => coverageRunner.run(request, token), (request, token) => runner.run(request, token, { updateSnapshots: true }), + (request, token) => benchRunner.runProfile(request, token), ); controller.testController.refreshHandler = async () => { @@ -125,6 +133,16 @@ export function activate(context: vscode.ExtensionContext): void { coverageStore, ); + const benchGateDiagnostics = new BenchGateDiagnostics(cache); + benchRunner = new BenchRunner( + controller, + cache, + benchResultStore, + outputChannel, + (report) => benchGateDiagnostics.apply(report), + ); + context.subscriptions.push(benchRunner, benchGateDiagnostics); + const specViewRefreshDisposable = runner.onDidComplete((jsonOutput) => { specView.refresh(jsonOutput, "run"); }); @@ -142,10 +160,11 @@ export function activate(context: vscode.ExtensionContext): void { const diagnostics = new FocusDiagnostics(cache); debugLauncher.registerCleanupOnSessionEnd(context); - const providerDisposables = registerProviders(cache); + const providerDisposables = registerProviders(cache, benchResultStore); const commandDisposables = registerCommands({ controller, runner, + benchRunner, debugLauncher, discoveryService, diagnostics, @@ -220,13 +239,26 @@ function resolveActiveWorkspaceDir(): string | undefined { return folder?.uri.fsPath; } -function registerProviders(cache: DiscoveryCache): vscode.Disposable[] { - const codeLensProvider = new GoTestCodeLensProvider(cache); +function registerProviders( + cache: DiscoveryCache, + benchResultStore?: BenchResultStore, +): vscode.Disposable[] { + const codeLensProvider = new GoTestCodeLensProvider(cache, benchResultStore); const codeLensDisposable = vscode.languages.registerCodeLensProvider( { language: "go", pattern: "**/*_test.go" }, codeLensProvider, ); + const extraDisposables: vscode.Disposable[] = []; + if (benchResultStore) { + extraDisposables.push( + vscode.languages.registerHoverProvider( + { language: "go", pattern: "**/*_test.go" }, + new BenchHoverProvider(cache, benchResultStore), + ), + ); + } + const focusExcludeProvider = new FocusExcludeProvider(cache); const codeActionsDisposable = vscode.languages.registerCodeActionsProvider( { language: "go", pattern: "**/*_test.go" }, @@ -248,6 +280,7 @@ function registerProviders(cache: DiscoveryCache): vscode.Disposable[] { return [ codeLensProvider, codeLensDisposable, + ...extraDisposables, focusExcludeProvider, codeActionsDisposable, scaffoldProvider, @@ -258,6 +291,7 @@ function registerProviders(cache: DiscoveryCache): vscode.Disposable[] { function registerCommands(deps: { controller: GoTestController; runner: TestRunner; + benchRunner: BenchRunner; debugLauncher: DebugLauncher; discoveryService: DiscoveryService; diagnostics: FocusDiagnostics; @@ -271,6 +305,7 @@ function registerCommands(deps: { const { controller, runner, + benchRunner, debugLauncher, discoveryService, diagnostics, @@ -301,6 +336,100 @@ function registerCommands(deps: { }, ), + vscode.commands.registerCommand( + "gotest.runBench", + async (importPath: string, suiteName: string, methodName?: string) => { + await benchRunner.runTarget({ importPath, suiteName, methodName }); + }, + ), + + vscode.commands.registerCommand( + "gotest.runBenchStable", + async (importPath: string, suiteName: string, methodName?: string) => { + await benchRunner.runTarget({ + importPath, + suiteName, + methodName, + count: 5, + }); + }, + ), + + vscode.commands.registerCommand( + "gotest.profileBench", + async (importPath?: string, suiteName?: string, methodName?: string) => { + // Palette invocations carry no target: offer every discovered + // benchmark. CodeLens/explorer callers pass the target directly. + if (!importPath || !suiteName) { + const picks: Array< + vscode.QuickPickItem & { + target: { + importPath: string; + suiteName: string; + methodName: string; + }; + } + > = []; + for (const pkg of cache.packages) { + for (const suite of pkg.suites) { + for (const bench of suite.benchmarks ?? []) { + picks.push({ + label: `${suite.name}/${bench.name}`, + description: pkg.importPath, + target: { + importPath: pkg.importPath, + suiteName: suite.name, + methodName: bench.name, + }, + }); + } + } + } + if (picks.length === 0) { + vscode.window.showInformationMessage( + "No benchmarks discovered in this workspace.", + ); + return; + } + const picked = await vscode.window.showQuickPick(picks, { + title: "Profile Benchmark", + }); + if (!picked) return; + ({ importPath, suiteName, methodName } = picked.target); + } + const kind = await vscode.window.showQuickPick( + [ + { label: "CPU", profileKind: "cpu" as const }, + { label: "Memory", profileKind: "mem" as const }, + ], + { title: "Profile kind" }, + ); + if (!kind) return; + await benchRunner.profileTarget( + { importPath, suiteName, methodName }, + kind.profileKind, + ); + }, + ), + + vscode.commands.registerCommand("gotest.saveBenchBaseline", async () => { + const wsDir = resolveActiveWorkspaceDir(); + if (!wsDir) { + outputChannel.warn("[command] saveBenchBaseline: no workspace dir"); + return; + } + await benchRunner.saveBaseline(wsDir); + }), + + vscode.commands.registerCommand("gotest.compareBenchBaseline", async () => { + const wsDir = resolveActiveWorkspaceDir(); + if (!wsDir) { + outputChannel.warn("[command] compareBenchBaseline: no workspace dir"); + return; + } + await benchRunner.compareBaseline(wsDir); + }), + vscode.commands.registerCommand( "gotest.debugTest", async (testId: string) => { diff --git a/vscode-gotest/src/runner.ts b/vscode-gotest/src/runner.ts index b9b4ee1e..1d9a23b8 100644 --- a/vscode-gotest/src/runner.ts +++ b/vscode-gotest/src/runner.ts @@ -76,8 +76,19 @@ export class TestRunner { let recordId: string | undefined; try { - const items = collectItems(this.controller, request); + // Benchmarks never run as part of a normal test run: they answer with + // a number, and that number is garbage when tests hammer the machine + // at the same time. The Bench profile (tag-scoped) is their only door. + const collected = collectItems(this.controller, request); + const items = collected.filter( + (i) => !i.tags.some((t) => t.id === "benchmark"), + ); if (items.length === 0) { + if (collected.length > 0) { + vscode.window.showInformationMessage( + "Benchmarks run through the Bench profile or the ▶ Bench CodeLens, not a test run.", + ); + } return; } @@ -304,3 +315,7 @@ export class TestRunner { return this._lastJsonOutput.includes("MatchSnapshot: snapshot mismatch"); } } + +// Benchmark execution lives in benchRunner.ts: `gotest bench --json` parsed +// into typed results, recorded in the BenchResultStore, rendered as CodeLens +// annotations. Nothing here runs benchmarks. diff --git a/vscode-gotest/src/testController.test.ts b/vscode-gotest/src/testController.test.ts index 686934cd..d9b6e072 100644 --- a/vscode-gotest/src/testController.test.ts +++ b/vscode-gotest/src/testController.test.ts @@ -686,3 +686,89 @@ describe("GoTestController broken packages", () => { expect(pkgItem.error).toBeUndefined(); }); }); + +describe("GoTestController benchmark items", () => { + beforeEach(() => { + mockWorkspaceFolders.length = 0; + mockGetWorkspaceFolder.mockReset(); + mockWorkspaceFolders.push({ name: "ws", uri: { fsPath: "/ws" } }); + }); + + it("registers benchmark methods beside tests, tagged for the Bench profile", () => { + const suite = makeSuite("CacheSuite"); + suite.benchmarks = [ + { + name: "BenchmarkGetHit", + parallel: false, + focused: false, + excluded: false, + file: "cachesuite_test.go", + line: 20, + col: 1, + }, + ]; + const cache = createMockCache([ + { + importPath: "example.com/proj/pkg", + dir: "/ws/pkg", + wsDir: "/ws", + suites: [suite], + }, + ]); + + const ctrl = createController(cache); + ctrl.rebuild(); + + const pkgItem = (ctrl.testController.items as any)._map.get( + "example.com/proj/pkg", + ); + const suiteItem = pkgItem.children.get("example.com/proj/pkg/CacheSuite"); + const benchItem = suiteItem.children.get( + "example.com/proj/pkg/CacheSuite/BenchmarkGetHit", + ); + expect(benchItem).toBeDefined(); + expect(benchItem.description).toBe("bench"); + expect(benchItem.tags.map((t: any) => t.id)).toContain("benchmark"); + + const testItem = suiteItem.children.get( + "example.com/proj/pkg/CacheSuite/TestOne", + ); + expect(testItem.tags.map((t: any) => t.id)).not.toContain("benchmark"); + }); + + it("removes a benchmark item when the method disappears from discovery", () => { + const suite = makeSuite("CacheSuite"); + suite.benchmarks = [ + { + name: "BenchmarkGetHit", + parallel: false, + focused: false, + excluded: false, + file: "cachesuite_test.go", + line: 20, + col: 1, + }, + ]; + const cache = createMockCache([ + { + importPath: "example.com/proj/pkg", + dir: "/ws/pkg", + wsDir: "/ws", + suites: [suite], + }, + ]); + + const ctrl = createController(cache); + ctrl.rebuild(); + suite.benchmarks = []; + ctrl.rebuild(); + + const pkgItem = (ctrl.testController.items as any)._map.get( + "example.com/proj/pkg", + ); + const suiteItem = pkgItem.children.get("example.com/proj/pkg/CacheSuite"); + expect( + suiteItem.children.get("example.com/proj/pkg/CacheSuite/BenchmarkGetHit"), + ).toBeUndefined(); + }); +}); diff --git a/vscode-gotest/src/testController.ts b/vscode-gotest/src/testController.ts index 09ebb4d9..8d17398b 100644 --- a/vscode-gotest/src/testController.ts +++ b/vscode-gotest/src/testController.ts @@ -32,6 +32,10 @@ export class GoTestController implements vscode.Disposable { request: vscode.TestRunRequest, token: vscode.CancellationToken, ) => Promise, + benchHandler?: ( + request: vscode.TestRunRequest, + token: vscode.CancellationToken, + ) => Promise, ) { this.controller = vscode.tests.createTestController("gotest", "gotest"); @@ -63,6 +67,21 @@ export class GoTestController implements vscode.Disposable { false, ); + // Benchmarks get their own profile, scoped to the "benchmark" tag so it + // is only offered on benchmark items — and benchmark items never run as + // part of a normal test run (the run handler filters the tag out). + // A benchmark answers with a number, not pass/fail; mixing the two run + // kinds would produce meaningless timings on loaded machines. + if (benchHandler) { + this.controller.createRunProfile( + "Bench", + vscode.TestRunProfileKind.Run, + (request, token) => benchHandler(request, token), + false, + new vscode.TestTag("benchmark"), + ); + } + let rebuildTimer: ReturnType | undefined; this.disposables.push( this.cache.onDidUpdate(() => { @@ -406,6 +425,35 @@ export class GoTestController implements vscode.Disposable { suiteItem.children.add(methodItem); } + // Benchmark methods sit beside tests and fuzzers under their suite — + // names are disjoint by prefix (Benchmark*), so they share the id + // namespace. They carry the "benchmark" tag: the tag routes them to + // the Bench run profile and keeps them out of ordinary runs. + for (const bench of suite.benchmarks ?? []) { + const benchId = `${suiteId}/${bench.name}`; + seenMethodIds.add(benchId); + + const benchUri = vscode.Uri.file(path.join(pkg.dir, bench.file)); + let benchItem = suiteItem.children.get(benchId); + if (!benchItem) { + benchItem = this.controller.createTestItem( + benchId, + bench.name, + benchUri, + ); + } + benchItem.range = new vscode.Range( + new vscode.Position(bench.line - 1, bench.col - 1), + new vscode.Position(bench.line - 1, bench.col - 1), + ); + benchItem.tags = [ + new vscode.TestTag("benchmark"), + ...this.buildTags(bench.focused, bench.excluded, bench.parallel), + ]; + benchItem.description = "bench"; + suiteItem.children.add(benchItem); + } + suiteItem.children.forEach((child) => { if (!seenMethodIds.has(child.id) && !child.id.includes("/dynamic/")) { suiteItem.children.delete(child.id); diff --git a/vscode-gotest/src/types.ts b/vscode-gotest/src/types.ts index 36b0e74d..f735ef0a 100644 --- a/vscode-gotest/src/types.ts +++ b/vscode-gotest/src/types.ts @@ -35,6 +35,7 @@ export interface DiscoverSuite { lifecycle: string[]; fixtures: string[]; methods: DiscoverMethod[]; + benchmarks: DiscoverMethod[]; } export interface DiscoverMethod {