Repository navigation
Expand file tree
/
Copy pathcheck.go
More file actions
1140 lines (1094 loc) · 50.2 KB
/
Copy pathcheck.go
File metadata and controls
1140 lines (1094 loc) · 50.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// This file prepares the required tests in a working copy, judges the copy
// against a Story, and records an accepted copy as a commit on its build
// branch.
//
// Only files inside the test module may change. A check requires the copy to
// have the build's tests branch checked out with no Git operation in progress,
// and measures the copy's changes from the build's base commit, so the test
// writer's own commits on the branch count as changes too.
package testwriter
import (
"bytes"
"context"
"crypto/rand"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"slices"
"strings"
"syscall"
"time"
"unicode/utf8"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/goadesign/surla/internal/abspath"
"github.com/goadesign/surla/internal/designeval"
"github.com/goadesign/surla/internal/genpublish"
"github.com/goadesign/surla/internal/procgroup"
"github.com/goadesign/surla/verify/candidate"
"github.com/goadesign/surla/verify/suite"
)
type (
// Suite locates the project's evaluator suite: the Go module holding the
// evaluation design, its generated gen/ folder and the test writer's
// scenario hooks, and the suite command in it that follows Surla's suite
// command contract (verify/suite). A surla init starter has Module "eval"
// and Command "./gen/cmd/<name>-evals", the program the generator writes
// in every Surla test module (designeval.SuiteCommandPackage).
Suite struct {
// Module is the module's slash-separated path relative to the
// project folder.
Module string
// Command is the suite command's package relative to Module: "." or
// a path starting with "./".
Command string
}
// Checker prepares and checks test writers' working copies. Its methods
// may be called concurrently, but not for the same working copy.
Checker struct {
// evaluator generates the required tests; its owner closes it.
evaluator *designeval.Evaluator
// workDir is the owner's private folder, outside every project, that
// holds check scratch and, by default, the checks' private compiler
// cache. The owner removes it after the last call.
workDir string
// buildCacheDir is supplied by the project owner, or empty for the
// private cache under workDir. Removing check scratch never removes it.
buildCacheDir string
// build and list run a check's two steps that compile and run the
// author's code: candidate.BuildModule and candidate.Output, each through
// runInGroup. Tests wrap them with stand-ins that sleep.
build func(context.Context, candidate.BuildSpec, func(*exec.Cmd) error) error
list func(context.Context, candidate.Artifact, candidate.Spec, int, func(*exec.Cmd) error) ([]byte, error)
// download downloads, with the go command goBin, the dependencies of
// the module in dir: downloadDependencies, for which tests stand in
// so that they need no network.
download func(ctx context.Context, goBin, dir string) error
// buildLimit and listLimit bound those two steps: buildProgramLimit
// and listTestsLimit, which tests shorten.
buildLimit, listLimit time.Duration
}
// Verdict is Check's judgment of one working copy against a Story.
Verdict struct {
// Findings lists what the test writer must change, in the order the
// check found them; it is empty exactly when the copy is accepted.
Findings []Finding
// DeclaredScenarios are the exact IDs listed from the checked tree,
// set when the structural check succeeds. The report owner uses them
// to validate writer statements without another program execution.
DeclaredScenarios []string
// copy is the working copy that was judged.
copy WorkingCopy
// tree is the git tree that was built and described: the copy's
// files, as a commit would hold them, with the Story table and the
// generated folder written again from the Story. It is set exactly
// when Findings is empty; Record commits it.
tree string
}
// Finding is one reason a working copy is not accepted, worded for the
// test writer.
Finding struct {
// Kind is what failed.
Kind FindingKind
// Message says what failed and how, quoting the tool's output, at
// most maxFindingBytes long.
Message string
}
// FindingKind names the step of the check that failed.
FindingKind string
// AuthorError is a failure of Prepare or Record that the working copy's
// author can fix: generation failed on module files, a module path cannot
// be opened/read, or the copy's branch/checked tip changed before recording.
// Finding states the recovery with paths relative to the working copy.
// Callers find it with errors.As; other errors retain operational causes.
AuthorError struct {
// Finding is what the author must change, with paths relative to the
// working copy.
Finding Finding
// cause is the underlying failure, kept for errors.Is and traces.
cause error
}
// toolchain is the Go toolchain and module cache a check builds with.
toolchain struct {
// goroot is the resolved toolchain root.
goroot string
// modCache is the module cache the toolchain downloads into.
modCache string
}
)
const (
// FindingGeneration means the required tests could not be generated from
// the Story with the copy's suite module, for example because its design
// does not compile, does not declare exactly one evaluation suite, or a
// folder stands where Surla writes the module's Story table.
FindingGeneration FindingKind = "generation"
// FindingDependencies means the suite module's dependencies could not be
// downloaded from its go.mod and go.sum, and the cause may be in the
// author's changes to them (dependencyFailure).
FindingDependencies FindingKind = "dependencies"
// FindingBuild means the suite command does not compile, for example
// because a required hook method is missing. The message quotes the
// compiler, which names the method.
FindingBuild FindingKind = "build"
// FindingListing means the suite command's --describe failed or did not
// print exactly one suite declaration on standard output.
FindingListing FindingKind = "listing"
// FindingMissingScenario means the suite command does not declare a
// scenario the Story requires. The message names the scenario, its flow
// and its hook method.
FindingMissingScenario FindingKind = "missing_scenario"
// FindingCoverageReference means a report statement names a scenario
// absent from the exact test program the check listed.
FindingCoverageReference FindingKind = "coverage_reference"
// FindingOutsideModule means a file outside the test module differs
// from the copy's base. The message names the file, the base commit and
// the required working-copy contents without a Git recipe.
FindingOutsideModule FindingKind = "outside_module"
// FindingHiddenTag means a declared scenario is tagged holdout. Tests in
// files the builder can read are not hidden.
FindingHiddenTag FindingKind = "hidden_tag"
// FindingWorkingCopy means the test writer's working copy is not on the
// build's tests branch, has a Git operation in progress, or its tip no
// longer belongs to that build. The message names the copy's state and
// tells the test writer to put its test
// files uncommitted on the build's tests branch and report again.
FindingWorkingCopy FindingKind = "working_copy"
// buildProgramLimit bounds one build of the test program. It
// compiles the test module and its dependencies with an empty or warm
// build cache, which takes a few minutes at most for a starter's
// dependencies; ten leaves room for a slower machine while still
// leaving the whole check room to finish.
buildProgramLimit = 10 * time.Minute
// listTestsLimit bounds one run of the test program's --describe, which
// declares the scenarios without running any: it should take well under
// a second, so sixty seconds is reached only by a program that waits on
// something while starting.
listTestsLimit = 60 * time.Second
// maxOutsideFindings bounds how many changed files outside the test
// module a verdict names one by one; one more finding counts the rest.
maxOutsideFindings = 20
// holdoutTag is the scenario tag that marks a hidden test.
holdoutTag = "holdout"
// maxFindingBytes bounds one finding's message.
maxFindingBytes = 8 << 10
// maxDeclarationBytes bounds what --describe may print.
maxDeclarationBytes = 1 << 20
// groupStopGrace bounds how long a canceled build or listing gets to
// exit, and how long its process group may take to empty.
groupStopGrace = 10 * time.Second
)
// ErrGeneration and ErrWriteGenerated mark the two steps of Prepare and Check
// that write the required tests, for a failure the copy's author cannot fix.
// Callers tell them apart with errors.Is to say what failed.
var (
// ErrGeneration marks a failure of the generator itself: while it
// prepared or removed its own files, started its program or read its
// output, or because ctx ended; or Story
// flows that map to the same scenario or hook. It wraps the evaluator's
// error, from which designeval.GenerationDiagnostic derives the
// caller-safe text.
ErrGeneration = errors.New("generate the required tests")
// ErrWriteGenerated marks a failure to write the test module's Story
// table, or to replace its generated folder after generation succeeded,
// for example because the disk is full or the module cannot be written.
// Its text names files only inside the module, apart from absolute paths
// that WithoutAbsolutePaths removes.
ErrWriteGenerated = errors.New("write the generated folder")
)
// errTableIsFolder reports a folder at the path of the test module's Story
// table, which writeStoryTests cannot replace.
var errTableIsFolder = errors.New("a folder is at the Story table's path")
// ErrDependencies reports that the test module's dependencies could not be
// downloaded where Surla runs, for a cause that is not in the author's
// changes: the module's go.mod and go.sum are the ones of the copy's base, or
// the base's own could not be downloaded either (dependencyFailure). The
// download's output is withheld, so the error's text holds only what Surla
// decided and the base commit; the user finds the cause by running the
// download where Surla runs.
var ErrDependencies = errors.New("the test module's dependencies could not be downloaded where Surla runs")
// absolutePath matches an absolute file path where a word starts: a slash at
// the start of a line or after a space, a quote, a parenthesis, a bracket, an
// equals sign or a comma, followed by a name character, up to the next
// space, quote, parenthesis, bracket, comma or colon (so a compiler's
// file.go:12:3 keeps its line and column). A URL's slashes follow a colon or
// another slash, an import path starts with a name, and a relative path with
// a name or a dot, so none of them matches.
var absolutePath = regexp.MustCompile(`(?m)(^|[\s"'(\[=,])/[A-Za-z0-9_.~+@-][^\s"'()\[\],:]*`)
// ErrLinkedFolder refuses a suite module reached through a symbolic link.
// Prepare and Check read and write the module only through real folders from
// the working copy's root, so a link the copy's author places on the way
// cannot lead them outside the copy, for example into the project.
var ErrLinkedFolder = errors.New("the suite module must be reached through real folders, not a symbolic link")
// NewChecker returns a Checker that generates with evaluator and works in
// workDir, a private folder its owner keeps outside every project and
// removes after the Checker's last call. Nothing is created until a call
// needs it.
func NewChecker(evaluator *designeval.Evaluator, workDir string) *Checker {
return &Checker{
evaluator: evaluator,
workDir: workDir,
build: candidate.BuildModule,
list: candidate.Output,
download: downloadDependencies,
buildLimit: buildProgramLimit,
listLimit: listTestsLimit,
}
}
// NewCheckerWithCache returns a checker with independent scratch and an
// owner-retained Go package cache. The owner supplies an absolute trusted cache
// outside source and scratch folders, shared only on one local filesystem.
// Removing workDir after the last check leaves the cache intact. Construction
// creates no files and rejects an empty or unresolved cache path.
func NewCheckerWithCache(
evaluator *designeval.Evaluator, workDir, buildCacheDir string,
) (*Checker, error) {
if !filepath.IsAbs(buildCacheDir) {
return nil, errors.New("test compiler build cache directory must be absolute")
}
cache, err := abspath.EvalMissing(buildCacheDir)
if err != nil {
return nil, fmt.Errorf("resolve test compiler build cache directory: %w", err)
}
c := NewChecker(evaluator, workDir)
c.buildCacheDir = cache
return c, nil
}
// Record commits an accepted verdict's tree, exactly the files Check built
// and described, on branch in the working copy's repository with message and
// the user's own git identity, never signed and with hooks disabled, and
// returns the commit. checkedTip is the full commit ID the caller checked
// with CheckBuildCommit; the commit uses exactly that parent, or refuses if
// the branch changed. The message carries no attribution. When the branch
// already holds that tree, as when the copy did not change since the last
// record, it returns the branch's commit and makes
// none, after confirming that the tip still equals checkedTip. Edits made
// after Check are not recorded. AuthorError marks a changed tip/branch or a
// Git operation the author can fix. Other errors preserve operational causes;
// if the reference was published but its index could not be updated, both the
// returned commit and error name that publication. Empty branch, checkedTip
// or message, and a verdict not made by Check or with findings are refused.
func Record(
ctx context.Context, v Verdict, branch, checkedTip, message string,
) (string, error) {
if !v.Accepted() {
return "", errors.New("record: the verdict has findings; only an accepted working copy is recorded")
}
if v.tree == "" {
return "", errors.New("record: the verdict did not come from Check")
}
if strings.TrimSpace(branch) == "" {
return "", errors.New("record: the branch is empty")
}
if strings.TrimSpace(checkedTip) == "" {
return "", errors.New("record: the checked tip is empty")
}
if strings.TrimSpace(message) == "" {
return "", errors.New("record: the commit message is empty")
}
tip, err := v.copy.requireCommit(ctx, checkedTip)
if err != nil {
return "", err
}
if tip != checkedTip {
return "", errors.New("record: the checked tip must be a full commit ID")
}
rawBranch, facts, err := v.copy.readFacts(ctx)
if err != nil {
return "", err
}
if facts.Detached || rawBranch != branch || facts.Operation != "none" {
return "", &AuthorError{Finding: workingCopyFinding(branch, facts)}
}
return commitTree(ctx, v.copy, "refs/heads/"+branch, v.tree, checkedTip, message)
}
// WithoutAbsolutePaths replaces every absolute file path in text with
// <absolute path>, leaving URLs, import paths and relative paths such as
// ./gen/cmd/shop-evals as they are. Every finding and every reason a check
// gives passes through it, so no path of Surla's folders or of the user's
// machine reaches the test writer, the builder or the page.
func WithoutAbsolutePaths(text string) string {
return absolutePath.ReplaceAllString(text, "${1}<absolute path>")
}
// Prepare writes the required tests for stories into the working copy on
// branch at base: it first verifies that the copy has branch checked out with
// no Git operation in progress, then renders the copy's
// suite module's Story table (designeval.StoryTestsPath) from stories, runs
// the design evaluator's generator on the module with it (comparing any
// dependency failure against base), and writes the table and replaces the
// module's gen/ folder (and its AGENTS_QUICKSTART.md) with the result. The
// table is a source file of the module's design, so the test writer's own
// regeneration of the module keeps the required tests. It returns the required
// scenarios, one per finished flow, named as designeval.StoryScenarios names
// them. The project folder is not touched: the module is opened one real
// folder at a time from the project folder through the copy, and a symbolic
// link on the way is refused, naming it, before anything is read or written.
//
// A failure the copy's author can fix is an *AuthorError carrying the finding
// Check would give, worded relative to the copy (including FindingWorkingCopy
// when the copy is off branch or has a Git operation in progress); a symbolic
// link on the way also wraps ErrLinkedFolder. Any
// other error is a failure the author cannot fix. Two are marked so a caller
// can say what failed: ErrGeneration, when the generator failed (the context
// ended, or it failed while preparing its own files), and ErrWriteGenerated,
// when the generated folder could not be written. A third is ErrDependencies: generation failed while loading the
// module's dependencies, and they cannot be downloaded where Surla runs for a
// cause outside the author's changes since base (loadFailure). A staging
// folder that an interrupted publication left in the module is removed and
// never a failure (publish).
func (c *Checker) Prepare(
ctx context.Context, wc WorkingCopy, s Suite, branch, base string, stories []designeval.StorySummary,
) ([]designeval.StoryScenario, error) {
if err := errors.Join(wc.opened(), s.validate()); err != nil {
return nil, err
}
if strings.TrimSpace(branch) == "" {
return nil, errors.New("prepare: the build's tests branch is empty")
}
baseCommit, err := wc.requireCommit(ctx, base)
if err != nil {
return nil, err
}
rawBranch, facts, err := wc.readFacts(ctx)
if err != nil {
return nil, err
}
if facts.Detached || rawBranch != branch || facts.Operation != "none" {
return nil, &AuthorError{Finding: workingCopyFinding(branch, facts)}
}
rel, err := filepath.Rel(wc.dir, wc.project)
if err != nil {
return nil, err
}
modulePath := path.Join(filepath.ToSlash(rel), s.Module)
return c.generate(ctx, wc, baseCommit, filepath.Dir(wc.dir), filepath.Base(wc.dir), modulePath, s, stories)
}
// Check judges the working copy on branch against stories and base without
// trusting its author. It first verifies that the copy has branch checked out
// with no Git operation in progress, then snapshots the copy's files as a
// commit would hold them into a private folder, compares them with base,
// writes the suite module's Story table there again from the stories (so the
// author cannot change what is required) and generates its folder with it,
// downloads the module's dependencies, compiles the generated suite command with
// candidate.BuildModule, lists the scenarios it declares by running its --describe
// with candidate.Output, reading standard output alone, each in its own
// process group, and compares them with the required ones by the generator's
// own names.
//
// The verdict is accepted, or carries findings for the first step the
// author's files failed: a working copy not on branch or with a Git operation
// in progress (FindingWorkingCopy), each file changed outside the test module,
// the generation (including a module that does not declare exactly one
// evaluation suite), dependencies, the build (quoting the compiler, which
// names a missing hook method), the listing, or each declared scenario
// tagged holdout and each required scenario not declared. A build that does
// not finish within buildProgramLimit, or a listing within listTestsLimit,
// is a finding too. No finding holds an absolute path
// (WithoutAbsolutePaths). The working copy itself is not written. Errors are
// failures that say nothing about the author's work: ctx ended, the module's
// dependencies cannot be downloaded where Surla runs for a cause outside the
// author's changes (ErrDependencies), or git, the file system or the build's
// process cleanup failed. ctx bounds the whole check.
//
// The author's code runs as an ordinary process of the user, with the user's
// file and network access. In its own folders, only time bounds what it
// writes: what the build writes to its log and to the build cache in the
// Checker's folder within its limit, and what --describe writes to its
// standard error and its state folder within its own. Its standard output is
// kept in memory, at most 1 MiB. The check's own folder, holding that log,
// standard error and state, is removed when Check returns; the build cache
// stays until the owner removes the Checker's folder.
func (c *Checker) Check(
ctx context.Context, wc WorkingCopy, s Suite, branch, base string, stories []designeval.StorySummary,
) (_ Verdict, err error) {
if err := errors.Join(wc.opened(), s.validate()); err != nil {
return Verdict{}, err
}
if strings.TrimSpace(branch) == "" {
return Verdict{}, errors.New("check: the build's tests branch is empty")
}
baseCommit, err := wc.requireCommit(ctx, base)
if err != nil {
return Verdict{}, err
}
rawBranch, facts, err := wc.readFacts(ctx)
if err != nil {
return Verdict{}, err
}
if facts.Detached || rawBranch != branch || facts.Operation != "none" {
return Verdict{Findings: []Finding{workingCopyFinding(branch, facts)}, copy: wc}, nil
}
if err := os.MkdirAll(c.workDir, 0o700); err != nil {
return Verdict{}, fmt.Errorf("create the check folder: %w", err)
}
tmp, err := os.MkdirTemp(c.workDir, "check-*")
if err != nil {
return Verdict{}, fmt.Errorf("create the check folder: %w", err)
}
keep := false
defer func() {
if keep {
err = fmt.Errorf("%w; %s is kept because processes the check started may still use it", err, tmp)
return
}
err = errors.Join(err, candidate.RemoveStore(filepath.Join(tmp, "store")), os.RemoveAll(tmp))
}()
index, src := filepath.Join(tmp, "index"), filepath.Join(tmp, "src")
if err := stageCopy(ctx, wc, index, src); err != nil {
return Verdict{}, err
}
rel, err := filepath.Rel(wc.dir, wc.project)
if err != nil {
return Verdict{}, err
}
modulePath := path.Join(filepath.ToSlash(rel), s.Module)
changed, err := changedSince(ctx, wc, index, baseCommit)
if err != nil {
return Verdict{}, err
}
if findings := outsideModule(changed, modulePath, baseCommit); len(findings) > 0 {
return Verdict{Findings: findings, copy: wc}, nil
}
required, err := c.generate(ctx, wc, baseCommit, src, ".", modulePath, s, stories)
if author := (*AuthorError)(nil); errors.As(err, &author) {
return Verdict{Findings: []Finding{author.Finding}, copy: wc}, nil
}
if err != nil {
return Verdict{}, err
}
tree, err := writeTree(ctx, wc, index, src)
if err != nil {
return Verdict{}, err
}
declared, findings, err := c.listScenarios(ctx, wc, baseCommit, tmp, src, modulePath, s.Command, &keep)
if err != nil {
return Verdict{}, err
}
if len(findings) > 0 {
return Verdict{Findings: findings, copy: wc}, nil
}
ids := make([]string, 0, len(declared))
for _, sc := range declared {
ids = append(ids, sc.ID)
if slices.Contains(sc.Tags, holdoutTag) {
findings = append(findings, Finding{Kind: FindingHiddenTag, Message: fmt.Sprintf(
"Scenario %q is tagged %s. Tests in files the builder can read are not hidden: "+
"remove the tag from the test module's design, then report again.", sc.ID, holdoutTag)})
}
}
for _, r := range required {
if !slices.Contains(ids, r.Scenario) {
findings = append(findings, Finding{Kind: FindingMissingScenario, Message: fmt.Sprintf(
"The suite command does not declare scenario %q, which the Story requires for flow %q of story %q: "+
"its --describe must list every scenario of the generated suite, whose hook %s runs it.",
r.Scenario, r.Key, r.Story, r.Hook)})
}
}
if len(findings) > 0 {
return Verdict{Findings: findings, copy: wc}, nil
}
return Verdict{copy: wc, tree: tree, DeclaredScenarios: ids}, nil
}
// Error returns the finding's message.
func (e *AuthorError) Error() string {
return e.Finding.Message
}
// Unwrap returns the underlying failure.
func (e *AuthorError) Unwrap() error {
return e.cause
}
// Accepted reports whether the working copy passed the check.
func (v Verdict) Accepted() bool {
return len(v.Findings) == 0
}
// publish replaces the gen/ folder of the module open at root with files. An
// error means gen/ was not replaced and wraps ErrWriteGenerated.
//
// It first removes the staging folders that publications cut short by a
// server's death left in the module (genpublish.RemoveLeftoverStages). Surla
// writes the generated folder of a working copy and of its snapshots one
// publication at a time, in the project's working-copy turn, so none of its
// own publications is using one. A surla gen the author runs in the copy at
// that moment loses its staging folder and fails; the copy's generated folder
// is Surla's to write, and this publication writes it. An entry with a
// staging folder's name that is not one Surla made, such as a link or a file,
// is the author's: it is left as it is, like any other file the author keeps
// in the module.
//
// Nothing else fails the publication. A staging folder that could not be
// removed and the warnings of genpublish.Publish (the agent guide, the change
// listing, this publication's own staging folder) all describe work beside a
// gen/ that already holds the generated files, so they are recorded as events
// on ctx's span and the caller goes on.
func publish(ctx context.Context, root *os.Root, files map[string][]byte) error {
span := trace.SpanFromContext(ctx)
if err := genpublish.RemoveLeftoverStages(root); err != nil {
span.AddEvent("staging folders of earlier publications were left in the test module",
trace.WithAttributes(attribute.String("surla.error", err.Error())))
}
res, err := genpublish.Publish(ctx, root, files)
if err != nil {
return fmt.Errorf("%w: %w", ErrWriteGenerated, err)
}
if len(res.Warnings) > 0 {
span.AddEvent("the generated folder was written with warnings",
trace.WithAttributes(attribute.StringSlice("surla.warnings", res.Warnings)))
}
return nil
}
// runInGroup runs a check's build or listing in its own process group, the
// command's owning boundary: whatever ends it, the processes it started that
// are still in the group are killed and waited for. A group that does not
// empty is reported with the command's own error and marked by stuck. A
// process that leaves the group (setsid) is not stopped.
func runInGroup(stuck *bool) func(*exec.Cmd) error {
return func(cmd *exec.Cmd) error {
runErr, stopErr := procgroup.Run(cmd, syscall.SIGKILL, groupStopGrace, groupStopGrace)
if stopErr != nil {
*stuck = true
return errors.Join(fmt.Errorf("processes did not exit: %w", stopErr), runErr)
}
return runErr
}
}
// goEnv is the environment of the go commands a check runs to download
// modules: the user's module download settings, with cgo, workspaces,
// toolchain switching and injected go flags turned off.
func goEnv() []string {
env := []string{
"PATH=" + os.Getenv("PATH"), "HOME=" + os.Getenv("HOME"),
"CGO_ENABLED=0", "GOWORK=off", "GOTOOLCHAIN=local", "GOFLAGS=",
}
for _, name := range []string{"GOPATH", "GOMODCACHE", "GOPROXY", "GOSUMDB", "GOPRIVATE", "GONOPROXY", "GONOSUMDB"} {
if v, ok := os.LookupEnv(name); ok {
env = append(env, name+"="+v)
}
}
return env
}
// downloadDependencies runs go mod download with the go command goBin in the
// module at dir with goEnv, so with the user's module
// download settings. Its output is discarded and never reaches a finding or a
// reason: it may print those settings, a proxy's address and credentials
// among them.
func downloadDependencies(ctx context.Context, goBin, dir string) error {
cmd := exec.CommandContext(ctx, goBin, "mod", "download")
cmd.Dir, cmd.Env = dir, goEnv()
return cmd.Run()
}
// forAuthor words message for the test writer: the check's private snapshot
// folder src is removed from the paths it quotes, so they read relative to
// the working copy, and it is cut to at most maxFindingBytes on a character
// boundary, saying how much was left out.
func forAuthor(message, src string) string {
s := WithoutAbsolutePaths(strings.TrimSpace(strings.ReplaceAll(message, src+string(filepath.Separator), "")))
if len(s) <= maxFindingBytes {
return s
}
const more = "\n… (%d more bytes)"
k := maxFindingBytes - len(more) - 8
for k > 0 && !utf8.RuneStart(s[k]) {
k--
}
return s[:k] + fmt.Sprintf(more, len(s)-k)
}
// logFinding returns one finding of kind whose message is head followed by
// the end of the log file at path: as much as fits in maxFindingBytes after
// head, read with candidate.ExcerptFile so a log of any size is never loaded
// whole, starting on a character boundary and saying when earlier output was
// left out. src is removed from quoted paths, as forAuthor does.
func logFinding(kind FindingKind, head, path, src string) ([]Finding, error) {
const leftOut = "… (earlier output left out)\n"
out, cut, err := candidate.ExcerptFile(path, int64(maxFindingBytes-len(head)-len(leftOut)))
if err != nil {
return nil, err
}
if cut {
for len(out) > 0 && !utf8.RuneStart(out[0]) {
out = out[1:]
}
out = leftOut + out
}
return []Finding{{Kind: kind, Message: forAuthor(head+out, src)}}, nil
}
// withinLimit runs step under ctx with its own deadline limit from now() and
// reports whether the limit was reached: the deadline had passed when step
// returned, by now(), whatever step returned, while ctx itself had not ended.
// A step that returns exactly at the deadline has reached it. Callers pass
// time.Now; a test passes a clock that places the return just below, at or
// just above the deadline. The caller's own deadline ending is never the limit.
func withinLimit(
ctx context.Context,
now func() time.Time,
limit time.Duration,
step func(context.Context) error,
) (bool, error) {
deadline := now().Add(limit)
stepCtx, cancel := context.WithDeadline(ctx, deadline)
defer cancel()
err := step(stepCtx)
return ctx.Err() == nil && !now().Before(deadline), err
}
// limitText words a limit for a finding: two or more whole minutes as
// minutes, other whole seconds as seconds, anything else as Go prints a
// duration. For example, 10 minutes, 60 seconds, 500ms.
func limitText(d time.Duration) string {
switch {
case d >= 2*time.Minute && d%time.Minute == 0:
return fmt.Sprintf("%d minutes", d/time.Minute)
case d%time.Second == 0:
return fmt.Sprintf("%d seconds", d/time.Second)
default:
return d.String()
}
}
// workingCopyFinding words a FindingWorkingCopy for a test writer's working
// copy that is not on branch or has a Git operation in progress.
func workingCopyFinding(branch string, facts CopyFacts) Finding {
where := fmt.Sprintf("is on branch %s", ShownBranch(facts.Branch))
if facts.Detached {
where = "is on a detached HEAD"
}
var opClause string
if facts.Operation != "none" {
opClause = fmt.Sprintf(" with a git %s in progress", strings.ReplaceAll(facts.Operation, "_", "-"))
}
var uncommittedClause string
if facts.UncommittedCount > 0 {
listed := strings.Join(facts.Uncommitted, ", ")
if extra := facts.UncommittedCount - len(facts.Uncommitted); extra > 0 {
listed = fmt.Sprintf("%s, and %d more", listed, extra)
}
uncommittedClause = fmt.Sprintf(" Uncommitted files (%d): %s.", facts.UncommittedCount, listed)
}
goal := fmt.Sprintf("Put your test files on %s, uncommitted (Surla commits what it accepts), and report again.", branch)
if facts.Operation != "none" {
goal = fmt.Sprintf(
"A %s is in progress in your working copy: abort it, then put your test files on %s, uncommitted (Surla commits what it accepts), and report again.",
strings.ReplaceAll(facts.Operation, "_", "-"),
branch,
)
}
if facts.IndexLock != "" {
goal += " " + IndexLockStep("report_tests_finished", facts.IndexLock)
}
msg := fmt.Sprintf("Your working copy (%s) %s%s.%s %s",
TestsRole.Folder, where, opClause, uncommittedClause, goal)
return Finding{Kind: FindingWorkingCopy, Message: WithoutAbsolutePaths(msg)}
}
// outsideModule names changed paths outside the test module, bounded by
// maxOutsideFindings plus one remainder count. Recovery states the required
// working-copy contents relative to base, without prescribing Git commands or
// deleting the copy/branch. A file absent from base must be absent from the copy.
func outsideModule(changed []changedPath, modulePath, base string) []Finding {
var outside []changedPath
for _, c := range changed {
if c.path != modulePath && !strings.HasPrefix(c.path, modulePath+"/") {
outside = append(outside, c)
}
}
var findings []Finding
for i, c := range outside {
if i == maxOutsideFindings {
findings = append(findings, Finding{Kind: FindingOutsideModule, Message: fmt.Sprintf(
"%d more files outside the test module %s differ from the working copy's base, commit %s.",
len(outside)-i, modulePath, base)})
break
}
goal := fmt.Sprintf("make it match its version in commit %s, the build's base", base)
if c.added {
goal = fmt.Sprintf("commit %s, the build's base, has no such file: make your working copy match that absence", base)
}
findings = append(findings, Finding{Kind: FindingOutsideModule, Message: WithoutAbsolutePaths(fmt.Sprintf(
"%s is changed outside the test module %s. Only files inside the test module may change: %s; "+
"keep your own tests inside %s uncommitted (Surla commits what it accepts), then report again.",
c.path, modulePath, goal, modulePath))})
}
return findings
}
// decodeDeclaration decodes exactly one suite.Declaration from out: unknown
// fields and anything after it are errors.
func decodeDeclaration(out []byte) (suite.Declaration, error) {
dec := json.NewDecoder(bytes.NewReader(out))
dec.DisallowUnknownFields()
var d suite.Declaration
if err := dec.Decode(&d); err != nil {
return suite.Declaration{}, err
}
if err := dec.Decode(&struct{}{}); !errors.Is(err, io.EOF) {
return suite.Declaration{}, errors.New("unexpected data after the declaration")
}
return d, nil
}
// resolveToolchain resolves the local Go toolchain's root and module cache,
// from dir, outside any module, so the author's go.mod cannot select another
// toolchain.
func resolveToolchain(ctx context.Context, dir string) (toolchain, error) {
cmd := exec.CommandContext(ctx, "go", "env", "GOROOT", "GOMODCACHE")
cmd.Dir, cmd.Env = dir, goEnv()
out, err := cmd.Output()
if err != nil {
return toolchain{}, fmt.Errorf("resolve go env GOROOT GOMODCACHE: %w", err)
}
lines := strings.Split(strings.TrimSpace(string(out)), "\n")
if len(lines) != 2 || lines[0] == "" || lines[1] == "" {
return toolchain{}, fmt.Errorf("unexpected go env output %q", out)
}
goroot, err := abspath.Eval(lines[0])
if err != nil {
return toolchain{}, fmt.Errorf("resolve the Go toolchain: %w", err)
}
return toolchain{goroot: goroot, modCache: lines[1]}, nil
}
// openModule opens the folder at the slash path rel below the copy's root,
// which is the slash path copyDir below the real folder top, one folder at a
// time, each through the folder above it. Each step is checked with Lstat and
// then opened, and the opened folder must be the one checked, so a symbolic
// link anywhere on the way, even one swapped in meanwhile, is refused with
// ErrLinkedFolder naming it, and nothing outside top is opened. A message
// names a folder on rel relative to the copy's root, which is how the copy's
// author sees it, and a folder on copyDir, the copy's own folder, relative to
// top. The caller closes the result.
func openModule(top, copyDir, rel string) (_ *os.Root, err error) {
cur, err := os.OpenRoot(top)
if err != nil {
return nil, err
}
for _, part := range []string{copyDir, rel} {
walked := ""
for _, name := range strings.Split(path.Clean(part), "/") {
if name == "." {
continue
}
walked = path.Join(walked, name)
next, err := openFolder(cur, name, walked)
if err := errors.Join(err, cur.Close()); err != nil {
if next != nil {
err = errors.Join(err, next.Close())
}
return nil, err
}
cur = next
}
}
return cur, nil
}
// openFolder opens the folder name inside dir, which walked names for
// messages, refusing a symbolic link or anything but a folder.
func openFolder(dir *os.Root, name, walked string) (*os.Root, error) {
info, err := dir.Lstat(name)
if err != nil {
return nil, err
}
if info.Mode()&fs.ModeSymlink != 0 {
return nil, fmt.Errorf("%w: %s is a symbolic link", ErrLinkedFolder, walked)
}
if !info.IsDir() {
return nil, fmt.Errorf("%s is not a folder", walked)
}
next, err := dir.OpenRoot(name)
if err != nil {
return nil, err
}
held, err := next.Stat(".")
if err != nil || !os.SameFile(info, held) {
return nil, errors.Join(fmt.Errorf("%w: %s changed while it was opened", ErrLinkedFolder, walked), err, next.Close())
}
return next, nil
}
// generate renders the suite module's Story table (designeval.StoryTestsPath)
// from stories, generates the module with that table in place of whatever
// the author left at its path, so the author's version never counts, then
// writes the table (writeStoryTests) and the generated folder, and returns the
// required scenarios. A generation that fails writes neither, so a working
// copy whose preparation failed stays as it was and a later build can still
// bring it forward; only a link at the table's path is replaced first,
// because a module snapshot holds only regular files. The module is at the
// slash path modulePath below the copy's root, and the copy's root is the
// slash path copyDir below the real folder top. Prepare runs it on the
// working copy (top is the project folder and copyDir the copy's folder in
// it) and Check on its snapshot (top is the snapshot and copyDir is "."). A
// cause the module's author can fix is an *AuthorError whose finding
// gives paths relative to the copy's root, in Prepare as in Check: the module
// cannot be opened, including through a symbolic link, or read; its design
// package cannot be named; a folder is at the table's path; its generation
// failed while working on the module's own files, which includes a design that
// does not declare exactly one evaluation suite, a rule of the table; or no
// program was generated at s.Command. A failure of the generator that is not
// the author's, or stories whose flows collide, wraps ErrGeneration, and a
// failure to write the table or the generated folder ErrWriteGenerated. A
// generation that fails while loading the module's dependencies is the
// author's unless they cannot be downloaded for a cause outside the changes
// the author made to wc, which is an error wrapping ErrDependencies
// (loadFailure).
func (c *Checker) generate(
ctx context.Context, wc WorkingCopy, base, top, copyDir, modulePath string, s Suite,
stories []designeval.StorySummary,
) (_ []designeval.StoryScenario, err error) {
copyRoot := filepath.Join(top, filepath.FromSlash(copyDir))
author := func(cause error, message string) error {
return &AuthorError{Finding: Finding{Kind: FindingGeneration, Message: forAuthor(message, copyRoot)}, cause: cause}
}
root, err := openModule(top, copyDir, modulePath)
if err != nil {
return nil, author(err, fmt.Sprintf("The suite module %s could not be opened: %v", modulePath, err))
}
defer func() { err = errors.Join(err, root.Close()) }()
table, required, err := designeval.StoryTests(root.FS(), stories)
switch {
case errors.Is(err, designeval.ErrDesignPackage):
return nil, author(err, fmt.Sprintf("Surla could not write the tests the Story requires into %s/%s: %v. "+
"Keep design/ holding the Go files of one package, then report again.", modulePath, designeval.StoryTestsPath, err))
case err != nil:
return nil, fmt.Errorf("%w: %w", ErrGeneration, err)
}
writeTable := func() error {
err := writeStoryTests(root, table)
if errors.Is(err, errTableIsFolder) {
return author(err, fmt.Sprintf("%s/%s is a folder. Surla writes the tests the Story requires in that file: "+
"delete the folder, then report again.", modulePath, designeval.StoryTestsPath))
}
if err != nil {
return fmt.Errorf("%w: %w", ErrWriteGenerated, err)
}
return nil
}
// A module snapshot holds only regular files, so a link the author left
// at the table's path is replaced before the module is read, and a
// folder there is the author's to delete.
if info, err := root.Lstat(designeval.StoryTestsPath); err == nil && (info.IsDir() || info.Mode()&fs.ModeSymlink != 0) {
if err := writeTable(); err != nil {
return nil, err
}
}
files, err := designeval.ReadModuleSnapshotFromRoot(filepath.Join(copyRoot, filepath.FromSlash(modulePath)), root)
if _, ok := files["go.mod"]; err == nil && !ok {
err = errors.New("it has no go.mod")
}
if err != nil {
return nil, author(err, fmt.Sprintf("The suite module could not be read: %v", err))
}
files[designeval.StoryTestsPath] = string(table)
generated, err := c.evaluator.Generate(ctx, files)
if err != nil {
if ctx.Err() != nil || !designeval.FailedOnModule(err) {
return nil, fmt.Errorf("%w: %w", ErrGeneration, err)
}
cause := err
if designeval.FailedLoadingDependencies(err) {
// The load may have failed because the dependencies cannot be
// downloaded where Surla runs, which is not the author's to fix
// unless the author's own lists are what cannot be downloaded.
depErr := c.loadFailure(ctx, wc, base, modulePath, files)
switch {
case errors.Is(depErr, ErrDependencies):
return nil, depErr
case ctx.Err() != nil:
return nil, fmt.Errorf("%w: %w", ErrGeneration, err)
case depErr != nil:
// Deciding failed: the finding stands, as when nothing decides.
cause = errors.Join(err, depErr)
}
}
return nil, author(cause, designeval.GenerationDiagnostic(err))
}
program := path.Join(strings.TrimPrefix(s.Command, "./"), "main.go")
if _, ok := generated[program]; !ok {
return nil, author(nil, fmt.Sprintf("Surla generated no test program at %s, because the test module is not "+
"a Surla test module: its go.mod must declare the module path <service module>/%s, as surla init writes it, "+
"and it must keep its %s package, which provides Name, Hooks, New, Live and LiveThroughGateway.",
s.Command, designeval.TestModuleDir, designeval.AcceptancePackage))
}
if err := writeTable(); err != nil {
return nil, err
}
if err := publish(ctx, root, generated); err != nil {
return nil, err
}
return required, nil
}
// writeStoryTests replaces the Story table of the module open at root with
// table. It writes a new file beside it and renames that file over the
// table's path, so a symbolic link there is replaced, never followed, and the
// path never lacks a table. A folder at the path is errTableIsFolder, the
// author's to delete; any other failure leaves the old table, removes the new
// file when it can, and is the caller's to report.
func writeStoryTests(root *os.Root, table []byte) (err error) {
tmp := path.Join(path.Dir(designeval.StoryTestsPath), ".story_tests-"+rand.Text()+".tmp")
f, err := root.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o644)
if err != nil {
return fmt.Errorf("write %s: %w", designeval.StoryTestsPath, err)
}
defer func() {
if err != nil {
err = errors.Join(err, root.Remove(tmp))
}
}()
if _, err := f.Write(table); err != nil {
return errors.Join(fmt.Errorf("write %s: %w", designeval.StoryTestsPath, err), f.Close())
}
if err := f.Close(); err != nil {
return fmt.Errorf("write %s: %w", designeval.StoryTestsPath, err)
}
if err := root.Rename(tmp, designeval.StoryTestsPath); err != nil {
if info, statErr := root.Lstat(designeval.StoryTestsPath); statErr == nil && info.IsDir() {
return fmt.Errorf("%w: %w", errTableIsFolder, err)
}
return fmt.Errorf("write %s: %w", designeval.StoryTestsPath, err)
}
return nil
}
// listScenarios builds the suite command of the module at the slash path
// modulePath of the snapshot src with workspaces off within c.buildLimit,
// runs its --describe within c.listLimit, each in its own process group, and returns the
// scenarios it declares, or findings when the module's dependencies, its build
// or its listing fail or reach their limit. Dependencies that cannot be
// downloaded for a cause outside the author's changes to wc are an error
// wrapping ErrDependencies instead (dependencyFailure). tmp is the check's
// private folder; keep is set when processes the build or the listing started
// may still use it.
func (c *Checker) listScenarios(