@@ -31,6 +31,74 @@ type TestCase struct {
3131 // nil inherits the eval-level value; 0 disables the check for this task.
3232 FirstEventTimeoutSec * int `yaml:"first_event_timeout_seconds,omitempty" json:"first_event_timeout_sec,omitempty"`
3333 Validators []ValidatorInline `yaml:"graders,omitempty" json:"validators,omitempty"`
34+ // Checkpoints attach graders to intermediate turn boundaries in a
35+ // multi-turn run. Each checkpoint specifies `after_turn: N` (1-based,
36+ // where turn 1 is the initial prompt) and a list of graders that run
37+ // against the cumulative conversation state at the end of that turn.
38+ // Checkpoints are additive — task-level `graders:` still run against
39+ // the final state after all turns complete.
40+ Checkpoints []Checkpoint `yaml:"checkpoints,omitempty" json:"checkpoints,omitempty"`
41+ }
42+
43+ // CheckpointOnFailure controls multi-turn behavior when a checkpoint fails.
44+ type CheckpointOnFailure string
45+
46+ const (
47+ // CheckpointContinue (default) records the failure and continues with the
48+ // remaining turns and the final grader pass.
49+ CheckpointContinue CheckpointOnFailure = "continue"
50+ // CheckpointStop records the failure, sets the run's error message, and
51+ // short-circuits the multi-turn loop so no further turns execute.
52+ CheckpointStop CheckpointOnFailure = "stop"
53+ )
54+
55+ // Checkpoint runs a slice of graders against the conversation state at the
56+ // end of a specific turn. See TestCase.Checkpoints for details.
57+ type Checkpoint struct {
58+ // AfterTurn is the 1-based turn index at whose boundary the graders run.
59+ // Turn 1 is the initial prompt; turns 2..N are follow-ups or responder
60+ // replies. Must be >= 1.
61+ AfterTurn int `yaml:"after_turn" json:"after_turn"`
62+ // Graders is the slice of grader configurations to run at this turn
63+ // boundary. Required. Reuses the same ValidatorInline shape as task-level
64+ // `graders:` so any existing grader type is supported.
65+ Graders []ValidatorInline `yaml:"graders" json:"graders"`
66+ // OnFailure controls multi-turn behavior when any grader in this
67+ // checkpoint fails. "continue" (default) records the failure and keeps
68+ // going. "stop" short-circuits the multi-turn loop after the failure.
69+ OnFailure CheckpointOnFailure `yaml:"on_failure,omitempty" json:"on_failure,omitempty"`
70+ }
71+
72+ // EffectiveOnFailure returns the OnFailure value to use, defaulting to
73+ // CheckpointContinue when unset.
74+ func (c * Checkpoint ) EffectiveOnFailure () CheckpointOnFailure {
75+ if c .OnFailure == "" {
76+ return CheckpointContinue
77+ }
78+ return c .OnFailure
79+ }
80+
81+ // Validate checks the checkpoint has the required fields. Reuses
82+ // ValidatorInline.Validate for inner grader validation.
83+ func (c * Checkpoint ) Validate () error {
84+ if c .AfterTurn < 1 {
85+ return fmt .Errorf ("after_turn must be at least 1, got %d" , c .AfterTurn )
86+ }
87+ if len (c .Graders ) == 0 {
88+ return fmt .Errorf ("after_turn %d: graders is required and must contain at least one grader" , c .AfterTurn )
89+ }
90+ switch c .OnFailure {
91+ case "" , CheckpointContinue , CheckpointStop :
92+ default :
93+ return fmt .Errorf ("after_turn %d: on_failure must be %q or %q, got %q" ,
94+ c .AfterTurn , CheckpointContinue , CheckpointStop , c .OnFailure )
95+ }
96+ for i := range c .Graders {
97+ if err := c .Graders [i ].Validate (); err != nil {
98+ return fmt .Errorf ("after_turn %d: grader[%d]: %w" , c .AfterTurn , i , err )
99+ }
100+ }
101+ return nil
34102}
35103
36104// TaskStimulus defines the input for a task.
@@ -375,6 +443,43 @@ func (tc *TestCase) Validate() error {
375443 }
376444 }
377445
446+ // Validate checkpoints. When follow-up prompts (not a responder) drive
447+ // the multi-turn run, the turn count is known statically so we can also
448+ // reject checkpoints that exceed it.
449+ if len (tc .Checkpoints ) > 0 {
450+ name := tc .TestID
451+ if name == "" {
452+ name = tc .DisplayName
453+ }
454+ prefix := "test case"
455+ if name != "" {
456+ prefix = fmt .Sprintf ("test case %q" , name )
457+ }
458+
459+ // Static turn count: 1 initial + len(FollowUps) follow-ups when no
460+ // responder is configured. Responder is dynamic — skip the upper bound.
461+ maxTurns := 0
462+ if tc .Stimulus .Responder == nil {
463+ maxTurns = 1 + len (tc .Stimulus .FollowUps )
464+ }
465+
466+ seen := make (map [int ]bool , len (tc .Checkpoints ))
467+ for i := range tc .Checkpoints {
468+ if err := tc .Checkpoints [i ].Validate (); err != nil {
469+ return fmt .Errorf ("%s: checkpoints[%d]: %w" , prefix , i , err )
470+ }
471+ at := tc .Checkpoints [i ].AfterTurn
472+ if seen [at ] {
473+ return fmt .Errorf ("%s: checkpoints[%d]: duplicate after_turn %d (each turn may have at most one checkpoint)" , prefix , i , at )
474+ }
475+ seen [at ] = true
476+ if maxTurns > 0 && at > maxTurns {
477+ return fmt .Errorf ("%s: checkpoints[%d].after_turn %d exceeds total turns (1 initial + %d follow_up_prompts = %d turns)" ,
478+ prefix , i , at , len (tc .Stimulus .FollowUps ), maxTurns )
479+ }
480+ }
481+ }
482+
378483 return nil
379484}
380485
0 commit comments