feat: per-turn checkpoint graders (closes #358) (#386)

* feat: per-turn checkpoint graders (closes #358)

Add an additive `checkpoints:` field to task YAML so multi-turn evals
can grade conversation state at specific turn boundaries instead of
only the final output.

- New `Checkpoint` model with after_turn, graders, on_failure (continue/stop)
- New `CheckpointOutcome` recorded per task on results.json
- Per-turn hook in runner (initial + follow_ups + responder loop)
- on_failure: stop aborts remaining turns and flips status to error
- Bumped schemaVersion to 1.1 (additive, MINOR bump per #382 policy)
- Reuses existing grader plumbing (graders.RunAll + buildGraderContext)
- Honors --skip-graders by short-circuiting checkpoint evaluation
- Full unit + integration tests; docs (guide + schema + changelog)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: address Copilot review feedback on #386

- Add Type field to synthesized _checkpoint_error GraderResults
- Fix docs to reference 'graders:' (the actual YAML key) instead of 'validators:'
- Update schema-changes.md Policy section to reflect current 1.1 default emission while preserving 1.0 reader fallback for back-compat

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* chore: gitignore .impeccable/ cache directory

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: spboyer

.gitignore

@@ -120,3 +120,4 @@ docs/research/
 .squad/sessions/
 .squad/.cache/
 .squad-workstream
+.impeccable/

cmd/waza/cmd_migrate_test.go

@@ -18,7 +18,7 @@ func TestMigrateCommandNoOpForCurrentMajor(t *testing.T) {
 	err := runMigrate(&out, path)
 
 	require.NoError(t, err)
-	require.Contains(t, out.String(), "already compatible with schemaVersion 1.0")
+	require.Contains(t, out.String(), "compatible with waza schemaVersion 1.1")
 }
 
 func TestMigrateCommandRejectsIncompatibleMajor(t *testing.T) {
@@ -43,7 +43,7 @@ func TestMigrateCommandDefaultsMissingSchemaVersion(t *testing.T) {
 	err := runMigrate(&out, path)
 
 	require.NoError(t, err)
-	require.Contains(t, out.String(), "already compatible with schemaVersion 1.0")
+	require.Contains(t, out.String(), "already compatible with schemaVersion 1.1")
 }
 
 func TestMigrateCommandRejectsUnknownJSONShape(t *testing.T) {
@@ -69,7 +69,7 @@ func TestMigrateCommandDetectsResultsJSONByShape(t *testing.T) {
 	err := runMigrate(&out, path)
 
 	require.NoError(t, err)
-	require.Contains(t, out.String(), "results.json is already compatible")
+	require.Contains(t, out.String(), "results.json uses schemaVersion 1.0")
 }
 
 func TestMigrateCommandMissingFile(t *testing.T) {

internal/models/outcome.go

@@ -192,6 +192,28 @@ type RunResult struct {
 	WorkspaceDir     string                   `json:"workspace_dir,omitempty"`
 	FailureArtifacts *FailureArtifacts        `json:"failure_artifacts,omitempty"`
 	Responder        *ResponderInfo           `json:"responder,omitempty"`
+	// Checkpoints captures per-turn checkpoint grader results, one entry per
+	// configured TestCase.Checkpoint that actually ran (i.e., turn index was
+	// reached). Empty / omitted when the task defines no checkpoints.
+	Checkpoints []CheckpointOutcome `json:"checkpoints,omitempty"`
+}
+
+// CheckpointOutcome captures the results of a single TestCase.Checkpoint that
+// ran during a multi-turn run. It mirrors the shape of the final-grader
+// validations map so dashboards can present per-turn pass/fail uniformly.
+type CheckpointOutcome struct {
+	// AfterTurn echoes Checkpoint.AfterTurn so consumers can sort/group.
+	AfterTurn int `json:"after_turn"`
+	// Status is StatusPassed when every grader in this checkpoint passed,
+	// StatusFailed when at least one grader failed.
+	Status Status `json:"status"`
+	// Validations maps grader identifier to result, identical to
+	// RunResult.Validations.
+	Validations map[string]GraderResults `json:"validations"`
+	// Stopped is true when this checkpoint had `on_failure: stop` and at
+	// least one grader failed, terminating the multi-turn loop after this
+	// turn.
+	Stopped bool `json:"stopped,omitempty"`
 }
 
 // FailureArtifacts captures diagnostic information when a run fails

internal/models/outcome_schema_test.go

@@ -84,7 +84,7 @@ func TestEvaluationOutcomeMarshalDefaultsSchemaVersion(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Marshal() error = %v", err)
 	}
-	if !strings.Contains(string(data), `"schemaVersion":"1.0"`) {
+	if !strings.Contains(string(data), `"schemaVersion":"1.1"`) {
 		t.Fatalf("marshaled outcome missing default schemaVersion: %s", data)
 	}
 }

internal/models/schema_version.go

@@ -15,7 +15,10 @@ import (
 
 const (
 	// CurrentSchemaVersion is the current MAJOR.MINOR schema version for public artifacts.
-	CurrentSchemaVersion = "1.0"
+	//
+	// 1.1 adds per-turn checkpoints (TestCase.Checkpoints / RunResult.Checkpoints) — purely
+	// additive over 1.0, so 1.0 artifacts continue to load without migration.
+	CurrentSchemaVersion = "1.1"
 )
 
 func defaultSchemaVersion(version string) string {

internal/models/testcase.go

@@ -31,6 +31,74 @@ type TestCase struct {
 	// nil inherits the eval-level value; 0 disables the check for this task.
 	FirstEventTimeoutSec *int              `yaml:"first_event_timeout_seconds,omitempty" json:"first_event_timeout_sec,omitempty"`
 	Validators           []ValidatorInline `yaml:"graders,omitempty" json:"validators,omitempty"`
+	// Checkpoints attach graders to intermediate turn boundaries in a
+	// multi-turn run. Each checkpoint specifies `after_turn: N` (1-based,
+	// where turn 1 is the initial prompt) and a list of graders that run
+	// against the cumulative conversation state at the end of that turn.
+	// Checkpoints are additive — task-level `graders:` still run against
+	// the final state after all turns complete.
+	Checkpoints []Checkpoint `yaml:"checkpoints,omitempty" json:"checkpoints,omitempty"`
+}
+
+// CheckpointOnFailure controls multi-turn behavior when a checkpoint fails.
+type CheckpointOnFailure string
+
+const (
+	// CheckpointContinue (default) records the failure and continues with the
+	// remaining turns and the final grader pass.
+	CheckpointContinue CheckpointOnFailure = "continue"
+	// CheckpointStop records the failure, sets the run's error message, and
+	// short-circuits the multi-turn loop so no further turns execute.
+	CheckpointStop CheckpointOnFailure = "stop"
+)
+
+// Checkpoint runs a slice of graders against the conversation state at the
+// end of a specific turn. See TestCase.Checkpoints for details.
+type Checkpoint struct {
+	// AfterTurn is the 1-based turn index at whose boundary the graders run.
+	// Turn 1 is the initial prompt; turns 2..N are follow-ups or responder
+	// replies. Must be >= 1.
+	AfterTurn int `yaml:"after_turn" json:"after_turn"`
+	// Graders is the slice of grader configurations to run at this turn
+	// boundary. Required. Reuses the same ValidatorInline shape as task-level
+	// `graders:` so any existing grader type is supported.
+	Graders []ValidatorInline `yaml:"graders" json:"graders"`
+	// OnFailure controls multi-turn behavior when any grader in this
+	// checkpoint fails. "continue" (default) records the failure and keeps
+	// going. "stop" short-circuits the multi-turn loop after the failure.
+	OnFailure CheckpointOnFailure `yaml:"on_failure,omitempty" json:"on_failure,omitempty"`
+}
+
+// EffectiveOnFailure returns the OnFailure value to use, defaulting to
+// CheckpointContinue when unset.
+func (c *Checkpoint) EffectiveOnFailure() CheckpointOnFailure {
+	if c.OnFailure == "" {
+		return CheckpointContinue
+	}
+	return c.OnFailure
+}
+
+// Validate checks the checkpoint has the required fields. Reuses
+// ValidatorInline.Validate for inner grader validation.
+func (c *Checkpoint) Validate() error {
+	if c.AfterTurn < 1 {
+		return fmt.Errorf("after_turn must be at least 1, got %d", c.AfterTurn)
+	}
+	if len(c.Graders) == 0 {
+		return fmt.Errorf("after_turn %d: graders is required and must contain at least one grader", c.AfterTurn)
+	}
+	switch c.OnFailure {
+	case "", CheckpointContinue, CheckpointStop:
+	default:
+		return fmt.Errorf("after_turn %d: on_failure must be %q or %q, got %q",
+			c.AfterTurn, CheckpointContinue, CheckpointStop, c.OnFailure)
+	}
+	for i := range c.Graders {
+		if err := c.Graders[i].Validate(); err != nil {
+			return fmt.Errorf("after_turn %d: grader[%d]: %w", c.AfterTurn, i, err)
+		}
+	}
+	return nil
 }
 
 // TaskStimulus defines the input for a task.
@@ -375,6 +443,43 @@ func (tc *TestCase) Validate() error {
 		}
 	}
 
+	// Validate checkpoints. When follow-up prompts (not a responder) drive
+	// the multi-turn run, the turn count is known statically so we can also
+	// reject checkpoints that exceed it.
+	if len(tc.Checkpoints) > 0 {
+		name := tc.TestID
+		if name == "" {
+			name = tc.DisplayName
+		}
+		prefix := "test case"
+		if name != "" {
+			prefix = fmt.Sprintf("test case %q", name)
+		}
+
+		// Static turn count: 1 initial + len(FollowUps) follow-ups when no
+		// responder is configured. Responder is dynamic — skip the upper bound.
+		maxTurns := 0
+		if tc.Stimulus.Responder == nil {
+			maxTurns = 1 + len(tc.Stimulus.FollowUps)
+		}
+
+		seen := make(map[int]bool, len(tc.Checkpoints))
+		for i := range tc.Checkpoints {
+			if err := tc.Checkpoints[i].Validate(); err != nil {
+				return fmt.Errorf("%s: checkpoints[%d]: %w", prefix, i, err)
+			}
+			at := tc.Checkpoints[i].AfterTurn
+			if seen[at] {
+				return fmt.Errorf("%s: checkpoints[%d]: duplicate after_turn %d (each turn may have at most one checkpoint)", prefix, i, at)
+			}
+			seen[at] = true
+			if maxTurns > 0 && at > maxTurns {
+				return fmt.Errorf("%s: checkpoints[%d].after_turn %d exceeds total turns (1 initial + %d follow_up_prompts = %d turns)",
+					prefix, i, at, len(tc.Stimulus.FollowUps), maxTurns)
+			}
+		}
+	}
+
 	return nil
 }
 

internal/models/testcase_test.go

@@ -340,3 +340,114 @@ func TestResponderValidationAcceptsValidConfig(t *testing.T) {
 	}
 	require.NoError(t, tc.Validate())
 }
+
+// -- Checkpoint validation tests --
+
+func TestLoadTestCase_CheckpointsParsed(t *testing.T) {
+	yamlData := `id: tc-cp
+name: Checkpoint loader
+inputs:
+  prompt: do work
+  follow_up_prompts:
+    - keep going
+checkpoints:
+  - after_turn: 1
+    graders:
+      - name: mid-check
+        type: text
+        config:
+          contains:
+            - hello
+  - after_turn: 2
+    on_failure: stop
+    graders:
+      - name: end-check
+        type: text
+        config:
+          contains:
+            - bye
+`
+	dir := t.TempDir()
+	p := filepath.Join(dir, "tc.yaml")
+	require.NoError(t, os.WriteFile(p, []byte(yamlData), 0o644))
+
+	tc, err := LoadTestCase(p)
+	require.NoError(t, err)
+	require.Len(t, tc.Checkpoints, 2)
+	require.Equal(t, 1, tc.Checkpoints[0].AfterTurn)
+	require.Equal(t, CheckpointContinue, tc.Checkpoints[0].EffectiveOnFailure())
+	require.Equal(t, CheckpointStop, tc.Checkpoints[1].EffectiveOnFailure())
+	require.Len(t, tc.Checkpoints[0].Graders, 1)
+	require.Equal(t, "mid-check", tc.Checkpoints[0].Graders[0].Identifier)
+}
+
+func TestCheckpointValidation(t *testing.T) {
+	makeTC := func(cps []Checkpoint) *TestCase {
+		return &TestCase{
+			TestID:      "t",
+			DisplayName: "t",
+			Stimulus: TaskStimulus{
+				Message:   "go",
+				FollowUps: []string{"again", "more"},
+			},
+			Checkpoints: cps,
+		}
+	}
+	validGrader := ValidatorInline{
+		Identifier: "g",
+		Kind:       GraderKindText,
+		Parameters: TextGraderParameters{Contains: []string{"hi"}},
+	}
+
+	t.Run("after_turn < 1 rejected", func(t *testing.T) {
+		tc := makeTC([]Checkpoint{{AfterTurn: 0, Graders: []ValidatorInline{validGrader}}})
+		require.ErrorContains(t, tc.Validate(), "after_turn")
+	})
+
+	t.Run("missing graders rejected", func(t *testing.T) {
+		tc := makeTC([]Checkpoint{{AfterTurn: 1}})
+		require.ErrorContains(t, tc.Validate(), "graders")
+	})
+
+	t.Run("duplicate after_turn rejected", func(t *testing.T) {
+		tc := makeTC([]Checkpoint{
+			{AfterTurn: 1, Graders: []ValidatorInline{validGrader}},
+			{AfterTurn: 1, Graders: []ValidatorInline{validGrader}},
+		})
+		require.ErrorContains(t, tc.Validate(), "duplicate")
+	})
+
+	t.Run("after_turn exceeds turns rejected", func(t *testing.T) {
+		// 1 initial + 2 follow-ups = 3 turns; 4 is out of range.
+		tc := makeTC([]Checkpoint{{AfterTurn: 4, Graders: []ValidatorInline{validGrader}}})
+		require.ErrorContains(t, tc.Validate(), "exceeds")
+	})
+
+	t.Run("invalid on_failure rejected", func(t *testing.T) {
+		tc := makeTC([]Checkpoint{{AfterTurn: 1, OnFailure: "maybe", Graders: []ValidatorInline{validGrader}}})
+		require.ErrorContains(t, tc.Validate(), "on_failure")
+	})
+
+	t.Run("valid case accepted", func(t *testing.T) {
+		tc := makeTC([]Checkpoint{
+			{AfterTurn: 1, Graders: []ValidatorInline{validGrader}},
+			{AfterTurn: 3, OnFailure: CheckpointStop, Graders: []ValidatorInline{validGrader}},
+		})
+		require.NoError(t, tc.Validate())
+	})
+
+	t.Run("upper bound skipped for responder", func(t *testing.T) {
+		// With responder, total turn count is unknown at validation time;
+		// only a structural sanity check applies.
+		tc := &TestCase{
+			TestID:      "t",
+			DisplayName: "t",
+			Stimulus: TaskStimulus{
+				Message:   "go",
+				Responder: &ResponderConfig{Instructions: "x", MaxFollowups: 4},
+			},
+			Checkpoints: []Checkpoint{{AfterTurn: 5, Graders: []ValidatorInline{validGrader}}},
+		}
+		require.NoError(t, tc.Validate())
+	})
+}