From b2656e137c8a07627313b2d7a7ce522c6d3b1bcf Mon Sep 17 00:00:00 2001
From: Copilot <223556219+Copilot@users.noreply.github.com>
Date: Sun, 28 Jun 2026 08:09:54 -0400
Subject: [PATCH 1/3] feat: per-turn checkpoint graders (closes #358)

Add an additive `checkpoints:` field to task YAML so multi-turn evals
can grade conversation state at specific turn boundaries instead of
only the final output.

- New `Checkpoint` model with after_turn, graders, on_failure (continue/stop)
- New `CheckpointOutcome` recorded per task on results.json
- Per-turn hook in runner (initial + follow_ups + responder loop)
- on_failure: stop aborts remaining turns and flips status to error
- Bumped schemaVersion to 1.1 (additive, MINOR bump per #382 policy)
- Reuses existing grader plumbing (graders.RunAll + buildGraderContext)
- Honors --skip-graders by short-circuiting checkpoint evaluation
- Full unit + integration tests; docs (guide + schema + changelog)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 cmd/waza/cmd_migrate_test.go                  |   6 +-
 internal/models/outcome.go                    |  22 ++
 internal/models/outcome_schema_test.go        |   2 +-
 internal/models/schema_version.go             |   5 +-
 internal/models/testcase.go                   | 105 +++++++++
 internal/models/testcase_test.go              | 111 ++++++++++
 internal/orchestration/checkpoints.go         | 202 ++++++++++++++++++
 internal/orchestration/runner.go              |  59 ++++-
 .../runner_orchestration_test.go              | 183 ++++++++++++++++
 site/src/content/docs/guides/eval-yaml.mdx    |  27 +++
 .../content/docs/reference/schema-changes.md  |  11 +-
 site/src/content/docs/reference/schema.mdx    |  30 ++-
 12 files changed, 747 insertions(+), 16 deletions(-)
 create mode 100644 internal/orchestration/checkpoints.go

diff --git a/cmd/waza/cmd_migrate_test.go b/cmd/waza/cmd_migrate_test.go
index 0d46bcf3..77c239cf 100644
--- a/cmd/waza/cmd_migrate_test.go
+++ b/cmd/waza/cmd_migrate_test.go
@@ -18,7 +18,7 @@ func TestMigrateCommandNoOpForCurrentMajor(t *testing.T) {
 	err := runMigrate(&out, path)
 
 	require.NoError(t, err)
-	require.Contains(t, out.String(), "already compatible with schemaVersion 1.0")
+	require.Contains(t, out.String(), "compatible with waza schemaVersion 1.1")
 }
 
 func TestMigrateCommandRejectsIncompatibleMajor(t *testing.T) {
@@ -43,7 +43,7 @@ func TestMigrateCommandDefaultsMissingSchemaVersion(t *testing.T) {
 	err := runMigrate(&out, path)
 
 	require.NoError(t, err)
-	require.Contains(t, out.String(), "already compatible with schemaVersion 1.0")
+	require.Contains(t, out.String(), "already compatible with schemaVersion 1.1")
 }
 
 func TestMigrateCommandRejectsUnknownJSONShape(t *testing.T) {
@@ -69,7 +69,7 @@ func TestMigrateCommandDetectsResultsJSONByShape(t *testing.T) {
 	err := runMigrate(&out, path)
 
 	require.NoError(t, err)
-	require.Contains(t, out.String(), "results.json is already compatible")
+	require.Contains(t, out.String(), "results.json uses schemaVersion 1.0")
 }
 
 func TestMigrateCommandMissingFile(t *testing.T) {
diff --git a/internal/models/outcome.go b/internal/models/outcome.go
index b2b221e6..124e00a4 100644
--- a/internal/models/outcome.go
+++ b/internal/models/outcome.go
@@ -192,6 +192,28 @@ type RunResult struct {
 	WorkspaceDir     string                   `json:"workspace_dir,omitempty"`
 	FailureArtifacts *FailureArtifacts        `json:"failure_artifacts,omitempty"`
 	Responder        *ResponderInfo           `json:"responder,omitempty"`
+	// Checkpoints captures per-turn checkpoint grader results, one entry per
+	// configured TestCase.Checkpoint that actually ran (i.e., turn index was
+	// reached). Empty / omitted when the task defines no checkpoints.
+	Checkpoints []CheckpointOutcome `json:"checkpoints,omitempty"`
+}
+
+// CheckpointOutcome captures the results of a single TestCase.Checkpoint that
+// ran during a multi-turn run. It mirrors the shape of the final-grader
+// validations map so dashboards can present per-turn pass/fail uniformly.
+type CheckpointOutcome struct {
+	// AfterTurn echoes Checkpoint.AfterTurn so consumers can sort/group.
+	AfterTurn int `json:"after_turn"`
+	// Status is StatusPassed when every grader in this checkpoint passed,
+	// StatusFailed when at least one grader failed.
+	Status Status `json:"status"`
+	// Validations maps grader identifier to result, identical to
+	// RunResult.Validations.
+	Validations map[string]GraderResults `json:"validations"`
+	// Stopped is true when this checkpoint had `on_failure: stop` and at
+	// least one grader failed, terminating the multi-turn loop after this
+	// turn.
+	Stopped bool `json:"stopped,omitempty"`
 }
 
 // FailureArtifacts captures diagnostic information when a run fails
diff --git a/internal/models/outcome_schema_test.go b/internal/models/outcome_schema_test.go
index 59a7c2ce..24d9dd49 100644
--- a/internal/models/outcome_schema_test.go
+++ b/internal/models/outcome_schema_test.go
@@ -84,7 +84,7 @@ func TestEvaluationOutcomeMarshalDefaultsSchemaVersion(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Marshal() error = %v", err)
 	}
-	if !strings.Contains(string(data), `"schemaVersion":"1.0"`) {
+	if !strings.Contains(string(data), `"schemaVersion":"1.1"`) {
 		t.Fatalf("marshaled outcome missing default schemaVersion: %s", data)
 	}
 }
diff --git a/internal/models/schema_version.go b/internal/models/schema_version.go
index c09929b2..5d4729b1 100644
--- a/internal/models/schema_version.go
+++ b/internal/models/schema_version.go
@@ -15,7 +15,10 @@ import (
 
 const (
 	// CurrentSchemaVersion is the current MAJOR.MINOR schema version for public artifacts.
-	CurrentSchemaVersion = "1.0"
+	//
+	// 1.1 adds per-turn checkpoints (TestCase.Checkpoints / RunResult.Checkpoints) — purely
+	// additive over 1.0, so 1.0 artifacts continue to load without migration.
+	CurrentSchemaVersion = "1.1"
 )
 
 func defaultSchemaVersion(version string) string {
diff --git a/internal/models/testcase.go b/internal/models/testcase.go
index e024cdbe..c88d1227 100644
--- a/internal/models/testcase.go
+++ b/internal/models/testcase.go
@@ -31,6 +31,74 @@ type TestCase struct {
 	// nil inherits the eval-level value; 0 disables the check for this task.
 	FirstEventTimeoutSec *int              `yaml:"first_event_timeout_seconds,omitempty" json:"first_event_timeout_sec,omitempty"`
 	Validators           []ValidatorInline `yaml:"graders,omitempty" json:"validators,omitempty"`
+	// Checkpoints attach graders to intermediate turn boundaries in a
+	// multi-turn run. Each checkpoint specifies `after_turn: N` (1-based,
+	// where turn 1 is the initial prompt) and a list of graders that run
+	// against the cumulative conversation state at the end of that turn.
+	// Checkpoints are additive — task-level `graders:` still run against
+	// the final state after all turns complete.
+	Checkpoints []Checkpoint `yaml:"checkpoints,omitempty" json:"checkpoints,omitempty"`
+}
+
+// CheckpointOnFailure controls multi-turn behavior when a checkpoint fails.
+type CheckpointOnFailure string
+
+const (
+	// CheckpointContinue (default) records the failure and continues with the
+	// remaining turns and the final grader pass.
+	CheckpointContinue CheckpointOnFailure = "continue"
+	// CheckpointStop records the failure, sets the run's error message, and
+	// short-circuits the multi-turn loop so no further turns execute.
+	CheckpointStop CheckpointOnFailure = "stop"
+)
+
+// Checkpoint runs a slice of graders against the conversation state at the
+// end of a specific turn. See TestCase.Checkpoints for details.
+type Checkpoint struct {
+	// AfterTurn is the 1-based turn index at whose boundary the graders run.
+	// Turn 1 is the initial prompt; turns 2..N are follow-ups or responder
+	// replies. Must be >= 1.
+	AfterTurn int `yaml:"after_turn" json:"after_turn"`
+	// Graders is the slice of grader configurations to run at this turn
+	// boundary. Required. Reuses the same ValidatorInline shape as task-level
+	// `graders:` so any existing grader type is supported.
+	Graders []ValidatorInline `yaml:"graders" json:"graders"`
+	// OnFailure controls multi-turn behavior when any grader in this
+	// checkpoint fails. "continue" (default) records the failure and keeps
+	// going. "stop" short-circuits the multi-turn loop after the failure.
+	OnFailure CheckpointOnFailure `yaml:"on_failure,omitempty" json:"on_failure,omitempty"`
+}
+
+// EffectiveOnFailure returns the OnFailure value to use, defaulting to
+// CheckpointContinue when unset.
+func (c *Checkpoint) EffectiveOnFailure() CheckpointOnFailure {
+	if c.OnFailure == "" {
+		return CheckpointContinue
+	}
+	return c.OnFailure
+}
+
+// Validate checks the checkpoint has the required fields. Reuses
+// ValidatorInline.Validate for inner grader validation.
+func (c *Checkpoint) Validate() error {
+	if c.AfterTurn < 1 {
+		return fmt.Errorf("after_turn must be at least 1, got %d", c.AfterTurn)
+	}
+	if len(c.Graders) == 0 {
+		return fmt.Errorf("after_turn %d: graders is required and must contain at least one grader", c.AfterTurn)
+	}
+	switch c.OnFailure {
+	case "", CheckpointContinue, CheckpointStop:
+	default:
+		return fmt.Errorf("after_turn %d: on_failure must be %q or %q, got %q",
+			c.AfterTurn, CheckpointContinue, CheckpointStop, c.OnFailure)
+	}
+	for i := range c.Graders {
+		if err := c.Graders[i].Validate(); err != nil {
+			return fmt.Errorf("after_turn %d: grader[%d]: %w", c.AfterTurn, i, err)
+		}
+	}
+	return nil
 }
 
 // TaskStimulus defines the input for a task.
@@ -375,6 +443,43 @@ func (tc *TestCase) Validate() error {
 		}
 	}
 
+	// Validate checkpoints. When follow-up prompts (not a responder) drive
+	// the multi-turn run, the turn count is known statically so we can also
+	// reject checkpoints that exceed it.
+	if len(tc.Checkpoints) > 0 {
+		name := tc.TestID
+		if name == "" {
+			name = tc.DisplayName
+		}
+		prefix := "test case"
+		if name != "" {
+			prefix = fmt.Sprintf("test case %q", name)
+		}
+
+		// Static turn count: 1 initial + len(FollowUps) follow-ups when no
+		// responder is configured. Responder is dynamic — skip the upper bound.
+		maxTurns := 0
+		if tc.Stimulus.Responder == nil {
+			maxTurns = 1 + len(tc.Stimulus.FollowUps)
+		}
+
+		seen := make(map[int]bool, len(tc.Checkpoints))
+		for i := range tc.Checkpoints {
+			if err := tc.Checkpoints[i].Validate(); err != nil {
+				return fmt.Errorf("%s: checkpoints[%d]: %w", prefix, i, err)
+			}
+			at := tc.Checkpoints[i].AfterTurn
+			if seen[at] {
+				return fmt.Errorf("%s: checkpoints[%d]: duplicate after_turn %d (each turn may have at most one checkpoint)", prefix, i, at)
+			}
+			seen[at] = true
+			if maxTurns > 0 && at > maxTurns {
+				return fmt.Errorf("%s: checkpoints[%d].after_turn %d exceeds total turns (1 initial + %d follow_up_prompts = %d turns)",
+					prefix, i, at, len(tc.Stimulus.FollowUps), maxTurns)
+			}
+		}
+	}
+
 	return nil
 }
 
diff --git a/internal/models/testcase_test.go b/internal/models/testcase_test.go
index 68875616..9756c036 100644
--- a/internal/models/testcase_test.go
+++ b/internal/models/testcase_test.go
@@ -340,3 +340,114 @@ func TestResponderValidationAcceptsValidConfig(t *testing.T) {
 	}
 	require.NoError(t, tc.Validate())
 }
+
+// -- Checkpoint validation tests --
+
+func TestLoadTestCase_CheckpointsParsed(t *testing.T) {
+	yamlData := `id: tc-cp
+name: Checkpoint loader
+inputs:
+  prompt: do work
+  follow_up_prompts:
+    - keep going
+checkpoints:
+  - after_turn: 1
+    graders:
+      - name: mid-check
+        type: text
+        config:
+          contains:
+            - hello
+  - after_turn: 2
+    on_failure: stop
+    graders:
+      - name: end-check
+        type: text
+        config:
+          contains:
+            - bye
+`
+	dir := t.TempDir()
+	p := filepath.Join(dir, "tc.yaml")
+	require.NoError(t, os.WriteFile(p, []byte(yamlData), 0o644))
+
+	tc, err := LoadTestCase(p)
+	require.NoError(t, err)
+	require.Len(t, tc.Checkpoints, 2)
+	require.Equal(t, 1, tc.Checkpoints[0].AfterTurn)
+	require.Equal(t, CheckpointContinue, tc.Checkpoints[0].EffectiveOnFailure())
+	require.Equal(t, CheckpointStop, tc.Checkpoints[1].EffectiveOnFailure())
+	require.Len(t, tc.Checkpoints[0].Graders, 1)
+	require.Equal(t, "mid-check", tc.Checkpoints[0].Graders[0].Identifier)
+}
+
+func TestCheckpointValidation(t *testing.T) {
+	makeTC := func(cps []Checkpoint) *TestCase {
+		return &TestCase{
+			TestID:      "t",
+			DisplayName: "t",
+			Stimulus: TaskStimulus{
+				Message:   "go",
+				FollowUps: []string{"again", "more"},
+			},
+			Checkpoints: cps,
+		}
+	}
+	validGrader := ValidatorInline{
+		Identifier: "g",
+		Kind:       GraderKindText,
+		Parameters: TextGraderParameters{Contains: []string{"hi"}},
+	}
+
+	t.Run("after_turn < 1 rejected", func(t *testing.T) {
+		tc := makeTC([]Checkpoint{{AfterTurn: 0, Graders: []ValidatorInline{validGrader}}})
+		require.ErrorContains(t, tc.Validate(), "after_turn")
+	})
+
+	t.Run("missing graders rejected", func(t *testing.T) {
+		tc := makeTC([]Checkpoint{{AfterTurn: 1}})
+		require.ErrorContains(t, tc.Validate(), "graders")
+	})
+
+	t.Run("duplicate after_turn rejected", func(t *testing.T) {
+		tc := makeTC([]Checkpoint{
+			{AfterTurn: 1, Graders: []ValidatorInline{validGrader}},
+			{AfterTurn: 1, Graders: []ValidatorInline{validGrader}},
+		})
+		require.ErrorContains(t, tc.Validate(), "duplicate")
+	})
+
+	t.Run("after_turn exceeds turns rejected", func(t *testing.T) {
+		// 1 initial + 2 follow-ups = 3 turns; 4 is out of range.
+		tc := makeTC([]Checkpoint{{AfterTurn: 4, Graders: []ValidatorInline{validGrader}}})
+		require.ErrorContains(t, tc.Validate(), "exceeds")
+	})
+
+	t.Run("invalid on_failure rejected", func(t *testing.T) {
+		tc := makeTC([]Checkpoint{{AfterTurn: 1, OnFailure: "maybe", Graders: []ValidatorInline{validGrader}}})
+		require.ErrorContains(t, tc.Validate(), "on_failure")
+	})
+
+	t.Run("valid case accepted", func(t *testing.T) {
+		tc := makeTC([]Checkpoint{
+			{AfterTurn: 1, Graders: []ValidatorInline{validGrader}},
+			{AfterTurn: 3, OnFailure: CheckpointStop, Graders: []ValidatorInline{validGrader}},
+		})
+		require.NoError(t, tc.Validate())
+	})
+
+	t.Run("upper bound skipped for responder", func(t *testing.T) {
+		// With responder, total turn count is unknown at validation time;
+		// only a structural sanity check applies.
+		tc := &TestCase{
+			TestID:      "t",
+			DisplayName: "t",
+			Stimulus: TaskStimulus{
+				Message:   "go",
+				Responder: &ResponderConfig{Instructions: "x", MaxFollowups: 4},
+			},
+			Checkpoints: []Checkpoint{{AfterTurn: 5, Graders: []ValidatorInline{validGrader}}},
+		}
+		require.NoError(t, tc.Validate())
+	})
+}
diff --git a/internal/orchestration/checkpoints.go b/internal/orchestration/checkpoints.go
new file mode 100644
index 00000000..a7f06666
--- /dev/null
+++ b/internal/orchestration/checkpoints.go
@@ -0,0 +1,202 @@
+package orchestration
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/microsoft/waza/internal/execution"
+	"github.com/microsoft/waza/internal/graders"
+	"github.com/microsoft/waza/internal/models"
+)
+
+// checkpointRunner manages per-turn checkpoint evaluation during a multi-turn
+// run. It owns the list of pending checkpoints for a task and exposes a
+// single entry point that the multi-turn drivers (executeFollowUps and
+// executeResponderLoop) call after each turn.
+//
+// The cumulative ExecutionResponse passed at each turn boundary already
+// aggregates events, tool calls, skill invocations, usage, and final output
+// through the just-completed turn, so we build a turn-scoped graders.Context
+// directly from it without slicing.
+type checkpointRunner struct {
+	runner *EvalRunner
+	tc     *models.TestCase
+	// pending maps after_turn -> *Checkpoint for fast turn-boundary lookup
+	// and to detect whether any checkpoints are configured.
+	pending map[int]*models.Checkpoint
+	// outcomes accumulates per-turn checkpoint results in turn order.
+	outcomes []models.CheckpointOutcome
+	// stopped is true once any "on_failure: stop" checkpoint has failed,
+	// at which point the multi-turn driver should break out of its loop.
+	stopped bool
+}
+
+// newCheckpointRunner builds a checkpointRunner from a TestCase. Returns nil
+// when the task has no checkpoints — callers should always check for nil
+// before invoking methods.
+func newCheckpointRunner(runner *EvalRunner, tc *models.TestCase) *checkpointRunner {
+	if tc == nil || len(tc.Checkpoints) == 0 {
+		return nil
+	}
+	cr := &checkpointRunner{
+		runner:  runner,
+		tc:      tc,
+		pending: make(map[int]*models.Checkpoint, len(tc.Checkpoints)),
+	}
+	for i := range tc.Checkpoints {
+		cp := &tc.Checkpoints[i]
+		cr.pending[cp.AfterTurn] = cp
+	}
+	return cr
+}
+
+// runForTurn evaluates the checkpoint (if any) scheduled for the given
+// 1-based turn number. It returns true when the multi-turn loop should stop
+// (i.e., an "on_failure: stop" checkpoint failed). When stop is requested,
+// it also sets resp.ErrorMsg so the run is recorded as failed.
+//
+// Safe to call on a nil receiver, in which case it is a no-op returning false.
+func (cr *checkpointRunner) runForTurn(ctx context.Context, turn int, resp *execution.ExecutionResponse) bool {
+	if cr == nil {
+		return false
+	}
+	cp, ok := cr.pending[turn]
+	if !ok {
+		return false
+	}
+	// Remove from pending so duplicate-turn callers (none expected, but
+	// defensive) cannot re-run the same checkpoint.
+	delete(cr.pending, turn)
+
+	// Honor --skip-graders: if the user asked to skip grading, skip
+	// checkpoint graders too. We do not record a CheckpointOutcome in that
+	// case (mirroring how skipped runs omit final-pass validations).
+	if cr.runner.skipGraders {
+		return false
+	}
+
+	gCtx := cr.runner.buildGraderContext(cr.tc, resp)
+
+	// Run only the checkpoint's graders. Reuse graders.RunAll by passing a
+	// synthetic TestCase whose Validators field carries the checkpoint
+	// graders. We do NOT want task-level expectation strings (output_contains
+	// etc.) or task-level validators to run again here — those belong to the
+	// final pass. Using a stripped TestCase ensures only the checkpoint
+	// graders fire at this boundary.
+	stub := &models.TestCase{
+		TestID:      cr.tc.TestID,
+		DisplayName: cr.tc.DisplayName,
+		Validators:  cp.Graders,
+	}
+	results, err := graders.RunAll(
+		ctx,
+		nil, // no spec-level graders at checkpoint boundaries
+		stub,
+		gCtx,
+		cr.runner.cfg.Spec().Config.JudgeModel,
+		cr.runner.updateSnapshots,
+	)
+	outcome := models.CheckpointOutcome{
+		AfterTurn:   turn,
+		Status:      models.StatusPassed,
+		Validations: results,
+	}
+	if err != nil {
+		// Synthesize a single failed-grader entry so downstream consumers
+		// can still see what went wrong, mirroring how runGraders surfaces
+		// errors via resp.ErrorMsg.
+		if outcome.Validations == nil {
+			outcome.Validations = make(map[string]models.GraderResults)
+		}
+		outcome.Validations["_checkpoint_error"] = models.GraderResults{
+			Name:     "_checkpoint_error",
+			Score:    0,
+			Passed:   false,
+			Feedback: err.Error(),
+			Weight:   1.0,
+		}
+		outcome.Status = models.StatusError
+	}
+
+	failed := outcome.Status != models.StatusPassed
+	if !failed {
+		for _, gr := range outcome.Validations {
+			if !gr.Passed {
+				outcome.Status = models.StatusFailed
+				failed = true
+				break
+			}
+		}
+	}
+
+	// Surface checkpoint grader results as progress events for verbose CLI
+	// users, mirroring the final-pass output.
+	if cr.runner.verbose {
+		names := make([]string, 0, len(outcome.Validations))
+		for n := range outcome.Validations {
+			names = append(names, n)
+		}
+		sort.Strings(names)
+		for _, n := range names {
+			gr := outcome.Validations[n]
+			cr.runner.notifyProgress(ProgressEvent{
+				EventType:  EventGraderResult,
+				TestName:   cr.tc.DisplayName,
+				DurationMs: gr.DurationMs,
+				Details: map[string]any{
+					"grader":      n,
+					"grader_type": gr.Type,
+					"passed":      gr.Passed,
+					"score":       gr.Score,
+					"feedback":    gr.Feedback,
+					"checkpoint":  turn,
+				},
+			})
+		}
+	}
+
+	if failed && cp.EffectiveOnFailure() == models.CheckpointStop {
+		outcome.Stopped = true
+		cr.stopped = true
+		failedNames := failedGraderNames(outcome.Validations)
+		msg := fmt.Sprintf("checkpoint after_turn=%d failed (on_failure: stop)", turn)
+		if len(failedNames) > 0 {
+			msg += ": " + strings.Join(failedNames, ", ")
+		}
+		// Preserve any prior error rather than overwriting silently.
+		if resp.ErrorMsg == "" {
+			resp.ErrorMsg = msg
+		} else {
+			resp.ErrorMsg = resp.ErrorMsg + "; " + msg
+		}
+	}
+
+	cr.outcomes = append(cr.outcomes, outcome)
+	return outcome.Stopped
+}
+
+// results returns the accumulated per-turn outcomes (sorted by turn).
+func (cr *checkpointRunner) results() []models.CheckpointOutcome {
+	if cr == nil || len(cr.outcomes) == 0 {
+		return nil
+	}
+	out := make([]models.CheckpointOutcome, len(cr.outcomes))
+	copy(out, cr.outcomes)
+	sort.SliceStable(out, func(i, j int) bool {
+		return out[i].AfterTurn < out[j].AfterTurn
+	})
+	return out
+}
+
+func failedGraderNames(m map[string]models.GraderResults) []string {
+	var names []string
+	for n, gr := range m {
+		if !gr.Passed {
+			names = append(names, n)
+		}
+	}
+	sort.Strings(names)
+	return names
+}
diff --git a/internal/orchestration/runner.go b/internal/orchestration/runner.go
index 7ddbafd5..6c464ffb 100644
--- a/internal/orchestration/runner.go
+++ b/internal/orchestration/runner.go
@@ -1153,13 +1153,23 @@ func (r *EvalRunner) executeRun(ctx context.Context, tc *models.TestCase, runNum
 		})
 	}
 
+	// Build the per-turn checkpoint runner. Nil when the task has no
+	// checkpoints; safe to invoke methods on a nil receiver below.
+	cps := newCheckpointRunner(r, tc)
+
+	// Run any checkpoint scheduled after turn 1 before the multi-turn loop.
+	// If it requests stop, skip follow-ups/responder entirely.
+	stop := cps.runForTurn(ctx, 1, resp)
+
 	// Drive multi-turn: responder loop takes precedence; otherwise static
 	// follow-ups. Validation guarantees these are mutually exclusive.
 	var responderInfo *models.ResponderInfo
-	if tc.Stimulus.Responder != nil {
-		responderInfo = r.executeResponderLoop(ctx, tc, resp)
-	} else if len(tc.Stimulus.FollowUps) > 0 {
-		r.executeFollowUps(ctx, tc, resp)
+	if !stop && resp.ErrorMsg == "" {
+		if tc.Stimulus.Responder != nil {
+			responderInfo = r.executeResponderLoop(ctx, tc, resp, cps)
+		} else if len(tc.Stimulus.FollowUps) > 0 {
+			r.executeFollowUps(ctx, tc, resp, cps)
+		}
 	}
 
 	// Build validation context
@@ -1219,6 +1229,19 @@ func (r *EvalRunner) executeRun(ctx context.Context, tc *models.TestCase, runNum
 		}
 	}
 
+	// Surface checkpoint failures in the run status even when graders are
+	// skipped or when the final-pass graders all passed. A failed checkpoint
+	// without on_failure: stop should still mark the run as failed.
+	checkpointOutcomes := cps.results()
+	if status != models.StatusError {
+		for _, co := range checkpointOutcomes {
+			if co.Status != models.StatusPassed {
+				status = models.StatusFailed
+				break
+			}
+		}
+	}
+
 	// Build transcript
 	transcript := r.buildTranscript(resp)
 
@@ -1239,6 +1262,7 @@ func (r *EvalRunner) executeRun(ctx context.Context, tc *models.TestCase, runNum
 		SkillInvocations: skillInvocations,
 		WorkspaceDir:     resp.WorkspaceDir,
 		Responder:        responderInfo,
+		Checkpoints:      checkpointOutcomes,
 	})
 }
 
@@ -1340,8 +1364,10 @@ func rejectRelativePathPromptWithEmptySandbox(tc *models.TestCase, resources []e
 }
 
 // executeFollowUps sends follow-up prompts using the same workspace and session,
-// aggregating results into the original response.
-func (r *EvalRunner) executeFollowUps(ctx context.Context, tc *models.TestCase, resp *execution.ExecutionResponse) {
+// aggregating results into the original response. When cps is non-nil, after
+// each follow-up turn it runs the matching checkpoint (if any) and stops the
+// loop early when a checkpoint with on_failure: stop fails.
+func (r *EvalRunner) executeFollowUps(ctx context.Context, tc *models.TestCase, resp *execution.ExecutionResponse, cps *checkpointRunner) {
 	for i, prompt := range tc.Stimulus.FollowUps {
 		followReq, err := r.buildExecutionRequest(tc)
 		if err != nil {
@@ -1410,6 +1436,14 @@ func (r *EvalRunner) executeFollowUps(ctx context.Context, tc *models.TestCase,
 				resp.Usage = models.AggregateUsageStats([]*models.UsageStats{resp.Usage, followResp.Usage})
 			}
 		}
+
+		// Run any checkpoint scheduled after this turn. Turn number is
+		// i+2 because the initial Execute was turn 1 and follow-ups are
+		// 0-indexed in tc.Stimulus.FollowUps. Stop the loop early when
+		// the checkpoint requested on_failure: stop.
+		if stop := cps.runForTurn(ctx, i+2, resp); stop {
+			break
+		}
 	}
 }
 
@@ -1417,7 +1451,10 @@ func (r *EvalRunner) executeFollowUps(ctx context.Context, tc *models.TestCase,
 // user. After each agent turn it classifies the agent's latest message and
 // either replies (sending a new agent prompt), stops, or aborts on abstain.
 // It mutates resp in place (mirroring executeFollowUps) and returns a summary.
-func (r *EvalRunner) executeResponderLoop(ctx context.Context, tc *models.TestCase, resp *execution.ExecutionResponse) *models.ResponderInfo {
+// When cps is non-nil, checkpoints scheduled at each turn boundary run after
+// every successful reply, and the loop stops early when a checkpoint with
+// on_failure: stop fails.
+func (r *EvalRunner) executeResponderLoop(ctx context.Context, tc *models.TestCase, resp *execution.ExecutionResponse, cps *checkpointRunner) *models.ResponderInfo {
 	cfg := *tc.Stimulus.Responder
 	classifier := r.newClassifier(cfg, r.cfg.Spec().Config.ModelID)
 	defer func() {
@@ -1462,6 +1499,14 @@ func (r *EvalRunner) executeResponderLoop(ctx context.Context, tc *models.TestCa
 			}
 			info.FollowupsSent++
 			left--
+
+			// Run any checkpoint scheduled after this turn. The reply we
+			// just sent produced agent turn `info.FollowupsSent + 1`
+			// (initial Execute was turn 1, replies are turns 2..N).
+			if stop := cps.runForTurn(ctx, info.FollowupsSent+1, resp); stop {
+				info.Outcome = models.ResponderOutcomeStopped
+				return info
+			}
 		}
 	}
 
diff --git a/internal/orchestration/runner_orchestration_test.go b/internal/orchestration/runner_orchestration_test.go
index 4127f72b..8ad4f1bc 100644
--- a/internal/orchestration/runner_orchestration_test.go
+++ b/internal/orchestration/runner_orchestration_test.go
@@ -719,3 +719,186 @@ expected:
 	assert.Equal(t, "follow-up 1", eng.calls[1].Message)
 	assert.Equal(t, "follow-up 2", eng.calls[2].Message)
 }
+
+// --- Per-turn checkpoint tests (#358) ---
+
+func TestExecuteRun_Checkpoints_PassAfterEachTurn(t *testing.T) {
+	eng := &trackingEngine{}
+	spec := &models.EvalSpec{
+		SpecIdentity: models.SpecIdentity{Name: "cp-pass"},
+		Config:       models.Config{TrialsPerTask: 1, TimeoutSec: 30},
+	}
+	cfg := config.NewEvalConfig(spec)
+	runner := NewEvalRunner(cfg, eng)
+
+	tc := &models.TestCase{
+		TestID:      "cp-pass",
+		DisplayName: "checkpoint pass",
+		Stimulus: models.TaskStimulus{
+			Message:   "alpha",
+			FollowUps: []string{"beta"},
+		},
+		Checkpoints: []models.Checkpoint{
+			{
+				AfterTurn: 1,
+				Graders: []models.ValidatorInline{{
+					Identifier: "after-1",
+					Kind:       models.GraderKindText,
+					Parameters: models.TextGraderParameters{Contains: []string{"alpha"}},
+				}},
+			},
+			{
+				AfterTurn: 2,
+				Graders: []models.ValidatorInline{{
+					Identifier: "after-2",
+					Kind:       models.GraderKindText,
+					Parameters: models.TextGraderParameters{Contains: []string{"beta"}},
+				}},
+			},
+		},
+	}
+
+	result := runner.executeRun(context.Background(), tc, 1)
+	assert.Equal(t, models.StatusPassed, result.Status)
+	require.Len(t, result.Checkpoints, 2)
+	assert.Equal(t, 1, result.Checkpoints[0].AfterTurn)
+	assert.Equal(t, models.StatusPassed, result.Checkpoints[0].Status)
+	assert.False(t, result.Checkpoints[0].Stopped)
+	assert.Equal(t, 2, result.Checkpoints[1].AfterTurn)
+	assert.Equal(t, models.StatusPassed, result.Checkpoints[1].Status)
+	// Both turns ran.
+	assert.Equal(t, 2, len(eng.calls))
+}
+
+func TestExecuteRun_Checkpoints_FailContinue(t *testing.T) {
+	eng := &trackingEngine{}
+	spec := &models.EvalSpec{
+		SpecIdentity: models.SpecIdentity{Name: "cp-cont"},
+		Config:       models.Config{TrialsPerTask: 1, TimeoutSec: 30},
+	}
+	cfg := config.NewEvalConfig(spec)
+	runner := NewEvalRunner(cfg, eng)
+
+	tc := &models.TestCase{
+		TestID:      "cp-cont",
+		DisplayName: "checkpoint failed but continue",
+		Stimulus: models.TaskStimulus{
+			Message:   "alpha",
+			FollowUps: []string{"beta", "gamma"},
+		},
+		Checkpoints: []models.Checkpoint{
+			{
+				AfterTurn: 2,
+				// trackingEngine returns "response to: beta" after turn 2;
+				// asking for an impossible substring forces a fail.
+				Graders: []models.ValidatorInline{{
+					Identifier: "must-fail",
+					Kind:       models.GraderKindText,
+					Parameters: models.TextGraderParameters{Contains: []string{"NEVER_PRESENT"}},
+				}},
+			},
+		},
+	}
+
+	result := runner.executeRun(context.Background(), tc, 1)
+	// All 3 turns should still have run.
+	assert.Equal(t, 3, len(eng.calls), "loop must continue past failed checkpoint")
+	// Run status is failed because a checkpoint failed.
+	assert.Equal(t, models.StatusFailed, result.Status)
+	require.Len(t, result.Checkpoints, 1)
+	assert.Equal(t, models.StatusFailed, result.Checkpoints[0].Status)
+	assert.False(t, result.Checkpoints[0].Stopped)
+	assert.Empty(t, result.ErrorMsg)
+}
+
+func TestExecuteRun_Checkpoints_FailStop(t *testing.T) {
+	eng := &trackingEngine{}
+	spec := &models.EvalSpec{
+		SpecIdentity: models.SpecIdentity{Name: "cp-stop"},
+		Config:       models.Config{TrialsPerTask: 1, TimeoutSec: 30},
+	}
+	cfg := config.NewEvalConfig(spec)
+	runner := NewEvalRunner(cfg, eng)
+
+	tc := &models.TestCase{
+		TestID:      "cp-stop",
+		DisplayName: "checkpoint stop short-circuits",
+		Stimulus: models.TaskStimulus{
+			Message:   "alpha",
+			FollowUps: []string{"beta", "gamma"},
+		},
+		Checkpoints: []models.Checkpoint{
+			{
+				AfterTurn: 2,
+				OnFailure: models.CheckpointStop,
+				Graders: []models.ValidatorInline{{
+					Identifier: "must-fail",
+					Kind:       models.GraderKindText,
+					Parameters: models.TextGraderParameters{Contains: []string{"NEVER_PRESENT"}},
+				}},
+			},
+		},
+	}
+
+	result := runner.executeRun(context.Background(), tc, 1)
+	// Only 2 turns ran (initial + first follow-up); the loop must stop.
+	assert.Equal(t, 2, len(eng.calls), "loop must stop after failed on_failure=stop checkpoint")
+	assert.Equal(t, models.StatusError, result.Status)
+	require.Len(t, result.Checkpoints, 1)
+	assert.True(t, result.Checkpoints[0].Stopped)
+	assert.Contains(t, result.ErrorMsg, "checkpoint after_turn=2")
+}
+
+func TestExecuteRun_Checkpoints_BackwardCompat(t *testing.T) {
+	eng := &trackingEngine{}
+	spec := &models.EvalSpec{
+		SpecIdentity: models.SpecIdentity{Name: "cp-none"},
+		Config:       models.Config{TrialsPerTask: 1, TimeoutSec: 30},
+	}
+	cfg := config.NewEvalConfig(spec)
+	runner := NewEvalRunner(cfg, eng, WithSkipGraders())
+
+	tc := &models.TestCase{
+		TestID:      "cp-none",
+		DisplayName: "no checkpoints",
+		Stimulus:    models.TaskStimulus{Message: "alpha", FollowUps: []string{"beta"}},
+	}
+
+	result := runner.executeRun(context.Background(), tc, 1)
+	assert.Equal(t, models.StatusSkipped, result.Status)
+	assert.Nil(t, result.Checkpoints, "no checkpoints means nil slice")
+	assert.Equal(t, 2, len(eng.calls))
+}
+
+func TestExecuteRun_Checkpoints_SkipGradersHonored(t *testing.T) {
+	eng := &trackingEngine{}
+	spec := &models.EvalSpec{
+		SpecIdentity: models.SpecIdentity{Name: "cp-skip"},
+		Config:       models.Config{TrialsPerTask: 1, TimeoutSec: 30},
+	}
+	cfg := config.NewEvalConfig(spec)
+	runner := NewEvalRunner(cfg, eng, WithSkipGraders())
+
+	tc := &models.TestCase{
+		TestID:      "cp-skip",
+		DisplayName: "skip graders also skips checkpoints",
+		Stimulus:    models.TaskStimulus{Message: "alpha", FollowUps: []string{"beta"}},
+		Checkpoints: []models.Checkpoint{
+			{
+				AfterTurn: 1,
+				OnFailure: models.CheckpointStop,
+				Graders: []models.ValidatorInline{{
+					Identifier: "would-fail",
+					Kind:       models.GraderKindText,
+					Parameters: models.TextGraderParameters{Contains: []string{"NEVER_PRESENT"}},
+				}},
+			},
+		},
+	}
+
+	result := runner.executeRun(context.Background(), tc, 1)
+	// All turns ran because checkpoints are skipped along with graders.
+	assert.Equal(t, 2, len(eng.calls))
+	assert.Equal(t, models.StatusSkipped, result.Status)
+	assert.Empty(t, result.Checkpoints)
+}
diff --git a/site/src/content/docs/guides/eval-yaml.mdx b/site/src/content/docs/guides/eval-yaml.mdx
index fbb5c880..36961340 100644
--- a/site/src/content/docs/guides/eval-yaml.mdx
+++ b/site/src/content/docs/guides/eval-yaml.mdx
@@ -421,6 +421,33 @@ inputs:
 
 After each agent turn the responder either **replies** (the answer is sent back, continuing the conversation), **stops** (the agent is done), or **abstains** — which fails the run with a distinct `abstained` outcome, signalling the brief is too vague. If `max_followups` is reached while the agent is still asking questions, the loop stops with outcome `cap_exhausted` and graders evaluate the final state. Each task carries its own responder, so the same skill can be tested against several target configurations.
 
+### Per-Turn Checkpoints
+
+By default graders only run once, against the final state of a multi-turn conversation. For long conversations you can run graders at specific turn boundaries using a top-level `checkpoints:` list:
+
+```yaml
+checkpoints:
+  - after_turn: 1
+    graders:
+      - type: text
+        contains: ["analyzing", "files"]
+  - after_turn: 2
+    on_failure: stop # abort the run if this checkpoint fails
+    graders:
+      - type: tool_calls
+        required: ["read_file"]
+```
+
+Each checkpoint accepts:
+
+| Field        | Type     | Description                                                                       |
+| ------------ | -------- | --------------------------------------------------------------------------------- |
+| `after_turn` | int      | 1-based turn number this checkpoint runs after (initial prompt is turn 1).        |
+| `graders`    | array    | Inline graders, same schema as task `validators:` / eval-level `graders:`.        |
+| `on_failure` | string   | `continue` (default) or `stop` — abort remaining turns when this checkpoint fails.|
+
+Outcomes are recorded per-checkpoint on `results.json` under `checkpoints[]`, alongside the final `validations`. `waza gate` still uses final-pass status. Available with `schemaVersion: "1.1"` and above (additive — older 1.0 files load unchanged).
+
 Prompt supports templating:
 
 ```yaml
diff --git a/site/src/content/docs/reference/schema-changes.md b/site/src/content/docs/reference/schema-changes.md
index 5f350e6b..501b4b77 100644
--- a/site/src/content/docs/reference/schema-changes.md
+++ b/site/src/content/docs/reference/schema-changes.md
@@ -9,10 +9,10 @@ Waza public artifacts use an explicit `schemaVersion` field so checked-in eval s
 
 | Artifact | Field | Current version |
 |---|---|---|
-| `eval.yaml` | `schemaVersion` | `1.0` |
-| `results.json` | `schemaVersion` | `1.0` |
+| `eval.yaml` | `schemaVersion` | `1.1` |
+| `results.json` | `schemaVersion` | `1.1` |
 | `snapshot.json` | `schemaVersion` | Not yet emitted |
-| Dashboard/SSE event envelope | `schemaVersion` | `1.0` |
+| Dashboard/SSE event envelope | `schemaVersion` | `1.1` |
 
 ## Policy
 
@@ -36,6 +36,11 @@ For schema `1.0`, the command is a no-op because there is no prior major version
 
 ## Changelog
 
+### 1.1
+
+- Added optional `checkpoints[]` to task YAML for per-turn graders (`after_turn`, `graders`, `on_failure`). Backward-compatible: 1.0 task files load unchanged.
+- Added optional `checkpoints[]` to `results.json` task results, recording per-turn grader outcomes.
+
 ### 1.0
 
 - Added `schemaVersion` to `eval.yaml`.
diff --git a/site/src/content/docs/reference/schema.mdx b/site/src/content/docs/reference/schema.mdx
index 774db448..c8d31e55 100644
--- a/site/src/content/docs/reference/schema.mdx
+++ b/site/src/content/docs/reference/schema.mdx
@@ -88,7 +88,7 @@ version: "1.0"
 
 **Type:** string  
 **Required:** no  
-**Default:** `1.0`
+**Default:** `1.1`
 
 Schema version for the public artifact shape. It uses `MAJOR.MINOR` format with no patch component. Same-major minor additions are backward-compatible; readers ignore unknown fields and warn. Different major versions are rejected with a migration hint.
 
@@ -564,6 +564,34 @@ inputs:
 
 The responder classifies each agent message as **reply**, **stop**, or **abstain**. An abstain marks the run as an error with outcome `abstained`, distinct from model timeouts or network errors. If `max_followups` is reached while the agent is still asking questions, the loop stops with outcome `cap_exhausted` and graders evaluate the final state.
 
+### checkpoints
+
+**Type:** array of objects  
+**Required:** no
+
+Per-turn graders that run at specific turn boundaries during multi-turn conversations. Useful for asserting intermediate state instead of only the final output. Added in `schemaVersion: "1.1"`.
+
+| Field        | Type    | Required | Description                                                                                  |
+|--------------|---------|----------|----------------------------------------------------------------------------------------------|
+| `after_turn` | integer | yes      | 1-based turn number to grade after. Turn 1 is the initial prompt; follow-ups are 2, 3, ...   |
+| `graders`    | array   | yes      | Inline graders (same schema as task `validators:`).                                          |
+| `on_failure` | string  | no       | `continue` (default) or `stop`. `stop` aborts remaining turns when this checkpoint fails.    |
+
+```yaml
+checkpoints:
+  - after_turn: 1
+    graders:
+      - type: text
+        contains: ["analyzing", "files"]
+  - after_turn: 2
+    on_failure: stop
+    graders:
+      - type: tool_calls
+        required: ["read_file"]
+```
+
+Each checkpoint's outcomes appear on `results.json` under `checkpoints[]`. The final-pass `validations` still run independently. `waza gate` continues to use the final-pass status; checkpoint failures flip the task status to `failed` (or `error` when `on_failure: stop` fired).
+
 ### files
 
 **Type:** array  

From ea8a134b8c4311e8f5144b7c63efa5c0c6bea4d3 Mon Sep 17 00:00:00 2001
From: Copilot <223556219+Copilot@users.noreply.github.com>
Date: Sun, 28 Jun 2026 08:17:53 -0400
Subject: [PATCH 2/3] fix: address Copilot review feedback on #386

- Add Type field to synthesized _checkpoint_error GraderResults
- Fix docs to reference 'graders:' (the actual YAML key) instead of 'validators:'
- Update schema-changes.md Policy section to reflect current 1.1 default emission while preserving 1.0 reader fallback for back-compat

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .impeccable/hook.cache.json                       |  1 +
 internal/orchestration/checkpoints.go             |  6 ++++--
 internal/orchestration/runner.go                  | 15 +++++++++++----
 site/src/content/docs/guides/eval-yaml.mdx        |  2 +-
 site/src/content/docs/reference/schema-changes.md |  4 ++--
 site/src/content/docs/reference/schema.mdx        |  2 +-
 6 files changed, 20 insertions(+), 10 deletions(-)
 create mode 100644 .impeccable/hook.cache.json

diff --git a/.impeccable/hook.cache.json b/.impeccable/hook.cache.json
new file mode 100644
index 00000000..ebc7c5d0
--- /dev/null
+++ b/.impeccable/hook.cache.json
@@ -0,0 +1 @@
+{"version":1,"sessions":{}}
\ No newline at end of file
diff --git a/internal/orchestration/checkpoints.go b/internal/orchestration/checkpoints.go
index a7f06666..8f2496c0 100644
--- a/internal/orchestration/checkpoints.go
+++ b/internal/orchestration/checkpoints.go
@@ -34,8 +34,9 @@ type checkpointRunner struct {
 }
 
 // newCheckpointRunner builds a checkpointRunner from a TestCase. Returns nil
-// when the task has no checkpoints — callers should always check for nil
-// before invoking methods.
+// when the task has no checkpoints, but all methods on *checkpointRunner are
+// nil-safe — callers can invoke runForTurn/results on the returned value
+// directly without a nil guard.
 func newCheckpointRunner(runner *EvalRunner, tc *models.TestCase) *checkpointRunner {
 	if tc == nil || len(tc.Checkpoints) == 0 {
 		return nil
@@ -112,6 +113,7 @@ func (cr *checkpointRunner) runForTurn(ctx context.Context, turn int, resp *exec
 		}
 		outcome.Validations["_checkpoint_error"] = models.GraderResults{
 			Name:     "_checkpoint_error",
+			Type:     models.GraderKind("checkpoint_error"),
 			Score:    0,
 			Passed:   false,
 			Feedback: err.Error(),
diff --git a/internal/orchestration/runner.go b/internal/orchestration/runner.go
index 6c464ffb..323392b0 100644
--- a/internal/orchestration/runner.go
+++ b/internal/orchestration/runner.go
@@ -1231,13 +1231,20 @@ func (r *EvalRunner) executeRun(ctx context.Context, tc *models.TestCase, runNum
 
 	// Surface checkpoint failures in the run status even when graders are
 	// skipped or when the final-pass graders all passed. A failed checkpoint
-	// without on_failure: stop should still mark the run as failed.
+	// without on_failure: stop should still mark the run as failed; a
+	// checkpoint that recorded StatusError (grader-execution error) should
+	// promote the run to StatusError so consumers can distinguish
+	// infrastructure problems from assertion failures.
 	checkpointOutcomes := cps.results()
 	if status != models.StatusError {
 		for _, co := range checkpointOutcomes {
-			if co.Status != models.StatusPassed {
-				status = models.StatusFailed
-				break
+			switch co.Status {
+			case models.StatusError:
+				status = models.StatusError
+			case models.StatusFailed:
+				if status != models.StatusError {
+					status = models.StatusFailed
+				}
 			}
 		}
 	}
diff --git a/site/src/content/docs/guides/eval-yaml.mdx b/site/src/content/docs/guides/eval-yaml.mdx
index 36961340..ef216687 100644
--- a/site/src/content/docs/guides/eval-yaml.mdx
+++ b/site/src/content/docs/guides/eval-yaml.mdx
@@ -443,7 +443,7 @@ Each checkpoint accepts:
 | Field        | Type     | Description                                                                       |
 | ------------ | -------- | --------------------------------------------------------------------------------- |
 | `after_turn` | int      | 1-based turn number this checkpoint runs after (initial prompt is turn 1).        |
-| `graders`    | array    | Inline graders, same schema as task `validators:` / eval-level `graders:`.        |
+| `graders`    | array    | Inline graders, same schema as the task-level `graders:` / eval-level `graders:` field. |
 | `on_failure` | string   | `continue` (default) or `stop` — abort remaining turns when this checkpoint fails.|
 
 Outcomes are recorded per-checkpoint on `results.json` under `checkpoints[]`, alongside the final `validations`. `waza gate` still uses final-pass status. Available with `schemaVersion: "1.1"` and above (additive — older 1.0 files load unchanged).
diff --git a/site/src/content/docs/reference/schema-changes.md b/site/src/content/docs/reference/schema-changes.md
index 501b4b77..21b39f61 100644
--- a/site/src/content/docs/reference/schema-changes.md
+++ b/site/src/content/docs/reference/schema-changes.md
@@ -20,8 +20,8 @@ Schema versions use `MAJOR.MINOR` format with no patch component.
 
 - **MINOR** changes are backward-compatible additions, usually optional fields. Readers accept same-major artifacts and warn when they see unknown fields.
 - **MAJOR** changes are breaking. Readers refuse artifacts from a different major version and point to `waza migrate <file>`.
-- Missing `schemaVersion` defaults to `1.0` for backward compatibility with existing `eval.yaml` and `results.json` files.
-- New artifacts should include `schemaVersion: "1.0"` or `"schemaVersion": "1.0"` explicitly.
+- Missing `schemaVersion` defaults to `1.0` for backward compatibility with eval suites authored before the field was introduced. Same-major readers continue to accept `1.0`-defaulted files unchanged.
+- New artifacts should emit the current `schemaVersion` (currently `1.1`). The version is automatically populated by the writer; you only need to set it manually when authoring fixtures or schema-pinned test data.
 
 ## Migration command
 
diff --git a/site/src/content/docs/reference/schema.mdx b/site/src/content/docs/reference/schema.mdx
index c8d31e55..c819374d 100644
--- a/site/src/content/docs/reference/schema.mdx
+++ b/site/src/content/docs/reference/schema.mdx
@@ -574,7 +574,7 @@ Per-turn graders that run at specific turn boundaries during multi-turn conversa
 | Field        | Type    | Required | Description                                                                                  |
 |--------------|---------|----------|----------------------------------------------------------------------------------------------|
 | `after_turn` | integer | yes      | 1-based turn number to grade after. Turn 1 is the initial prompt; follow-ups are 2, 3, ...   |
-| `graders`    | array   | yes      | Inline graders (same schema as task `validators:`).                                          |
+| `graders`    | array   | yes      | Inline graders (same schema as the task-level `graders:` field).                             |
 | `on_failure` | string  | no       | `continue` (default) or `stop`. `stop` aborts remaining turns when this checkpoint fails.    |
 
 ```yaml

From b1fa9bc1e99c09ee713743bb8cd6c97e3e306429 Mon Sep 17 00:00:00 2001
From: Copilot <223556219+Copilot@users.noreply.github.com>
Date: Sun, 28 Jun 2026 08:18:03 -0400
Subject: [PATCH 3/3] chore: gitignore .impeccable/ cache directory

---
 .gitignore                  | 1 +
 .impeccable/hook.cache.json | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)
 delete mode 100644 .impeccable/hook.cache.json

diff --git a/.gitignore b/.gitignore
index 80604014..f83798ca 100644
--- a/.gitignore
+++ b/.gitignore
@@ -120,3 +120,4 @@ docs/research/
 .squad/sessions/
 .squad/.cache/
 .squad-workstream
+.impeccable/
diff --git a/.impeccable/hook.cache.json b/.impeccable/hook.cache.json
deleted file mode 100644
index ebc7c5d0..00000000
--- a/.impeccable/hook.cache.json
+++ /dev/null
@@ -1 +0,0 @@
-{"version":1,"sessions":{}}
\ No newline at end of file