feat: schema versioning policy (closes #368) (#382)

* feat: add schema versioning policy #368

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: harden schema migration guidance #368

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* test: stabilize inline grader duration check #368

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: detect schema artifact json shape #368

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* fix: skip incompatible stored results #368

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: spboyer

README.md

@@ -146,6 +146,9 @@ waza grade eval.yaml --results results.json
 # Compare results across models
 waza compare results-gpt4.json results-sonnet.json
 
+# Check whether a schema artifact needs migration
+waza migrate eval.yaml
+
 # Generate eval coverage grid
 waza coverage --format markdown
 
@@ -409,6 +412,15 @@ Compare results from multiple evaluation runs side by side — per-task score de
 |------|-------|-------------|
 | `--format <fmt>` | `-f` | Output format: `table` or `json` (default: `table`) |
 
+### `waza migrate <file>`
+
+Check a public schema artifact and migrate it to the current schema version when a future major schema requires it. The current schema is `1.0`, so v1 `eval.yaml` and `results.json` files are already current and no file changes are made.
+
+```bash
+waza migrate eval.yaml
+waza migrate results.json
+```
+
 ### `waza coverage [root]`
 
 Generate a skill-to-eval coverage grid showing which skills are fully covered, partially covered, or missing evals.
@@ -931,6 +943,7 @@ skills/                Example skills
 ```yaml
 name: my-eval
 skill: my-skill
+schemaVersion: "1.0"
 version: "1.0"
 
 config:
@@ -1002,6 +1015,8 @@ tasks:
 # range: [1, 10]  # Only include rows 1-10 (0-indexed, skips header)
 ```
 
+`schemaVersion` uses `MAJOR.MINOR` format and defaults to `1.0` when omitted for backward compatibility. Readers allow same-major minor additions with warnings for unknown fields, but reject different majors with a hint to run `waza migrate <file>`.
+
 ### Custom Input Variables
 
 Use the `inputs` section to define key-value variables available throughout your evaluation as `{{.Vars.key}}`:

cmd/waza/cmd_compare.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
-	"os"
 	"strings"
 
 	"github.com/microsoft/waza/internal/models"
@@ -79,15 +78,7 @@ func compareCommandE(_ *cobra.Command, args []string) error {
 }
 
 func loadOutcomeFile(path string) (*models.EvaluationOutcome, error) {
-	data, err := os.ReadFile(path)
-	if err != nil {
-		return nil, err
-	}
-	var outcome models.EvaluationOutcome
-	if err := json.Unmarshal(data, &outcome); err != nil {
-		return nil, err
-	}
-	return &outcome, nil
+	return models.LoadEvaluationOutcome(path)
 }
 
 func buildComparisonReport(files []string, outcomes []*models.EvaluationOutcome) *comparisonReport {

cmd/waza/cmd_grade.go

@@ -82,8 +82,8 @@ func runGrade(ctx context.Context, w, errW io.Writer, specPath, taskID, resultsF
 	if readErr != nil {
 		return fmt.Errorf("failed to read results file: %w", readErr)
 	}
-	var outcome models.EvaluationOutcome
-	if jsonErr := json.Unmarshal(data, &outcome); jsonErr != nil {
+	outcome, jsonErr := models.ParseEvaluationOutcome(data, resultsFile)
+	if jsonErr != nil {
 		return fmt.Errorf("failed to parse results JSON: %w", jsonErr)
 	}
 
@@ -181,7 +181,7 @@ func runGrade(ctx context.Context, w, errW io.Writer, specPath, taskID, resultsF
 			}
 		}
 
-		graded := orchestration.RegradeOutcome(&outcome, finalOutcomes, effectiveJudgeModel)
+		graded := orchestration.RegradeOutcome(outcome, finalOutcomes, effectiveJudgeModel)
 		if err := saveOutcome(graded, outputFile); err != nil {
 			return fmt.Errorf("failed to save graded outcome: %w", err)
 		}

cmd/waza/cmd_migrate.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/microsoft/waza/internal/models"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
+)
+
+func newMigrateCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "migrate <file>",
+		Short: "Migrate a waza schema artifact to the current schema version",
+		Long: `Migrate a waza schema artifact to the current schema version.
+
+The current schema version is 1.0, so v1 artifacts are already current and the
+command performs no file changes. Future major schema versions will add explicit
+migration steps here.`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runMigrate(cmd.OutOrStdout(), args[0])
+		},
+	}
+	return cmd
+}
+
+func runMigrate(out io.Writer, path string) error {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("reading %s: %w", path, err)
+	}
+
+	artifact, version, err := readArtifactSchemaVersion(path, data)
+	if err != nil {
+		return err
+	}
+
+	version, err = models.ValidateSchemaVersion(artifact, path, version)
+	if err != nil {
+		return err
+	}
+
+	if version != models.CurrentSchemaVersion {
+		_, err = fmt.Fprintf(out, "%s uses schemaVersion %s, which is compatible with waza schemaVersion %s; no migration needed.\n", artifact, version, models.CurrentSchemaVersion)
+		return err
+	}
+
+	_, err = fmt.Fprintf(out, "%s is already compatible with schemaVersion %s; no migration needed.\n", artifact, models.CurrentSchemaVersion)
+	return err
+}
+
+func readArtifactSchemaVersion(path string, data []byte) (artifact string, version string, err error) {
+	switch filepath.Base(path) {
+	case "eval.yaml", "eval.yml":
+		artifact = "eval.yaml"
+		var header struct {
+			SchemaVersion string `yaml:"schemaVersion"`
+		}
+		if err := yaml.Unmarshal(data, &header); err != nil {
+			return "", "", fmt.Errorf("parsing %s: %w", path, err)
+		}
+		return artifact, header.SchemaVersion, nil
+	default:
+		if filepath.Ext(path) == ".json" {
+			version, ok, err := models.ProbeEvaluationOutcomeSchemaVersion(data)
+			if err != nil {
+				return "", "", fmt.Errorf("parsing %s: %w", path, err)
+			}
+			if !ok {
+				return "", "", fmt.Errorf("unsupported JSON schema artifact %s: expected a results.json object with top-level eval_id, eval_name, summary, or tasks", path)
+			}
+			return "results.json", version, nil
+		}
+	}
+	return "", "", fmt.Errorf("unsupported schema artifact %s: expected eval.yaml, eval.yml, or a JSON results artifact", path)
+}

cmd/waza/cmd_migrate_test.go

@@ -0,0 +1,81 @@
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMigrateCommandNoOpForCurrentMajor(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "eval.yaml")
+	require.NoError(t, os.WriteFile(path, []byte("schemaVersion: \"1.0\"\n"), 0o644))
+
+	var out bytes.Buffer
+	err := runMigrate(&out, path)
+
+	require.NoError(t, err)
+	require.Contains(t, out.String(), "already compatible with schemaVersion 1.0")
+}
+
+func TestMigrateCommandRejectsIncompatibleMajor(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "results.json")
+	require.NoError(t, os.WriteFile(path, []byte(`{"schemaVersion":"2.0","tasks":[]}`), 0o644))
+
+	var out bytes.Buffer
+	err := runMigrate(&out, path)
+
+	require.Error(t, err)
+	require.Contains(t, err.Error(), `waza migrate <file>`)
+	require.Empty(t, out.String())
+}
+
+func TestMigrateCommandDefaultsMissingSchemaVersion(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "results.json")
+	require.NoError(t, os.WriteFile(path, []byte(`{"eval_id":"run-1"}`), 0o644))
+
+	var out bytes.Buffer
+	err := runMigrate(&out, path)
+
+	require.NoError(t, err)
+	require.Contains(t, out.String(), "already compatible with schemaVersion 1.0")
+}
+
+func TestMigrateCommandRejectsUnknownJSONShape(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "settings.json")
+	require.NoError(t, os.WriteFile(path, []byte(`{"schemaVersion":"1.0","name":"not-results"}`), 0o644))
+
+	var out bytes.Buffer
+	err := runMigrate(&out, path)
+
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "unsupported JSON schema artifact")
+	require.Contains(t, err.Error(), "expected a results.json object")
+	require.Empty(t, out.String())
+}
+
+func TestMigrateCommandDetectsResultsJSONByShape(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "custom-name.json")
+	require.NoError(t, os.WriteFile(path, []byte(`{"schemaVersion":"1.0","tasks":[]}`), 0o644))
+
+	var out bytes.Buffer
+	err := runMigrate(&out, path)
+
+	require.NoError(t, err)
+	require.Contains(t, out.String(), "results.json is already compatible")
+}
+
+func TestMigrateCommandMissingFile(t *testing.T) {
+	var out bytes.Buffer
+	err := runMigrate(&out, filepath.Join(t.TempDir(), "missing.json"))
+
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "reading")
+}

cmd/waza/root.go

@@ -70,6 +70,7 @@ performance against predefined test cases.`,
 	cmd.AddCommand(newQualityCommand())
 	cmd.AddCommand(newUpdateCommand())
 	cmd.AddCommand(newSpecCommand())
+	cmd.AddCommand(newMigrateCommand())
 
 	return cmd
 }

docs/PRD.md

@@ -69,6 +69,7 @@ Port existing Python waza functionality to Go for single-binary distribution.
 | E1-06 | As a developer, I can execute against Copilot SDK | Full integration with streaming responses |
 | E1-07 | As a developer, I can use verbose mode for debugging | Real-time conversation display |
 | E1-08 | As a developer, I can save transcripts for analysis | JSON log output with full conversation |
+| E1-09 | As a developer, public artifacts are schema-versioned | `eval.yaml` and `results.json` include `schemaVersion`; readers warn on same-major drift and reject cross-major versions with a migration hint |
 
 ### Epic 2: Sensei Engine (P0)
 

examples/code-explainer/eval.yaml

@@ -9,6 +9,7 @@ description: |
   - Code patterns (recursion, async/await, comprehensions, joins)
 
 skill: code-explainer
+schemaVersion: "1.0"
 version: "1.0"
 
 config:

examples/custom-agent/eval.yaml

@@ -1,6 +1,7 @@
 name: security-reviewer-eval
 description: Evaluates the security-reviewer custom agent
 skill: security-reviewer
+schemaVersion: "1.0"
 version: "1.0"
 
 config:

examples/grader-showcase/eval.yaml

@@ -16,6 +16,7 @@ description: |
   of configuration options and use cases.
 
 skill: general-coding-assistant
+schemaVersion: "1.0"
 version: "1.0"
 
 config: