feat: synthetic test case generation flags for waza suggest (#357)

Extend `waza suggest` with steerable generation and merge-safe apply:

- New flags: --count, --focus, --force (in addition to existing
  --dry-run / --apply / --model / --output-dir / --format)
- Per-case confidence (0-1) and rationale referencing the SKILL.md span
- --apply is merge-safe: existing eval.yaml is preserved (new task
  files picked up via existing tasks: glob), and existing task /
  fixture files (by path or task id) are never overwritten unless
  --force is also passed
- Generated tasks are schema-validated via internal/validation before
  being written to disk
- Focus categories: triggers, negative-triggers, edge-fixtures,
  do-not-use-for, parameters
- Tests cover focus directives, count guidance, overwrite safety
  (with and without --force), duplicate task id detection (in-batch
  and against existing tasks), and schema validation failures
- README.md and site CLI reference updated with new flags + worked
  examples

Closes #357

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: Copilot

README.md

@@ -566,17 +566,30 @@ Use an LLM to analyze `SKILL.md` and generate suggested evaluation artifacts.
 | `--model <model>` | Model to use for suggestions (default: project default model) |
 | `--dry-run` | Print suggested output to stdout (default) |
 | `--apply` | Write files to disk |
+| `--force` | Allow `--apply` to overwrite existing eval/task/fixture files (requires `--apply`) |
+| `--count <n>` | Generate exactly N tasks (default: at least 3 + 1 negative) |
+| `--focus <category>` | Steer generation toward one of `triggers`, `negative-triggers`, `edge-fixtures`, `do-not-use-for`, `parameters` |
 | `--output-dir <dir>` | Output directory (default: `<skill-path>/evals`) |
 | `--format yaml\|json` | Output format (default: `yaml`) |
 
+Each generated task entry carries a `confidence` score in `[0, 1]` and a `rationale` string pointing to the SKILL.md span it was derived from. Both appear in dry-run output but are kept outside the written task YAML (the task schema rejects unknown fields).
+
+`--apply` is merge-safe: an existing `eval.yaml` is never overwritten (new task files are picked up by its existing `tasks:` glob), and existing task / fixture files (by path or by task `id`) cause `--apply` to fail unless `--force` is also passed.
+
 **Examples:**
 ```bash
 # Preview generated eval/task/fixture files as YAML
 waza suggest skills/code-explainer --dry-run
 
+# Generate exactly 5 negative-trigger tasks and merge them into the existing suite
+waza suggest skills/code-explainer --focus negative-triggers --count 5 --apply
+
 # Write generated files to disk
 waza suggest skills/code-explainer --apply
 
+# Overwrite a previously generated suite
+waza suggest skills/code-explainer --apply --force
+
 # Print JSON-formatted suggestion payload
 waza suggest skills/code-explainer --format json
 ```

cmd/waza/cmd_suggest.go

@@ -25,8 +25,11 @@ type suggestFlags struct {
 	model     string
 	dryRun    bool
 	apply     bool
+	force     bool
 	outputDir string
 	format    string
+	count     int
+	focus     string
 }
 
 func newSuggestCommand() *cobra.Command {
@@ -44,7 +47,12 @@ func newSuggestCommand() *cobra.Command {
 
 This command is experimental. Because an LLM generates suggestions, they should be
 reviewed by a human before applying. By default, suggestions are printed to stdout
-(--dry-run). Use --apply to write suggested eval.yaml, tasks, and fixtures to disk.`,
+(--dry-run). Use --apply to write suggested eval.yaml, tasks, and fixtures to disk.
+
+Use --count to control how many cases are proposed, --focus to steer generation
+toward a category (` + strings.Join(suggestpkg.AvailableFocusCategories(), ", ") + `),
+and --force to overwrite existing files (default is merge-safe: existing eval.yaml
+and task ids are preserved).`,
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return runSuggestCommand(cmd, args[0], flags)
@@ -55,8 +63,11 @@ reviewed by a human before applying. By default, suggestions are printed to stdo
 	cmd.Flags().StringVar(&flags.model, "model", flags.model, "Model to use for suggestions")
 	cmd.Flags().BoolVar(&flags.dryRun, "dry-run", true, "Print suggestions to stdout without writing files")
 	cmd.Flags().BoolVar(&flags.apply, "apply", false, "Write suggested files to disk")
+	cmd.Flags().BoolVar(&flags.force, "force", false, "Overwrite existing files and duplicate task ids (use with --apply)")
 	cmd.Flags().StringVar(&flags.outputDir, "output-dir", "", "Directory for output (default: <skill-path>/evals)")
 	cmd.Flags().StringVar(&flags.format, "format", "yaml", "Output format: yaml|json")
+	cmd.Flags().IntVar(&flags.count, "count", 0, "Number of test cases to propose (0 = model default)")
+	cmd.Flags().StringVar(&flags.focus, "focus", "", "Steer generation toward a category: "+strings.Join(suggestpkg.AvailableFocusCategories(), "|"))
 
 	return cmd
 }
@@ -65,12 +76,21 @@ func runSuggestCommand(cmd *cobra.Command, skillPath string, flags *suggestFlags
 	if flags.format != "yaml" && flags.format != "json" {
 		return fmt.Errorf("invalid format %q: must be yaml or json", flags.format)
 	}
+	if err := suggestpkg.ValidateFocus(flags.focus); err != nil {
+		return err
+	}
+	if flags.count < 0 {
+		return fmt.Errorf("invalid --count %d: must be >= 0", flags.count)
+	}
 	if flags.apply {
 		flags.dryRun = false
 	}
 	if !flags.apply && !flags.dryRun {
 		return errors.New("either --dry-run or --apply must be enabled")
 	}
+	if flags.force && !flags.apply {
+		return errors.New("--force requires --apply")
+	}
 
 	outputDir := flags.outputDir
 	if outputDir == "" {
@@ -93,6 +113,8 @@ func runSuggestCommand(cmd *cobra.Command, skillPath string, flags *suggestFlags
 	suggestion, err := suggestpkg.Generate(cmd.Context(), engine, suggestpkg.Options{
 		SkillPath:  skillPath,
 		GraderDocs: waza.GraderDocsFS,
+		Count:      flags.count,
+		Focus:      flags.focus,
 	})
 	if err != nil {
 		return err
@@ -110,7 +132,7 @@ func runSuggestCommand(cmd *cobra.Command, skillPath string, flags *suggestFlags
 		return nil
 	}
 
-	written, err := suggestion.WriteToDir(outputDir)
+	written, err := suggestion.WriteToDir(outputDir, suggestpkg.WriteOptions{Force: flags.force})
 	if err != nil {
 		return err
 	}

internal/suggest/prompt.go

@@ -68,6 +68,41 @@ type promptData struct {
 	ContentSummary string
 	GraderTypes    string
 	SkillContent   string
+	// Count is the desired number of test cases (<= 0 means use default).
+	Count int
+	// Focus is an optional focus category that steers the kinds of cases
+	// the LLM should emit (e.g. "negative-triggers"). Empty means balanced.
+	Focus string
+}
+
+// focusDirective returns prompt text that steers generation toward a focus
+// category. Returns "" when focus is empty.
+func focusDirective(focus string) string {
+	switch FocusCategory(focus) {
+	case "":
+		return ""
+	case FocusTriggers:
+		return "FOCUS: Generate tasks that exercise the skill's positive trigger phrases " +
+			"(the 'USE FOR' list). Each task should match a distinct trigger so we can verify " +
+			"the skill activates when it should."
+	case FocusNegativeTriggers:
+		return "FOCUS: Generate tasks that look superficially like triggers but should NOT " +
+			"invoke this skill. Use skill_invocation graders with forbidden_skills to assert " +
+			"the skill stays out of these cases."
+	case FocusEdgeFixtures:
+		return "FOCUS: Generate tasks that stress edge cases in the input fixtures — empty " +
+			"files, very large files, malformed inputs, unicode, ambiguous content. Each task " +
+			"should carry a realistic edge-case fixture under fixtures/."
+	case FocusDoNotUseFor:
+		return "FOCUS: Generate tasks drawn from the skill's 'DO NOT USE FOR' anti-trigger " +
+			"list. Each task must assert the skill refuses or routes elsewhere (use " +
+			"skill_invocation forbidden_skills, or text graders that check for refusal language)."
+	case FocusParameters:
+		return "FOCUS: Generate tasks that vary the parameters and tool arguments the skill " +
+			"would supply — required vs optional, valid vs invalid values, boundary numerics. " +
+			"Use tool_constraint or action_sequence graders where appropriate."
+	}
+	return ""
 }
 
 // renderSelectionPrompt builds the pass-1 prompt that asks the LLM
@@ -102,6 +137,8 @@ func renderImplementationPrompt(data promptData, graderDocs string) string {
 	b.WriteString("  <full eval.yaml content>\n")
 	b.WriteString("tasks:\n")
 	b.WriteString("  - path: tasks/<task-file>.yaml\n")
+	b.WriteString("    confidence: 0.0-1.0\n")
+	b.WriteString("    rationale: <one sentence pointing to the SKILL.md span this case came from>\n")
 	b.WriteString("    content: |\n")
 	b.WriteString("      <task yaml>\n")
 	b.WriteString("fixtures:\n")
@@ -113,11 +150,21 @@ func renderImplementationPrompt(data promptData, graderDocs string) string {
 	b.WriteString("- Each grader entry MUST be an object with at least 'type' and 'name' fields. NEVER use bare strings like '- grader_name'. Always use '- name: grader_name' with a 'type' field.\n")
 	b.WriteString("- Include required config fields for each grader type (see grader documentation below).\n")
 	b.WriteString("- For skill_invocation graders, use required_skills + mode (exact_match, in_order, any_order) for positive checks, forbidden_skills for negative checks, or both together.\n")
-	b.WriteString("- Include at least 3 diverse tasks and at least 1 negative/anti-trigger task.\n")
+	if data.Count > 0 {
+		fmt.Fprintf(&b, "- Generate EXACTLY %d tasks. Do not generate more or fewer.\n", data.Count)
+	} else {
+		b.WriteString("- Include at least 3 diverse tasks and at least 1 negative/anti-trigger task.\n")
+	}
 	b.WriteString("- Use grader types from the allowed list only.\n")
 	b.WriteString("- Keep task IDs deterministic and kebab-case.\n")
 	b.WriteString("- Task YAML must use inputs: { prompt: ... } (do not use a top-level prompt field).\n")
-	b.WriteString("- Make fixtures minimal and realistic for the tasks.\n\n")
+	b.WriteString("- Task YAML must NOT include 'confidence' or 'rationale' inside the task content; those belong on the outer suggestion entry and will be stripped before writing.\n")
+	b.WriteString("- Each task entry MUST carry a 'confidence' float in [0,1] and a 'rationale' string citing the SKILL.md span (e.g. \"matches USE FOR: summarize bullet 2\").\n")
+	b.WriteString("- Make fixtures minimal and realistic for the tasks.\n")
+	if directive := focusDirective(data.Focus); directive != "" {
+		b.WriteString("- " + directive + "\n")
+	}
+	b.WriteString("\n")
 	b.WriteString("Skill metadata:\n")
 	fmt.Fprintf(&b, "- Name: %s\n", data.SkillName)
 	fmt.Fprintf(&b, "- Description: %s\n", data.Description)

internal/suggest/resolve_test.go

@@ -163,7 +163,7 @@ func TestWriteToDir_EmptyTaskPath(t *testing.T) {
 	}
 
 	dir := t.TempDir()
-	written, err := s.WriteToDir(dir)
+	written, err := s.WriteToDir(dir, WriteOptions{})
 	require.NoError(t, err)
 	assert.Len(t, written, 2) // eval.yaml + auto-named task
 }
@@ -177,7 +177,7 @@ func TestWriteToDir_EmptyFixturePath(t *testing.T) {
 	}
 
 	dir := t.TempDir()
-	written, err := s.WriteToDir(dir)
+	written, err := s.WriteToDir(dir, WriteOptions{})
 	require.NoError(t, err)
 	assert.Len(t, written, 2) // eval.yaml + auto-named fixture
 }
@@ -195,7 +195,7 @@ func TestWriteToDir_AbsolutePathRejected(t *testing.T) {
 	}
 
 	dir := t.TempDir()
-	_, err := s.WriteToDir(dir)
+	_, err := s.WriteToDir(dir, WriteOptions{})
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "invalid generated path")
 }
@@ -209,7 +209,7 @@ func TestWriteToDir_TraversalPathRejected(t *testing.T) {
 	}
 
 	dir := t.TempDir()
-	_, err := s.WriteToDir(dir)
+	_, err := s.WriteToDir(dir, WriteOptions{})
 	require.Error(t, err)
 	assert.Contains(t, err.Error(), "invalid generated path")
 }
@@ -220,7 +220,7 @@ func TestWriteToDir_InvalidEvalYAML(t *testing.T) {
 	}
 
 	dir := t.TempDir()
-	_, err := s.WriteToDir(dir)
+	_, err := s.WriteToDir(dir, WriteOptions{})
 	require.Error(t, err)
 }