feat: enhance suggest case generation #357

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: Copilot

README.md

@@ -127,7 +127,7 @@ waza new task from-prompt "Explain this code and suggest fixes" evals/code-expla
 waza check skills/my-skill
 
 # Suggest an eval suite from SKILL.md
-waza suggest skills/my-skill --dry-run
+waza suggest skills/my-skill --count 5 --focus edge-fixtures --dry-run
 waza suggest skills/my-skill --apply
 
 # Note: 'generate' is available as an alias for 'new' (see below for new command)
@@ -564,23 +564,31 @@ Use an LLM to analyze `SKILL.md` and generate suggested evaluation artifacts.
 | Flag | Description |
 |------|-------------|
 | `--model <model>` | Model to use for suggestions (default: project default model) |
+| `--count <n>` | Number of task cases to propose (default: `3`) |
+| `--focus <category>` | Focus category: `triggers`, `negative-triggers`, `edge-fixtures`, `do-not-use-for`, or `parameters` |
 | `--dry-run` | Print suggested output to stdout (default) |
 | `--apply` | Write files to disk |
+| `--force` | Overwrite conflicting generated files or task IDs when applying |
 | `--output-dir <dir>` | Output directory (default: `<skill-path>/evals`) |
 | `--format yaml\|json` | Output format (default: `yaml`) |
 
 **Examples:**
 ```bash
 # Preview generated eval/task/fixture files as YAML
-waza suggest skills/code-explainer --dry-run
+waza suggest skills/code-explainer --count 5 --focus negative-triggers --dry-run
 
 # Write generated files to disk
 waza suggest skills/code-explainer --apply
 
+# Replace conflicting generated task files after review
+waza suggest skills/code-explainer --apply --force
+
 # Print JSON-formatted suggestion payload
 waza suggest skills/code-explainer --format json
 ```
 
+Dry-run output includes per-task `confidence` and `rationale` fields so authors can review why each case was proposed. Apply mode validates generated eval and task YAML against the built-in schemas before writing, merges task references into an existing `eval.yaml`, and refuses to overwrite an existing fixture, task file, or task `id` unless `--force` is set.
+
 ### `waza tokens count [paths...]`
 
 Count tokens in markdown files. Paths may be files or directories (scanned recursively for `.md`/`.mdx`).

cmd/waza/cmd_suggest.go

@@ -25,8 +25,11 @@ type suggestFlags struct {
 	model     string
 	dryRun    bool
 	apply     bool
+	force     bool
 	outputDir string
 	format    string
+	count     int
+	focus     string
 }
 
 func newSuggestCommand() *cobra.Command {
@@ -35,6 +38,7 @@ func newSuggestCommand() *cobra.Command {
 		model:  defaultModel,
 		dryRun: true,
 		format: "yaml",
+		count:  suggestpkg.DefaultCaseCount,
 	}
 
 	cmd := &cobra.Command{
@@ -55,8 +59,11 @@ reviewed by a human before applying. By default, suggestions are printed to stdo
 	cmd.Flags().StringVar(&flags.model, "model", flags.model, "Model to use for suggestions")
 	cmd.Flags().BoolVar(&flags.dryRun, "dry-run", true, "Print suggestions to stdout without writing files")
 	cmd.Flags().BoolVar(&flags.apply, "apply", false, "Write suggested files to disk")
+	cmd.Flags().BoolVar(&flags.force, "force", false, "Overwrite conflicting generated files and task ids when applying")
 	cmd.Flags().StringVar(&flags.outputDir, "output-dir", "", "Directory for output (default: <skill-path>/evals)")
 	cmd.Flags().StringVar(&flags.format, "format", "yaml", "Output format: yaml|json")
+	cmd.Flags().IntVar(&flags.count, "count", suggestpkg.DefaultCaseCount, "Number of task cases to propose")
+	cmd.Flags().StringVar(&flags.focus, "focus", "", fmt.Sprintf("Focus category: %s", strings.Join(suggestFocusValues(), "|")))
 
 	return cmd
 }
@@ -65,6 +72,13 @@ func runSuggestCommand(cmd *cobra.Command, skillPath string, flags *suggestFlags
 	if flags.format != "yaml" && flags.format != "json" {
 		return fmt.Errorf("invalid format %q: must be yaml or json", flags.format)
 	}
+	if flags.count < 1 {
+		return fmt.Errorf("invalid count %d: must be at least 1", flags.count)
+	}
+	focus, err := suggestpkg.ParseFocusCategory(flags.focus)
+	if err != nil {
+		return err
+	}
 	if flags.apply {
 		flags.dryRun = false
 	}
@@ -93,6 +107,8 @@ func runSuggestCommand(cmd *cobra.Command, skillPath string, flags *suggestFlags
 	suggestion, err := suggestpkg.Generate(cmd.Context(), engine, suggestpkg.Options{
 		SkillPath:  skillPath,
 		GraderDocs: waza.GraderDocsFS,
+		Count:      flags.count,
+		Focus:      focus,
 	})
 	if err != nil {
 		return err
@@ -110,7 +126,7 @@ func runSuggestCommand(cmd *cobra.Command, skillPath string, flags *suggestFlags
 		return nil
 	}
 
-	written, err := suggestion.WriteToDir(outputDir)
+	written, err := suggestion.WriteToDirWithOptions(outputDir, suggestpkg.WriteOptions{Force: flags.force})
 	if err != nil {
 		return err
 	}
@@ -134,6 +150,15 @@ func runSuggestCommand(cmd *cobra.Command, skillPath string, flags *suggestFlags
 	return nil
 }
 
+func suggestFocusValues() []string {
+	categories := suggestpkg.AllFocusCategories()
+	values := make([]string, 0, len(categories))
+	for _, category := range categories {
+		values = append(values, string(category))
+	}
+	return values
+}
+
 func defaultSuggestOutputDir(skillPath string) string {
 	candidate := skillPath
 	if strings.EqualFold(filepath.Base(skillPath), "SKILL.md") {

cmd/waza/cmd_suggest_test.go

@@ -18,21 +18,25 @@ type suggestTestEngine struct {
 	err        error
 	callIdx    int
 	initCalled bool
+	requests   []string
 }
 
 func (e *suggestTestEngine) Initialize(context.Context) error {
 	e.initCalled = true
 	return nil
 }
 
-func (e *suggestTestEngine) Execute(_ context.Context, _ *execution.ExecutionRequest) (*execution.ExecutionResponse, error) {
+func (e *suggestTestEngine) Execute(_ context.Context, req *execution.ExecutionRequest) (*execution.ExecutionResponse, error) {
 	if !e.initCalled {
 		return nil, fmt.Errorf("initialize was not called before Execute!")
 	}
 
 	if e.err != nil {
 		return nil, e.err
 	}
+	if req != nil {
+		e.requests = append(e.requests, req.Message)
+	}
 	idx := e.callIdx
 	e.callIdx++
 	if idx < len(e.outputs) {
@@ -90,6 +94,8 @@ func TestSuggestCommand_DryRunYAML(t *testing.T) {
     - "tasks/*.yaml"
 tasks:
   - path: tasks/basic.yaml
+    confidence: 0.9
+    rationale: "SKILL.md overview"
     content: |
       id: basic-001
       name: Basic
@@ -117,6 +123,65 @@ tasks:
 	require.NoFileExists(t, filepath.Join(skillDir, "evals", "eval.yaml"))
 }
 
+func TestSuggestCommand_CountFocusAndCaseMetadata(t *testing.T) {
+	skillDir := writeSuggestSkill(t)
+	engineOutput := `eval_yaml: |
+  name: generated-eval
+  description: generated
+  skill: suggest-skill
+  version: "1.0"
+  config:
+    trials_per_task: 1
+    timeout_seconds: 120
+    parallel: false
+    executor: mock
+    model: test
+  graders:
+    - type: text
+      name: has_text
+      config:
+        contains:
+          - hello
+  metrics:
+    - name: completion
+      weight: 1.0
+      threshold: 0.8
+  tasks:
+    - "tasks/*.yaml"
+tasks:
+  - path: tasks/params.yaml
+    confidence: 0.77
+    rationale: "SKILL.md parameters section"
+    content: |
+      id: params-001
+      name: Parameters
+      inputs:
+        prompt: "hello"
+`
+
+	selectionOutput := "graders:\n  - text\n"
+	engine := &suggestTestEngine{outputs: []string{selectionOutput, engineOutput}}
+
+	orig := newSuggestEngine
+	newSuggestEngine = func(string) execution.AgentEngine {
+		return engine
+	}
+	t.Cleanup(func() { newSuggestEngine = orig })
+
+	cmd := newSuggestCommand()
+	var out bytes.Buffer
+	cmd.SetOut(&out)
+	cmd.SetErr(&out)
+	cmd.SetArgs([]string{skillDir, "--count", "1", "--focus", "parameters"})
+
+	require.NoError(t, cmd.Execute())
+	require.Contains(t, out.String(), "confidence: 0.77")
+	require.Contains(t, out.String(), "rationale: SKILL.md parameters section")
+	require.Len(t, engine.requests, 2)
+	require.Contains(t, engine.requests[1], "Generate exactly 1 task case(s).")
+	require.Contains(t, engine.requests[1], "parameter extraction")
+}
+
 func TestSuggestCommand_ApplyWritesFiles(t *testing.T) {
 	skillDir := writeSuggestSkill(t)
 	engineOutput := `eval_yaml: |
@@ -144,6 +209,8 @@ func TestSuggestCommand_ApplyWritesFiles(t *testing.T) {
     - "tasks/*.yaml"
 tasks:
   - path: tasks/basic.yaml
+    confidence: 0.9
+    rationale: "SKILL.md overview"
     content: |
       id: basic-001
       name: Basic

internal/suggest/prompt.go

@@ -68,12 +68,16 @@ type promptData struct {
 	ContentSummary string
 	GraderTypes    string
 	SkillContent   string
+	Count          int
+	Focus          string
 }
 
 // renderSelectionPrompt builds the pass-1 prompt that asks the LLM
 // to choose appropriate grader types for the skill.
 func renderSelectionPrompt(data promptData) string {
 	var b strings.Builder
+	count := effectivePromptCount(data.Count)
+	focus := effectivePromptFocus(data.Focus)
 	b.WriteString("You are selecting grader types for a waza evaluation suite.\n")
 	b.WriteString("Given the skill description below, choose which grader types are most appropriate.\n\n")
 	b.WriteString("Return ONLY a YAML list of grader type names, one per line, like:\n")
@@ -86,6 +90,8 @@ func renderSelectionPrompt(data promptData) string {
 	fmt.Fprintf(&b, "- Triggers (USE FOR): %s\n", data.Triggers)
 	fmt.Fprintf(&b, "- Anti-triggers (DO NOT USE FOR): %s\n", data.AntiTriggers)
 	fmt.Fprintf(&b, "- Content summary: %s\n\n", data.ContentSummary)
+	fmt.Fprintf(&b, "Requested case count: %d\n", count)
+	fmt.Fprintf(&b, "Requested focus: %s\n\n", focus)
 	b.WriteString("Available grader types:\n")
 	b.WriteString(GraderSummaries())
 	b.WriteString("\n")
@@ -96,12 +102,16 @@ func renderSelectionPrompt(data promptData) string {
 // the full eval YAML, with detailed docs for the selected grader types.
 func renderImplementationPrompt(data promptData, graderDocs string) string {
 	var b strings.Builder
+	count := effectivePromptCount(data.Count)
+	focus := effectivePromptFocus(data.Focus)
 	b.WriteString("You are generating a waza evaluation suite for a skill.\n")
 	b.WriteString("Return ONLY YAML in this exact schema:\n\n")
 	b.WriteString("eval_yaml: |\n")
 	b.WriteString("  <full eval.yaml content>\n")
 	b.WriteString("tasks:\n")
 	b.WriteString("  - path: tasks/<task-file>.yaml\n")
+	b.WriteString("    confidence: <0.0-1.0>\n")
+	b.WriteString("    rationale: <SKILL.md span or phrase that supports this case>\n")
 	b.WriteString("    content: |\n")
 	b.WriteString("      <task yaml>\n")
 	b.WriteString("fixtures:\n")
@@ -113,7 +123,10 @@ func renderImplementationPrompt(data promptData, graderDocs string) string {
 	b.WriteString("- Each grader entry MUST be an object with at least 'type' and 'name' fields. NEVER use bare strings like '- grader_name'. Always use '- name: grader_name' with a 'type' field.\n")
 	b.WriteString("- Include required config fields for each grader type (see grader documentation below).\n")
 	b.WriteString("- For skill_invocation graders, use required_skills + mode (exact_match, in_order, any_order) for positive checks, forbidden_skills for negative checks, or both together.\n")
-	b.WriteString("- Include at least 3 diverse tasks and at least 1 negative/anti-trigger task.\n")
+	b.WriteString("- Make the task set diverse when count allows; include negative/anti-trigger coverage when it fits the requested focus.\n")
+	fmt.Fprintf(&b, "- Generate exactly %d task case(s).\n", count)
+	fmt.Fprintf(&b, "- Focus guidance: %s\n", focus)
+	b.WriteString("- For every task case, include confidence as a number from 0.0 to 1.0 and rationale referencing the SKILL.md heading, phrase, or span that supports the case.\n")
 	b.WriteString("- Use grader types from the allowed list only.\n")
 	b.WriteString("- Keep task IDs deterministic and kebab-case.\n")
 	b.WriteString("- Task YAML must use inputs: { prompt: ... } (do not use a top-level prompt field).\n")
@@ -135,12 +148,27 @@ func renderImplementationPrompt(data promptData, graderDocs string) string {
 		b.WriteString(graderDocs)
 		b.WriteString("\n\n")
 	}
+
 	b.WriteString("Skill content (SKILL.md):\n")
 	b.WriteString(data.SkillContent)
 	b.WriteString("\n")
 	return b.String()
 }
 
+func effectivePromptCount(count int) int {
+	if count <= 0 {
+		return DefaultCaseCount
+	}
+	return count
+}
+
+func effectivePromptFocus(focus string) string {
+	if strings.TrimSpace(focus) == "" {
+		return focusInstruction("")
+	}
+	return focus
+}
+
 // renderPrompt builds a single-pass prompt (used when no grader docs FS is available).
 func renderPrompt(data promptData) string {
 	return renderImplementationPrompt(data, "")

internal/suggest/prompt_test.go

@@ -73,8 +73,9 @@ func TestRenderImplementationPrompt_ContainsRequirements(t *testing.T) {
 	assert.Contains(t, prompt, "NEVER use bare strings")
 	assert.Contains(t, prompt, "required_skills")
 	assert.Contains(t, prompt, "Task YAML must use inputs")
-	assert.Contains(t, prompt, "at least 3 diverse tasks")
-	assert.Contains(t, prompt, "at least 1 negative/anti-trigger task")
+	assert.Contains(t, prompt, "Generate exactly 3 task case(s)")
+	assert.Contains(t, prompt, "confidence")
+	assert.Contains(t, prompt, "rationale")
 }
 
 func TestRenderPrompt_IsSinglePassFallback(t *testing.T) {

internal/suggest/resolve_test.go

@@ -112,7 +112,7 @@ func TestBuildPromptData_ExtractsFields(t *testing.T) {
 	rawContent, sk, err := loadSkill(skillPath)
 	require.NoError(t, err)
 
-	data := buildPromptData(sk, rawContent)
+	data := buildPromptData(sk, rawContent, DefaultCaseCount, "")
 	assert.Equal(t, "build-prompt-skill", data.SkillName)
 	assert.NotEmpty(t, data.Description)
 	assert.Contains(t, data.Triggers, "summarize")
@@ -133,7 +133,7 @@ func TestBuildPromptData_FallbackSkillName(t *testing.T) {
 	rawContent, sk, err := loadSkill(skillPath)
 	require.NoError(t, err)
 
-	data := buildPromptData(sk, rawContent)
+	data := buildPromptData(sk, rawContent, DefaultCaseCount, "")
 	// Name not set in frontmatter, should fall back to parent dir name
 	assert.Equal(t, "my-cool-skill", data.SkillName)
 }
@@ -147,7 +147,7 @@ func TestBuildPromptData_NoTriggers(t *testing.T) {
 	rawContent, sk, err := loadSkill(skillPath)
 	require.NoError(t, err)
 
-	data := buildPromptData(sk, rawContent)
+	data := buildPromptData(sk, rawContent, DefaultCaseCount, "")
 	assert.Equal(t, "none", data.Triggers)
 	assert.Equal(t, "none", data.AntiTriggers)
 }
@@ -158,7 +158,7 @@ func TestWriteToDir_EmptyTaskPath(t *testing.T) {
 	s := &Suggestion{
 		EvalYAML: validEvalYAML(),
 		Tasks: []GeneratedFile{
-			{Path: "", Content: "id: auto\nname: Auto\ninputs:\n  prompt: hi"},
+			{Path: "", Confidence: testConfidence(0.8), Rationale: "SKILL.md overview", Content: "id: auto\nname: Auto\ninputs:\n  prompt: hi"},
 		},
 	}
 
@@ -190,7 +190,7 @@ func TestWriteToDir_AbsolutePathRejected(t *testing.T) {
 	s := &Suggestion{
 		EvalYAML: validEvalYAML(),
 		Tasks: []GeneratedFile{
-			{Path: absPath, Content: "id: x\nname: X\ninputs:\n  prompt: hi"},
+			{Path: absPath, Confidence: testConfidence(0.8), Rationale: "SKILL.md overview", Content: "id: x\nname: X\ninputs:\n  prompt: hi"},
 		},
 	}