eval.yaml

# Example eval.yaml — Agent Eval Harness configuration
#
# Place this file in your project root.
# Run with: /eval-run --model opus

name: my-skill-eval
description: Evaluate the main skill pipeline

# Skill to test
skill: my-skill-name

# Execution — how the skill processes test cases (runner-agnostic)
execution:
  mode: case              # per-case (default) or batch
  arguments: "{prompt}"   # resolved per case from input.yaml fields
  # For batch skills: mode: batch, arguments: "--input batch.yaml --headless"
  # timeout: 3600         # Per-invocation wall-clock timeout (seconds)
  # max_budget_usd: 5.0   # Per-invocation cost cap
  # env:                  # Inject env vars into workspace settings
  #   JIRA_SERVER: http://localhost:8080   # Literal value
  #   JIRA_TOKEN: $JIRA_TOKEN              # $VAR resolved from caller's env

# Runner — agent harness + runner-specific knobs
runner:
  type: claude-code              # Discriminator: claude-code, opencode, etc.
  # effort: high                 # Reasoning effort: low | medium | high | xhigh | max
  # settings: {}                 # Claude Code settings merged into workspace
  # plugin_dirs: []              # Plugin dirs for sub-skills the evaluated skill calls
  # env_strip: [JIRA_TOKEN]      # Env vars to remove from subprocess
  # system_prompt: ""            # Appended to harness system prompt

# Models — defaults for each role (CLI flags override)
models:
  skill: claude-opus-4-6         # Required (or pass --model)
  # subagent: claude-sonnet-4-6  # Defaults to skill model
  judge: claude-opus-4-6         # Used by LLM and pairwise judges
  # hook: claude-sonnet-4-6      # Model for LLM-based AskUserQuestion answering

# Permissions — tool access during headless execution
permissions:
  allow: []            # Tool patterns to allow (empty = all allowed)
  deny:                # Tool patterns to block
    - "mcp__*"         # Block all MCP tools during eval

# MLflow experiment (optional — results logged here)
mlflow:
  experiment: my-skill-eval
  # tracking_uri: sqlite:///mlflow.db   # Override env var for self-contained runs
  # tags: { team: ml }

# Dataset — where test cases live and what they look like
dataset:
  path: eval/dataset/cases
  schema: |
    Each case directory contains:
    - input.yaml: YAML file. The 'prompt' field is the main input to
      the skill. Optionally 'context' with additional context.
    - reference.md: Gold standard output for comparison scoring.

# Inputs — tool interception for headless execution
inputs:
  tools:
    # Auto-answer user questions during eval
    # - match: Questions asked to the user via AskUserQuestion.
    #   prompt: |
    #     Answer based on the test case context. Default to "yes"
    #     for confirmations, "Normal" for priority.

    # Control external service access
    # - match: |
    #     Any interaction with Jira — whether via MCP tools
    #     (mcp__atlassian__*) or Bash scripts calling the Jira API.
    #   prompt: |
    #     Only allow if the target is a test Jira instance or emulator.
    #     Block any requests to production Jira servers.

# Outputs — what the skill produces (files on disk or tool calls)
outputs:
  # File artifacts on disk
  - path: artifacts
    # batch_pattern: "RFE-{n:03d}"  # Map output files to cases in batch mode
    schema: |
      One markdown file per case, named NNN-slug.md where NNN is the
      case number (001, 002, ...).

  # Tool call outputs (for side effects like Jira creation)
  # - tool: mcp__atlassian__create_issue
  #   schema: |
  #     Creates a Jira issue. The tool input contains title,
  #     description, priority, labels.

# Traces — execution data to capture for judges
traces:
  stdout: true    # Capture stdout.log
  stderr: true    # Capture stderr.log
  events: false   # Capture execution events: tool calls, reasoning, results (verbose)
  metrics: true   # Capture exit code, tokens, cost, duration

# Judges — evaluate output quality
judges:
  # Inline code check
  - name: has_content
    description: |
      Check that the generated output is non-empty and has at least
      100 characters of content.
    check: |
      content = outputs["main_content"]
      if len(content.strip()) < 100:
          return False, f"Output too short ({len(content.strip())} chars)"
      return True, f"Output has {len(content.strip())} chars"

  # LLM judge with inline prompt (conditional — skipped when condition is false)
  - name: output_quality
    if: "not annotations.get('skip_quality', False)"  # Skip based on case annotations
    description: |
      Evaluate the quality of the generated output compared to the
      reference. Score 1-5.
    prompt: |
      Compare the generated output against the reference.
      Consider: completeness, clarity, accuracy, and relevance.
      Score 1-5 where 5 is excellent.

  # LLM judge with prompt file and context
  # - name: detailed_quality
  #   description: Detailed quality assessment with rubric
  #   prompt_file: eval/prompts/quality-judge.md
  #   context:
  #     - eval/prompts/scoring-rubric.md
  #     - eval/prompts/domain-guidelines.md

  # External code judge (for complex validation)
  # - name: schema_valid
  #   description: Validate output schema
  #   module: eval.judges.schema_checks
  #   function: check_schema

  # Execution efficiency check (uses trace metrics)
  # - name: cost_reasonable
  #   description: Verify cost stays under $0.50 per case
  #   check: |
  #     cost = outputs.get("cost_usd", 0)
  #     if cost and cost > 0.50:
  #         return False, f"Cost ${cost:.2f} exceeds limit"
  #     return True, f"Cost ${cost:.2f}"

  # Tool call check (uses tool outputs)
  # - name: jira_created
  #   description: Verify the skill created a Jira issue
  #   check: |
  #     calls = outputs.get("tool_calls", [])
  #     jira = [c for c in calls if "create_issue" in c.get("name", "")]
  #     if not jira:
  #         return False, "No Jira issue created"
  #     return True, f"Created issue"

  # Pairwise comparison judge (used with: score.py pairwise --baseline <id>)
  # - name: pairwise
  #   description: Compare two runs and pick the better output
  #   prompt_file: eval/prompts/comparison-judge.md
  #   # model: claude-sonnet-4-6  # Optional override; default is models.judge

# Thresholds for regression detection
thresholds:
  output_quality:
    min_mean: 3.5            # Minimum average score
  has_content:
    min_pass_rate: 1.0       # Minimum fraction of cases passing (0.0–1.0)
  # pairwise:
  #   min_win_rate: 0.6      # Minimum pairwise win rate