test-validator.sh

#!/usr/bin/env bash
# Self-test for validate-skills.sh.
#
# Creates a minimal temporary skill tree, runs the validator against it, and
# asserts that specific checks pass or fail as expected. Exits 0 if all
# assertions hold, 1 if any fail.
#
# Usage:
#   ./scripts/test-validator.sh

set -euo pipefail

REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
VALIDATOR="$REPO_ROOT/scripts/validate-skills.sh"

PASS=0
FAIL=0

# ── helpers ───────────────────────────────────────────────────────────────────

green()  { echo -e "\033[32m$*\033[0m"; }
red()    { echo -e "\033[31m$*\033[0m"; }

assert_contains() {
    local label="$1" pattern="$2" output="$3"
    if echo "$output" | grep -qF "$pattern"; then
        green "  PASS: $label"
        PASS=$((PASS + 1))
    else
        red "  FAIL: $label"
        red "        expected pattern: $pattern"
        FAIL=$((FAIL + 1))
    fi
}

assert_not_contains() {
    local label="$1" pattern="$2" output="$3"
    if ! echo "$output" | grep -qF "$pattern"; then
        green "  PASS: $label"
        PASS=$((PASS + 1))
    else
        red "  FAIL: $label"
        red "        unexpected pattern: $pattern"
        FAIL=$((FAIL + 1))
    fi
}

# ── fixture setup ─────────────────────────────────────────────────────────────

TMPDIR_ROOT=$(mktemp -d)
trap 'rm -rf "$TMPDIR_ROOT"' EXIT

SKILLS_DIR="$TMPDIR_ROOT/skills"
SCRIPTS_DIR="$TMPDIR_ROOT/scripts"
mkdir -p "$SKILLS_DIR" "$SCRIPTS_DIR"

# Copy the validator into the temp tree so REPO_ROOT resolves to TMPDIR_ROOT
cp "$VALIDATOR" "$SCRIPTS_DIR/validate-skills.sh"

# Replace the allowlist in the copied validator with just the one cycle present
# in our fixture (good-skill|other-skill). This avoids 100+ stale-entry failures
# from the real allowlist being checked against a 3-skill tree.
perl -i -0pe 's/(    local ALLOWLIST=")(\n.*?)^(")/\1\ngood-skill|other-skill\n\3/ms' \
    "$SCRIPTS_DIR/validate-skills.sh"

# Minimal valid discovery.yaml (bad-skill intentionally absent — tests missing-from-discovery)
cat > "$TMPDIR_ROOT/discovery.yaml" <<YAML
skills:
  - skill: good-skill
    priority: primary
    use_when: user asks about good-skill
    triggers:
      - good skill
      - good-skill usage
      - how to use good-skill
  - skill: other-skill
    priority: primary
    use_when: user asks about other-skill
    triggers:
      - other skill
      - other-skill usage
      - how to use other-skill
YAML

# Minimal routing.yaml — good-skill <-> other-skill bidirectional (allowlisted above)
cat > "$TMPDIR_ROOT/routing.yaml" <<YAML
routes:
  - from: good-skill
    to: other-skill
    condition: "user needs other-skill"
    type: topic
  - from: other-skill
    to: good-skill
    condition: "user needs good-skill"
    type: topic
YAML

# ── fixture: good-skill (valid) ───────────────────────────────────────────────

mkdir -p "$SKILLS_DIR/good-skill/examples"
cat > "$SKILLS_DIR/good-skill/SKILL.md" <<MD
---
name: good-skill
summary: A well-formed skill for testing.
metadata:
  last_verified: "2026-05"
  handoff:  # generated from routing.yaml — do not edit manually
    - condition: "user needs other-skill"
      skill: other-skill
---

# Good Skill

This skill is valid.
MD

cat > "$SKILLS_DIR/good-skill/examples/examples.md" <<MD
---
skill: good-skill
cases:
  - id: basic-question
    input: "How do I use good-skill?"
    expect:
      - good
      - skill
---

## Example 1

**Input:** How do I use good-skill?

**Output:**
Use good-skill like this.
MD

# ── fixture: other-skill (valid) ─────────────────────────────────────────────

mkdir -p "$SKILLS_DIR/other-skill/examples"
cat > "$SKILLS_DIR/other-skill/SKILL.md" <<MD
---
name: other-skill
summary: Another well-formed skill for testing.
metadata:
  last_verified: "2026-05"
  handoff:  # generated from routing.yaml — do not edit manually
    - condition: "user needs good-skill"
      skill: good-skill
---

# Other Skill

This skill is also valid.
MD

cat > "$SKILLS_DIR/other-skill/examples/examples.md" <<MD
---
skill: other-skill
cases:
  - id: basic-question
    input: "How do I use other-skill?"
    expect:
      - other
      - skill
---

## Example 1

**Input:** How do I use other-skill?

**Output:**
Use other-skill like this.
MD

# ── fixture: bad-skill (intentionally broken) ────────────────────────────────

mkdir -p "$SKILLS_DIR/bad-skill/examples"
cat > "$SKILLS_DIR/bad-skill/SKILL.md" <<MD
---
name: wrong-name
summary:
metadata:
  last_verified: "2026-05"
---

# Bad Skill
MD

cat > "$SKILLS_DIR/bad-skill/examples/examples.md" <<MD
---
skill: bad-skill
cases:
  - id: bad-case
    input: "test"
    expect:
      - something
    code_block: notareallanguage
---
MD

# ── run validator against fixture ─────────────────────────────────────────────

echo "Running validator against test fixture..."
echo ""

VALIDATOR_OUTPUT=$(bash "$SCRIPTS_DIR/validate-skills.sh" 2>&1 || true)

# ── assertions ────────────────────────────────────────────────────────────────

echo "Assertions:"

# Passing checks are silent (ok increments PASS but prints nothing).
# Assert their absence from the failure/warning output instead.

# good-skill: name check passes — no name-mismatch error for good-skill
assert_not_contains \
    "good-skill: no name mismatch error" \
    "good-skill: name 'good-skill' does not match" \
    "$VALIDATOR_OUTPUT"

# good-skill: summary check passes — no summary-missing error for good-skill
assert_not_contains \
    "good-skill: no summary-missing error" \
    "good-skill: summary missing or empty" \
    "$VALIDATOR_OUTPUT"

# good-skill: handoff target resolves — no broken-target error for other-skill
assert_not_contains \
    "good-skill: handoff target 'other-skill' not flagged as missing" \
    "handoff target 'other-skill' does not exist" \
    "$VALIDATOR_OUTPUT"

# bad-skill: name mismatch detected
assert_contains \
    "bad-skill: name mismatch detected" \
    "wrong-name' does not match directory 'bad-skill'" \
    "$VALIDATOR_OUTPUT"

# bad-skill: empty summary flagged
assert_contains \
    "bad-skill: empty summary flagged" \
    "summary missing or empty" \
    "$VALIDATOR_OUTPUT"

# bad-skill: unrecognised code_block language flagged
assert_contains \
    "bad-skill: unrecognised code_block language flagged" \
    "notareallanguage" \
    "$VALIDATOR_OUTPUT"

# good-skill|other-skill cycle is allowlisted — must not appear as a failure
assert_not_contains \
    "allowlisted cycle not flagged as error" \
    "2-node cycle detected: good-skill|other-skill" \
    "$VALIDATOR_OUTPUT"

echo ""

# ── summary ───────────────────────────────────────────────────────────────────

TOTAL=$((PASS + FAIL))
echo "Results: $PASS/$TOTAL assertions passed"
echo ""

if [ "$FAIL" -gt 0 ]; then
    red "$FAIL assertion(s) failed"
    echo ""
    echo "Validator output (for debugging):"
    echo "$VALIDATOR_OUTPUT"
    exit 1
else
    green "All $PASS assertions passed"
fi