feat(slop-test-detector): benchmark eval fixtures (#21)

## Summary

- 13 eval fixtures covering all 18 slop detection rules, validated
against `analyzeTestFile()` ground truth
- `evals.json` with per-eval expectations, rule coverage matrix, and
holdout split for benchmark runs
- SKILL.md updated with pushier description (better triggering) and
explicit guidance on reporting findings with impact
- Validation scripts (`validate-fixtures.ts`, `run-quick-eval.ts`) for
ground truth regression testing

## Key design decisions

- **Fixture headers stripped of ground truth comments** — baseline
testing revealed models read JSDoc hints and reproduce expected rule
names, invalidating with/without comparisons
- **13 fixtures, not 12** — added `self_referential_assertion` fixture
to close the last rule coverage gap
- **3 baseline runs (Sonnet, no skill)** confirmed the skill's value is
precision (0 FPs on clean code vs 4 baseline FPs) and consistency
(stable rule names vs ad hoc labels), not just detection

## Rule coverage

All 18 rules exercised across fixtures. Distribution: 3 clearly bad, 3
mixed, 3 clean/FP-resistant, 4 rule-specific.

## Test plan

- [x] All 13 evals pass ground truth validation (172 checks, 0 failures)
- [x] All 157 existing slop-detector tests pass
- [x] Baseline comparison run without answer leak confirms skill adds
value
- [ ] Full benchmark (5 runs × 2 configs × 13 evals) — deferred pending
cost/value assessment

🤖 Generated with [Claude Code](https://claude.com/claude-code)

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: apankov1

biome.json

@@ -25,6 +25,6 @@
   },
   "files": {
     "include": ["skills/**/*.ts"],
-    "ignore": ["node_modules"]
+    "ignore": ["node_modules", "skills/*/evals/fixtures"]
   }
 }

skills/slop-test-detector/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: slop-test-detector
-description: "Static analyzer that detects 18 slop patterns in test code — tests that compile and pass but catch zero bugs."
+description: "Static analyzer that detects 18 slop patterns in test code — tests that compile and pass but catch zero bugs. Use when reviewing test files, auditing test quality, checking for weak tests, validating generated tests, or when the user mentions test slop, empty tests, missing assertions, or tautological checks. TRIGGER on any test review or test quality task."
 ---
 
 # Slop Test Detector
@@ -134,7 +134,18 @@ const report = analyzeTestFile(source, 'my-module.spec.ts');
 console.log(formatReport(report));
 ```
 
-### Step 2: Validate During Generation
+### Step 2: Report Findings with Impact
+
+When presenting findings, always include:
+- **Rule name** — the specific slop pattern (e.g., `tautological_assertion`)
+- **Test name and line number** — so the user can navigate to it
+- **Severity** — `must-fail` (test is structurally broken) vs `should-fail` (quality concern)
+- **Why it matters** — what production bug could slip through because of this pattern
+- **What clean tests should NOT be flagged** — avoid false positives on well-written tests
+
+If a test file has assertion-equivalent helpers (functions named `assert*()` or `test*()` that internally call `assert.*`), configure `assertionEquivalents` to prevent false `empty_test_body` findings.
+
+### Step 3: Validate During Generation
 
 Before writing a generated test to disk, check for slop:
 
@@ -151,15 +162,15 @@ if (findings.some(f => f.severity === 'must-fail')) {
 }
 ```
 
-### Step 3: Machine-Readable Output (CI)
+### Step 4: Machine-Readable Output (CI)
 
 ```typescript
 const report = analyzeTestFile(source, filePath, getPreset('balanced'));
 const json = formatReportJSON(report);
 // Outputs structured JSON with filePath, score, summary, findings[]
 ```
 
-### Step 4: Interpret the Score
+### Step 5: Interpret the Score
 
 The score formula weights must-fail findings (1.0) higher than should-fail (0.3):
 

skills/slop-test-detector/evals/evals.json

@@ -0,0 +1,78 @@
+/**
+ * Notification service contract tests
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { NotificationService } from "../src/notification-service.js";
+
+describe("NotificationService", () => {
+  it("should deliver email notification with correct subject", () => {
+    const service = new NotificationService({ provider: "ses" });
+    const result = service.send({
+      channel: "email",
+      to: "user@example.com",
+      subject: "Order Confirmed",
+      body: "Your order #1234 has been confirmed.",
+    });
+    assert.equal(result.status, "delivered");
+    assert.equal(result.channel, "email");
+    assert.ok(result.messageId.startsWith("msg_"));
+  });
+
+  it("should batch multiple recipients in single API call", () => {
+    const service = new NotificationService({ provider: "ses", batchSize: 50 });
+    const recipients = Array.from({ length: 120 }, (_, i) => `user${i}@example.com`);
+    const batches = service.prepareBatches(recipients);
+    assert.equal(batches.length, 3);
+    assert.equal(batches[0].length, 50);
+    assert.equal(batches[1].length, 50);
+    assert.equal(batches[2].length, 20);
+  });
+
+  it("should throw on unsupported notification channel", () => {
+    const service = new NotificationService({ provider: "ses" });
+    assert.throws(
+      () =>
+        service.send({
+          channel: "carrier-pigeon" as any,
+          to: "user@example.com",
+          subject: "test",
+          body: "test",
+        }),
+      { code: "UNSUPPORTED_CHANNEL" },
+    );
+  });
+
+  it("should respect quiet hours by deferring delivery", () => {
+    const service = new NotificationService({
+      provider: "ses",
+      quietHours: { start: 22, end: 7, timezone: "America/New_York" },
+    });
+    const result = service.send({
+      channel: "push",
+      to: "device_token_abc",
+      subject: "Reminder",
+      body: "Don't forget your appointment",
+      sendAt: new Date("2026-03-15T03:00:00-05:00"),
+    });
+    assert.equal(result.status, "deferred");
+    assert.ok(result.scheduledFor);
+    assert.ok(new Date(result.scheduledFor).getHours() >= 7);
+  });
+
+  it("should redact PII from notification logs", () => {
+    const service = new NotificationService({ provider: "ses" });
+    service.send({
+      channel: "email",
+      to: "sensitive@example.com",
+      subject: "Password Reset",
+      body: "Click here to reset: https://app.com/reset?token=abc123",
+    });
+    const logs = service.getAuditLog();
+    assert.ok(logs.length > 0);
+    const lastLog = logs[logs.length - 1];
+    assert.equal(lastLog.to, "s*******e@example.com");
+    assert.ok(!lastLog.body.includes("abc123"));
+  });
+});

skills/slop-test-detector/evals/fixtures/clean-contract.spec.ts

@@ -0,0 +1,71 @@
+/**
+ * Order lifecycle state machine tests
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { OrderStateMachine } from "../src/order-state-machine.js";
+
+describe("OrderStateMachine", () => {
+  it("should start in 'pending' state", () => {
+    const order = new OrderStateMachine("order_001");
+    assert.equal(order.currentState, "pending");
+    assert.deepEqual(order.history, [{ state: "pending", timestamp: order.createdAt }]);
+  });
+
+  it("should transition pending → confirmed on payment", () => {
+    const order = new OrderStateMachine("order_002");
+    order.transition("confirm", { paymentId: "pay_abc" });
+    assert.equal(order.currentState, "confirmed");
+    assert.equal(order.history.length, 2);
+    assert.equal(order.metadata.paymentId, "pay_abc");
+  });
+
+  it("should transition confirmed → shipped with tracking", () => {
+    const order = new OrderStateMachine("order_003");
+    order.transition("confirm", { paymentId: "pay_def" });
+    order.transition("ship", { trackingNumber: "1Z999AA10123456784", carrier: "UPS" });
+    assert.equal(order.currentState, "shipped");
+    assert.equal(order.metadata.trackingNumber, "1Z999AA10123456784");
+    assert.equal(order.metadata.carrier, "UPS");
+  });
+
+  it("should reject invalid transitions", () => {
+    const order = new OrderStateMachine("order_004");
+    assert.throws(() => order.transition("ship", { trackingNumber: "TRACK123" }), {
+      message: /Cannot transition from 'pending' via 'ship'/,
+    });
+  });
+
+  it("should allow cancellation from pending or confirmed", () => {
+    const pendingOrder = new OrderStateMachine("order_005");
+    pendingOrder.transition("cancel", { reason: "Customer request" });
+    assert.equal(pendingOrder.currentState, "cancelled");
+
+    const confirmedOrder = new OrderStateMachine("order_006");
+    confirmedOrder.transition("confirm", { paymentId: "pay_ghi" });
+    confirmedOrder.transition("cancel", { reason: "Out of stock" });
+    assert.equal(confirmedOrder.currentState, "cancelled");
+  });
+
+  it("should reject cancellation of shipped orders", () => {
+    const order = new OrderStateMachine("order_007");
+    order.transition("confirm", { paymentId: "pay_jkl" });
+    order.transition("ship", { trackingNumber: "TRACK789", carrier: "FedEx" });
+    assert.throws(() => order.transition("cancel", { reason: "Changed mind" }), {
+      message: /Cannot transition from 'shipped' via 'cancel'/,
+    });
+  });
+
+  it("should record full audit trail of state changes", () => {
+    const order = new OrderStateMachine("order_008");
+    order.transition("confirm", { paymentId: "pay_mno" });
+    order.transition("ship", { trackingNumber: "TRACK456", carrier: "DHL" });
+    order.transition("deliver", {});
+    assert.equal(order.history.length, 4);
+    assert.deepEqual(
+      order.history.map((h) => h.state),
+      ["pending", "confirmed", "shipped", "delivered"],
+    );
+  });
+});

skills/slop-test-detector/evals/fixtures/clean-state-machine.spec.ts

@@ -0,0 +1,55 @@
+/**
+ * Cache eviction policy tests
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { LRUCache } from "../src/lru-cache.js";
+
+describe("LRUCache", () => {
+  it("should evict least recently used entry when at capacity", () => {
+    const cache = new LRUCache<string>({ maxSize: 3 });
+    cache.set("a", "alpha");
+    cache.set("b", "bravo");
+    cache.set("c", "charlie");
+    cache.get("a"); // access 'a' to make 'b' the LRU
+    cache.set("d", "delta"); // should evict 'b'
+    assert.equal(cache.get("b"), undefined);
+    assert.equal(cache.get("a"), "alpha");
+    assert.equal(cache.get("d"), "delta");
+  });
+
+  it("should respect max size limit", () => {
+    const cache = new LRUCache<number>({ maxSize: 2 });
+    cache.set("x", 1);
+    cache.set("y", 2);
+    cache.set("z", 3);
+    assert.equal(cache.size, 2);
+    assert.equal(cache.has("x"), false);
+  });
+
+  // Defect: TTL bug
+  it("should handle TTL expiry correctly", () => {
+    const cache = new LRUCache<string>({ maxSize: 10, ttlMs: 100 });
+    cache.set("temp", "value");
+    // Simulate time passage
+    cache._advanceClock(150);
+    assert.equal(cache.get("temp"), undefined);
+    assert.equal(cache.size, 0);
+  });
+
+  // Defect: get() must update eviction order to most-recently-used, otherwise hot keys
+  // get evicted and hit rate degrades under load — breaks LRU invariant.
+  it("should update access order on get to maintain LRU invariant", () => {
+    const cache = new LRUCache<string>({ maxSize: 3 });
+    cache.set("a", "1");
+    cache.set("b", "2");
+    cache.set("c", "3");
+    cache.get("a"); // touch 'a', making 'b' the LRU
+    cache.set("d", "4"); // evicts 'b', not 'a'
+    assert.equal(cache.has("a"), true);
+    assert.equal(cache.has("b"), false);
+    assert.equal(cache.has("c"), true);
+    assert.equal(cache.has("d"), true);
+  });
+});

skills/slop-test-detector/evals/fixtures/defect-comments.spec.ts

@@ -0,0 +1,43 @@
+/**
+ * URL shortener service tests
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { UrlShortener, classify } from "../src/url-shortener.js";
+
+describe("UrlShortener", () => {
+  it("should shorten HTTP URL to 8-char slug", () => {
+    const shortener = new UrlShortener({ slugLength: 8 });
+    const result = shortener.shorten("https://example.com/very/long/path");
+    assert.ok(result.slug);
+    assert.equal(result.slug.length, 8);
+    assert.equal(result.originalUrl, "https://example.com/very/long/path");
+  });
+
+  it("should shorten HTTPS URL to 8-char slug", () => {
+    const shortener = new UrlShortener({ slugLength: 8 });
+    const result = shortener.shorten("https://example.com/very/long/path");
+    assert.ok(result.slug);
+    assert.equal(result.slug.length, 8);
+    assert.equal(result.originalUrl, "https://example.com/very/long/path");
+  });
+
+  it("should classify URL protocol as HTTP", () => {
+    assert.equal(classify("https://example.com"), "secure");
+  });
+
+  it("should classify URL protocol for redirect", () => {
+    assert.equal(classify("https://example.com"), "secure");
+  });
+
+  it("should track click count per slug", () => {
+    const shortener = new UrlShortener({ slugLength: 8 });
+    const { slug } = shortener.shorten("https://docs.example.com/guide");
+    shortener.resolve(slug);
+    shortener.resolve(slug);
+    shortener.resolve(slug);
+    const stats = shortener.getStats(slug);
+    assert.equal(stats.clicks, 3);
+  });
+});

skills/slop-test-detector/evals/fixtures/duplicate-no-variation.spec.ts

@@ -0,0 +1,41 @@
+/**
+ * Payment gateway integration tests
+ */
+
+import assert from "node:assert/strict";
+import { describe, it } from "node:test";
+import { PaymentGateway } from "../src/payment-gateway.js";
+
+describe("PaymentGateway", () => {
+  it("should process refund for valid transaction", () => {
+    const gateway = new PaymentGateway({ apiKey: "sk_test_123" });
+    const transaction = gateway.createTransaction({
+      amount: 4999,
+      currency: "USD",
+      cardToken: "tok_visa_debit",
+    });
+    // Will add assertions after integration env is up
+  });
+
+  it("should validate card number format before charging", () => {
+    const gateway = new PaymentGateway({ apiKey: "sk_test_123" });
+    const result = gateway.validateCard("4111111111111111");
+    // assert.equal(result.valid, true);
+    // assert.equal(result.brand, "visa");
+    // assert.equal(result.lastFour, "1111");
+  });
+
+  it("should log transaction with correlation ID", () => {
+    const logs: string[] = [];
+    const correlationId = "corr-abc-123";
+    const timestamp = new Date().toISOString();
+    const entry = `[${timestamp}] ${correlationId} PAYMENT_INIT`;
+    logs.push(entry);
+    assert.ok(logs.length > 0);
+  });
+
+  it("should reject expired cards", () => {
+    const gateway = new PaymentGateway({ apiKey: "sk_test_123" });
+    assert.throws(() => gateway.charge({ cardToken: "tok_expired", amount: 1000 }), { code: "CARD_EXPIRED" });
+  });
+});