diff --git a/packages/evals-core/src/graders/executors/llm-judge.ts b/packages/evals-core/src/graders/executors/llm-judge.ts index 274a8ae2..f3113fcf 100644 --- a/packages/evals-core/src/graders/executors/llm-judge.ts +++ b/packages/evals-core/src/graders/executors/llm-judge.ts @@ -16,12 +16,18 @@ import { logger } from '../../utils/logger.js'; * are dropped so credential values never reach the judge LLM — security graders verify * secret absence in source deterministically via `notContainsInSource`, and "wired into * .env" is an event-based (`wroteFile`) concern, so the judge never needs `.env` content. + * + * `.md` / `.txt` files are dropped because agents routinely emit large standalone docs + * (integration guides, summaries, checklists) that are not source code. They inflate the + * corpus — sometimes past `maxCodeChars`, which fails the judge outright — while adding no + * signal the judge needs. */ const JUDGE_EXCLUDED_PATTERNS = [ /^tsconfig(\.\w+)?\.json$/, /^angular\.json$/, /^tsconfig\.tsbuildinfo$/, /\.md$/i, + /\.txt$/i, /^\.env(\..*)?$/, ]; diff --git a/packages/evals-core/tests/graders/executors.test.ts b/packages/evals-core/tests/graders/executors.test.ts index 37c235c5..582969cb 100644 --- a/packages/evals-core/tests/graders/executors.test.ts +++ b/packages/evals-core/tests/graders/executors.test.ts @@ -349,6 +349,12 @@ describe('isJudgeExcluded', () => { expect(isJudgeExcluded('docs/SETUP.md')).toBe(true); expect(isJudgeExcluded('CHANGELOG.MD')).toBe(true); }); + + it('excludes text files (agent-emitted docs, summaries, checklists)', () => { + expect(isJudgeExcluded('IMPLEMENTATION_SUMMARY.txt')).toBe(true); + expect(isJudgeExcluded('docs/VERIFICATION_CHECKLIST.txt')).toBe(true); + expect(isJudgeExcluded('NOTES.TXT')).toBe(true); + }); }); // ── compile ───────────────────────────────────────────────────────────────────