Merge pull request #4250 from ava-labs/agent-score-guardrail #2
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Agent Score CI | |
| # Deterministic, in-repo guardrail for AI-readiness (Fern Agent Score). | |
| # | |
| # The live Agent Score is noisy: it samples ~10 random pages per run and its | |
| # check suite evolves, so the headline number drifts a few points run-to-run | |
| # regardless of our changes. This job instead asserts the structural invariants | |
| # we actually control, so a refactor that would quietly degrade the score fails | |
| # *here* — at the source — instead of on the next external audit. | |
| # | |
| # The live score itself is tracked separately by agent-score-monitor.yml. | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| paths: | |
| - 'lib/llm-utils.ts' | |
| - 'app/layout.tsx' | |
| - 'app/robots.ts' | |
| - 'app/sitemap.ts' | |
| - 'app/llms.txt/**' | |
| - 'app/llms-full.txt/**' | |
| - 'app/api/raw/**' | |
| - 'app/docs/[...slug]/page.tsx' | |
| - 'app/academy/[...slug]/page.tsx' | |
| - 'app/blog/[...slug]/page.tsx' | |
| - 'app/integrations/[...slug]/page.tsx' | |
| - 'next.config.mjs' | |
| - 'tests/unit/seo/**' | |
| - '.github/workflows/agent-score-ci.yml' | |
| concurrency: | |
| group: agent-score-ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| ci: | |
| name: Agent Score invariants | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 | |
| with: | |
| node-version: '22' | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| run: yarn install --frozen-lockfile | |
| # ── Markdown invariants (unit-tested) ──────────────── | |
| # Guarantees every .md document starts with an H1 (content-start-position) | |
| # and references /llms.txt (llms-txt-directive, markdown variant). | |
| - name: SEO unit tests | |
| run: npx vitest run tests/unit/seo | |
| # ── Source invariants (grep) ───────────────────────── | |
| - name: llms.txt directive present everywhere it must be | |
| run: | | |
| errors=0 | |
| # 1. Markdown discovery directive lives in the shared helper. | |
| if ! grep -q "LLMS_TXT_DIRECTIVE" lib/llm-utils.ts \ | |
| || ! grep -q "/llms.txt" lib/llm-utils.ts; then | |
| echo "::error file=lib/llm-utils.ts::getLLMText must append the /llms.txt discovery directive (LLMS_TXT_DIRECTIVE)" | |
| errors=$((errors + 1)) | |
| fi | |
| # 2. Site-wide HTML directive in the root layout metadata. | |
| if ! grep -q "/llms.txt" app/layout.tsx; then | |
| echo "::error file=app/layout.tsx::root metadata must advertise /llms.txt via alternates (llms-txt-directive, html variant)" | |
| errors=$((errors + 1)) | |
| fi | |
| # 3. Per-section doc pages advertise both their .md alternate and llms.txt. | |
| for page in \ | |
| 'app/docs/[...slug]/page.tsx' \ | |
| 'app/academy/[...slug]/page.tsx' \ | |
| 'app/blog/[...slug]/page.tsx' \ | |
| 'app/integrations/[...slug]/page.tsx'; do | |
| if [ ! -f "$page" ]; then | |
| echo "::error::$page is missing" | |
| errors=$((errors + 1)) | |
| continue | |
| fi | |
| if ! grep -q "text/markdown" "$page"; then | |
| echo "::error file=$page::generateMetadata must declare a text/markdown alternate (markdown-url-support)" | |
| errors=$((errors + 1)) | |
| fi | |
| if ! grep -q "/llms.txt" "$page"; then | |
| echo "::error file=$page::generateMetadata must declare the /llms.txt alternate (llms-txt-directive, html variant)" | |
| errors=$((errors + 1)) | |
| fi | |
| done | |
| [ $errors -eq 0 ] && echo "✓ llms.txt directives present in HTML metadata and markdown helper" || exit 1 | |
| - name: Discovery routes exist | |
| run: | | |
| errors=0 | |
| for f in \ | |
| app/llms.txt/route.ts \ | |
| app/llms-full.txt/route.ts \ | |
| app/robots.ts \ | |
| app/sitemap.ts \ | |
| 'app/api/raw/[...slug]/route.ts'; do | |
| if [ ! -f "$f" ]; then | |
| echo "::error::$f is missing — AI agents lose a discovery/markdown entry point" | |
| errors=$((errors + 1)) | |
| fi | |
| done | |
| [ $errors -eq 0 ] && echo "✓ llms.txt, sitemap, robots and raw-markdown routes all present" || exit 1 | |
| - name: .md URLs are wired for every doc section | |
| run: | | |
| errors=0 | |
| for section in docs academy blog integrations; do | |
| if ! grep -q "/${section}/:path\*.md" next.config.mjs; then | |
| echo "::error file=next.config.mjs::missing .md rewrite for /${section} (markdown-url-support / content negotiation)" | |
| errors=$((errors + 1)) | |
| fi | |
| done | |
| [ $errors -eq 0 ] && echo "✓ .md rewrites present for docs, academy, blog, integrations" || exit 1 |