fix(seo): adapt llms.txt back #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Agent Score CI | |
| # Regression guardrail for the AI-readiness infrastructure (Fern Agent Score). | |
| # | |
| # PR #4065 ("fern agent score") got the audit to 100% by adding a set of | |
| # load-bearing, easy-to-break-by-accident pieces: the /llms.txt <link> in the | |
| # root <head>, the .md rewrites, the proxy.ts content-negotiation matcher, the | |
| # /api/raw markdown route, and includeProcessedMarkdown for HTML↔markdown | |
| # parity. None of those are obviously "SEO" when you're editing them for some | |
| # other reason (e.g. adding an auth route to proxy.ts's matcher), so they can | |
| # silently regress. | |
| # | |
| # This job re-asserts each of those invariants whenever a relevant file is | |
| # touched, so the regression fails *here* — at the source, on the PR — instead | |
| # of weeks later on the next external audit. It deliberately does NOT call the | |
| # live afdocs score: that number is noisy (samples ~10 random pages per run, the | |
| # check suite evolves) and is better watched out-of-band, not gated in CI. | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| paths: | |
| - 'lib/llm-utils.ts' | |
| - 'app/layout.tsx' | |
| - 'app/robots.ts' | |
| - 'app/sitemap.ts' | |
| - 'app/llms.txt/**' | |
| - 'app/llms-full.txt/**' | |
| - 'app/api/raw/**' | |
| - 'app/docs/[...slug]/page.tsx' | |
| - 'app/academy/[...slug]/page.tsx' | |
| - 'app/blog/[...slug]/page.tsx' | |
| - 'app/integrations/[...slug]/page.tsx' | |
| - 'next.config.mjs' | |
| - 'proxy.ts' | |
| - 'source.config.ts' | |
| - 'tests/unit/seo/**' | |
| - '.github/workflows/agent-score-ci.yml' | |
| concurrency: | |
| group: agent-score-ci-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| ci: | |
| name: Agent Score invariants | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 | |
| - name: Setup Node.js | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 | |
| with: | |
| node-version: '22' | |
| cache: 'yarn' | |
| - name: Install dependencies | |
| run: yarn install --frozen-lockfile | |
| # ── Markdown invariants (unit-tested) ──────────────── | |
| # Guarantees every .md document starts with an H1 (content-start-position) | |
| # and references /llms.txt (llms-txt-directive, markdown variant). | |
| - name: SEO unit tests | |
| run: npx vitest run tests/unit/seo | |
| # ── Source invariants (grep) ───────────────────────── | |
| - name: llms.txt directive present everywhere it must be | |
| run: | | |
| errors=0 | |
| # 1. Markdown discovery directive lives in the shared helper. | |
| if ! grep -q "LLMS_TXT_DIRECTIVE" lib/llm-utils.ts \ | |
| || ! grep -q "/llms.txt" lib/llm-utils.ts; then | |
| echo "::error file=lib/llm-utils.ts::getLLMText must append the /llms.txt discovery directive (LLMS_TXT_DIRECTIVE)" | |
| errors=$((errors + 1)) | |
| fi | |
| # 2. Root layout must render a VISIBLE (sr-only) in-body link to | |
| # /llms.txt. afdocs does NOT credit the <head> <link rel="alternate"> | |
| # tag for llms-txt-directive-html — only an in-body anchor near the | |
| # top of the page counts. | |
| if ! grep -q 'href="/llms.txt"' app/layout.tsx; then | |
| echo "::error file=app/layout.tsx::root layout must render an in-body <a href=\"/llms.txt\"> (llms-txt-directive-html — the <head> tag alone does not count)" | |
| errors=$((errors + 1)) | |
| fi | |
| # 3. Per-section doc pages advertise both their .md alternate and llms.txt. | |
| for page in \ | |
| 'app/docs/[...slug]/page.tsx' \ | |
| 'app/academy/[...slug]/page.tsx' \ | |
| 'app/blog/[...slug]/page.tsx' \ | |
| 'app/integrations/[...slug]/page.tsx'; do | |
| if [ ! -f "$page" ]; then | |
| echo "::error::$page is missing" | |
| errors=$((errors + 1)) | |
| continue | |
| fi | |
| if ! grep -q "text/markdown" "$page"; then | |
| echo "::error file=$page::generateMetadata must declare a text/markdown alternate (markdown-url-support)" | |
| errors=$((errors + 1)) | |
| fi | |
| if ! grep -q "/llms.txt" "$page"; then | |
| echo "::error file=$page::generateMetadata must declare the /llms.txt alternate (llms-txt-directive, html variant)" | |
| errors=$((errors + 1)) | |
| fi | |
| done | |
| [ $errors -eq 0 ] && echo "✓ llms.txt directives present in HTML metadata and markdown helper" || exit 1 | |
| - name: Discovery routes exist | |
| run: | | |
| errors=0 | |
| for f in \ | |
| app/llms.txt/route.ts \ | |
| app/llms-full.txt/route.ts \ | |
| app/robots.ts \ | |
| app/sitemap.ts \ | |
| 'app/api/raw/[...slug]/route.ts'; do | |
| if [ ! -f "$f" ]; then | |
| echo "::error::$f is missing — AI agents lose a discovery/markdown entry point" | |
| errors=$((errors + 1)) | |
| fi | |
| done | |
| [ $errors -eq 0 ] && echo "✓ llms.txt, sitemap, robots and raw-markdown routes all present" || exit 1 | |
| - name: .md URLs are wired for every doc section | |
| run: | | |
| errors=0 | |
| for section in docs academy blog integrations; do | |
| if ! grep -q "/${section}/:path\*.md" next.config.mjs; then | |
| echo "::error file=next.config.mjs::missing .md rewrite for /${section} (markdown-url-support / content negotiation)" | |
| errors=$((errors + 1)) | |
| fi | |
| done | |
| [ $errors -eq 0 ] && echo "✓ .md rewrites present for docs, academy, blog, integrations" || exit 1 | |
| # proxy.ts serves markdown on `Accept: text/markdown` by rewriting to | |
| # /api/raw, gated by its config.matcher. It also carries unrelated auth | |
| # routing, so an edit there (the events routes were added this way) can | |
| # silently drop the content paths and kill content negotiation. | |
| - name: Markdown content negotiation intact (proxy.ts) | |
| run: | | |
| errors=0 | |
| if ! grep -q "text/markdown" proxy.ts; then | |
| echo "::error file=proxy.ts::content negotiation removed — must serve markdown on 'Accept: text/markdown'" | |
| errors=$((errors + 1)) | |
| fi | |
| if ! grep -q "/api/raw" proxy.ts; then | |
| echo "::error file=proxy.ts::markdown requests must be rewritten to /api/raw" | |
| errors=$((errors + 1)) | |
| fi | |
| # Every doc section must stay in both the negotiation list and the matcher. | |
| for prefix in "/docs/" "/academy/" "/blog/" "/integrations/"; do | |
| if ! grep -q "'${prefix}'" proxy.ts; then | |
| echo "::error file=proxy.ts::content prefix '${prefix}' dropped from contentPrefixes" | |
| errors=$((errors + 1)) | |
| fi | |
| done | |
| for m in "/docs/:path\*" "/blog/:path\*" "/integrations/:path\*"; do | |
| if ! grep -q "\"${m}\"" proxy.ts; then | |
| echo "::error file=proxy.ts::config.matcher missing content path \"${m}\" — proxy won't run for markdown negotiation" | |
| errors=$((errors + 1)) | |
| fi | |
| done | |
| [ $errors -eq 0 ] && echo "✓ proxy.ts markdown content negotiation + matcher intact" || exit 1 | |
| # getText('processed') (used by getLLMText) only returns rendered-equivalent | |
| # markdown when includeProcessedMarkdown is enabled; without it the .md | |
| # output diverges from the HTML (markdown-content-parity). | |
| - name: Processed markdown enabled (content parity) | |
| run: | | |
| if ! grep -q "includeProcessedMarkdown" source.config.ts; then | |
| echo "::error file=source.config.ts::includeProcessedMarkdown must stay enabled so .md mirrors the rendered HTML (markdown-content-parity)" | |
| exit 1 | |
| fi | |
| echo "✓ includeProcessedMarkdown enabled — markdown mirrors rendered content" |