fix: added new tests

Author: separatrixxx

schemas/yfm-schema.yaml

@@ -367,6 +367,12 @@ properties:
     type: boolean
     description: Emit yfm-build-manifest.json with all documentation paths.
 
+  crawlerManifest:
+    type: boolean
+    description: >
+      Emit crawler-manifest.json mapping each source file to its external links.
+      Useful for link-health checks (detecting 404, 403, etc.).
+
   changelogs:
     oneOf:
       - type: boolean

src/commands/build/features/crawler-manifest/index.spec.ts

@@ -1,6 +1,13 @@
-import {describe, expect, it} from 'vitest';
+import {describe, expect, it, vi} from 'vitest';
 
-import {extractExternalLinks, extractIncludePaths} from './index';
+import type {Run} from '~/commands/build';
+
+import {
+    collectExternalLinksFromYaml,
+    collectLinks,
+    extractExternalLinks,
+    extractIncludePaths,
+} from './index';
 
 describe('CrawlerManifest feature', () => {
     describe('extractExternalLinks', () => {
@@ -217,4 +224,145 @@ plain https://plain.example.com text
             );
         });
     });
+
+    describe('collectExternalLinksFromYaml', () => {
+        it('extracts external link from href field', () => {
+            expect(collectExternalLinksFromYaml('href: https://example.com')).toContain(
+                'https://example.com',
+            );
+        });
+
+        it('extracts external link from nested url field', () => {
+            expect(
+                collectExternalLinksFromYaml('navigation:\n  href: https://nav.example.com'),
+            ).toContain('https://nav.example.com');
+        });
+
+        it('ignores relative href value', () => {
+            expect(collectExternalLinksFromYaml('href: ./relative.md')).toEqual([]);
+        });
+
+        it('returns empty on invalid YAML', () => {
+            expect(collectExternalLinksFromYaml(': invalid: {unclosed')).toEqual([]);
+        });
+
+        it('returns empty when YAML is null', () => {
+            expect(collectExternalLinksFromYaml('')).toEqual([]);
+        });
+
+        it('returns empty when YAML is a scalar', () => {
+            expect(collectExternalLinksFromYaml('just a string')).toEqual([]);
+        });
+    });
+
+    describe('stripFencedBlocks (via extractExternalLinks)', () => {
+        it('strips unclosed fenced block to end of content', () => {
+            expect(extractExternalLinks('```\n[link](https://unclosed.example.com)')).toEqual([]);
+        });
+
+        it('tilde fence does not close backtick fence', () => {
+            const content = '```\n[inside](https://inside.example.com)\n~~~\nstill inside\n```';
+
+            expect(extractExternalLinks(content)).toEqual([]);
+        });
+
+        it('shorter fence does not close longer opening fence', () => {
+            const content = '````\n[inside](https://inside.example.com)\n```\nnot closed\n````';
+
+            expect(extractExternalLinks(content)).toEqual([]);
+        });
+
+        it('handles CRLF line endings correctly', () => {
+            const content =
+                '```\r\n[inside](https://inside.example.com)\r\n```\r\n[outside](https://outside.example.com)';
+            const links = extractExternalLinks(content);
+
+            expect(links).not.toContain('https://inside.example.com');
+            expect(links).toContain('https://outside.example.com');
+        });
+
+        it('content after closed fence is included', () => {
+            const content =
+                '```\n[inside](https://inside.example.com)\n```\n[outside](https://outside.example.com)';
+            const links = extractExternalLinks(content);
+
+            expect(links).not.toContain('https://inside.example.com');
+            expect(links).toContain('https://outside.example.com');
+        });
+    });
+
+    describe('collectLinks', () => {
+        const makeRun = (files: Record<string, string>) =>
+            ({
+                input: '/input',
+                read: vi.fn(async (path: string) => {
+                    const relative = path.slice('/input/'.length);
+                    if (relative in files) return files[relative];
+                    throw new Error(`File not found: ${path}`);
+                }),
+            }) as unknown as Run;
+
+        it('extracts links from a markdown file', async () => {
+            const run = makeRun({'page.md': '[link](https://example.com)'});
+
+            expect(await collectLinks(run, 'page.md')).toContain('https://example.com');
+        });
+
+        it('returns empty array when file cannot be read', async () => {
+            const run = makeRun({});
+
+            expect(await collectLinks(run, 'missing.md')).toEqual([]);
+        });
+
+        it('returns empty array for already-visited file (circular protection)', async () => {
+            const run = makeRun({'page.md': '[link](https://example.com)'});
+            const visited = new Set(['page.md']);
+
+            expect(await collectLinks(run, 'page.md', visited)).toEqual([]);
+        });
+
+        it('extracts links from YAML file including structured keys', async () => {
+            const run = makeRun({
+                'data.yaml': 'href: https://yaml.example.com',
+            });
+
+            expect(await collectLinks(run, 'data.yaml')).toContain('https://yaml.example.com');
+        });
+
+        it('extracts links from .yml file', async () => {
+            const run = makeRun({
+                'data.yml': 'href: https://yml.example.com',
+            });
+
+            expect(await collectLinks(run, 'data.yml')).toContain('https://yml.example.com');
+        });
+
+        it('recursively extracts links from included markdown files', async () => {
+            const run = makeRun({
+                'page.md': '{% include [snippet](./_includes/snippet.md) %}',
+                '_includes/snippet.md': '[link](https://included.example.com)',
+            });
+
+            expect(await collectLinks(run, 'page.md')).toContain('https://included.example.com');
+        });
+
+        it('does not follow includes from YAML files', async () => {
+            const run = makeRun({
+                'data.yaml': 'href: https://yaml.example.com',
+                'snippet.md': '[link](https://shouldnotappear.example.com)',
+            });
+            const links = await collectLinks(run, 'data.yaml');
+
+            expect(links).toContain('https://yaml.example.com');
+            expect(links).not.toContain('https://shouldnotappear.example.com');
+        });
+
+        it('skips unreadable include files gracefully', async () => {
+            const run = makeRun({
+                'page.md': '{% include [missing](./missing.md) %}\n[link](https://example.com)',
+            });
+
+            expect(await collectLinks(run, 'page.md')).toContain('https://example.com');
+        });
+    });
 });

src/commands/build/features/crawler-manifest/index.ts

@@ -24,23 +24,52 @@ export type CrawlerManifestConfig = {
 
 const MANIFEST_FILENAME = 'crawler-manifest.json';
 
-const INLINE_LINK_RE = /\[[^\]]*\]\(([^)\s]+)[^)]*\)/g;
-const AUTOLINK_REGEX = /<([^>]+)>/g;
+const INLINE_LINK_REGEX = /\[[^\]]*\]\(([^)\s"']+)/g;
+const AUTOLINK_REGEX = /<([^>\s]+)>/g;
 const BARE_URL_REGEX = /\bhttps?:\/\/[^\s<>)[\]"']+/g;
-const FILE_BLOCK_REGEX = /{%\s*file\s[^%]*src="([^"]+)"[^%]*%}/g;
+const FILE_BLOCK_REGEX = /{%\s*file\s[^%]*?src="([^"]+)"/g;
+
+function stripFencedBlocks(content: string): string {
+    const lines = content.split('\n');
+    const out: string[] = [];
+    let fenceChar = '';
+    let fenceLen = 0;
+
+    for (const line of lines) {
+        const normalized = line.replace(/\r$/, '');
+
+        if (fenceLen === 0) {
+            const fence = /^(`{3,}|~{3,})/.exec(normalized);
+
+            if (fence) {
+                fenceChar = fence[1][0];
+                fenceLen = fence[1].length;
+            } else {
+                out.push(line);
+            }
+        } else {
+            const re = fenceChar === '`' ? /^(`{3,})[ \t]*$/ : /^(~{3,})[ \t]*$/;
+            const fence = re.exec(normalized);
+
+            if (fence && fence[1].length >= fenceLen) {
+                fenceChar = '';
+                fenceLen = 0;
+            }
+        }
+    }
+
+    return out.join('\n');
+}
 
 function stripNonContent(content: string): string {
-    return content
-        .replace(/<!--[\s\S]*?-->/g, '')
-        .replace(/^(`{3,}|~{3,})[^\n]*\n[\s\S]*?\n\1[ \t]*$/gm, '')
-        .replace(/`[^`\n]+`/g, '');
+    return stripFencedBlocks(content.replace(/<!--[\s\S]*?-->/g, '')).replace(/`[^`\n]+`/g, '');
 }
 
 export function extractExternalLinks(content: string): string[] {
     const links = new Set<string>();
     const stripped = stripNonContent(content);
 
-    for (const match of stripped.matchAll(INLINE_LINK_RE)) {
+    for (const match of stripped.matchAll(INLINE_LINK_REGEX)) {
         if (isExternalHref(match[1])) {
             links.add(match[1]);
         }
@@ -65,13 +94,19 @@ export function extractExternalLinks(content: string): string[] {
     }
 
     for (const match of stripped.matchAll(BARE_URL_REGEX)) {
-        links.add(match[0].replace(/[.,;:!?]+$/, ''));
+        let url = match[0];
+
+        while (url.length > 0 && '.,;:!?'.includes(url[url.length - 1])) {
+            url = url.slice(0, -1);
+        }
+
+        links.add(url);
     }
 
     return [...links];
 }
 
-function collectExternalLinksFromYaml(content: string): string[] {
+export function collectExternalLinksFromYaml(content: string): string[] {
     const links: string[] = [];
 
     try {
@@ -103,7 +138,7 @@ export function extractIncludePaths(content: string): string[] {
     return paths;
 }
 
-async function collectLinks(
+export async function collectLinks(
     run: Run,
     filePath: string,
     visited = new Set<string>(),

src/core/markdown/utils.ts

@@ -68,8 +68,8 @@ export function getPcIconTitle(iconPath: string): string {
 }
 
 export const PC_REGEX = /^([ \t]*):::\s*page-constructor[ \t]*\r?\n?/m;
-export const REF_DEF_REGEX = /^\s*\[(.*?)]:\s*([^\s]+)/gm;
-export const INCLUDE_REGEX = /{%\s*include\s*.+?%}/g;
+export const REF_DEF_REGEX = /^\s*\[([^\]]*)]:\s*([^\s]+)/gm;
+export const INCLUDE_REGEX = /{%\s*include\s+[^%]+%}/g;
 
 export function findPcImages(content: string): AssetInfo[] {
     const pcImages: AssetInfo[] = [];

tests/e2e/crawler-manifest.spec.ts

@@ -1,12 +1,12 @@
 import {describe, expect, test} from 'vitest';
 import {join} from 'node:path';
-import {readFile} from 'node:fs/promises';
+import {access, readFile} from 'node:fs/promises';
 
 import {TestAdapter, getTestPaths} from '../fixtures';
 
 describe('Crawler manifest', () => {
     test('generates crawler-manifest.json', async () => {
-        const {inputPath, outputPath} = getTestPaths('mocks/crawler-manifest');
+        const {inputPath, outputPath} = getTestPaths('mocks/crawler-manifest/with-links');
 
         await TestAdapter.testBuildPass(inputPath, outputPath, {
             md2md: true,
@@ -18,4 +18,41 @@ describe('Crawler manifest', () => {
 
         expect(JSON.parse(manifestContent)).toMatchSnapshot();
     });
+
+    test('does not generate manifest when --crawler-manifest flag is absent', async () => {
+        const {inputPath, outputPath} = getTestPaths('mocks/crawler-manifest/with-links');
+
+        await TestAdapter.testBuildPass(inputPath, outputPath, {
+            md2md: true,
+            md2html: false,
+        });
+
+        await expect(access(join(outputPath, 'crawler-manifest.json'))).rejects.toThrow();
+    });
+
+    test('does not generate manifest when there are no external links', async () => {
+        const {inputPath, outputPath} = getTestPaths('mocks/crawler-manifest/no-links');
+
+        await TestAdapter.testBuildPass(inputPath, outputPath, {
+            md2md: true,
+            md2html: false,
+            args: '--crawler-manifest',
+        });
+
+        await expect(access(join(outputPath, 'crawler-manifest.json'))).rejects.toThrow();
+    });
+
+    test('reads crawlerManifest setting from .yfm config file', async () => {
+        const {inputPath, outputPath} = getTestPaths('mocks/crawler-manifest/config');
+
+        await TestAdapter.testBuildPass(inputPath, outputPath, {
+            md2md: true,
+            md2html: false,
+        });
+
+        const manifestContent = await readFile(join(outputPath, 'crawler-manifest.json'), 'utf-8');
+        const manifest = JSON.parse(manifestContent);
+
+        expect(manifest['index.md']).toContain('https://config.example.com');
+    });
 });

tests/mocks/crawler-manifest/config/input/.yfm

@@ -0,0 +1 @@
+crawlerManifest: true

tests/mocks/crawler-manifest/config/input/index.md

@@ -0,0 +1,3 @@
+# Index
+
+[External link](https://config.example.com)

tests/mocks/crawler-manifest/config/input/toc.yaml

@@ -0,0 +1,4 @@
+title: Crawler manifest config test
+items:
+  - name: Index
+    href: index.md

tests/mocks/crawler-manifest/no-links/input/index.md

@@ -0,0 +1,5 @@
+# Index
+
+Local link: [other page](./other.md).
+
+Absolute local link: [absolute](/docs/page.md).

tests/mocks/crawler-manifest/no-links/input/toc.yaml

@@ -0,0 +1,4 @@
+title: No external links test
+items:
+  - name: Index
+    href: index.md