diff --git a/packages/core/src/node/route/extractPageData.test.ts b/packages/core/src/node/route/extractPageData.test.ts index f1d376a19e..efc08c86f0 100644 --- a/packages/core/src/node/route/extractPageData.test.ts +++ b/packages/core/src/node/route/extractPageData.test.ts @@ -330,6 +330,35 @@ describe('getPageIndexInfoByRoute', async () => { `); }); + it('should strip MDX syntax while keeping visible content', async () => { + const pageIndexInfo = await getPageIndexInfoByRoute( + createRoute('with-mdx-syntax.mdx', fixtureContentProcessingDir), + { + alias: {}, + replaceRules: [], + root: fixtureContentProcessingDir, + searchCodeBlocks: true, + }, + ); + + expect(pageIndexInfo.content).toBe( + [ + 'Visible component content.', + 'Text before text after.', + [ + '{', + ' "name": "create_shopping_plan",', + ' "description": "keep spacing"', + '}', + ].join('\n'), + ].join('\n\n'), + ); + expect(pageIndexInfo.content).not.toContain('```json'); + expect(pageIndexInfo.content).not.toContain('should not be indexed'); + expect(pageIndexInfo.content).not.toContain('hiddenExpression'); + expect(pageIndexInfo.content).not.toContain('anotherHiddenExpression'); + }); + it('should remove images from content', async () => { const pageIndexInfo = await getPageIndexInfoByRoute( createRoute('with-images.mdx', fixtureContentProcessingDir), diff --git a/packages/core/src/node/route/extractPageData.ts b/packages/core/src/node/route/extractPageData.ts index b40253a7e9..c4a25a5251 100644 --- a/packages/core/src/node/route/extractPageData.ts +++ b/packages/core/src/node/route/extractPageData.ts @@ -10,6 +10,7 @@ import { import { loadFrontMatter } from '@rspress/shared/node-utils'; import type { Node, Nodes, Root } from 'mdast'; import remarkGFM from 'remark-gfm'; +import remarkMdx from 'remark-mdx'; import remarkParse from 'remark-parse'; import type { Plugin } from 'unified'; import { unified } from 'unified'; @@ -77,6 +78,7 @@ const remarkRemoveImages: Plugin<[], Root> = () => { const createProcessor = (searchCodeBlocks: boolean) => unified() .use(remarkParse) + .use(remarkMdx) .use(remarkGFM) .use(remarkRemoveImages) .use(searchCodeBlocks ? [] : [remarkRemoveCodeBlocks]); @@ -84,6 +86,8 @@ const createProcessor = (searchCodeBlocks: boolean) => const processorWithCode = createProcessor(true); const processorWithoutCode = createProcessor(false); +const MDX_TEXT_EXPRESSION_PLACEHOLDER = '\0'; + /** * Extract text content from a node recursively */ @@ -108,10 +112,7 @@ const SEARCH_SKIP_TYPES = new Set([ 'footnoteReference', 'html', 'thematicBreak', - 'mdxJsxFlowElement', - 'mdxJsxTextElement', 'mdxFlowExpression', - 'mdxTextExpression', 'mdxjsEsm', ]); @@ -131,6 +132,11 @@ const SEARCH_BLOCK_TYPES = new Set([ function extractSearchText(node: Nodes, codeblocks: boolean): Array { const { type } = node; + // Keep a placeholder so only whitespace around removed inline expressions is normalized later. + if (type === 'mdxTextExpression') { + return [MDX_TEXT_EXPRESSION_PLACEHOLDER]; + } + // Return an empty string for any kind of "non-content" node if (SEARCH_SKIP_TYPES.has(type)) { return []; @@ -202,6 +208,8 @@ function buildSearchContent( .join('') // \t\n replace so we don't have trailing whitespace on table rows that aren't at the end of the text .replaceAll('\t\n', '\n') + // Replace only removed inline MDX expressions and their surrounding spaces. + .replaceAll(/(?: *\0 *)+/g, ' ') .trim(); if (!text) { continue; diff --git a/packages/core/src/node/route/fixtures/content-processing/with-mdx-syntax.mdx b/packages/core/src/node/route/fixtures/content-processing/with-mdx-syntax.mdx new file mode 100644 index 0000000000..5b1e606c16 --- /dev/null +++ b/packages/core/src/node/route/fixtures/content-processing/with-mdx-syntax.mdx @@ -0,0 +1,17 @@ +# Page with MDX syntax + +export const hiddenExport = 'should not be indexed'; + + + Visible component content. + {hiddenExpression} + + +Text before {anotherHiddenExpression} text after. + +```json +{ + "name": "create_shopping_plan", + "description": "keep spacing" +} +```