build(docs-gen): add raw extension data extractor

infra/docs-gen/EXTRACTION_PIPELINE.md

@@ -0,0 +1,42 @@
+# Extension Extraction Pipeline
+
+```mermaid
+flowchart TD
+    CLI["extract-extension-data.mjs<br/>Parse CLI options"] --> Extractor["ExtensionExtractor<br/>src/extractor/index.mjs"]
+    Extractor --> Categories["Extension categories<br/>src/config.mjs"]
+    Categories --> ScanAll["scanAll()<br/>Skip internal extensions"]
+    ScanAll --> Scan["scanExtension()<br/>src/extractor/scan.mjs"]
+
+    Scan --> Files["readExtensionSources()<br/>src/utils.mjs"]
+    Files --> Selectors["Source file selectors<br/>src/extractor/source-files.mjs"]
+    Selectors --> SourceText["Production source text"]
+    Selectors --> TestFiles["Test files"]
+    Selectors --> SerializerFiles["Serializer and Specs files"]
+
+    SourceText --> Constants["extractConstants()<br/>src/extractor/constants.mjs"]
+    SourceText --> RegexScanners["Source scanners<br/>src/extractor/regex.mjs + patterns.mjs"]
+    SourceText --> Options["Option declarations<br/>src/extractor/options.mjs"]
+    SerializerFiles --> SerializerHints["Serializer hints"]
+    TestFiles --> MarkupExamples["Markup examples<br/>src/extractor/examples.mjs"]
+
+    Constants --> Schema["Schema names<br/>nodes and marks"]
+    RegexScanners --> ExtractedFields["Actions, keymaps, input rules,<br/>plugins, md plugins"]
+    Options --> ExtractedFields
+    Schema --> IR["Extension IR record"]
+    ExtractedFields --> IR
+    SerializerHints --> IR
+    MarkupExamples --> IR
+
+    Extractor --> Presets["Preset membership<br/>src/extractor/presets.mjs"]
+    Presets --> IR
+    IR --> JsonOut["extensions.json<br/>src/extractor/output.mjs"]
+    IR --> MarkdownOut["raw/*.md<br/>output.mjs -> markdown-gen.mjs"]
+```
+
+The extractor keeps orchestration and parsing separate:
+
+- `index.mjs` decides which extension directories are scanned and when output is written.
+- `scan.mjs` builds one extension record from source files and parser results.
+- `source-files.mjs` owns file selection rules.
+- `regex.mjs`, `options.mjs`, `examples.mjs`, `constants.mjs`, and `patterns.mjs` own source parsing details.
+- `output.mjs` and `markdown-gen.mjs` own generated artifacts.

infra/docs-gen/README.md

@@ -0,0 +1,48 @@
+# Docs Generator
+
+`infra/docs-gen` contains tooling for two documentation flows:
+
+- building Diplodoc input from `docs/*.md`;
+- extracting raw extension metadata from `packages/editor/src/extensions`.
+
+## Commands
+
+- `pnpm --filter @markdown-editor/docs-gen run build` creates `tmp/docs-src` and builds `dist/docs`.
+- `pnpm --filter @markdown-editor/docs-gen run extract` creates `tmp/docs-gen/extensions.json` and `tmp/docs-gen/raw/*.md`.
+- `pnpm --filter @markdown-editor/docs-gen run test` runs the colocated Node unit tests.
+
+## Files
+
+- `package.json` defines the local docs-gen package, scripts, and Nx target.
+- `EXTRACTION_PIPELINE.md` documents the raw extension extraction flow with a Mermaid diagram.
+- `src/config.mjs` stores shared paths, extension categories, ignored internal extensions, preset definitions, and generator regex constants.
+- `src/extract-extension-data.mjs` provides the CLI entry point for raw extension extraction.
+- `src/generate-docs.mjs` builds the Diplodoc source tree from repository markdown files.
+- `src/logger.mjs` contains the small console logger used by docs-gen CLIs.
+- `src/utils.mjs` contains filesystem helpers shared by the build and extraction flows.
+
+## Extractor Files
+
+- `src/extractor/index.mjs` contains `ExtensionExtractor`, the high-level orchestrator that scans extension categories, enriches records with presets, and writes output.
+- `src/extractor/scan.mjs` scans one extension directory and assembles the raw extension IR record.
+- `src/extractor/source-files.mjs` selects source, spec, serializer, root index, specs index, and test files from an extension directory.
+- `src/extractor/output.mjs` writes extracted JSON and raw Markdown artifacts.
+- `src/extractor/markdown-gen.mjs` renders one raw extension Markdown file from extracted metadata.
+- `src/extractor/presets.mjs` parses editor preset files and resolves inherited preset membership.
+- `src/extractor/constants.mjs` extracts string constants, enum values, object scalar members, and resolves references between them.
+- `src/extractor/options.mjs` extracts extension option fields from local TypeScript option declarations.
+- `src/extractor/examples.mjs` extracts serializer test markup examples from `same(...)` calls and simple local string expressions.
+- `src/extractor/regex.mjs` contains shared scanner helpers and the source scanners for schema names, actions, keymaps, plugins, input rules, markdown-it plugins, and serializer hints.
+- `src/extractor/patterns.mjs` keeps the regular expressions used by extractor modules.
+- `src/extractor/actions.test.mjs` covers action extraction behavior.
+- `src/extractor/cli.test.mjs` covers extraction CLI argument parsing.
+- `src/extractor/constants.test.mjs` covers constant extraction and reference resolution behavior.
+- `src/extractor/examples.test.mjs` covers markup example extraction behavior.
+- `src/extractor/keymaps.test.mjs` covers keymap extraction behavior.
+- `src/extractor/options.test.mjs` covers option declaration extraction behavior.
+
+## Output
+
+- `tmp/docs-gen/extensions.json` is the machine-readable raw IR for all extracted extensions.
+- `tmp/docs-gen/raw/*.md` is the raw Markdown view of each extension record.
+- `tmp/docs-src` is the generated Diplodoc input tree used by the docs build.

infra/docs-gen/package.json

@@ -14,7 +14,9 @@
     }
   },
   "scripts": {
-    "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs"
+    "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs",
+    "extract": "node src/extract-extension-data.mjs",
+    "test": "node --test src/extractor/*.test.mjs"
   },
   "dependencies": {
     "@diplodoc/cli": "5.43.0"

infra/docs-gen/src/config.mjs

@@ -0,0 +1,37 @@
+import {dirname, join, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+export const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..');
+export const DOCS_DIR = join(REPO_ROOT, 'docs');
+export const DOCS_SRC_DIR = join(REPO_ROOT, 'tmp/docs-src');
+export const DOCS_GEN_DIR = join(REPO_ROOT, 'tmp/docs-gen');
+export const EDITOR_PKG_DIR = join(REPO_ROOT, 'packages/editor');
+
+export const GITHUB_RAW_RE =
+    /https:\/\/raw\.githubusercontent\.com\/gravity-ui\/markdown-editor\/(?:refs\/heads\/[^/]+|[^/]+)\/docs\//g;
+export const HEADER_RE = /^#{5}\s+(.+)$/;
+
+export const EXTENSION_CATEGORIES = ['base', 'behavior', 'markdown', 'yfm', 'additional'];
+
+export const INTERNAL_EXTENSIONS = [
+    'BaseInputRules',
+    'BaseKeymap',
+    'BaseStyles',
+    'ReactRenderer',
+    'SharedState',
+];
+
+export const PRESET_DEFS = [
+    {name: 'ZeroPreset', file: 'zero.ts', parent: null},
+    {name: 'CommonMarkPreset', file: 'commonmark.ts', parent: 'ZeroPreset'},
+    {name: 'DefaultPreset', file: 'default.ts', parent: 'CommonMarkPreset'},
+    {name: 'YfmPreset', file: 'yfm.ts', parent: 'DefaultPreset'},
+    {name: 'FullPreset', file: 'full.ts', parent: 'YfmPreset'},
+];
+
+/**
+ * Checks whether an extension is internal-only infrastructure.
+ */
+export function isInternalExtension(name) {
+    return INTERNAL_EXTENSIONS.includes(name);
+}

infra/docs-gen/src/extract-extension-data.mjs

@@ -0,0 +1,133 @@
+#!/usr/bin/env node
+import {isAbsolute, join} from 'node:path';
+import process from 'node:process';
+import {fileURLToPath} from 'node:url';
+
+import {DOCS_GEN_DIR, EDITOR_PKG_DIR, REPO_ROOT} from './config.mjs';
+import {ExtensionExtractor} from './extractor/index.mjs';
+import {logger} from './logger.mjs';
+
+/**
+ * Resolves a path from the repository root.
+ */
+function resolveFromRoot(path) {
+    return isAbsolute(path) ? path : join(REPO_ROOT, path);
+}
+
+/**
+ * Creates default CLI options.
+ */
+function createDefaultOptions() {
+    return {
+        editorPkg: EDITOR_PKG_DIR,
+        outDir: DOCS_GEN_DIR,
+        only: null,
+    };
+}
+
+/**
+ * Reads a required option value.
+ */
+function readOptionValue(args, index, optionName) {
+    const value = args[index + 1];
+    if (!value || value.startsWith('--')) {
+        throw new Error(`Missing value for ${optionName}`);
+    }
+
+    return value;
+}
+
+/**
+ * Parses comma-separated extension names.
+ */
+function parseOnlyOption(value) {
+    return value
+        .split(',')
+        .map((item) => item.trim())
+        .filter(Boolean);
+}
+
+/**
+ * Applies one CLI option to a new options object.
+ */
+function applyOption(opts, args, index) {
+    const arg = args[index];
+
+    switch (arg) {
+        case '--editor-pkg':
+            return {
+                nextIndex: index + 1,
+                opts: {...opts, editorPkg: resolveFromRoot(readOptionValue(args, index, arg))},
+            };
+        case '--out-dir':
+            return {
+                nextIndex: index + 1,
+                opts: {...opts, outDir: resolveFromRoot(readOptionValue(args, index, arg))},
+            };
+        case '--only':
+            return {
+                nextIndex: index + 1,
+                opts: {...opts, only: parseOnlyOption(readOptionValue(args, index, arg))},
+            };
+        case '--help':
+            return {nextIndex: index, opts: {...opts, help: true}};
+        default:
+            throw new Error(`Unknown option: ${arg}`);
+    }
+}
+
+/**
+ * Parses CLI options for extension data extraction.
+ */
+export function parseArgs(args = process.argv.slice(2)) {
+    let opts = createDefaultOptions();
+
+    for (let index = 0; index < args.length; index++) {
+        const arg = args[index];
+        if (arg === '--') return opts;
+
+        const parsedOption = applyOption(opts, args, index);
+        opts = parsedOption.opts;
+        index = parsedOption.nextIndex;
+    }
+
+    return opts;
+}
+
+/**
+ * Prints command usage information.
+ */
+function printHelp() {
+    logger.info('Usage: pnpm --filter @markdown-editor/docs-gen run extract [options]');
+    logger.info('');
+    logger.info('Options:');
+    logger.info('  --only Bold,Link       Extract selected extension names');
+    logger.info('  --out-dir tmp/docs-gen Override output directory');
+    logger.info('  --editor-pkg path      Override packages/editor path');
+}
+
+/**
+ * Runs extension data extraction from CLI arguments.
+ */
+export function main(args = process.argv.slice(2)) {
+    const opts = parseArgs(args);
+    if (opts.help) {
+        printHelp();
+        return;
+    }
+
+    new ExtensionExtractor({
+        editorPkg: opts.editorPkg,
+        outDir: opts.outDir,
+        repoRoot: REPO_ROOT,
+    }).run({only: opts.only});
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+    try {
+        main();
+    } catch (error) {
+        logger.error(error);
+        process.exit(1);
+    }
+}

infra/docs-gen/src/extractor/actions.test.mjs

@@ -0,0 +1,25 @@
+import assert from 'node:assert/strict';
+import {test} from 'node:test';
+
+import {extractActions} from './regex.mjs';
+
+test('extractActions captures direct and chained builder.addAction calls', () => {
+    const content = [
+        "builder.addAction('bold', () => boldAction);",
+        'builder',
+        '    .addAction(BoldAction.Toggle, () => toggle)',
+        '    .addAction(BoldAction.Off, () => off);',
+    ].join('\n');
+
+    assert.deepEqual(extractActions(content), ['bold', 'BoldAction.Toggle', 'BoldAction.Off']);
+});
+
+test('extractActions ignores non-builder addAction calls', () => {
+    const content = [
+        "tr.addAction('shouldNotMatch', cb);",
+        "service.addAction('alsoSkip', cb);",
+        "builder.addAction('keepMe', cb);",
+    ].join('\n');
+
+    assert.deepEqual(extractActions(content), ['keepMe']);
+});

infra/docs-gen/src/extractor/cli.test.mjs

@@ -0,0 +1,17 @@
+import assert from 'node:assert/strict';
+import {test} from 'node:test';
+
+import {parseArgs} from '../extract-extension-data.mjs';
+
+test('parseArgs parses selected extensions', () => {
+    assert.deepEqual(parseArgs(['--only', 'Bold, Link']).only, ['Bold', 'Link']);
+});
+
+test('parseArgs stops option parsing after separator', () => {
+    assert.equal(parseArgs(['--help', '--', '--unknown']).help, true);
+});
+
+test('parseArgs rejects missing option values', () => {
+    assert.throws(() => parseArgs(['--out-dir']), /Missing value for --out-dir/u);
+    assert.throws(() => parseArgs(['--only', '--help']), /Missing value for --only/u);
+});