build(docs-gen): add simple extension name extractor

infra/docs-gen/package.json

@@ -14,9 +14,12 @@
     }
   },
   "scripts": {
-    "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs"
+    "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs",
+    "extract": "node src/extract-extension-data.mjs",
+    "test": "node --test src/*.test.mjs"
   },
   "dependencies": {
-    "@diplodoc/cli": "5.43.0"
+    "@diplodoc/cli": "5.43.0",
+    "typescript": "catalog:ts"
   }
 }

infra/docs-gen/src/extension-ast.mjs

@@ -0,0 +1,120 @@
+/* eslint-disable jsdoc/require-param, jsdoc/require-returns */
+import ts from 'typescript';
+
+import {EXTENSION_BUILDER_TYPE_NAMES, EXTENSION_TYPE_NAMES} from './extension-config.mjs';
+
+/** Reads the visible name from a TypeScript type reference. */
+function getTypeReferenceName(typeName) {
+    if (ts.isIdentifier(typeName)) return typeName.text;
+    if (ts.isQualifiedName(typeName)) return typeName.right.text;
+
+    return null;
+}
+
+/** Checks that a type annotation references one of the configured names. */
+function isTypeReferenceTo(typeNode, names) {
+    return (
+        typeNode &&
+        ts.isTypeReferenceNode(typeNode) &&
+        names.has(getTypeReferenceName(typeNode.typeName))
+    );
+}
+
+/** Removes syntax wrappers that do not change the checked expression. */
+function unwrapExpression(expression) {
+    let current = expression;
+
+    while (
+        ts.isParenthesizedExpression(current) ||
+        ts.isAsExpression(current) ||
+        ts.isSatisfiesExpression(current) ||
+        ts.isNonNullExpression(current) ||
+        ts.isTypeAssertionExpression(current)
+    ) {
+        current = current.expression;
+    }
+
+    return current;
+}
+
+/** Detects direct `export` modifiers on a top-level declaration statement. */
+function hasExportModifier(node) {
+    return ts.getModifiers(node)?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword);
+}
+
+/** Collects top-level variable declarations from a parsed source file. */
+function readVariableDeclarations(sourceFile) {
+    return sourceFile.statements.flatMap((statement) => {
+        if (!ts.isVariableStatement(statement)) return [];
+
+        return statement.declarationList.declarations
+            .filter((declaration) => ts.isIdentifier(declaration.name))
+            .map((declaration) => ({statement, declaration}));
+    });
+}
+
+/** Detects declarations typed as configured extensions. */
+function isExtensionDeclaration(declaration) {
+    return isTypeReferenceTo(declaration.type, EXTENSION_TYPE_NAMES);
+}
+
+/** Detects builder-style extension initializers. */
+function isBuilderExtensionInitializer(initializer) {
+    const current = initializer && unwrapExpression(initializer);
+
+    return (
+        current &&
+        ts.isArrowFunction(current) &&
+        isTypeReferenceTo(current.parameters[0]?.type, EXTENSION_BUILDER_TYPE_NAMES)
+    );
+}
+
+/** Detects public exports built from a known extension implementation. */
+function isObjectAssignFromKnownExtension(initializer, extensionImplementations) {
+    const current = initializer && unwrapExpression(initializer);
+    if (
+        !current ||
+        !ts.isCallExpression(current) ||
+        !ts.isPropertyAccessExpression(current.expression)
+    ) {
+        return false;
+    }
+
+    const callee = current.expression;
+    const firstArg = current.arguments[0];
+
+    return (
+        ts.isIdentifier(callee.expression) &&
+        callee.expression.text === 'Object' &&
+        callee.name.text === 'assign' &&
+        firstArg &&
+        ts.isIdentifier(firstArg) &&
+        extensionImplementations.has(firstArg.text)
+    );
+}
+
+/** Detects exported declarations that represent extensions. */
+function isExtensionExport({statement, declaration}, extensionImplementations) {
+    if (!hasExportModifier(statement)) return false;
+
+    return (
+        isExtensionDeclaration(declaration) ||
+        isBuilderExtensionInitializer(declaration.initializer) ||
+        isObjectAssignFromKnownExtension(declaration.initializer, extensionImplementations)
+    );
+}
+
+/** Reads extension export names from a source file. */
+export function readExtensionExportNames(content) {
+    const sourceFile = ts.createSourceFile('source.tsx', content, ts.ScriptTarget.Latest, true);
+    const declarations = readVariableDeclarations(sourceFile);
+    const extensionImplementations = new Set(
+        declarations
+            .filter(({declaration}) => isExtensionDeclaration(declaration))
+            .map(({declaration}) => declaration.name.text),
+    );
+
+    return declarations
+        .filter((entry) => isExtensionExport(entry, extensionImplementations))
+        .map(({declaration}) => declaration.name.text);
+}

infra/docs-gen/src/extension-config.mjs

@@ -0,0 +1,25 @@
+import {dirname, join, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+
+export const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..');
+export const DOCS_GEN_DIR = join(REPO_ROOT, 'tmp/docs-gen');
+export const EDITOR_EXTENSIONS_DIR = join(REPO_ROOT, 'packages/editor/src/extensions');
+export const PAGE_CONSTRUCTOR_EXTENSION_DIR = join(
+    REPO_ROOT,
+    'packages/page-constructor-extension/src/extension',
+);
+
+export const EXTENSION_CATEGORIES = ['base', 'behavior', 'markdown', 'yfm', 'additional'];
+export const EXTRA_EXTENSION_DIRS = [PAGE_CONSTRUCTOR_EXTENSION_DIR];
+export const EXTENSION_BLACKLIST = [
+    'BaseInputRules',
+    'BaseKeymap',
+    'BaseStyles',
+    'ReactRendererExtension',
+    'Resizable',
+    'SharedState',
+    'YfmCut',
+];
+
+export const EXTENSION_TYPE_NAMES = new Set(['Extension', 'ExtensionAuto', 'ExtensionWithOptions']);
+export const EXTENSION_BUILDER_TYPE_NAMES = new Set(['ExtensionBuilder']);

infra/docs-gen/src/extract-extension-data.mjs

@@ -0,0 +1,78 @@
+#!/usr/bin/env node
+/* eslint-disable jsdoc/require-param, jsdoc/require-returns */
+import {existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync} from 'node:fs';
+import {join} from 'node:path';
+import process from 'node:process';
+import {fileURLToPath} from 'node:url';
+
+import {readExtensionExportNames} from './extension-ast.mjs';
+import {
+    DOCS_GEN_DIR,
+    EDITOR_EXTENSIONS_DIR,
+    EXTENSION_BLACKLIST,
+    EXTENSION_CATEGORIES,
+    EXTRA_EXTENSION_DIRS,
+} from './extension-config.mjs';
+
+/** Reads TypeScript sources from a directory recursively. */
+function readSourceFiles(dir) {
+    if (!existsSync(dir)) return [];
+
+    const files = [];
+    for (const entry of readdirSync(dir, {withFileTypes: true})) {
+        const fullPath = join(dir, entry.name);
+
+        if (entry.isDirectory()) {
+            files.push(...readSourceFiles(fullPath));
+        } else if (entry.isFile() && (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx'))) {
+            files.push(readFileSync(fullPath, 'utf-8'));
+        }
+    }
+
+    return files;
+}
+
+/** Builds configured source roots for extension scanning. */
+function listExtensionSourceDirs(extensionsDir, categories, extraExtensionDirs) {
+    return categories.map((category) => join(extensionsDir, category)).concat(extraExtensionDirs);
+}
+
+/** Builds the filtered list of public extension names. */
+export function listExtensionNames({
+    extensionsDir = EDITOR_EXTENSIONS_DIR,
+    categories = EXTENSION_CATEGORIES,
+    extraExtensionDirs = EXTRA_EXTENSION_DIRS,
+    blacklist = EXTENSION_BLACKLIST,
+} = {}) {
+    const blacklistSet = new Set(blacklist);
+    const names = listExtensionSourceDirs(extensionsDir, categories, extraExtensionDirs)
+        .flatMap(readSourceFiles)
+        .flatMap(readExtensionExportNames);
+
+    return [...new Set(names)]
+        .filter((name) => !blacklistSet.has(name))
+        .sort((left, right) => left.localeCompare(right));
+}
+
+/** Converts extension names into JSON records. */
+export function createExtensionRecords(names) {
+    return names.map((name) => ({name}));
+}
+
+/** Writes generated extension records into the docs-gen output directory. */
+export function writeExtensionsJson(outDir = DOCS_GEN_DIR, names = listExtensionNames()) {
+    mkdirSync(outDir, {recursive: true});
+    writeFileSync(
+        join(outDir, 'extensions.json'),
+        `${JSON.stringify({extensions: createExtensionRecords(names)}, null, 2)}\n`,
+    );
+}
+
+/** Runs the extension data extraction CLI. */
+export function main() {
+    writeExtensionsJson();
+}
+
+if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {
+    main();
+}

infra/docs-gen/src/extract-extension-data.test.mjs

@@ -0,0 +1,91 @@
+import assert from 'node:assert/strict';
+import {mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {afterEach, test} from 'node:test';
+
+import {
+    createExtensionRecords,
+    listExtensionNames,
+    writeExtensionsJson,
+} from './extract-extension-data.mjs';
+
+const cleanupDirs = [];
+
+function makeExtensionsRoot() {
+    const root = mkdtempSync(join(tmpdir(), 'docs-gen-extensions-'));
+    cleanupDirs.push(root);
+
+    return root;
+}
+
+function addExtensionFile(root, category, name, content) {
+    const dir = join(root, category, name);
+
+    mkdirSync(dir, {recursive: true});
+    writeFileSync(join(dir, 'index.ts'), content);
+}
+
+afterEach(() => {
+    for (const dir of cleanupDirs.splice(0)) {
+        rmSync(dir, {recursive: true, force: true});
+    }
+});
+
+test('listExtensionNames extracts AST-backed extension names and applies blacklist to exports', () => {
+    const extensionsDir = makeExtensionsRoot();
+    const extraDir = makeExtensionsRoot();
+
+    addExtensionFile(
+        extensionsDir,
+        'base',
+        'base-keymap',
+        'export const BaseKeymap: ExtensionAuto = () => {};',
+    );
+    addExtensionFile(extensionsDir, 'base', 'bold', 'export const Bold: ExtensionAuto = () => {};');
+    addExtensionFile(
+        extensionsDir,
+        'base',
+        'mixed',
+        [
+            'export const One: ExtensionAuto = () => {};',
+            'export const Two: ExtensionAuto = () => {};',
+        ].join('\n'),
+    );
+    addExtensionFile(
+        extensionsDir,
+        'behavior',
+        'Resizable',
+        'export const Resizable: React.FC = () => null;',
+    );
+    addExtensionFile(
+        extensionsDir,
+        'additional',
+        'GPT',
+        'export const gptExtension = (builder: ExtensionBuilder) => builder;',
+    );
+    addExtensionFile(extensionsDir, 'additional', 'Widget', 'export const Widget = () => null;');
+    writeFileSync(
+        join(extraDir, 'index.ts'),
+        'export const YfmPageConstructorExtension: ExtensionAuto = () => {};',
+    );
+
+    assert.deepEqual(
+        listExtensionNames({
+            extensionsDir,
+            categories: ['base', 'behavior', 'additional'],
+            extraExtensionDirs: [extraDir],
+        }),
+        ['Bold', 'One', 'Two', 'YfmPageConstructorExtension', 'gptExtension'],
+    );
+});
+
+test('writeExtensionsJson writes extension name records', () => {
+    const outDir = makeExtensionsRoot();
+
+    writeExtensionsJson(outDir, ['Bold', 'GPT']);
+
+    assert.deepEqual(JSON.parse(readFileSync(join(outDir, 'extensions.json'), 'utf-8')), {
+        extensions: createExtensionRecords(['Bold', 'GPT']),
+    });
+});

infra/docs-gen/src/generate-docs.mjs

@@ -11,6 +11,8 @@ import {dirname, join, resolve} from 'node:path';
 import process from 'node:process';
 import {fileURLToPath} from 'node:url';
 
+import {listExtensionNames} from './extract-extension-data.mjs';
+
 const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..');
 const DOCS_DIR = join(REPO_ROOT, 'docs');
 const OUT_DIR = join(REPO_ROOT, 'tmp/docs-src');
@@ -126,6 +128,15 @@ function computeOutputPath(doc) {
     return slugify(doc.title) + '.md';
 }
 
+function collectExtensionDocs() {
+    return listExtensionNames().map((name) => ({
+        sourceFile: `generated:extension:${name}`,
+        category: 'Extension Reference',
+        title: name,
+        content: `# ${name}\n`,
+    }));
+}
+
 /**
  * Ensures no two docs resolve to the same output path; exits on collision.
  * @param docs
@@ -257,7 +268,7 @@ function writeYfmConfig() {
 function main() {
     cleanOutDir();
 
-    const docs = collectDocs();
+    const docs = [...collectDocs(), ...collectExtensionDocs()];
     const {categories, topLevel} = groupByCategory(docs);
 
     writeYfmConfig();