build(docs-gen): add simple extension list extractor

Author: makhnatkin

infra/docs-gen/package.json

@@ -16,7 +16,7 @@
   "scripts": {
     "build": "node src/generate-docs.mjs && yfm -i ../../tmp/docs-src -o ../../dist/docs",
     "extract": "node src/extract-extension-data.mjs",
-    "test": "node --test src/extractor/*.test.mjs"
+    "test": "node --test src/*.test.mjs"
   },
   "dependencies": {
     "@diplodoc/cli": "5.43.0",

infra/docs-gen/src/extract-extension-data.mjs

@@ -1,12 +1,201 @@
 #!/usr/bin/env node
+import {existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync} from 'node:fs';
+import {dirname, join, resolve} from 'node:path';
+import process from 'node:process';
 import {fileURLToPath} from 'node:url';
 
-import {DOCS_GEN_DIR} from './extractor/config.mjs';
-import {extractExtensionData} from './extractor/index.mjs';
-import {writeExtensionsJson} from './extractor/output.mjs';
+import ts from 'typescript';
+
+export const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '../../..');
+export const DOCS_GEN_DIR = join(REPO_ROOT, 'tmp/docs-gen');
+export const EDITOR_EXTENSIONS_DIR = join(REPO_ROOT, 'packages/editor/src/extensions');
+export const PAGE_CONSTRUCTOR_EXTENSION_DIR = join(
+    REPO_ROOT,
+    'packages/page-constructor-extension/src/extension',
+);
+export const EXTENSION_CATEGORIES = ['base', 'behavior', 'markdown', 'yfm', 'additional'];
+export const EXTRA_EXTENSION_REFS = [
+    {name: 'YfmPageConstructorExtension', dir: PAGE_CONSTRUCTOR_EXTENSION_DIR},
+];
+export const EXTENSION_BLACKLIST = [
+    'BaseInputRules',
+    'BaseKeymap',
+    'BaseStyles',
+    'ReactRenderer',
+    'Resizable',
+    'SharedState',
+    'YfmCut',
+];
+
+const EXTENSION_TYPE_NAMES = new Set(['Extension', 'ExtensionAuto', 'ExtensionWithOptions']);
+const EXTENSION_BUILDER_TYPE_NAMES = new Set(['ExtensionBuilder']);
+
+function startsWithUppercaseLetter(name) {
+    const firstChar = name.charAt(0);
+
+    return (
+        firstChar !== '' &&
+        firstChar === firstChar.toUpperCase() &&
+        firstChar !== firstChar.toLowerCase()
+    );
+}
+
+function listExtensionRefs(dir) {
+    if (!existsSync(dir)) return [];
+
+    return readdirSync(dir, {withFileTypes: true})
+        .filter((entry) => entry.isDirectory() && startsWithUppercaseLetter(entry.name))
+        .map((entry) => ({name: entry.name, dir: join(dir, entry.name)}))
+        .sort((left, right) => left.name.localeCompare(right.name));
+}
+
+function readSourceFiles(dir) {
+    if (!existsSync(dir)) return [];
+
+    const files = [];
+    for (const entry of readdirSync(dir, {withFileTypes: true})) {
+        const fullPath = join(dir, entry.name);
+
+        if (entry.isDirectory()) {
+            files.push(...readSourceFiles(fullPath));
+        } else if (entry.isFile() && (entry.name.endsWith('.ts') || entry.name.endsWith('.tsx'))) {
+            files.push(readFileSync(fullPath, 'utf-8'));
+        }
+    }
+
+    return files;
+}
+
+function getTypeReferenceName(typeName) {
+    if (ts.isIdentifier(typeName)) return typeName.text;
+    if (ts.isQualifiedName(typeName)) return typeName.right.text;
+
+    return null;
+}
+
+function isTypeReferenceTo(typeNode, names) {
+    return (
+        typeNode &&
+        ts.isTypeReferenceNode(typeNode) &&
+        names.has(getTypeReferenceName(typeNode.typeName))
+    );
+}
+
+function unwrapExpression(expression) {
+    let current = expression;
+
+    while (
+        ts.isParenthesizedExpression(current) ||
+        ts.isAsExpression(current) ||
+        ts.isSatisfiesExpression(current) ||
+        ts.isNonNullExpression(current) ||
+        ts.isTypeAssertionExpression(current)
+    ) {
+        current = current.expression;
+    }
+
+    return current;
+}
+
+function hasExportModifier(node) {
+    return ts.getModifiers(node)?.some((modifier) => modifier.kind === ts.SyntaxKind.ExportKeyword);
+}
+
+function readVariableDeclarations(sourceFile) {
+    return sourceFile.statements.flatMap((statement) => {
+        if (!ts.isVariableStatement(statement)) return [];
+
+        return statement.declarationList.declarations
+            .filter((declaration) => ts.isIdentifier(declaration.name))
+            .map((declaration) => ({statement, declaration}));
+    });
+}
+
+function isExtensionInitializer(initializer) {
+    const current = initializer && unwrapExpression(initializer);
+
+    return (
+        current &&
+        ts.isArrowFunction(current) &&
+        isTypeReferenceTo(current.parameters[0]?.type, EXTENSION_BUILDER_TYPE_NAMES)
+    );
+}
+
+function isObjectAssignFromKnownExtension(initializer, extensionImplementations) {
+    const current = initializer && unwrapExpression(initializer);
+    if (
+        !current ||
+        !ts.isCallExpression(current) ||
+        !ts.isPropertyAccessExpression(current.expression)
+    ) {
+        return false;
+    }
+
+    const callee = current.expression;
+    const firstArg = current.arguments[0];
+
+    return (
+        ts.isIdentifier(callee.expression) &&
+        callee.expression.text === 'Object' &&
+        callee.name.text === 'assign' &&
+        firstArg &&
+        ts.isIdentifier(firstArg) &&
+        extensionImplementations.has(firstArg.text)
+    );
+}
+
+function sourceHasExtensionExport(content) {
+    const sourceFile = ts.createSourceFile('source.tsx', content, ts.ScriptTarget.Latest, true);
+    const declarations = readVariableDeclarations(sourceFile);
+    const extensionImplementations = new Set(
+        declarations
+            .filter(({declaration}) => isTypeReferenceTo(declaration.type, EXTENSION_TYPE_NAMES))
+            .map(({declaration}) => declaration.name.text),
+    );
+
+    return declarations.some(({statement, declaration}) => {
+        if (!hasExportModifier(statement)) return false;
+
+        return (
+            isTypeReferenceTo(declaration.type, EXTENSION_TYPE_NAMES) ||
+            isExtensionInitializer(declaration.initializer) ||
+            isObjectAssignFromKnownExtension(declaration.initializer, extensionImplementations)
+        );
+    });
+}
+
+function refHasExtensionExport(ref) {
+    return readSourceFiles(ref.dir).some(sourceHasExtensionExport);
+}
+
+export function listExtensionNames({
+    extensionsDir = EDITOR_EXTENSIONS_DIR,
+    categories = EXTENSION_CATEGORIES,
+    extraExtensionRefs = EXTRA_EXTENSION_REFS,
+    blacklist = EXTENSION_BLACKLIST,
+} = {}) {
+    const blacklistSet = new Set(blacklist);
+    const refs = categories.flatMap((category) => listExtensionRefs(join(extensionsDir, category)));
+
+    return [...refs, ...extraExtensionRefs]
+        .filter((ref) => !blacklistSet.has(ref.name) && refHasExtensionExport(ref))
+        .map((ref) => ref.name);
+}
+
+export function createExtensionRecords(names) {
+    return names.map((name) => ({name}));
+}
+
+export function writeExtensionsJson(outDir = DOCS_GEN_DIR, names = listExtensionNames()) {
+    mkdirSync(outDir, {recursive: true});
+    writeFileSync(
+        join(outDir, 'extensions.json'),
+        `${JSON.stringify({extensions: createExtensionRecords(names)}, null, 2)}\n`,
+    );
+}
 
 export function main() {
-    writeExtensionsJson(DOCS_GEN_DIR, extractExtensionData());
+    writeExtensionsJson();
 }
 
 if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) {

infra/docs-gen/src/extract-extension-data.test.mjs

@@ -0,0 +1,82 @@
+import assert from 'node:assert/strict';
+import {mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync} from 'node:fs';
+import {tmpdir} from 'node:os';
+import {join} from 'node:path';
+import {afterEach, test} from 'node:test';
+
+import {
+    createExtensionRecords,
+    listExtensionNames,
+    writeExtensionsJson,
+} from './extract-extension-data.mjs';
+
+const cleanupDirs = [];
+
+function makeExtensionsRoot() {
+    const root = mkdtempSync(join(tmpdir(), 'docs-gen-extensions-'));
+    cleanupDirs.push(root);
+
+    return root;
+}
+
+function addExtensionFile(root, category, name, content) {
+    const dir = join(root, category, name);
+
+    mkdirSync(dir, {recursive: true});
+    writeFileSync(join(dir, 'index.ts'), content);
+}
+
+afterEach(() => {
+    for (const dir of cleanupDirs.splice(0)) {
+        rmSync(dir, {recursive: true, force: true});
+    }
+});
+
+test('listExtensionNames keeps AST-backed extension dirs and applies blacklist', () => {
+    const extensionsDir = makeExtensionsRoot();
+    const extraDir = makeExtensionsRoot();
+
+    addExtensionFile(
+        extensionsDir,
+        'base',
+        'BaseKeymap',
+        'export const BaseKeymap: ExtensionAuto = () => {};',
+    );
+    addExtensionFile(extensionsDir, 'base', 'Bold', 'export const Bold: ExtensionAuto = () => {};');
+    addExtensionFile(
+        extensionsDir,
+        'behavior',
+        'Resizable',
+        'export const Resizable: React.FC = () => null;',
+    );
+    addExtensionFile(
+        extensionsDir,
+        'additional',
+        'GPT',
+        'export const gptExtension = (builder: ExtensionBuilder) => builder;',
+    );
+    addExtensionFile(extensionsDir, 'additional', 'Widget', 'export const Widget = () => null;');
+    writeFileSync(
+        join(extraDir, 'index.ts'),
+        'export const YfmPageConstructorExtension: ExtensionAuto = () => {};',
+    );
+
+    assert.deepEqual(
+        listExtensionNames({
+            extensionsDir,
+            categories: ['base', 'behavior', 'additional'],
+            extraExtensionRefs: [{name: 'YfmPageConstructorExtension', dir: extraDir}],
+        }),
+        ['Bold', 'GPT', 'YfmPageConstructorExtension'],
+    );
+});
+
+test('writeExtensionsJson writes extension name records', () => {
+    const outDir = makeExtensionsRoot();
+
+    writeExtensionsJson(outDir, ['Bold', 'GPT']);
+
+    assert.deepEqual(JSON.parse(readFileSync(join(outDir, 'extensions.json'), 'utf-8')), {
+        extensions: createExtensionRecords(['Bold', 'GPT']),
+    });
+});