Reworkd format logic to use the markdown parser instead of the old hacky regex-based approach

Author: jeremy-rifkin

src/components/format.ts

@@ -13,60 +13,19 @@ import { Wheatley } from "../wheatley.js";
 import { MINUTE } from "../common.js";
 import { MessageContextMenuInteractionBuilder } from "../command-abstractions/context-menu.js";
 import { async_exec_file } from "../utils/filesystem.js";
+import { markdown_node, MarkdownParser, CodeBlockRule, InlineCodeRule, TextRule } from "dismark";
 import { Mutex } from "../utils/containers.js";
 
 const color = 0x7e78fe; //0xA931FF;
 
+const code_only_parser = new MarkdownParser([new CodeBlockRule(), new InlineCodeRule(), new TextRule()]);
+
 const clang_format_path = "/usr/bin/clang-format";
 
 const max_attachment_size = 1024 * 10;
 
-// highlight js accepts all
-// prettier-ignore
-const languages = [
-    "1c", "4d", "abnf", "accesslog", "actionscript", "ada", "adoc", "alan", "angelscript", "apache", "apacheconf",
-    "applescript", "arcade", "arduino", "arm", "armasm", "as", "asc", "asciidoc", "aspectj", "atom", "autohotkey",
-    "autoit", "avrasm", "awk", "axapta", "bash", "basic", "bat", "bbcode", "bf", "bind", "blade", "bnf", "brainfuck",
-    "c", "c++", "cal", "capnp", "capnproto", "cc", "chaos", "chapel", "chpl", "cisco", "clj", "clojure", "cls",
-    "cmake.in", "cmake", "cmd", "coffee", "coffeescript", "console", "coq", "cos", "cpc", "cpp", "cr", "craftcms",
-    "crm", "crmsh", "crystal", "cs", "csharp", "cshtml", "cson", "csp", "css", "cxx", "cypher", "d", "dart", "delphi",
-    "dfm", "diff", "django", "dns", "docker", "dockerfile", "dos", "dpr", "dsconfig", "dst", "dts", "dust", "dylan",
-    "ebnf", "elixir", "elm", "erl", "erlang", "excel", "extempore", "f90", "f95", "fix", "fortran", "freepascal", "fs",
-    "fsharp", "gams", "gauss", "gawk", "gcode", "gdscript", "gemspec", "gf", "gherkin", "glimmer", "glsl", "gms", "gn",
-    "gni", "go", "godot", "golang", "golo", "gololang", "gradle", "graph", "groovy", "gss", "gyp", "h", "h++", "haml",
-    "handlebars", "haskell", "haxe", "hbs", "hbs", "hcl", "hh", "hlsl", "hpp", "hs", "html.handlebars",
-    "html.handlebars", "html.hbs", "html.hbs", "html", "htmlbars", "http", "https", "hx", "hxx", "hy", "hylang", "i",
-    "i7", "iced", "iecst", "inform7", "ini", "ino", "instances", "iol", "irb", "irpf90", "java", "javascript", "jinja",
-    "jolie", "js", "json", "jsp", "jsx", "julia-repl", "julia", "k", "kaos", "kdb", "kotlin", "kt", "lasso",
-    "lassoscript", "lazarus", "ldif", "leaf", "lean", "less", "lfm", "lisp", "livecodeserver", "livescript", "ln",
-    "lpr", "ls", "ls", "lua", "mak", "make", "makefile", "markdown", "mathematica", "matlab", "mawk", "maxima", "md",
-    "mel", "mercury", "mirc", "mizar", "mk", "mkd", "mkdown", "ml", "ml", "mm", "mma", "mojolicious", "monkey", "moon",
-    "moonscript", "mrc", "n1ql", "nawk", "nc", "never", "nginx", "nginxconf", "nim", "nimrod", "nix", "nsis", "obj-c",
-    "obj-c++", "objc", "objective-c++", "objectivec", "ocaml", "ocl", "ol", "openscad", "osascript", "oxygene", "p21",
-    "papyrus", "parser3", "pas", "pascal", "patch", "pcmk", "perl", "pf.conf", "pf", "pgsql", "php", "php3", "php4",
-    "php5", "php6", "php7", "php8", "pl", "plaintext", "plist", "pm", "podspec", "pony", "postgres", "postgresql",
-    "powershell", "pp", "processing", "profile", "prolog", "properties", "protobuf", "ps", "ps1", "psc", "puppet", "py",
-    "pycon", "python-repl", "python", "qml", "qsharp", "r", "razor-cshtml", "razor", "rb", "re", "reasonml", "rebol",
-    "red-system", "red", "redbol", "rf", "rib", "risc", "riscript", "robot", "rpm-spec", "rpm-specfile", "rpm", "rs",
-    "rsl", "rss", "ruby", "ruleslanguage", "rust", "sas", "SAS", "sc", "scad", "scala", "scheme", "sci", "scilab",
-    "scl", "scss", "sh", "shell", "shexc", "smali", "smalltalk", "sml", "sol", "solidity", "spec", "specfile", "spl",
-    "sql", "st", "stan", "stanfuncs", "stata", "step", "stl", "stp", "structured-text", "styl", "stylus", "subunit",
-    "supercollider", "svelte", "svg", "swift", "tao", "tap", "tcl", "terraform", "tex", "text", "tf", "thor", "thrift",
-    "tk", "toml", "tp", "ts", "tsql", "twig", "txt", "typescript", "unicorn-rails-log", "v", "vala", "vb", "vba",
-    "vbnet", "vbs", "vbscript", "verilog", "vhdl", "vim", "wl", "x++", "x86asm", "xhtml", "xjb", "xl", "xls", "xlsx",
-    "xml", "xpath", "xq", "xquery", "xsd", "xsl", "xtlang", "xtm", "yaml", "yml", "zenscript", "zep", "zephir", "zone",
-    "zs", "zsh"
-];
-
 const c_cpp_language_codes = new Set(["c", "h", "cpp", "hpp", "cc", "hh", "cxx", "cxx", "c++", "h++"]);
 
-const languages_re = new RegExp(
-    languages
-        .sort((a, b) => b.length - a.length)
-        .map(x => x.replaceAll("+", "\\+"))
-        .join("|"),
-);
-
 const code_begin = [
     "//",
     "/\\*",
@@ -109,8 +68,6 @@ function word_boundary(regex: string) {
 
 const code_begin_re = new RegExp(`(?:${code_begin.map(word_boundary).join("|")})`);
 
-const code_block_re = new RegExp(`(\`\`\`(?:${languages_re.source}\b)?)(.*?)\`\`\``, "gims");
-
 const default_clang_format_language = "cpp";
 
 const ignore_prefixes = [";compile", ";asm"];
@@ -148,48 +105,55 @@ export async function clang_format_general(text: string) {
     return await clang_format(text, [`-style={${clang_format_style.join(", ")}}`]);
 }
 
-// https://stackoverflow.com/questions/12568097/how-can-i-replace-a-string-by-range
-function replace_range(s: string, start: number, end: number, substitute: string) {
-    return s.substring(0, start) + substitute + s.substring(end);
-}
-
-async function format(replying_to: Discord.Message) {
-    let content = replying_to.content;
-    // does the message have code blocks?
-    const code_blocks: { language: string; content: string }[] = [];
-    content = content.replaceAll(code_block_re, (_, starter: string, block: string) => {
-        const language = starter.length > 3 ? starter.substring(3) : "cpp";
-        code_blocks.push({ language, content: block });
-        return `<[<[<[<[${code_blocks.length - 1}]>]>]>]>`;
-    });
-    // else ...
-    if (code_blocks.length == 0) {
-        const start = content.search(code_begin_re);
-        if (start > -1) {
-            const end = Math.max(...[...";}"].map(c => content.lastIndexOf(c)));
-            if (end > start) {
-                code_blocks.push({
-                    language: default_clang_format_language,
-                    content: content.substring(start, end + 1),
-                });
-                content = replace_range(content, start, end + 1, `<[<[<[<[${code_blocks.length - 1}]>]>]>]>`);
+async function format_message_content(ast: markdown_node): Promise<{ content: string; found_code: boolean }> {
+    let found_code = false;
+
+    const format_node = async (node: markdown_node): Promise<string> => {
+        switch (node.type) {
+            case "doc":
+                return (await Promise.all(node.content.map(format_node))).join("");
+            case "code_block": {
+                found_code = true;
+                const language = node.language ?? default_clang_format_language;
+                if (c_cpp_language_codes.has(language)) {
+                    const formatted = await clang_format_general(node.content);
+                    return `\`\`\`${node.language ?? ""}\n${formatted}\n\`\`\``;
+                } else {
+                    return `\`\`\`${node.language ?? ""}\n${node.content}\n\`\`\``;
+                }
             }
+            case "inline_code":
+                return `\`${node.content}\``;
+            case "plain": {
+                const code_start = node.content.search(code_begin_re);
+                if (code_start > -1) {
+                    const end = Math.max(...[...";}"].map(c => node.content.lastIndexOf(c)));
+                    if (end > code_start) {
+                        found_code = true;
+                        const code_content = node.content.substring(code_start, end + 1);
+                        const formatted = await clang_format_general(code_content);
+                        return (
+                            node.content.substring(0, code_start) +
+                            `\`\`\`${default_clang_format_language}\n${formatted}\n\`\`\`` +
+                            node.content.substring(end + 1)
+                        );
+                    }
+                }
+                return node.content;
+            }
+            default:
+                throw new Error(`Unknown ast node ${(node as markdown_node).type}`);
         }
-    }
+    };
 
-    for (const [i, block] of code_blocks.entries()) {
-        if (c_cpp_language_codes.has(block.language)) {
-            content = content.replace(
-                `<[<[<[<[${i}]>]>]>]>`,
-                `\`\`\`${block.language}\n${await clang_format_general(block.content)}\n\`\`\``,
-            );
-        } else {
-            // don't format, just put it back
-            content = content.replace(`<[<[<[<[${i}]>]>]>]>`, `\`\`\`${block.language}\n${block.content}\n\`\`\``);
-        }
-    }
+    const content = await format_node(ast);
+    return { content, found_code };
+}
+
+async function format(replying_to: Discord.Message) {
+    const ast = code_only_parser.parse(replying_to.content);
+    const { content, found_code } = await format_message_content(ast);
 
-    // does the message have attachments?
     const attachments = await Promise.all(
         [...replying_to.attachments.values()]
             .filter(attachment => attachment.contentType?.startsWith("text/") ?? false)
@@ -208,7 +172,11 @@ async function format(replying_to: Discord.Message) {
             }),
     );
 
-    return { content, attachments, found_code_blocks: code_blocks.length > 0 };
+    return {
+        content,
+        attachments,
+        found_code_blocks: found_code,
+    };
 }
 
 function should_replace_original(replying_to: Discord.Message, request_timestamp: Date) {