fix(website): avoid exponential pandoc parse on sidebar titles (#14576)

The navigation envelope concentrated every injected inline title into a
single hidden markdown paragraph. Pandoc's markdown reader backtracks
exponentially over unresolved emphasis candidates inside consecutive
bracketed inlines (jgm/pandoc#11687), so a site whose page titles merely
contain double-underscore names (e.g. `Class.__method__()`) hung on every
page render: ~20 such titles pushed parse time past 180s, with nothing
user-visible to debug.

Joining the spans with a blank line puts each title in its own paragraph,
so pandoc parses each independently and parse time stays linear. This keeps
markdown processing intact (shortcodes, icons, bold/code in titles still
render), unlike escaping or raw-wrapping the injected content.

Author: cderv

news/changelog-1.10.md

@@ -102,6 +102,7 @@ All changes included in 1.10:
 - ([#14461](https://github.com/quarto-dev/quarto-cli/issues/14461)): Fix `quarto render --to pdf` aborting with `ERROR: Problem running 'fmtutil-sys --all' to rebuild format tree.` when an automatically-installed LaTeX package's post-update format rebuild fails. Format-tree rebuild is now treated as best-effort housekeeping (matching upstream `tinytex` R behavior) — the failure is logged as a warning and the package install completes.
 - ([#14472](https://github.com/quarto-dev/quarto-cli/issues/14472)): Add support for Kotlin in code annotations and YAML cell options. (author: @barendgehrels)
 - ([#14529](https://github.com/quarto-dev/quarto-cli/issues/14529)): Fix bundled Julia engine path leaking into rendered YAML metadata and pandoc log output when running an installed Quarto. The internal subtree-engine filter only matched the source-tree share-path layout (`resources/extension-subtrees/`) and missed installed layouts where the path is `share/extension-subtrees/`.
+- ([#14576](https://github.com/quarto-dev/quarto-cli/issues/14576)): Fix website render hanging when many sidebar titles contain double-underscore (dunder) names such as `Transcript.__getitem__()`.
 - ([#14582](https://github.com/quarto-dev/quarto-cli/issues/14582)): Fix format detection for extension formats (e.g. `acm-pdf`) in project preview, manuscript notebooks, MECA bundles, and website format ordering.
 - ([#14583](https://github.com/quarto-dev/quarto-cli/issues/14583)): Fix a shortcode used as an image source (e.g. `![]({{< meta logo >}})`) getting the `default-image-extension` appended, producing a doubled extension once the shortcode resolves.
 - ([#14595](https://github.com/quarto-dev/quarto-cli/issues/14595)): Fix reload preview in code-server environment

src/core/markdown-pipeline.ts

@@ -116,7 +116,7 @@ const markdownEnvelopeWriter = (envelopeId: string) => {
       renderList.push(hiddenSpan(id, value));
     },
     toMarkdown: () => {
-      const contents = renderList.join("\n");
+      const contents = renderList.join("\n\n");
       return `\n:::{#${envelopeId} .hidden}\n${contents}\n:::\n`;
     },
   };

tests/smoke/site/render-sidebar-markdown-titles.test.ts

@@ -0,0 +1,121 @@
+/*
+ * render-sidebar-markdown-titles.test.ts
+ *
+ * Copyright (C) 2020-2024 Posit Software, PBC
+ *
+ */
+import { join } from "../../../src/deno_ral/path.ts";
+import { ensureDirSync, safeRemoveSync } from "../../../src/deno_ral/fs.ts";
+import { testQuartoCmd } from "../../test.ts";
+import { ensureFileRegexMatches, noErrors } from "../../verify.ts";
+
+// Regression test for GH #14576 / upstream jgm/pandoc#11687.
+//
+// Sidebar titles are rendered through the navigation "envelope": each title
+// becomes an inline span, the spans are joined into a single markdown document,
+// pandoc renders it, and the results are read back. When many sidebar titles
+// wrap text around a dunder name (e.g. `Transcript.__getitem__()`), joining the
+// spans into one paragraph made pandoc's markdown reader backtrack
+// exponentially over the unresolved `_`-emphasis candidates, hanging the
+// render. The fix joins each span as its own paragraph so the parse stays
+// linear. Pre-fix this render hangs (the test fails by timing out); post-fix it
+// completes quickly.
+//
+// The project is generated on the fly rather than stored as fixtures: the many
+// dunder pages would otherwise bloat the repo and each be picked up as its own
+// smoke-all render target.
+
+const dunderTitles = [
+  "__init__", "__call__", "__repr__", "__str__", "__len__",
+  "__getitem__", "__setitem__", "__delitem__", "__iter__", "__next__",
+  "__enter__", "__exit__", "__add__", "__sub__", "__mul__",
+  "__eq__", "__hash__", "__new__", "__del__", "__contains__",
+];
+
+const pageName = (i: number) => `page${String(i + 1).padStart(2, "0")}.qmd`;
+
+const projectDir = Deno.makeTempDirSync({ prefix: "quarto-sidebar-titles" });
+const outputDir = join(projectDir, "_site");
+
+const writeProject = () => {
+  ensureDirSync(projectDir);
+
+  const sidebarContents = [
+    "index.qmd",
+    ...dunderTitles.map((_, i) => pageName(i)),
+    "bold-page.qmd",
+    "code-page.qmd",
+  ].map((p) => `      - ${p}`).join("\n");
+
+  Deno.writeTextFileSync(
+    join(projectDir, "_quarto.yml"),
+    `project:
+  type: website
+website:
+  title: "Sidebar Titles"
+  sidebar:
+    contents:
+${sidebarContents}
+format:
+  html:
+    theme: cosmo
+`,
+  );
+
+  Deno.writeTextFileSync(
+    join(projectDir, "index.qmd"),
+    `---\ntitle: "Home"\n---\n\nSidebar markdown-title regression guard for #14576.\n`,
+  );
+
+  dunderTitles.forEach((dunder, i) => {
+    Deno.writeTextFileSync(
+      join(projectDir, pageName(i)),
+      `---\ntitle: "Transcript.${dunder}()"\n---\n\nPage for \`Transcript.${dunder}()\`.\n`,
+    );
+  });
+
+  Deno.writeTextFileSync(
+    join(projectDir, "bold-page.qmd"),
+    `---\ntitle: "A **bold** word"\n---\n\nBold title page.\n`,
+  );
+  Deno.writeTextFileSync(
+    join(projectDir, "code-page.qmd"),
+    `---\ntitle: "A \`code\` word"\n---\n\nCode title page.\n`,
+  );
+};
+
+testQuartoCmd(
+  "render",
+  [projectDir],
+  [
+    noErrors,
+    ensureFileRegexMatches(
+      join(outputDir, "index.html"),
+      [
+        // Dunder titles render literally (intraword_underscores keeps `__x__`
+        // out of emphasis), and reaching this assertion at all proves the
+        // render did not hang.
+        /Transcript\.__getitem__\(\)/,
+        // Markdown inside titles still renders correctly through the envelope.
+        /A <strong>bold<\/strong> word/,
+        /A <code>code<\/code> word/,
+      ],
+      [
+        // A dunder name must not be turned into spurious emphasis.
+        /<strong>getitem<\/strong>/,
+      ],
+    ),
+  ],
+  {
+    // Performance budget: the healthy render finishes in well under this.
+    // Pre-fix the exponential parse blows far past it, so the test fails fast
+    // (instead of riding the default 10-minute timeout) when the fix regresses.
+    timeout: 120000,
+    setup: async () => {
+      writeProject();
+    },
+    teardown: async () => {
+      safeRemoveSync(projectDir, { recursive: true });
+    },
+  },
+);

tests/test.ts

@@ -85,6 +85,11 @@ export interface TestContext {
 
   // environment to pass to downstream processes
   env?: Record<string, string>;
+
+  // Maximum time (ms) the quarto command may run before the test fails.
+  // Defaults to 600000 (10 minutes). Lower it to assert a performance budget
+  // (e.g. a render that must not regress into a hang).
+  timeout?: number;
 }
 
 // Allow to merge test contexts in Tests helpers
@@ -124,6 +129,8 @@ export function mergeTestContexts(baseContext: TestContext, additionalContext?:
     ignore: additionalContext.ignore ?? baseContext.ignore,
     // merge env with additional context taking precedence
     env: { ...baseContext.env, ...additionalContext.env },
+    // override timeout if provided
+    timeout: additionalContext.timeout ?? baseContext.timeout,
   };
 }
 
@@ -141,8 +148,13 @@ export function testQuartoCmd(
   test({
     name,
     execute: async () => {
+      const timeoutMs = context?.timeout ?? 600000;
       const timeout = new Promise((_resolve, reject) => {
-        setTimeout(reject, 600000, "timed out after 10 minutes");
+        setTimeout(
+          reject,
+          timeoutMs,
+          `timed out after ${timeoutMs}ms`,
+        );
       });
       await Promise.race([
         quarto([cmd, ...args], undefined, context?.env),

tests/unit/markdown-pipeline.test.ts

@@ -0,0 +1,32 @@
+/*
+ * markdown-pipeline.test.ts
+ *
+ * Copyright (C) 2020-2022 Posit Software, PBC
+ *
+ */
+import { unitTest } from "../test.ts";
+import { assertStringIncludes } from "testing/asserts";
+import { createMarkdownRenderEnvelope } from "../../src/core/markdown-pipeline.ts";
+
+// Each inline span must be separated by a blank line (double newline) so that
+// pandoc's markdown reader parses each as its own paragraph. A single newline
+// joins all titles into one paragraph, triggering exponential emphasis-matcher
+// parse time on many `__x__`-style titles (GH #14576).
+// deno-lint-ignore require-await
+unitTest(
+  "markdown-pipeline - inline spans separated by blank line (#14576)",
+  async () => {
+    const envelope = createMarkdownRenderEnvelope("test-envelope", {
+      inlines: {
+        first: "alpha",
+        second: "beta",
+      },
+    });
+
+    assertStringIncludes(
+      envelope,
+      "render-id=\"Zmlyc3Q=\"}\n\n[beta]",
+      "Inline spans must be separated by a blank line so pandoc parses each as its own paragraph",
+    );
+  },
+);