Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 33 additions & 1 deletion frontend/src/lib/utils/content-parser.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ describe("parseContent", () => {

it("preserves leading whitespace in tail text", () => {
const segments =
parseContent("```code\ncontent```\n Trailing text");
parseContent("```code\ncontent\n```\n Trailing text");
expect(segments).toHaveLength(2);
expect(segments[0]).toMatchObject({ type: "code" });
expect(segments[1]).toEqual({
Expand Down Expand Up @@ -111,6 +111,38 @@ describe("parseContent", () => {
]);
});

it("keeps nested shorter fences inside longer code blocks", () => {
const content =
"````markdown\nSome context paragraph.\n\n```qmd\nauthor: \"Jane Doe\"\n```\n\nMore context here.\n````";
const segments = parseContent(content);
expect(segments).toEqual([
{
type: "code",
content:
"Some context paragraph.\n\n```qmd\nauthor: \"Jane Doe\"\n```\n\nMore context here.\n",
label: "markdown",
},
]);
});

it("keeps inline same-length backtick runs inside code blocks", () => {
const content =
"```javascript\nconst fence = \"```\";\n[Thinking]\nnot parsed\n```\nAfter";
const segments = parseContent(content);
expect(segments).toEqual([
{
type: "code",
content:
"const fence = \"```\";\n[Thinking]\nnot parsed\n",
label: "javascript",
},
{
type: "text",
content: "\nAfter",
},
]);
});

it("omits label for code blocks without language", () => {
const segments = parseContent("```\nplain code\n```");
expect(segments[0]).toEqual({
Expand Down
106 changes: 90 additions & 16 deletions frontend/src/lib/utils/content-parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,6 @@ const TOOL_RE = new RegExp(
"g",
);

const CODE_BLOCK_RE = /```(\w*)\n([\s\S]*?)```/g;

/** Returns true if text[from..to) contains a backtick run of
* exactly `len` characters. Used to detect a closing inline
* code delimiter on the same line as the opener. */
Expand All @@ -95,7 +93,7 @@ function hasRunBefore(
* backtick run of length N is closed by the next run of exactly
* N backticks. Fenced code blocks (triple-backtick at line
* start followed by a newline) are excluded — those are handled
* separately by CODE_BLOCK_RE.
* separately by codeBlockMatches.
*/
function scanInlineCodeSpans(
text: string,
Expand Down Expand Up @@ -125,15 +123,13 @@ function scanInlineCodeSpans(
}

// Scan for a closing run of exactly the same length.
let found = false;
for (let j = i; j < text.length; j++) {
if (text[j] !== "`") continue;
const closeStart = j;
while (j < text.length && text[j] === "`") j++;
if (j - closeStart === runLen) {
spans.push([openStart, j]);
i = j;
found = true;
break;
}
}
Expand Down Expand Up @@ -190,6 +186,92 @@ function insideInlineCode(
return spans.some(([s, e]) => pos > s && pos < e);
}

function atFenceLineStart(text: string, pos: number): boolean {
const lineStart = text.lastIndexOf("\n", pos - 1) + 1;
return /^[ \t]{0,3}$/.test(text.slice(lineStart, pos));
}

function countBackticks(text: string, pos: number): number {
let end = pos;
while (end < text.length && text[end] === "`") end++;
return end - pos;
}

function closingFence(
text: string,
contentStart: number,
fenceLen: number,
): { contentEnd: number; end: number } | undefined {
let pos = contentStart;
while (pos < text.length) {
const tickStart = text.indexOf("`", pos);
if (tickStart < 0) return undefined;
const lineStart = text.lastIndexOf("\n", tickStart - 1) + 1;
const nextLineStart = text.indexOf("\n", tickStart);
const lineEnd =
nextLineStart >= 0 ? nextLineStart : text.length;

const tickCount = countBackticks(text, tickStart);
const rest = text.slice(tickStart + tickCount, lineEnd);
if (
tickCount >= fenceLen &&
atFenceLineStart(text, tickStart) &&
/^[ \t]*$/.test(rest)
) {
return { contentEnd: lineStart, end: lineEnd };
}

pos = tickStart + tickCount;
}
return undefined;
}

function codeBlockMatches(text: string): Match[] {
const matches: Match[] = [];
let pos = 0;

while (pos < text.length) {
const start = text.indexOf("```", pos);
if (start < 0) break;

if (!atFenceLineStart(text, start)) {
pos = start + 1;
continue;
}

const fenceLen = countBackticks(text, start);
const infoStart = start + fenceLen;
const lineEnd = text.indexOf("\n", infoStart);
if (lineEnd < 0) break;

const info = text.slice(infoStart, lineEnd);
if (info.includes("`")) {
pos = infoStart;
continue;
}

const contentStart = lineEnd + 1;
const close = closingFence(text, contentStart, fenceLen);
if (close === undefined) {
pos = infoStart;
continue;
}

matches.push({
start,
end: close.end,
segment: {
type: "code",
content: text.slice(contentStart, close.contentEnd),
label: info.trim() || undefined,
},
});
pos = close.end;
}

return matches;
}

function extractMatches(text: string, parseTools = true): Match[] {
const matches: Match[] = [];

Expand Down Expand Up @@ -271,22 +353,14 @@ function extractMatches(text: string, parseTools = true): Match[] {
}
}

for (const m of text.matchAll(CODE_BLOCK_RE)) {
const idx = m.index!;
for (const m of codeBlockMatches(text)) {
const idx = m.start;
const insideOther = matches.some(
(o) => idx >= o.start && idx < o.end,
);
if (insideOther) continue;

matches.push({
start: idx,
end: idx + m[0].length,
segment: {
type: "code",
content: m[2] ?? "",
label: m[1] || undefined,
},
});
matches.push(m);
}

return matches;
Expand Down