Skip to content

Commit ae0e19b

Browse files
pi0pi0
committed
refactor: split parser from compiler
1 parent 55d5085 commit ae0e19b

4 files changed

Lines changed: 119 additions & 117 deletions

File tree

‎src/_parser.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
export type Token = {
2+
type: "text" | "code" | "expr";
3+
contents: string;
4+
};
5+
6+
export function parseTemplate(template: string): Token[] {
7+
if (!template) {
8+
return [];
9+
}
10+
11+
// convert <script server> ... </script> to <?js ... ?>
12+
template = template.replace(
13+
/<script\s+server\s*>([\s\S]*?)<\/script>/gi,
14+
(_m, code) => `<?js${code}?>`,
15+
);
16+
17+
const tokens: Token[] = [];
18+
const re = /<\?(?:js)?(?<equals>=)?(?<value>[\s\S]*?)\?>/g;
19+
let cursor = 0;
20+
let match;
21+
while ((match = re.exec(template))) {
22+
const { equals, value } = match.groups || {};
23+
const matchStart = match.index;
24+
const matchEnd = matchStart + match[0].length;
25+
if (matchStart > cursor) {
26+
const textContent = template.slice(cursor, matchStart);
27+
if (textContent) {
28+
tokens.push({ type: "text", contents: textContent });
29+
}
30+
}
31+
if (equals) {
32+
// Expression tag: <?= ... ?>
33+
tokens.push({ type: "expr", contents: value || "" });
34+
} else {
35+
// Code tag: <? ... ?> or <?js ... ?>
36+
tokens.push({ type: "code", contents: value || "" });
37+
}
38+
cursor = matchEnd;
39+
}
40+
41+
if (cursor < template.length) {
42+
const remainingText = template.slice(cursor);
43+
if (remainingText) {
44+
tokens.push({ type: "text", contents: remainingText });
45+
}
46+
}
47+
48+
return tokens;
49+
}

‎src/compiler.ts‎

Lines changed: 2 additions & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import { parseTemplate } from "./_parser.ts";
12
import { runtimeStream, runtimeText } from "./_runtime.ts";
23

34
export type CompileTemplateOptions = {
@@ -68,7 +69,7 @@ export function compileTemplateToString(
6869
asyncWrapper?: boolean,
6970
): string {
7071
const parts: string[] = [];
71-
const tokens = tokenize(template);
72+
const tokens = parseTemplate(template);
7273
for (const token of tokens) {
7374
switch (token.type) {
7475
case "text": {
@@ -107,55 +108,3 @@ export function hasTemplateSyntax(template: string): boolean {
107108
template,
108109
);
109110
}
110-
111-
// --- Tokenizer ---
112-
113-
export type Token = {
114-
type: "text" | "code" | "expr";
115-
contents: string;
116-
};
117-
118-
export function tokenize(template: string): Token[] {
119-
if (!template) {
120-
return [];
121-
}
122-
123-
// convert <script server> ... </script> to <?js ... ?>
124-
template = template.replace(
125-
/<script\s+server\s*>([\s\S]*?)<\/script>/gi,
126-
(_m, code) => `<?js${code}?>`,
127-
);
128-
129-
const tokens: Token[] = [];
130-
const re = /<\?(?:js)?(?<equals>=)?(?<value>[\s\S]*?)\?>/g;
131-
let cursor = 0;
132-
let match;
133-
while ((match = re.exec(template))) {
134-
const { equals, value } = match.groups || {};
135-
const matchStart = match.index;
136-
const matchEnd = matchStart + match[0].length;
137-
if (matchStart > cursor) {
138-
const textContent = template.slice(cursor, matchStart);
139-
if (textContent) {
140-
tokens.push({ type: "text", contents: textContent });
141-
}
142-
}
143-
if (equals) {
144-
// Expression tag: <?= ... ?>
145-
tokens.push({ type: "expr", contents: value || "" });
146-
} else {
147-
// Code tag: <? ... ?> or <?js ... ?>
148-
tokens.push({ type: "code", contents: value || "" });
149-
}
150-
cursor = matchEnd;
151-
}
152-
153-
if (cursor < template.length) {
154-
const remainingText = template.slice(cursor);
155-
if (remainingText) {
156-
tokens.push({ type: "text", contents: remainingText });
157-
}
158-
}
159-
160-
return tokens;
161-
}

‎test/compiler.test.ts‎

Lines changed: 1 addition & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { describe, expect, it } from "vitest";
2-
import { compileTemplate, tokenize } from "../src/compiler.ts";
2+
import { compileTemplate } from "../src/compiler.ts";
33
import { format } from "prettier";
44

55
describe("compileTemplater", () => {
@@ -25,67 +25,4 @@ describe("compileTemplater", () => {
2525
).toMatchFileSnapshot("snapshots/compileTemplated-stream.js");
2626
});
2727
});
28-
29-
describe("tokenize", () => {
30-
it("plain text", () => {
31-
const tokens = tokenize("Hello, World!");
32-
expect(tokens).toMatchObject([
33-
{ type: "text", contents: "Hello, World!" },
34-
]);
35-
});
36-
37-
it("expression", () => {
38-
const tokens = tokenize("<?js= name ?>");
39-
expect(tokens).toMatchObject([{ type: "expr", contents: " name " }]);
40-
});
41-
42-
it("expression (short)", () => {
43-
const tokens = tokenize("<?= name ?>");
44-
expect(tokens).toMatchObject([{ type: "expr", contents: " name " }]);
45-
});
46-
47-
it("code", () => {
48-
const tokens = tokenize("<?js if (true) { ?>123<?js } ?>");
49-
expect(tokens).toMatchObject([
50-
{ type: "code", contents: " if (true) { " },
51-
{ type: "text", contents: "123" },
52-
{ type: "code", contents: " } " },
53-
]);
54-
});
55-
56-
it("code (short)", () => {
57-
const tokens = tokenize("<? if (true) { ?>123<? } ?>");
58-
expect(tokens).toMatchObject([
59-
{ type: "code", contents: " if (true) { " },
60-
{ type: "text", contents: "123" },
61-
{ type: "code", contents: " } " },
62-
]);
63-
});
64-
65-
it("mixed", () => {
66-
const template = [
67-
"",
68-
"Hello, <?= name ?>!",
69-
"<?js if (age >= 18) { ?>",
70-
" You are an adult.",
71-
"<?js } else { ?>",
72-
" You are a minor.",
73-
"<?js } ?>",
74-
"",
75-
].join("\n");
76-
const tokens = tokenize(template);
77-
// console.log(tokens);
78-
expect(tokens).toMatchObject([
79-
{ type: "text", contents: "\nHello, " },
80-
{ type: "expr", contents: " name " },
81-
{ type: "text", contents: "!\n" },
82-
{ type: "code", contents: " if (age >= 18) { " },
83-
{ type: "text", contents: "\n You are an adult.\n" },
84-
{ type: "code", contents: " } else { " },
85-
{ type: "text", contents: "\n You are a minor.\n" },
86-
{ type: "code", contents: " } " },
87-
{ type: "text", contents: "\n" },
88-
]);
89-
});
90-
});
9128
});

‎test/parser.test.ts‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
import { describe, expect, it } from "vitest";
2+
import { parseTemplate } from "../src/_parser.ts";
3+
4+
describe("parser", () => {
5+
describe("parseTemplate", () => {
6+
it("plain text", () => {
7+
const tokens = parseTemplate("Hello, World!");
8+
expect(tokens).toMatchObject([
9+
{ type: "text", contents: "Hello, World!" },
10+
]);
11+
});
12+
13+
it("expression", () => {
14+
const tokens = parseTemplate("<?js= name ?>");
15+
expect(tokens).toMatchObject([{ type: "expr", contents: " name " }]);
16+
});
17+
18+
it("expression (short)", () => {
19+
const tokens = parseTemplate("<?= name ?>");
20+
expect(tokens).toMatchObject([{ type: "expr", contents: " name " }]);
21+
});
22+
23+
it("code", () => {
24+
const tokens = parseTemplate("<?js if (true) { ?>123<?js } ?>");
25+
expect(tokens).toMatchObject([
26+
{ type: "code", contents: " if (true) { " },
27+
{ type: "text", contents: "123" },
28+
{ type: "code", contents: " } " },
29+
]);
30+
});
31+
32+
it("code (short)", () => {
33+
const tokens = parseTemplate("<? if (true) { ?>123<? } ?>");
34+
expect(tokens).toMatchObject([
35+
{ type: "code", contents: " if (true) { " },
36+
{ type: "text", contents: "123" },
37+
{ type: "code", contents: " } " },
38+
]);
39+
});
40+
41+
it("mixed", () => {
42+
const template = [
43+
"",
44+
"Hello, <?= name ?>!",
45+
"<?js if (age >= 18) { ?>",
46+
" You are an adult.",
47+
"<?js } else { ?>",
48+
" You are a minor.",
49+
"<?js } ?>",
50+
"",
51+
].join("\n");
52+
const tokens = parseTemplate(template);
53+
// console.log(tokens);
54+
expect(tokens).toMatchObject([
55+
{ type: "text", contents: "\nHello, " },
56+
{ type: "expr", contents: " name " },
57+
{ type: "text", contents: "!\n" },
58+
{ type: "code", contents: " if (age >= 18) { " },
59+
{ type: "text", contents: "\n You are an adult.\n" },
60+
{ type: "code", contents: " } else { " },
61+
{ type: "text", contents: "\n You are a minor.\n" },
62+
{ type: "code", contents: " } " },
63+
{ type: "text", contents: "\n" },
64+
]);
65+
});
66+
});
67+
});

0 commit comments

Comments
 (0)