Skip to content

Commit b8180e3

Browse files
DmitrySharabinLeaVerou
DmitrySharabin
and
LeaVerou
authored
First stab at functional $inside (#4032)
The code from the `simplify` branch, authored mainly by @LeaVerou. - Add `tokenizeByNamedGroups()` and `camelToKebabCase()` utils - `resolve()` now accepts functional tokens - Adjust `_matchGrammar()` to not choke on functional tokens - Transform the Markdown language to use named token groups and adjust existing tests accordingly (as a proof of concept) - In the pattern and coverage tests, add support for named capture groups and regexp flags - Tweak TS types accordingly Co-authored-by: Lea Verou <lea@verou.me>
1 parent f427052 commit b8180e3

10 files changed

Lines changed: 200 additions & 102 deletions

File tree

‎src/core/tokenize/match.js‎

Lines changed: 62 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { Token } from '../classes/token.js';
22
import singleton from '../prism.js';
33
import { tokenize } from './tokenize.js';
4-
import { resolve } from './util.js';
4+
import { resolve, tokenizeByNamedGroups } from './util.js';
55

66
/**
77
* @this {Prism}
@@ -21,7 +21,12 @@ export function _matchGrammar (text, tokenList, grammar, startNode, startPos, re
2121

2222
for (const token in grammar) {
2323
const tokenValue = grammar[token];
24-
if (!grammar.hasOwnProperty(token) || token.startsWith('$') || !tokenValue) {
24+
if (
25+
!grammar.hasOwnProperty(token) ||
26+
token.startsWith('$') ||
27+
!tokenValue ||
28+
typeof tokenValue === 'function' // functional tokens ($inside for now) are handled on L170, and we should ignore them in all other cases
29+
) {
2530
continue;
2631
}
2732

@@ -36,9 +41,20 @@ export function _matchGrammar (text, tokenList, grammar, startNode, startPos, re
3641
let { pattern, lookbehind = false, greedy = false, alias, inside } = patternObj;
3742
const insideGrammar = resolve.call(prism, inside);
3843

44+
let flagsToAdd = '';
45+
3946
if (greedy && !pattern.global) {
4047
// Without the global flag, lastIndex won't work
41-
patternObj.pattern = pattern = RegExp(pattern.source, pattern.flags + 'g');
48+
flagsToAdd += 'g';
49+
}
50+
51+
if (pattern.source?.includes('(?<') && pattern.hasIndices === false) {
52+
// Has named groups, we need to be able to capture their indices
53+
flagsToAdd += 'd';
54+
}
55+
56+
if (flagsToAdd) {
57+
patternObj.pattern = pattern = RegExp(pattern.source, pattern.flags + flagsToAdd);
4258
}
4359

4460
for (
@@ -63,7 +79,8 @@ export function _matchGrammar (text, tokenList, grammar, startNode, startPos, re
6379
}
6480

6581
let removeCount = 1; // this is the to parameter of removeBetween
66-
let match;
82+
/** @type {RegExpExecArray | null} */
83+
let match = null;
6784

6885
if (greedy) {
6986
match = matchPattern(pattern, pos, text, lookbehind);
@@ -117,6 +134,10 @@ export function _matchGrammar (text, tokenList, grammar, startNode, startPos, re
117134

118135
const from = match.index;
119136
const matchStr = match[0];
137+
138+
/** @type {TokenStream | string} */
139+
let content = matchStr;
140+
120141
const before = str.slice(0, from);
121142
const after = str.slice(from + matchStr.length);
122143

@@ -134,14 +155,42 @@ export function _matchGrammar (text, tokenList, grammar, startNode, startPos, re
134155

135156
tokenList.removeRange(removeFrom, removeCount);
136157

137-
const wrapped = new Token(
138-
token,
139-
insideGrammar
140-
? tokenize.call(prism, matchStr, /** @type {Grammar} */ (insideGrammar))
141-
: matchStr,
142-
alias,
143-
matchStr
144-
);
158+
const byGroups = match.groups ? tokenizeByNamedGroups(match) : null;
159+
if (byGroups && byGroups.length > 1) {
160+
content = byGroups
161+
.map(arg => {
162+
let content = typeof arg === 'string' ? arg : arg.content;
163+
const type = typeof arg === 'string' ? undefined : arg.type;
164+
165+
if (insideGrammar) {
166+
let localInsideGrammar = type ? insideGrammar[type] : insideGrammar;
167+
168+
if (typeof localInsideGrammar === 'function') {
169+
// Late resolving
170+
localInsideGrammar = resolve.call(
171+
prism,
172+
localInsideGrammar(match.groups)
173+
);
174+
}
175+
176+
if (localInsideGrammar) {
177+
// @ts-ignore
178+
content = tokenize.call(prism, content, localInsideGrammar);
179+
}
180+
}
181+
182+
return typeof arg === 'object' && arg.type
183+
? new Token(arg.type, content)
184+
: content;
185+
})
186+
.flat(); // Flatten tokens like ['foo']
187+
}
188+
else if (insideGrammar) {
189+
// @ts-ignore
190+
content = tokenize.call(prism, content, insideGrammar);
191+
}
192+
193+
const wrapped = new Token(token, content, alias, matchStr);
145194
currentNode = tokenList.addAfter(removeFrom, wrapped);
146195

147196
if (after) {
@@ -216,7 +265,7 @@ function toGrammarToken (pattern) {
216265

217266
/**
218267
* @import { Prism } from '../prism.js';
219-
* @import { Grammar, GrammarToken, GrammarTokens, RegExpLike } from '../../types.d.ts';
268+
* @import { Grammar, GrammarToken, GrammarTokens, TokenStream, RegExpLike } from '../../types.d.ts';
220269
*/
221270

222271
/**

‎src/core/tokenize/util.js‎

Lines changed: 44 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1+
import { camelToKebabCase } from '../../shared/util.js';
12
import singleton from '../prism.js';
23

34
/**
45
* @this {Prism}
5-
* @param {Grammar | string | null | undefined} reference
6-
* @returns {Grammar | undefined}
6+
* @param {Grammar | string | Function | null | undefined} reference
7+
* @returns {Grammar | Function | undefined}
78
*/
89
export function resolve (reference) {
910
const prism = this ?? singleton;
@@ -13,6 +14,11 @@ export function resolve (reference) {
1314
ret = prism.languageRegistry.getLanguage(ret)?.resolvedGrammar;
1415
}
1516

17+
if (typeof ret === 'function' && ret.length === 0) {
18+
// Function with no arguments, resolve eagerly
19+
ret = ret.call(prism);
20+
}
21+
1622
if (typeof ret === 'object' && ret.$rest) {
1723
const restGrammar = resolve.call(prism, ret.$rest) ?? {};
1824
if (typeof restGrammar === 'object') {
@@ -25,6 +31,42 @@ export function resolve (reference) {
2531
return /** @type {Grammar | undefined} */ (ret);
2632
}
2733

34+
/**
35+
*
36+
* @param {RegExpExecArray} match
37+
* @returns {({type: string, content: string} | string)[]}
38+
*/
39+
export function tokenizeByNamedGroups (match) {
40+
const str = match[0];
41+
const result = [];
42+
let i = 0;
43+
44+
const entries = Object.entries(match.indices?.groups || {})
45+
.map(([type, [start, end]]) => ({
46+
type,
47+
start: start - match.index,
48+
end: end - match.index,
49+
}))
50+
.sort((a, b) => a.start - b.start);
51+
52+
for (let { type, start, end } of entries) {
53+
if (start > i) {
54+
result.push(str.slice(i, start));
55+
}
56+
57+
const content = str.slice(start, end);
58+
type = camelToKebabCase(type);
59+
result.push({ type, content });
60+
i = end;
61+
}
62+
63+
if (i < str.length) {
64+
result.push(str.slice(i));
65+
}
66+
67+
return result;
68+
}
69+
2870
/**
2971
* @import { Prism } from '../prism.js';
3072
* @import { Grammar, LanguageRegistry } from '../../types.d.ts';

‎src/languages/markdown.js‎

Lines changed: 15 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -99,73 +99,24 @@ export default {
9999
// ```optional language
100100
// code block
101101
// ```
102-
pattern: /^```[\s\S]*?^```$/m,
103-
greedy: true,
104-
inside: /** @type {Grammar} */ ({
105-
'code-block': {
106-
pattern: /^(```.*(?:\n|\r\n?))[\s\S]+?(?=(?:\n|\r\n?)^```$)/m,
107-
lookbehind: true,
108-
},
109-
'code-language': {
110-
pattern: /^(```).+/,
111-
lookbehind: true,
112-
},
113-
'punctuation': /```/,
114-
/** @type {Grammar['$tokenize']} */
115-
$tokenize (code, grammar, Prism) {
116-
const tokens = Prism.tokenize(code, withoutTokenize(grammar));
117-
118-
/*
119-
* Add the correct `language-xxxx` class to this code block. Keep in mind that the `code-language` token
120-
* is optional. But the grammar is defined so that there is only one case we have to handle:
121-
*
122-
* token.content = [
123-
* <span class="punctuation">```</span>,
124-
* <span class="code-language">xxxx</span>,
125-
* '\n', // exactly one new lines (\r or \n or \r\n)
126-
* <span class="code-block">...</span>,
127-
* '\n', // exactly one new lines again
128-
* <span class="punctuation">```</span>
129-
* ];
130-
*/
131-
132-
const codeLang = tokens[1];
133-
const codeBlock = tokens[3];
134-
135-
if (
136-
typeof codeLang === 'object' &&
137-
typeof codeBlock === 'object' &&
138-
codeLang.type === 'code-language' &&
139-
codeBlock.type === 'code-block'
140-
) {
141-
// this might be a language that Prism does not support
142-
143-
// do some replacements to support C++, C#, and F#
144-
const lang = getTextContent(codeLang.content)
145-
.replace(/\b#/g, 'sharp')
146-
.replace(/\b\+\+/g, 'pp');
147-
// only use the first word
148-
const langName = /[a-z][\w-]*/i.exec(lang)?.[0].toLowerCase();
149-
if (langName) {
150-
codeBlock.addAlias('language-' + langName);
151-
152-
const grammar =
153-
Prism.languageRegistry.getLanguage(lang)?.resolvedGrammar;
154-
if (grammar) {
155-
codeBlock.content = Prism.tokenize(
156-
getTextContent(codeBlock),
157-
grammar
158-
);
159-
}
160-
else {
161-
codeBlock.addAlias('needs-highlighting');
162-
}
102+
pattern:
103+
/^```\s*(?<codeLanguage>\{[^{}]*\}|[a-z+#-]+)(?:[ \t][^\n\r]*)?(?:\n|\r\n?)(?<codeBlock>[\s\S]*?)(?:\n|\r\n?)```$/im,
104+
inside: {
105+
'code-block': groups => {
106+
let lang = groups.codeLanguage;
107+
// Extract language code from curly braces like {r pressure, echo=FALSE} → r
108+
if (lang.startsWith('{') && lang.endsWith('}')) {
109+
const match = lang.slice(1, -1).match(/^\s*([a-z+#-]+)/i);
110+
if (match) {
111+
lang = match[0];
163112
}
164113
}
165-
166-
return tokens;
114+
// Apply transformations: c++ → cpp, c# → csharp, f# → fsharp, etc.
115+
lang = lang.replace(/\b#/g, 'sharp').replace(/\b\+\+/g, 'pp');
116+
return lang.toLowerCase();
167117
},
168-
}),
118+
'punctuation': /```/,
119+
},
169120
},
170121
],
171122
'title': [

‎src/shared/languages/templating.js‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -127,9 +127,9 @@ export function templating (code, hostGrammar, templateGrammar, Prism) {
127127
hostGrammar = resolve.call(Prism, hostGrammar);
128128
templateGrammar = resolve.call(Prism, templateGrammar);
129129

130-
const { hostCode, tokenStack } = buildPlaceholders(code, templateGrammar, Prism);
130+
const { hostCode, tokenStack } = buildPlaceholders(code, /** @type {Grammar | undefined} */ (templateGrammar), Prism);
131131

132-
const tokens = hostGrammar ? Prism.tokenize(hostCode, hostGrammar) : [hostCode];
132+
const tokens = hostGrammar ? Prism.tokenize(hostCode, /** @type {Grammar} */ (hostGrammar)) : [hostCode];
133133
insertIntoHostToken(tokens, tokenStack);
134134
return tokens;
135135
}
@@ -145,10 +145,10 @@ export function embeddedIn (hostGrammar) {
145145
}
146146

147147
/**
148-
* @import { Prism, Token } from '../../core.js';
149-
* @import { TokenStream, TokenStack, Grammar, LanguageRegistry} from '../../types.d.ts';
148+
* @import { Prism } from '../../core.js';
149+
* @import { TokenStream, TokenStack, Grammar } from '../../types.d.ts';
150150
*/
151151

152152
/**
153-
* @typedef {Grammar | string | undefined | null} GrammarRef
153+
* @typedef {Grammar | Function | string | undefined | null} GrammarRef
154154
*/

‎src/shared/util.js‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,3 +76,13 @@ export function kebabToCamelCase (kebab) {
7676
const [first, ...others] = kebab.split(/-/);
7777
return first + others.map(capitalize).join('');
7878
}
79+
80+
/**
81+
* Converts the given camel case identifier to a kebab case identifier.
82+
*
83+
* @param {string} str
84+
* @returns
85+
*/
86+
export function camelToKebabCase (str) {
87+
return (str + '').replace(/[A-Z]/g, l => '-' + l.toLowerCase());
88+
}

0 commit comments

Comments
 (0)