Skip to content

Commit 16918b7

Browse files
AjaniBilbyAjaniBilby
committed
~Fixed memory overgrowth
1 parent 737de85 commit 16918b7

8 files changed

Lines changed: 91 additions & 44 deletions

File tree

‎dist/bnf.d.ts‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type _Shared from './shared.js';
2+
export type _Literal = { type: "literal", value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange };
23
export type Term_Program = {
34
type: 'program',
45
start: number,
@@ -23,7 +24,7 @@ export type Term_W = {
2324
count: number,
2425
ref: _Shared.ReferenceRange,
2526
value: [
26-
(Term_Comment | { type: 'literal', value: '\x20', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x09', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x0a', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x0d\x0a', start: number, end: number, count: number, ref: _Shared.ReferenceRange })
27+
(Term_Comment | _Literal & {value: "\x20"} | _Literal & {value: "\x09"} | _Literal & {value: "\x0a"} | _Literal & {value: "\x0d\x0a"})
2728
]
2829
}
2930
export declare function Parse_W (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -40,9 +41,9 @@ export type Term_Comment = {
4041
count: number,
4142
ref: _Shared.ReferenceRange,
4243
value: [
43-
{ type: 'literal', value: '\x23', start: number, end: number, count: number, ref: _Shared.ReferenceRange },
44-
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange },
45-
{ type: 'literal', value: '\x0a', start: number, end: number, count: number, ref: _Shared.ReferenceRange }
44+
_Literal & {value: "\x23"},
45+
_Literal,
46+
_Literal & {value: "\x0a"}
4647
]
4748
}
4849
export declare function Parse_Comment (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -60,7 +61,7 @@ export type Term_Name = {
6061
ref: _Shared.ReferenceRange,
6162
value: [
6263
Term_Letter,
63-
{ type: '(...)*', value: Array<(Term_Letter | Term_Digit | { type: 'literal', value: '\x5f', start: number, end: number, count: number, ref: _Shared.ReferenceRange })>, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
64+
{ type: '(...)*', value: Array<(Term_Letter | Term_Digit | _Literal & {value: "\x5f"})>, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
6465
]
6566
}
6667
export declare function Parse_Name (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -77,7 +78,7 @@ export type Term_Letter = {
7778
count: number,
7879
ref: _Shared.ReferenceRange,
7980
value: [
80-
({ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange })
81+
(_Literal | _Literal)
8182
]
8283
}
8384
export declare function Parse_Letter (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -94,7 +95,7 @@ export type Term_Digit = {
9495
count: number,
9596
ref: _Shared.ReferenceRange,
9697
value: [
97-
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
98+
_Literal
9899
]
99100
}
100101
export declare function Parse_Digit (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -111,7 +112,7 @@ export type Term_Hex = {
111112
count: number,
112113
ref: _Shared.ReferenceRange,
113114
value: [
114-
({ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange })
115+
(_Literal | _Literal | _Literal)
115116
]
116117
}
117118
export declare function Parse_Hex (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -145,7 +146,7 @@ export type Term_Frag = {
145146
count: number,
146147
ref: _Shared.ReferenceRange,
147148
value: [
148-
(Term_Escape | Term_Byte | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange })
149+
(Term_Escape | Term_Byte | _Literal)
149150
]
150151
}
151152
export declare function Parse_Frag (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -162,7 +163,7 @@ export type Term_Escape = {
162163
count: number,
163164
ref: _Shared.ReferenceRange,
164165
value: [
165-
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
166+
_Literal
166167
]
167168
}
168169
export declare function Parse_Escape (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -179,7 +180,7 @@ export type Term_Byte = {
179180
count: number,
180181
ref: _Shared.ReferenceRange,
181182
value: [
182-
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
183+
_Literal
183184
]
184185
}
185186
export declare function Parse_Byte (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -196,7 +197,7 @@ export type Term_Def = {
196197
count: number,
197198
ref: _Shared.ReferenceRange,
198199
value: [
199-
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange },
200+
_Literal,
200201
Term_Expr
201202
]
202203
}
@@ -222,7 +223,7 @@ export type Term_Expr = {
222223
count: number,
223224
ref: _Shared.ReferenceRange,
224225
value: [
225-
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange },
226+
_Literal,
226227
Term_Expr_arg
227228
]
228229
}>, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
@@ -243,8 +244,8 @@ export type Term_Expr_arg = {
243244
ref: _Shared.ReferenceRange,
244245
value: [
245246
Term_Expr_prefix,
246-
(Term_Constant | Term_Expr_brackets | { type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }),
247-
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
247+
(Term_Constant | Term_Expr_brackets | _Literal),
248+
_Literal
248249
]
249250
}
250251
export declare function Parse_Expr_arg (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -261,9 +262,9 @@ export type Term_Expr_prefix = {
261262
count: number,
262263
ref: _Shared.ReferenceRange,
263264
value: [
264-
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange },
265-
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange },
266-
{ type: 'literal', value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange }
265+
_Literal,
266+
_Literal,
267+
_Literal
267268
]
268269
}
269270
export declare function Parse_Expr_prefix (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -280,7 +281,7 @@ export type Term_Expr_infix = {
280281
count: number,
281282
ref: _Shared.ReferenceRange,
282283
value: [
283-
({ type: 'literal', value: '\x2d\x3e', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x7c', start: number, end: number, count: number, ref: _Shared.ReferenceRange })
284+
(_Literal & {value: "\x2d\x3e"} | _Literal & {value: "\x7c"})
284285
]
285286
}
286287
export declare function Parse_Expr_infix (i: string, refMapping?: boolean): _Shared.ParseError | {
@@ -297,7 +298,7 @@ export type Term_Expr_suffix = {
297298
count: number,
298299
ref: _Shared.ReferenceRange,
299300
value: [
300-
({ type: 'literal', value: '\x2a', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x3f', start: number, end: number, count: number, ref: _Shared.ReferenceRange } | { type: 'literal', value: '\x2b', start: number, end: number, count: number, ref: _Shared.ReferenceRange })
301+
(_Literal & {value: "\x2a"} | _Literal & {value: "\x3f"} | _Literal & {value: "\x2b"})
301302
]
302303
}
303304
export declare function Parse_Expr_suffix (i: string, refMapping?: boolean): _Shared.ParseError | {

‎dist/shared.js‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,16 @@
11
const OFFSET = {"TYPE":0,"TYPE_LEN":4,"START":8,"END":12,"COUNT":16,"DATA":20};
22
export function InitParse(ctx, data) {
3-
const memory = ctx.exports.memory;
4-
memory.grow(1); // grow memory if needed
3+
const bytesPerPage = 64 * 1024;
54
// Convert the string to UTF-8 bytes
65
const utf8Encoder = new TextEncoder();
76
const stringBytes = utf8Encoder.encode(data);
7+
const memory = ctx.exports.memory;
8+
// ONLY grow memory if needed
9+
const chunks = memory.buffer.byteLength / bytesPerPage;
10+
const desireChunks = stringBytes.byteLength * 6 / bytesPerPage;
11+
if (desireChunks > chunks) {
12+
memory.grow(desireChunks - chunks);
13+
}
814
// Copy the string bytes to WebAssembly memory
915
const wasmMemory = new Uint8Array(memory.buffer);
1016
wasmMemory.set(stringBytes, ctx.exports.input.value);

‎docs/source/changelog.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
11
# Changelog
22

3+
## Version 4.0.4
4+
5+
### Fixes:
6+
- [x] Memory over growth: Available memory kept growing after every single parse
7+
- [x] Better error handling for `bnf-compile` when given a bad directory
8+
9+
## Version 4.0.3
10+
11+
### Additions:
12+
- [x] `_Literal` helper type in compiled artifacts
13+
314
## Version 4.0.2
415

516
### Fixes:

‎docs/source/static/dist/bnf-parser.js‎

Lines changed: 25 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1500,7 +1500,7 @@ function CompileOmit() {
15001500
return '';
15011501
}
15021502
function CompileGather() {
1503-
return TemplateNode(`'literal'`, 'string');
1503+
return "_Literal";
15041504
}
15051505
function CompileTerm(expr) {
15061506
const once = CompileTermOnce(expr);
@@ -1515,10 +1515,10 @@ function CompileTermOnce(expr) {
15151515
return 'Term_' + expr.value[0].toUpperCase() + expr.value.slice(1);
15161516
}
15171517
function CompileNot() {
1518-
return TemplateNode(`'literal'`, 'string');
1518+
return "_Literal";
15191519
}
15201520
function CompileRange() {
1521-
return TemplateNode(`'literal'`, 'string');
1521+
return "_Literal";
15221522
}
15231523
function CompileLiteral(expr) {
15241524
const once = CompileLiteralOnce(expr);
@@ -1531,7 +1531,7 @@ function CompileLiteral(expr) {
15311531
}
15321532
function CompileLiteralOnce(expr) {
15331533
let safe = expr.value.replace(/[^a-zA-Z0-9]/g, (char) => '\\x' + char.charCodeAt(0).toString(16).padStart(2, '0'));
1534-
return TemplateNode(`'literal'`, `'${safe}'`);
1534+
return `_Literal & {value: "${safe}"}`;
15351535
}
15361536
function CompileRepeat(innerType, repetitions) {
15371537
switch (repetitions) {
@@ -1571,9 +1571,11 @@ function CompileRule(rule) {
15711571
`export declare function Parse_${capName} (i: string, refMapping?: boolean): _Shared.ParseError | {\n\troot: _Shared.SyntaxNode & ${typeName},\n\treachBytes: number,\n\treach: null | _Shared.Reference,\n\tisPartial: boolean\n}\n`;
15721572
}
15731573
function CompileTypes(lang) {
1574-
return `import type _Shared from './shared.js';\n` + [...lang.terms.keys()]
1575-
.map(x => CompileRule(lang.terms.get(x))) // hush Typescript it's okay
1576-
.join('\n');
1574+
return `import type _Shared from './shared.js';\n` +
1575+
`export type _Literal = { type: "literal", value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange };\n` +
1576+
[...lang.terms.keys()]
1577+
.map(x => CompileRule(lang.terms.get(x))) // hush Typescript it's okay
1578+
.join('\n');
15771579
}
15781580

15791581
function FlattenConstant(syntax) {
@@ -1745,11 +1747,17 @@ function Create(wasm) {
17451747
return bundle;
17461748
}
17471749
function InitParse$1(ctx, data) {
1748-
const memory = ctx.exports.memory;
1749-
memory.grow(1); // grow memory if needed
1750+
const bytesPerPage = 64 * 1024;
17501751
// Convert the string to UTF-8 bytes
17511752
const utf8Encoder = new TextEncoder();
17521753
const stringBytes = utf8Encoder.encode(data);
1754+
const memory = ctx.exports.memory;
1755+
// ONLY grow memory if needed
1756+
const chunks = memory.buffer.byteLength / bytesPerPage;
1757+
const desireChunks = stringBytes.byteLength * 6 / bytesPerPage;
1758+
if (desireChunks > chunks) {
1759+
memory.grow(desireChunks - chunks);
1760+
}
17531761
// Copy the string bytes to WebAssembly memory
17541762
const wasmMemory = new Uint8Array(memory.buffer);
17551763
wasmMemory.set(stringBytes, ctx.exports.input.value);
@@ -1902,11 +1910,17 @@ var run = /*#__PURE__*/Object.freeze({
19021910

19031911
const OFFSET = {"TYPE":0,"TYPE_LEN":4,"START":8,"END":12,"COUNT":16,"DATA":20};
19041912
function InitParse(ctx, data) {
1905-
const memory = ctx.exports.memory;
1906-
memory.grow(1); // grow memory if needed
1913+
const bytesPerPage = 64 * 1024;
19071914
// Convert the string to UTF-8 bytes
19081915
const utf8Encoder = new TextEncoder();
19091916
const stringBytes = utf8Encoder.encode(data);
1917+
const memory = ctx.exports.memory;
1918+
// ONLY grow memory if needed
1919+
const chunks = memory.buffer.byteLength / bytesPerPage;
1920+
const desireChunks = stringBytes.byteLength * 6 / bytesPerPage;
1921+
if (desireChunks > chunks) {
1922+
memory.grow(desireChunks - chunks);
1923+
}
19101924
// Copy the string bytes to WebAssembly memory
19111925
const wasmMemory = new Uint8Array(memory.buffer);
19121926
wasmMemory.set(stringBytes, ctx.exports.input.value);

‎package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "bnf-parser",
3-
"version": "4.0.2",
3+
"version": "4.0.4",
44
"description": "Deterministic BNF compiler/parser",
55
"homepage": "https://bnf-parser.ajanibilby.com",
66
"main": "./bin/index.js",

‎source/cli.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,10 +51,15 @@ export { ready };\n`;
5151

5252

5353
const root = process.argv[2] || "./";
54+
if (!existsSync(root)) {
55+
console.error(`Unknown path ${root}`);
56+
process.exit(1);
57+
}
58+
5459
const isFile = statSync(root).isFile();
5560
const root_dir = isFile ? dirname(root) : root.slice(0, -1);
5661

57-
if (!existsSync(root)) {
62+
if (!existsSync(root_dir)) {
5863
console.error(`Unknown path ${root}`);
5964
process.exit(1);
6065
}

‎source/types.ts‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ function CompileOmit(): string {
6565
}
6666

6767
function CompileGather(): string {
68-
return TemplateNode(`'literal'`, 'string');
68+
return "_Literal";
6969
}
7070

7171

@@ -85,11 +85,11 @@ function CompileTermOnce(expr: Term): string {
8585

8686

8787
function CompileNot(): string {
88-
return TemplateNode(`'literal'`, 'string');
88+
return "_Literal";
8989
}
9090

9191
function CompileRange(): string {
92-
return TemplateNode(`'literal'`, 'string');
92+
return "_Literal";
9393
}
9494

9595

@@ -109,7 +109,7 @@ function CompileLiteralOnce(expr: Literal): string {
109109
(char) => '\\x' + char.charCodeAt(0).toString(16).padStart(2, '0')
110110
);
111111

112-
return TemplateNode(`'literal'`, `'${safe}'`);
112+
return `_Literal & {value: "${safe}"}`;
113113
}
114114

115115

@@ -160,7 +160,9 @@ function CompileRule(rule: Rule) {
160160

161161

162162
export function CompileTypes(lang: Parser) {
163-
return `import type _Shared from './shared.js';\n`+ [...lang.terms.keys()]
164-
.map(x => CompileRule(lang.terms.get(x) as any)) // hush Typescript it's okay
165-
.join('\n');
163+
return `import type _Shared from './shared.js';\n`+
164+
`export type _Literal = { type: "literal", value: string, start: number, end: number, count: number, ref: _Shared.ReferenceRange };\n` +
165+
[...lang.terms.keys()]
166+
.map(x => CompileRule(lang.terms.get(x) as any)) // hush Typescript it's okay
167+
.join('\n');
166168
}

‎source/wasm/run.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,21 @@ export function Create(wasm: BufferSource){
2626
}
2727

2828
function InitParse(ctx: WasmParser, data: string) {
29-
const memory = ctx.exports.memory;
30-
memory.grow(1); // grow memory if needed
29+
const bytesPerPage = 64 * 1024
3130

3231
// Convert the string to UTF-8 bytes
3332
const utf8Encoder = new TextEncoder();
3433
const stringBytes = utf8Encoder.encode(data);
3534

35+
const memory = ctx.exports.memory;
36+
37+
// ONLY grow memory if needed
38+
const chunks = memory.buffer.byteLength / bytesPerPage;
39+
const desireChunks = stringBytes.byteLength * 10 / bytesPerPage;
40+
if (desireChunks > chunks) {
41+
memory.grow(desireChunks - chunks);
42+
}
43+
3644
// Copy the string bytes to WebAssembly memory
3745
const wasmMemory = new Uint8Array(memory.buffer);
3846
wasmMemory.set(stringBytes, ctx.exports.input.value);

0 commit comments

Comments
 (0)