Migrate to antlr4ng 3

alessiostalla · alessiostalla · commit 12d668809384 · 2024-03-25T18:25:03.000+01:00
diff --git a/package.json b/package.json
@@ -83,7 +83,7 @@
 	},
 	"peerDependencies": {
 		"@lionweb/core": "^0.6.2",
-		"antlr4ng": "^2.0.3",
+		"antlr4ng": "^3.0.4",
 		"cmd-ts": "^0.11.0",
 		"ecore": "^0.12.0"
 	},
@@ -122,8 +122,8 @@
 		"@types/node": "^18.19.2",
 		"@typescript-eslint/eslint-plugin": "^6.13.2",
 		"@typescript-eslint/parser": "^6.13.2",
-		"antlr4ng": "^2.0.3",
-		"antlr4ng-cli": "^1.0.6",
+		"antlr4ng": "^3.0.4",
+		"antlr4ng-cli": "^2.0.0",
 		"chai": "^4.3.10",
 		"cmd-ts": "^0.11.0",
 		"cross-env": "^7.0.3",
diff --git a/src/parsing/parse-tree.ts b/src/parsing/parse-tree.ts
@@ -1,5 +1,5 @@
 import {Node, Origin, Point, Position} from "../";
-import {Interval, ParserRuleContext, ParseTree, TerminalNode, Token} from "antlr4ng";
+import {ParserRuleContext, ParseTree, TerminalNode, Token} from "antlr4ng";
 
 // Note: we cannot provide Kolasu-style extension methods on ParseTree because it's an interface.
 // Also, Kolasu-style extension methods on Token are problematic because we use Token.EOF below.
@@ -116,6 +116,5 @@ Object.defineProperty(Node.prototype, "parseTree", {
 ParserRuleContext.prototype.getOriginalText = function () {
     const a = this.start.start;
     const b = this.stop.stop;
-    const interval = new Interval(a, b);
-    return this.start.inputStream.getText(interval);
+    return this.start.inputStream?.getTextFromRange(a, b);
 }
diff --git a/src/parsing/tylasu-parser.ts b/src/parsing/tylasu-parser.ts
@@ -4,7 +4,6 @@ import {
     CharStream,
     CommonTokenStream,
     ErrorNode,
-    Interval,
     Lexer,
     Parser as ANTLRParser,
     ParserRuleContext,
@@ -103,7 +102,7 @@ export abstract class TylasuANTLRLexer<T extends TylasuToken> implements TylasuL
             issues.push(Issue.syntactic(message, IssueSeverity.WARNING, Position.ofTokenEnd(t)))
         }
 
-        const code = inputStream.getText(Interval.of(0, inputStream.size - 1));
+        const code = inputStream.getTextFromRange(0, inputStream.size - 1);
         return new LexingResult(code, tokens, issues, performance.now() - time);
     }
 
@@ -183,12 +182,7 @@ export abstract class TylasuParser<
 
         processDescendantsAndErrors(
             root,
-            it => {
-                if (it.exception != null) {
-                    const message = `Recognition exception: ${it.exception.message}`;
-                    issues.push(Issue.syntactic(message, IssueSeverity.ERROR, Position.ofParseTree(it)));
-                }
-            },
+            () => {},
             it => {
                 const message = `Error node found (token: ${it.symbol?.text})`;
                 issues.push(Issue.syntactic(message, IssueSeverity.ERROR, Position.ofParseTree(it)));
@@ -217,7 +211,7 @@ export abstract class TylasuParser<
         if (root != null) {
             this.verifyParseTree(parser, issues, root);
         }
-        const code = inputStream.getText(Interval.of(0, inputStream.size - 1));
+        const code = inputStream.getTextFromRange(0, inputStream.size - 1);
         return new FirstStageParsingResult(code, root, issues, parser, performance.now() - time, lexingTime);
     }
 
@@ -234,7 +228,7 @@ export abstract class TylasuParser<
             if (!source) {
                 source = new StringSource(code);
             }
-            code = new CharStream(code);
+            code = CharStream.fromString(code);
         }
         const start = performance.now();
         const firstStage = this.parseFirstStage(code, measureLexingTime);
@@ -251,7 +245,7 @@ export abstract class TylasuParser<
                 delete node.origin;
             }
         }
-        const text = code.getText(Interval.of(0, code.size - 1));
+        const text = code.getTextFromRange(0, code.size - 1);
         return new ParsingResult(text, ast, issues, undefined, firstStage, performance.now() - start);
     }
 
diff --git a/tests/mapping.test.ts b/tests/mapping.test.ts
@@ -2,7 +2,7 @@ import {expect} from "chai";
 
 import {ASTTransformer, Child, GenericErrorNode, GenericNode, Node, Position} from "../src";
 import {SimpleLangLexer} from "./parser/SimpleLangLexer";
-import {CharStreams, CommonTokenStream, ParserRuleContext} from "antlr4ng";
+import {CharStream, CommonTokenStream, ParserRuleContext} from "antlr4ng";
 import {CompilationUnitContext, DisplayStmtContext, SetStmtContext, SimpleLangParser} from "./parser/SimpleLangParser";
 import {ParseTreeOrigin} from "../src/parsing";
 import {ParseTreeToASTTransformer} from "../src/mapping";
@@ -56,14 +56,14 @@ describe('Mapping of Parse Trees to ASTs', function() {
     it("Generic node",
         function () {
             const transformer = new ParseTreeToASTTransformer();
-            const node = transformer.transform(new ParserRuleContext());
+            const node = transformer.transform(new ParserRuleContext(null));
             expect(node).not.to.be.undefined;
             expect(node instanceof GenericNode).to.be.true;
         });
     it("Node registered declaratively",
         function () {
             const code = "set foo = 123 + 45";
-            const lexer = new SimpleLangLexer(CharStreams.fromString(code));
+            const lexer = new SimpleLangLexer(CharStream.fromString(code));
             const parser = new SimpleLangParser(new CommonTokenStream(lexer));
             const cu = parser.compilationUnit();
             const setStmt = cu.statement(0) as SetStmtContext;
@@ -84,7 +84,7 @@ describe('Mapping of Parse Trees to ASTs', function() {
 describe('ParseTreeToASTTransformer', function () {
     it("Test ParseTree Transformer", function () {
         const code = "set foo = 123\ndisplay 456";
-        const lexer = new SimpleLangLexer(CharStreams.fromString(code));
+        const lexer = new SimpleLangLexer(CharStream.fromString(code));
         const parser = new SimpleLangParser(new CommonTokenStream(lexer));
         const pt = parser.compilationUnit();
 
@@ -104,7 +104,7 @@ describe('ParseTreeToASTTransformer', function () {
     });
     it("Test transformation with errors", function () {
         const code = "set foo = \ndisplay @@@";
-        const lexer = new SimpleLangLexer(CharStreams.fromString(code));
+        const lexer = new SimpleLangLexer(CharStream.fromString(code));
         const parser = new SimpleLangParser(new CommonTokenStream(lexer));
         parser.removeErrorListeners();
         const pt = parser.compilationUnit();
@@ -127,7 +127,7 @@ describe('ParseTreeToASTTransformer', function () {
     });
     it("Test generic node", function () {
         const code = "set foo = 123\ndisplay 456";
-        const lexer = new SimpleLangLexer(CharStreams.fromString(code));
+        const lexer = new SimpleLangLexer(CharStream.fromString(code));
         const parser = new SimpleLangParser(new CommonTokenStream(lexer));
         const pt = parser.compilationUnit();
 
@@ -136,7 +136,7 @@ describe('ParseTreeToASTTransformer', function () {
     });
     it("test generic AST transformer", function () {
         const code = "set foo = 123\ndisplay 456";
-        const lexer = new SimpleLangLexer(CharStreams.fromString(code));
+        const lexer = new SimpleLangLexer(CharStream.fromString(code));
         const parser = new SimpleLangParser(new CommonTokenStream(lexer));
         const pt = parser.compilationUnit();
 
@@ -163,20 +163,12 @@ const configure = function(transformer: ASTTransformer) : void {
     transformer.registerNodeFactory<DisplayStmtContext, DisplayIntStatement>(
         DisplayStmtContext,
         source => {
-            if (source.exception || source.expression().exception) {
-                // We throw a custom error so that we can check that it's recorded in the AST
-                throw new Error("Parse error");
-            }
             return new DisplayIntStatement(parseInt(source.expression().INT_LIT()!.getText()));
         });
 
     transformer.registerNodeFactory<SetStmtContext, SetStatement>(
         SetStmtContext,
         source => {
-            if (source.exception || source.expression().exception) {
-                // We throw a custom error so that we can check that it's recorded in the AST
-                throw new Error("Parse error");
-            }
             const setStatement = new SetStatement();
             setStatement.variable = source.ID().getText();
             setStatement.value = parseInt(source.expression().INT_LIT()!.getText());
diff --git a/tests/model/origin.test.ts b/tests/model/origin.test.ts
@@ -1,5 +1,5 @@
 import {SimpleLangLexer} from "../parser/SimpleLangLexer";
-import {CharStreams, CommonTokenStream} from "antlr4ng";
+import {CharStream, CommonTokenStream} from "antlr4ng";
 import {SimpleLangParser} from "../parser/SimpleLangParser";
 import {ParseTreeOrigin} from "../../src/parsing";
 import {expect} from "chai";
@@ -11,7 +11,7 @@ describe('Origin', function () {
             const code = `set a = 1 + 2
 input c is string
 display 2 * 3`;
-            const lexer = new SimpleLangLexer(CharStreams.fromString(code));
+            const lexer = new SimpleLangLexer(CharStream.fromString(code));
             const parser = new SimpleLangParser(new CommonTokenStream(lexer));
             const parseTreeRoot = parser.compilationUnit();
             expect(parser.numberOfSyntaxErrors).to.equal(0);
@@ -31,7 +31,7 @@ display 2 * 3`;
             const code = `set a = 1 + 2
 input c is string
 display 2 * 3`;
-            const lexer = new SimpleLangLexer(CharStreams.fromString(code));
+            const lexer = new SimpleLangLexer(CharStream.fromString(code));
             const parser = new SimpleLangParser(new CommonTokenStream(lexer));
             const parseTreeRoot = parser.compilationUnit();
             expect(parser.numberOfSyntaxErrors).to.equal(0);
diff --git a/tests/model/position.test.ts b/tests/model/position.test.ts
@@ -2,7 +2,7 @@ import {expect} from "chai";
 
 import {Point, START_POINT, Node, Position} from "../../src";
 import {SimpleLangLexer} from "../parser/SimpleLangLexer";
-import {CharStreams, CommonTokenStream} from "antlr4ng";
+import {CharStream, CommonTokenStream} from "antlr4ng";
 import {SetStmtContext, SimpleLangParser} from "../parser/SimpleLangParser";
 import {positionOfParseTree} from "../../src/parsing";
 
@@ -49,7 +49,7 @@ describe('Position', function() {
     it("ParserRuleContext position",
         function () {
             const code = "set foo = 123";
-            const lexer = new SimpleLangLexer(CharStreams.fromString(code));
+            const lexer = new SimpleLangLexer(CharStream.fromString(code));
             const parser = new SimpleLangParser(new CommonTokenStream(lexer));
             const cu = parser.compilationUnit();
             const setStmt = cu.statement(0) as SetStmtContext;
@@ -60,7 +60,7 @@ describe('Position', function() {
     it("Position derived from parse tree node",
         function () {
             const code = "set foo = 123";
-            const lexer = new SimpleLangLexer(CharStreams.fromString(code));
+            const lexer = new SimpleLangLexer(CharStream.fromString(code));
             const parser = new SimpleLangParser(new CommonTokenStream(lexer));
             const cu = parser.compilationUnit();
             const setStmt = cu.statement(0) as SetStmtContext;
diff --git a/tests/strumenta-playground.test.ts b/tests/strumenta-playground.test.ts
@@ -4,16 +4,33 @@ import {Issue, Source} from "../src";
 import {registerECoreModel} from "../src/interop/ecore";
 import {ParserTraceLoader, saveForStrumentaPlayground} from "../src/interop/strumenta-playground";
 import {NodeSubclass} from "./nodes";
-import {CharStream, CommonToken, Lexer, TerminalNode, Token, TokenStream} from "antlr4ng";
+import {CharStream, CommonToken, Lexer, TerminalNode, Token, TokenSource, TokenStream} from "antlr4ng";
 import * as fs from "fs";
 import ECore from "ecore/dist/ecore";
 import {ANTLRTokenFactory, ParsingResult} from "../src/parsing";
 import {EcoreEnabledParser} from "../src/interop/ecore-enabled-parser";
 
+class SyntheticToken extends CommonToken {
+
+    constructor(details: {
+        source?: [TokenSource | null, CharStream | null];
+        type: number;
+        channel?: number;
+        start?: number;
+        stop?: number;
+        text?: string;
+        line?: number;
+        tokenIndex?: number;
+        column?: number
+    }) {
+        super({...details, source: [null, null]});
+    }
+}
+
 describe('Strumenta Playground', function() {
     it("Export round-trip", function () {
         // We assign a fake parse tree, to ensure that we don't attempt to serialize ANTLR parse trees into the model.
-        const fakePT = new TerminalNode(new CommonToken([null, null], Token.EOF, Token.DEFAULT_CHANNEL, 0, 0));
+        const fakePT = new TerminalNode(new SyntheticToken({ type: Token.EOF }));
         (fakePT.symbol as CommonToken).line = 1;
         (fakePT.symbol as CommonToken).column = 0;
         /* TODO not supported yet
diff --git a/yarn.lock b/yarn.lock
@@ -973,15 +973,15 @@ ansi-styles@^5.0.0:
   resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b"
   integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==
 
-antlr4ng-cli@^1.0.6:
-  version "1.0.6"
-  resolved "https://registry.yarnpkg.com/antlr4ng-cli/-/antlr4ng-cli-1.0.6.tgz#db22d74b4a931489ef0c59f146fc90624c6871ae"
-  integrity sha512-9ZeVmXu8v/UO4mLhSTyNSCi2fEEBzya2G04bqigwZVg5YdTlS6HhuAa2ruZy/HdV5erhL12USqPCJPMsq+OeKQ==
+antlr4ng-cli@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/antlr4ng-cli/-/antlr4ng-cli-2.0.0.tgz#4f86f3c3818a2170aa1041d4e1633b489af00131"
+  integrity sha512-oAt5OSSYhRQn1PgahtpAP4Vp3BApCoCqlzX7Q8ZUWWls4hX59ryYuu0t7Hwrnfk796OxP/vgIJaqxdltd/oEvQ==
 
-antlr4ng@^2.0.3:
-  version "2.0.3"
-  resolved "https://registry.yarnpkg.com/antlr4ng/-/antlr4ng-2.0.3.tgz#2dde0c90474597395917ed48b9f4394ec5563b3b"
-  integrity sha512-usw0hZsNbTQrJ899P6sHLlQbmKgwQ9UBYq5rh6ZFicPf5qtRJogPVTBj4fSB3OY5ZLP5TSZ7d8tR2p4I5KyiLQ==
+antlr4ng@^3.0.4:
+  version "3.0.4"
+  resolved "https://registry.yarnpkg.com/antlr4ng/-/antlr4ng-3.0.4.tgz#71a47e6148ae75f72fa5f27fbed5ef3462815c7c"
+  integrity sha512-u1Ww6wVv9hq70E9AaYe5qW3ba8hvnjJdO3ZsKnb3iJWFV/medLEEhbyWwXCvvD2ef0ptdaiIUgmaazS/WE6uyQ==
 
 anymatch@^3.0.3:
   version "3.1.3"

Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,6 @@ import {`
`4`	`4`	`CharStream,`
`5`	`5`	`CommonTokenStream,`
`6`	`6`	`ErrorNode,`
`7`		`- Interval,`
`8`	`7`	`Lexer,`
`9`	`8`	`Parser as ANTLRParser,`
`10`	`9`	`ParserRuleContext,`
`@@ -103,7 +102,7 @@ export abstract class TylasuANTLRLexer<T extends TylasuToken> implements TylasuL`
`103`	`102`	`issues.push(Issue.syntactic(message, IssueSeverity.WARNING, Position.ofTokenEnd(t)))`
`104`	`103`	`}`
`105`	`104`
`106`		`- const code = inputStream.getText(Interval.of(0, inputStream.size - 1));`
	`105`	`+ const code = inputStream.getTextFromRange(0, inputStream.size - 1);`
`107`	`106`	`return new LexingResult(code, tokens, issues, performance.now() - time);`
`108`	`107`	`}`
`109`	`108`
`@@ -183,12 +182,7 @@ export abstract class TylasuParser<`
`183`	`182`
`184`	`183`	`processDescendantsAndErrors(`
`185`	`184`	`root,`
`186`		`- it => {`
`187`		`- if (it.exception != null) {`
`188`		- const message = `Recognition exception: ${it.exception.message}`;
`189`		`- issues.push(Issue.syntactic(message, IssueSeverity.ERROR, Position.ofParseTree(it)));`
`190`		`- }`
`191`		`- },`
	`185`	`+ () => {},`
`192`	`186`	`it => {`
`193`	`187`	const message = `Error node found (token: ${it.symbol?.text})`;
`194`	`188`	`issues.push(Issue.syntactic(message, IssueSeverity.ERROR, Position.ofParseTree(it)));`
`@@ -217,7 +211,7 @@ export abstract class TylasuParser<`
`217`	`211`	`if (root != null) {`
`218`	`212`	`this.verifyParseTree(parser, issues, root);`
`219`	`213`	`}`
`220`		`- const code = inputStream.getText(Interval.of(0, inputStream.size - 1));`
	`214`	`+ const code = inputStream.getTextFromRange(0, inputStream.size - 1);`
`221`	`215`	`return new FirstStageParsingResult(code, root, issues, parser, performance.now() - time, lexingTime);`
`222`	`216`	`}`
`223`	`217`
`@@ -234,7 +228,7 @@ export abstract class TylasuParser<`
`234`	`228`	`if (!source) {`
`235`	`229`	`source = new StringSource(code);`
`236`	`230`	`}`
`237`		`- code = new CharStream(code);`
	`231`	`+ code = CharStream.fromString(code);`
`238`	`232`	`}`
`239`	`233`	`const start = performance.now();`
`240`	`234`	`const firstStage = this.parseFirstStage(code, measureLexingTime);`
`@@ -251,7 +245,7 @@ export abstract class TylasuParser<`
`251`	`245`	`delete node.origin;`
`252`	`246`	`}`
`253`	`247`	`}`
`254`		`- const text = code.getText(Interval.of(0, code.size - 1));`
	`248`	`+ const text = code.getTextFromRange(0, code.size - 1);`
`255`	`249`	`return new ParsingResult(text, ast, issues, undefined, firstStage, performance.now() - start);`
`256`	`250`	`}`
`257`	`251`