use enum approach to sourceLocation

Author: jitsedesmet

engines/parser-sparql-1-1/test/objectListParser.test.ts

@@ -1,11 +1,11 @@
-import type { SparqlContext, Triple, Wrap } from '@traqula/rules-sparql-1-1';
+import type { SparqlContext, TripleNesting, Wrap } from '@traqula/rules-sparql-1-1';
 import { TraqulaFactory, completeParseContext, lex as l } from '@traqula/rules-sparql-1-1';
 import { describe, it } from 'vitest';
 import { triplesTemplateParserBuilder } from '../lib';
 
 describe('a SPARQL 1.1 objectlist parser', () => {
   const F = new TraqulaFactory();
-  function parse(query: string, context: Partial<SparqlContext>): Wrap<Triple[]> {
+  function parse(query: string, context: Partial<SparqlContext>): Wrap<TripleNesting[]> {
     const parser = triplesTemplateParserBuilder.consumeToParser({
       tokenVocabulary: l.sparql11Tokens.build(),
     });

engines/parser-sparql-1-1/test/triplesTemplateParser.test.ts

@@ -0,0 +1,83 @@
+import type { IToken } from 'chevrotain';
+
+import type {
+  SourceLocation,
+  SourceLocationNodeAutoGenerate,
+  SourceLocationNodeReplace,
+  SourceLocationNoMaterialize,
+  SourceLocationSource,
+  SourceLocationStringReplace,
+  Node,
+} from './nodeTypings';
+
+export class CoreFactory {
+  public sourceLocation(...elements: (IToken | SourceLocation)[]): SourceLocation {
+    const filtered =
+      elements.filter(element => !this.isSourceLocation(element) || this.isSourceLocationSource(element));
+    if (filtered.length === 0) {
+      return this.sourceLocationNoMaterialize();
+    }
+    const first = filtered[0];
+    const last = filtered.at(-1)!;
+    return {
+      sourceLocationType: 'source',
+      start: this.isSourceLocationSource(first) ? first.start : first.startOffset,
+      end: this.isSourceLocationSource(last) ? last.end : (last.endOffset! + 1),
+    };
+  }
+
+  public sourceLocationNoMaterialize(): SourceLocationNoMaterialize {
+    return { sourceLocationType: 'noMaterialize' };
+  }
+
+  public isSourceLocation(loc: object): loc is SourceLocation {
+    return 'sourceLocationType' in loc;
+  }
+
+  public sourceLocationSource(start: number, end: number): SourceLocationSource {
+    return {
+      sourceLocationType: 'source',
+      start,
+      end,
+    };
+  }
+
+  public isSourceLocationSource(loc: object): loc is SourceLocationSource {
+    return this.isSourceLocation(loc) && loc.sourceLocationType === 'source';
+  }
+
+  public isSourceLocationStringReplace(loc: object): loc is SourceLocationStringReplace {
+    return this.isSourceLocation(loc) && loc.sourceLocationType === 'stringReplace';
+  }
+
+  public sourceLocationNodeReplace(start: number, end: number): SourceLocationNodeReplace {
+    return {
+      sourceLocationType: 'nodeReplace',
+      start,
+      end,
+    };
+  }
+
+  public isSourceLocationNodeReplace(loc: object): loc is SourceLocationNodeReplace {
+    return this.isSourceLocation(loc) && loc.sourceLocationType === 'nodeReplace';
+  }
+
+  public isSourceLocationNodeAutoGenerate(loc: object): loc is SourceLocationNodeAutoGenerate {
+    return this.isSourceLocation(loc) && loc.sourceLocationType === 'autoGenerate';
+  }
+
+  public nodeShouldPrint(node: Node): boolean {
+    return this.isSourceLocationNodeReplace(node.loc) ||
+      this.isSourceLocationNodeAutoGenerate(node.loc);
+  }
+
+  public printFilter(node: Node, callback: () => void): void {
+    if (this.nodeShouldPrint(node)) {
+      callback();
+    }
+  }
+
+  public isSourceLocationNoMaterialize(loc: object): loc is SourceLocationNoMaterialize {
+    return this.isSourceLocation(loc) && loc.sourceLocationType === 'noMaterialize';
+  }
+}

packages/core/lib/CoreFactory.ts

@@ -5,8 +5,10 @@ export type Range<A extends number = number, B extends number = number> = [A, B]
 
 export class RangeArithmetic {
   public ranges: Range[] = [];
+  private readonly initRange: Range;
   public constructor(start: number, end: number) {
-    this.ranges.push([ start, end ]);
+    this.initRange = RangeArithmetic.validate(start, end);
+    this.ranges.push(this.initRange);
   }
 
   private static validate(...range: Range): Range {
@@ -21,7 +23,7 @@ export class RangeArithmetic {
     const [ sMinus, eMinus ] = RangeArithmetic.validate(...range);
 
     return included.flatMap(([ sCur, eCur ]) => {
-    // Split in half
+      // Split in half
       if (sCur < sMinus && eMinus < eCur) {
         return [[ sCur, sMinus ], [ eMinus, eCur ]];
       }
@@ -43,6 +45,16 @@ export class RangeArithmetic {
     return this;
   }
 
+  public negate(): this {
+    // Can be optimized
+    let iter = [ this.initRange ];
+    for (const range of this.ranges) {
+      iter = RangeArithmetic.substractRange(iter, ...range);
+    }
+    this.ranges = iter;
+    return this;
+  }
+
   public projection(...range: Range): Range[] {
     const [ sProj, eProj ] = range;
     if (sProj >= eProj) {

packages/core/lib/RangeArithmetic.ts

@@ -27,5 +27,5 @@ export type GenRulesToObject<
 
 export type GeneratorFromRules<Context, Names extends string, RuleDefs extends GenRuleMap<Names>> = {
   [K in Names]: RuleDefs[K] extends GeneratorRule<Context, K, infer RET, infer ARGS> ?
-      (input: RET, context: Context & { skipRanges: [number, number][] }, args: ARGS) => string : never
+      (input: RET, context: Context & { origSource: string; offset?: number }, args: ARGS) => string : never
 };

packages/core/lib/generator-builder/builderTypes.ts

@@ -1,5 +1,6 @@
-import { RangeArithmetic } from '../RangeArithmetic';
-import type { Node, CheckOverlap } from '../utils';
+import { CoreFactory } from '../CoreFactory';
+import type { Node } from '../nodeTypings';
+import type { CheckOverlap } from '../utils';
 import type { GeneratorFromRules, GenRuleMap, GenRulesToObject, GenNamesFromList } from './builderTypes';
 import type { GeneratorRule, RuleDefArg } from './generatorTypes';
 
@@ -179,21 +180,21 @@ export class GeneratorBuilder<Context, Names extends string, RuleDefs extends Ge
 }
 
 export class Generator<Context, Names extends string, RuleDefs extends GenRuleMap<Names>> {
+  protected readonly factory = new CoreFactory();
   protected __context: Context | undefined = undefined;
-  protected readonly sourceStack: { source: string; generatedUntil: number; allowedRanges: RangeArithmetic }[] = [];
+  protected origSource = '';
+  protected generatedUntil = 0;
   protected readonly stringBuilder: string[] = [];
-  protected readonly skipRanges: [number, number][] = [];
 
   public constructor(protected rules: RuleDefs) {
     // eslint-disable-next-line ts/no-unnecessary-type-assertion
     for (const rule of <GeneratorRule[]> Object.values(rules)) {
       // Define function implementation
       this[<keyof (typeof this)> rule.name] =
-        <any> ((input: any, context: Context & { skipRanges: [number, number][] }, args: any) => {
+        <any> ((input: any, context: Context & { origSource: string; offset?: number }, args: any) => {
           this.stringBuilder.length = 0;
-          this.sourceStack.length = 0;
-          this.skipRanges.length = 0;
-          this.skipRanges.push(...context.skipRanges);
+          this.origSource = context.origSource;
+          this.generatedUntil = context?.offset ?? 0;
           this.setContext(context);
 
           this.subrule(rule, input, args);
@@ -216,56 +217,41 @@ export class Generator<Context, Names extends string, RuleDefs extends GenRuleMa
     if (!def) {
       throw new Error(`Rule ${cstDef.name} not found`);
     }
-    if (ast.loc && ast.loc.noStringManifestation) {
+    if (this.factory.isSourceLocationNoMaterialize(ast.loc)) {
       return;
     }
-    if (ast.loc?.source) {
-      this.pushSource(ast.loc.source);
+    if (this.factory.isSourceLocationStringReplace(ast.loc)) {
+      this.catchup(ast.loc.start);
+      this.print(ast.loc.newSource);
+      this.generatedUntil = ast.loc.end;
+      return;
     }
-    if (ast.loc) {
+    if (this.factory.isSourceLocationNodeReplace(ast.loc) || this.factory.isSourceLocationSource(ast.loc)) {
       this.catchup(ast.loc.start);
     }
+    // If autoGenerate - do nothing
+
+    // Do call generation
     def.gImpl({
       SUBRULE: this.subrule,
       PRINT: this.print,
       PRINT_WORD: this.printWord,
       CATCHUP: this.catchup,
-      PUSH_SOURCE: this.pushSource,
-      POP_SOURCE: this.popSource,
     })(ast, this.getSafeContext(), arg);
-    // TODO: You cannot close at end because scopes dont work like that for blankNodes like `[ ... ]`
-    //    That than gives errors since you don't close at the time you will insert something new.
-    // if (ast.loc) {
-    //   this.catchup(ast.loc.end);
-    // }
-    if (ast.loc?.source) {
-      this.popSource();
-    }
-  };
 
-  protected readonly catchup: RuleDefArg['CATCHUP'] = (until) => {
-    const currentSource = this.sourceStack.at(-1);
-    if (!currentSource) {
-      throw new Error('No source to catchup to');
-    }
-    const { source, generatedUntil, allowedRanges } = currentSource;
-    for (const range of allowedRanges.projection(generatedUntil, until)) {
-      this.stringBuilder.push(source.slice(...range));
+    if (this.factory.isSourceLocationNodeReplace(ast.loc)) {
+      this.generatedUntil = ast.loc.end;
+    } else if (this.factory.isSourceLocationSource(ast.loc)) {
+      this.catchup(ast.loc.end);
     }
-    currentSource.generatedUntil = Math.max(generatedUntil, until);
   };
 
-  protected readonly pushSource: RuleDefArg['PUSH_SOURCE'] = (source) => {
-    const allowedRanges = new RangeArithmetic(0, source.length);
-    for (const skip of this.skipRanges) {
-      allowedRanges.subtract(...skip);
+  protected readonly catchup: RuleDefArg['CATCHUP'] = (until) => {
+    const start = this.generatedUntil;
+    if (start < until) {
+      this.stringBuilder.push(this.origSource.slice(start, until));
     }
-    this.sourceStack.push({ source, generatedUntil: 0, allowedRanges });
-  };
-
-  protected readonly popSource: RuleDefArg['POP_SOURCE'] = () => {
-    this.catchup(Number.MAX_VALUE);
-    this.sourceStack.pop();
+    this.generatedUntil = Math.max(this.generatedUntil, until);
   };
 
   protected readonly print: RuleDefArg['PRINT'] = (...args) => {

packages/core/lib/generator-builder/generatorBuilder.ts

@@ -1,4 +1,4 @@
-import type { Node } from '../utils';
+import type { Node } from '../nodeTypings';
 
 /**
  * Type used to declare generator rules.
@@ -13,9 +13,9 @@ export type GeneratorRule<
    */
   NameType extends string = string,
   /**
-   * Type that will be returned after a correct parse of this rule.
-   * This type will be the return type of calling SUBRULE with this grammar rule.
-   * Generation typically happens on a per AST node basis.
+   * Type that of the AST that we will generate the string for.
+   * This type will be the provided when calling SUBRULE with this generator rule.
+   * Generation happens on a per AST node basis.
    * The core will implement the generation as such. If this ever changes, we will cross that bridge when we get there.
    */
   ReturnType = any,
@@ -34,6 +34,4 @@ export interface RuleDefArg {
   PRINT: (...args: string[]) => void;
   PRINT_WORD: (...args: string[]) => void;
   CATCHUP: (until: number) => void;
-  PUSH_SOURCE: (source: string) => void;
-  POP_SOURCE: () => void;
 }

packages/core/lib/generator-builder/generatorTypes.ts

@@ -12,3 +12,5 @@ export * from './parser-builder/ruleDefTypes';
 // Export general types
 export * from './utils';
 export * from './RangeArithmetic';
+export * from './CoreFactory';
+export * from "./nodeTypings";

packages/core/lib/index.ts

@@ -0,0 +1,58 @@
+/**
+ * A AST node. Nodes are indexable by their types.
+ * When generating, the SUBRULES called should be located within the current location range.
+ */
+export type Node = {
+  type: string;
+  /**
+   * Location undefined means the node does have a string representation, but it was not clarified.
+   * This happens when an AST node is patched by a client of the lib.
+   */
+  loc: SourceLocation;
+};
+
+export interface SourceLocationBase {
+  sourceLocationType: string;
+}
+
+export interface SourceLocationSource extends SourceLocationBase {
+  sourceLocationType: 'source';
+  start: number;
+  end: number;
+}
+
+/**
+ * NoStringManifestation means the node does not have a string representation.
+ * For example the literal '5' has an integer type (which is an AST node),
+ * but the type does not have an associated string representation.
+ * When set to true, the node will not be printed, start and end are meaningless in this case.
+ */
+export interface SourceLocationNoMaterialize extends SourceLocationBase {
+  sourceLocationType: 'noMaterialize';
+}
+
+export interface SourceLocationStringReplace extends SourceLocationBase {
+  sourceLocationType: 'stringReplace';
+  newSource: string;
+  start: number;
+  end: number;
+}
+
+export interface SourceLocationNodeReplace extends SourceLocationBase {
+  sourceLocationType: 'nodeReplace';
+  start: number;
+  end: number;
+}
+/**
+ * Must have an ancestor of type {@link SourceLocationNodeReplace}
+ */
+export interface SourceLocationNodeAutoGenerate extends SourceLocationBase {
+  sourceLocationType: 'autoGenerate';
+}
+
+export type SourceLocation =
+  | SourceLocationSource
+  | SourceLocationNoMaterialize
+  | SourceLocationStringReplace
+  | SourceLocationNodeReplace
+  | SourceLocationNodeAutoGenerate;

packages/core/lib/nodeTypings.ts

@@ -7,28 +7,3 @@ export function unCapitalize<T extends string>(str: T): Uncapitalize<T> {
   return <Uncapitalize<T>> (str.charAt(0).toLowerCase() + str.slice(1));
 }
 
-/**
- * A AST node. Nodes are indexable by their types.
- */
-export type Node = {
-  type: string;
-  /**
-   * Location undefined means the node does have a string representation, but it was not clarified.
-   * This happens when an AST node is patched by a client of the lib.
-   */
-  loc: undefined | SourceLocation;
-};
-
-export type SourceLocation = {
-  // When null, traverse the tree up until you find a non-null source
-  source?: string | undefined;
-  /**
-   * NoStringManifestation means the node does not have a string representation.
-   * For example the literal '5' has an integer type (which is an AST node),
-   * but the type does not have an associated string representation.
-   * When set to true, the node will not be printed, start and end are meaningless in this case.
-   */
-  noStringManifestation?: true;
-  start: number;
-  end: number;
-};