{feature} place number suffix parsing behind a flag

Number suffix parsing was added as an experiment, and the
experimental implementation is very unlikely to resemble
what will finally land in the spec.

The 0.4 release is stuck on pre-releases as long as it
deviates from the spec, so hiding this deviation behind a
flag makes it possible to release 0.4.0 without waiting
on the spec to land the number suffix PR.

Author: bgotink

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "test/upstream-v1"]
 	path = test/upstream-v1
 	url = https://github.com/kdl-org/kdl
+[submodule "test/upstream-suffixed-numbers"]
+	path = test/upstream-suffixed-numbers
+	url = https://github.com/kdl-org/kdl

documentation/src/api/parse.md

@@ -52,6 +52,35 @@ The pride flag that consists of four code points is a single grapheme.
 Tracking by code points is the default for the simple reason that it seems to match how columns are tracked in editors like VS Code or Zed.
 There's also a 6.5x speed difference between the two methods, but even with `graphemeLocations` enabled the parser succeeds in parsing thousands of documents per second.
 
+## Parser flags
+
+The `parse` function accepts optional flags to define parser behaviour.
+There is currently a single flag:
+
+- [`experimentalSuffixedNumbers`](./reference/index/index.md#experimentalsuffixednumbers): if enabled, numbers can have a suffix tag (e.g. `10px` is equivalent to `(px)10`).
+  This suffix is used as tag for the value, which implies a number cannot have both a tag and a suffix.
+  The following limitations apply unless a `#` is used as separator:
+
+  - Binary, octal, and hexadecimal numbers cannot have a suffix, only decimal numbers
+  - Decimal numbers cannot have both an exponent and a suffix, e.g. `1e1lorem` is invalid
+  - A suffix cannot start with a letter followed by a digit or an underscore (`_`)
+  - A suffix cannot start with `x` or `X` followed by the letter a through f or A through F
+  - A suffix cannot start on a dot (`.`) or comma (`,`)
+
+```js
+import {parse} from "@bgotink/kdl";
+
+assert.throws(() => parse("node 10px"));
+
+assert.doesNotThrow(() =>
+	parse("node 10px", {
+		flags: {
+			experimentalSuffixedNumbers: true,
+		},
+	}),
+);
+```
+
 ## Quirks
 
 This package turns KDL documents into JavaScript objects and vice versa. It is therefore limited by the JavaScript language.

src/flags.js

@@ -0,0 +1,28 @@
+/**
+ * @typedef {object} ParserFlags
+ * Flags to turn language features on or off
+ *
+ * @prop {boolean} experimentalSuffixedNumbers
+ * Support suffixed numbers using a proposal that might not make it into the language
+ *
+ * If enabled, decimal numbers and can have a suffix.
+ * This suffix is used as tag for the value, which implies a number cannot have both a tag and a suffix.
+ *
+ * The following limitations apply:
+ * - Binary, octal, and hexadecimal numbers cannot have a suffix, only decimal numbers
+ * - Decimal numbers cannot have both an exponent and a suffix, e.g. `1e1lorem` is invalid
+ * - A suffix cannot start with a letter followed by a digit or an underscore (`_`)
+ * - A suffix cannot start with `x` or `X` followed by the letter a through f or A through F
+ * - A suffix cannot start on a dot (`.`) or comma (`,`)
+ *
+ * A suffix can start with a `#` character, in which case none of the limitations above apply.
+ * The `#` itself is a separator character, and not part of the suffix itself.
+ */
+
+/**
+ * @param {Partial<ParserFlags>} [flags]
+ * @returns {ParserFlags}
+ */
+export function resolveFlags({experimentalSuffixedNumbers = false} = {}) {
+	return {experimentalSuffixedNumbers};
+}

src/index.js

@@ -5,6 +5,10 @@ export {format} from "./format.js";
 export {getLocation} from "./locations.js";
 export {parse} from "./parse.js";
 
+/**
+ * @typedef {import('./flags.js').ParserFlags} ParserFlags
+ */
+
 /**
  * @typedef {import('./model.js').Primitive} Primitive
  */

src/parse.d.ts

@@ -1,3 +1,4 @@
+import type {ParserFlags} from "./flags.js";
 import type {Document, Entry, Identifier, Node, Value} from "./model.js";
 import type {LineSpace, NodeSpace} from "./model/whitespace.js";
 
@@ -35,6 +36,7 @@ export function parse(
 		as: "value";
 		storeLocations?: boolean;
 		graphemeLocations?: boolean;
+		flags?: Partial<ParserFlags>;
 	},
 ): Value;
 /**
@@ -58,6 +60,7 @@ export function parse(
 		as: "identifier";
 		storeLocations?: boolean;
 		graphemeLocations?: boolean;
+		flags?: Partial<ParserFlags>;
 	},
 ): Identifier;
 /**
@@ -81,6 +84,7 @@ export function parse(
 		as: "entry";
 		storeLocations?: boolean;
 		graphemeLocations?: boolean;
+		flags?: Partial<ParserFlags>;
 	},
 ): Entry;
 /**
@@ -103,6 +107,7 @@ export function parse(
 		as: "node";
 		storeLocations?: boolean;
 		graphemeLocations?: boolean;
+		flags?: Partial<ParserFlags>;
 	},
 ): Node;
 /**
@@ -123,6 +128,7 @@ export function parse(
 		as: "whitespace in document";
 		storeLocations?: boolean;
 		graphemeLocations?: boolean;
+		flags?: Partial<ParserFlags>;
 	},
 ): LineSpace;
 /**
@@ -143,6 +149,7 @@ export function parse(
 		as: "whitespace in node";
 		storeLocations?: boolean;
 		graphemeLocations?: boolean;
+		flags?: Partial<ParserFlags>;
 	},
 ): NodeSpace;
 /**
@@ -165,6 +172,7 @@ export function parse(
 		as?: "document";
 		storeLocations?: boolean;
 		graphemeLocations?: boolean;
+		flags?: Partial<ParserFlags>;
 	},
 ): Document;
 /**
@@ -183,5 +191,10 @@ export function parse<T extends keyof ParserResult>(
 		| Uint32Array
 		| Int32Array
 		| DataView,
-	options: {as: T; storeLocations?: boolean; graphemeLocations?: boolean},
+	options: {
+		as: T;
+		storeLocations?: boolean;
+		graphemeLocations?: boolean;
+		flags?: Partial<ParserFlags>;
+	},
 ): ParserResult[T];

src/parse.js

@@ -1,3 +1,5 @@
+import {InvalidKdlError} from "./error.js";
+import {resolveFlags} from "./flags.js";
 import {tokenize} from "./parser/tokenize.js";
 import {
 	finalize,
@@ -12,7 +14,6 @@ import {
 	parseWhitespaceInDocument,
 	parseWhitespaceInNode,
 } from "./parser/parse-whitespace.js";
-import {InvalidKdlError} from "./error.js";
 
 const methods = /** @type {const} */ ({
 	value: parseValue,
@@ -31,8 +32,9 @@ const methods = /** @type {const} */ ({
  * @param {keyof typeof methods} [options.as]
  * @param {boolean} [options.storeLocations]
  * @param {boolean} [options.graphemeLocations]
+ * @param {Partial<import('./flags.js').ParserFlags>} [options.flags]
  */
-export function parse(text, {as = "document", ...parserOptions} = {}) {
+export function parse(text, {as = "document", flags, ...parserOptions} = {}) {
 	const parserMethod = methods[as];
 	if (parserMethod == null) {
 		throw new TypeError(`Invalid "as" target passed: ${JSON.stringify(as)}`);
@@ -50,10 +52,15 @@ export function parse(text, {as = "document", ...parserOptions} = {}) {
 		text = decoder.decode(text);
 	}
 
-	const tokens = tokenize(text, parserOptions);
+	const reoslvedFlags = resolveFlags(flags);
+
+	const tokens = tokenize(text, {...parserOptions, flags: reoslvedFlags});
 	// console.log(Array.from(tokens));
 
-	const ctx = createParserCtx(text, tokens, parserOptions);
+	const ctx = createParserCtx(text, tokens, {
+		...parserOptions,
+		flags: reoslvedFlags,
+	});
 
 	let value;
 	try {

src/parser/parse-whitespace.js

@@ -227,7 +227,15 @@ export function parseWhitespaceInNode(ctx) {
 			if (slashDash[1][0]) {
 				result.push(
 					...parseWhitespaceInNode(
-						createParserCtx(slashDash[1][0], tokenize(slashDash[1][0], {})),
+						createParserCtx(
+							slashDash[1][0],
+							tokenize(slashDash[1][0], {
+								flags: ctx.flags,
+							}),
+							{
+								flags: ctx.flags,
+							},
+						),
 					),
 				);
 			}

src/parser/parse.js

@@ -46,6 +46,7 @@ import {
  * @prop {Token} lastToken
  * @prop {boolean} storeLocations
  * @prop {InvalidKdlError[]} errors
+ * @prop {import('../flags.js').ParserFlags} flags
  */
 
 /** @param {ParserCtx} ctx */
@@ -103,11 +104,12 @@ export function concatenate(one = "", two = "", three = "") {
 /**
  * @param {string} text
  * @param {Iterable<Token>} tokens
- * @param {object} [options]
+ * @param {object} options
  * @param {boolean} [options.storeLocations]
+ * @param {import('../flags.js').ParserFlags} options.flags
  * @returns {ParserCtx}
  */
-export function createParserCtx(text, tokens, {storeLocations = false} = {}) {
+export function createParserCtx(text, tokens, {storeLocations = false, flags}) {
 	const iterator = tokens[Symbol.iterator]();
 
 	return {
@@ -131,6 +133,7 @@ export function createParserCtx(text, tokens, {storeLocations = false} = {}) {
 			errors: null,
 		},
 		errors: [],
+		flags,
 	};
 }
 
@@ -220,6 +223,13 @@ function parseNonStringValue(ctx) {
 	switch (token?.type) {
 		case T_NUMBER_WITH_SUFFIX:
 			{
+				if (!ctx.flags.experimentalSuffixedNumbers) {
+					throw new InvalidKdlError(
+						"Unreachable code: got a suffixed number from the tokenizer but the suffixed number flag is not set",
+						{token},
+					);
+				}
+
 				const startOfSuffix = token.text.search(/[^0-9._]/);
 
 				representation = token.text.slice(0, startOfSuffix);
@@ -327,7 +337,9 @@ function parseNonStringValue(ctx) {
 
 	pop(ctx);
 
-	const tagToken = consume(ctx, T_KEYWORD_OR_HASHED_IDENT);
+	const tagToken =
+		ctx.flags.experimentalSuffixedNumbers &&
+		consume(ctx, T_KEYWORD_OR_HASHED_IDENT);
 	if (tagToken) {
 		if (checkSeparatedSuffix) {
 			suffixTagRepresentation = tagToken.text;

src/parser/tokenize/context.js

@@ -40,16 +40,17 @@ function iterateCodePoints(text) {
  * @prop {Location} start
  * @prop {boolean} graphemeLocations
  * @prop {InvalidKdlError[] | null} errorsInToken
+ * @prop {import('../../flags.js').ParserFlags} flags
  */
 
 /**
  * @param {string} text
- * @param {{graphemeLocations?: boolean}} opts
+ * @param {object} opts
+ * @param {boolean} [opts.graphemeLocations]
+ * @param {import('../../flags.js').ParserFlags} opts.flags
  * @returns {TokenizeContext}
  */
-export function createContext(text, opts) {
-	const graphemeLocations = opts.graphemeLocations ?? false;
-
+export function createContext(text, {flags, graphemeLocations = false}) {
 	const iterator =
 		graphemeLocations ? iterateGraphemes(text) : iterateCodePoints(text);
 	const currentIter = iterator.next();
@@ -86,6 +87,8 @@ export function createContext(text, opts) {
 		start: {line: 1, column: 1, offset: 0},
 
 		errorsInToken: null,
+
+		flags,
 	};
 }
 

src/parser/tokenize/tokenize-query.js

@@ -146,7 +146,9 @@ characterHandlers[0xa0] = handleWhitespaceCharacter; // No-Break Space
 
 /**
  * @param {string} t
- * @param {{graphemeLocations?: boolean}} opts
+ * @param {object} opts
+ * @param {boolean} [opts.graphemeLocations]
+ * @param {import('../../flags.js').ParserFlags} opts.flags
  * @returns {Generator<Token, void>}
  */
 export function* tokenizeQuery(t, opts) {