From 25611e70e3e53a71e69ed3a152944a6efc5c2357 Mon Sep 17 00:00:00 2001 From: Romain Menke Date: Sat, 10 Aug 2024 14:52:16 +0200 Subject: [PATCH] porffor --- package-lock.json | 283 ++-- packages/css-tokenizer/dist/index.cjs | 1595 +++++++++++++++++++- packages/css-tokenizer/dist/index.mjs | 1553 ++++++++++++++++++- packages/css-tokenizer/dist/porffor.mjs | 1561 +++++++++++++++++++ packages/css-tokenizer/dist/porffor.wasm | Bin 0 -> 1391122 bytes packages/css-tokenizer/package.json | 8 +- packages/css-tokenizer/prepare-porffor.mjs | 18 + rollup/presets/package-typescript.mjs | 28 +- 8 files changed, 4911 insertions(+), 135 deletions(-) create mode 100644 packages/css-tokenizer/dist/porffor.mjs create mode 100644 packages/css-tokenizer/dist/porffor.wasm create mode 100644 packages/css-tokenizer/prepare-porffor.mjs diff --git a/package-lock.json b/package-lock.json index 5f3f35bcd..7308c6c62 100644 --- a/package-lock.json +++ b/package-lock.json @@ -685,6 +685,22 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/plugin-proposal-explicit-resource-management": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-explicit-resource-management/-/plugin-proposal-explicit-resource-management-7.24.7.tgz", + "integrity": "sha512-sYvUjHrKctxFiNe/YFyzNyXoblzP4YaZrWSk0js7QU3Lw2lYjLAxnzV8l/4+7PB5+NDFKGblPUq2YbdTxs3Ppg==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.24.7", + "@babel/plugin-syntax-explicit-resource-management": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, "node_modules/@babel/plugin-proposal-private-property-in-object": { "version": "7.21.0-placeholder-for-preset-env.2", "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-private-property-in-object/-/plugin-proposal-private-property-in-object-7.21.0-placeholder-for-preset-env.2.tgz", @@ -763,6 +779,21 @@ "@babel/core": "^7.0.0-0" } }, + "node_modules/@babel/plugin-syntax-explicit-resource-management": { + "version": "7.24.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-explicit-resource-management/-/plugin-syntax-explicit-resource-management-7.24.7.tgz", + "integrity": "sha512-gEl7s4tkyWl3EcfUCLVcJJRSogDQju6qX2hdJmZVO0wdpVGzss5bJU8wySoZ7JQw1jBjztEY1a/eYstgv+AQkA==", + "dev": true, + "dependencies": { + "@babel/helper-plugin-utils": "^7.24.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, "node_modules/@babel/plugin-syntax-export-namespace-from": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-export-namespace-from/-/plugin-syntax-export-namespace-from-7.8.3.tgz", @@ -2413,9 +2444,9 @@ } }, "node_modules/@eslint/js": { - "version": "9.8.0", - "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.8.0.tgz", - "integrity": "sha512-MfluB7EUfxXtv3i/++oh89uzAr4PDI4nn201hsp+qaXqsjAWzinlZEHEfPgAX4doIlKvPG/i0A9dpKxOLII8yA==", + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/@eslint/js/-/js-9.9.0.tgz", + "integrity": "sha512-hhetes6ZHP3BlXLxmd8K2SNgkhNSi+UcecbnwWKwpP7kyi/uC75DJ1lOOBO3xrC4jyojtGE3YxKZPHfk4yrgug==", "dev": true, "engines": { "node": "^18.18.0 || ^20.9.0 || >=21.1.0" @@ -3543,9 +3574,9 @@ } }, "node_modules/@stryker-mutator/api": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@stryker-mutator/api/-/api-8.3.0.tgz", - "integrity": "sha512-sYCf0xgD0smLvqlMI+HJuvgMuRzSWJa8H3RIq98UwC8cBBpYVDbUw7WMunsqaSGPDuRfCMnfvpRPiGzP6UqW/A==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@stryker-mutator/api/-/api-8.4.0.tgz", + "integrity": "sha512-VccQ0s4WOxVkIN0slBNM4CIo5jMf46Mc6gazwIWyyng/MvjuEaXO/yYl3ihL9wFww41FDKXiTy3alOc4WWyIVQ==", "dev": true, "dependencies": { "mutation-testing-metrics": "3.1.1", @@ -3558,15 +3589,15 @@ } }, "node_modules/@stryker-mutator/core": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@stryker-mutator/core/-/core-8.3.0.tgz", - "integrity": "sha512-GF02YcEQ1ErHKCSwG2pEt1e8K6YQOsOEBcuBW3ZONeqZ55qoRtL6tisVuz82gHDaBR4DXctSKPjAVB5VLSY2vw==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@stryker-mutator/core/-/core-8.4.0.tgz", + "integrity": "sha512-vqdggnLFpr6uG/JOwiRYmk62xDJmTrj42gx7Kvth1AAj8txIVeCxcmh0o6gMG31vS22NGkkSzW8uYIxT/gfyKg==", "dev": true, "dependencies": { "@inquirer/prompts": "^5.3.6", - "@stryker-mutator/api": "8.3.0", - "@stryker-mutator/instrumenter": "8.3.0", - "@stryker-mutator/util": "8.3.0", + "@stryker-mutator/api": "8.4.0", + "@stryker-mutator/instrumenter": "8.4.0", + "@stryker-mutator/util": "8.4.0", "ajv": "~8.17.1", "chalk": "~5.3.0", "commander": "~12.1.0", @@ -3575,10 +3606,9 @@ "execa": "~9.3.0", "file-url": "~4.0.0", "get-port": "~7.1.0", - "glob": "~11.0.0", "lodash.groupby": "~4.6.0", "log4js": "~6.9.1", - "minimatch": "~10.0.1", + "minimatch": "~9.0.5", "mutation-testing-elements": "3.1.1", "mutation-testing-metrics": "3.1.1", "mutation-testing-report-schema": "3.1.1", @@ -3636,82 +3666,16 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/@stryker-mutator/core/node_modules/glob": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-11.0.0.tgz", - "integrity": "sha512-9UiX/Bl6J2yaBbxKoEBRm4Cipxgok8kQYcOPEhScPwebu2I0HoQOuYdIO6S3hLuWoZgpDpwQZMzTFxgpkyT76g==", - "dev": true, - "dependencies": { - "foreground-child": "^3.1.0", - "jackspeak": "^4.0.1", - "minimatch": "^10.0.0", - "minipass": "^7.1.2", - "package-json-from-dist": "^1.0.0", - "path-scurry": "^2.0.0" - }, - "bin": { - "glob": "dist/esm/bin.mjs" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@stryker-mutator/core/node_modules/jackspeak": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-4.0.1.tgz", - "integrity": "sha512-cub8rahkh0Q/bw1+GxP7aeSe29hHHn2V4m29nnDlvCdlgU+3UGxkZp7Z53jLUdpX3jdTO0nJZUDl3xvbWc2Xog==", - "dev": true, - "dependencies": { - "@isaacs/cliui": "^8.0.2" - }, - "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - }, - "optionalDependencies": { - "@pkgjs/parseargs": "^0.11.0" - } - }, - "node_modules/@stryker-mutator/core/node_modules/lru-cache": { - "version": "11.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.0.0.tgz", - "integrity": "sha512-Qv32eSV1RSCfhY3fpPE2GNZ8jgM9X7rdAfemLWqTUxwiyIC4jJ6Sy0fZ8H+oLWevO6i4/bizg7c8d8i6bxrzbA==", - "dev": true, - "engines": { - "node": "20 || >=22" - } - }, "node_modules/@stryker-mutator/core/node_modules/minimatch": { - "version": "10.0.1", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.0.1.tgz", - "integrity": "sha512-ethXTt3SGGR+95gudmqJ1eNhRO7eGEGIgYA9vnPatK4/etz2MEVDno5GMCibdMTuBMyElzIlgxMna3K94XDIDQ==", + "version": "9.0.5", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.5.tgz", + "integrity": "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow==", "dev": true, "dependencies": { "brace-expansion": "^2.0.1" }, "engines": { - "node": "20 || >=22" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/@stryker-mutator/core/node_modules/path-scurry": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-2.0.0.tgz", - "integrity": "sha512-ypGJsmGtdXUOeM5u93TyeIEfEhM6s+ljAhrk5vAvSx8uyY/02OvrZnA0YNGUrPXfpJMgI1ODd3nwz8Npx4O4cg==", - "dev": true, - "dependencies": { - "lru-cache": "^11.0.0", - "minipass": "^7.1.2" - }, - "engines": { - "node": "20 || >=22" + "node": ">=16 || 14 >=14.17" }, "funding": { "url": "https://github.com/sponsors/isaacs" @@ -3739,18 +3703,19 @@ } }, "node_modules/@stryker-mutator/instrumenter": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@stryker-mutator/instrumenter/-/instrumenter-8.3.0.tgz", - "integrity": "sha512-EdXCD7KPaNBaqnX0+GTaTFcUDJPtAo2kBaUjz+pVZlMXLHiUtLSuVQjbl0kjC5pyV3olU5P3kapCn3DNaG1Cwg==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@stryker-mutator/instrumenter/-/instrumenter-8.4.0.tgz", + "integrity": "sha512-thLblaLuH+hIWk53bVxFJ9gfpnlOY+zNSxGV09986QoztYXejSSqRUWe8cxPc7PKF5xVewEh4yMTJaewX7Nzfw==", "dev": true, "dependencies": { "@babel/core": "~7.25.2", "@babel/generator": "~7.25.0", "@babel/parser": "~7.25.0", "@babel/plugin-proposal-decorators": "~7.24.7", + "@babel/plugin-proposal-explicit-resource-management": "^7.24.7", "@babel/preset-typescript": "~7.24.7", - "@stryker-mutator/api": "8.3.0", - "@stryker-mutator/util": "8.3.0", + "@stryker-mutator/api": "8.4.0", + "@stryker-mutator/util": "8.4.0", "angular-html-parser": "~6.0.2", "semver": "~7.6.3", "weapon-regex": "~1.3.2" @@ -3772,9 +3737,9 @@ } }, "node_modules/@stryker-mutator/util": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/@stryker-mutator/util/-/util-8.3.0.tgz", - "integrity": "sha512-750ldEV3CRdDuBhxD1c2HEY4CYW4oz45u6Hm0Ca/FVefU4tCWYwf9kS87khw3FZGZC/gbkI6Xn3fGnXYyHa+cA==", + "version": "8.4.0", + "resolved": "https://registry.npmjs.org/@stryker-mutator/util/-/util-8.4.0.tgz", + "integrity": "sha512-dvDBmIrKpkqICO/7kcmfTzs+dsS1dA61q6XbbF+uB+WSsNLUHMfft0XKQg6BwLY+NLlis4pFN5CJz6gI7bfwxA==", "dev": true }, "node_modules/@tootallnate/quickjs-emscripten": { @@ -3805,9 +3770,9 @@ } }, "node_modules/@types/node": { - "version": "22.1.0", - "resolved": "https://registry.npmjs.org/@types/node/-/node-22.1.0.tgz", - "integrity": "sha512-AOmuRF0R2/5j1knA3c6G3HOk523Ga+l+ZXltX8SF1+5oqcXijjfTd8fY3XRZqSihEu9XhtQnKYLmkFaoxgsJHw==", + "version": "22.2.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.2.0.tgz", + "integrity": "sha512-bm6EG6/pCpkxDf/0gDNDdtDILMOHgaQBVOJGdwsqClnxA3xL6jtMv76rLBc006RVMWbmaf0xbmom4Z/5o2nRkQ==", "dev": true, "dependencies": { "undici-types": "~6.13.0" @@ -4549,9 +4514,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001649", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001649.tgz", - "integrity": "sha512-fJegqZZ0ZX8HOWr6rcafGr72+xcgJKI9oWfDW5DrD7ExUtgZC7a7R7ZYmZqplh7XDocFdGeIFn7roAxhOeYrPQ==", + "version": "1.0.30001651", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001651.tgz", + "integrity": "sha512-9Cf+Xv1jJNe1xPZLGuUXLNkE1BoDkqRqYyFJ9TDYSqhduqA4hu4oR9HluGoWYQC/aj8WHjsGVV+bwkh0+tegRg==", "funding": [ { "type": "opencollective", @@ -5078,9 +5043,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.5.4", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.4.tgz", - "integrity": "sha512-orzA81VqLyIGUEA77YkVA1D+N+nNfl2isJVjjmOyrlxuooZ19ynb+dOlaDTqd/idKRS9lDCSBmtzM+kyCsMnkA==" + "version": "1.5.6", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.6.tgz", + "integrity": "sha512-jwXWsM5RPf6j9dPYzaorcBSUg6AiqocPEyMpkchkvntaH9HGfOOMZwxMJjDY/XEs3T5dM7uyH1VhRMkqUU9qVw==" }, "node_modules/emoji-regex": { "version": "10.3.0", @@ -5097,6 +5062,19 @@ "once": "^1.4.0" } }, + "node_modules/enhanced-resolve": { + "version": "5.17.1", + "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.17.1.tgz", + "integrity": "sha512-LMHl3dXhTcfv8gM4kEzIUeTQ+7fpdA0l2tUf34BddXPkz2A5xJ5L/Pchd5BL6rdccM9QGvu0sWZzK1Z1t4wwyg==", + "dev": true, + "dependencies": { + "graceful-fs": "^4.2.4", + "tapable": "^2.2.0" + }, + "engines": { + "node": ">=10.13.0" + } + }, "node_modules/env-paths": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz", @@ -5175,16 +5153,16 @@ } }, "node_modules/eslint": { - "version": "9.8.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.8.0.tgz", - "integrity": "sha512-K8qnZ/QJzT2dLKdZJVX6W4XOwBzutMYmt0lqUS+JdXgd+HTYFlonFgkJ8s44d/zMPPCnOOk0kMWCApCPhiOy9A==", + "version": "9.9.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.9.0.tgz", + "integrity": "sha512-JfiKJrbx0506OEerjK2Y1QlldtBxkAlLxT5OEcRF8uaQ86noDe2k31Vw9rnSWv+MXZHj7OOUV/dA0AhdLFcyvA==", "dev": true, "dependencies": { "@eslint-community/eslint-utils": "^4.2.0", "@eslint-community/regexpp": "^4.11.0", "@eslint/config-array": "^0.17.1", "@eslint/eslintrc": "^3.1.0", - "@eslint/js": "9.8.0", + "@eslint/js": "9.9.0", "@humanwhocodes/module-importer": "^1.0.1", "@humanwhocodes/retry": "^0.3.0", "@nodelib/fs.walk": "^1.2.8", @@ -5223,6 +5201,14 @@ }, "funding": { "url": "https://eslint.org/donate" + }, + "peerDependencies": { + "jiti": "*" + }, + "peerDependenciesMeta": { + "jiti": { + "optional": true + } } }, "node_modules/eslint-scope": { @@ -5685,9 +5671,9 @@ "dev": true }, "node_modules/foreground-child": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.2.1.tgz", - "integrity": "sha512-PXUUyLqrR2XCWICfv6ukppP96sdFwWbNEnfEMt7jNsISjMsvaLNinAHNDYyvkyU+SZG2BTSbT5NjG+vZslfGTA==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.3.0.tgz", + "integrity": "sha512-Ld2g8rrAyMYFXBhEqMz8ZAHBi4J4uS1i/CxGMDnjyFWddMXLVcDp051DZfu+t7+ab7Wv6SMqpWmyFIj5UbfFvg==", "dev": true, "dependencies": { "cross-spawn": "^7.0.0", @@ -6077,6 +6063,23 @@ "node": ">= 0.4" } }, + "node_modules/hermes-estree": { + "version": "0.18.2", + "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.18.2.tgz", + "integrity": "sha512-KoLsoWXJ5o81nit1wSyEZnWUGy9cBna9iYMZBR7skKh7okYAYKqQ9/OczwpMHn/cH0hKDyblulGsJ7FknlfVxQ==", + "dev": true, + "optional": true + }, + "node_modules/hermes-parser": { + "version": "0.18.2", + "resolved": "https://registry.npmjs.org/hermes-parser/-/hermes-parser-0.18.2.tgz", + "integrity": "sha512-1eQfvib+VPpgBZ2zYKQhpuOjw1tH+Emuib6QmjkJWJMhyjM8xnXMvA+76o9LhF0zOAJDZgPfQhg43cyXEyl5Ew==", + "dev": true, + "optional": true, + "dependencies": { + "hermes-estree": "0.18.2" + } + }, "node_modules/html-tags": { "version": "3.3.1", "resolved": "https://registry.npmjs.org/html-tags/-/html-tags-3.3.1.tgz", @@ -6526,9 +6529,9 @@ } }, "node_modules/knip": { - "version": "5.27.0", - "resolved": "https://registry.npmjs.org/knip/-/knip-5.27.0.tgz", - "integrity": "sha512-W8+jhO7i5pXRUqOzhJGm2DT5/d9aQjyrYTCSojqJxFOvi7ku/nHKzpBO3WNf4eflJo0t3zitmUkM69g53qoZQw==", + "version": "5.27.2", + "resolved": "https://registry.npmjs.org/knip/-/knip-5.27.2.tgz", + "integrity": "sha512-Mya1XEDq1oygibQf0uocQd02Fil8RtvNVhcFAcxypjcc6zakT7wsJtS0xvuwEitilfI0tiFC9PghmJQ3DMKuTg==", "dev": true, "funding": [ { @@ -6548,6 +6551,7 @@ "@nodelib/fs.walk": "1.2.8", "@snyk/github-codeowners": "1.1.0", "easy-table": "1.2.0", + "enhanced-resolve": "^5.17.1", "fast-glob": "^3.3.2", "jiti": "^1.21.6", "js-yaml": "^4.1.0", @@ -6555,7 +6559,6 @@ "picocolors": "^1.0.0", "picomatch": "^4.0.1", "pretty-ms": "^9.0.0", - "resolve": "^1.22.8", "smol-toml": "^1.1.4", "strip-json-comments": "5.0.1", "summary": "2.1.0", @@ -6757,6 +6760,16 @@ "node": ">= 8" } }, + "node_modules/meriyah": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/meriyah/-/meriyah-4.5.0.tgz", + "integrity": "sha512-Rbiu0QPIxTXgOXwiIpRVJfZRQ2FWyfzYrOGBs9SN5RbaXg1CN5ELn/plodwWwluX93yzc4qO/bNIen1ThGFCxw==", + "dev": true, + "optional": true, + "engines": { + "node": ">=10.4.0" + } + }, "node_modules/micromatch": { "version": "4.0.7", "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.7.tgz", @@ -6885,6 +6898,12 @@ "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.18.tgz", "integrity": "sha512-d9VeXT4SJ7ZeOqGX6R5EM022wpL+eWPooLI+5UpWn2jCT1aosUQEhQP214x33Wkwx3JQMvIm+tIoVOdodFS40g==" }, + "node_modules/node-repl-polyfill": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/node-repl-polyfill/-/node-repl-polyfill-0.1.1.tgz", + "integrity": "sha512-RrSiqvuXhtMf4cD1W8Gllyi8bEQtm6FAtSWobL8B+IU5j5vLVL4vLfh4jS7HN4YcjwmL7uh+5bTJ/j+72gnGqA==", + "dev": true + }, "node_modules/normalize-path": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", @@ -7194,6 +7213,24 @@ "node": ">=0.10.0" } }, + "node_modules/porffor": { + "version": "0.37.0", + "resolved": "https://registry.npmjs.org/porffor/-/porffor-0.37.0.tgz", + "integrity": "sha512-WaRAKviG6gVOAgHf4pRjuLjm+Vs71Z9Idbpsl870Y9UwWc86tbTTltGGTTuOFIBrbzV0LXz2aMemSgheyEpeow==", + "dev": true, + "dependencies": { + "acorn": "^8.11.3", + "node-repl-polyfill": "^0.1.1" + }, + "bin": { + "porf": "runner/index.js" + }, + "optionalDependencies": { + "@babel/parser": "^7.24.4", + "hermes-parser": "^0.18.2", + "meriyah": "^4.3.9" + } + }, "node_modules/postcss": { "version": "8.4.41", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.41.tgz", @@ -7433,9 +7470,9 @@ } }, "node_modules/postcss-resolve-nested-selector": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.4.tgz", - "integrity": "sha512-R6vHqZWgVnTAPq0C+xjyHfEZqfIYboCBVSy24MjxEDm+tIh1BU4O6o7DP7AA7kHzf136d+Qc5duI4tlpHjixDw==", + "version": "0.1.5", + "resolved": "https://registry.npmjs.org/postcss-resolve-nested-selector/-/postcss-resolve-nested-selector-0.1.5.tgz", + "integrity": "sha512-tum2m18S22ZSNjXatMG0FSk5ZL83pTttymeJx5Gzxg7RU0s1jNDU9rXltro4osQrukjyNormcb07IEjqEyPNaA==", "dev": true }, "node_modules/postcss-safe-parser": { @@ -8670,6 +8707,15 @@ "node": ">=8" } }, + "node_modules/tapable": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", + "integrity": "sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/tar-fs": { "version": "3.0.6", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-3.0.6.tgz", @@ -8696,9 +8742,9 @@ } }, "node_modules/terser": { - "version": "5.31.3", - "resolved": "https://registry.npmjs.org/terser/-/terser-5.31.3.tgz", - "integrity": "sha512-pAfYn3NIZLyZpa83ZKigvj6Rn9c/vd5KfYGX7cN1mnzqgDcxWvrU5ZtAfIKhEXz9nRecw4z3LXkjaq96/qZqAA==", + "version": "5.31.5", + "resolved": "https://registry.npmjs.org/terser/-/terser-5.31.5.tgz", + "integrity": "sha512-YPmas0L0rE1UyLL/llTWA0SiDOqIcAQYLeUj7cJYzXHlRTAnMSg9pPe4VJ5PlKvTrPQsdVFuiRiwyeNlYgwh2Q==", "dev": true, "dependencies": { "@jridgewell/source-map": "^0.3.3", @@ -9525,6 +9571,7 @@ "license": "MIT", "devDependencies": { "@rmenke/css-tokenizer-tests": "^1.1.6", + "porffor": "^0.37.0", "postcss": "^8.4.38", "postcss-parser-tests": "^8.8.0" }, diff --git a/packages/css-tokenizer/dist/index.cjs b/packages/css-tokenizer/dist/index.cjs index 59fdf698b..9fe2ee0f9 100644 --- a/packages/css-tokenizer/dist/index.cjs +++ b/packages/css-tokenizer/dist/index.cjs @@ -1 +1,1594 @@ -"use strict";class ParseError extends Error{sourceStart;sourceEnd;parserState;constructor(e,n,o,t){super(e),this.name="ParseError",this.sourceStart=n,this.sourceEnd=o,this.parserState=t}}class ParseErrorWithToken extends ParseError{token;constructor(e,n,o,t,r){super(e,n,o,t),this.token=r}}const e={UnexpectedNewLineInString:"Unexpected newline while consuming a string token.",UnexpectedEOFInString:"Unexpected EOF while consuming a string token.",UnexpectedEOFInComment:"Unexpected EOF while consuming a comment.",UnexpectedEOFInURL:"Unexpected EOF while consuming a url token.",UnexpectedEOFInEscapedCodePoint:"Unexpected EOF while consuming an escaped code point.",UnexpectedCharacterInURL:"Unexpected character while consuming a url token.",InvalidEscapeSequenceInURL:"Invalid escape sequence while consuming a url token.",InvalidEscapeSequenceAfterBackslash:'Invalid escape sequence after "\\"'},n="undefined"!=typeof globalThis&&"structuredClone"in globalThis;const o=39,t=42,r=8,s=13,i=9,c=58,a=44,u=64,d=127,p=33,T=12,P=46,k=62,C=45,l=31,f=69,x=101,S=123,m=40,E=91,h=60,y=10,v=11,A=95,g=1114111,I=0,U=35,O=37,D=43,R=34,w=65533,L=92,W=125,N=41,F=93,q=59,b=14,H=47,B=32,V=117,z=85,K=114,M=82,$=108,J=76,_=63,j=48,Q=70;function checkIfFourCodePointsWouldStartCDO(e){return e.source.codePointAt(e.cursor)===h&&e.source.codePointAt(e.cursor+1)===p&&e.source.codePointAt(e.cursor+2)===C&&e.source.codePointAt(e.cursor+3)===C}function isDigitCodePoint(e){return void 0!==e&&e>=48&&e<=57}function isUppercaseLetterCodePoint(e){return void 0!==e&&e>=65&&e<=90}function isLowercaseLetterCodePoint(e){return void 0!==e&&e>=97&&e<=122}function isHexDigitCodePoint(e){return void 0!==e&&(e>=48&&e<=57||e>=97&&e<=102||e>=65&&e<=70)}function isLetterCodePoint(e){return isLowercaseLetterCodePoint(e)||isUppercaseLetterCodePoint(e)}function isIdentStartCodePoint(e){return isLetterCodePoint(e)||isNonASCII_IdentCodePoint(e)||e===A}function isIdentCodePoint(e){return isIdentStartCodePoint(e)||isDigitCodePoint(e)||e===C}function isNonASCII_IdentCodePoint(e){return 183===e||8204===e||8205===e||8255===e||8256===e||8204===e||void 0!==e&&(192<=e&&e<=214||216<=e&&e<=246||248<=e&&e<=893||895<=e&&e<=8191||8304<=e&&e<=8591||11264<=e&&e<=12271||12289<=e&&e<=55295||63744<=e&&e<=64975||65008<=e&&e<=65533||e>=65536)}function isNewLine(e){return e===y||e===s||e===T}function isWhitespace(e){return e===B||e===y||e===i||e===s||e===T}function checkIfTwoCodePointsAreAValidEscape(e){return e.source.codePointAt(e.cursor)===L&&!isNewLine(e.source.codePointAt(e.cursor+1))}function checkIfThreeCodePointsWouldStartAnIdentSequence(e,n){return n.source.codePointAt(n.cursor)===C?n.source.codePointAt(n.cursor+1)===C||(!!isIdentStartCodePoint(n.source.codePointAt(n.cursor+1))||n.source.codePointAt(n.cursor+1)===L&&!isNewLine(n.source.codePointAt(n.cursor+2))):!!isIdentStartCodePoint(n.source.codePointAt(n.cursor))||checkIfTwoCodePointsAreAValidEscape(n)}function checkIfThreeCodePointsWouldStartANumber(e){return e.source.codePointAt(e.cursor)===D||e.source.codePointAt(e.cursor)===C?!!isDigitCodePoint(e.source.codePointAt(e.cursor+1))||e.source.codePointAt(e.cursor+1)===P&&isDigitCodePoint(e.source.codePointAt(e.cursor+2)):e.source.codePointAt(e.cursor)===P?isDigitCodePoint(e.source.codePointAt(e.cursor+1)):isDigitCodePoint(e.source.codePointAt(e.cursor))}function checkIfTwoCodePointsStartAComment(e){return e.source.codePointAt(e.cursor)===H&&e.source.codePointAt(e.cursor+1)===t}function checkIfThreeCodePointsWouldStartCDC(e){return e.source.codePointAt(e.cursor)===C&&e.source.codePointAt(e.cursor+1)===C&&e.source.codePointAt(e.cursor+2)===k}var G,X,Y;function consumeComment(n,o){for(o.advanceCodePoint(2);;){const r=o.readCodePoint();if(void 0===r){const t=[exports.TokenType.Comment,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInComment,o.representationStart,o.representationEnd,["4.3.2. Consume comments","Unexpected EOF"],t)),t}if(r===t&&(void 0!==o.source.codePointAt(o.cursor)&&o.source.codePointAt(o.cursor)===H)){o.advanceCodePoint();break}}return[exports.TokenType.Comment,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,void 0]}function consumeEscapedCodePoint(n,o){const t=o.readCodePoint();if(void 0===t)return n.onParseError(new ParseError(e.UnexpectedEOFInEscapedCodePoint,o.representationStart,o.representationEnd,["4.3.7. Consume an escaped code point","Unexpected EOF"])),w;if(isHexDigitCodePoint(t)){const e=[t];let n;for(;void 0!==(n=o.source.codePointAt(o.cursor))&&isHexDigitCodePoint(n)&&e.length<6;)e.push(n),o.advanceCodePoint();isWhitespace(o.source.codePointAt(o.cursor))&&o.advanceCodePoint();const s=parseInt(String.fromCodePoint(...e),16);return 0===s?w:void 0!==(r=s)&&r>=55296&&r<=57343||s>g?w:s}var r;return t}function consumeIdentSequence(e,n){const o=[];for(;;){const t=n.source.codePointAt(n.cursor);if(isIdentCodePoint(t))o.push(t),n.advanceCodePoint(+(t>65535)+1);else{if(!checkIfTwoCodePointsAreAValidEscape(n))return o;n.advanceCodePoint(),o.push(consumeEscapedCodePoint(e,n))}}}function consumeHashToken(e,n){n.advanceCodePoint();const o=n.source.codePointAt(n.cursor);if(void 0!==o&&(isIdentCodePoint(o)||checkIfTwoCodePointsAreAValidEscape(n))){let o=exports.HashType.Unrestricted;checkIfThreeCodePointsWouldStartAnIdentSequence(0,n)&&(o=exports.HashType.ID);const t=consumeIdentSequence(e,n);return[exports.TokenType.Hash,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...t),type:o}]}return[exports.TokenType.Delim,"#",n.representationStart,n.representationEnd,{value:"#"}]}function consumeNumber(e,n){let o=exports.NumberType.Integer;for(n.source.codePointAt(n.cursor)!==D&&n.source.codePointAt(n.cursor)!==C||n.advanceCodePoint();isDigitCodePoint(n.source.codePointAt(n.cursor));)n.advanceCodePoint();if(n.source.codePointAt(n.cursor)===P&&isDigitCodePoint(n.source.codePointAt(n.cursor+1)))for(n.advanceCodePoint(2),o=exports.NumberType.Number;isDigitCodePoint(n.source.codePointAt(n.cursor));)n.advanceCodePoint();if(n.source.codePointAt(n.cursor)===x||n.source.codePointAt(n.cursor)===f){if(isDigitCodePoint(n.source.codePointAt(n.cursor+1)))n.advanceCodePoint(2);else{if(n.source.codePointAt(n.cursor+1)!==C&&n.source.codePointAt(n.cursor+1)!==D||!isDigitCodePoint(n.source.codePointAt(n.cursor+2)))return o;n.advanceCodePoint(3)}for(o=exports.NumberType.Number;isDigitCodePoint(n.source.codePointAt(n.cursor));)n.advanceCodePoint()}return o}function consumeNumericToken(e,n){let o;{const e=n.source.codePointAt(n.cursor);e===C?o="-":e===D&&(o="+")}const t=consumeNumber(0,n),r=parseFloat(n.source.slice(n.representationStart,n.representationEnd+1));if(checkIfThreeCodePointsWouldStartAnIdentSequence(0,n)){const s=consumeIdentSequence(e,n);return[exports.TokenType.Dimension,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:r,signCharacter:o,type:t,unit:String.fromCodePoint(...s)}]}return n.source.codePointAt(n.cursor)===O?(n.advanceCodePoint(),[exports.TokenType.Percentage,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:r,signCharacter:o}]):[exports.TokenType.Number,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:r,signCharacter:o,type:t}]}function consumeWhiteSpace(e){for(;isWhitespace(e.source.codePointAt(e.cursor));)e.advanceCodePoint();return[exports.TokenType.Whitespace,e.source.slice(e.representationStart,e.representationEnd+1),e.representationStart,e.representationEnd,void 0]}exports.TokenType=void 0,(G=exports.TokenType||(exports.TokenType={})).Comment="comment",G.AtKeyword="at-keyword-token",G.BadString="bad-string-token",G.BadURL="bad-url-token",G.CDC="CDC-token",G.CDO="CDO-token",G.Colon="colon-token",G.Comma="comma-token",G.Delim="delim-token",G.Dimension="dimension-token",G.EOF="EOF-token",G.Function="function-token",G.Hash="hash-token",G.Ident="ident-token",G.Number="number-token",G.Percentage="percentage-token",G.Semicolon="semicolon-token",G.String="string-token",G.URL="url-token",G.Whitespace="whitespace-token",G.OpenParen="(-token",G.CloseParen=")-token",G.OpenSquare="[-token",G.CloseSquare="]-token",G.OpenCurly="{-token",G.CloseCurly="}-token",G.UnicodeRange="unicode-range-token",exports.NumberType=void 0,(X=exports.NumberType||(exports.NumberType={})).Integer="integer",X.Number="number",exports.HashType=void 0,(Y=exports.HashType||(exports.HashType={})).Unrestricted="unrestricted",Y.ID="id";class Reader{cursor=0;source="";representationStart=0;representationEnd=-1;constructor(e){this.source=e}advanceCodePoint(e=1){this.cursor=this.cursor+e,this.representationEnd=this.cursor-1}readCodePoint(){const e=this.source.codePointAt(this.cursor);if(void 0!==e)return this.cursor=this.cursor+1,this.representationEnd=this.cursor-1,e}unreadCodePoint(e=1){this.cursor=this.cursor-e,this.representationEnd=this.cursor-1}resetRepresentation(){this.representationStart=this.cursor,this.representationEnd=-1}}function consumeStringToken(n,o){let t="";const r=o.readCodePoint();for(;;){const i=o.readCodePoint();if(void 0===i){const r=[exports.TokenType.String,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,{value:t}];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInString,o.representationStart,o.representationEnd,["4.3.5. Consume a string token","Unexpected EOF"],r)),r}if(isNewLine(i)){o.unreadCodePoint();const t=[exports.TokenType.BadString,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.UnexpectedNewLineInString,o.representationStart,o.source.codePointAt(o.cursor)===s&&o.source.codePointAt(o.cursor+1)===y?o.representationEnd+2:o.representationEnd+1,["4.3.5. Consume a string token","Unexpected newline"],t)),t}if(i===r)return[exports.TokenType.String,o.source.slice(o.representationStart,o.representationEnd+1),o.representationStart,o.representationEnd,{value:t}];if(i!==L)t+=String.fromCodePoint(i);else{if(void 0===o.source.codePointAt(o.cursor))continue;if(isNewLine(o.source.codePointAt(o.cursor))){o.source.codePointAt(o.cursor)===s&&o.source.codePointAt(o.cursor+1)===y&&o.advanceCodePoint(),o.advanceCodePoint();continue}t+=String.fromCodePoint(consumeEscapedCodePoint(n,o))}}}function checkIfCodePointsMatchURLIdent(e){return!(3!==e.length||e[0]!==V&&e[0]!==z||e[1]!==K&&e[1]!==M||e[2]!==$&&e[2]!==J)}function consumeBadURL(e,n){for(;;){const o=n.source.codePointAt(n.cursor);if(void 0===o)return;if(o===N)return void n.advanceCodePoint();checkIfTwoCodePointsAreAValidEscape(n)?(n.advanceCodePoint(),consumeEscapedCodePoint(e,n)):n.advanceCodePoint()}}function consumeUrlToken(n,t){for(;isWhitespace(t.source.codePointAt(t.cursor));)t.advanceCodePoint();let s="";for(;;){if(void 0===t.source.codePointAt(t.cursor)){const o=[exports.TokenType.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:s}];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","Unexpected EOF"],o)),o}if(t.source.codePointAt(t.cursor)===N)return t.advanceCodePoint(),[exports.TokenType.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:s}];if(isWhitespace(t.source.codePointAt(t.cursor))){for(t.advanceCodePoint();isWhitespace(t.source.codePointAt(t.cursor));)t.advanceCodePoint();if(void 0===t.source.codePointAt(t.cursor)){const o=[exports.TokenType.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:s}];return n.onParseError(new ParseErrorWithToken(e.UnexpectedEOFInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","Consume as much whitespace as possible","Unexpected EOF"],o)),o}return t.source.codePointAt(t.cursor)===N?(t.advanceCodePoint(),[exports.TokenType.URL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,{value:s}]):(consumeBadURL(n,t),[exports.TokenType.BadURL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0])}const c=t.source.codePointAt(t.cursor);if(c===R||c===o||c===m||void 0!==(i=c)&&(i===v||i===d||I<=i&&i<=r||b<=i&&i<=l)){consumeBadURL(n,t);const o=[exports.TokenType.BadURL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.UnexpectedCharacterInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","Unexpected U+0022 QUOTATION MARK (\"), U+0027 APOSTROPHE ('), U+0028 LEFT PARENTHESIS (() or non-printable code point"],o)),o}if(c===L){if(checkIfTwoCodePointsAreAValidEscape(t)){t.advanceCodePoint(),s+=String.fromCodePoint(consumeEscapedCodePoint(n,t));continue}consumeBadURL(n,t);const o=[exports.TokenType.BadURL,t.source.slice(t.representationStart,t.representationEnd+1),t.representationStart,t.representationEnd,void 0];return n.onParseError(new ParseErrorWithToken(e.InvalidEscapeSequenceInURL,t.representationStart,t.representationEnd,["4.3.6. Consume a url token","U+005C REVERSE SOLIDUS (\\)","The input stream does not start with a valid escape sequence"],o)),o}s+=t.source[t.cursor],t.advanceCodePoint()}var i}function consumeIdentLikeToken(e,n){const t=consumeIdentSequence(e,n);if(n.source.codePointAt(n.cursor)!==m)return[exports.TokenType.Ident,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...t)}];if(checkIfCodePointsMatchURLIdent(t)){n.advanceCodePoint();let r=0;for(;;){const e=isWhitespace(n.source.codePointAt(n.cursor)),s=isWhitespace(n.source.codePointAt(n.cursor+1));if(e&&s){r+=1,n.advanceCodePoint(1);continue}const i=e?n.source.codePointAt(n.cursor+1):n.source.codePointAt(n.cursor);if(i===R||i===o)return r>0&&n.unreadCodePoint(r),[exports.TokenType.Function,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...t)}];break}return consumeUrlToken(e,n)}return n.advanceCodePoint(),[exports.TokenType.Function,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{value:String.fromCodePoint(...t)}]}function checkIfThreeCodePointsWouldStartAUnicodeRange(e){return!(e.source.codePointAt(e.cursor)!==V&&e.source.codePointAt(e.cursor)!==z||e.source.codePointAt(e.cursor+1)!==D||e.source.codePointAt(e.cursor+2)!==_&&!isHexDigitCodePoint(e.source.codePointAt(e.cursor+2)))}function consumeUnicodeRangeToken(e,n){n.advanceCodePoint(2);const o=[],t=[];let r;for(;void 0!==(r=n.source.codePointAt(n.cursor))&&o.length<6&&isHexDigitCodePoint(r);)o.push(r),n.advanceCodePoint();for(;void 0!==(r=n.source.codePointAt(n.cursor))&&o.length<6&&r===_;)0===t.length&&t.push(...o),o.push(j),t.push(Q),n.advanceCodePoint();if(!t.length&&n.source.codePointAt(n.cursor)===C&&isHexDigitCodePoint(n.source.codePointAt(n.cursor+1)))for(n.advanceCodePoint();void 0!==(r=n.source.codePointAt(n.cursor))&&t.length<6&&isHexDigitCodePoint(r);)t.push(r),n.advanceCodePoint();if(!t.length){const e=parseInt(String.fromCodePoint(...o),16);return[exports.TokenType.UnicodeRange,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{startOfRange:e,endOfRange:e}]}const s=parseInt(String.fromCodePoint(...o),16),i=parseInt(String.fromCodePoint(...t),16);return[exports.TokenType.UnicodeRange,n.source.slice(n.representationStart,n.representationEnd+1),n.representationStart,n.representationEnd,{startOfRange:s,endOfRange:i}]}function tokenizer(n,t){const r=n.css.valueOf(),d=n.unicodeRangesAllowed??!1,p=new Reader(r),k={onParseError:t?.onParseError??noop};return{nextToken:function nextToken(){p.resetRepresentation();const n=p.source.codePointAt(p.cursor);if(void 0===n)return[exports.TokenType.EOF,"",-1,-1,void 0];if(n===H&&checkIfTwoCodePointsStartAComment(p))return consumeComment(k,p);if(d&&(n===V||n===z)&&checkIfThreeCodePointsWouldStartAUnicodeRange(p))return consumeUnicodeRangeToken(0,p);if(isIdentStartCodePoint(n))return consumeIdentLikeToken(k,p);if(isDigitCodePoint(n))return consumeNumericToken(k,p);switch(n){case a:return p.advanceCodePoint(),[exports.TokenType.Comma,",",p.representationStart,p.representationEnd,void 0];case c:return p.advanceCodePoint(),[exports.TokenType.Colon,":",p.representationStart,p.representationEnd,void 0];case q:return p.advanceCodePoint(),[exports.TokenType.Semicolon,";",p.representationStart,p.representationEnd,void 0];case m:return p.advanceCodePoint(),[exports.TokenType.OpenParen,"(",p.representationStart,p.representationEnd,void 0];case N:return p.advanceCodePoint(),[exports.TokenType.CloseParen,")",p.representationStart,p.representationEnd,void 0];case E:return p.advanceCodePoint(),[exports.TokenType.OpenSquare,"[",p.representationStart,p.representationEnd,void 0];case F:return p.advanceCodePoint(),[exports.TokenType.CloseSquare,"]",p.representationStart,p.representationEnd,void 0];case S:return p.advanceCodePoint(),[exports.TokenType.OpenCurly,"{",p.representationStart,p.representationEnd,void 0];case W:return p.advanceCodePoint(),[exports.TokenType.CloseCurly,"}",p.representationStart,p.representationEnd,void 0];case o:case R:return consumeStringToken(k,p);case U:return consumeHashToken(k,p);case D:case P:return checkIfThreeCodePointsWouldStartANumber(p)?consumeNumericToken(k,p):(p.advanceCodePoint(),[exports.TokenType.Delim,p.source[p.representationStart],p.representationStart,p.representationEnd,{value:p.source[p.representationStart]}]);case y:case s:case T:case i:case B:return consumeWhiteSpace(p);case C:return checkIfThreeCodePointsWouldStartANumber(p)?consumeNumericToken(k,p):checkIfThreeCodePointsWouldStartCDC(p)?(p.advanceCodePoint(3),[exports.TokenType.CDC,"--\x3e",p.representationStart,p.representationEnd,void 0]):checkIfThreeCodePointsWouldStartAnIdentSequence(0,p)?consumeIdentLikeToken(k,p):(p.advanceCodePoint(),[exports.TokenType.Delim,"-",p.representationStart,p.representationEnd,{value:"-"}]);case h:return checkIfFourCodePointsWouldStartCDO(p)?(p.advanceCodePoint(4),[exports.TokenType.CDO,"\x3c!--",p.representationStart,p.representationEnd,void 0]):(p.advanceCodePoint(),[exports.TokenType.Delim,"<",p.representationStart,p.representationEnd,{value:"<"}]);case u:if(p.advanceCodePoint(),checkIfThreeCodePointsWouldStartAnIdentSequence(0,p)){const e=consumeIdentSequence(k,p);return[exports.TokenType.AtKeyword,p.source.slice(p.representationStart,p.representationEnd+1),p.representationStart,p.representationEnd,{value:String.fromCodePoint(...e)}]}return[exports.TokenType.Delim,"@",p.representationStart,p.representationEnd,{value:"@"}];case L:{if(checkIfTwoCodePointsAreAValidEscape(p))return consumeIdentLikeToken(k,p);p.advanceCodePoint();const n=[exports.TokenType.Delim,"\\",p.representationStart,p.representationEnd,{value:"\\"}];return k.onParseError(new ParseErrorWithToken(e.InvalidEscapeSequenceAfterBackslash,p.representationStart,p.representationEnd,["4.3.1. Consume a token","U+005C REVERSE SOLIDUS (\\)","The input stream does not start with a valid escape sequence"],n)),n}}return p.advanceCodePoint(),[exports.TokenType.Delim,p.source[p.representationStart],p.representationStart,p.representationEnd,{value:p.source[p.representationStart]}]},endOfFile:function endOfFile(){return void 0===p.source.codePointAt(p.cursor)}}}function noop(){}function ensureThatValueRoundTripsAsIdent(e){let n=0;e[0]===C&&e[1]===C?n=2:e[0]===C&&e[1]?(n=2,isIdentStartCodePoint(e[1])||(n+=insertEscapedCodePoint(e,1,e[1]))):isIdentStartCodePoint(e[0])?n=1:(n=1,n+=insertEscapedCodePoint(e,0,e[0]));for(let o=n;o */ +const GREATER_THAN_SIGN = 0x003e; +/** - */ +const HYPHEN_MINUS = 0x002d; +/** \x1F */ +const INFORMATION_SEPARATOR_ONE = 0x001f; +/** E */ +const LATIN_CAPITAL_LETTER_E = 0x0045; +/** e */ +const LATIN_SMALL_LETTER_E = 0x0065; +/** { */ +const LEFT_CURLY_BRACKET = 0x007b; +/** ( */ +const LEFT_PARENTHESIS = 0x0028; +/** [ */ +const LEFT_SQUARE_BRACKET = 0x005b; +/** < */ +const LESS_THAN_SIGN = 0x003c; +/** \n */ +const LINE_FEED = 0x00a; +/** \v */ +const LINE_TABULATION = 0x00b; +/** _ */ +const LOW_LINE = 0x005f; +/** \x10FFFF */ +const MAXIMUM_ALLOWED_CODEPOINT = 0x10FFFF; +/** \x00 */ +const NULL = 0x000; +/** # */ +const NUMBER_SIGN = 0x0023; +/** % */ +const PERCENTAGE_SIGN = 0x0025; +/** + */ +const PLUS_SIGN = 0x002b; +/** " */ +const QUOTATION_MARK = 0x0022; +/** � */ +const REPLACEMENT_CHARACTER = 0xFFFD; +/** \ */ +const REVERSE_SOLIDUS = 0x005c; +/** } */ +const RIGHT_CURLY_BRACKET = 0x007d; +/** ) */ +const RIGHT_PARENTHESIS = 0x0029; +/** ] */ +const RIGHT_SQUARE_BRACKET = 0x005d; +/** ; */ +const SEMICOLON = 0x003b; +/** \u0E */ +const SHIFT_OUT = 0x00e; +/** / */ +const SOLIDUS = 0x002f; +/** \u20 */ +const SPACE = 0x0020; +/** u */ +const LATIN_SMALL_LETTER_U = 0x0075; +/** U */ +const LATIN_CAPITAL_LETTER_U = 0x0055; +/** r */ +const LATIN_SMALL_LETTER_R = 0x0072; +/** R */ +const LATIN_CAPITAL_LETTER_R = 0x0052; +/** l */ +const LATIN_SMALL_LETTER_L = 0x006c; +/** L */ +const LATIN_CAPITAL_LETTER_L = 0x004c; +/** ? */ +const QUESTION_MARK = 0x003f; +/** 0 */ +const DIGIT_ZERO = 0x0030; +/** F */ +const LATIN_CAPITAL_LETTER_F = 0x0046; + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-token +function checkIfFourCodePointsWouldStartCDO(reader) { + return reader.source.codePointAt(reader.cursor) === LESS_THAN_SIGN && reader.source.codePointAt(reader.cursor + 1) === EXCLAMATION_MARK && reader.source.codePointAt(reader.cursor + 2) === HYPHEN_MINUS && reader.source.codePointAt(reader.cursor + 3) === HYPHEN_MINUS; +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokenizer-definitions +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#digit +function isDigitCodePoint(search) { + return (typeof search !== "undefined") && search >= 0x0030 && search <= 0x0039; +} +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#uppercase-letter +function isUppercaseLetterCodePoint(search) { + return (typeof search !== "undefined") && search >= 0x0041 && search <= 0x005a; +} +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#lowercase-letter +function isLowercaseLetterCodePoint(search) { + return (typeof search !== "undefined") && search >= 0x0061 && search <= 0x007a; +} +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#hex-digit +function isHexDigitCodePoint(search) { + return (typeof search !== "undefined") && ((search >= 0x0030 && search <= 0x0039) || // 0 .. 9 + (search >= 0x0061 && search <= 0x0066) || // a .. f + (search >= 0x0041 && search <= 0x0046) // A .. F + ); +} +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#letter +function isLetterCodePoint(search) { + return isLowercaseLetterCodePoint(search) || isUppercaseLetterCodePoint(search); +} +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#ident-start-code-point +function isIdentStartCodePoint(search) { + return isLetterCodePoint(search) || isNonASCII_IdentCodePoint(search) || search === LOW_LINE; +} +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#ident-code-point +function isIdentCodePoint(search) { + return isIdentStartCodePoint(search) || isDigitCodePoint(search) || search === HYPHEN_MINUS; +} +// https://drafts.csswg.org/css-syntax/#non-ascii-ident-code-point +function isNonASCII_IdentCodePoint(search) { + if (search === 0x00B7 || + search === 0x200C || + search === 0x200D || + search === 0x203F || + search === 0x2040 || + search === 0x200C) { + return true; + } + if (typeof search === "undefined") { + return false; + } + if ((0x00C0 <= search && search <= 0x00D6) || + (0x00D8 <= search && search <= 0x00F6) || + (0x00F8 <= search && search <= 0x037D) || + (0x037F <= search && search <= 0x1FFF) || + (0x2070 <= search && search <= 0x218F) || + (0x2C00 <= search && search <= 0x2FEF) || + (0x3001 <= search && search <= 0xD7FF) || + (0xF900 <= search && search <= 0xFDCF) || + (0xFDF0 <= search && search <= 0xFFFD)) { + return true; + } + return search >= 0x10000; +} +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#non-printable-code-point +function isNonPrintableCodePoint(search) { + return (typeof search !== "undefined") && ((search === LINE_TABULATION) || + (search === DELETE) || + (NULL <= search && search <= BACKSPACE) || + (SHIFT_OUT <= search && search <= INFORMATION_SEPARATOR_ONE)); +} +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#whitespace +function isNewLine(search) { + return search === LINE_FEED || search === CARRIAGE_RETURN || search === FORM_FEED; +} +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#whitespace +function isWhitespace(search) { + return search === SPACE || search === LINE_FEED || search === CHARACTER_TABULATION || search === CARRIAGE_RETURN || search === FORM_FEED; +} +// https://infra.spec.whatwg.org/#surrogate +function isSurrogate(search) { + return (typeof search !== "undefined") && search >= 0xd800 && search <= 0xdfff; +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#starts-with-a-valid-escape +function checkIfTwoCodePointsAreAValidEscape(reader) { + return ( + // If the first code point is not U+005C REVERSE SOLIDUS (\), return false. + reader.source.codePointAt(reader.cursor) === REVERSE_SOLIDUS && + // Otherwise, if the second code point is a newline, return false. + !isNewLine(reader.source.codePointAt(reader.cursor + 1))); +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#would-start-an-identifier +function checkIfThreeCodePointsWouldStartAnIdentSequence(ctx, reader) { + // // U+002D HYPHEN-MINUS + if (reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS) { + // If the second code point is a U+002D HYPHEN-MINUS return true + if (reader.source.codePointAt(reader.cursor + 1) === HYPHEN_MINUS) { + return true; + } + // If the second code point is an ident-start code point return true + if (isIdentStartCodePoint(reader.source.codePointAt(reader.cursor + 1))) { + return true; + } + // If the second and third code points are a valid escape return true + if (reader.source.codePointAt(reader.cursor + 1) === REVERSE_SOLIDUS && !isNewLine(reader.source.codePointAt(reader.cursor + 2))) { + return true; + } + return false; + } + // ident-start code point + // Return true. + if (isIdentStartCodePoint(reader.source.codePointAt(reader.cursor))) { + return true; + } + // U+005C REVERSE SOLIDUS (\) + return checkIfTwoCodePointsAreAValidEscape(reader); +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#starts-with-a-number +function checkIfThreeCodePointsWouldStartANumber(reader) { + if (reader.source.codePointAt(reader.cursor) === PLUS_SIGN || reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS) { // U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-) + // If the second code point is a digit, return true. + if (isDigitCodePoint(reader.source.codePointAt(reader.cursor + 1))) { + return true; + } + // Otherwise, if the second code point is a U+002E FULL STOP (.) + if (reader.source.codePointAt(reader.cursor + 1) === FULL_STOP) { + // and the third code point is a digit, return true. + return isDigitCodePoint(reader.source.codePointAt(reader.cursor + 2)); + } + // Otherwise, return false. + return false; + } + else if (reader.source.codePointAt(reader.cursor) === FULL_STOP) { // U+002E FULL STOP (.) + // If the second code point is a digit, return true. + // Otherwise, return false. + return isDigitCodePoint(reader.source.codePointAt(reader.cursor + 1)); + } + return isDigitCodePoint(reader.source.codePointAt(reader.cursor)); // digit +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-comments +function checkIfTwoCodePointsStartAComment(reader) { + return (reader.source.codePointAt(reader.cursor) === SOLIDUS && + reader.source.codePointAt(reader.cursor + 1) === ASTERISK); +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-token +function checkIfThreeCodePointsWouldStartCDC(reader) { + return reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS && reader.source.codePointAt(reader.cursor + 1) === HYPHEN_MINUS && reader.source.codePointAt(reader.cursor + 2) === GREATER_THAN_SIGN; +} + +/* eslint-disable @typescript-eslint/no-empty-object-type */ +/** + * All possible CSS token types + */ +exports.TokenType = void 0; +(function (TokenType) { + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#comment-diagram} + */ + TokenType["Comment"] = "comment"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-at-keyword-token} + */ + TokenType["AtKeyword"] = "at-keyword-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-bad-string-token} + */ + TokenType["BadString"] = "bad-string-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-bad-url-token} + */ + TokenType["BadURL"] = "bad-url-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-cdc-token} + */ + TokenType["CDC"] = "CDC-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-cdo-token} + */ + TokenType["CDO"] = "CDO-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-colon-token} + */ + TokenType["Colon"] = "colon-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-comma-token} + */ + TokenType["Comma"] = "comma-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-delim-token} + */ + TokenType["Delim"] = "delim-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-dimension-token} + */ + TokenType["Dimension"] = "dimension-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-eof-token} + */ + TokenType["EOF"] = "EOF-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-function-token} + */ + TokenType["Function"] = "function-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-hash-token} + */ + TokenType["Hash"] = "hash-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-ident-token} + */ + TokenType["Ident"] = "ident-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-percentage-token} + */ + TokenType["Number"] = "number-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-percentage-token} + */ + TokenType["Percentage"] = "percentage-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-semicolon-token} + */ + TokenType["Semicolon"] = "semicolon-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-string-token} + */ + TokenType["String"] = "string-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-url-token} + */ + TokenType["URL"] = "url-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#typedef-whitespace-token} + */ + TokenType["Whitespace"] = "whitespace-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-open-paren} + */ + TokenType["OpenParen"] = "(-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-close-paren} + */ + TokenType["CloseParen"] = ")-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-open-square} + */ + TokenType["OpenSquare"] = "[-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-close-square} + */ + TokenType["CloseSquare"] = "]-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-open-curly} + */ + TokenType["OpenCurly"] = "{-token"; + /** + * @see {@link https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#tokendef-close-curly} + */ + TokenType["CloseCurly"] = "}-token"; + /** + * Only appears in the token stream when the `unicodeRangesAllowed` option is set to true. + * + * @example + * ```js + * import { tokenize } from '@csstools/css-tokenizer'; + * + * const tokens = tokenize({ + * css: `U+0025-00FF, U+4??`, + * unicodeRangesAllowed: true, + * }); + * + * console.log(tokens); + * ``` + * + * @see {@link https://drafts.csswg.org/css-syntax/#typedef-unicode-range-token} + */ + TokenType["UnicodeRange"] = "unicode-range-token"; +})(exports.TokenType || (exports.TokenType = {})); +/** + * The type of number token + * Either `integer` or `number` + */ +exports.NumberType = void 0; +(function (NumberType) { + NumberType["Integer"] = "integer"; + NumberType["Number"] = "number"; +})(exports.NumberType || (exports.NumberType = {})); +/** + * The type of hash token + */ +exports.HashType = void 0; +(function (HashType) { + /** + * The hash token did not start with an ident sequence (e.g. `#-2`) + */ + HashType["Unrestricted"] = "unrestricted"; + /** + * The hash token started with an ident sequence (e.g. `#foo`) + * Only hash tokens with the "id" type are valid ID selectors. + */ + HashType["ID"] = "id"; +})(exports.HashType || (exports.HashType = {})); +/** + * Get the mirror variant type of a given token type + * + * @example + * + * ```js + * const input = TokenType.OpenParen; + * const output = mirrorVariantType(input); + * + * console.log(output); // TokenType.CloseParen + * ``` + */ +function mirrorVariantType(type) { + switch (type) { + case exports.TokenType.OpenParen: + return exports.TokenType.CloseParen; + case exports.TokenType.CloseParen: + return exports.TokenType.OpenParen; + case exports.TokenType.OpenCurly: + return exports.TokenType.CloseCurly; + case exports.TokenType.CloseCurly: + return exports.TokenType.OpenCurly; + case exports.TokenType.OpenSquare: + return exports.TokenType.CloseSquare; + case exports.TokenType.CloseSquare: + return exports.TokenType.OpenSquare; + default: + return null; + } +} +/** + * Get the mirror variant of a given token + * + * @example + * + * ```js + * const input = [TokenType.OpenParen, '(', 0, 1, undefined]; + * const output = mirrorVariant(input); + * + * console.log(output); // [TokenType.CloseParen, ')', -1, -1, undefined] + * ``` + */ +function mirrorVariant(token) { + switch (token[0]) { + case exports.TokenType.OpenParen: + return [exports.TokenType.CloseParen, ')', -1, -1, undefined]; + case exports.TokenType.CloseParen: + return [exports.TokenType.OpenParen, '(', -1, -1, undefined]; + case exports.TokenType.OpenCurly: + return [exports.TokenType.CloseCurly, '}', -1, -1, undefined]; + case exports.TokenType.CloseCurly: + return [exports.TokenType.OpenCurly, '{', -1, -1, undefined]; + case exports.TokenType.OpenSquare: + return [exports.TokenType.CloseSquare, ']', -1, -1, undefined]; + case exports.TokenType.CloseSquare: + return [exports.TokenType.OpenSquare, '[', -1, -1, undefined]; + default: + return null; + } +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-comment +function consumeComment(ctx, reader) { + reader.advanceCodePoint(2); + while (true) { + const codePoint = reader.readCodePoint(); + if (typeof codePoint === "undefined") { + const token = [ + exports.TokenType.Comment, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + undefined, + ]; + ctx.onParseError(new ParseErrorWithToken(ParseErrorMessage.UnexpectedEOFInComment, reader.representationStart, reader.representationEnd, [ + '4.3.2. Consume comments', + 'Unexpected EOF', + ], token)); + return token; + } + if (codePoint !== ASTERISK) { + continue; + } + if (typeof reader.source.codePointAt(reader.cursor) === "undefined") { + continue; + } + if (reader.source.codePointAt(reader.cursor) === SOLIDUS) { + reader.advanceCodePoint(); + break; + } + } + return [ + exports.TokenType.Comment, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + undefined, + ]; +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-escaped-code-point +function consumeEscapedCodePoint(ctx, reader) { + const codePoint = reader.readCodePoint(); + if (typeof codePoint === "undefined") { + ctx.onParseError(new ParseError(ParseErrorMessage.UnexpectedEOFInEscapedCodePoint, reader.representationStart, reader.representationEnd, [ + '4.3.7. Consume an escaped code point', + 'Unexpected EOF', + ])); + return REPLACEMENT_CHARACTER; + } + if (isHexDigitCodePoint(codePoint)) { + const hexSequence = [codePoint]; + let nextCodePoint; + while ((typeof (nextCodePoint = reader.source.codePointAt(reader.cursor)) !== "undefined") && isHexDigitCodePoint(nextCodePoint) && hexSequence.length < 6) { + hexSequence.push(nextCodePoint); + reader.advanceCodePoint(); + } + if (isWhitespace(reader.source.codePointAt(reader.cursor))) { + reader.advanceCodePoint(); + } + const codePointLiteral = parseInt(String.fromCodePoint(...hexSequence), 16); + if (codePointLiteral === 0) { + return REPLACEMENT_CHARACTER; + } + if (isSurrogate(codePointLiteral)) { + return REPLACEMENT_CHARACTER; + } + if (codePointLiteral > MAXIMUM_ALLOWED_CODEPOINT) { + return REPLACEMENT_CHARACTER; + } + return codePointLiteral; + } + return codePoint; +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-name +function consumeIdentSequence(ctx, reader) { + const result = []; + while (true) { + const codePoint = reader.source.codePointAt(reader.cursor); + if (isIdentCodePoint(codePoint)) { + result.push(codePoint); + reader.advanceCodePoint(1 + +(codePoint > 0xffff)); + continue; + } + if (checkIfTwoCodePointsAreAValidEscape(reader)) { + reader.advanceCodePoint(); + result.push(consumeEscapedCodePoint(ctx, reader)); + continue; + } + return result; + } +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-token +function consumeHashToken(ctx, reader) { + reader.advanceCodePoint(); + const codePoint = reader.source.codePointAt(reader.cursor); + if ((typeof codePoint !== "undefined") && (isIdentCodePoint(codePoint) || + checkIfTwoCodePointsAreAValidEscape(reader))) { + let hashType = exports.HashType.Unrestricted; + if (checkIfThreeCodePointsWouldStartAnIdentSequence(ctx, reader)) { + hashType = exports.HashType.ID; + } + const identSequence = consumeIdentSequence(ctx, reader); + return [ + exports.TokenType.Hash, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: String.fromCodePoint(...identSequence), + type: hashType, + }, + ]; + } + return [ + exports.TokenType.Delim, + '#', + reader.representationStart, + reader.representationEnd, + { + value: '#', + }, + ]; +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-number +function consumeNumber(ctx, reader) { + // 1. Initially set type to "integer". + let type = exports.NumberType.Integer; + // 2. If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-), consume it and append it to repr. + if (reader.source.codePointAt(reader.cursor) === PLUS_SIGN || reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS) { + reader.advanceCodePoint(); + } + // 3. While the next input code point is a digit, consume it and append it to repr. + while (isDigitCodePoint(reader.source.codePointAt(reader.cursor))) { + reader.advanceCodePoint(); + } + // 4. If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then: + if (reader.source.codePointAt(reader.cursor) === FULL_STOP && isDigitCodePoint(reader.source.codePointAt(reader.cursor + 1))) { + // 4.1. Consume them. + reader.advanceCodePoint(2); + // 4.3. Set type to "number". + type = exports.NumberType.Number; + // 4.4. While the next input code point is a digit, consume it and append it to repr. + while (isDigitCodePoint(reader.source.codePointAt(reader.cursor))) { + reader.advanceCodePoint(); + } + } + // 5. If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E) or U+0065 LATIN SMALL LETTER E (e), + // optionally followed by U+002D HYPHEN-MINUS (-) or U+002B PLUS SIGN (+), + // followed by a digit, then: + if (reader.source.codePointAt(reader.cursor) === LATIN_SMALL_LETTER_E || reader.source.codePointAt(reader.cursor) === LATIN_CAPITAL_LETTER_E) { + if (isDigitCodePoint(reader.source.codePointAt(reader.cursor + 1))) { + // 5.1. Consume them. + reader.advanceCodePoint(2); + } + else if ((reader.source.codePointAt(reader.cursor + 1) === HYPHEN_MINUS || reader.source.codePointAt(reader.cursor + 1) === PLUS_SIGN) && + isDigitCodePoint(reader.source.codePointAt(reader.cursor + 2))) { + // 5.1. Consume them. + reader.advanceCodePoint(3); + } + else { + return type; + } + // 5.3. Set type to "number". + type = exports.NumberType.Number; + // 5.4. While the next input code point is a digit, consume it and append it to repr. + while (isDigitCodePoint(reader.source.codePointAt(reader.cursor))) { + reader.advanceCodePoint(); + } + } + return type; +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-numeric-token +function consumeNumericToken(ctx, reader) { + let signCharacter = undefined; + { + const peeked = reader.source.codePointAt(reader.cursor); + if (peeked === HYPHEN_MINUS) { + signCharacter = '-'; + } + else if (peeked === PLUS_SIGN) { + signCharacter = '+'; + } + } + const numberType = consumeNumber(ctx, reader); + const numberValue = parseFloat(reader.source.slice(reader.representationStart, reader.representationEnd + 1)); + if (checkIfThreeCodePointsWouldStartAnIdentSequence(ctx, reader)) { + const unit = consumeIdentSequence(ctx, reader); + return [ + exports.TokenType.Dimension, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: numberValue, + signCharacter: signCharacter, + type: numberType, + unit: String.fromCodePoint(...unit), + }, + ]; + } + if (reader.source.codePointAt(reader.cursor) === PERCENTAGE_SIGN) { + reader.advanceCodePoint(); + return [ + exports.TokenType.Percentage, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: numberValue, + signCharacter: signCharacter, + }, + ]; + } + return [ + exports.TokenType.Number, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: numberValue, + signCharacter: signCharacter, + type: numberType, + }, + ]; +} + +function consumeWhiteSpace(reader) { + while (isWhitespace(reader.source.codePointAt(reader.cursor))) { + reader.advanceCodePoint(); + } + return [ + exports.TokenType.Whitespace, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + undefined, + ]; +} + +/** + * @internal + */ +class Reader { + cursor = 0; + source = ''; + representationStart = 0; + representationEnd = -1; + constructor(source) { + this.source = source; + } + advanceCodePoint(n = 1) { + this.cursor = this.cursor + n; + this.representationEnd = this.cursor - 1; + } + readCodePoint() { + const codePoint = this.source.codePointAt(this.cursor); + if (typeof codePoint === "undefined") { + return undefined; + } + this.cursor = this.cursor + 1; + this.representationEnd = this.cursor - 1; + return codePoint; + } + unreadCodePoint(n = 1) { + this.cursor = this.cursor - n; + this.representationEnd = this.cursor - 1; + return; + } + resetRepresentation() { + this.representationStart = this.cursor; + this.representationEnd = -1; + } +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-string-token +function consumeStringToken(ctx, reader) { + let result = ''; + const first = reader.readCodePoint(); + while (true) { + const next = reader.readCodePoint(); + if (typeof next === "undefined") { + const token = [exports.TokenType.String, reader.source.slice(reader.representationStart, reader.representationEnd + 1), reader.representationStart, reader.representationEnd, { value: result }]; + ctx.onParseError(new ParseErrorWithToken(ParseErrorMessage.UnexpectedEOFInString, reader.representationStart, reader.representationEnd, [ + '4.3.5. Consume a string token', + 'Unexpected EOF', + ], token)); + return token; + } + if (isNewLine(next)) { + reader.unreadCodePoint(); + const token = [exports.TokenType.BadString, reader.source.slice(reader.representationStart, reader.representationEnd + 1), reader.representationStart, reader.representationEnd, undefined]; + ctx.onParseError(new ParseErrorWithToken(ParseErrorMessage.UnexpectedNewLineInString, reader.representationStart, ((reader.source.codePointAt(reader.cursor) === CARRIAGE_RETURN && + reader.source.codePointAt(reader.cursor + 1) === LINE_FEED) ? + // CR LF + reader.representationEnd + 2 : + // LF + reader.representationEnd + 1), [ + '4.3.5. Consume a string token', + 'Unexpected newline', + ], token)); + return token; + } + if (next === first) { + return [exports.TokenType.String, reader.source.slice(reader.representationStart, reader.representationEnd + 1), reader.representationStart, reader.representationEnd, { value: result }]; + } + if (next === REVERSE_SOLIDUS) { + if (typeof reader.source.codePointAt(reader.cursor) === "undefined") { + continue; + } + if (isNewLine(reader.source.codePointAt(reader.cursor))) { + if (reader.source.codePointAt(reader.cursor) === CARRIAGE_RETURN && + reader.source.codePointAt(reader.cursor + 1) === LINE_FEED) { + reader.advanceCodePoint(); + } + reader.advanceCodePoint(); + continue; + } + result = result + String.fromCodePoint(consumeEscapedCodePoint(ctx, reader)); + continue; + } + result = result + String.fromCodePoint(next); + } +} + +function checkIfCodePointsMatchURLIdent(codePoints) { + return (codePoints.length === 3 && + (codePoints[0] === LATIN_SMALL_LETTER_U || + codePoints[0] === LATIN_CAPITAL_LETTER_U) && + (codePoints[1] === LATIN_SMALL_LETTER_R || + codePoints[1] === LATIN_CAPITAL_LETTER_R) && + (codePoints[2] === LATIN_SMALL_LETTER_L || + codePoints[2] === LATIN_CAPITAL_LETTER_L)); +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-remnants-of-bad-url +function consumeBadURL(ctx, reader) { + while (true) { + const codePoint = reader.source.codePointAt(reader.cursor); + if (typeof codePoint === "undefined") { + return; + } + if (codePoint === RIGHT_PARENTHESIS) { + reader.advanceCodePoint(); + return; + } + if (checkIfTwoCodePointsAreAValidEscape(reader)) { + reader.advanceCodePoint(); + consumeEscapedCodePoint(ctx, reader); + continue; + } + reader.advanceCodePoint(); + continue; + } +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-url-token +function consumeUrlToken(ctx, reader) { + while (isWhitespace(reader.source.codePointAt(reader.cursor))) { + reader.advanceCodePoint(); + } + let string = ''; + while (true) { + if (typeof reader.source.codePointAt(reader.cursor) === "undefined") { + const token = [ + exports.TokenType.URL, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: string, + }, + ]; + ctx.onParseError(new ParseErrorWithToken(ParseErrorMessage.UnexpectedEOFInURL, reader.representationStart, reader.representationEnd, [ + '4.3.6. Consume a url token', + 'Unexpected EOF', + ], token)); + return token; + } + if (reader.source.codePointAt(reader.cursor) === RIGHT_PARENTHESIS) { + reader.advanceCodePoint(); + return [ + exports.TokenType.URL, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: string, + }, + ]; + } + if (isWhitespace(reader.source.codePointAt(reader.cursor))) { + reader.advanceCodePoint(); + while (isWhitespace(reader.source.codePointAt(reader.cursor))) { + reader.advanceCodePoint(); + } + if (typeof reader.source.codePointAt(reader.cursor) === "undefined") { + const token = [ + exports.TokenType.URL, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: string, + }, + ]; + ctx.onParseError(new ParseErrorWithToken(ParseErrorMessage.UnexpectedEOFInURL, reader.representationStart, reader.representationEnd, [ + '4.3.6. Consume a url token', + 'Consume as much whitespace as possible', + 'Unexpected EOF', + ], token)); + return token; + } + if (reader.source.codePointAt(reader.cursor) === RIGHT_PARENTHESIS) { + reader.advanceCodePoint(); + return [ + exports.TokenType.URL, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: string, + }, + ]; + } + consumeBadURL(ctx, reader); + return [ + exports.TokenType.BadURL, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + undefined, + ]; + } + const codePoint = reader.source.codePointAt(reader.cursor); + if (codePoint === QUOTATION_MARK || codePoint === APOSTROPHE || codePoint === LEFT_PARENTHESIS || isNonPrintableCodePoint(codePoint)) { + consumeBadURL(ctx, reader); + const token = [ + exports.TokenType.BadURL, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + undefined, + ]; + ctx.onParseError(new ParseErrorWithToken(ParseErrorMessage.UnexpectedCharacterInURL, reader.representationStart, reader.representationEnd, [ + '4.3.6. Consume a url token', + 'Unexpected U+0022 QUOTATION MARK ("), U+0027 APOSTROPHE (\'), U+0028 LEFT PARENTHESIS (() or non-printable code point', + ], token)); + return token; + } + if (codePoint === REVERSE_SOLIDUS) { + if (checkIfTwoCodePointsAreAValidEscape(reader)) { + reader.advanceCodePoint(); + string = string + String.fromCodePoint(consumeEscapedCodePoint(ctx, reader)); + continue; + } + consumeBadURL(ctx, reader); + const token = [ + exports.TokenType.BadURL, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + undefined, + ]; + ctx.onParseError(new ParseErrorWithToken(ParseErrorMessage.InvalidEscapeSequenceInURL, reader.representationStart, reader.representationEnd, [ + '4.3.6. Consume a url token', + 'U+005C REVERSE SOLIDUS (\\)', + 'The input stream does not start with a valid escape sequence', + ], token)); + return token; + } + string = string + reader.source[reader.cursor]; + reader.advanceCodePoint(); + } +} + +// https://www.w3.org/TR/2021/CRD-css-syntax-3-20211224/#consume-ident-like-token +function consumeIdentLikeToken(ctx, reader) { + const codePoints = consumeIdentSequence(ctx, reader); + if (reader.source.codePointAt(reader.cursor) !== LEFT_PARENTHESIS) { + return [ + exports.TokenType.Ident, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: String.fromCodePoint(...codePoints), + }, + ]; + } + if (checkIfCodePointsMatchURLIdent(codePoints)) { + reader.advanceCodePoint(); + let read = 0; + while (true) { + const firstIsWhitespace = isWhitespace(reader.source.codePointAt(reader.cursor)); + const secondIsWhitespace = isWhitespace(reader.source.codePointAt(reader.cursor + 1)); + if (firstIsWhitespace && secondIsWhitespace) { + read = read + 1; + reader.advanceCodePoint(1); + continue; + } + const firstNonWhitespace = firstIsWhitespace ? reader.source.codePointAt(reader.cursor + 1) : reader.source.codePointAt(reader.cursor); + if (firstNonWhitespace === QUOTATION_MARK || firstNonWhitespace === APOSTROPHE) { + if (read > 0) { + // https://github.com/w3c/csswg-drafts/issues/8280#issuecomment-1370566921 + reader.unreadCodePoint(read); + } + return [ + exports.TokenType.Function, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: String.fromCodePoint(...codePoints), + }, + ]; + } + break; + } + return consumeUrlToken(ctx, reader); + } + reader.advanceCodePoint(); + return [ + exports.TokenType.Function, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + value: String.fromCodePoint(...codePoints), + }, + ]; +} + +// https://drafts.csswg.org/css-syntax/#starts-a-unicode-range +function checkIfThreeCodePointsWouldStartAUnicodeRange(reader) { + if ( + // The first code point is either U+0055 LATIN CAPITAL LETTER U (U) or U+0075 LATIN SMALL LETTER U (u) + (reader.source.codePointAt(reader.cursor) === LATIN_SMALL_LETTER_U || + reader.source.codePointAt(reader.cursor) === LATIN_CAPITAL_LETTER_U) && + // The second code point is U+002B PLUS SIGN (+). + reader.source.codePointAt(reader.cursor + 1) === PLUS_SIGN && + // The third code point is either U+003F QUESTION MARK (?) or a hex digit + (reader.source.codePointAt(reader.cursor + 2) === QUESTION_MARK || + isHexDigitCodePoint(reader.source.codePointAt(reader.cursor + 2)))) { + // then return true. + return true; + } + // Otherwise return false. + return false; +} + +// https://drafts.csswg.org/css-syntax/#starts-a-unicode-range +function consumeUnicodeRangeToken(ctx, reader) { + // 1. Consume the next two input code points and discard them. + reader.advanceCodePoint(2); + const firstSegment = []; + const secondSegment = []; + // 2. Consume as many hex digits as possible, + // but no more than 6. + let codePoint; + while ((typeof (codePoint = reader.source.codePointAt(reader.cursor)) !== "undefined") && + firstSegment.length < 6 && + isHexDigitCodePoint(codePoint)) { + firstSegment.push(codePoint); + reader.advanceCodePoint(); + } + // 2. If less than 6 hex digits were consumed, + // consume as many U+003F QUESTION MARK (?) code points as possible, + // but no more than enough to make the total of hex digits and U+003F QUESTION MARK (?) code points equal to 6. + while ((typeof (codePoint = reader.source.codePointAt(reader.cursor)) !== "undefined") && + firstSegment.length < 6 && + codePoint === QUESTION_MARK) { + if (secondSegment.length === 0) { + secondSegment.push(...firstSegment); + } + // 3. If first segment contains any question mark code points, then: + // 3.1 Replace the question marks in first segment with U+0030 DIGIT ZERO (0) code points. + firstSegment.push(DIGIT_ZERO); + // 3.2.Replace the question marks in first segment with U+0046 LATIN CAPITAL LETTER F (F) code points. + secondSegment.push(LATIN_CAPITAL_LETTER_F); + reader.advanceCodePoint(); + } + if (!secondSegment.length) { + // 5. If the next 2 input code points are U+002D HYPHEN-MINUS (-) followed by a hex digit + if (reader.source.codePointAt(reader.cursor) === HYPHEN_MINUS && + isHexDigitCodePoint(reader.source.codePointAt(reader.cursor + 1))) { + // 5.1. Consume the next input code point. + reader.advanceCodePoint(); + // 5.2 Consume as many hex digits as possible, + // but no more than 6. + while ((typeof (codePoint = reader.source.codePointAt(reader.cursor)) !== "undefined") && + secondSegment.length < 6 && + isHexDigitCodePoint(codePoint)) { + secondSegment.push(codePoint); + reader.advanceCodePoint(); + } + } + } + if (!secondSegment.length) { + // Interpret the consumed code points as a hexadecimal number. + const startOfRange = parseInt(String.fromCodePoint(...firstSegment), 16); + // Return a new both starting and ending at start of range. + return [ + exports.TokenType.UnicodeRange, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + startOfRange: startOfRange, + endOfRange: startOfRange, + }, + ]; + } + // Interpret the consumed code points as a hexadecimal number. + const startOfRange = parseInt(String.fromCodePoint(...firstSegment), 16); + const endOfRange = parseInt(String.fromCodePoint(...secondSegment), 16); + // Return a new starting at start of range and ending at end of range. + return [ + exports.TokenType.UnicodeRange, + reader.source.slice(reader.representationStart, reader.representationEnd + 1), + reader.representationStart, + reader.representationEnd, + { + startOfRange: startOfRange, + endOfRange: endOfRange, + }, + ]; +} + +/** + * Tokenize a CSS string into a list of tokens. + */ +function tokenize(input, options) { + const t = tokenizer(input, options); + const tokens = []; + { + while (!t.endOfFile()) { + const token = t.nextToken(); + if (token) { + tokens.push(token); + } + } + const token = t.nextToken(); // EOF-token + if (token) { + tokens.push(token); + } + } + return tokens; +} +/** + * Create a tokenizer for a CSS string. + */ +function tokenizer(input, options) { + const css = input.css.valueOf(); + console.log(css.codePointAt(0)); + const unicodeRangesAllowed = input.unicodeRangesAllowed ?? false; + const reader = new Reader(css); + console.log(reader.source.codePointAt(0)); + const ctx = { + onParseError: options?.onParseError ?? noop, + }; + const endOfFile = () => { + console.log('ok? eof'); + console.log(typeof reader); + const y = reader.source.codePointAt(reader.cursor); + console.log(y); + const x = typeof y === "undefined"; + console.log('ok eof'); + return x; + }; + const nextToken = () => { + reader.resetRepresentation(); + console.log('ok?'); + const peeked = reader.source.codePointAt(reader.cursor); + console.log('ok'); + if (typeof peeked === "undefined") { + return [exports.TokenType.EOF, '', -1, -1, undefined]; + } + if (peeked === SOLIDUS && checkIfTwoCodePointsStartAComment(reader)) { + return consumeComment(ctx, reader); + } + if (unicodeRangesAllowed && (peeked === LATIN_SMALL_LETTER_U || + peeked === LATIN_CAPITAL_LETTER_U) && + checkIfThreeCodePointsWouldStartAUnicodeRange(reader)) { + return consumeUnicodeRangeToken(ctx, reader); + } + if (isIdentStartCodePoint(peeked)) { + return consumeIdentLikeToken(ctx, reader); + } + if (isDigitCodePoint(peeked)) { + return consumeNumericToken(ctx, reader); + } + // Simple, one character tokens: + switch (peeked) { + case COMMA: + reader.advanceCodePoint(); + return [exports.TokenType.Comma, ',', reader.representationStart, reader.representationEnd, undefined]; + case COLON: + reader.advanceCodePoint(); + return [exports.TokenType.Colon, ':', reader.representationStart, reader.representationEnd, undefined]; + case SEMICOLON: + reader.advanceCodePoint(); + return [exports.TokenType.Semicolon, ';', reader.representationStart, reader.representationEnd, undefined]; + case LEFT_PARENTHESIS: + reader.advanceCodePoint(); + return [exports.TokenType.OpenParen, '(', reader.representationStart, reader.representationEnd, undefined]; + case RIGHT_PARENTHESIS: + reader.advanceCodePoint(); + return [exports.TokenType.CloseParen, ')', reader.representationStart, reader.representationEnd, undefined]; + case LEFT_SQUARE_BRACKET: + reader.advanceCodePoint(); + return [exports.TokenType.OpenSquare, '[', reader.representationStart, reader.representationEnd, undefined]; + case RIGHT_SQUARE_BRACKET: + reader.advanceCodePoint(); + return [exports.TokenType.CloseSquare, ']', reader.representationStart, reader.representationEnd, undefined]; + case LEFT_CURLY_BRACKET: + reader.advanceCodePoint(); + return [exports.TokenType.OpenCurly, '{', reader.representationStart, reader.representationEnd, undefined]; + case RIGHT_CURLY_BRACKET: + reader.advanceCodePoint(); + return [exports.TokenType.CloseCurly, '}', reader.representationStart, reader.representationEnd, undefined]; + case APOSTROPHE: + case QUOTATION_MARK: + return consumeStringToken(ctx, reader); + case NUMBER_SIGN: + return consumeHashToken(ctx, reader); + case PLUS_SIGN: + case FULL_STOP: + if (checkIfThreeCodePointsWouldStartANumber(reader)) { + return consumeNumericToken(ctx, reader); + } + reader.advanceCodePoint(); + return [exports.TokenType.Delim, reader.source[reader.representationStart], reader.representationStart, reader.representationEnd, { + value: reader.source[reader.representationStart], + }]; + case LINE_FEED: + case CARRIAGE_RETURN: + case FORM_FEED: + case CHARACTER_TABULATION: + case SPACE: + return consumeWhiteSpace(reader); + case HYPHEN_MINUS: + if (checkIfThreeCodePointsWouldStartANumber(reader)) { + return consumeNumericToken(ctx, reader); + } + if (checkIfThreeCodePointsWouldStartCDC(reader)) { + reader.advanceCodePoint(3); + return [exports.TokenType.CDC, '-->', reader.representationStart, reader.representationEnd, undefined]; + } + if (checkIfThreeCodePointsWouldStartAnIdentSequence(ctx, reader)) { + return consumeIdentLikeToken(ctx, reader); + } + reader.advanceCodePoint(); + return [exports.TokenType.Delim, '-', reader.representationStart, reader.representationEnd, { + value: '-', + }]; + case LESS_THAN_SIGN: + if (checkIfFourCodePointsWouldStartCDO(reader)) { + reader.advanceCodePoint(4); + return [exports.TokenType.CDO, '', reader.representationStart, reader.representationEnd, undefined]; + } + if (checkIfThreeCodePointsWouldStartAnIdentSequence(ctx, reader)) { + return consumeIdentLikeToken(ctx, reader); + } + reader.advanceCodePoint(); + return [TokenType.Delim, '-', reader.representationStart, reader.representationEnd, { + value: '-', + }]; + case LESS_THAN_SIGN: + if (checkIfFourCodePointsWouldStartCDO(reader)) { + reader.advanceCodePoint(4); + return [TokenType.CDO, '', reader.representationStart, reader.representationEnd, undefined]; + } + if (checkIfThreeCodePointsWouldStartAnIdentSequence(ctx, reader)) { + return consumeIdentLikeToken(ctx, reader); + } + reader.advanceCodePoint(); + return [TokenType.Delim, '-', reader.representationStart, reader.representationEnd, { + value: '-', + }]; + case LESS_THAN_SIGN: + if (checkIfFourCodePointsWouldStartCDO(reader)) { + reader.advanceCodePoint(4); + return [TokenType.CDO, '