feat: support Unicode properties added in ES2023 (#59)

ota-meshi · web-flow · commit f66121da1189 · 2023-03-29T09:54:52.000+09:00
diff --git a/.github/workflows/cron.yml b/.github/workflows/cron.yml
@@ -0,0 +1,24 @@
+name: cron
+on:
+  schedule:
+    - cron: 0 0 * * 0
+
+jobs:
+  check-resource-update:
+    name: check-resource-update
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: Install Packages
+        run: npm install
+      - name: Update
+        run: npm run update:unicode
+      - name: Check changes
+        run: |
+            git add --all && \
+            git diff-index --cached HEAD --stat --exit-code
diff --git a/scripts/fetch-lines.ts b/scripts/fetch-lines.ts
@@ -0,0 +1,4 @@
+export async function* fetchLines(url: string): AsyncIterable<string> {
+    const response = await fetch(url)
+    yield* (await response.text()).split("\n")
+}
diff --git a/scripts/get-latest-unicode-general-category-values.ts b/scripts/get-latest-unicode-general-category-values.ts
@@ -0,0 +1,11 @@
+import { getLatestUnicodePropertyValues } from "./get-latest-unicode-property-values"
+
+export async function* getLatestUnicodeGeneralCategoryValues(): AsyncIterable<string> {
+    for await (const value of getLatestUnicodePropertyValues()) {
+        if (value.propertyAlias !== "gc") {
+            continue
+        }
+
+        yield* value.aliases
+    }
+}
diff --git a/scripts/get-latest-unicode-property-values.ts b/scripts/get-latest-unicode-property-values.ts
@@ -0,0 +1,40 @@
+import { fetchLines } from "./fetch-lines"
+
+const DB_URL =
+    "https://unicode.org/Public/UCD/latest/ucd/PropertyValueAliases.txt"
+const logger = console
+export type UnicodePropertyValue = {
+    propertyAlias: string
+    aliases: string[]
+    canonical: string
+}
+let cache: UnicodePropertyValue[] | undefined = undefined
+export async function* getLatestUnicodePropertyValues(): AsyncIterable<UnicodePropertyValue> {
+    logger.log("Fetching data... (%s)", DB_URL)
+    const iterable = cache
+        ? cache
+        : (async function* () {
+              const newCache: UnicodePropertyValue[] = []
+              for await (const line of fetchLines(DB_URL)) {
+                  if (!line || line.startsWith("#")) {
+                      continue
+                  }
+                  const [propertyAlias, alias, canonical, ...remaining] = line
+                      .split("#")[0] // strip comments
+                      .split(";") // split by semicolon
+                      .map((x) => x.trim()) // trim
+
+                  const value: UnicodePropertyValue = {
+                      propertyAlias,
+                      aliases: [canonical, alias, ...remaining],
+                      canonical,
+                  }
+                  newCache.push(value)
+                  yield value
+              }
+              cache = newCache
+          })()
+    for await (const value of iterable) {
+        yield value
+    }
+}
diff --git a/scripts/get-latest-unicode-script-values.ts b/scripts/get-latest-unicode-script-values.ts
@@ -0,0 +1,11 @@
+import { getLatestUnicodePropertyValues } from "./get-latest-unicode-property-values"
+
+export async function* getLatestUnicodeScriptValues(): AsyncIterable<string> {
+    for await (const value of getLatestUnicodePropertyValues()) {
+        if (value.propertyAlias !== "sc") {
+            continue
+        }
+
+        yield* value.aliases
+    }
+}
diff --git a/scripts/update-unicode-ids.ts b/scripts/update-unicode-ids.ts
@@ -1,8 +1,8 @@
 import fs from "fs"
-import http from "http"
 import { ESLint } from "eslint"
+import { fetchLines } from "./fetch-lines"
 
-const DB_URL = "http://unicode.org/Public/UNIDATA/DerivedCoreProperties.txt"
+const DB_URL = "https://unicode.org/Public/UNIDATA/DerivedCoreProperties.txt"
 const FILE_PATH = "src/unicode/ids.ts"
 const ID_START = /^([0-9a-z]+)(?:\.\.([0-9a-z]+))?[^;]*; ID_Start /iu
 const ID_CONTINUE = /^([0-9a-z]+)(?:\.\.([0-9a-z]+))?[^;]*; ID_Continue /iu
@@ -19,7 +19,7 @@ const main = async () => {
     const idContinueLarge: [number, number][] = []
 
     logger.log("Fetching data... (%s)", DB_URL)
-    await processEachLine((line) => {
+    for await (const line of fetchLines(DB_URL)) {
         let m: RegExpExecArray | null = null
         if (banner === "") {
             logger.log("Processing data... (%s)", line.slice(2))
@@ -42,7 +42,7 @@ const main = async () => {
                 idContinueLarge.push([min, max])
             }
         }
-    })
+    }
 
     logger.log("Normalizing data...")
     normalizeRanges(idStartSmall)
@@ -127,33 +127,6 @@ main().catch((err) => {
     process.exitCode = 1
 })
 
-function processEachLine(processLine: (line: string) => void): Promise<void> {
-    return new Promise((resolve, reject) => {
-        http.get(DB_URL, (res) => {
-            let buffer = ""
-            res.setEncoding("utf8")
-            res.on("data", (chunk) => {
-                const lines = (buffer + String(chunk)).split("\n")
-                if (lines.length === 1) {
-                    buffer = lines[0]
-                } else {
-                    buffer = lines.pop()!
-                    for (const line of lines) {
-                        processLine(line)
-                    }
-                }
-            })
-            res.on("end", () => {
-                if (buffer) {
-                    processLine(buffer)
-                }
-                resolve()
-            })
-            res.on("error", reject)
-        }).on("error", reject)
-    })
-}
-
 function normalizeRanges(ranges: [number, number][]): void {
     for (let i = ranges.length - 1; i >= 1; --i) {
         const currRange = ranges[i]
diff --git a/scripts/update-unicode-properties.ts b/scripts/update-unicode-properties.ts
@@ -2,6 +2,8 @@ import fs from "fs"
 import type { DOMWindow } from "jsdom"
 import { JSDOM } from "jsdom"
 import { ESLint } from "eslint"
+import { getLatestUnicodeGeneralCategoryValues } from "./get-latest-unicode-general-category-values"
+import { getLatestUnicodeScriptValues } from "./get-latest-unicode-script-values"
 
 const DATA_SOURCES = [
     {
@@ -43,8 +45,8 @@ const DATA_SOURCES = [
         url: "https://tc39.es/ecma262/multipage/text-processing.html",
         version: 2023,
         binProperties: "#table-binary-unicode-properties",
-        gcValues: "#table-unicode-general-category-values",
-        scValues: "#table-unicode-script-values",
+        gcValues: getLatestUnicodeGeneralCategoryValues,
+        scValues: getLatestUnicodeScriptValues,
     },
 ]
 const FILE_PATH = "src/unicode/properties.ts"
@@ -96,13 +98,21 @@ type Datum = {
         } while (window == null)
 
         logger.log("Parsing tables")
-        datum.binProperties = collectValues(
+        datum.binProperties = await collectValues(
             window,
             binProperties,
             existing.binProperties,
         )
-        datum.gcValues = collectValues(window, gcValues, existing.gcValues)
-        datum.scValues = collectValues(window, scValues, existing.scValues)
+        datum.gcValues = await collectValues(
+            window,
+            gcValues,
+            existing.gcValues,
+        )
+        datum.scValues = await collectValues(
+            window,
+            scValues,
+            existing.scValues,
+        )
 
         logger.log("Done")
     }
@@ -169,32 +179,55 @@ export function isValidLoneUnicodeProperty(version: number, value: string): bool
     process.exitCode = 1
 })
 
-function collectValues(
+async function collectValues(
     window: DOMWindow,
-    id: string,
+    idSelectorOrProvider: string | (() => AsyncIterable<string>),
     existingSet: Set<string>,
-): string[] {
-    const selector = `${id} td:nth-child(1) code`
-    const nodes = window.document.querySelectorAll(selector)
-    const values = Array.from(nodes, (node) => node.textContent ?? "")
-        .filter((value) => {
-            if (existingSet.has(value)) {
-                return false
-            }
-            existingSet.add(value)
-            return true
-        })
-        .sort(undefined)
+): Promise<string[]> {
+    const getValues =
+        typeof idSelectorOrProvider === "function"
+            ? idSelectorOrProvider
+            : function* (): Iterable<string> {
+                  const selector = `${idSelectorOrProvider} td:nth-child(1) code`
+                  const nodes = window.document.querySelectorAll(selector)
+                  if (nodes.length === 0) {
+                      throw new Error(`No nodes found for selector ${selector}`)
+                  }
+                  logger.log(
+                      "%o nodes of %o were found.",
+                      nodes.length,
+                      selector,
+                  )
+                  for (const node of Array.from(nodes)) {
+                      yield node.textContent ?? ""
+                  }
+              }
+
+    const missing = new Set(existingSet)
+    const values = new Set<string>()
+    let allCount = 0
+
+    for await (const value of getValues()) {
+        allCount++
+        missing.delete(value)
+        if (existingSet.has(value)) {
+            continue
+        }
+        existingSet.add(value)
+        values.add(value)
+    }
+
+    if (missing.size > 0) {
+        throw new Error(`Missing values: ${Array.from(missing).join(", ")}`)
+    }
 
     logger.log(
-        "%o nodes of %o were found, then %o adopted and %o ignored as duplication.",
-        nodes.length,
-        selector,
-        values.length,
-        nodes.length - values.length,
+        "%o adopted and %o ignored as duplication.",
+        values.size,
+        allCount - values.size,
     )
 
-    return values
+    return [...values].sort((a, b) => (a > b ? 1 : a < b ? -1 : 0))
 }
 
 function makeClassDeclarationCode(versions: string[]): string {
diff --git a/src/ecma-versions.ts b/src/ecma-versions.ts
@@ -8,3 +8,4 @@ export type EcmaVersion =
     | 2020
     | 2021
     | 2022
+    | 2023
diff --git a/src/parser.ts b/src/parser.ts
@@ -45,7 +45,7 @@ class RegExpParserState {
 
     public constructor(options?: RegExpParser.Options) {
         this.strict = Boolean(options?.strict)
-        this.ecmaVersion = options?.ecmaVersion ?? 2022
+        this.ecmaVersion = options?.ecmaVersion ?? 2023
     }
 
     public get pattern(): Pattern {
@@ -519,12 +519,13 @@ export namespace RegExpParser {
         strict?: boolean
 
         /**
-         * ECMAScript version. Default is `2022`.
+         * ECMAScript version. Default is `2023`.
          * - `2015` added `u` and `y` flags.
          * - `2018` added `s` flag, Named Capturing Group, Lookbehind Assertion,
          *   and Unicode Property Escape.
          * - `2019`, `2020`, and `2021` added more valid Unicode Property Escapes.
          * - `2022` added `d` flag.
+         * - `2023` added more valid Unicode Property Escapes.
          */
         ecmaVersion?: EcmaVersion
     }
diff --git a/src/unicode/properties.ts b/src/unicode/properties.ts
@@ -94,7 +94,7 @@ const scValueSets = new DataSet(
     "Elym Elymaic Hmnp Nand Nandinagari Nyiakeng_Puachue_Hmong Wancho Wcho",
     "Chorasmian Chrs Diak Dives_Akuru Khitan_Small_Script Kits Yezi Yezidi",
     "Cpmn Cypro_Minoan Old_Uyghur Ougr Tangsa Tnsa Toto Vith Vithkuqi",
-    "",
+    "Hrkt Katakana_Or_Hiragana Kawi Nag_Mundari Nagm Unknown Zzzz",
 )
 const binPropertySets = new DataSet(
     "AHex ASCII ASCII_Hex_Digit Alpha Alphabetic Any Assigned Bidi_C Bidi_Control Bidi_M Bidi_Mirrored CI CWCF CWCM CWKCF CWL CWT CWU Case_Ignorable Cased Changes_When_Casefolded Changes_When_Casemapped Changes_When_Lowercased Changes_When_NFKC_Casefolded Changes_When_Titlecased Changes_When_Uppercased DI Dash Default_Ignorable_Code_Point Dep Deprecated Dia Diacritic Emoji Emoji_Component Emoji_Modifier Emoji_Modifier_Base Emoji_Presentation Ext Extender Gr_Base Gr_Ext Grapheme_Base Grapheme_Extend Hex Hex_Digit IDC IDS IDSB IDST IDS_Binary_Operator IDS_Trinary_Operator ID_Continue ID_Start Ideo Ideographic Join_C Join_Control LOE Logical_Order_Exception Lower Lowercase Math NChar Noncharacter_Code_Point Pat_Syn Pat_WS Pattern_Syntax Pattern_White_Space QMark Quotation_Mark RI Radical Regional_Indicator SD STerm Sentence_Terminal Soft_Dotted Term Terminal_Punctuation UIdeo Unified_Ideograph Upper Uppercase VS Variation_Selector White_Space XIDC XIDS XID_Continue XID_Start space",
@@ -119,7 +119,8 @@ export function isValidUnicodeProperty(
             (version >= 2019 && scValueSets.es2019.has(value)) ||
             (version >= 2020 && scValueSets.es2020.has(value)) ||
             (version >= 2021 && scValueSets.es2021.has(value)) ||
-            (version >= 2022 && scValueSets.es2022.has(value))
+            (version >= 2022 && scValueSets.es2022.has(value)) ||
+            (version >= 2023 && scValueSets.es2023.has(value))
         )
     }
     return false
diff --git a/src/validator.ts b/src/validator.ts
@@ -127,12 +127,13 @@ export namespace RegExpValidator {
         strict?: boolean
 
         /**
-         * ECMAScript version. Default is `2022`.
+         * ECMAScript version. Default is `2023`.
          * - `2015` added `u` and `y` flags.
          * - `2018` added `s` flag, Named Capturing Group, Lookbehind Assertion,
          *   and Unicode Property Escape.
          * - `2019`, `2020`, and `2021` added more valid Unicode Property Escapes.
          * - `2022` added `d` flag.
+         * - `2023` added more valid Unicode Property Escapes.
          */
         ecmaVersion?: EcmaVersion
 
@@ -614,7 +615,7 @@ export class RegExpValidator {
     }
 
     private get ecmaVersion() {
-        return this._options.ecmaVersion ?? 2022
+        return this._options.ecmaVersion ?? 2023
     }
 
     private onLiteralEnter(start: number): void {
diff --git a/test/fixtures/parser/literal.ts b/test/fixtures/parser/literal.ts
@@ -1,21 +1,13 @@
 import fs from "fs"
 import path from "path"
+import type { EcmaVersion } from "../../../src/ecma-versions"
 
 type FixtureData = Record<
     string,
     {
         options: {
             strict: boolean
-            ecmaVersion:
-                | 5
-                | 2015
-                | 2016
-                | 2017
-                | 2018
-                | 2019
-                | 2020
-                | 2021
-                | 2022
+            ecmaVersion: EcmaVersion
         }
         patterns: Record<
             string,
diff --git a/test/fixtures/visitor/index.ts b/test/fixtures/visitor/index.ts
@@ -1,21 +1,13 @@
 import fs from "fs"
 import path from "path"
+import type { EcmaVersion } from "../../../src/ecma-versions"
 
 type FixtureData = Record<
     string,
     {
         options: {
             strict?: boolean
-            ecmaVersion?:
-                | 5
-                | 2015
-                | 2016
-                | 2017
-                | 2018
-                | 2019
-                | 2020
-                | 2021
-                | 2022
+            ecmaVersion?: EcmaVersion
         }
         patterns: Record<string, string[]>
     }

-Original file line number
+Diff line change
     | 2020
     | 2021
     | 2022
 +    | 2023
Original file line number	Diff line number	Diff line change
`@@ -94,7 +94,7 @@ const scValueSets = new DataSet(`
`94`	`94`	`"Elym Elymaic Hmnp Nand Nandinagari Nyiakeng_Puachue_Hmong Wancho Wcho",`
`95`	`95`	`"Chorasmian Chrs Diak Dives_Akuru Khitan_Small_Script Kits Yezi Yezidi",`
`96`	`96`	`"Cpmn Cypro_Minoan Old_Uyghur Ougr Tangsa Tnsa Toto Vith Vithkuqi",`
`97`		`- "",`
	`97`	`+ "Hrkt Katakana_Or_Hiragana Kawi Nag_Mundari Nagm Unknown Zzzz",`
`98`	`98`	`)`
`99`	`99`	`const binPropertySets = new DataSet(`
`100`	`100`	"AHex ASCII ASCII_Hex_Digit Alpha Alphabetic Any Assigned Bidi_C Bidi_Control Bidi_M Bidi_Mirrored CI CWCF CWCM CWKCF CWL CWT CWU Case_Ignorable Cased Changes_When_Casefolded Changes_When_Casemapped Changes_When_Lowercased Changes_When_NFKC_Casefolded Changes_When_Titlecased Changes_When_Uppercased DI Dash Default_Ignorable_Code_Point Dep Deprecated Dia Diacritic Emoji Emoji_Component Emoji_Modifier Emoji_Modifier_Base Emoji_Presentation Ext Extender Gr_Base Gr_Ext Grapheme_Base Grapheme_Extend Hex Hex_Digit IDC IDS IDSB IDST IDS_Binary_Operator IDS_Trinary_Operator ID_Continue ID_Start Ideo Ideographic Join_C Join_Control LOE Logical_Order_Exception Lower Lowercase Math NChar Noncharacter_Code_Point Pat_Syn Pat_WS Pattern_Syntax Pattern_White_Space QMark Quotation_Mark RI Radical Regional_Indicator SD STerm Sentence_Terminal Soft_Dotted Term Terminal_Punctuation UIdeo Unified_Ideograph Upper Uppercase VS Variation_Selector White_Space XIDC XIDS XID_Continue XID_Start space",
`@@ -119,7 +119,8 @@ export function isValidUnicodeProperty(`
`119`	`119`	`(version >= 2019 && scValueSets.es2019.has(value)) \|\|`
`120`	`120`	`(version >= 2020 && scValueSets.es2020.has(value)) \|\|`
`121`	`121`	`(version >= 2021 && scValueSets.es2021.has(value)) \|\|`
`122`		`- (version >= 2022 && scValueSets.es2022.has(value))`
	`122`	`+ (version >= 2022 && scValueSets.es2022.has(value)) \|\|`
	`123`	`+ (version >= 2023 && scValueSets.es2023.has(value))`
`123`	`124`	`)`
`124`	`125`	`}`
`125`	`126`	`return false`