Build CodePointTries in datagen

Author: robertbastian

components/collections/codepointtrie_builder/src/common.rs

@@ -2,7 +2,9 @@
 // called LICENSE at the top level of the ICU4X source tree
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
-use icu_collections::codepointtrie::TrieType;
+use icu_collections::codepointtrie::{TrieType, TrieValue};
+
+use crate::{CodePointTrieBuilder, CodePointTrieBuilderData};
 
 /// Returns the type and width arguments for `umutablecptrie_buildImmutable`
 pub(crate) fn args_for_build_immutable<U>(trie_type: TrieType) -> (u32, u32) {
@@ -18,3 +20,37 @@ pub(crate) fn args_for_build_immutable<U>(trie_type: TrieType) -> (u32, u32) {
     };
     (trie_type, width)
 }
+
+impl<T> CodePointTrieBuilder<'_, T>
+where
+    T: TrieValue,
+{
+    pub(crate) fn for_each_code_point(&self, mut f: impl FnMut((u32, T))) {
+        match self.data {
+            CodePointTrieBuilderData::ValuesByCodePoint(values) => {
+                for (cp, &value) in values.iter().enumerate() {
+                    if value != self.default_value {
+                        f((cp as u32, value))
+                    }
+                }
+            }
+            CodePointTrieBuilderData::ByCodePoint(ref v) => {
+                for cp in 0..=(char::MAX as u32) {
+                    let Some(value) = v(cp) else {
+                        continue;
+                    };
+                    if value != self.default_value {
+                        f((cp, value))
+                    }
+                }
+            }
+            CodePointTrieBuilderData::Map(ref map) => {
+                for (&cp, &value) in map {
+                    if value != self.default_value {
+                        f((cp, value))
+                    }
+                }
+            }
+        }
+    }
+}

components/collections/codepointtrie_builder/src/lib.rs

@@ -85,6 +85,10 @@
     )
 )]
 
+use std::collections::HashMap;
+use std::panic::RefUnwindSafe;
+use std::panic::UnwindSafe;
+
 use icu_collections::codepointtrie::TrieType;
 use icu_collections::codepointtrie::TrieValue;
 
@@ -103,13 +107,30 @@ mod native;
 ///
 /// [`CodePointTrie`]: icu_collections::codepointtrie::CodePointTrie
 #[non_exhaustive]
-#[derive(Debug)]
 pub enum CodePointTrieBuilderData<'a, T> {
     /// A list of values for each code point, starting from code point 0.
     ///
     /// For example, the value for U+0020 (space) should be at index 32 in the slice.
     /// Index 0 sets the value for the U+0000 (NUL).
     ValuesByCodePoint(&'a [T]),
+    /// A closure that returns a value for a code point.
+    ///
+    /// This is called for every code point.
+    ByCodePoint(Box<dyn Fn(u32) -> Option<T> + Send + Sync + UnwindSafe + RefUnwindSafe + 'a>),
+    /// A map from code points to values.
+    Map(HashMap<u32, T>),
+}
+
+impl<'a, T: std::fmt::Debug> std::fmt::Debug for CodePointTrieBuilderData<'a, T> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Self::ValuesByCodePoint(arg0) => {
+                f.debug_tuple("ValuesByCodePoint").field(arg0).finish()
+            }
+            Self::ByCodePoint(_) => f.debug_tuple("ByCodePoint").finish(),
+            Self::Map(arg0) => f.debug_tuple("Map").field(arg0).finish(),
+        }
+    }
 }
 
 /// Settings for building a [`CodePointTrie`].

components/collections/codepointtrie_builder/src/native.rs

@@ -3,7 +3,6 @@
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
 use crate::CodePointTrieBuilder;
-use crate::CodePointTrieBuilderData;
 
 use icu_collections::codepointtrie::TrieType;
 use icu_collections::codepointtrie::TrieValue;
@@ -94,21 +93,16 @@ where
         panic!("cpt builder returned error code {error}");
     }
 
-    let CodePointTrieBuilderData::ValuesByCodePoint(values) = cpt_builder.data;
-
-    for (cp, value) in values.iter().enumerate() {
-        let value = value.to_u32();
-        if value != cpt_builder.default_value.to_u32() {
-            unsafe {
-                // safety: builder is a valid UMutableCPTrie
-                // safety: we're passing a valid error pointer
-                umutablecptrie_set(builder, cp as u32, value, &mut error);
-            }
-            if error != 0 {
-                panic!("cpt builder returned error code {error}");
-            }
+    cpt_builder.for_each_code_point(|(cp, value)| {
+        unsafe {
+            // safety: builder is a valid UMutableCPTrie
+            // safety: we're passing a valid error pointer
+            umutablecptrie_set(builder, cp, value.to_u32(), &mut error);
         }
-    }
+        if error != 0 {
+            panic!("cpt builder returned error code {error}");
+        }
+    });
 
     let (trie_type, width) =
         crate::common::args_for_build_immutable::<T::ULE>(cpt_builder.trie_type);

components/collections/codepointtrie_builder/src/wasm.rs

@@ -3,7 +3,6 @@
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
 use crate::CodePointTrieBuilder;
-use crate::CodePointTrieBuilderData;
 use icu_collections::codepointtrie::CodePointTrie;
 use icu_collections::codepointtrie::CodePointTrieHeader;
 use icu_collections::codepointtrie::TrieValue;
@@ -229,13 +228,9 @@ where
         &error_code_ptr,
     );
 
-    let CodePointTrieBuilderData::ValuesByCodePoint(values) = builder.data;
-    for (cp, value) in values.iter().enumerate() {
-        let num = value.to_u32();
-        if num != builder.default_value.to_u32() {
-            wasm.umutablecptrie_set(&trie_ptr, cp as u32, num, &error_code_ptr);
-        }
-    }
+    builder.for_each_code_point(|(cp, value)| {
+        wasm.umutablecptrie_set(&trie_ptr, cp, value.to_u32(), &error_code_ptr);
+    });
 
     let (trie_type, width) = crate::common::args_for_build_immutable::<T::ULE>(builder.trie_type);
 

provider/data/properties/data/property_enum_bidi_class_v1.rs.data

@@ -68,7 +68,7 @@ property/binary/white/space/v1, <singleton>, 92B, 62B, 8f045d65fbaa6724
 property/binary/xdigit/v1, <singleton>, 488B, 459B, 334cc78d29ee5f0e
 property/binary/xid/continue/v1, <singleton>, 4868B, 4839B, 386b045eb6522e2f
 property/binary/xid/start/v1, <singleton>, 4178B, 4149B, 5090f2f1b95a3480
-property/enum/bidi/class/v1, <singleton>, 9884B, 9828B, 327160efebade591
+property/enum/bidi/class/v1, <singleton>, 9888B, 9832B, 6e461ef365a53dcc
 property/enum/bidi/mirroring/glyph/v1, <singleton>, 4739B, 4673B, 34ba0898d34a25fb
 property/enum/canonical/combining/class/v1, <singleton>, 5452B, 5394B, 6c5392d6fc5f2889
 property/enum/east/asian/width/v1, <singleton>, 5004B, 4947B, 881a7ee1402b1937
@@ -81,7 +81,7 @@ property/enum/joining/group/v1, <singleton>, 1428B, 1370B, a24c126ff86065d8
 property/enum/joining/type/v1, <singleton>, 7136B, 7079B, 39f05bd0b9931b55
 property/enum/line/break/v1, <singleton>, 15328B, 15272B, d392409dfdd00a74
 property/enum/numeric/type/v1, <singleton>, 5900B, 5842B, 9ffe79ebbb54f863
-property/enum/script/v1, <singleton>, 25902B, 25847B, 1b972fe75ad7b369
+property/enum/script/v1, <singleton>, 25900B, 25845B, f1ad61c965485250
 property/enum/sentence/break/v1, <singleton>, 13964B, 13907B, 36e6dafc005907f8
 property/enum/vertical/orientation/v1, <singleton>, 3280B, 3222B, 36f945b6b4d90bc8
 property/enum/word/break/v1, <singleton>, 11136B, 11079B, ae4831ff49f66415