evolve special primaries data struct

Author: robertbastian

components/collator/src/comparison.rs

@@ -37,8 +37,8 @@ use crate::provider::CollationMetadataV1;
 use crate::provider::CollationReordering;
 use crate::provider::CollationReorderingV1;
 use crate::provider::CollationRootV1;
+use crate::provider::CollationSpecialPrimaries;
 use crate::provider::CollationSpecialPrimariesV1;
-use crate::provider::CollationSpecialPrimariesValidated;
 use crate::provider::CollationTailoringV1;
 use core::cmp::Ordering;
 use core::convert::Infallible;
@@ -48,7 +48,6 @@ use icu_normalizer::provider::NormalizerNfdDataV1;
 use icu_normalizer::provider::NormalizerNfdTablesV1;
 use icu_normalizer::DecomposingNormalizerBorrowed;
 use icu_normalizer::Decomposition;
-use icu_provider::marker::ErasedMarker;
 use icu_provider::prelude::*;
 use smallvec::SmallVec;
 use utf16_iter::Utf16CharsEx;
@@ -546,7 +545,7 @@ impl LocaleSpecificDataHolder {
 /// Compares strings according to culturally-relevant ordering.
 #[derive(Debug)]
 pub struct Collator {
-    special_primaries: DataPayload<ErasedMarker<CollationSpecialPrimariesValidated<'static>>>,
+    special_primaries: DataPayload<CollationSpecialPrimariesV1>,
     root: DataPayload<CollationRootV1>,
     tailoring: Option<DataPayload<CollationTailoringV1>>,
     jamo: DataPayload<CollationJamoV1>,
@@ -646,8 +645,6 @@ impl Collator {
         let locale_dependent =
             LocaleSpecificDataHolder::try_new_unstable_internal(provider, prefs, options)?;
 
-        let special_primaries = special_primaries.map_project(|csp, _| csp.validated());
-
         Ok(Collator {
             special_primaries,
             root,
@@ -693,7 +690,7 @@ macro_rules! compare {
 /// borrowed version.
 #[derive(Debug)]
 pub struct CollatorBorrowed<'a> {
-    special_primaries: &'a CollationSpecialPrimariesValidated<'a>,
+    special_primaries: &'a CollationSpecialPrimaries<'a>,
     root: &'a CollationData<'a>,
     tailoring: Option<&'a CollationData<'a>>,
     jamo: &'a CollationJamo<'a>,
@@ -720,13 +717,10 @@ impl CollatorBorrowed<'static> {
         let tables = icu_normalizer::provider::Baked::SINGLETON_NORMALIZER_NFD_TABLES_V1;
         let root = crate::provider::Baked::SINGLETON_COLLATION_ROOT_V1;
         let jamo = crate::provider::Baked::SINGLETON_COLLATION_JAMO_V1;
-
         let locale_dependent =
             LocaleSpecificDataHolder::try_new_unstable_internal(provider, prefs, options)?;
 
-        let special_primaries = const {
-            &crate::provider::Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V1.const_validated()
-        };
+        let special_primaries = crate::provider::Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V1;
 
         // Attribute belongs closer to `unwrap`, but
         // https://github.com/rust-lang/rust/issues/15701

components/collator/src/provider.rs

@@ -588,20 +588,22 @@ impl CollationMetadata {
 /// to be stable, their Rust representation might not be. Use with caution.
 /// </div>
 #[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
-#[cfg_attr(feature = "datagen", derive(serde::Serialize, databake::Bake))]
+#[cfg_attr(feature = "datagen", derive(databake::Bake))]
 #[cfg_attr(feature = "datagen", databake(path = icu_collator::provider))]
 pub struct CollationSpecialPrimaries<'data> {
     /// The primaries corresponding to `MaxVariable`
     /// character classes packed so that each fits in
     /// 16 bits. Length must match the number of enum
     /// variants in `MaxVariable`, currently 4.
-    ///
-    /// This is potentially followed by 256 bits
-    /// (packed in 16 u16s) to classify every possible
-    /// byte into compressible or non-compressible.
     pub last_primaries: ZeroVec<'data, u16>,
     /// The high 8 bits of the numeric primary
     pub numeric_primary: u8,
+    /// 256 bits (packed in 16 u16s) to classify every possible
+    /// byte into compressible or non-compressible.
+    ///
+    /// In the serde encoding, this is appended to `last_primaries`,
+    /// or might be missing.
+    pub compressible_bytes: ZeroVec<'data, u16>,
 }
 
 #[cfg(feature = "serde")]
@@ -613,126 +615,87 @@ impl<'de> serde::Deserialize<'de> for CollationSpecialPrimaries<'de> {
         #[derive(serde::Deserialize)]
         struct Raw<'data> {
             #[cfg_attr(feature = "serde", serde(borrow))]
-            last_primaries: ZeroVec<'data, u16>,
+            concatenated: &'data ZeroSlice<u16>,
             numeric_primary: u8,
         }
 
         let Raw {
-            last_primaries,
+            concatenated,
             numeric_primary,
         } = Raw::deserialize(deserializer)?;
 
-        // `variant_count` isn't stable yet:
-        // https://github.com/rust-lang/rust/issues/73662
-        if last_primaries.len() <= (MaxVariable::Currency as usize) {
+        let Some((l, c)) = concatenated
+            .as_ule_slice()
+            // `variant_count` isn't stable yet:
+            // https://github.com/rust-lang/rust/issues/73662
+            .split_at_checked(MaxVariable::Currency as usize)
+        else {
             return Err(serde::de::Error::custom("invalid"));
+        };
+
+        let last_primaries = ZeroSlice::from_ule_slice(l).as_zerovec();
+        let mut compressible_bytes = ZeroSlice::from_ule_slice(c).as_zerovec();
+
+        if c.len() != 16 {
+            compressible_bytes = zerovec::zerovec!(
+                u16; <u16 as AsULE>::ULE::from_unsigned; [
+                0b0000_0000_0000_0000,
+                0b0000_0000_0000_0000,
+                0b0000_0000_0000_0000,
+                0b0000_0000_0000_0000,
+                0b0000_0000_0000_0000,
+                0b0000_0000_0000_0000,
+                0b1111_1111_1111_1110,
+                0b1111_1111_1111_1111,
+                0b0000_0000_0000_0001,
+                0b0000_0000_0000_0000,
+                0b0000_0000_0000_0000,
+                0b0000_0000_0000_0000,
+                0b0000_0000_0000_0000,
+                0b0000_0000_0000_0000,
+                0b0000_0000_0000_0000,
+                0b0100_0000_0000_0000,
+            ]);
         }
 
         Ok(Self {
             last_primaries,
             numeric_primary,
+            compressible_bytes,
         })
     }
 }
 
-#[derive(Debug, PartialEq, Clone, yoke::Yokeable, zerofrom::ZeroFrom)]
-pub(crate) struct CollationSpecialPrimariesValidated<'data> {
-    /// The primaries corresponding to `MaxVariable`
-    /// character classes packed so that each fits in
-    /// 16 bits. Length must match the number of enum
-    /// variants in `MaxVariable`, currently 4.
-    pub last_primaries: ZeroVec<'data, u16>,
-    /// The high 8 bits of the numeric primary
-    pub numeric_primary: u8,
-    /// 256 bits (packed in 16 u16s) to classify every possible
-    /// byte into compressible or non-compressible.
-    pub compressible_bytes: &'data [<u16 as AsULE>::ULE; 16],
-}
-
-impl<'a> CollationSpecialPrimaries<'a> {
-    const HARDCODED_COMPRESSIBLE_BYTES_FALLBACK: &'static [<u16 as AsULE>::ULE; 16] = &[
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b1111_1111_1111_1110),
-        <u16 as AsULE>::ULE::from_unsigned(0b1111_1111_1111_1111),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0001),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0000_0000_0000_0000),
-        <u16 as AsULE>::ULE::from_unsigned(0b0100_0000_0000_0000),
-    ];
-
-    pub(crate) fn validated(self) -> CollationSpecialPrimariesValidated<'a> {
-        let (last_primaries, compressible_bytes) =
-            if let Some(borrowed) = self.last_primaries.as_maybe_borrowed() {
-                let (l, c) = borrowed
-                    .as_ule_slice()
-                    // by invariant
-                    .split_at(MaxVariable::Currency as usize);
-                (
-                    l,
-                    c.try_into()
-                        .unwrap_or(Self::HARDCODED_COMPRESSIBLE_BYTES_FALLBACK),
-                )
-            } else {
-                (
-                    self.last_primaries.as_slice().as_ule_slice(),
-                    Self::HARDCODED_COMPRESSIBLE_BYTES_FALLBACK,
-                )
-            };
-
-        let last_primaries_truncate_len = last_primaries.len();
-        CollationSpecialPrimariesValidated {
-            last_primaries: self.last_primaries.truncated(last_primaries_truncate_len),
-            numeric_primary: self.numeric_primary,
-            compressible_bytes,
+#[cfg(feature = "datagen")]
+impl serde::Serialize for CollationSpecialPrimaries<'_> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: serde::Serializer,
+    {
+        #[derive(serde::Serialize)]
+        struct Raw {
+            concatenated: ZeroVec<'static, u16>,
+            numeric_primary: u8,
         }
-    }
 
-    pub(crate) const fn const_validated(&'static self) -> CollationSpecialPrimariesValidated<'a> {
-        let borrowed = self.last_primaries.as_slice();
-        let (last_primaries, compressible_bytes) = borrowed
-            .as_ule_slice()
-            // by invariant
-            .split_at(MaxVariable::Currency as usize);
-        // TODO: use c.as_array() on MSRV 1.93
-        let compressible_bytes = if compressible_bytes.len() == 16 {
-            unsafe { &*(compressible_bytes.as_ptr() as *const [<u16 as AsULE>::ULE; 16]) }
-        } else {
-            Self::HARDCODED_COMPRESSIBLE_BYTES_FALLBACK
-        };
-
-        CollationSpecialPrimariesValidated {
-            last_primaries: ZeroSlice::from_ule_slice(last_primaries).as_zerovec(),
+        Raw {
+            concatenated: self
+                .last_primaries
+                .iter()
+                .chain(self.compressible_bytes.iter())
+                .collect(),
             numeric_primary: self.numeric_primary,
-            compressible_bytes,
         }
+        .serialize(serializer)
     }
 }
 
-#[test]
-fn compressible_bytes() {
-    assert_eq!(
-        Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V1
-            .clone()
-            .validated(),
-        Baked::SINGLETON_COLLATION_SPECIAL_PRIMARIES_V1.const_validated(),
-    );
-}
-
 icu_provider::data_struct!(
     CollationSpecialPrimaries<'_>,
     #[cfg(feature = "datagen")]
 );
 
-impl CollationSpecialPrimariesValidated<'_> {
+impl CollationSpecialPrimaries<'_> {
     #[expect(clippy::unwrap_used)]
     pub(crate) fn last_primary_for_group(&self, max_variable: MaxVariable) -> u32 {
         // `unwrap` is OK, because `Collator::try_new` validates the length.
@@ -744,11 +707,10 @@ impl CollationSpecialPrimariesValidated<'_> {
 
     #[allow(dead_code)]
     pub(crate) fn is_compressible(&self, b: u8) -> bool {
-        // Indexing slicing OK by construction and pasting this
-        // into Compiler Explorer shows that the panic
-        // is optimized away.
-        #[expect(clippy::indexing_slicing)]
-        let field = u16::from_unaligned(self.compressible_bytes[usize::from(b >> 4)]);
+        let field = self
+            .compressible_bytes
+            .get(usize::from(b >> 4))
+            .unwrap_or_default();
         let mask = 1 << (b & 0b1111);
         (field & mask) != 0
     }

provider/data/collator/data/collation_special_primaries_v1.rs.data

@@ -181,7 +181,7 @@ collation/reordering/v1, und-Hans, -> und-Hani/pinyin
 collation/reordering/v1, und-Hant, -> und-Hani/stroke
 collation/reordering/v1, ur, -> ar
 collation/root/v1, <singleton>, 131040B, 130923B, 787ce37ea65e1e9
-collation/special/primaries/v1, <singleton>, 72B, 42B, 46181a77c61fe445
+collation/special/primaries/v1, <singleton>, 96B, 42B, 46181a77c61fe445
 collation/tailoring/v1, <lookup>, 608B, 102 identifiers
 collation/tailoring/v1, <total>, 920210B, 908458B, 93 unique payloads
 collation/tailoring/v1, af, 1004B, 877B, 8e3ca7ba0c0efe4b

provider/data/collator/fingerprints.csv

@@ -352,13 +352,9 @@ impl TryInto<CollationSpecialPrimaries<'static>> for &collator_serde::CollationS
         }
 
         Ok(CollationSpecialPrimaries {
-            last_primaries: self
-                .last_primaries
-                .iter()
-                .copied()
-                .chain(packed_compressible_bytes)
-                .collect(),
+            last_primaries: self.last_primaries.iter().copied().collect(),
             numeric_primary: self.numeric_primary,
+            compressible_bytes: packed_compressible_bytes.into_iter().collect(),
         })
     }
 }