From 3993bc1dd5ec921ec63080e2bed8761e98f82228 Mon Sep 17 00:00:00 2001 From: Robert Bastian <4706271+robertbastian@users.noreply.github.com> Date: Mon, 23 Feb 2026 15:46:53 +0100 Subject: [PATCH 1/4] add missing collation testdata --- .../metadata/v1/pinyin/und-Hani.json | 3 + .../metadata/v1/stroke/und-Hani.json | 3 + .../debug/collation/metadata/v1/und-Hani.json | 3 + .../debug/collation/metadata/v1/und-Hans.json | 3 + .../debug/collation/metadata/v1/und-Hant.json | 3 + .../metadata/v1/unihan/und-Hani.json | 3 + .../metadata/v1/zhuyin/und-Hani.json | 3 + .../reordering/v1/pinyin/und-Hani.json | 262 +++++++++++++++++ .../reordering/v1/stroke/und-Hani.json | 266 ++++++++++++++++++ .../collation/reordering/v1/und-Hans.json | 262 +++++++++++++++++ .../collation/reordering/v1/und-Hant.json | 266 ++++++++++++++++++ .../reordering/v1/unihan/und-Hani.json | 266 ++++++++++++++++++ .../reordering/v1/zhuyin/und-Hani.json | 266 ++++++++++++++++++ provider/source/src/tests/data.rs | 8 + .../collation/implicithan/zh_pinyin_meta.toml | 6 + .../implicithan/zh_pinyin_reord.toml | 27 ++ .../collation/implicithan/zh_stroke_meta.toml | 6 + .../implicithan/zh_stroke_reord.toml | 27 ++ .../collation/implicithan/zh_unihan_meta.toml | 6 + .../implicithan/zh_unihan_reord.toml | 27 ++ .../collation/implicithan/zh_zhuyin_meta.toml | 6 + .../implicithan/zh_zhuyin_reord.toml | 27 ++ .../make/download-repo-sources/globs.rs.data | 8 + 23 files changed, 1757 insertions(+) create mode 100644 provider/source/data/debug/collation/metadata/v1/pinyin/und-Hani.json create mode 100644 provider/source/data/debug/collation/metadata/v1/stroke/und-Hani.json create mode 100644 provider/source/data/debug/collation/metadata/v1/und-Hani.json create mode 100644 provider/source/data/debug/collation/metadata/v1/und-Hans.json create mode 100644 provider/source/data/debug/collation/metadata/v1/und-Hant.json create mode 100644 provider/source/data/debug/collation/metadata/v1/unihan/und-Hani.json create mode 100644 provider/source/data/debug/collation/metadata/v1/zhuyin/und-Hani.json create mode 100644 provider/source/data/debug/collation/reordering/v1/pinyin/und-Hani.json create mode 100644 provider/source/data/debug/collation/reordering/v1/stroke/und-Hani.json create mode 100644 provider/source/data/debug/collation/reordering/v1/und-Hans.json create mode 100644 provider/source/data/debug/collation/reordering/v1/und-Hant.json create mode 100644 provider/source/data/debug/collation/reordering/v1/unihan/und-Hani.json create mode 100644 provider/source/data/debug/collation/reordering/v1/zhuyin/und-Hani.json create mode 100644 provider/source/tests/data/icuexport/collation/implicithan/zh_pinyin_meta.toml create mode 100644 provider/source/tests/data/icuexport/collation/implicithan/zh_pinyin_reord.toml create mode 100644 provider/source/tests/data/icuexport/collation/implicithan/zh_stroke_meta.toml create mode 100644 provider/source/tests/data/icuexport/collation/implicithan/zh_stroke_reord.toml create mode 100644 provider/source/tests/data/icuexport/collation/implicithan/zh_unihan_meta.toml create mode 100644 provider/source/tests/data/icuexport/collation/implicithan/zh_unihan_reord.toml create mode 100644 provider/source/tests/data/icuexport/collation/implicithan/zh_zhuyin_meta.toml create mode 100644 provider/source/tests/data/icuexport/collation/implicithan/zh_zhuyin_reord.toml diff --git a/provider/source/data/debug/collation/metadata/v1/pinyin/und-Hani.json b/provider/source/data/debug/collation/metadata/v1/pinyin/und-Hani.json new file mode 100644 index 00000000000..5929352390a --- /dev/null +++ b/provider/source/data/debug/collation/metadata/v1/pinyin/und-Hani.json @@ -0,0 +1,3 @@ +{ + "bits": 41 +} diff --git a/provider/source/data/debug/collation/metadata/v1/stroke/und-Hani.json b/provider/source/data/debug/collation/metadata/v1/stroke/und-Hani.json new file mode 100644 index 00000000000..5929352390a --- /dev/null +++ b/provider/source/data/debug/collation/metadata/v1/stroke/und-Hani.json @@ -0,0 +1,3 @@ +{ + "bits": 41 +} diff --git a/provider/source/data/debug/collation/metadata/v1/und-Hani.json b/provider/source/data/debug/collation/metadata/v1/und-Hani.json new file mode 100644 index 00000000000..39521269d4c --- /dev/null +++ b/provider/source/data/debug/collation/metadata/v1/und-Hani.json @@ -0,0 +1,3 @@ +{ + "bits": 1 +} diff --git a/provider/source/data/debug/collation/metadata/v1/und-Hans.json b/provider/source/data/debug/collation/metadata/v1/und-Hans.json new file mode 100644 index 00000000000..5929352390a --- /dev/null +++ b/provider/source/data/debug/collation/metadata/v1/und-Hans.json @@ -0,0 +1,3 @@ +{ + "bits": 41 +} diff --git a/provider/source/data/debug/collation/metadata/v1/und-Hant.json b/provider/source/data/debug/collation/metadata/v1/und-Hant.json new file mode 100644 index 00000000000..5929352390a --- /dev/null +++ b/provider/source/data/debug/collation/metadata/v1/und-Hant.json @@ -0,0 +1,3 @@ +{ + "bits": 41 +} diff --git a/provider/source/data/debug/collation/metadata/v1/unihan/und-Hani.json b/provider/source/data/debug/collation/metadata/v1/unihan/und-Hani.json new file mode 100644 index 00000000000..5929352390a --- /dev/null +++ b/provider/source/data/debug/collation/metadata/v1/unihan/und-Hani.json @@ -0,0 +1,3 @@ +{ + "bits": 41 +} diff --git a/provider/source/data/debug/collation/metadata/v1/zhuyin/und-Hani.json b/provider/source/data/debug/collation/metadata/v1/zhuyin/und-Hani.json new file mode 100644 index 00000000000..5929352390a --- /dev/null +++ b/provider/source/data/debug/collation/metadata/v1/zhuyin/und-Hani.json @@ -0,0 +1,3 @@ +{ + "bits": 41 +} diff --git a/provider/source/data/debug/collation/reordering/v1/pinyin/und-Hani.json b/provider/source/data/debug/collation/reordering/v1/pinyin/und-Hani.json new file mode 100644 index 00000000000..18418a67702 --- /dev/null +++ b/provider/source/data/debug/collation/reordering/v1/pinyin/und-Hani.json @@ -0,0 +1,262 @@ +{ + "min_high_no_reorder": 4261412864, + "reorder_table": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 254, + 255 + ], + "reorder_ranges": [] +} diff --git a/provider/source/data/debug/collation/reordering/v1/stroke/und-Hani.json b/provider/source/data/debug/collation/reordering/v1/stroke/und-Hani.json new file mode 100644 index 00000000000..5a496800b4b --- /dev/null +++ b/provider/source/data/debug/collation/reordering/v1/stroke/und-Hani.json @@ -0,0 +1,266 @@ +{ + "min_high_no_reorder": 4261412864, + "reorder_table": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 0, + 249, + 250, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 254, + 255 + ], + "reorder_ranges": [ + 2121662503, + 2164260986, + 4261478311 + ] +} diff --git a/provider/source/data/debug/collation/reordering/v1/und-Hans.json b/provider/source/data/debug/collation/reordering/v1/und-Hans.json new file mode 100644 index 00000000000..18418a67702 --- /dev/null +++ b/provider/source/data/debug/collation/reordering/v1/und-Hans.json @@ -0,0 +1,262 @@ +{ + "min_high_no_reorder": 4261412864, + "reorder_table": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 254, + 255 + ], + "reorder_ranges": [] +} diff --git a/provider/source/data/debug/collation/reordering/v1/und-Hant.json b/provider/source/data/debug/collation/reordering/v1/und-Hant.json new file mode 100644 index 00000000000..5a496800b4b --- /dev/null +++ b/provider/source/data/debug/collation/reordering/v1/und-Hant.json @@ -0,0 +1,266 @@ +{ + "min_high_no_reorder": 4261412864, + "reorder_table": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 0, + 249, + 250, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 254, + 255 + ], + "reorder_ranges": [ + 2121662503, + 2164260986, + 4261478311 + ] +} diff --git a/provider/source/data/debug/collation/reordering/v1/unihan/und-Hani.json b/provider/source/data/debug/collation/reordering/v1/unihan/und-Hani.json new file mode 100644 index 00000000000..5a496800b4b --- /dev/null +++ b/provider/source/data/debug/collation/reordering/v1/unihan/und-Hani.json @@ -0,0 +1,266 @@ +{ + "min_high_no_reorder": 4261412864, + "reorder_table": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 0, + 249, + 250, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 254, + 255 + ], + "reorder_ranges": [ + 2121662503, + 2164260986, + 4261478311 + ] +} diff --git a/provider/source/data/debug/collation/reordering/v1/zhuyin/und-Hani.json b/provider/source/data/debug/collation/reordering/v1/zhuyin/und-Hani.json new file mode 100644 index 00000000000..5a496800b4b --- /dev/null +++ b/provider/source/data/debug/collation/reordering/v1/zhuyin/und-Hani.json @@ -0,0 +1,266 @@ +{ + "min_high_no_reorder": 4261412864, + "reorder_table": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, + 25, + 26, + 27, + 28, + 29, + 30, + 31, + 32, + 33, + 34, + 35, + 36, + 37, + 38, + 39, + 40, + 41, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 0, + 249, + 250, + 40, + 41, + 42, + 43, + 44, + 45, + 46, + 47, + 48, + 49, + 50, + 51, + 52, + 53, + 54, + 55, + 56, + 57, + 58, + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 254, + 255 + ], + "reorder_ranges": [ + 2121662503, + 2164260986, + 4261478311 + ] +} diff --git a/provider/source/src/tests/data.rs b/provider/source/src/tests/data.rs index 6b0fc65f54e..8fb0ed3e6a0 100644 --- a/provider/source/src/tests/data.rs +++ b/provider/source/src/tests/data.rs @@ -574,6 +574,14 @@ impl SourceDataProvider { ("collation/implicithan/zh_stroke_data.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_stroke_data.toml").as_slice()), ("collation/implicithan/zh_unihan_data.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_unihan_data.toml").as_slice()), ("collation/implicithan/zh_zhuyin_data.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_zhuyin_data.toml").as_slice()), + ("collation/implicithan/zh_pinyin_meta.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_pinyin_meta.toml").as_slice()), + ("collation/implicithan/zh_stroke_meta.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_stroke_meta.toml").as_slice()), + ("collation/implicithan/zh_unihan_meta.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_unihan_meta.toml").as_slice()), + ("collation/implicithan/zh_zhuyin_meta.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_zhuyin_meta.toml").as_slice()), + ("collation/implicithan/zh_pinyin_reord.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_pinyin_reord.toml").as_slice()), + ("collation/implicithan/zh_stroke_reord.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_stroke_reord.toml").as_slice()), + ("collation/implicithan/zh_unihan_reord.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_unihan_reord.toml").as_slice()), + ("collation/implicithan/zh_zhuyin_reord.toml", include_bytes!("../../tests/data/icuexport/collation/implicithan/zh_zhuyin_reord.toml").as_slice()), ("norm/small/compositions.toml", include_bytes!("../../tests/data/icuexport/norm/small/compositions.toml").as_slice()), ("norm/small/decompositionex.toml", include_bytes!("../../tests/data/icuexport/norm/small/decompositionex.toml").as_slice()), ("norm/fast/nfd.toml", include_bytes!("../../tests/data/icuexport/norm/fast/nfd.toml").as_slice()), diff --git a/provider/source/tests/data/icuexport/collation/implicithan/zh_pinyin_meta.toml b/provider/source/tests/data/icuexport/collation/implicithan/zh_pinyin_meta.toml new file mode 100644 index 00000000000..5c6c2ba16c9 --- /dev/null +++ b/provider/source/tests/data/icuexport/collation/implicithan/zh_pinyin_meta.toml @@ -0,0 +1,6 @@ +# +# file name: zh_pinyin_meta +# +# machine-generated by: genrb -X + +bits = 0x29 diff --git a/provider/source/tests/data/icuexport/collation/implicithan/zh_pinyin_reord.toml b/provider/source/tests/data/icuexport/collation/implicithan/zh_pinyin_reord.toml new file mode 100644 index 00000000000..8aac7863ef0 --- /dev/null +++ b/provider/source/tests/data/icuexport/collation/implicithan/zh_pinyin_reord.toml @@ -0,0 +1,27 @@ +# +# file name: zh_pinyin_reord +# +# machine-generated by: genrb -X + +min_high_no_reorder = 0xFE000000 +reorder_table = [ + 0,1,2,3,4,5,6,7,8,9,0xa,0xb,0xc,0xd,0xe,0xf, + 0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f, + 0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa, + 0xab,0xac,0xad,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,0xb4,0xb5,0xb6,0xb7,0xb8,0xb9,0xba, + 0xbb,0xbc,0xbd,0xbe,0xbf,0xc0,0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca, + 0xcb,0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda, + 0xdb,0xda,0xdb,0xdc,0xdd,0xde,0xdf,0xe0,0xe1,0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8, + 0xe9,0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8, + 0xf9,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,0x35,0x36, + 0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44,0x45,0x46, + 0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,0x55,0x56, + 0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64,0x65,0x66, + 0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,0x75,0x76, + 0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0x80,0x81,0x82,0x83,0x84,0x85,0x86, + 0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,0x90,0x91,0x92,0x93,0x94,0x95,0x96, + 0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f,0xa0,0xa1,0xa2,0xa3,0xa4,0xfe,0xff +] +reorder_ranges = [ + +] diff --git a/provider/source/tests/data/icuexport/collation/implicithan/zh_stroke_meta.toml b/provider/source/tests/data/icuexport/collation/implicithan/zh_stroke_meta.toml new file mode 100644 index 00000000000..e9355b74eb0 --- /dev/null +++ b/provider/source/tests/data/icuexport/collation/implicithan/zh_stroke_meta.toml @@ -0,0 +1,6 @@ +# +# file name: zh_stroke_meta +# +# machine-generated by: genrb -X + +bits = 0x29 diff --git a/provider/source/tests/data/icuexport/collation/implicithan/zh_stroke_reord.toml b/provider/source/tests/data/icuexport/collation/implicithan/zh_stroke_reord.toml new file mode 100644 index 00000000000..69b4e1c4dd7 --- /dev/null +++ b/provider/source/tests/data/icuexport/collation/implicithan/zh_stroke_reord.toml @@ -0,0 +1,27 @@ +# +# file name: zh_stroke_reord +# +# machine-generated by: genrb -X + +min_high_no_reorder = 0xFE000000 +reorder_table = [ + 0,1,2,3,4,5,6,7,8,9,0xa,0xb,0xc,0xd,0xe,0xf, + 0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f, + 0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0xa6,0xa7,0xa8,0xa9,0xaa,0xab, + 0xac,0xad,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,0xb4,0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb, + 0xbc,0xbd,0xbe,0xbf,0xc0,0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb, + 0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb, + 0xdc,0xdb,0xdc,0xdd,0xde,0xdf,0xe0,0xe1,0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9, + 0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0,0xf9, + 0xfa,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,0x35,0x36, + 0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44,0x45,0x46, + 0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,0x55,0x56, + 0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64,0x65,0x66, + 0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,0x75,0x76, + 0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0x80,0x81,0x82,0x83,0x84,0x85,0x86, + 0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,0x90,0x91,0x92,0x93,0x94,0x95,0x96, + 0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f,0xa0,0xa1,0xa2,0xa3,0xa4,0xfe,0xff +] +reorder_ranges = [ + 0x7e760027,0x8100007a,0xfe00ffa7 +] diff --git a/provider/source/tests/data/icuexport/collation/implicithan/zh_unihan_meta.toml b/provider/source/tests/data/icuexport/collation/implicithan/zh_unihan_meta.toml new file mode 100644 index 00000000000..1f1a742efbb --- /dev/null +++ b/provider/source/tests/data/icuexport/collation/implicithan/zh_unihan_meta.toml @@ -0,0 +1,6 @@ +# +# file name: zh_unihan_meta +# +# machine-generated by: genrb -X + +bits = 0x29 diff --git a/provider/source/tests/data/icuexport/collation/implicithan/zh_unihan_reord.toml b/provider/source/tests/data/icuexport/collation/implicithan/zh_unihan_reord.toml new file mode 100644 index 00000000000..cad4ae9ea30 --- /dev/null +++ b/provider/source/tests/data/icuexport/collation/implicithan/zh_unihan_reord.toml @@ -0,0 +1,27 @@ +# +# file name: zh_unihan_reord +# +# machine-generated by: genrb -X + +min_high_no_reorder = 0xFE000000 +reorder_table = [ + 0,1,2,3,4,5,6,7,8,9,0xa,0xb,0xc,0xd,0xe,0xf, + 0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f, + 0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0xa6,0xa7,0xa8,0xa9,0xaa,0xab, + 0xac,0xad,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,0xb4,0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb, + 0xbc,0xbd,0xbe,0xbf,0xc0,0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb, + 0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb, + 0xdc,0xdb,0xdc,0xdd,0xde,0xdf,0xe0,0xe1,0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9, + 0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0,0xf9, + 0xfa,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,0x35,0x36, + 0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44,0x45,0x46, + 0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,0x55,0x56, + 0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64,0x65,0x66, + 0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,0x75,0x76, + 0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0x80,0x81,0x82,0x83,0x84,0x85,0x86, + 0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,0x90,0x91,0x92,0x93,0x94,0x95,0x96, + 0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f,0xa0,0xa1,0xa2,0xa3,0xa4,0xfe,0xff +] +reorder_ranges = [ + 0x7e760027,0x8100007a,0xfe00ffa7 +] diff --git a/provider/source/tests/data/icuexport/collation/implicithan/zh_zhuyin_meta.toml b/provider/source/tests/data/icuexport/collation/implicithan/zh_zhuyin_meta.toml new file mode 100644 index 00000000000..cc6c5d6d4f0 --- /dev/null +++ b/provider/source/tests/data/icuexport/collation/implicithan/zh_zhuyin_meta.toml @@ -0,0 +1,6 @@ +# +# file name: zh_zhuyin_meta +# +# machine-generated by: genrb -X + +bits = 0x29 diff --git a/provider/source/tests/data/icuexport/collation/implicithan/zh_zhuyin_reord.toml b/provider/source/tests/data/icuexport/collation/implicithan/zh_zhuyin_reord.toml new file mode 100644 index 00000000000..02f54184ddf --- /dev/null +++ b/provider/source/tests/data/icuexport/collation/implicithan/zh_zhuyin_reord.toml @@ -0,0 +1,27 @@ +# +# file name: zh_zhuyin_reord +# +# machine-generated by: genrb -X + +min_high_no_reorder = 0xFE000000 +reorder_table = [ + 0,1,2,3,4,5,6,7,8,9,0xa,0xb,0xc,0xd,0xe,0xf, + 0x10,0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f, + 0x20,0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,0x29,0xa6,0xa7,0xa8,0xa9,0xaa,0xab, + 0xac,0xad,0xae,0xaf,0xb0,0xb1,0xb2,0xb3,0xb4,0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xbb, + 0xbc,0xbd,0xbe,0xbf,0xc0,0xc1,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xcb, + 0xcc,0xcd,0xce,0xcf,0xd0,0xd1,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xdb, + 0xdc,0xdb,0xdc,0xdd,0xde,0xdf,0xe0,0xe1,0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9, + 0xea,0xeb,0xec,0xed,0xee,0xef,0xf0,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0,0xf9, + 0xfa,0x28,0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,0x31,0x32,0x33,0x34,0x35,0x36, + 0x37,0x38,0x39,0x3a,0x3b,0x3c,0x3d,0x3e,0x3f,0x40,0x41,0x42,0x43,0x44,0x45,0x46, + 0x47,0x48,0x49,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50,0x51,0x52,0x53,0x54,0x55,0x56, + 0x57,0x58,0x59,0x5a,0x5b,0x5c,0x5d,0x5e,0x5f,0x60,0x61,0x62,0x63,0x64,0x65,0x66, + 0x67,0x68,0x69,0x6a,0x6b,0x6c,0x6d,0x6e,0x6f,0x70,0x71,0x72,0x73,0x74,0x75,0x76, + 0x77,0x78,0x79,0x7a,0x7b,0x7c,0x7d,0x7e,0x7f,0x80,0x81,0x82,0x83,0x84,0x85,0x86, + 0x87,0x88,0x89,0x8a,0x8b,0x8c,0x8d,0x8e,0x8f,0x90,0x91,0x92,0x93,0x94,0x95,0x96, + 0x97,0x98,0x99,0x9a,0x9b,0x9c,0x9d,0x9e,0x9f,0xa0,0xa1,0xa2,0xa3,0xa4,0xfe,0xff +] +reorder_ranges = [ + 0x7e760027,0x8100007a,0xfe00ffa7 +] diff --git a/tools/make/download-repo-sources/globs.rs.data b/tools/make/download-repo-sources/globs.rs.data index 5a88160cd33..299aee34b43 100644 --- a/tools/make/download-repo-sources/globs.rs.data +++ b/tools/make/download-repo-sources/globs.rs.data @@ -100,6 +100,14 @@ const ICUEXPORTDATA_GLOB: &[&str] = &[ "collation/implicithan/zh_stroke_data.toml", "collation/implicithan/zh_unihan_data.toml", "collation/implicithan/zh_zhuyin_data.toml", + "collation/implicithan/zh_pinyin_meta.toml", + "collation/implicithan/zh_stroke_meta.toml", + "collation/implicithan/zh_unihan_meta.toml", + "collation/implicithan/zh_zhuyin_meta.toml", + "collation/implicithan/zh_pinyin_reord.toml", + "collation/implicithan/zh_stroke_reord.toml", + "collation/implicithan/zh_unihan_reord.toml", + "collation/implicithan/zh_zhuyin_reord.toml", "norm/small/compositions.toml", "norm/small/decompositionex.toml", "norm/fast/nfd.toml", From c7c6c30fd78e3851e96cd11491200aa3fbe97963 Mon Sep 17 00:00:00 2001 From: Robert Bastian <4706271+robertbastian@users.noreply.github.com> Date: Mon, 23 Feb 2026 15:50:56 +0100 Subject: [PATCH 2/4] add more attributes filters to testdata --- components/decimal/src/provider.rs | 4 ++-- .../experimental/src/transliterate/provider.rs | 2 ++ provider/source/src/tests/make_testdata.rs | 18 ++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/components/decimal/src/provider.rs b/components/decimal/src/provider.rs index aba6ec69634..a3941aceceb 100644 --- a/components/decimal/src/provider.rs +++ b/components/decimal/src/provider.rs @@ -157,13 +157,13 @@ icu_provider::data_marker!( ); icu_provider::data_marker!( - /// The digits for a given numbering system. This data ought to be stored in the `und` locale with an auxiliary key + /// The digits for a given numbering system. This data ought to be stored in the `und` locale with a marker attribute /// set to the numbering system code. DecimalDigitsV1, "decimal/digits/v1", [char; 10], #[cfg(feature = "datagen")] - attributes_domain = "numbering_system" + attributes_domain = "numbering-system" ); #[cfg(feature = "datagen")] diff --git a/components/experimental/src/transliterate/provider.rs b/components/experimental/src/transliterate/provider.rs index 681974623da..13717735ed9 100644 --- a/components/experimental/src/transliterate/provider.rs +++ b/components/experimental/src/transliterate/provider.rs @@ -38,6 +38,8 @@ icu_provider::data_marker!( RuleBasedTransliterator<'static>, #[cfg(feature = "datagen")] expose_baked_consts = true, + #[cfg(feature = "datagen")] + attributes_domain = "transliterator", ); /// The data struct representing [UTS #35 transform rules](https://unicode.org/reports/tr35/tr35-general.html#Transforms). diff --git a/provider/source/src/tests/make_testdata.rs b/provider/source/src/tests/make_testdata.rs index 21ecda74ee5..10599c2052b 100644 --- a/provider/source/src/tests/make_testdata.rs +++ b/provider/source/src/tests/make_testdata.rs @@ -85,6 +85,24 @@ fn make_testdata() { "001" | "419" | "FR" | "JP" | "CG" | "HK" | "ZA" ) }) + .with_marker_attributes_filter("numbering_system", |attrs| { + matches!(attrs.as_str(), "arab" | "beng" | "cakm" | "latn" | "thai") + }) + .with_marker_attributes_filter("transliterator", |attrs| { + matches!( + attrs.as_str(), + "de-t-de-d0-ascii" + | "el-latn-t-s0-ascii" + | "el-latn-t-el-m0-bgn" + | "und-arab-t-und-beng" + | "und-latn-t-s0-ascii" + | "und-t-d0-publish" + | "und-t-s0-publish" + | "und-t-und-latn-d0-ascii" + | "und-x-bengali-interind" + | "und-x-interind-arabic" + ) + }) .export(&provider, exporter) .unwrap(); } From 8ecf73ddbca42b5e16511f2ebd456e733e450e2a Mon Sep 17 00:00:00 2001 From: Robert Bastian <4706271+robertbastian@users.noreply.github.com> Date: Mon, 23 Feb 2026 19:01:25 +0100 Subject: [PATCH 3/4] - --- provider/source/src/tests/make_testdata.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/provider/source/src/tests/make_testdata.rs b/provider/source/src/tests/make_testdata.rs index 10599c2052b..236faca4199 100644 --- a/provider/source/src/tests/make_testdata.rs +++ b/provider/source/src/tests/make_testdata.rs @@ -85,7 +85,7 @@ fn make_testdata() { "001" | "419" | "FR" | "JP" | "CG" | "HK" | "ZA" ) }) - .with_marker_attributes_filter("numbering_system", |attrs| { + .with_marker_attributes_filter("numbering-system", |attrs| { matches!(attrs.as_str(), "arab" | "beng" | "cakm" | "latn" | "thai") }) .with_marker_attributes_filter("transliterator", |attrs| { From f1c812f108420e4818808147a917ce9979197ba8 Mon Sep 17 00:00:00 2001 From: Robert Bastian <4706271+robertbastian@users.noreply.github.com> Date: Mon, 23 Feb 2026 22:57:24 +0100 Subject: [PATCH 4/4] _ --- components/decimal/src/provider.rs | 2 +- provider/source/src/tests/make_testdata.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/decimal/src/provider.rs b/components/decimal/src/provider.rs index a3941aceceb..9d9c9daf6d5 100644 --- a/components/decimal/src/provider.rs +++ b/components/decimal/src/provider.rs @@ -163,7 +163,7 @@ icu_provider::data_marker!( "decimal/digits/v1", [char; 10], #[cfg(feature = "datagen")] - attributes_domain = "numbering-system" + attributes_domain = "numbering_system" ); #[cfg(feature = "datagen")] diff --git a/provider/source/src/tests/make_testdata.rs b/provider/source/src/tests/make_testdata.rs index 236faca4199..10599c2052b 100644 --- a/provider/source/src/tests/make_testdata.rs +++ b/provider/source/src/tests/make_testdata.rs @@ -85,7 +85,7 @@ fn make_testdata() { "001" | "419" | "FR" | "JP" | "CG" | "HK" | "ZA" ) }) - .with_marker_attributes_filter("numbering-system", |attrs| { + .with_marker_attributes_filter("numbering_system", |attrs| { matches!(attrs.as_str(), "arab" | "beng" | "cakm" | "latn" | "thai") }) .with_marker_attributes_filter("transliterator", |attrs| {