fix(rust): gate trigram-based shared-dict merging on corpus size (#1505)

Fixes shared dictionary grouping inflation on datasets with many
high-cardinality string columns (e.g. OSM hiking relations with `@id`,
`relation_ids`, `relation_lengths_m`).

`max(exact, trigram) > 0.075` merges columns that share character
trigrams (digits, URLs) despite sharing zero actual values. Through
transitive union-find closure, nearly all string columns get merged into
one giant dictionary.

**Fix**:
After forming groups with the existing aggressive heuristic, each group
whose combined corpus exceeds 100 KB is checked for actual cross-column
value deduplication. Groups with <5% dedup are dissolved.
Tjios means that existing OMT groupings are preserved, since they have
genuine value overlap (multilingual name translations share most values
across `name`, `name:latin`, `name_de`, `name_en`, `name_int`):

| Tile | Before | After | Change |
|---|---:|---:|---:|
| OMT z14 (23K features) | 378.1 KB | 378.1 KB | **+0.0%** |
| OMT z7 | 178.1 KB | 178.1 KB | **+0.0%** |
| Hiking z0 | 180.0 KB | 155.5 KB | **−13.6%** |
| Hiking z2 | 678.9 KB | 555.2 KB | **−18.2%** |
| Hiking z4 (86K features) | 5.2 MB | 4.4 MB | **−14.8%** |
| Hiking z6 | 3.4 MB | 2.9 MB | **−16.0%** |
| Hiking z8 | 543.0 KB | 445.7 KB | **−17.9%** |
| Hiking z10 | 173.6 KB | 138.6 KB | **−20.2%** |

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: 3 people

rust/mlt-core/src/encoder/property/shared_dict.rs

@@ -28,9 +28,18 @@ const MINHASH_PERMUTATIONS: usize = 128;
 /// grouped into a single shared dictionary.
 const MINHASH_SIMILARITY_THRESHOLD: f64 = 0.075;
 
+/// Groups whose sum of per-column unique-corpus bytes exceeds this are validated via [`group_is_beneficial`].
+/// Smaller groups are kept unconditionally.
+const VALIDATE_CORPUS_THRESHOLD: usize = 100_000;
+
+/// Minimum dedup ratio (`1 − union/sum_individual`) for a validated group to be retained.
+const MIN_DEDUP_RATIO: f64 = 0.05;
+
 struct StringProfile<'a> {
     col_idx: usize,
     name: &'a str,
+    /// Sorted, deduplicated values for [`group_is_beneficial`].
+    unique_values: Vec<&'a str>,
     /// `MinHash` over exact string values.
     exact_hashes: Vec<u64>,
     /// `MinHash` over byte trigrams (empty when all strings are shorter than 3 bytes).
@@ -61,7 +70,7 @@ impl TileLayer {
             .iter()
             .enumerate()
             .filter_map(|(col_idx, name)| {
-                let vals: Vec<&str> = self
+                let mut vals: Vec<&str> = self
                     .features()
                     .iter()
                     .filter_map(|f| match f.properties().get(col_idx) {
@@ -72,6 +81,8 @@ impl TileLayer {
                 if vals.is_empty() {
                     return None;
                 }
+                vals.sort_unstable();
+                vals.dedup();
                 let exact_hashes = exact_mh.get_min_hashes(vals.iter().copied());
                 let trigrams: Vec<[u8; 3]> = vals
                     .iter()
@@ -85,6 +96,7 @@ impl TileLayer {
                 Some(StringProfile {
                     col_idx,
                     name,
+                    unique_values: vals,
                     exact_hashes,
                     trigram_hashes,
                 })
@@ -120,6 +132,34 @@ fn minhash_similarity(a: &[u64], b: &[u64]) -> f64 {
     matches as f64 / a.len() as f64
 }
 
+/// Check whether a proposed shared-dictionary group has enough cross-column
+/// value deduplication to justify the combined dictionary.
+///
+/// Groups below [`VALIDATE_CORPUS_THRESHOLD`] are always kept. Larger groups
+/// must achieve at least [`MIN_DEDUP_RATIO`] dedup savings.
+#[allow(clippy::cast_precision_loss)]
+fn group_is_beneficial(group: &[StringProfile<'_>]) -> bool {
+    let sum_individual: usize = group
+        .iter()
+        .map(|p| p.unique_values.iter().map(|s| s.len()).sum::<usize>())
+        .sum();
+
+    if sum_individual <= VALIDATE_CORPUS_THRESHOLD {
+        return true;
+    }
+
+    let mut all_values: Vec<&str> = group
+        .iter()
+        .flat_map(|p| p.unique_values.iter().copied())
+        .collect();
+    all_values.sort_unstable();
+    all_values.dedup();
+    let union_bytes: usize = all_values.iter().map(|s| s.len()).sum();
+
+    let dedup_savings = sum_individual.saturating_sub(union_bytes);
+    dedup_savings as f64 / sum_individual as f64 >= MIN_DEDUP_RATIO
+}
+
 fn cluster_by_similarity(profiles: Vec<StringProfile<'_>>) -> Vec<Vec<StringProfile<'_>>> {
     if profiles.is_empty() {
         return Vec::new();
@@ -146,7 +186,7 @@ fn cluster_by_similarity(profiles: Vec<StringProfile<'_>>) -> Vec<Vec<StringProf
     let mut groups: Vec<Vec<StringProfile<'_>>> = groups_map
         .into_values()
         .filter_map(|mut v| {
-            if v.len() >= 2 {
+            if v.len() >= 2 && group_is_beneficial(&v) {
                 v.sort_unstable_by_key(|p| p.col_idx);
                 Some(v)
             } else {

rust/mlt-core/src/encoder/property/tests.rs

@@ -1295,6 +1295,76 @@ fn mixed_scalars_and_grouped_strings() {
     assert_eq!(res.properties[3].stats.shared_dict(), SharedDictRole::None);
 }
 
+#[test]
+fn high_cardinality_digit_columns_not_grouped() {
+    let n = 5000;
+    let col_a: Vec<PropValue> = (0..n)
+        .map(|i| PropValue::Str(Some(format!("way/{}", 100_000 + i))))
+        .collect();
+    let col_b: Vec<PropValue> = (0..n)
+        .map(|i| PropValue::Str(Some(format!("{},{}", 200_000 + i, 300_000 + i))))
+        .collect();
+    let col_c: Vec<PropValue> = (0..n)
+        .map(|i| {
+            PropValue::Str(Some(format!(
+                "{}:{}|{}:{}",
+                200_000 + i,
+                4 + i,
+                300_000 + i,
+                8 + i
+            )))
+        })
+        .collect();
+
+    let tile = tile_from_cols(&[
+        ("@id", col_a),
+        ("relation_ids", col_b),
+        ("relation_data", col_c),
+    ]);
+    let res = tile.analyze(true).unwrap();
+
+    for (idx, prop) in res.properties.iter().enumerate() {
+        assert_eq!(
+            prop.stats.shared_dict(),
+            SharedDictRole::None,
+            "column {idx} should not be in a shared dict"
+        );
+    }
+}
+
+#[test]
+fn overlapping_name_columns_stay_grouped() {
+    let n = 500;
+    let shared_names: Vec<String> = (0..n).map(|i| format!("Place_{i}")).collect();
+    let col_name: Vec<PropValue> = shared_names
+        .iter()
+        .map(|s| PropValue::Str(Some(s.clone())))
+        .collect();
+    let col_en: Vec<PropValue> = shared_names
+        .iter()
+        .enumerate()
+        .map(|(i, s)| {
+            if i % 10 == 0 {
+                PropValue::Str(Some(format!("{s} (EN)")))
+            } else {
+                PropValue::Str(Some(s.clone()))
+            }
+        })
+        .collect();
+
+    let tile = tile_from_cols(&[("name", col_name), ("name:en", col_en)]);
+    let res = tile.analyze(true).unwrap();
+
+    assert_eq!(
+        res.properties[0].stats.shared_dict(),
+        SharedDictRole::Owner("name".to_string())
+    );
+    assert_eq!(
+        res.properties[1].stats.shared_dict(),
+        SharedDictRole::Member(0)
+    );
+}
+
 #[test]
 fn encode_with_explicit_encoder_works() {
     let props = vec![StagedProperty::u32("id", (1_000u32..2_000).collect())];