Merge pull request #95 from LaurenzV/3.3.2

3.3.2

Author: RazrFalcon

scripts/gen-shaping-tests.py

@@ -38,15 +38,6 @@
     'morx_36_001',
     # ttf-parser uses different rounding, not a bug
     'fallback_positioning_001',
-
-    # text-rendering-tests tests
-    # Unknown issue. Investigate.
-    'cmap_1_004',
-    'shknda_3_031',
-    'shlana_10_028',
-    'shlana_10_041',
-    'shlana_5_010',
-    'shlana_5_012',
 ]
 
 

scripts/gen-tag-table.py

@@ -319,6 +319,10 @@ class OpenTypeRegistryParser(HTMLParser):
         from_bcp_47(DefaultDict[str, AbstractSet[str]]): ``to_bcp_47``
             inverted. Its values start as unsorted sets;
             ``sort_languages`` converts them to sorted lists.
+        from_bcp_47_uninherited (Optional[Dict[str, AbstractSet[str]]]):
+            A copy of ``from_bcp_47``. It starts as ``None`` and is
+            populated at the beginning of the first call to
+            ``inherit_from_macrolanguages``.
 
     """
     def __init__(self):
@@ -328,6 +332,7 @@ def __init__(self):
         self.ranks = collections.defaultdict(int)
         self.to_bcp_47 = collections.defaultdict(set)
         self.from_bcp_47 = collections.defaultdict(set)
+        self.from_bcp_47_uninherited = None
         # Whether the parser is in a <td> element
         self._td = False
         # Whether the parser is after a <br> element within the current <tr> element
@@ -448,34 +453,56 @@ def inherit_from_macrolanguages(self):
 
         If a BCP 47 tag for an individual mapping has no OpenType
         mapping but its macrolanguage does, the mapping is copied to
-        the individual language. For example, als(Tosk Albanian) has no
-        explicit mapping, so it inherits from sq(Albanian) the mapping
+        the individual language. For example, als (Tosk Albanian) has no
+        explicit mapping, so it inherits from sq (Albanian) the mapping
         to SQI.
 
+        However, if an OpenType tag maps to a BCP 47 macrolanguage and
+        some but not all of its individual languages, the mapping is not
+        inherited from the macrolanguage to the missing individual
+        languages. For example, INUK (Nunavik Inuktitut) is mapped to
+        ike (Eastern Canadian Inuktitut) and iu (Inuktitut) but not to
+        ikt (Inuinnaqtun, which is an individual language of iu), so
+        this method does not add a mapping from ikt to INUK.
+
+
         If a BCP 47 tag for a macrolanguage has no OpenType mapping but
-        all of its individual languages do and they all map to the same
-        tags, the mapping is copied to the macrolanguage.
+        some of its individual languages do, their mappings are copied
+        to the macrolanguage.
         """
         global bcp_47
-        original_ot_from_bcp_47 = dict(self.from_bcp_47)
+        first_time = self.from_bcp_47_uninherited is None
+        if first_time:
+            self.from_bcp_47_uninherited = dict(self.from_bcp_47)
         for macrolanguage, languages in dict(bcp_47.macrolanguages).items():
-            ot_macrolanguages = set(original_ot_from_bcp_47.get(macrolanguage, set()))
+            ot_macrolanguages = {
+                ot_macrolanguage for ot_macrolanguage in self.from_bcp_47_uninherited.get(macrolanguage, set ())
+            }
+            blocked_ot_macrolanguages = set()
+            if 'retired code' not in bcp_47.scopes.get(macrolanguage, ''):
+                for ot_macrolanguage in ot_macrolanguages:
+                    round_trip_macrolanguages = {
+                        l for l in self.to_bcp_47[ot_macrolanguage]
+                        if 'retired code' not in bcp_47.scopes.get(l, '')
+                    }
+                    round_trip_languages = {
+                        l for l in languages
+                        if 'retired code' not in bcp_47.scopes.get(l, '')
+                    }
+                    intersection = round_trip_macrolanguages & round_trip_languages
+                    if intersection and intersection != round_trip_languages:
+                        blocked_ot_macrolanguages.add(ot_macrolanguage)
             if ot_macrolanguages:
                 for ot_macrolanguage in ot_macrolanguages:
-                    for language in languages:
-                        self.add_language(language, ot_macrolanguage)
-                        self.ranks[ot_macrolanguage] += 1
-            else:
+                    if ot_macrolanguage not in blocked_ot_macrolanguages:
+                        for language in languages:
+                            self.add_language(language, ot_macrolanguage)
+                            if not blocked_ot_macrolanguages:
+                                self.ranks[ot_macrolanguage] += 1
+            elif first_time:
                 for language in languages:
-                    if language in original_ot_from_bcp_47:
-                        if ot_macrolanguages:
-                            ml = original_ot_from_bcp_47[language]
-                            if ml:
-                                ot_macrolanguages &= ml
-                            else:
-                                pass
-                        else:
-                            ot_macrolanguages |= original_ot_from_bcp_47[language]
+                    if language in self.from_bcp_47_uninherited:
+                        ot_macrolanguages |= self.from_bcp_47_uninherited[language]
                     else:
                         ot_macrolanguages.clear()
                     if not ot_macrolanguages:
@@ -561,7 +588,7 @@ def parse(self, filename):
                         if scope == 'macrolanguage':
                             scope = ' [macrolanguage]'
                         elif scope == 'collection':
-                            scope = ' [family]'
+                            scope = ' [collection]'
                         else:
                             continue
                         self.scopes[subtag] = scope
@@ -710,6 +737,7 @@ def get_name(self, lt):
 
 ot.add_language('oc-provenc', 'PRO')
 
+ot.remove_language_ot('QUZ')
 ot.add_language('qu', 'QUZ')
 ot.add_language('qub', 'QWH')
 ot.add_language('qud', 'QVI')
@@ -742,7 +770,6 @@ def get_name(self, lt):
 ot.add_language('qxt', 'QWH')
 ot.add_language('qxw', 'QWH')
 
-bcp_47.macrolanguages['ro'].remove('mo')
 bcp_47.macrolanguages['ro-MD'].add('mo')
 
 ot.remove_language_ot('SYRE')
@@ -993,6 +1020,8 @@ def print_subtag_matches(subtag, new_line):
         continue
 
     for lt, tags in items:
+        if not tags:
+            continue
         if lt.variant in bcp_47.prefixes:
             expect(next(iter(bcp_47.prefixes[lt.variant])) == lt.language,
                    '%s is not a valid prefix of %s' %(lt.language, lt.variant))
@@ -1022,6 +1051,8 @@ def print_subtag_matches(subtag, new_line):
 
     print("        b'%s' => {" % initial)
     for lt, tags in items:
+        if not tags:
+            continue
         print('            if ', end='')
         script = lt.script
         region = lt.region

scripts/gen-vowel-constraints.py

@@ -185,6 +185,6 @@ def __str__(self, index=0, depth=4):
 
 print('        _ => {}')
 print('    }')
-print('    buffer.swap_buffers();')
+print('    buffer.sync();')
 print('}')
 print()

scripts/ms-use/IndicPositionalCategory-Additional.txt

@@ -2,7 +2,7 @@
 # Not derivable
 # Initial version based on Unicode 7.0 by Andrew Glass 2014-03-17
 # Updated for Unicode 10.0 by Andrew Glass 2017-07-25
-# Ammended for Unicode 10.0 by Andrew Glass 2018-09-21
+# Amended for Unicode 10.0 by Andrew Glass 2018-09-21
 # Updated for L2/19-083    by Andrew Glass 2019-05-06
 # Updated for Unicode 12.1 by Andrew Glass 2019-05-30
 # Updated for Unicode 13.0 by Andrew Glass 2020-07-28
@@ -58,16 +58,16 @@ AA35          ; Top     # Mn       CHAM CONSONANT SIGN
 # Indic_Positional_Category=Bottom
 0859..085B    ; Bottom # Mn   [3] MANDAIC AFFRICATION MARK..MANDAIC GEMINATION MARK
 18A9          ; Bottom # Mn       MONGOLIAN LETTER ALI GALI DAGALGA
-10AE5         ; Bottom # Mn       MANICHAEAN ABBREVIATION MARK ABOVE  # Overriden, ccc controls order
+10AE5         ; Bottom # Mn       MANICHAEAN ABBREVIATION MARK ABOVE  # Overridden, ccc controls order
 10AE6         ; Bottom # Mn       MANICHAEAN ABBREVIATION MARK BELOW
 10F46..10F47  ; Bottom # Mn   [2] SOGDIAN COMBINING DOT BELOW..SOGDIAN COMBINING TWO DOTS BELOW
-10F48..10F4A  ; Bottom # Mn   [3] SOGDIAN COMBINING DOT ABOVE..SOGDIAN COMBINING CURVE ABOVE     # Overriden, ccc controls order
+10F48..10F4A  ; Bottom # Mn   [3] SOGDIAN COMBINING DOT ABOVE..SOGDIAN COMBINING CURVE ABOVE     # Overridden, ccc controls order
 10F4B         ; Bottom # Mn       SOGDIAN COMBINING CURVE BELOW
-10F4C         ; Bottom # Mn       SOGDIAN COMBINING HOOK ABOVE        # Overriden, ccc controls order
+10F4C         ; Bottom # Mn       SOGDIAN COMBINING HOOK ABOVE        # Overridden, ccc controls order
 10F4D..10F50  ; Bottom # Mn   [4] SOGDIAN COMBINING HOOK BELOW..SOGDIAN COMBINING STROKE BELOW
-10F82         ; Bottom # Mn       OLD UYGHUR COMBINING DOT ABOVE      # Overriden, ccc controls order
+10F82         ; Bottom # Mn       OLD UYGHUR COMBINING DOT ABOVE      # Overridden, ccc controls order
 10F83         ; Bottom # Mn       OLD UYGHUR COMBINING DOT BELOW
-10F84         ; Bottom # Mn       OLD UYGHUR COMBINING TWO DOTS ABOVE # Overriden, ccc controls order
+10F84         ; Bottom # Mn       OLD UYGHUR COMBINING TWO DOTS ABOVE # Overridden, ccc controls order
 10F85         ; Bottom # Mn       OLD UYGHUR COMBINING TWO DOTS BELOW
 16F4F         ; Bottom # Mn       MIAO SIGN CONSONANT MODIFIER BAR
 16F51..16F87  ; Bottom # Mc  [55] MIAO SIGN ASPIRATION..MIAO VOWEL SIGN UI

src/aat/extended_kerning.rs

@@ -92,6 +92,8 @@ pub(crate) fn apply(plan: &ShapePlan, face: &Face, buffer: &mut Buffer) -> Optio
                     continue;
                 }
 
+                buffer.unsafe_to_concat(None, None);
+
                 apply_simple_kerning(&subtable, plan, face, buffer);
             }
             kerx::Format::Format4(ref sub) => {
@@ -140,7 +142,10 @@ fn apply_simple_kerning(
         }
 
         let mut iter = SkippyIter::new(&ctx, i, 1, false);
-        if !iter.next() {
+
+        let mut unsafe_to = 0;
+        if !iter.next(Some(&mut unsafe_to)) {
+            ctx.buffer.unsafe_to_concat(Some(i), Some(unsafe_to));
             i += 1;
             continue;
         }
@@ -179,7 +184,7 @@ fn apply_simple_kerning(
                 }
             }
 
-            ctx.buffer.unsafe_to_break(i, j + 1)
+            ctx.buffer.unsafe_to_break(Some(i), Some(j + 1))
         }
 
         i = j;
@@ -235,7 +240,10 @@ fn apply_state_machine_kerning<T, E>(
             // If there's no value and we're just epsilon-transitioning to state 0, safe to break.
             if entry.is_actionable() || !(entry.new_state == START_OF_TEXT && !entry.has_advance())
             {
-                buffer.unsafe_to_break_from_outbuffer(buffer.backtrack_len() - 1, buffer.idx + 1);
+                buffer.unsafe_to_break_from_outbuffer(
+                    Some(buffer.backtrack_len() - 1),
+                    Some(buffer.idx + 1),
+                );
             }
         }
 
@@ -249,7 +257,7 @@ fn apply_state_machine_kerning<T, E>(
             };
 
             if end_entry.is_actionable() {
-                buffer.unsafe_to_break(buffer.idx, buffer.idx + 2);
+                buffer.unsafe_to_break(Some(buffer.idx), Some(buffer.idx + 2));
             }
         }
 

src/aat/metamorphosis.rs

@@ -210,7 +210,10 @@ fn drive<T: FromData>(
         };
 
         if !is_safe_to_break() && buffer.backtrack_len() > 0 && buffer.idx < buffer.len {
-            buffer.unsafe_to_break_from_outbuffer(buffer.backtrack_len() - 1, buffer.idx + 1);
+            buffer.unsafe_to_break_from_outbuffer(
+                Some(buffer.backtrack_len() - 1),
+                Some(buffer.idx + 1),
+            );
         }
 
         c.transition(&entry, buffer);
@@ -232,7 +235,7 @@ fn drive<T: FromData>(
     }
 
     if !c.in_place() {
-        buffer.swap_buffers();
+        buffer.sync();
     }
 }
 
@@ -459,7 +462,7 @@ impl Driver<morx::ContextualEntryData> for ContextualCtx<'_> {
         }
 
         if let Some(replacement) = replacement {
-            buffer.unsafe_to_break(self.mark, (buffer.idx + 1).min(buffer.len));
+            buffer.unsafe_to_break(Some(self.mark), Some((buffer.idx + 1).min(buffer.len)));
             buffer.info[self.mark].glyph_id = u32::from(replacement);
 
             if let Some(face) = self.face_if_has_glyph_classes {
@@ -565,8 +568,8 @@ impl Driver<morx::InsertionEntryData> for InsertionCtx<'_> {
             buffer.move_to(end + usize::from(count));
 
             buffer.unsafe_to_break_from_outbuffer(
-                self.mark as usize,
-                (buffer.idx + 1).min(buffer.len),
+                Some(self.mark as usize),
+                Some((buffer.idx + 1).min(buffer.len)),
             );
         }