Use MF2 syntax for xcode & xliff strings

Author: eemeli

pontoon/base/migrations/0100_android_as_mf2.py

@@ -3,10 +3,11 @@
 
 from moz.l10n.formats.android import android_parse_message, android_serialize_message
 from moz.l10n.formats.mf2 import mf2_parse_message, mf2_serialize_message
+from moz.l10n.model import PatternMessage
 
 from django.db import migrations
 
-from pontoon.base.simple_preview import android_simple_preview
+from pontoon.base.simple_preview import preview_placeholder
 
 
 android_nl = compile(r"\s*\n\s*")
@@ -32,6 +33,10 @@ def mf2_string_changed(obj):
     return True
 
 
+def android_simple_preview(msg: PatternMessage) -> str:
+    return "".join(preview_placeholder(part) for part in msg.pattern)
+
+
 def mf2_tm_changed(tm):
     changed = False
     src_prev = tm.source

pontoon/base/migrations/0102_xliff_as_mf2.py

@@ -0,0 +1,82 @@
+from html import escape, unescape
+
+from moz.l10n.formats.mf2 import mf2_parse_message, mf2_serialize_message
+from moz.l10n.formats.xliff import xliff_parse_message, xliff_serialize_message
+from moz.l10n.message import message_from_json
+
+from django.db import migrations
+
+
+def mf2_entity_changed(entity):
+    db_source = entity.string
+    msg = message_from_json(entity.value)
+    mf2_source = mf2_serialize_message(msg)
+    if mf2_source == db_source:
+        return False
+    entity.string = mf2_source
+    return True
+
+
+def mf2_translation_changed(translation, is_xcode: bool):
+    db_source = translation.string
+    msg = xliff_parse_message(escape(db_source), is_xcode=is_xcode)
+    mf2_source = mf2_serialize_message(msg)
+    if mf2_source == db_source:
+        return False
+    translation.string = mf2_source
+    return True
+
+
+def xliff_as_mf2(apps, schema_editor):
+    Entity = apps.get_model("base", "Entity")
+    Translation = apps.get_model("base", "Translation")
+
+    entities = Entity.objects.filter(resource__format__in=["xliff", "xcode"])
+    ent_fixed = [e for e in entities if mf2_entity_changed(e)]
+    n = Entity.objects.bulk_update(ent_fixed, ["string"], batch_size=10_000)
+    print(f" ({n} entities)", end="", flush=True)
+
+    translations = Translation.objects.filter(
+        entity__resource__format="xcode"
+    ).select_related("entity")
+    trans_fixed = [t for t in translations if mf2_translation_changed(t, True)]
+    n = Translation.objects.bulk_update(trans_fixed, ["string"], batch_size=10_000)
+    print(f" ({n} xcode translations)", end="", flush=True)
+
+    translations = Translation.objects.filter(
+        entity__resource__format="xliff"
+    ).select_related("entity")
+    trans_fixed = [t for t in translations if mf2_translation_changed(t, False)]
+    n = Translation.objects.bulk_update(trans_fixed, ["string"], batch_size=10_000)
+    print(f" ({n} xliff translations)", end="", flush=True)
+
+
+def xliff_string_changed(obj):
+    mf2_source = obj.string
+    msg = mf2_parse_message(mf2_source)
+    string = unescape(xliff_serialize_message(msg))
+    if string == mf2_source:
+        return False
+    obj.string = string
+    return True
+
+
+def mf2_as_xliff(apps, schema_editor):
+    Entity = apps.get_model("base", "Entity")
+    entities = Entity.objects.filter(resource__format__in=["xliff", "xcode"])
+    ent_fixed = [e for e in entities if xliff_string_changed(e)]
+    n = Entity.objects.bulk_update(ent_fixed, ["meta", "string"], batch_size=10_000)
+    print(f" ({n} entities)", end="", flush=True)
+
+    Translation = apps.get_model("base", "Translation")
+    translations = Translation.objects.filter(
+        entity__resource__format__in=["xliff", "xcode"]
+    )
+    trans_fixed = [t for t in translations if xliff_string_changed(t)]
+    n = Translation.objects.bulk_update(trans_fixed, ["string"], batch_size=10_000)
+    print(f" ({n} translations)", end="", flush=True)
+
+
+class Migration(migrations.Migration):
+    dependencies = [("base", "0101_webext_as_mf2")]
+    operations = [migrations.RunPython(xliff_as_mf2, reverse_code=mf2_as_xliff)]

pontoon/base/simple_preview.py

@@ -1,24 +1,21 @@
 from json import dumps
 
-from moz.l10n.formats import Format
-from moz.l10n.formats.fluent import fluent_parse_entry
+from moz.l10n.formats.fluent import fluent_parse_entry, fluent_serialize_message
 from moz.l10n.formats.mf2 import mf2_parse_message, mf2_serialize_message
-from moz.l10n.message import serialize_message
 from moz.l10n.model import (
     CatchallKey,
     Expression,
     Markup,
     Message,
     Pattern,
     PatternMessage,
-    SelectMessage,
     VariableRef,
 )
 
 from pontoon.base.models import Resource
 
 
-def get_simple_preview(format: str, string: str):
+def get_simple_preview(format: str, msg: str | Message | Pattern) -> str:
     """
     Flatten a message entry as a simple string.
 
@@ -27,58 +24,58 @@ def get_simple_preview(format: str, string: str):
     For Fluent, selects the value if it's not empty,
     or the first non-empty attribute.
     """
-    try:
-        match format:
-            case Resource.Format.FLUENT:
-                entry = fluent_parse_entry(string, with_linepos=False)
-                if not entry.value.is_empty():
-                    msg = entry.value
-                else:
-                    msg = next(
+    if format == Resource.Format.FLUENT:
+        if isinstance(msg, str):
+            try:
+                entry = fluent_parse_entry(msg, with_linepos=False)
+                msg = (
+                    entry.value
+                    if not entry.value.is_empty()
+                    else next(
                         prop
                         for prop in entry.properties.values()
                         if not prop.is_empty()
                     )
-                msg = as_pattern_message(msg)
-                return serialize_message(Format.fluent, msg)
+                )
+            except Exception:
+                return msg
+        pattern = as_simple_pattern(msg)
+        return fluent_serialize_message(PatternMessage(pattern))
 
-            case Resource.Format.ANDROID:
-                msg = mf2_parse_message(string)
-                return android_simple_preview(msg)
-
-            case Resource.Format.GETTEXT | Resource.Format.WEBEXT:
-                msg = mf2_parse_message(string)
-                msg = as_pattern_message(msg)
-                return serialize_message(None, msg)
-    except Exception:
-        pass
-    return string
-
-
-def as_pattern_message(msg: Message) -> PatternMessage:
-    if isinstance(msg, SelectMessage):
-        default_pattern = next(
-            pattern
-            for keys, pattern in msg.variants.items()
-            if all(isinstance(key, CatchallKey) for key in keys)
-        )
-        return PatternMessage(default_pattern)
-    else:
+    if format in (
+        Resource.Format.ANDROID,
+        Resource.Format.GETTEXT,
+        Resource.Format.WEBEXT,
+        Resource.Format.XCODE,
+        Resource.Format.XLIFF,
+    ):
+        if isinstance(msg, str):
+            try:
+                msg = mf2_parse_message(msg)
+            except Exception:
+                return msg
+    elif isinstance(msg, str):
         return msg
 
-
-def android_simple_preview(msg: Message | Pattern) -> str:
-    """
-    Matches the JS androidEditPattern() from translate/src/utils/message/android.ts
-    """
     preview = ""
-    pattern = msg if isinstance(msg, list) else as_pattern_message(msg).pattern
-    for part in pattern:
-        preview += android_placeholder_preview(part)
+    for part in as_simple_pattern(msg):
+        preview += preview_placeholder(part)
     return preview
 
 
-def android_placeholder_preview(part: str | Expression | Markup) -> str:
+def as_simple_pattern(msg: Message | Pattern) -> Pattern:
+    if isinstance(msg, list):
+        return msg
+    if isinstance(msg, PatternMessage):
+        return msg.pattern
+    return next(
+        pattern
+        for keys, pattern in msg.variants.items()
+        if all(isinstance(key, CatchallKey) for key in keys)
+    )
+
+
+def preview_placeholder(part: str | Expression | Markup) -> str:
     if isinstance(part, str):
         return part
     if isinstance(ps := part.attributes.get("source", None), str):
@@ -88,12 +85,12 @@ def android_placeholder_preview(part: str | Expression | Markup) -> str:
             return part.arg
         elif part.function == "entity" and isinstance(part.arg, VariableRef):
             return part.arg.name
-    elif part.kind == "open":
+    elif part.kind in ("open", "standalone"):
         res = "<" + part.name
         for name, val in part.options.items():
             valstr = dumps(val) if isinstance(val, str) else "$" + val.name
             res += f" {name}={valstr}"
-        res += ">"
+        res += ">" if part.kind == "open" else " />"
         return res
     elif part.kind == "close" and not part.options:
         return f"</{part.name}>"

pontoon/checks/libraries/__init__.py

@@ -3,7 +3,7 @@
 from moz.l10n.model import CatchallKey, Pattern, PatternMessage, SelectMessage
 
 from pontoon.base.models import Entity, Resource
-from pontoon.base.simple_preview import android_simple_preview
+from pontoon.base.simple_preview import get_simple_preview
 
 from . import compare_locales, translate_toolkit
 from .custom import run_custom_checks
@@ -75,19 +75,21 @@ def run_checks(
             case Resource.Format.ANDROID:
                 src_msg = mf2_parse_message(entity.string)
                 tgt_msg = mf2_parse_message(string)
-                src0 = android_simple_preview(src_msg)
+                src0 = get_simple_preview(res_format, src_msg)
                 if isinstance(src_msg, SelectMessage) and isinstance(
                     tgt_msg, SelectMessage
                 ):
                     for keys, pattern in tgt_msg.variants.items():
                         src = (
-                            android_simple_preview(src_msg.variants[keys])
+                            get_simple_preview(res_format, src_msg.variants[keys])
                             if keys == ("one",) and keys in src_msg.variants
                             else src0
                         )
-                        tt_patterns.append((src, android_simple_preview(pattern)))
+                        tt_patterns.append(
+                            (src, get_simple_preview(res_format, pattern))
+                        )
                 else:
-                    tt_patterns.append((src0, android_simple_preview(tgt_msg)))
+                    tt_patterns.append((src0, get_simple_preview(res_format, tgt_msg)))
 
             case Resource.Format.GETTEXT:
                 src_msg = mf2_parse_message(entity.string)

pontoon/checks/libraries/custom.py

@@ -17,10 +17,7 @@
 )
 
 from pontoon.base.models import Entity, Resource
-from pontoon.base.simple_preview import (
-    android_placeholder_preview,
-    android_simple_preview,
-)
+from pontoon.base.simple_preview import get_simple_preview, preview_placeholder
 
 
 parser = FluentParser()
@@ -81,7 +78,7 @@ def run_custom_checks(entity: Entity, string: str) -> dict[str, list[str]]:
                     for el in pattern
                     if not isinstance(el, str)
                 )
-                orig_ps = {android_placeholder_preview(ph) for ph in orig_ph_iter}
+                orig_ps = {preview_placeholder(ph) for ph in orig_ph_iter}
             except ValueError:
                 orig_msg = None
                 orig_ps = set()
@@ -99,11 +96,11 @@ def run_custom_checks(entity: Entity, string: str) -> dict[str, list[str]]:
             try:
                 for pattern in patterns:
                     android_msg = android_parse_message(
-                        escape(android_simple_preview(pattern))
+                        escape(get_simple_preview(Resource.Format.ANDROID, pattern))
                     )
                     for el in android_msg.pattern:
                         if not isinstance(el, str):
-                            ps = android_placeholder_preview(el)
+                            ps = preview_placeholder(el)
                             if ps in orig_ps:
                                 found_ps.add(ps)
                             else:

pontoon/pretranslation/pretranslate.py

@@ -62,6 +62,8 @@ def get_pretranslation(
             Resource.Format.ANDROID,
             Resource.Format.GETTEXT,
             Resource.Format.WEBEXT,
+            Resource.Format.XCODE,
+            Resource.Format.XLIFF,
         }:
             format = Format.mf2
             msg = parse_message(format, entity.string)
@@ -90,6 +92,8 @@ def __init__(self, entity: Entity, locale: Locale, preserve_placeables: bool):
                 Resource.Format.ANDROID
                 | Resource.Format.GETTEXT
                 | Resource.Format.WEBEXT
+                | Resource.Format.XCODE
+                | Resource.Format.XLIFF
             ):
                 self.format = Format.mf2
             case _:

pontoon/sync/core/translations_to_repo.py

@@ -347,7 +347,7 @@ def set_translation(
             return False
 
     match format:
-        case Format.android | Format.gettext | Format.webext:
+        case Format.android | Format.gettext | Format.webext | Format.xliff:
             msg = parse_message(Format.mf2, tx.string)
             if isinstance(entry.value, SelectMessage):
                 entry.value.variants = (