Skip to content

Switch to new moz-l10n based parsers #1050

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
flodolo merged 2 commits into from
Sep 25, 2025
Merged

Switch to new moz-l10n based parsers #1050

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c3e6f12
Switch to new moz-l10n based parsers
flodolo Sep 25, 2025
578ffbb
Update actions
flodolo Sep 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/tests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ jobs:
- '8.4'
steps:
- name: Check out repository
uses: actions/checkout@v4
uses: actions/checkout@v5

- name: Set up Node
uses: actions/setup-node@v4
uses: actions/setup-node@v5
with:
node-version: '16'

Expand Down
4 changes: 2 additions & 2 deletions app/scripts/glossaire.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ function updateMultirepo() {
local locale="$2"
echogreen "Create cache for $path/$locale"
mkdir -p "${root}TMX/${locale}/"
nice -20 python $install/app/scripts/tmx/tmx_products.py --path $path/$locale/ --locale $locale --ref en-US --repo $repo_name
nice -20 python $install/app/scripts/tmx/tmx_repository.py --path $path/$locale/ --locale $locale --ref en-US --repo $repo_name
}

local repo_name="$1"
Expand Down Expand Up @@ -166,7 +166,7 @@ function updateSeamonkey() {
# $1: Locale code
echogreen "Create cache for $repo_name/$1"
mkdir -p "${root}TMX/${locale}/"
nice -20 python $install/app/scripts/tmx/tmx_products.py --path $repo_folder/$1/ --locale $1 --ref en-US --repo $repo_name
nice -20 python $install/app/scripts/tmx/tmx_repository.py --path $repo_folder/$1/ --locale $1 --ref en-US --repo $repo_name
}

local repo_name="seamonkey"
Expand Down
93 changes: 93 additions & 0 deletions app/scripts/tmx/functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
from configparser import ConfigParser
import argparse
import os


def get_config() -> str:
# Get absolute path of ../../config from the current script location (not the
# current folder)
config_folder = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "config")
)
# Read Transvision's configuration file from ../../config/config.ini
# If not available use a default storage folder to store data
config_file = os.path.join(config_folder, "config.ini")
if not os.path.isfile(config_file):
print(
"Configuration file /app/config/config.ini is missing. "
"Default settings will be used."
)
root_folder = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir)
)
storage_path = os.path.join(root_folder, "TMX")
os.makedirs(storage_path, exist_ok=True)
else:
config_parser = ConfigParser()
config_parser.read(config_file)
storage_path = os.path.join(config_parser.get("config", "root"), "TMX")

return storage_path


def get_cli_parameters(config: bool = False) -> argparse.Namespace:
# Read command line input parameters
parser = argparse.ArgumentParser()

if config:
parser.add_argument("toml_path", help="Path to root l10n.toml file")
parser.add_argument(
"--android",
dest="android_project",
action="store_true",
help="If passed, the script will parse the config file using Android locale codes",
default=False,
)
else:
parser.add_argument(
"--path",
dest="repo_path",
help="Path to locale files",
required=True,
)
parser.add_argument(
"--locale",
dest="locale_code",
help="Locale code",
required=True,
)

# Common parameters
parser.add_argument(
"--ref",
dest="reference_code",
help="Reference locale code",
required=True,
)
parser.add_argument(
"--repo", dest="repository_name", help="Repository name", required=True
)
parser.add_argument(
"--append",
dest="append_mode",
action="store_true",
help="If set to 'append', translations will be added to an existing cache file",
)
parser.add_argument(
"--prefix",
dest="storage_prefix",
nargs="?",
help="This prefix will be prependended to the identified "
"path in string IDs (e.g. extensions/irc for Chatzilla)",
default="",
)
parser.add_argument(
"--output",
nargs="?",
type=str,
choices=["json", "php"],
help="Store only one type of output.",
default="",
)

return parser.parse_args()
157 changes: 58 additions & 99 deletions app/scripts/tmx/tmx_projectconfig.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,37 +1,15 @@
#!/usr/bin/env python

from compare_locales import parser
from configparser import ConfigParser
from functions import get_cli_parameters, get_config
from moz.l10n.formats import Format
from moz.l10n.message import serialize_message
from moz.l10n.model import Entry
from moz.l10n.paths import L10nConfigPaths, get_android_locale
import argparse
from moz.l10n.resource import parse_resource
import codecs
import json
import logging
import os

logging.basicConfig()
# Get absolute path of ../../config from the current script location (not the
# current folder)
config_folder = os.path.abspath(
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "config")
)

# Read Transvision's configuration file from ../../config/config.ini
# If not available use a default storage folder to store data
config_file = os.path.join(config_folder, "config.ini")
if not os.path.isfile(config_file):
print(
"Configuration file /app/config/config.ini is missing. "
"Default settings will be used."
)
root_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
storage_path = os.path.join(root_folder, "TMX")
os.makedirs(storage_path, exist_ok=True)
else:
config_parser = ConfigParser()
config_parser.read(config_file)
storage_path = os.path.join(config_parser.get("config", "root"), "TMX")


class StringExtraction:
def __init__(
Expand Down Expand Up @@ -80,10 +58,24 @@ def readExistingJSON(locale):

return translations

def getEntryValue(resource, value):
entry_value = serialize_message(resource.format, value)
if resource.format == Format.android:
# In Android resources, unescape quotes
entry_value = entry_value.replace('\\"', '"').replace("\\'", "'")

return entry_value

def readFiles(locale):
"""Read files for locale"""

if locale != self.reference_locale:
is_ref_locale = locale == self.reference_locale
if is_ref_locale:
locale_files = [
(os.path.abspath(ref_path), os.path.abspath(ref_path))
for ref_path in project_config_paths.ref_paths
]
else:
locale_files = [
(
os.path.abspath(ref_path),
Expand All @@ -100,11 +92,6 @@ def readFiles(locale):
)
)
]
else:
locale_files = [
(os.path.abspath(ref_path), os.path.abspath(ref_path))
for ref_path in project_config_paths.ref_paths
]

for reference_file, l10n_file in locale_files:
if not os.path.exists(l10n_file):
Expand All @@ -119,32 +106,45 @@ def readFiles(locale):
# Prepend storage_prefix if defined
if self.storage_prefix != "":
key_path = f"{self.storage_prefix}/{key_path}"
try:
p = parser.getParser(reference_file)
except UserWarning:
continue

p.readFile(l10n_file)
if isinstance(p, parser.android.AndroidParser):
# As of https://github.com/mozilla/pontoon/pull/3611, Pontoon
# uses moz.l10n for resource parsing, resulting in quotes being
# escaped. compare-locales doesn't escape them, so need to
# manually remove escapes.
self.translations[locale].update(
(
f"{self.repository_name}/{key_path}:{entity.key}",
entity.raw_val.replace("\\'", "'").replace('\\"', '"'),
try:
if is_ref_locale:
resource = parse_resource(
reference_file, android_literal_quotes=True
)
for entity in p.parse()
)
else:
self.translations[locale].update(
(
f"{self.repository_name}/{key_path}:{entity.key}",
entity.raw_val,
else:
resource = parse_resource(
l10n_file, android_literal_quotes=True
)
for entity in p.parse()
)
for section in resource.sections:
for entry in section.entries:
if isinstance(entry, Entry):
entry_id = ".".join(section.id + entry.id)
string_id = (
f"{self.repository_name}/{key_path}:{entry_id}"
)
if entry.properties:
# Store the value of an entry with attributes only
# if the value is not empty.
if not entry.value.is_empty():
self.translations[locale][string_id] = (
getEntryValue(resource, entry.value)
)
for (
attribute,
attr_value,
) in entry.properties.items():
attr_id = f"{string_id}.{attribute}"
self.translations[locale][attr_id] = (
getEntryValue(resource, attr_value)
)
else:
self.translations[locale][string_id] = (
getEntryValue(resource, entry.value)
)
except Exception as e:
print(f"Error parsing file: {reference_file}")
print(e)

basedir = os.path.dirname(self.toml_path)
if self.android_project:
Expand Down Expand Up @@ -234,49 +234,8 @@ def escape(self, translation):


def main():
# Read command line input parameters
parser = argparse.ArgumentParser()
parser.add_argument("toml_path", help="Path to root l10n.toml file")
parser.add_argument(
"--ref",
dest="reference_code",
help="Reference language code",
required=True,
)
parser.add_argument(
"--repo", dest="repository_name", help="Repository name", required=True
)
parser.add_argument(
"--append",
dest="append_mode",
action="store_true",
help="If set to 'append', translations will be added to an existing cache file",
)
parser.add_argument(
"--android",
dest="android_project",
action="store_true",
help="If passed, the script will parse the config file using Android locale codes",
default=False,
)
parser.add_argument(
"--prefix",
dest="storage_prefix",
nargs="?",
help="This prefix will be prependended to the identified "
"path in string IDs (e.g. extensions/irc for Chatzilla)",
default="",
)
parser.add_argument(
"--output",
nargs="?",
type=str,
choices=["json", "php"],
help="Store only one type of output.",
default="",
)
args = parser.parse_args()

args = get_cli_parameters(config=True)
storage_path = get_config()
extracted_strings = StringExtraction(
args.toml_path,
storage_path,
Expand Down
Loading