Skip to content

Commit f2d5102

Browse files
flodoloflodolo
authored
Switch to new moz-l10n based parsers (mozfr#1050)
1 parent b732281 commit f2d5102

File tree

6 files changed

+184
-215
lines changed

6 files changed

+184
-215
lines changed

‎.github/workflows/tests.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,10 @@ jobs:
2222
- '8.4'
2323
steps:
2424
- name: Check out repository
25-
uses: actions/checkout@v4
25+
uses: actions/checkout@v5
2626

2727
- name: Set up Node
28-
uses: actions/setup-node@v4
28+
uses: actions/setup-node@v5
2929
with:
3030
node-version: '16'
3131

‎app/scripts/glossaire.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ function updateMultirepo() {
120120
local locale="$2"
121121
echogreen "Create cache for $path/$locale"
122122
mkdir -p "${root}TMX/${locale}/"
123-
nice -20 python $install/app/scripts/tmx/tmx_products.py --path $path/$locale/ --locale $locale --ref en-US --repo $repo_name
123+
nice -20 python $install/app/scripts/tmx/tmx_repository.py --path $path/$locale/ --locale $locale --ref en-US --repo $repo_name
124124
}
125125

126126
local repo_name="$1"
@@ -166,7 +166,7 @@ function updateSeamonkey() {
166166
# $1: Locale code
167167
echogreen "Create cache for $repo_name/$1"
168168
mkdir -p "${root}TMX/${locale}/"
169-
nice -20 python $install/app/scripts/tmx/tmx_products.py --path $repo_folder/$1/ --locale $1 --ref en-US --repo $repo_name
169+
nice -20 python $install/app/scripts/tmx/tmx_repository.py --path $repo_folder/$1/ --locale $1 --ref en-US --repo $repo_name
170170
}
171171

172172
local repo_name="seamonkey"

‎app/scripts/tmx/functions.py‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
from configparser import ConfigParser
2+
import argparse
3+
import os
4+
5+
6+
def get_config() -> str:
7+
# Get absolute path of ../../config from the current script location (not the
8+
# current folder)
9+
config_folder = os.path.abspath(
10+
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "config")
11+
)
12+
# Read Transvision's configuration file from ../../config/config.ini
13+
# If not available use a default storage folder to store data
14+
config_file = os.path.join(config_folder, "config.ini")
15+
if not os.path.isfile(config_file):
16+
print(
17+
"Configuration file /app/config/config.ini is missing. "
18+
"Default settings will be used."
19+
)
20+
root_folder = os.path.abspath(
21+
os.path.join(os.path.dirname(__file__), os.pardir)
22+
)
23+
storage_path = os.path.join(root_folder, "TMX")
24+
os.makedirs(storage_path, exist_ok=True)
25+
else:
26+
config_parser = ConfigParser()
27+
config_parser.read(config_file)
28+
storage_path = os.path.join(config_parser.get("config", "root"), "TMX")
29+
30+
return storage_path
31+
32+
33+
def get_cli_parameters(config: bool = False) -> argparse.Namespace:
34+
# Read command line input parameters
35+
parser = argparse.ArgumentParser()
36+
37+
if config:
38+
parser.add_argument("toml_path", help="Path to root l10n.toml file")
39+
parser.add_argument(
40+
"--android",
41+
dest="android_project",
42+
action="store_true",
43+
help="If passed, the script will parse the config file using Android locale codes",
44+
default=False,
45+
)
46+
else:
47+
parser.add_argument(
48+
"--path",
49+
dest="repo_path",
50+
help="Path to locale files",
51+
required=True,
52+
)
53+
parser.add_argument(
54+
"--locale",
55+
dest="locale_code",
56+
help="Locale code",
57+
required=True,
58+
)
59+
60+
# Common parameters
61+
parser.add_argument(
62+
"--ref",
63+
dest="reference_code",
64+
help="Reference locale code",
65+
required=True,
66+
)
67+
parser.add_argument(
68+
"--repo", dest="repository_name", help="Repository name", required=True
69+
)
70+
parser.add_argument(
71+
"--append",
72+
dest="append_mode",
73+
action="store_true",
74+
help="If set to 'append', translations will be added to an existing cache file",
75+
)
76+
parser.add_argument(
77+
"--prefix",
78+
dest="storage_prefix",
79+
nargs="?",
80+
help="This prefix will be prependended to the identified "
81+
"path in string IDs (e.g. extensions/irc for Chatzilla)",
82+
default="",
83+
)
84+
parser.add_argument(
85+
"--output",
86+
nargs="?",
87+
type=str,
88+
choices=["json", "php"],
89+
help="Store only one type of output.",
90+
default="",
91+
)
92+
93+
return parser.parse_args()

‎app/scripts/tmx/tmx_projectconfig.py‎

Lines changed: 58 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -1,37 +1,15 @@
11
#!/usr/bin/env python
22

3-
from compare_locales import parser
4-
from configparser import ConfigParser
3+
from functions import get_cli_parameters, get_config
4+
from moz.l10n.formats import Format
5+
from moz.l10n.message import serialize_message
6+
from moz.l10n.model import Entry
57
from moz.l10n.paths import L10nConfigPaths, get_android_locale
6-
import argparse
8+
from moz.l10n.resource import parse_resource
79
import codecs
810
import json
9-
import logging
1011
import os
1112

12-
logging.basicConfig()
13-
# Get absolute path of ../../config from the current script location (not the
14-
# current folder)
15-
config_folder = os.path.abspath(
16-
os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, "config")
17-
)
18-
19-
# Read Transvision's configuration file from ../../config/config.ini
20-
# If not available use a default storage folder to store data
21-
config_file = os.path.join(config_folder, "config.ini")
22-
if not os.path.isfile(config_file):
23-
print(
24-
"Configuration file /app/config/config.ini is missing. "
25-
"Default settings will be used."
26-
)
27-
root_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))
28-
storage_path = os.path.join(root_folder, "TMX")
29-
os.makedirs(storage_path, exist_ok=True)
30-
else:
31-
config_parser = ConfigParser()
32-
config_parser.read(config_file)
33-
storage_path = os.path.join(config_parser.get("config", "root"), "TMX")
34-
3513

3614
class StringExtraction:
3715
def __init__(
@@ -80,10 +58,24 @@ def readExistingJSON(locale):
8058

8159
return translations
8260

61+
def getEntryValue(resource, value):
62+
entry_value = serialize_message(resource.format, value)
63+
if resource.format == Format.android:
64+
# In Android resources, unescape quotes
65+
entry_value = entry_value.replace('\\"', '"').replace("\\'", "'")
66+
67+
return entry_value
68+
8369
def readFiles(locale):
8470
"""Read files for locale"""
8571

86-
if locale != self.reference_locale:
72+
is_ref_locale = locale == self.reference_locale
73+
if is_ref_locale:
74+
locale_files = [
75+
(os.path.abspath(ref_path), os.path.abspath(ref_path))
76+
for ref_path in project_config_paths.ref_paths
77+
]
78+
else:
8779
locale_files = [
8880
(
8981
os.path.abspath(ref_path),
@@ -100,11 +92,6 @@ def readFiles(locale):
10092
)
10193
)
10294
]
103-
else:
104-
locale_files = [
105-
(os.path.abspath(ref_path), os.path.abspath(ref_path))
106-
for ref_path in project_config_paths.ref_paths
107-
]
10895

10996
for reference_file, l10n_file in locale_files:
11097
if not os.path.exists(l10n_file):
@@ -119,32 +106,45 @@ def readFiles(locale):
119106
# Prepend storage_prefix if defined
120107
if self.storage_prefix != "":
121108
key_path = f"{self.storage_prefix}/{key_path}"
122-
try:
123-
p = parser.getParser(reference_file)
124-
except UserWarning:
125-
continue
126109

127-
p.readFile(l10n_file)
128-
if isinstance(p, parser.android.AndroidParser):
129-
# As of https://github.com/mozilla/pontoon/pull/3611, Pontoon
130-
# uses moz.l10n for resource parsing, resulting in quotes being
131-
# escaped. compare-locales doesn't escape them, so need to
132-
# manually remove escapes.
133-
self.translations[locale].update(
134-
(
135-
f"{self.repository_name}/{key_path}:{entity.key}",
136-
entity.raw_val.replace("\\'", "'").replace('\\"', '"'),
110+
try:
111+
if is_ref_locale:
112+
resource = parse_resource(
113+
reference_file, android_literal_quotes=True
137114
)
138-
for entity in p.parse()
139-
)
140-
else:
141-
self.translations[locale].update(
142-
(
143-
f"{self.repository_name}/{key_path}:{entity.key}",
144-
entity.raw_val,
115+
else:
116+
resource = parse_resource(
117+
l10n_file, android_literal_quotes=True
145118
)
146-
for entity in p.parse()
147-
)
119+
for section in resource.sections:
120+
for entry in section.entries:
121+
if isinstance(entry, Entry):
122+
entry_id = ".".join(section.id + entry.id)
123+
string_id = (
124+
f"{self.repository_name}/{key_path}:{entry_id}"
125+
)
126+
if entry.properties:
127+
# Store the value of an entry with attributes only
128+
# if the value is not empty.
129+
if not entry.value.is_empty():
130+
self.translations[locale][string_id] = (
131+
getEntryValue(resource, entry.value)
132+
)
133+
for (
134+
attribute,
135+
attr_value,
136+
) in entry.properties.items():
137+
attr_id = f"{string_id}.{attribute}"
138+
self.translations[locale][attr_id] = (
139+
getEntryValue(resource, attr_value)
140+
)
141+
else:
142+
self.translations[locale][string_id] = (
143+
getEntryValue(resource, entry.value)
144+
)
145+
except Exception as e:
146+
print(f"Error parsing file: {reference_file}")
147+
print(e)
148148

149149
basedir = os.path.dirname(self.toml_path)
150150
if self.android_project:
@@ -234,49 +234,8 @@ def escape(self, translation):
234234

235235

236236
def main():
237-
# Read command line input parameters
238-
parser = argparse.ArgumentParser()
239-
parser.add_argument("toml_path", help="Path to root l10n.toml file")
240-
parser.add_argument(
241-
"--ref",
242-
dest="reference_code",
243-
help="Reference language code",
244-
required=True,
245-
)
246-
parser.add_argument(
247-
"--repo", dest="repository_name", help="Repository name", required=True
248-
)
249-
parser.add_argument(
250-
"--append",
251-
dest="append_mode",
252-
action="store_true",
253-
help="If set to 'append', translations will be added to an existing cache file",
254-
)
255-
parser.add_argument(
256-
"--android",
257-
dest="android_project",
258-
action="store_true",
259-
help="If passed, the script will parse the config file using Android locale codes",
260-
default=False,
261-
)
262-
parser.add_argument(
263-
"--prefix",
264-
dest="storage_prefix",
265-
nargs="?",
266-
help="This prefix will be prependended to the identified "
267-
"path in string IDs (e.g. extensions/irc for Chatzilla)",
268-
default="",
269-
)
270-
parser.add_argument(
271-
"--output",
272-
nargs="?",
273-
type=str,
274-
choices=["json", "php"],
275-
help="Store only one type of output.",
276-
default="",
277-
)
278-
args = parser.parse_args()
279-
237+
args = get_cli_parameters(config=True)
238+
storage_path = get_config()
280239
extracted_strings = StringExtraction(
281240
args.toml_path,
282241
storage_path,

0 commit comments

Comments
 (0)