From d5bd79da3388c35d01813ce226f9e1b4c7f5eaf1 Mon Sep 17 00:00:00 2001 From: iTrooz Date: Sun, 19 Jul 2026 23:15:54 +0100 Subject: [PATCH 1/8] feat: error when unused lang is used + improve lang detection --- tests/check_langs.py | 62 ++++++++++++++++++++++++++++++++------------ 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/tests/check_langs.py b/tests/check_langs.py index e8ba83c3eec22..624504bd65808 100644 --- a/tests/check_langs.py +++ b/tests/check_langs.py @@ -1,13 +1,25 @@ #!/usr/bin/env python3 +"""Check for unused and non-existent language keys across the codebase. + +- Every string in C/C++ source must have a matching key in an en_US.json file. +- Every key in an en_US.json file must be referenced somewhere in C/C++ source. + +Usage: + python check_langs.py [--unused] + +Exit code 1 on any mismatch.""" import json import re import os import sys +from collections.abc import Generator -SHOW_UNUSED_LANGS = "--unused" in sys.argv +CHECK_UNUSED_LANGS = "--unused" in sys.argv -# use a regex on code to get all "hex.lang.id"_lang occurences -def get_lang_occurrences_in_code(path): + +def find_lang_keys_in_source(path: str) -> Generator[tuple[str, int, str], None, None]: + """Walk all C/C++ files under path and yield (filepath, line_number, key) + for every quoted string matching "hex.".""" for dir, _, files in os.walk(path): for file in files: @@ -18,11 +30,13 @@ def get_lang_occurrences_in_code(path): with open(filepath, encoding="utf8") as file: for line_num, line in enumerate(file): - for m in re.finditer('"([^"]*?)"_lang', line): + for m in re.finditer(r'"(hex\.[a-zA-Z0-9_.]+)"', line): yield (filepath, line_num+1, m.group(1)) -# Get langs in a specific json file -def get_langs(filepath) -> list[str]: + +def load_json_lang_keys(filepath: str | None) -> list[str]: + """Return the list of top-level keys from a lang JSON file, or [] if + the file doesn't exist.""" if filepath == None: return [] elif not os.path.exists(filepath): @@ -38,14 +52,23 @@ def get_langs(filepath) -> list[str]: return existing_langs -def check_langs(code_path, bonus_langs, specific_langs_path): + +def check_plugin_langs(code_path: str, bonus_langs: list[str], specific_langs_path: str | None) -> bool: + """For a single plugin (or main): verify every "hex.*" string found in + source has a matching JSON key, and report keys in the JSON that aren't + referenced in code (when --unused is set).""" print(f"--- Checking langs at {code_path}") - - specific_langs = get_langs(specific_langs_path) + + specific_langs = load_json_lang_keys(specific_langs_path) unused_langs = specific_langs.copy() ret = True - for filepath, line, match in get_lang_occurrences_in_code(code_path): + for lang in specific_langs: + if not lang.startswith("hex."): + ret = False + print(f"Problem: Lang '{lang}' in {specific_langs_path} doesn't start with 'hex.'") + + for filepath, line, match in find_lang_keys_in_source(code_path): try: unused_langs.remove(match) except ValueError: @@ -54,14 +77,19 @@ def check_langs(code_path, bonus_langs, specific_langs_path): if not match in bonus_langs + specific_langs: ret = False print(f"Problem: Lang '{match}' at {filepath}:{line} not found") - - if SHOW_UNUSED_LANGS and len(unused_langs) > 0: + + + if CHECK_UNUSED_LANGS and len(unused_langs) > 0: + ret = False print(f"Unused langs in {specific_langs_path}:") for unused_lang in unused_langs: print(unused_lang) return ret -def check_languages_exist(languages_file_path: str): + +def verify_language_files_exist(languages_file_path: str) -> bool: + """Check that a plugin's languages.json references language files that + actually exist on disk.""" languages_folder = os.path.dirname(languages_file_path) if not os.path.exists(languages_folder): return True @@ -87,10 +115,10 @@ def check_languages_exist(languages_file_path: str): return False -ui_langs = get_langs("./plugins/ui/romfs/lang/en_US.json") +ui_langs = load_json_lang_keys("./plugins/ui/romfs/lang/en_US.json") exit_ok = True -exit_ok &= check_langs("./main", ui_langs, None) +exit_ok &= check_plugin_langs("./main", ui_langs, None) for plugin in os.listdir("./plugins"): if plugin == "ui": continue @@ -98,7 +126,7 @@ def check_languages_exist(languages_file_path: str): path = f"./plugins/{plugin}" if not os.path.isdir(path): continue - exit_ok &= check_langs(path, ui_langs, f"./plugins/{plugin}/romfs/lang/en_US.json") - exit_ok &= check_languages_exist(f"./plugins/{plugin}/romfs/lang/languages.json") + exit_ok &= check_plugin_langs(path, ui_langs, f"./plugins/{plugin}/romfs/lang/en_US.json") + exit_ok &= verify_language_files_exist(f"./plugins/{plugin}/romfs/lang/languages.json") sys.exit(0 if exit_ok else 1) From baa5ab1f1c216e7ab87dc2e088f431110cb98418 Mon Sep 17 00:00:00 2001 From: iTrooz Date: Sun, 19 Jul 2026 23:26:09 +0100 Subject: [PATCH 2/8] feat: enhance language key validation --- tests/check_langs.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/check_langs.py b/tests/check_langs.py index 624504bd65808..bc41d00781999 100644 --- a/tests/check_langs.py +++ b/tests/check_langs.py @@ -49,41 +49,42 @@ def load_json_lang_keys(filepath: str | None) -> list[str]: for key, _ in data.items(): existing_langs.append(key) + + for lang in existing_langs: + if not lang.startswith("hex."): + ret = False + print(f"Problem: Lang '{lang}' doesn't start with 'hex.'") return existing_langs -def check_plugin_langs(code_path: str, bonus_langs: list[str], specific_langs_path: str | None) -> bool: +def check_plugin_langs(code_path: str, common_langs: list[str], specific_langs: list[str]) -> bool: """For a single plugin (or main): verify every "hex.*" string found in source has a matching JSON key, and report keys in the JSON that aren't referenced in code (when --unused is set).""" print(f"--- Checking langs at {code_path}") - specific_langs = load_json_lang_keys(specific_langs_path) + all_langs = common_langs + specific_langs + unused_langs = specific_langs.copy() ret = True - for lang in specific_langs: - if not lang.startswith("hex."): - ret = False - print(f"Problem: Lang '{lang}' in {specific_langs_path} doesn't start with 'hex.'") - + # Check that every "hex.*" string in source has a matching key in the JSON for filepath, line, match in find_lang_keys_in_source(code_path): try: unused_langs.remove(match) except ValueError: pass - if not match in bonus_langs + specific_langs: + if not match in all_langs: ret = False print(f"Problem: Lang '{match}' at {filepath}:{line} not found") - + # Check for unused keys in the JSON file if CHECK_UNUSED_LANGS and len(unused_langs) > 0: ret = False - print(f"Unused langs in {specific_langs_path}:") for unused_lang in unused_langs: - print(unused_lang) + print(f"Problem: Unused lang {unused_lang}") return ret @@ -115,10 +116,10 @@ def verify_language_files_exist(languages_file_path: str) -> bool: return False -ui_langs = load_json_lang_keys("./plugins/ui/romfs/lang/en_US.json") +common_langs = load_json_lang_keys("./plugins/ui/romfs/lang/en_US.json") + load_json_lang_keys("./plugins/builtin/romfs/lang/en_US.json") exit_ok = True -exit_ok &= check_plugin_langs("./main", ui_langs, None) +exit_ok &= check_plugin_langs("./main", [], common_langs) for plugin in os.listdir("./plugins"): if plugin == "ui": continue @@ -126,7 +127,8 @@ def verify_language_files_exist(languages_file_path: str) -> bool: path = f"./plugins/{plugin}" if not os.path.isdir(path): continue - exit_ok &= check_plugin_langs(path, ui_langs, f"./plugins/{plugin}/romfs/lang/en_US.json") + specific_langs = load_json_lang_keys(f"./plugins/{plugin}/romfs/lang/en_US.json") + exit_ok &= check_plugin_langs(path, common_langs, specific_langs) exit_ok &= verify_language_files_exist(f"./plugins/{plugin}/romfs/lang/languages.json") sys.exit(0 if exit_ok else 1) From 1b2f0e70bf7b7a507e78e2765715c86d5740379e Mon Sep 17 00:00:00 2001 From: iTrooz Date: Sun, 19 Jul 2026 23:26:36 +0100 Subject: [PATCH 3/8] add --nonexistent flag --- tests/check_langs.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/check_langs.py b/tests/check_langs.py index bc41d00781999..64fea2165b2f2 100644 --- a/tests/check_langs.py +++ b/tests/check_langs.py @@ -5,7 +5,7 @@ - Every key in an en_US.json file must be referenced somewhere in C/C++ source. Usage: - python check_langs.py [--unused] + python check_langs.py [--unused] [--nonexistent] Exit code 1 on any mismatch.""" import json @@ -15,6 +15,7 @@ from collections.abc import Generator CHECK_UNUSED_LANGS = "--unused" in sys.argv +CHECK_NONEXISTENT_LANGS = "--nonexistent" in sys.argv def find_lang_keys_in_source(path: str) -> Generator[tuple[str, int, str], None, None]: @@ -76,7 +77,7 @@ def check_plugin_langs(code_path: str, common_langs: list[str], specific_langs: except ValueError: pass - if not match in all_langs: + if CHECK_NONEXISTENT_LANGS and not match in all_langs: ret = False print(f"Problem: Lang '{match}' at {filepath}:{line} not found") From a44c42881d3494043faf4f2d605c3a1252f53335 Mon Sep 17 00:00:00 2001 From: iTrooz Date: Sun, 19 Jul 2026 23:39:20 +0100 Subject: [PATCH 4/8] fix: query source code in different ways based on what we want to check --- tests/check_langs.py | 77 +++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/tests/check_langs.py b/tests/check_langs.py index 64fea2165b2f2..8db9e3c0bee28 100644 --- a/tests/check_langs.py +++ b/tests/check_langs.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Check for unused and non-existent language keys across the codebase. -- Every string in C/C++ source must have a matching key in an en_US.json file. -- Every key in an en_US.json file must be referenced somewhere in C/C++ source. +- Every "_lang" string in C/C++ source must have a matching key in an en_US.json file. +- Every key in an en_US.json file must be referenced somewhere in C/C++ source starting with hex. Usage: python check_langs.py [--unused] [--nonexistent] @@ -18,9 +18,18 @@ CHECK_NONEXISTENT_LANGS = "--nonexistent" in sys.argv -def find_lang_keys_in_source(path: str) -> Generator[tuple[str, int, str], None, None]: - """Walk all C/C++ files under path and yield (filepath, line_number, key) - for every quoted string matching "hex.".""" +def find_lang_keys_in_source(path: str, mode: str) -> Generator[tuple[str, int, str], None, None]: + """Walk all C/C++ files under path and yield (filepath, line_number, key). + + mode="lang": match "..."_lang patterns. + mode="hex": match plain "hex.*" string literals.""" + if mode == "lang": + pattern = r'"([^"]*?)"_lang' + elif mode == "hex": + pattern = r'"(hex\.[a-zA-Z0-9_.]+)"' + else: + raise ValueError(f"Unknown mode: {mode}") + for dir, _, files in os.walk(path): for file in files: @@ -31,7 +40,7 @@ def find_lang_keys_in_source(path: str) -> Generator[tuple[str, int, str], None, with open(filepath, encoding="utf8") as file: for line_num, line in enumerate(file): - for m in re.finditer(r'"(hex\.[a-zA-Z0-9_.]+)"', line): + for m in re.finditer(pattern, line): yield (filepath, line_num+1, m.group(1)) @@ -50,45 +59,40 @@ def load_json_lang_keys(filepath: str | None) -> list[str]: for key, _ in data.items(): existing_langs.append(key) - + for lang in existing_langs: if not lang.startswith("hex."): - ret = False print(f"Problem: Lang '{lang}' doesn't start with 'hex.'") return existing_langs -def check_plugin_langs(code_path: str, common_langs: list[str], specific_langs: list[str]) -> bool: - """For a single plugin (or main): verify every "hex.*" string found in - source has a matching JSON key, and report keys in the JSON that aren't - referenced in code (when --unused is set).""" - print(f"--- Checking langs at {code_path}") - - all_langs = common_langs + specific_langs +def check_nonexistent_langs(code_path: str, all_langs: list[str]) -> bool: + """Check that every "_lang" key found in source exists in the JSON.""" + if not CHECK_NONEXISTENT_LANGS: + return True - unused_langs = specific_langs.copy() + print(f"--- Checking nonexistent langs at {code_path}") ret = True - # Check that every "hex.*" string in source has a matching key in the JSON - for filepath, line, match in find_lang_keys_in_source(code_path): - try: - unused_langs.remove(match) - except ValueError: - pass - - if CHECK_NONEXISTENT_LANGS and not match in all_langs: + for filepath, line, match in find_lang_keys_in_source(code_path, "lang"): + if not match in all_langs: ret = False print(f"Problem: Lang '{match}' at {filepath}:{line} not found") - - # Check for unused keys in the JSON file - if CHECK_UNUSED_LANGS and len(unused_langs) > 0: - ret = False - for unused_lang in unused_langs: - print(f"Problem: Unused lang {unused_lang}") return ret +def check_unused_langs(unused_langs: list[str], langs_path: str) -> bool: + """Check for keys in the JSON file that aren't referenced in code.""" + if not CHECK_UNUSED_LANGS or len(unused_langs) == 0: + return True + + print(f"--- Checking unused langs at {langs_path}") + for lang in unused_langs: + print(f"Problem: Unused lang {lang}") + return False + + def verify_language_files_exist(languages_file_path: str) -> bool: """Check that a plugin's languages.json references language files that actually exist on disk.""" @@ -120,16 +124,21 @@ def verify_language_files_exist(languages_file_path: str) -> bool: common_langs = load_json_lang_keys("./plugins/ui/romfs/lang/en_US.json") + load_json_lang_keys("./plugins/builtin/romfs/lang/en_US.json") exit_ok = True -exit_ok &= check_plugin_langs("./main", [], common_langs) for plugin in os.listdir("./plugins"): - if plugin == "ui": continue - path = f"./plugins/{plugin}" if not os.path.isdir(path): continue specific_langs = load_json_lang_keys(f"./plugins/{plugin}/romfs/lang/en_US.json") - exit_ok &= check_plugin_langs(path, common_langs, specific_langs) + + # Collect hex-mode keys from source to detect unused JSON keys + hex_keys = set() + for _, _, key in find_lang_keys_in_source(path, "hex"): + hex_keys.add(key) + unused = [k for k in specific_langs if k not in hex_keys] + + exit_ok &= check_nonexistent_langs(path, common_langs + specific_langs) + exit_ok &= check_unused_langs(unused, f"./plugins/{plugin}/romfs/lang/en_US.json") exit_ok &= verify_language_files_exist(f"./plugins/{plugin}/romfs/lang/languages.json") sys.exit(0 if exit_ok else 1) From eb74321f29906651e894055f07193233a2d8f348 Mon Sep 17 00:00:00 2001 From: iTrooz Date: Sun, 19 Jul 2026 23:39:44 +0100 Subject: [PATCH 5/8] ci: use --nonexistent in CI --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5a3a45a085eae..bcb1a1c18b1b0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -101,4 +101,4 @@ jobs: - name: Check langs run: - python3 tests/check_langs.py + python3 tests/check_langs.py --nonexistent From 6e4de3ae0d89768c20d51429d4a35431a214447e Mon Sep 17 00:00:00 2001 From: iTrooz Date: Tue, 21 Jul 2026 00:25:14 +0100 Subject: [PATCH 6/8] chore: merge langtool.py and check_langs.py --- .github/workflows/tests.yml | 2 +- dist/langtool.py | 535 ++++++++++++++++++++++++------------ 2 files changed, 366 insertions(+), 171 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bcb1a1c18b1b0..3bf46ce371449 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -101,4 +101,4 @@ jobs: - name: Check langs run: - python3 tests/check_langs.py --nonexistent + python3 dist/langtool.py check-nonexistent diff --git a/dist/langtool.py b/dist/langtool.py index 308d4687b37fd..5dd48bf623c60 100755 --- a/dist/langtool.py +++ b/dist/langtool.py @@ -1,203 +1,398 @@ #!/usr/bin/env python3 -from pathlib import Path +"""ImHex language tool. + +Manages translations and validates lang keys against C/C++ source code. +Run with --help to see available subcommands.""" import argparse import json +import re +from collections.abc import Callable, Generator +from pathlib import Path # This fixes a CJK full-width character input issue -# which makes left halves of deleted characters displayed on screen -# pylint: disable=unused-import -import re -import readline +# which makes left halves of deleted characters displayed on screen +import readline # noqa: F401 DEFAULT_LANG = "en_US" -DEFAULT_LANG_PATH = "plugins/*/romfs/lang/" INVALID_TRANSLATION = "" +def resolve_lang_dirs() -> tuple[list[Path], list[Path]]: + """Resolve langdir glob into (lang_dirs, source_roots). -def main(): - parser = argparse.ArgumentParser( - prog="langtool", - description="ImHex translate tool", - ) - parser.add_argument( - "command", - choices=[ - "check", - "translate", - "update", - "create", - "retranslate", - "untranslate", - "fmtzh", - ], - ) - parser.add_argument( - "-c", "--langdir", default=DEFAULT_LANG_PATH, help="Language folder glob" - ) - parser.add_argument("-l", "--lang", default="", help="Language to translate") - parser.add_argument( - "-r", "--reflang", default="", help="Language for reference when translating" - ) - parser.add_argument( - "-k", "--keys", help="Keys to re-translate (only in re/untranslate mode)" - ) - args = parser.parse_args() + lang_dirs are the actual lang folders (e.g. plugins/builtin/romfs/lang/). + source_roots are the corresponding C++ source roots (e.g. plugins/builtin/).""" + lang_dirs = sorted(Path(".").glob("plugins/*/romfs/lang/")) + source_roots = [] + for lang_dir in lang_dirs: + # plugins/builtin/romfs/lang/ -> plugins/builtin/ + source_root = lang_dir.parent.parent + if source_root not in source_roots: + source_roots.append(source_root) + return lang_dirs, source_roots - command = args.command - lang = args.lang - print(f"Running in {command} mode") - lang_files_glob = f"{lang}.json" if lang != "" else "*.json" +def load_json_data(filepath: Path) -> dict[str, str]: + """Return the full JSON contents as a dict, or {} if it doesn't exist.""" + if not filepath.exists(): + return {} + with filepath.open("r", encoding="utf-8-sig") as f: + return json.load(f) - lang_folders = set(Path(".").glob(args.langdir)) - if len(lang_folders) == 0: - print(f"Error: {args.langdir} matches nothing") - return 1 - for lang_folder in lang_folders: - if not lang_folder.is_dir(): - print(f"Error: {lang_folder} is not a folder") - return 1 +def write_json(filepath: Path, data: dict[str, str]) -> None: + """Write a dict to a JSON file with standard formatting.""" + with filepath.open("w", encoding="utf-8") as f: + json.dump(data, f, indent=4, sort_keys=False, ensure_ascii=False) + f.write("\n") - default_lang_data = {} - default_lang_path = lang_folder / Path(DEFAULT_LANG + ".json") - if not default_lang_path.exists(): - print( - f"Error: Default language file {default_lang_path} does not exist in {lang_folder}" + +def find_lang_keys_in_source(path: Path, mode: str) -> Generator[tuple[Path, int, str], None, None]: + """Walk C/C++ files under path and yield (filepath, line_number, key). + + mode="lang": match "..."_lang patterns. + mode="hex": match plain "hex.*" string literals.""" + if mode == "lang": + pattern = r'"([^"]*?)"_lang' + elif mode == "hex": + pattern = r'"(hex\.[a-zA-Z0-9_.]+)"' + else: + raise ValueError(f"Unknown mode: {mode}") + + for dirpath, _, files in path.walk(): + for filename in files: + if Path(filename).suffix not in (".cpp", ".c", ".hpp", ".h"): + continue + + filepath = dirpath / filename + with filepath.open(encoding="utf-8") as f: + for line_num, line in enumerate(f, 1): + for m in re.finditer(pattern, line): + yield (filepath, line_num, m.group(1)) + + +def verify_language_files_exist(languages_file_path: Path) -> bool: + """Check that a plugin's languages.json references files that exist on disk.""" + languages_folder = languages_file_path.parent + if not languages_folder.exists(): + return True + + if not languages_file_path.exists(): + print(f"Error: Languages file '{languages_file_path}' does not exist.") + return False + + try: + data = json.loads(languages_file_path.read_text(encoding="utf-8")) + except json.JSONDecodeError as e: + print(f"Error: Languages file '{languages_file_path}' is not valid JSON: {e}") + return False + + if not isinstance(data, list): + print(f"Error: Languages file '{languages_file_path}' is not a JSON array.") + return False + + for lang in data: + lang_path = languages_folder.parent / lang["path"] + if not lang_path.exists(): + print(f"Error: Language file '{lang['path']}' does not exist in '{languages_folder}'.") + return False + + return True + +def cmd_check(args: argparse.Namespace) -> int: + """Check that all non-English translation files have every key from en_US.""" + ret = 0 + + def check_callback(lang_data: dict[str, str], default_data: dict[str, str], path: Path) -> None: + nonlocal ret + for key, value in default_data.items(): + if key in lang_data and lang_data[key] != INVALID_TRANSLATION: + continue + print(f"Error: Translation {path} is missing translation for key '{key}'") + ret = 1 + + _for_each_lang_file(args.lang, check_callback) + return ret + + +# Print one interactive translation prompt line with optional existing value. +def _print_translation_prompt(key: str, value: str, current_value: str | None = None) -> None: + """Print a formatted prompt line for translation operations.""" + print(f"\033[1m'{key}' '{value}'\033[0m => ", end="") + if current_value is not None: + print(f" <= \033[1m'{current_value}'\033[0m") + print() + + +def _run_translate(args: argparse.Namespace, retranslate: bool = False) -> int: + """Run interactive translation, optionally limited to matching existing keys.""" + key_pattern = re.compile(args.keys) + + # Iterate keys and prompt for translated values. + def translate_callback(lang_data: dict[str, str], default_data: dict[str, str], path: Path) -> dict[str, str]: + for key, value in default_data.items(): + has_translation = key in lang_data and lang_data[key] != INVALID_TRANSLATION + + if retranslate: + if not has_translation or not key_pattern.fullmatch(key): + continue + elif has_translation: + continue + + _print_translation_prompt( + key, + value, + lang_data[key] if has_translation else None, ) - return 1 - with default_lang_path.open("r", encoding="utf-8") as file: - default_lang_data = json.load(file) - - reference_lang_data = None - reference_lang_path = lang_folder / Path(args.reflang + ".json") - if reference_lang_path.exists(): - with reference_lang_path.open("r", encoding="utf-8") as file: - reference_lang_data = json.load(file) - - if command == "create" and lang != "": - lang_file_path = lang_folder / Path(lang + ".json") - if lang_file_path.exists(): + + try: + new_value = input("=> ") + lang_data[key] = new_value + except KeyboardInterrupt: + break + + return lang_data + + _for_each_lang_file(args.lang, translate_callback) + return 0 + + +def _run_untranslate(args: argparse.Namespace) -> int: + """Clear translated values for matching keys.""" + key_pattern = re.compile(args.keys) + + # Limit clearing to keys that are translated and regex-matched. + def untranslate_callback(lang_data: dict[str, str], default_data: dict[str, str], path: Path) -> dict[str, str]: + for key, value in default_data.items(): + has_translation = key in lang_data and lang_data[key] != INVALID_TRANSLATION + if not has_translation or not key_pattern.fullmatch(key): continue - exist_lang_data = None - for lang_folder1 in lang_folders: - lang_file_path1 = lang_folder1 / Path(lang + ".json") - if lang_file_path1.exists(): - with lang_file_path1.open("r", encoding="utf-8") as file: - exist_lang_data = json.load(file) - break + _print_translation_prompt(key, value, lang_data[key]) + lang_data[key] = INVALID_TRANSLATION + + return lang_data + + _for_each_lang_file(args.lang, untranslate_callback) + return 0 + + +def cmd_sync_sublangs(args: argparse.Namespace) -> int: + """Sync non-English files with en_US (add missing blank entries, remove orphans).""" + + def sync_callback(lang_data: dict[str, str], default_data: dict[str, str], path: Path) -> dict[str, str]: + for key in default_data: + if key not in lang_data: + lang_data[key] = INVALID_TRANSLATION + _remove_orphaned_keys(lang_data, default_data, path) + return lang_data - print(f"Creating new language file '{lang_file_path}'") + _for_each_lang_file(args.lang, sync_callback) + return 0 - with lang_file_path.open("w", encoding="utf-8") as new_lang_file: - new_lang_data = { - } - json.dump(new_lang_data, new_lang_file, indent=4, ensure_ascii=False) +def cmd_fmtzh(args: argparse.Namespace) -> int: + """Fix CJK full-width punctuation in translations.""" + + def fmtzh_callback(lang_data: dict[str, str], default_data: dict[str, str], path: Path) -> dict[str, str]: + for key in default_data: + if key not in lang_data or lang_data[key] == INVALID_TRANSLATION: + continue + lang_data[key] = _fmtzh(lang_data[key]) + return lang_data + + _for_each_lang_file(args.lang, fmtzh_callback) + return 0 + +def cmd_check_nonexistent(args: argparse.Namespace) -> int: + """Verify every "_lang" key in C++ source has a matching JSON entry. + + Matches tests/check_langs.py: keys are accepted if present in either the + ui/builtin common en_US files or in the plugin's own en_US file.""" + lang_dirs, source_roots = resolve_lang_dirs() + + common_keys = set() + for common_plugin in ("ui", "builtin"): + common_path = Path(f"./plugins/{common_plugin}/romfs/lang/{DEFAULT_LANG}.json") + if common_path.exists(): + common_keys.update(load_json_data(common_path).keys()) + + plugin_keys: dict[Path, set[str]] = {} + for lang_dir in lang_dirs: + plugin_root = lang_dir.parent.parent + plugin_keys[plugin_root] = set(load_json_data(lang_dir / f"{DEFAULT_LANG}.json").keys()) + + ret = 0 + for source_root in source_roots: + print(f"--- Checking nonexistent langs at {source_root}") + allowed_keys = common_keys | plugin_keys.get(source_root, set()) + for filepath, line, key in find_lang_keys_in_source(source_root, "lang"): + if key not in allowed_keys: + ret = 1 + print(f"Problem: Lang '{key}' at {filepath}:{line} not found") + + return ret + +def _handle_unused(remove: bool) -> int: + """Shared logic for check-unused and remove-unused. + + Matches tests/check_langs.py: only a plugin's own en_US.json keys are checked + against hex.* string literals found in that same plugin's source.""" + _, source_roots = resolve_lang_dirs() + + ret = 0 + for source_root in source_roots: + lang_file = source_root / "romfs" / "lang" / f"{DEFAULT_LANG}.json" + lang_data = load_json_data(lang_file) + if not lang_data: + continue + + hex_keys: set[str] = set() + for _, _, key in find_lang_keys_in_source(source_root, "hex"): + hex_keys.add(key) + + keys_to_handle = [k for k in lang_data if k not in hex_keys] + if not keys_to_handle: + continue + + if remove: + print(f"--- Removing unused keys from {lang_file}") + for key in keys_to_handle: + del lang_data[key] + print(f" Removed '{key}'") + write_json(lang_file, lang_data) + else: + ret = 1 + print(f"--- Unused langs in {lang_file}") + for key in keys_to_handle: + print(f" {key}") + + return ret + +def _for_each_lang_file( + lang: str, + callback: Callable[[dict[str, str], dict[str, str], Path], dict[str, str] | None], +) -> None: + """Iterate over non-English lang files and apply a callback. + + Resolves lang dirs, loads the en_US default, and for each target lang file + calls callback(lang_data, default_data, path). If the callback returns a + dict, it is written back to the file.""" + lang_dirs, _ = resolve_lang_dirs() + + for lang_folder in lang_dirs: + print(f"\nProcessing lang folder '{lang_folder}'") + default_lang_path = lang_folder / f"{DEFAULT_LANG}.json" + if not default_lang_path.exists(): + print(f"Error: Default language file {default_lang_path} does not exist") + continue + + default_lang_data = load_json_data(default_lang_path) + lang_files = _get_target_lang_files(lang_folder, lang) - lang_files = set(lang_folder.glob(lang_files_glob)) - if len(lang_files) == 0: - print(f"Warn: Language file for '{lang}' does not exist in '{lang_folder}'") for lang_file_path in lang_files: - if ( - lang_file_path.stem == f"{DEFAULT_LANG}.json" - or lang_file_path.stem == f"{args.reflang}.json" - ): + print(f"Processing '{lang_file_path}'") + + lang_data = load_json_data(lang_file_path) + if not isinstance(lang_data, dict): + print(f"Skipping non-language file '{lang_file_path}'") continue - print(f"\nProcessing '{lang_file_path}'") - if not (command == "update" or command == "create"): - print("\n----------------------------\n") - - with lang_file_path.open("r+", encoding="utf-8") as target_lang_file: - lang_data = json.load(target_lang_file) - - for key, value in default_lang_data.items(): - has_translation = ( - key in lang_data - and lang_data[key] != INVALID_TRANSLATION - ) - if ( - has_translation - and not ( - (command == "retranslate" or command == "untranslate") - and re.compile(args.keys).fullmatch(key) - ) - and not command == "fmtzh" - ): - continue - if command == "check": - print( - f"Error: Translation {lang_file_path} is missing translation for key '{key}'" - ) - elif ( - command == "translate" - or command == "retranslate" - or command == "untranslate" - ): - if command == "untranslate" and not has_translation: - continue - reference_tranlsation = ( - " '%s'" % reference_lang_data[key] - if ( - reference_lang_data - and key in reference_lang_data - ) - else "" - ) - print( - f"\033[1m'{key}' '{value}'{reference_tranlsation}\033[0m => ", - end="", - ) - if has_translation: - translation = lang_data[key] - print(f" <= \033[1m'{translation}'\033[0m") - print() # for a new line - if command == "untranslate": - lang_data[key] = INVALID_TRANSLATION - continue - try: - new_value = input("=> ") - lang_data[key] = new_value - except KeyboardInterrupt: - break - elif command == "update" or command == "create": - lang_data[key] = INVALID_TRANSLATION - elif command == "fmtzh": - if has_translation: - lang_data[key] = fmtzh( - lang_data[key] - ) - - keys_to_remove = [] - for key, value in lang_data.items(): - if key not in default_lang_data: - keys_to_remove.append(key) - for key in keys_to_remove: - lang_data.pop(key) - print( - f"Removed unused key '{key}' from translation '{lang_file_path}'" - ) - - target_lang_file.seek(0) - target_lang_file.truncate() - json.dump( - lang_data, - target_lang_file, - indent=4, - sort_keys=True, - ensure_ascii=False, - ) - - -def fmtzh(text: str) -> str: + result = callback(lang_data, default_lang_data, lang_file_path) + + if result is not None: + write_json(lang_file_path, result) + + +def _get_target_lang_files(lang_folder: Path, lang: str) -> list[Path]: + """Get the list of non-default lang files to process.""" + glob_pattern = f"{lang}.json" if lang else "*.json" + results = [] + for p in sorted(lang_folder.glob(glob_pattern)): + if p.stem == DEFAULT_LANG: + continue + results.append(p) + return results + + +def _remove_orphaned_keys(lang_data: dict[str, str], default_data: dict[str, str], lang_file_path: Path) -> None: + """Remove keys from lang_data that don't exist in default_data.""" + keys_to_remove = [k for k in lang_data if k not in default_data] + for key in keys_to_remove: + del lang_data[key] + print(f"Removed unused key '{key}' from '{lang_file_path}'") + + +def _fmtzh(text: str) -> str: + """Fix CJK full-width punctuation.""" text = re.sub(r"(\.{3}|\.{6})", "……", text) text = text.replace("!", "！") - text = re.sub(r"([^\.\na-zA-Z\d])\.$", "\1。", text, flags=re.M) + text = re.sub(r"([^\.\na-zA-Z\d])\.$", r"\1。", text, flags=re.M) text = text.replace("?", "？") return text +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="langtool", + description="ImHex language tool", + ) + + subparsers = parser.add_subparsers(dest="command", required=True) + + # Translation commands + _add_translation_subparser(subparsers, "check", "Check non-English files have all translations") + p_translate = _add_translation_subparser( + subparsers, + "translate", + "Interactively translate keys (use --retranslate for already translated keys)", + ) + p_translate.add_argument( + "--retranslate", + action="store_true", + help="Only edit keys that already have a translation", + ) + p_translate.add_argument( + "-k", "--keys", default=".*", + help="Regex pattern used with --retranslate (default: %(default)s)", + ) + _add_translation_subparser(subparsers, "sync-sublangs", "Sync non-English files with en_US") + _add_translation_subparser(subparsers, "fmtzh", "Fix CJK punctuation in translations") + + # Untranslate needs --keys + p_untranslate = _add_translation_subparser(subparsers, "untranslate", "Clear translations matching a regex") + p_untranslate.add_argument("-k", "--keys", required=True, help="Regex pattern to match keys") + + # Source-checking commands (no --lang needed) + subparsers.add_parser("check-nonexistent", help="Verify _lang keys in C++ source exist in JSON") + subparsers.add_parser("check-unused", help="Report JSON keys not referenced in C++ source") + subparsers.add_parser("remove-unused", help="Remove JSON keys not referenced in C++ source") + + return parser + + +def _add_translation_subparser(subparsers, name: str, help_text: str) -> argparse.ArgumentParser: + """Add a subparser for a translation command with standard translation args.""" + p = subparsers.add_parser(name, help=help_text) + p.add_argument("-l", "--lang", default="", help="Target language (e.g. de_DE)") + return p + + +def main() -> int: + parser = build_parser() + args = parser.parse_args() + + commands = { + "check": cmd_check, + "translate": lambda args: _run_translate(args, retranslate=args.retranslate), + "untranslate": _run_untranslate, + "sync-sublangs": cmd_sync_sublangs, + "fmtzh": cmd_fmtzh, + "check-nonexistent": cmd_check_nonexistent, + "check-unused": lambda args: _handle_unused(remove=False), + "remove-unused": lambda args: _handle_unused(remove=True), + } + + return commands[args.command](args) + if __name__ == "__main__": exit(main()) From 910539f8c7d8b98767fb49177d1961883dd7e9ca Mon Sep 17 00:00:00 2001 From: iTrooz Date: Tue, 21 Jul 2026 00:38:50 +0100 Subject: [PATCH 7/8] ci: make CI check for nonexistent and unused langs --- .github/workflows/tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3bf46ce371449..6ef8b8867350a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -102,3 +102,6 @@ jobs: - name: Check langs run: python3 dist/langtool.py check-nonexistent + python3 dist/langtool.py check-unused + python3 dist/langtool.py sync-sublangs + git diff --exit-code From 27fe99ec2f031c7c78e920e76c6819d170ed4560 Mon Sep 17 00:00:00 2001 From: iTrooz Date: Tue, 21 Jul 2026 00:40:44 +0100 Subject: [PATCH 8/8] chore: remove check_langs.py --- tests/check_langs.py | 144 ------------------------------------------- 1 file changed, 144 deletions(-) delete mode 100644 tests/check_langs.py diff --git a/tests/check_langs.py b/tests/check_langs.py deleted file mode 100644 index 8db9e3c0bee28..0000000000000 --- a/tests/check_langs.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python3 -"""Check for unused and non-existent language keys across the codebase. - -- Every "_lang" string in C/C++ source must have a matching key in an en_US.json file. -- Every key in an en_US.json file must be referenced somewhere in C/C++ source starting with hex. - -Usage: - python check_langs.py [--unused] [--nonexistent] - -Exit code 1 on any mismatch.""" -import json -import re -import os -import sys -from collections.abc import Generator - -CHECK_UNUSED_LANGS = "--unused" in sys.argv -CHECK_NONEXISTENT_LANGS = "--nonexistent" in sys.argv - - -def find_lang_keys_in_source(path: str, mode: str) -> Generator[tuple[str, int, str], None, None]: - """Walk all C/C++ files under path and yield (filepath, line_number, key). - - mode="lang": match "..."_lang patterns. - mode="hex": match plain "hex.*" string literals.""" - if mode == "lang": - pattern = r'"([^"]*?)"_lang' - elif mode == "hex": - pattern = r'"(hex\.[a-zA-Z0-9_.]+)"' - else: - raise ValueError(f"Unknown mode: {mode}") - - for dir, _, files in os.walk(path): - for file in files: - - if not os.path.splitext(file)[1] in (".cpp", ".c", ".hpp", ".h"): - continue - - filepath = os.path.join(dir, file) - - with open(filepath, encoding="utf8") as file: - for line_num, line in enumerate(file): - for m in re.finditer(pattern, line): - yield (filepath, line_num+1, m.group(1)) - - -def load_json_lang_keys(filepath: str | None) -> list[str]: - """Return the list of top-level keys from a lang JSON file, or [] if - the file doesn't exist.""" - if filepath == None: - return [] - elif not os.path.exists(filepath): - print(f"Warning: no langs file found at {filepath}") - return [] - - with open(filepath, "r", encoding="utf8") as file: - data = json.loads(file.read()) - existing_langs = [] - - for key, _ in data.items(): - existing_langs.append(key) - - for lang in existing_langs: - if not lang.startswith("hex."): - print(f"Problem: Lang '{lang}' doesn't start with 'hex.'") - - return existing_langs - - -def check_nonexistent_langs(code_path: str, all_langs: list[str]) -> bool: - """Check that every "_lang" key found in source exists in the JSON.""" - if not CHECK_NONEXISTENT_LANGS: - return True - - print(f"--- Checking nonexistent langs at {code_path}") - ret = True - - for filepath, line, match in find_lang_keys_in_source(code_path, "lang"): - if not match in all_langs: - ret = False - print(f"Problem: Lang '{match}' at {filepath}:{line} not found") - return ret - - -def check_unused_langs(unused_langs: list[str], langs_path: str) -> bool: - """Check for keys in the JSON file that aren't referenced in code.""" - if not CHECK_UNUSED_LANGS or len(unused_langs) == 0: - return True - - print(f"--- Checking unused langs at {langs_path}") - for lang in unused_langs: - print(f"Problem: Unused lang {lang}") - return False - - -def verify_language_files_exist(languages_file_path: str) -> bool: - """Check that a plugin's languages.json references language files that - actually exist on disk.""" - languages_folder = os.path.dirname(languages_file_path) - if not os.path.exists(languages_folder): - return True - - if not os.path.exists(languages_file_path): - print(f"Error: Languages file '{languages_file_path}' does not exist.") - return False - with open(languages_file_path, "r", encoding="utf8") as file: - try: - data = json.load(file) - if not isinstance(data, list): - print(f"Error: Languages file '{languages_file_path}' is not a valid JSON object.") - return False - - for lang in data: - if not os.path.exists(os.path.join(languages_folder, "..", lang['path'])): - print(f"Error: Language file '{lang['path']}' does not exist in '{languages_folder}'.") - return False - return True - - except json.JSONDecodeError as e: - print(f"Error: Languages file '{languages_file_path}' is not a valid JSON file. {e}") - return False - - -common_langs = load_json_lang_keys("./plugins/ui/romfs/lang/en_US.json") + load_json_lang_keys("./plugins/builtin/romfs/lang/en_US.json") - -exit_ok = True - -for plugin in os.listdir("./plugins"): - path = f"./plugins/{plugin}" - if not os.path.isdir(path): continue - - specific_langs = load_json_lang_keys(f"./plugins/{plugin}/romfs/lang/en_US.json") - - # Collect hex-mode keys from source to detect unused JSON keys - hex_keys = set() - for _, _, key in find_lang_keys_in_source(path, "hex"): - hex_keys.add(key) - unused = [k for k in specific_langs if k not in hex_keys] - - exit_ok &= check_nonexistent_langs(path, common_langs + specific_langs) - exit_ok &= check_unused_langs(unused, f"./plugins/{plugin}/romfs/lang/en_US.json") - exit_ok &= verify_language_files_exist(f"./plugins/{plugin}/romfs/lang/languages.json") - -sys.exit(0 if exit_ok else 1)