Kept edited lines only

Some submods copy entire files whereas it is better to replace only new lines for multilingualism

Author: NicolasGrosjean

badges/coverage.svg

@@ -1,5 +1,6 @@
 poetry shell
 coverage run --omit "tests/*" -m pytest ./tests/ --junitxml=./report.xml -s
+python -m coverage xml
 genbadge tests -i ./report.xml -s -o ./badges/tests.svg
 genbadge coverage -i ./coverage.xml -s -o ./badges/coverage.svg
 coverage report

badges/tests.svg

@@ -0,0 +1,87 @@
+import argparse
+import os
+
+import sys
+
+sys.path.append(os.path.abspath(os.path.dirname(os.path.dirname(__file__))))
+from paradox_localization_utils.read_localization_file import (
+    file_to_keys_and_lines,
+    get_key,
+    BadLocalizationException,
+)
+
+
+def get_args():
+    parser = argparse.ArgumentParser(description="Keep edited lines only")
+    parser.add_argument("source_dir", type=str, help="Directory with Paradox files to filter")
+    parser.add_argument("original_dir", type=str, help="Directory with original Paradox files")
+    parser.add_argument("target_dir", type=str, help="Directory where to write filtered lines")
+    parser.add_argument("-language", type=str, help="Language to filter", default="english")
+    parser.add_argument("-target_prefix", type=str, help="Prefix added to files in target_dir", default="replace ")
+    return parser.parse_args()
+
+
+def write_kept_lines(source_file_path: str, output_file_path: str, original_lines_by_key: dict[str, str]):
+    """Write in output_file_path the lines of source_file_path that are not in or are edited from original_lines_by_key
+
+    :param source_file_path: File to filter
+    :param output_file_path: File where we write the filtered lines
+    :param original_lines_by_key: Dictionary with the original lines by key
+    """
+    with open(source_file_path, "r", encoding="utf8") as f:
+        lines = f.readlines()
+
+    with open(output_file_path, "w", encoding="utf8") as f:
+        language_found = False
+        for line in lines:
+            # Keep the language definition line
+            if not language_found and line.startswith("l_"):
+                f.write(line)
+                language_found = True
+                continue
+            try:
+                key = get_key(line)
+                if key not in original_lines_by_key or line != original_lines_by_key[key]:
+                    # It is a new or edited line, we keep it
+                    f.write(line)
+            except BadLocalizationException:
+                pass
+
+
+def keep_only_edited_lines(
+    source_dir: str,
+    original_dir: str,
+    target_dir: str,
+    language: str,
+    target_prefix: str
+):
+    # Store original file paths
+    original_file_paths_by_file_name: dict[str, str] = dict()
+    for root, _, files in os.walk(original_dir):
+        for file in files:
+            if file.endswith(language + ".yml"):
+                original_file_paths_by_file_name[file] = os.path.join(root, file) 
+
+    # Analyze each file in source_dir to kepts only edited lines
+    for root, _, files in os.walk(source_dir):
+        for file in files:
+            if file.endswith(language + ".yml"):
+                print(f"Processing {file}...")
+                # TODO Copy file when file is not in original_file_paths_by_file_name
+                original_lines_by_key, _ = file_to_keys_and_lines(original_file_paths_by_file_name[file])
+                write_kept_lines(
+                    os.path.join(root, file),
+                    os.path.join(target_dir, target_prefix + file),
+                    original_lines_by_key,
+                )
+
+
+if __name__ == "__main__":
+    args = get_args()
+    keep_only_edited_lines(
+        args.source_dir,
+        args.original_dir,
+        args.target_dir,
+        args.language,
+        args.target_prefix,
+    )

coverage.sh

@@ -86,7 +86,7 @@ def get_key(line):
     :return: (localization key, value corresponding to the localization key, version of the value)
     """
     i = 0
-    while i < len(line) and (line[i] == " " or line[i] == "\t"):
+    while i < len(line) and (line[i] == " " or line[i] == "\t" or line[i] == "\ufeff"):
         i += 1
     if i < len(line) and line[i] == "#":
         raise BadLocalizationException("Comment line")

paradox_localization_utils/keep_edited_lines_only.py

@@ -0,0 +1,54 @@
+from pathlib import Path
+from paradox_localization_utils.keep_edited_lines_only import write_kept_lines
+
+class TestWriteKeptLines:
+    def test_almost_empty_original(self, tmp_path: Path):
+        input_source_file_path = tmp_path / "toto_l_english.yml"
+        input_source_file_path.write_text("l_english:")
+        input_original_lines_by_key = {
+            "l_english": "l_english:"
+        }
+
+        output_file_path = tmp_path / "replace toto_l_english.yml"
+        write_kept_lines(input_source_file_path, output_file_path, input_original_lines_by_key)
+
+        assert output_file_path.read_text() == "l_english:"
+
+    def test_new_line(self, tmp_path: Path):
+        input_source_file_path = tmp_path / "toto_l_english.yml"
+        input_source_file_path.write_text("l_english:\nkey: value")
+        input_original_lines_by_key = {
+            "l_english": "l_english:"
+        }
+
+        output_file_path = tmp_path / "replace toto_l_english.yml"
+        write_kept_lines(input_source_file_path, output_file_path, input_original_lines_by_key)
+
+        assert output_file_path.read_text() == "l_english:\nkey: value"
+
+    def test_edited_line(self, tmp_path: Path):
+        input_source_file_path = tmp_path / "toto_l_english.yml"
+        input_source_file_path.write_text("l_english:\nkey: value2")
+        input_original_lines_by_key = {
+            "l_english": "l_english:",
+            "key": "key: value1"
+        }
+
+        output_file_path = tmp_path / "replace toto_l_english.yml"
+        write_kept_lines(input_source_file_path, output_file_path, input_original_lines_by_key)
+
+        assert output_file_path.read_text() == "l_english:\nkey: value2"
+
+    def test_unedited_line(self, tmp_path: Path):
+        input_source_file_path = tmp_path / "toto_l_english.yml"
+        input_source_file_path.write_text("l_english:\nkey: value")
+        input_original_lines_by_key = {
+            "l_english": "l_english:",
+            "key": "key: value"
+        }
+
+        output_file_path = tmp_path / "replace toto_l_english.yml"
+        write_kept_lines(input_source_file_path, output_file_path, input_original_lines_by_key)
+
+        assert output_file_path.read_text() == "l_english:\n"
+