Fix path issues (dirs no created automatically etc.)

Author: Rot127

.github/workflows/auto-sync.yml

@@ -30,13 +30,13 @@ jobs:
 
       - name: Build llvm-tblgen
         run: |
-          git clone https://github.com/capstone-engine/llvm-capstone.git
-          cd llvm-capstone
+          git clone https://github.com/capstone-engine/llvm-capstone.git vendor/llvm_root
+          cd vendor/llvm_root
           mkdir build
           cd build
           cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug ../llvm
           cmake --build . --target llvm-tblgen --config Debug
-          cd ../../
+          cd ../../../
 
       - name: Test generationof inc files
         run: |

suite/auto-sync/src/autosync/PathVarHandler.py

@@ -16,7 +16,6 @@ def __call__(cls, *args, **kwargs):
 
 
 class PathVarHandler(metaclass=Singleton):
-    paths = {}
 
     def __init__(self) -> None:
         try:
@@ -30,6 +29,7 @@ def __init__(self) -> None:
             exit(1)
         repo_root = res.stdout.decode("utf8").strip("\n")
         # The main directories
+        self.paths: dict[str:Path] = dict()
         self.paths["{CS_ROOT}"] = Path(repo_root)
         self.paths["{AUTO_SYNC_ROOT}"] = Path(repo_root).joinpath("suite/auto-sync/")
         self.paths["{AUTO_SYNC_SRC}"] = self.paths["{AUTO_SYNC_ROOT}"].joinpath(
@@ -41,20 +41,27 @@ def __init__(self) -> None:
         with open(path_config_file) as f:
             vars = json.load(f)
 
+        paths = vars["paths"]
+        create_during_runtime = vars["create_during_runtime"]
+
         missing = list()
-        for p_name, path in vars.items():
+        for p_name, path in paths.items():
             resolved = path
             for var_id in re.findall(r"\{.+}", resolved):
                 if var_id not in self.paths:
                     log.fatal(
                         f"{var_id} hasn't been added to the PathVarsHandler, yet. The var must be defined in a previous entry."
                     )
                     exit(1)
-                resolved = re.sub(var_id, str(self.paths[var_id]), resolved)
+                resolved: str = re.sub(var_id, str(self.paths[var_id]), resolved)
                 log.debug(f"Set {p_name} = {resolved}")
-                if not Path(resolved).exists():
+                if not Path(resolved).exists() and (
+                    p_name not in create_during_runtime
+                ):
                     missing.append(resolved)
-                self.paths[p_name] = resolved
+                elif var_id in create_during_runtime:
+                    self.create_path(var_id, resolved)
+                self.paths[p_name] = Path(resolved)
         if len(missing) > 0:
             log.fatal(f"Some paths from config file are missing!")
             for m in missing:
@@ -69,5 +76,25 @@ def get_path(self, name: str) -> Path:
     def complete_path(self, path_str: str) -> Path:
         resolved = path_str
         for p_name in re.findall(r"\{.+}", path_str):
-            resolved = re.sub(p_name, self.get_path(p_name), resolved)
+            resolved = re.sub(p_name, str(self.get_path(p_name)), resolved)
         return Path(resolved)
+
+    @staticmethod
+    def create_path(var_id: str, path: str):
+        pp = Path(path)
+        if pp.exists():
+            return
+
+        postfix = var_id.strip("}").split("_")[-1]
+        if postfix == "FILE":
+            if not pp.parent.exists():
+                pp.parent.mkdir(parents=True)
+            pp.touch()
+        elif postfix == "DIR":
+            pp.mkdir(parents=True)
+        else:
+            from autosync.Helper import fail_exit
+
+            fail_exit(
+                f"The var_id: {var_id} must end in _FILE or _DIR. It ends in '{postfix}'"
+            )

suite/auto-sync/src/autosync/cpptranslator/CppTranslator.py

@@ -181,7 +181,7 @@ def __init__(self, configure: Configurator):
         self.src_paths: [Path] = [
             get_path(sp["in"]) for sp in self.conf["files_to_translate"]
         ]
-        t_out_dir: Path = get_path(self.conf_general["translation_out_dir"])
+        t_out_dir: Path = get_path("{CPP_TRANSLATOR_TRANSLATION_OUT_DIR}")
         self.out_paths: [Path] = [
             t_out_dir.joinpath(sp["out"]) for sp in self.conf["files_to_translate"]
         ]

suite/auto-sync/src/autosync/cpptranslator/Differ.py

@@ -176,7 +176,7 @@ def __init__(self, configurator: Configurator, no_auto_apply: bool):
         self.parser = self.configurator.get_parser()
         self.differ = dl.Differ()
 
-        t_out_dir: Path = get_path(self.conf_general["translation_out_dir"])
+        t_out_dir: Path = get_path("{CPP_TRANSLATOR_TRANSLATION_OUT_DIR}")
         self.translated_files = [
             t_out_dir.joinpath(sp["out"]) for sp in self.conf_arch["files_to_translate"]
         ]
@@ -191,9 +191,7 @@ def __init__(self, configurator: Configurator, no_auto_apply: bool):
         self.load_persistence_file()
 
     def load_persistence_file(self) -> None:
-        self.persistence_filepath = get_path(
-            self.conf_general["patch_persistence_file"]
-        )
+        self.persistence_filepath = get_path("{CPP_TRANSLATOR_PATH_PERSISTENCE_FILE}")
         if not self.persistence_filepath.exists():
             self.saved_patches = dict()
             return
@@ -224,7 +222,7 @@ def copy_files(self) -> None:
         Copy translated files to diff directory for editing.
         """
         log.info("Copy files for editing")
-        diff_dir: Path = get_path(self.conf_general["diff_out_dir"])
+        diff_dir: Path = get_path("{CPP_TRANSLATOR_DIFF_OUT_DIR}")
         for f in self.translated_files:
             dest = diff_dir.joinpath(f.name)
             copy2(f, dest)

suite/auto-sync/src/autosync/cpptranslator/README.md

@@ -13,9 +13,6 @@ The configuration for each architecture is set in `arch_config.json`.
 The config values have the following meaning:
 
 - `General`: Settings valid for all architectures.
-   - `patch_persistent_file`: Path to the file which saves the selections from the `Differ`.
-   - `translation_out_dir`: Path to the directory where the `CppTranslator` stores its files.
-   - `diff_out_dir`: Path to the directory where the `Differ` stores its files.
    - `diff_color_new`: Color in the `Differ` for translated content.
    - `diff_color_old`: Color in the `Differ` for old/current Capstone content.
    - `diff_color_saved`: Color in the `Differ` for saved content.

suite/auto-sync/src/autosync/cpptranslator/Tests/test_config.json

@@ -1,8 +1,5 @@
 {
   "General": {
-    "patch_persistence_file": "",
-    "translation_out_dir": "",
-    "diff_out_dir": "",
     "diff_color_new": "green",
     "diff_color_old": "light_blue",
     "diff_color_saved": "yellow",

suite/auto-sync/src/autosync/cpptranslator/arch_config.json

@@ -1,8 +1,5 @@
 {
   "General": {
-    "patch_persistence_file": "{CPP_TRANSLATOR_DIR}/saved_patches.json",
-    "translation_out_dir": "{BUILD_DIR}/translate_out/",
-    "diff_out_dir": "{BUILD_DIR}/diff_out/",
     "diff_color_new": "green",
     "diff_color_old": "light_blue",
     "diff_color_saved": "yellow",

suite/auto-sync/src/autosync/path_vars.json

@@ -1,23 +1,25 @@
 {
-  "{LLVM_ROOT}": "{AUTO_SYNC_ROOT}/vendor/llvm_root//",
-  "{LLVM_TARGET_DIR}": "{LLVM_ROOT}/llvm/lib/Target/",
-  "{LLVM_TBLGEN_BIN}": "{LLVM_ROOT}/build/bin/llvm-tblgen",
-  "{LLVM_INCLUDE_DIR}": "{LLVM_ROOT}/llvm/include",
-
-  "{VENDOR_DIR}": "{AUTO_SYNC_ROOT}/vendor/",
-
-  "{CPP_TRANSLATOR_DIR}": "{AUTO_SYNC_SRC}/cpptranslator/",
-  "{CPP_TRANSLATOR_CONFIG}": "{CPP_TRANSLATOR_DIR}/arch_config.json",
-  "{CPP_TRANSLATOR_TEST_DIR}": "{CPP_TRANSLATOR_DIR}/Tests/",
-  "{CPP_TRANSLATOR_TEST_CONFIG}": "{CPP_TRANSLATOR_TEST_DIR}/test_config.json",
-  "{INC_PATCH_DIR}": "{AUTO_SYNC_ROOT}/inc_patches/",
-
-  "{CS_INCLUDE_DIR}": "{CS_ROOT}/include/capstone/",
-  "{CS_ARCH_MODULE_DIR}": "{CS_ROOT}/arch/",
-  "{CS_CLANG_FORMAT_FILE}": "{CS_ROOT}/.clang-format",
-
-  "{BUILD_DIR}": "{AUTO_SYNC_ROOT}/build/",
-  "{C_INC_OUT_DIR}": "{BUILD_DIR}/llvm_c_inc/",
-  "{CPP_INC_OUT_DIR}": "{BUILD_DIR}/llvm_cpp_inc/"
+  "paths": {
+    "{LLVM_ROOT}": "{AUTO_SYNC_ROOT}/vendor/llvm_root//",
+    "{LLVM_TARGET_DIR}": "{LLVM_ROOT}/llvm/lib/Target/",
+    "{LLVM_TBLGEN_BIN}": "{LLVM_ROOT}/build/bin/llvm-tblgen",
+    "{LLVM_INCLUDE_DIR}": "{LLVM_ROOT}/llvm/include/",
+    "{VENDOR_DIR}": "{AUTO_SYNC_ROOT}/vendor/",
+    "{BUILD_DIR}": "{AUTO_SYNC_ROOT}/build/",
+    "{C_INC_OUT_DIR}": "{BUILD_DIR}/llvm_c_inc/",
+    "{CPP_INC_OUT_DIR}": "{BUILD_DIR}/llvm_cpp_inc/",
+    "{CPP_TRANSLATOR_DIR}": "{AUTO_SYNC_SRC}/cpptranslator/",
+    "{CPP_TRANSLATOR_CONFIG}": "{CPP_TRANSLATOR_DIR}/arch_config.json",
+    "{CPP_TRANSLATOR_TEST_DIR}": "{CPP_TRANSLATOR_DIR}/Tests/",
+    "{CPP_TRANSLATOR_TEST_CONFIG}": "{CPP_TRANSLATOR_TEST_DIR}/test_config.json",
+    "{CPP_TRANSLATOR_PATH_PERSISTENCE_FILE}": "{CPP_TRANSLATOR_DIR}/saved_patches.json",
+    "{CPP_TRANSLATOR_TRANSLATION_OUT_DIR}": "{BUILD_DIR}/translate_out/",
+    "{CPP_TRANSLATOR_DIFF_OUT_DIR}": "{BUILD_DIR}/diff_out/",
+    "{INC_PATCH_DIR}": "{AUTO_SYNC_ROOT}/inc_patches/",
+    "{CS_INCLUDE_DIR}": "{CS_ROOT}/include/capstone/",
+    "{CS_ARCH_MODULE_DIR}": "{CS_ROOT}/arch/",
+    "{CS_CLANG_FORMAT_FILE}": "{CS_ROOT}/.clang-format"
+  },
+  "create_during_runtime": ["{BUILD_DIR}", "{C_INC_OUT_DIR}", "{CPP_INC_OUT_DIR}", "{CPP_TRANSLATOR_TRANSLATION_OUT_DIR}", "{CPP_TRANSLATOR_DIFF_OUT_DIR}"]
 }