Merge pull request #60 from cseptesting/59-git-repository-accessor-failing-to-manage-path-structure

59 git repository accessor failing to manage path structure

Author: pabloitu

floatcsep/experiment.py

@@ -5,6 +5,7 @@
 import os
 import shutil
 from os.path import join, abspath, relpath, dirname, isfile, split, exists
+from pathlib import Path
 from typing import Union, List, Dict, Sequence
 
 import numpy
@@ -113,12 +114,10 @@ def __init__(
         #    Or filter region?
         # Instantiate
 
-        workdir = abspath(kwargs.get("path", os.getcwd()))
+        workdir = Path(kwargs.get("path", os.getcwd())).resolve()
         if kwargs.get("timestamp", False):
-            rundir = os.path.join(
-                rundir, f"run_{datetime.datetime.utcnow().date().isoformat()}"
-            )
-        os.makedirs(os.path.join(workdir, rundir), exist_ok=True)
+            rundir = Path(rundir, f"run_{datetime.datetime.utcnow().date().isoformat()}")
+        os.makedirs(Path(workdir, rundir), exist_ok=True)
 
         self.name = name if name else "floatingExp"
         self.registry = ExperimentRegistry.factory(workdir=workdir, run_dir=rundir)
@@ -359,7 +358,6 @@ def set_input_cat(self, tstring: str, model: Model) -> None:
             model (:class:`~floatcsep.model.Model`): Model to give the input
              catalog
         """
-
         self.catalog_repo.set_input_cat(tstring, model)
 
     def set_tasks(self) -> None:
@@ -581,11 +579,14 @@ def make_repr(self) -> None:
         if not exists(target_cat):
             shutil.copy2(self.registry.abs(self.catalog_repo.cat_path), target_cat)
 
-        relative_path = os.path.relpath(
-            self.registry.workdir, os.path.join(self.registry.workdir, self.registry.run_dir)
+        # relative_path = self.registry.rel(self.registry.run_dir)
+        # print(self.registry.workdir.__class__, self.registry.run_dir.__class__)
+        relative_path = Path(
+            os.path.relpath(self.registry.workdir.as_posix(), self.registry.run_dir.as_posix())
         )
         self.registry.workdir = relative_path
-        self.to_yml(repr_config, extended=True)
+
+        self.to_yml(repr_config.as_posix(), extended=True)
 
     def as_dict(self, extra: Sequence = (), extended=False) -> dict:
         """
@@ -603,8 +604,8 @@ def as_dict(self, extra: Sequence = (), extended=False) -> dict:
         dict_walk = {
             "name": self.name,
             "config_file": self.config_file,
-            "path": self.registry.workdir,
-            "run_dir": self.registry.run_dir,
+            "path": self.registry.workdir.as_posix(),
+            "run_dir": self.registry.rel(self.registry.run_dir).as_posix(),
             "time_config": {
                 i: j
                 for i, j in self.time_config.items()
@@ -615,12 +616,11 @@ def as_dict(self, extra: Sequence = (), extended=False) -> dict:
                 for i, j in self.region_config.items()
                 if (i not in ("magnitudes", "depths") or extended)
             },
-            "catalog": self.catalog_repo.cat_path,
+            "catalog": self.registry.rel(self.catalog_repo.cat_path).as_posix(),
             "models": [i.as_dict() for i in self.models],
             "tests": [i.as_dict() for i in self.tests],
         }
         dict_walk.update(extra)
-
         return parse_nested_dicts(dict_walk)
 
     def to_yml(self, filename: str, **kwargs) -> None:
@@ -816,7 +816,9 @@ def get_filecomp(self):
                 results[test.name] = dict.fromkeys(models_orig)
                 for model in models_orig:
                     orig_path = self.original.registry.get_result_key(win_orig[-1], test, model)
-                    repr_path = self.reproduced.registry.get_result_key(win_orig[-1], test, model)
+                    repr_path = self.reproduced.registry.get_result_key(
+                        win_orig[-1], test, model
+                    )
                     results[test.name][model] = {
                         "hash": (self.get_hash(orig_path) == self.get_hash(repr_path)),
                         "byte2byte": filecmp.cmp(orig_path, repr_path),

floatcsep/infrastructure/engine.py

@@ -147,8 +147,9 @@ def add(self, task: Task):
         self.tasks[task] = []
         self.ntasks += 1
 
-    def add_dependency(self, task, dep_inst: Union[object, str] = None, dep_meth: str = None,
-                       dkw: Any = None):
+    def add_dependency(
+        self, task, dep_inst: Union[object, str] = None, dep_meth: str = None, dkw: Any = None
+    ):
         """
         Adds a dependency to a task already within the graph.
 

floatcsep/infrastructure/environments.py

@@ -429,7 +429,7 @@ def create_environment(self, force: bool = False) -> None:
                 rm=True,
                 decode=True,
                 buildargs=build_args,
-                nocache=False # todo: create model arg for --no-cache
+                nocache=False,  # todo: create model arg for --no-cache
             )
 
             # Stream each chunk
@@ -459,15 +459,15 @@ def env_exists(self) -> bool:
         except ImageNotFound:
             return False
 
-    def run_command(self,  command=None) -> None:
+    def run_command(self, command=None) -> None:
         """
         Runs the model’s Docker container with input/ and forecasts/ mounted.
         Streams logs and checks for non-zero exit codes.
         """
         model_root = os.path.abspath(self.model_directory)
         mounts = {
-            os.path.join(model_root, "input"): {'bind': '/app/input', 'mode': 'rw'},
-            os.path.join(model_root, "forecasts"): {'bind': '/app/forecasts', 'mode': 'rw'},
+            os.path.join(model_root, "input"): {"bind": "/app/input", "mode": "rw"},
+            os.path.join(model_root, "forecasts"): {"bind": "/app/forecasts", "mode": "rw"},
         }
 
         uid, gid = os.getuid(), os.getgid()

floatcsep/infrastructure/logger.py

@@ -62,8 +62,6 @@ def set_console_log_level(log_level):
             handler.setLevel(log_level)
 
 
-
-
 def log_models_tree(log, experiment_registry, time_windows):
     """
     Logs the forecasts for all models managed by this ExperimentFileRegistry.
@@ -112,7 +110,9 @@ def log_results_tree(log, experiment_registry):
 
                 for model_name, result_path in models.items():
                     total_results += 1
-                    result_full_path = experiment_registry.get_result_key(timewindow, test_name, model_name)
+                    result_full_path = experiment_registry.get_result_key(
+                        timewindow, test_name, model_name
+                    )
                     if os.path.exists(result_full_path):
                         results_exist_count += 1
                     else:
@@ -130,4 +130,4 @@ def log_results_tree(log, experiment_registry):
     log.debug(f"Total Results: {total_results}")
     log.debug(f"Results that Exist: {results_exist_count}")
     log.debug(f"Results that Do Not Exist: {results_not_exist_count}")
-    log.debug("===================")
+    log.debug("===================")