refac: changing os.path for pathlib across the application, having mostly changes in the registries module.

fix: from_git now correctly locates the repository content in the desired path set in model_config. incremented timeout for Zenodo requests.
ft: get_source methods are now different for TimeDependent and TimeIndependent. Expanded git and zenodo accessors to handle end-cases, and to become adapted to the type of model structure (e.g., src or files).
tests: updated corresponding unit tests, and expanded integration tests to catch more end-cases when querying a model from a repository.

Author: pabloitu

floatcsep/experiment.py

@@ -116,9 +116,7 @@ def __init__(
 
         workdir = Path(kwargs.get("path", os.getcwd())).resolve()
         if kwargs.get("timestamp", False):
-            rundir = Path(
-                rundir, f"run_{datetime.datetime.utcnow().date().isoformat()}"
-            )
+            rundir = Path(rundir, f"run_{datetime.datetime.utcnow().date().isoformat()}")
         os.makedirs(Path(workdir, rundir), exist_ok=True)
 
         self.name = name if name else "floatingExp"
@@ -360,7 +358,6 @@ def set_input_cat(self, tstring: str, model: Model) -> None:
             model (:class:`~floatcsep.model.Model`): Model to give the input
              catalog
         """
-
         self.catalog_repo.set_input_cat(tstring, model)
 
     def set_tasks(self) -> None:
@@ -582,11 +579,14 @@ def make_repr(self) -> None:
         if not exists(target_cat):
             shutil.copy2(self.registry.abs(self.catalog_repo.cat_path), target_cat)
 
-        relative_path = os.path.relpath(
-            self.registry.workdir, os.path.join(self.registry.workdir, self.registry.run_dir)
+        # relative_path = self.registry.rel(self.registry.run_dir)
+        # print(self.registry.workdir.__class__, self.registry.run_dir.__class__)
+        relative_path = Path(
+            os.path.relpath(self.registry.workdir.as_posix(), self.registry.run_dir.as_posix())
         )
         self.registry.workdir = relative_path
-        self.to_yml(repr_config, extended=True)
+
+        self.to_yml(repr_config.as_posix(), extended=True)
 
     def as_dict(self, extra: Sequence = (), extended=False) -> dict:
         """
@@ -604,8 +604,8 @@ def as_dict(self, extra: Sequence = (), extended=False) -> dict:
         dict_walk = {
             "name": self.name,
             "config_file": self.config_file,
-            "path": self.registry.workdir.resolve().as_posix(),
-            "run_dir": self.registry.run_dir.resolve().as_posix(),
+            "path": self.registry.workdir.as_posix(),
+            "run_dir": self.registry.rel(self.registry.run_dir).as_posix(),
             "time_config": {
                 i: j
                 for i, j in self.time_config.items()
@@ -616,11 +616,10 @@ def as_dict(self, extra: Sequence = (), extended=False) -> dict:
                 for i, j in self.region_config.items()
                 if (i not in ("magnitudes", "depths") or extended)
             },
-            "catalog": self.catalog_repo.cat_path.resolve().as_posix(),
+            "catalog": self.registry.rel(self.catalog_repo.cat_path).as_posix(),
             "models": [i.as_dict() for i in self.models],
             "tests": [i.as_dict() for i in self.tests],
         }
-        print(dict_walk)
         dict_walk.update(extra)
         return parse_nested_dicts(dict_walk)
 
@@ -817,7 +816,9 @@ def get_filecomp(self):
                 results[test.name] = dict.fromkeys(models_orig)
                 for model in models_orig:
                     orig_path = self.original.registry.get_result_key(win_orig[-1], test, model)
-                    repr_path = self.reproduced.registry.get_result_key(win_orig[-1], test, model)
+                    repr_path = self.reproduced.registry.get_result_key(
+                        win_orig[-1], test, model
+                    )
                     results[test.name][model] = {
                         "hash": (self.get_hash(orig_path) == self.get_hash(repr_path)),
                         "byte2byte": filecmp.cmp(orig_path, repr_path),

floatcsep/infrastructure/engine.py

@@ -147,8 +147,9 @@ def add(self, task: Task):
         self.tasks[task] = []
         self.ntasks += 1
 
-    def add_dependency(self, task, dep_inst: Union[object, str] = None, dep_meth: str = None,
-                       dkw: Any = None):
+    def add_dependency(
+        self, task, dep_inst: Union[object, str] = None, dep_meth: str = None, dkw: Any = None
+    ):
         """
         Adds a dependency to a task already within the graph.
 

floatcsep/infrastructure/environments.py

@@ -429,7 +429,7 @@ def create_environment(self, force: bool = False) -> None:
                 rm=True,
                 decode=True,
                 buildargs=build_args,
-                nocache=False # todo: create model arg for --no-cache
+                nocache=False,  # todo: create model arg for --no-cache
             )
 
             # Stream each chunk
@@ -459,15 +459,15 @@ def env_exists(self) -> bool:
         except ImageNotFound:
             return False
 
-    def run_command(self,  command=None) -> None:
+    def run_command(self, command=None) -> None:
         """
         Runs the model’s Docker container with input/ and forecasts/ mounted.
         Streams logs and checks for non-zero exit codes.
         """
         model_root = os.path.abspath(self.model_directory)
         mounts = {
-            os.path.join(model_root, "input"): {'bind': '/app/input', 'mode': 'rw'},
-            os.path.join(model_root, "forecasts"): {'bind': '/app/forecasts', 'mode': 'rw'},
+            os.path.join(model_root, "input"): {"bind": "/app/input", "mode": "rw"},
+            os.path.join(model_root, "forecasts"): {"bind": "/app/forecasts", "mode": "rw"},
         }
 
         uid, gid = os.getuid(), os.getgid()

floatcsep/infrastructure/logger.py

@@ -62,8 +62,6 @@ def set_console_log_level(log_level):
             handler.setLevel(log_level)
 
 
-
-
 def log_models_tree(log, experiment_registry, time_windows):
     """
     Logs the forecasts for all models managed by this ExperimentFileRegistry.
@@ -112,7 +110,9 @@ def log_results_tree(log, experiment_registry):
 
                 for model_name, result_path in models.items():
                     total_results += 1
-                    result_full_path = experiment_registry.get_result_key(timewindow, test_name, model_name)
+                    result_full_path = experiment_registry.get_result_key(
+                        timewindow, test_name, model_name
+                    )
                     if os.path.exists(result_full_path):
                         results_exist_count += 1
                     else:
@@ -130,4 +130,4 @@ def log_results_tree(log, experiment_registry):
     log.debug(f"Total Results: {total_results}")
     log.debug(f"Results that Exist: {results_exist_count}")
     log.debug(f"Results that Do Not Exist: {results_not_exist_count}")
-    log.debug("===================")
+    log.debug("===================")

floatcsep/infrastructure/registries.py

@@ -14,11 +14,13 @@
 
 log = logging.getLogger("floatLogger")
 
+
 class FilepathMixin:
     """
     Small mixin to provide filepath management functionality to Registries that uses files to
     store objects
     """
+
     workdir: Path
     path: Path
 
@@ -32,7 +34,7 @@ def dir(self) -> Path:
         return self.path.parents[0]
 
     @staticmethod
-    def _parse_arg(arg) -> Union[str , list[str]]:
+    def _parse_arg(arg) -> Union[str, list[str]]:
         if isinstance(arg, (list, tuple)):
             return timewindow2str(arg)
         elif isinstance(arg, str):
@@ -91,7 +93,7 @@ def abs_dir(self, *paths: Sequence[str | Path]) -> Path:
         _dir = _path.parents[0]
         return _dir
 
-    def rel(self, *paths: Union[Path | str | Sequence[str | Path]] ) -> Path:
+    def rel(self, *paths: Union[Path | str | Sequence[str | Path]]) -> Path:
         """
         Gets the relative path of an item, relative to the Registry workdir
 
@@ -133,6 +135,7 @@ def file_exists(self, *args: Sequence[str | Path]):
         file_abspath = self.get_attr(*args)
         return exists(file_abspath)
 
+
 class ModelRegistry(ABC):
     @abstractmethod
     def get_input_catalog_key(self, tstring: str) -> str:
@@ -147,18 +150,21 @@ def get_args_key(self, tstring: str) -> str:
         pass
 
     @classmethod
-    def factory(cls, registry_type: str = 'file', **kwargs) -> Union["ModelFileRegistry", "ModelHDF5Registry"]:
+    def factory(
+        cls, registry_type: str = "file", **kwargs
+    ) -> Union["ModelFileRegistry", "ModelHDF5Registry"]:
         """Factory method. Instantiate first on any explicit option provided in the model
         configuration.
         """
-        if registry_type == 'file':
+        if registry_type == "file":
             return ModelFileRegistry(**kwargs)
 
-        elif registry_type == 'hdf5':
+        elif registry_type == "hdf5":
             return ModelHDF5Registry(**kwargs)
         else:
             raise Exception("No valid model management schema was selected")
 
+
 class ModelFileRegistry(ModelRegistry, FilepathMixin):
     def __init__(
         self,
@@ -181,18 +187,16 @@ def __init__(
 
         self.model_name = model_name
         self.workdir = Path(workdir)
-        self.path = Path(path)
-
-        self.args_file = self.rel(self.path, args_file) if args_file else None
-        self.input_cat = self.rel(self.path, input_cat) if input_cat else None
+        self.path = self.abs(Path(path))
 
+        self.args_file = args_file if args_file else None
+        self.input_cat = input_cat if input_cat else None
         self.forecasts = {}
         self.input_args = {}
         self.input_cats = {}
         self.input_store = None
         self._fmt = fmt
 
-
     @property
     def fmt(self) -> str:
         """
@@ -265,8 +269,8 @@ def build_tree(
         time_windows: Sequence[Sequence[datetime]] = None,
         model_class: str = "TimeIndependentModel",
         prefix: str = None,
-        run_mode: str = 'sequential',
-        run_dir: Optional[str] = None
+        run_mode: str = "sequential",
+        run_dir: Optional[str] = None,
     ) -> None:
         """
         Creates the run directory, and reads the file structure inside.
@@ -295,20 +299,20 @@ def build_tree(
             for _, folder_ in dirtree.items():
                 os.makedirs(folder_, exist_ok=True)
 
-            if run_mode == 'sequential':
+            if run_mode == "sequential":
                 self.input_args = {
-                    win: Path(self.path, 'input', self.args_file) for win in windows
+                    win: Path(self.path, "input", self.args_file) for win in windows
                 }
                 self.input_cats = {
-                    win: Path(self.path, 'input', self.input_cat) for win in windows
+                    win: Path(self.path, "input", self.input_cat) for win in windows
                 }
-            elif run_mode == 'parallel':
+            elif run_mode == "parallel":
                 self.input_args = {
-                    win: Path(run_dir, win,  'input', self.model_name, self.args_file)
+                    win: Path(run_dir, win, "input", self.model_name, self.args_file)
                     for win in windows
                 }
                 self.input_cats = {
-                    win: Path(run_dir, win,  'input', self.model_name, self.input_cat)
+                    win: Path(run_dir, win, "input", self.model_name, self.input_cat)
                     for win in windows
                 }
 
@@ -330,16 +334,21 @@ def as_dict(self) -> dict:
             "forecasts": self.forecasts,
         }
 
+
 class ModelHDF5Registry(ModelRegistry):
 
     def __init__(self, workdir: str, path: str):
         pass
+
     def get_input_catalog_key(self, tstring: str) -> str:
-        return ''
+        return ""
+
     def get_forecast_key(self, tstring: str) -> str:
-        return ''
+        return ""
+
     def get_args_key(self, tstring: str) -> str:
-        return ''
+        return ""
+
 
 class ExperimentRegistry(ABC):
     @abstractmethod
@@ -372,16 +381,19 @@ def build_tree(
         pass
 
     @classmethod
-    def factory(cls, registry_type: str = 'file', **kwargs) -> Optional["ExperimentFileRegistry"]:
+    def factory(
+        cls, registry_type: str = "file", **kwargs
+    ) -> Optional["ExperimentFileRegistry"]:
         """Factory method. Instantiate first on any explicit option provided in the experiment
         configuration.
         """
 
-        if registry_type == 'file':
+        if registry_type == "file":
             return ExperimentFileRegistry(**kwargs)
         else:
             return None
 
+
 class ExperimentFileRegistry(ExperimentRegistry, FilepathMixin):
     """
     The class has the responsibility of managing the keys (based on models, timewindow and
@@ -398,7 +410,7 @@ def __init__(self, workdir: str, run_dir: str = "results") -> None:
             run_dir: The directory in which the results will be stored.
         """
         self.workdir = Path(workdir)
-        self.run_dir = Path(run_dir)
+        self.run_dir = self.abs(Path(run_dir))
         self.results = {}
         self.test_catalogs = {}
         self.figures = {}
@@ -510,7 +522,7 @@ def build_tree(
         time_windows: Sequence[Sequence[datetime]],
         models: Sequence["Model"],
         tests: Sequence["Evaluation"],
-        run_mode: str = 'sequential'
+        run_mode: str = "sequential",
     ) -> None:
         """
         Creates the run directory and reads the file structure inside.
@@ -529,8 +541,8 @@ def build_tree(
 
         run_folder = self.run_dir
         subfolders = ["catalog", "evaluations", "figures"]
-        if run_mode == 'parallel':
-            subfolders.append('input')
+        if run_mode == "parallel":
+            subfolders.append("input")
         dirtree = {
             win: {folder: self.abs(run_folder, win, folder) for folder in subfolders}
             for win in windows
@@ -540,8 +552,7 @@ def build_tree(
         for tw, tw_folder in dirtree.items():
             for _, folder_ in tw_folder.items():
                 os.makedirs(folder_, exist_ok=True)
-                if run_mode == 'parallel' and folder_.endswith('input'):
-                    print('a')
+                if run_mode == "parallel" and folder_.name == "input":
                     for model in models:
                         os.makedirs(join(folder_, model), exist_ok=True)
         results = {
@@ -578,4 +589,3 @@ def build_tree(
     def as_dict(self) -> Path:
 
         return self.workdir
-