Merge pull request #20 from Sage-Bionetworks-Workflows/bgrande/parsers-reports-cov

Close code coverage for `parsers` and `reports` submodules

Author: Bruno Grande

Pipfile.lock

@@ -76,6 +76,7 @@ testing =
     pytest-mock~=3.0
     hypothesis~=4.0
     nbmake~=1.3
+    pytest-xdist[psutil]~=3.1
 
 # Dependencies for development (used by Pipenv)
 dev =
@@ -108,8 +109,11 @@ console_scripts =
 # CAUTION: --cov flags may prohibit setting breakpoints while debugging.
 #          Comment those flags to avoid this pytest issue.
 addopts =
-    --cov dcqc --cov-report term-missing --cov-report xml
+    --cov "dcqc" --cov-report "term-missing" --cov-report "xml"
     -m "not slow"
+    # For now, the overhead with parallelizing the small number of tests makes
+    # them slower (2.49s vs 1.16s). Uncomment when we have more tests.
+    # --numprocesses "auto"
     --verbose
 norecursedirs =
     dist

setup.cfg

@@ -1,3 +1,4 @@
-from dcqc.main import app
+if __name__ == "__main__":
+    from dcqc.main import app
 
-app(prog_name="dcqc")
+    app(prog_name="dcqc")

src/dcqc/__main__.py

@@ -68,6 +68,15 @@ def register_file_type(self) -> None:
             raise ValueError(message)
         self._registry[name] = self
 
+    @classmethod
+    def list_file_types(cls) -> list[FileType]:
+        """Retrieve all available file type objects.
+
+        Returns:
+            The full list of file type objects.
+        """
+        return list(cls._registry.values())
+
     @classmethod
     def get_file_type(cls, file_type: str) -> FileType:
         """Retrieve file type object based on its name.

src/dcqc/file.py

@@ -4,7 +4,7 @@
 from pathlib import Path
 from typing import Any, Optional, Type, TypeVar, cast
 
-from dcqc.file import File
+from dcqc.file import File, FileType
 from dcqc.mixins import SerializableMixin
 from dcqc.suites.suite_abc import SuiteABC
 from dcqc.target import Target
@@ -85,14 +85,17 @@ def get_class(cls, cls_name: str) -> Type[SerializableMixin]:
         suite_classes = SuiteABC.list_subclasses()
         suite_cls_map = {cls.__name__: cls for cls in suite_classes}
 
-        if cls_name == "File":
-            return File
-        elif cls_name == "Target":
+        file_types = FileType.list_file_types()
+        file_types_names = {ft.name.lower() for ft in file_types}
+
+        if cls_name == "Target":
             return Target
         elif cls_name in test_cls_map:
             return test_cls_map[cls_name]
         elif cls_name in suite_cls_map:
             return suite_cls_map[cls_name]
+        elif cls_name.lower() in file_types_names:
+            return File
         else:
             message = f"Type ({cls_name}) is not recognized."
             raise ValueError(message)
@@ -126,7 +129,7 @@ def parse_objects(cls, path: Path, expected_cls: Type[T]) -> list[T]:
         contents = parser.load_json()
 
         if not isinstance(contents, list):
-            message = f"JSON file ({cls.path}) does not contain a list of objects."
+            message = f"JSON file ({parser.path}) does not contain a list of objects."
             raise ValueError(message)
 
         objects = [cls.from_dict(dictionary) for dictionary in contents]

src/dcqc/parsers.py

@@ -20,10 +20,10 @@ def __init__(self, paths_relative_to: Optional[Path] = None) -> None:
         self._fs: Optional[FS] = None
         self._fs_path: Optional[str] = None
 
+    # TODO: Move towards an FS mixin for these functions
     def _init_fs(self, url) -> tuple[FS, str]:
-        if self._url != url or self._fs is None or self._fs_path is None:
-            self._url = url
-            self._fs, self._fs_path = open_parent_fs(url)
+        self._url = url
+        self._fs, self._fs_path = open_parent_fs(url)
         return self._fs, self._fs_path
 
     def _create_parent_directories(self, url: str):
@@ -61,11 +61,11 @@ def _generate_single(self, item: SerializableMixin) -> SerializedObject:
     # the inputs and outputs: single to single, and many to many.
     @overload
     def generate(self, items: SerializableMixin) -> SerializedObject:
-        ...
+        """"""
 
     @overload
     def generate(self, items: Iterable[SerializableMixin]) -> list[SerializedObject]:
-        ...
+        """"""
 
     def generate(self, items):
         if isinstance(items, Iterable):
@@ -79,13 +79,13 @@ def generate(self, items):
     def save(
         self, items: SerializableMixin, url: str, overwrite: bool = False
     ) -> SerializedObject:
-        ...
+        """"""
 
     @overload
     def save(
         self, items: Iterable[SerializableMixin], url: str, overwrite: bool = False
     ) -> list[SerializedObject]:
-        ...
+        """"""
 
     def save(self, items, url: str, overwrite: bool = False):
         report = self.generate(items)

src/dcqc/reports.py

@@ -0,0 +1,9 @@
+{
+  "url": "tests/data/test.txt",
+  "metadata": {
+    "md5_checksum": "14758f1afd44c09b7992073ccf00b43d"
+  },
+  "type": "TIFF",
+  "name": "test.txt",
+  "local_path": "tests/data/test.txt"
+}

tests/data/file.json

@@ -4,6 +4,8 @@
 
 import os
 import sys
+from pathlib import Path
+from typing import Sequence
 
 from dcqc.file import File
 from dcqc.mixins import SerializableMixin
@@ -15,19 +17,22 @@
 # Shared values
 data_dir = sys.path[0]
 data_dir = os.path.relpath(data_dir)
-report = JsonReport()
+report = JsonReport(paths_relative_to=Path.cwd())
 
 
 # Shared functions
-def export(obj: SerializableMixin, filename: str):
+def export(obj: SerializableMixin | Sequence[SerializableMixin], filename: str):
     output_url = os.path.join(data_dir, filename)
     report.save(obj, output_url, overwrite=True)
 
 
-# target.json
+# file.json
 file_url = os.path.join(data_dir, "test.txt")
 metadata = {"file_type": "TIFF", "md5_checksum": "14758f1afd44c09b7992073ccf00b43d"}
 file = File(file_url, metadata)
+export(file, "file.json")
+
+# target.json
 target = Target(file, id="001")
 export(target, "target.json")
 
@@ -44,6 +49,10 @@ def export(obj: SerializableMixin, filename: str):
 computed_test.get_status()
 export(computed_test, "test.computed.json")
 
+# tests.json
+test_list = [internal_test, external_test, computed_test]
+export(test_list, "tests.json")
+
 # suite.json
 suite_tests = [internal_test, external_test]
 required_tests = ["Md5ChecksumTest"]

tests/data/generate.py

@@ -10,8 +10,8 @@
           "md5_checksum": "14758f1afd44c09b7992073ccf00b43d"
         },
         "type": "TIFF",
-        "local_path": "tests/data/test.txt",
-        "name": "test.txt"
+        "name": "test.txt",
+        "local_path": "tests/data/test.txt"
       }
     ]
   },

tests/data/suite.json

@@ -8,8 +8,8 @@
         "md5_checksum": "14758f1afd44c09b7992073ccf00b43d"
       },
       "type": "TIFF",
-      "local_path": "tests/data/test.txt",
-      "name": "test.txt"
+      "name": "test.txt",
+      "local_path": "tests/data/test.txt"
     }
   ]
 }