Merge pull request #5 from Sage-Bionetworks-Workflows/bgrande/tests-and-suites

Implement `TestABC` and `SuiteABC` as well as subclasses for the testing logic

Author: Bruno Grande

.pre-commit-config.yaml

@@ -53,4 +53,4 @@ repos:
 - repo: https://github.com/pre-commit/mirrors-mypy
   rev: 'v0.991'
   hooks:
-  -   id: mypy
+  - id: mypy

pyproject.toml

@@ -9,6 +9,18 @@ build-backend = "setuptools.build_meta"
 version_scheme = "no-guess-dev"
 
 [tool.mypy]
+
+[[tool.mypy.overrides]]
+module = "dcqc"
+disallow_untyped_calls = true
+disallow_untyped_defs = true
+disallow_incomplete_defs = true
+check_untyped_defs = true
+
 [[tool.mypy.overrides]]
 module = "synapseclient.*"
 ignore_missing_imports = true
+
+[[tool.mypy.overrides]]
+module = "dcqc.suites.suites"
+disable_error_code = "assignment"

src/dcqc/__init__.py

@@ -12,6 +12,15 @@
 
 from fs.opener import registry
 
+# isort: off
+
+# Import suites to ensure that they are defined and thus discoverable
+# It is located here to avoid a circular import
+from dcqc.suites import suite_abc  # isort: skip
+from dcqc.suites import suites  # isort: skip
+
+# isort: on
+
 from dcqc.filesystems.openers import SynapseFSOpener
 
 # Set default logging handler to avoid "No handler found" warnings

src/dcqc/enums.py

@@ -0,0 +1,8 @@
+from enum import Enum
+
+
+class TestStatus(Enum):
+    NONE = "pending"
+    FAIL = "failed"
+    PASS = "passed"
+    SKIP = "skipped"

src/dcqc/file.py

@@ -8,7 +8,9 @@
 from pathlib import Path, PurePosixPath
 from typing import Any, Optional
 
-from dcqc.mixins import SerializableMixin
+from fs.base import FS
+
+from dcqc.mixins import SerializableMixin, SerializedObject
 from dcqc.utils import open_parent_fs
 
 
@@ -29,7 +31,7 @@ def __init__(self, name: str, file_extensions: Collection[str]):
         self.register_file_type(self)
 
     @classmethod
-    def register_file_type(cls, self):
+    def register_file_type(cls, self: FileType) -> None:
         name = self.name.lower()
         if name in cls._registry:
             message = f"File type ({name}) is already registered ({self._registry})."
@@ -79,10 +81,11 @@ def __init__(
         self.metadata = dict(metadata)
         self.type = self._pop_file_type()
         self.file_name = self._get_file_name()
+        self._fs: Optional[FS]
         self._fs = None
 
     @property
-    def fs(self):
+    def fs(self) -> FS:
         if self._fs is None:
             fname = self.file_name
             fs, bname = open_parent_fs(self.url)
@@ -97,7 +100,7 @@ def _pop_file_type(self) -> str:
         del self.metadata["file_type"]
         return file_type
 
-    def _get_file_name(self):
+    def _get_file_name(self) -> str:
         path = PurePosixPath(self.url)
         return path.name
 
@@ -112,7 +115,7 @@ def get_metadata(self, key: str) -> Any:
             raise ValueError(message)
         return self.metadata[key]
 
-    def is_local(self, url: Optional[str] = None):
+    def is_local(self, url: Optional[str] = None) -> bool:
         url = url or self.url
         return self.LOCAL_REGEX.fullmatch(url) is not None
 
@@ -182,7 +185,7 @@ def stage(self, destination: Optional[str] = None, overwrite: bool = False) -> s
         self.url = destination
         return self.url
 
-    def to_dict(self):
+    def to_dict(self) -> SerializedObject:
         return asdict(self)
 
     @classmethod

src/dcqc/filesystems/openers.py

@@ -1,14 +1,22 @@
 import os
 
 from fs.opener import Opener
+from fs.opener.parse import ParseResult
 
 from dcqc.filesystems.synapsefs import SynapseFS
 
 
 class SynapseFSOpener(Opener):
     protocols = ["syn"]
 
-    def open_fs(self, fs_url, parse_result, writeable, create, cwd):
+    def open_fs(
+        self,
+        fs_url: str,
+        parse_result: ParseResult,
+        writeable: bool,
+        create: bool,
+        cwd: str,
+    ) -> SynapseFS:
         auth_token = os.environ.get("SYNAPSE_AUTH_TOKEN")
         root = parse_result.resource
         fs = SynapseFS(root, auth_token)

src/dcqc/filesystems/remote_file.py

@@ -2,56 +2,78 @@
 
 import io
 import os
-from typing import BinaryIO
+from array import array
+from collections.abc import Callable, Iterable
+from mmap import mmap
+from pickle import PickleBuffer
+from typing import IO, TYPE_CHECKING, Any, BinaryIO, Optional, Union
+
+from fs.mode import Mode
+
+if TYPE_CHECKING:
+    from ctypes import _CData
+
+    LinesType = Iterable[
+        Union[
+            bytes, Union[bytearray, memoryview, array[Any], mmap, _CData, PickleBuffer]
+        ]
+    ]
 
 
 class RemoteFile(io.IOBase, BinaryIO):
     """Proxy for a remote file."""
 
-    def __init__(self, f, filename, mode, on_close=None):
+    def __init__(
+        self,
+        f: IO,
+        filename: str,
+        mode: Mode,
+        on_close: Optional[Callable] = None,
+    ):
         self._f = f
         self.__filename = filename
         self.__mode = mode
         self._on_close = on_close
 
-    def __enter__(self):
+    def __enter__(self) -> RemoteFile:
         return self
 
-    def __exit__(self, exc_type, exc_value, traceback):
+    def __exit__(self, exc_type, exc_value, traceback) -> None:  # type: ignore
         self.close()
 
     @property
-    def raw(self):
+    def raw(self) -> IO:
         return self._f
 
-    def close(self):
+    def close(self) -> None:
         if self._on_close is not None:
             self._on_close(self)
 
     @property
-    def closed(self):
+    def closed(self) -> bool:
         return self._f.closed
 
     @property
-    def mode(self):
+    def mode(self) -> str:
         return self._f.mode
 
-    def fileno(self):
+    def fileno(self) -> int:
         return self._f.fileno()
 
-    def flush(self):
-        return self._f.flush()
+    def flush(self) -> None:
+        self._f.flush()
 
     # def isatty(self):
     #     return self._f.asatty()
 
-    def readable(self):
+    def readable(self) -> bool:
         return self.__mode.reading
 
-    def readline(self, limit=-1):
+    def readline(self, limit: Optional[int] = -1) -> bytes:
+        limit = limit or -1
         return self._f.readline(limit)
 
-    def readlines(self, hint=-1):
+    def readlines(self, hint: int = -1) -> list[bytes]:
         if hint == -1:
             return self._f.readlines(hint)
         else:
@@ -64,42 +86,42 @@ def readlines(self, hint=-1):
                     break
             return lines
 
-    def seek(self, offset, whence=os.SEEK_SET):
+    def seek(self, offset: int, whence: int = os.SEEK_SET) -> int:
         if whence not in (os.SEEK_CUR, os.SEEK_END, os.SEEK_SET):
             raise ValueError("invalid value for 'whence'")
         self._f.seek(offset, whence)
         return self._f.tell()
 
-    def seekable(self):
+    def seekable(self) -> bool:
         return True
 
-    def tell(self):
+    def tell(self) -> int:
         return self._f.tell()
 
-    def writable(self):
+    def writable(self) -> bool:
         return self.__mode.writing
 
-    def writelines(self, lines):
+    def writelines(self, lines: LinesType) -> None:
         return self._f.writelines(lines)
 
-    def read(self, n=-1):
+    def read(self, n: int = -1) -> bytes:
         if not self.__mode.reading:
             raise IOError("not open for reading")
         return self._f.read(n)
 
     # def readall(self):
     #     return self._f.readall()
 
-    def readinto(self, b):
-        return self._f.readinto(b)
+    # def readinto(self, b):
+    #     return self._f.readinto(b)
 
-    def write(self, b):
+    def write(self, b: bytes) -> int:
         if not self.__mode.writing:
             raise IOError("not open for reading")
         self._f.write(b)
         return len(b)
 
-    def truncate(self, size=None):
+    def truncate(self, size: Optional[int] = None) -> int:
         if size is None:
             size = self._f.tell()
         self._f.truncate(size)

src/dcqc/filesystems/synapsefs.py

@@ -8,7 +8,7 @@
 from contextlib import contextmanager
 from pathlib import Path, PurePosixPath
 from tempfile import NamedTemporaryFile, TemporaryDirectory, mkdtemp
-from typing import TYPE_CHECKING, Any, BinaryIO, Optional, Type
+from typing import TYPE_CHECKING, Any, BinaryIO, Generator, Optional, Type
 
 from fs import ResourceType
 from fs.base import FS
@@ -40,7 +40,7 @@
 
 
 @contextmanager
-def synapse_errors(path):
+def synapse_errors(path: str) -> Generator:
     """A context manager for mapping ``synapseclient`` errors to ``fs`` errors."""
     try:
         yield
@@ -116,12 +116,13 @@ def synapse(self) -> Synapse:
             Synapse: Authenticated Synapse client
         """
         if not hasattr(self._local, "synapse"):
-            # Override cache with temporary directory to avoid unwanted side effects
+            # Override cache with temporary directory
             self.synapse_args["cache_root_dir"] = mkdtemp()
             synapse = Synapse(**self.synapse_args)
             synapse.login(authToken=self.auth_token)
             self._local.synapse = synapse
-        # Clear the Synapse cache to ensure up-to-date
+        # Clear the Synapse cache to avoid unwanted side effects. More info here:
+        # https://github.com/Sage-Bionetworks-Workflows/py-dcqc/pull/3#discussion_r1068443214
         self._local.synapse.cache.purge(after_date=0)
         return self._local.synapse
 
@@ -237,7 +238,11 @@ def _path_to_synapse_id(
 
         return current_entity
 
-    def _synapse_id_to_entity(self, synapse_id: str, download_file=False) -> Entity:
+    def _synapse_id_to_entity(
+        self,
+        synapse_id: str,
+        download_file: bool = False,
+    ) -> Entity:
         """Retrieve and validate (meta)data for a Synapse entity
 
         Args:
@@ -261,7 +266,7 @@ def _synapse_id_to_entity(self, synapse_id: str, download_file=False) -> Entity:
             raise ResourceInvalid(message)
         return entity
 
-    def _path_to_entity(self, path: str, download_file=False) -> Entity:
+    def _path_to_entity(self, path: str, download_file: bool = False) -> Entity:
         """Perform the validation and retrieval steps for a Synapse entity.
 
         Arguments:
@@ -534,7 +539,7 @@ def openbin(
         temp_dir = TemporaryDirectory()
         temp_path = temp_dir.name
 
-        def on_close(remote_file):
+        def on_close(remote_file: RemoteFile) -> None:
             """Called when the S3 file closes, to upload data."""
             # If the file is empty, add a null byte to bypass
             # Synapse restriction on empty files
@@ -579,7 +584,7 @@ def on_close(remote_file):
 
         return RemoteFile(target_file, file_name, mode_obj, on_close)
 
-    def remove(self, path: str):
+    def remove(self, path: str) -> None:
         """Remove a file from the filesystem.
 
         Arguments:
@@ -600,7 +605,7 @@ def remove(self, path: str):
 
         self.synapse.delete(entity)
 
-    def removedir(self, path: str):
+    def removedir(self, path: str) -> None:
         """Remove a directory from the filesystem.
 
         Arguments:
@@ -639,7 +644,7 @@ def removedir(self, path: str):
 
         self.synapse.delete(entity)
 
-    def setinfo(self, path: str, info: RawInfo):
+    def setinfo(self, path: str, info: RawInfo) -> None:
         """Set info on a resource.
 
         This method is the complement to `~fs.base.FS.getinfo`