Merge pull request #40 from Sage-Bionetworks-Workflows/bwmac/orca-234/improved_file_mounting

[ORCA-234] Improve file path handling

Author: BWMac

src/dcqc/file.py

@@ -125,6 +125,8 @@ class File(SerializableMixin):
             current work directory (default).
     """
 
+    tmp_dir: ClassVar[str] = "dcqc-staged-"
+
     _serialized_properties = ["name", "local_path"]
 
     url: str
@@ -336,9 +338,7 @@ def stage(
             if self._local_path is not None:
                 return self._local_path
             else:
-                # TODO: This prefix is used by nf-dcqc to easily find the staged file.
-                #       It might be worth using a DCQCTMPDIR to avoid hard-coding this.
-                destination_str = mkdtemp(prefix="dcqc-staged-")
+                destination_str = mkdtemp(prefix=self.tmp_dir)
                 destination = Path(destination_str)
 
         # By this point, destination is defined (not None)

src/dcqc/tests/base_test.py

@@ -202,21 +202,6 @@ def _interpret_process_outputs(self, outputs: dict[str, Path]) -> TestStatus:
             status = TestStatus.FAIL
         return status
 
-    # TODO: make changes to this package or the nextflow
-    # workflow so that file mounting is handled cleaner
-    @staticmethod
-    def _short_string_path(path: Path, substring: str) -> str:
-        # chech if substring is in path
-        if substring not in path.as_posix():
-            raise ValueError(f"{substring} not in {path}")
-        # get index where staged folder is
-        index = next(i for i, item in enumerate(path.parts) if substring in item)
-        # shorten path starting from staged folder
-        short_path = Path(*path.parts[index:])
-        # wrap path string in quotes
-        quote_path = str(short_path)
-        return f"'{quote_path}'"
-
     # TODO: Include process in serialized test dictionary
     # def to_dict(self):
     #     dictionary = super(ExternalTestMixin, self).to_dict()

src/dcqc/tests/bioformats_info_test.py

@@ -9,14 +9,13 @@ class BioFormatsInfoTest(ExternalBaseTest):
 
     def generate_process(self) -> Process:
         path = self.target.file.stage()
-        string_path = self._short_string_path(path, "dcqc-staged-")
 
         command_args = [
             "/opt/bftools/showinf",
             "-nopix",
             "-novalid",
             "-nocore",
-            string_path,
+            f"'{path.name}'",
         ]
         process = Process(
             container="quay.io/sagebionetworks/bftools:latest",

src/dcqc/tests/grep_date_test.py

@@ -9,7 +9,6 @@ class GrepDateTest(ExternalBaseTest):
 
     def generate_process(self) -> Process:
         path = self.target.file.stage()
-        string_path = self._short_string_path(path, "dcqc-staged-")
 
         command_args = [
             "grep",
@@ -18,7 +17,7 @@ def generate_process(self) -> Process:
             "-a",  # treat input as text
             "-q",  # suppress output
             "'date|time'",  # match date or time
-            string_path,
+            f"'{path.name}'",
         ]
         process = Process(
             container="quay.io/biocontainers/coreutils:8.30--h14c3975_1000",

src/dcqc/tests/libtiff_info_test.py

@@ -9,11 +9,10 @@ class LibTiffInfoTest(ExternalBaseTest):
 
     def generate_process(self) -> Process:
         path = self.target.file.stage()
-        string_path = self._short_string_path(path, "dcqc-staged-")
 
         command_args = [
             "tiffinfo",
-            string_path,
+            f"'{path.name}'",
         ]
         process = Process(
             container="quay.io/sagebionetworks/libtiff:2.0",

src/dcqc/tests/ome_xml_schema_test.py

@@ -9,11 +9,10 @@ class OmeXmlSchemaTest(ExternalBaseTest):
 
     def generate_process(self) -> Process:
         path = self.target.file.stage()
-        string_path = self._short_string_path(path, "dcqc-staged-")
 
         command_args = [
             "/opt/bftools/xmlvalid",
-            string_path,
+            f"'{path.name}'",
         ]
         process = Process(
             container="quay.io/sagebionetworks/bftools:latest",

src/dcqc/tests/tiff_tag_306_date_time_test.py

@@ -9,12 +9,11 @@ class TiffTag306DateTimeTest(ExternalBaseTest):
 
     def generate_process(self) -> Process:
         path = self.target.file.stage()
-        string_path = self._short_string_path(path, "dcqc-staged-")
 
         command_args = [
             "tifftools",
             "dump",
-            string_path,
+            f"'{path.name}'",
             "|",
             "grep",  # pipe the output
             "-a",  # treat input as text

tests/test_tests.py

@@ -324,18 +324,3 @@ def test_that_paired_fastq_parity_test_correctly_handles_compressed_fastq_files(
     test = tests.PairedFastqParityTest(target)
     test_status = test.get_status()
     assert test_status == TestStatus.PASS
-
-
-def test_that_short_string_path_correctly_shortens_file_paths():
-    substring = "test-substring"
-    long_path = f"path/needs/to/be/shortened/{substring}/file.txt"
-    expected_short_path = f"'{substring}/file.txt'"
-    short_path = ExternalTestMixin._short_string_path(Path(long_path), substring)
-    assert short_path == expected_short_path
-
-
-def test_that_short_string_path_raises_valueerror_if_substring_not_in_path():
-    substring = "test-substring"
-    long_path = "path/needs/to/be/shortened/fail/file.txt"
-    with pytest.raises(ValueError):
-        ExternalTestMixin._short_string_path(Path(long_path), substring)