[SCHEMATIC-183] Use paths from file view for manifest generation

schematic/manifest/generator.py

@@ -1904,6 +1904,8 @@ def get_manifest(
         # TODO: avoid explicitly exposing Synapse store functionality
         # just instantiate a Store class and let it decide at runtime/config
         # the store type
+        # TODO: determine which parts of fileview are necessary for `get` operations
+        # and pass query parameters at object instantiation to avoid having to re-query
         if access_token:
             # for getting an existing manifest on AWS
             store = SynapseStorage(access_token=access_token)

schematic/models/validate_attribute.py

@@ -2119,7 +2119,9 @@ def filename_validation(
 
         where_clauses = []
 
-        dataset_clause = SynapseStorage.build_clause_from_dataset_id(dataset_scope)
+        dataset_clause = SynapseStorage.build_clause_from_dataset_id(
+            dataset_id=dataset_scope
+        )
         where_clauses.append(dataset_clause)
 
         self._login(

schematic/store/synapse.py

@@ -417,15 +417,28 @@ def query_fileview(
                     raise AccessCredentialsError(self.storageFileview)
 
     @staticmethod
-    def build_clause_from_dataset_id(dataset_id: Optional[str]) -> str:
+    def build_clause_from_dataset_id(
+        dataset_id: Optional[str] = None, dataset_folder_list: Optional[list] = None
+    ) -> str:
         """
         Method to build a where clause for a Synapse FileView query based on a dataset ID that can be used before an object is initialized.
         Args:
             dataset_id: Synapse ID of a dataset that should be used to limit the query
+            dataset_folder_list: List of Synapse IDs of a dataset and all its subfolders that should be used to limit the query
         Returns:
             clause for the query or an empty string if no dataset ID is provided
         """
-        return f"parentId='{dataset_id}'" if dataset_id else ""
+        # Calling this method without specifying synIDs will complete but will not scope the view
+        if (not dataset_id) and (not dataset_folder_list):
+            return ""
+
+        # This will be used to gather files under a dataset recursively with a fileview query instead of walking
+        if dataset_folder_list:
+            search_folders = ", ".join(f"'{synId}'" for synId in dataset_folder_list)
+            return f"parentId IN ({search_folders})"
+
+        # `dataset_id` should be provided when all files are stored directly under the dataset folder
+        return f"parentId='{dataset_id}'"
 
     def _build_query(
         self, columns: Optional[list] = None, where_clauses: Optional[list] = None
@@ -677,7 +690,7 @@ def getStorageDatasetsInProject(self, projectId: str) -> list[tuple[str, str]]:
     def getFilesInStorageDataset(
         self, datasetId: str, fileNames: List = None, fullpath: bool = True
     ) -> List[Tuple[str, str]]:
-        """Gets all files in a given dataset folder.
+        """Gets all files (excluding manifest files) in a given dataset folder.
 
         Args:
             datasetId: synapse ID of a storage dataset.
@@ -693,28 +706,35 @@ def getFilesInStorageDataset(
         """
         file_list = []
 
-        # Identify all folders nested under the dataset folder
-        folders = synapseutils.walk(self.syn, datasetId, includeTypes=["folder"])
-
-        # Getting the files directly under the dataset will be the beginning of the query
-        dataset_clause = f"parentId='{datasetId}' "
-
-        # The query will also be ammended to include everything containted in all the subdirectories of the dataset
-        for subfolder, _, path in folders:
-            dataset_clause += f"OR parentId='{subfolder[1]}' "
-        dataset_clause = f"({dataset_clause})"
+        # Get path to dataset folder from fileview to avoid building a new fileview and walking to determine folders and files within
+        child_path = self.storageFileviewTable.loc[
+            self.storageFileviewTable["parentId"] == datasetId, "path"
+        ][0]
+        parent = child_path.split("/")[0]
+        dataset_path = f"'{parent}/%'"
 
         # When querying, only include files to exclude entity files and subdirectories
-        where_clauses = [dataset_clause, "type='file'"]
+        where_clauses = [f"path like {dataset_path}", "type='file'"]
 
         # Requery the fileview to specifically get the files in the given dataset
         self.query_fileview(columns=["id", "path"], where_clauses=where_clauses)
 
         # Exclude manifest files
         non_manifest_files = self.storageFileviewTable.loc[
-            ~self.storageFileviewTable["path"].str.contains("manifest"), :
+            ~self.storageFileviewTable["path"].str.contains("synapse_storage_manifest"),
+            :,
         ]
 
+        # Remove all files that are not in the list of fileNames
+        if fileNames:
+            filename_regex = "|".join(fileNames)
+
+            matching_files = non_manifest_files["path"].str.contains(
+                filename_regex, case=False, regex=True
+            )
+
+            non_manifest_files = non_manifest_files.loc[matching_files, :]
+
         # Truncate path if necessary
         if not fullpath:
             non_manifest_files.path = non_manifest_files.path.apply(os.path.basename)

schematic/utils/cli_utils.py

@@ -8,7 +8,7 @@
 from functools import reduce
 from typing import Any, Mapping, Optional, Sequence, Union
 
-from schematic.utils.general import syn_id_regex
+from schematic.utils.general import SYN_ID_REGEX
 
 logger = logging.getLogger(__name__)
 
@@ -70,7 +70,7 @@ def parse_syn_ids(
     if not syn_ids:
         return None
 
-    project_regex = re.compile(syn_id_regex())
+    project_regex = re.compile(SYN_ID_REGEX)
     valid = project_regex.fullmatch(syn_ids)
 
     if not valid:

schematic/utils/general.py

@@ -24,6 +24,8 @@
 
 T = TypeVar("T")
 
+SYN_ID_REGEX = r"(syn\d+\,?)+"
+
 
 def find_duplicates(_list: list[T]) -> set[T]:
     """Find duplicate items in a list"""
@@ -332,14 +334,3 @@ def normalize_path(path: str, parent_folder: str) -> str:
     if not os.path.isabs(path):
         path = os.path.join(parent_folder, path)
     return os.path.normpath(path)
-
-
-def syn_id_regex() -> str:
-    """
-    Regex pattern to match synapse ids
-    Args:
-        None
-    Returns:
-        str: synID regex pattern
-    """
-    return r"(syn\d+\,?)+"

schematic_api/api/openapi/api.yaml

@@ -692,8 +692,8 @@ paths:
         - Synapse Storage
   /storage/dataset/files:
     get:
-      summary: Get all files in a given dataset folder
-      description: Get all files in a given dataset folder
+      summary: Get all files (excluding manifest files) in a given dataset folder
+      description: Get all files (excluding manifest files) in a given dataset folder
       operationId: schematic_api.api.routes.get_files_storage_dataset
       security:
         - access_token: []

tests/integration/test_commands.py

@@ -4,14 +4,14 @@
 import uuid
 from io import BytesIO
 
+import numpy as np
+import pandas as pd
 import pytest
 import requests
-from openpyxl import load_workbook
 from click.testing import CliRunner
-import pandas as pd
-import numpy as np
+from openpyxl import load_workbook
 
-from schematic.configuration.configuration import Configuration, CONFIG
+from schematic.configuration.configuration import CONFIG, Configuration
 from schematic.manifest.commands import manifest
 from schematic.models.commands import model
 from tests.conftest import ConfigurationForTesting
@@ -665,18 +665,18 @@ def test_generate_bulk_rna_google_sheet_manifest(
             # Reset config to it's default values
             CONFIG.load_config("config_example.yml")
 
-        assert result.output.split("\n")[7] == (
+        assert result.output.split("\n")[8] == (
             "Find the manifest template using this Google Sheet URL:"
         )
-        assert result.output.split("\n")[8].startswith(
+        assert result.output.split("\n")[9].startswith(
             "https://docs.google.com/spreadsheets/d/"
         )
-        assert result.output.split("\n")[9] == (
+        assert result.output.split("\n")[10] == (
             "Find the manifest template using this CSV file path: "
             "./CLI_gs_bulk_rna.csv"
         )
 
-        google_sheet_url = result.output.split("\n")[8]
+        google_sheet_url = result.output.split("\n")[9]
 
         # Download the Google Sheets content as an Excel file and load into openpyxl
         export_url = f"{google_sheet_url}/export?format=xlsx"
@@ -908,18 +908,18 @@ def test_generate_bulk_rna_google_sheet_manifest_with_annotations(
                 os.remove("tests/data/example.BulkRNA-seqAssay.schema.json")
             os.remove("./CLI_gs_bulk_rna_annos.csv")
 
-        assert result.output.split("\n")[10] == (
+        assert result.output.split("\n")[11] == (
             "Find the manifest template using this Google Sheet URL:"
         )
-        assert result.output.split("\n")[11].startswith(
+        assert result.output.split("\n")[12].startswith(
             "https://docs.google.com/spreadsheets/d/"
         )
-        assert result.output.split("\n")[12] == (
+        assert result.output.split("\n")[13] == (
             "Find the manifest template using this CSV file path: "
             "./CLI_gs_bulk_rna_annos.csv"
         )
 
-        google_sheet_url = result.output.split("\n")[11]
+        google_sheet_url = result.output.split("\n")[12]
 
         # Download the Google Sheets content as an Excel file and load into openpyxl
         export_url = f"{google_sheet_url}/export?format=xlsx"