Merge pull request #1317 from Sage-Bionetworks/develop

Schematic `v23.11.1`

Author: andrewelamb

.github/workflows/api_test.yml

@@ -15,7 +15,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     env:
-      POETRY_VERSION:  1.2.0
+      POETRY_VERSION:  1.3.0
     strategy:
       fail-fast: false
       matrix:
@@ -85,4 +85,4 @@ jobs:
         if: ${{ false == inputs.perform_benchmarking }}
         run: >
           source .venv/bin/activate;
-          pytest -m "schematic_api and not submission and not rule_benchmark"
+          pytest -m "schematic_api and not submission and not rule_benchmark"

.github/workflows/pdoc.yml

@@ -27,7 +27,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     env:
-      POETRY_VERSION:  1.2.0
+      POETRY_VERSION:  1.3.0
     strategy:
       matrix:
         python-version: ["3.9", "3.10"]
@@ -93,4 +93,4 @@ jobs:
       url: ${{ steps.deployment.outputs.page_url }}
     steps:
       - id: deployment
-        uses: actions/deploy-pages@v1
+        uses: actions/deploy-pages@v1

.github/workflows/publish.yml

@@ -9,7 +9,7 @@ jobs:
   pypi_release:
     runs-on: ubuntu-latest
     env:
-      POETRY_VERSION:  1.2.0
+      POETRY_VERSION:  1.3.0
     if: github.event_name == 'push' && contains(github.ref, 'refs/tags')
     steps:
       #----------------------------------------------

.github/workflows/test.yml

@@ -27,7 +27,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     env:
-      POETRY_VERSION:  1.2.0
+      POETRY_VERSION:  1.3.0
     strategy:
       fail-fast: false
       matrix:
@@ -127,7 +127,7 @@ jobs:
         if: ${{ contains(github.event.head_commit.message, 'runcombos') }}
         run: >
           source .venv/bin/activate;
-          pytest --cov-report=term --cov-report=html:htmlcov --cov=schematic/
+          pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov=schematic/
           -m "not (google_credentials_needed or schematic_api or table_operations)"
 
       - name: Run tests
@@ -137,7 +137,7 @@ jobs:
         if: ${{ false == contains(github.event.head_commit.message, 'runcombos') }}
         run: >
           source .venv/bin/activate;
-          pytest --cov-report=term --cov-report=html:htmlcov --cov=schematic/
+          pytest --durations=0 --cov-report=term --cov-report=html:htmlcov --cov=schematic/
           -m "not (google_credentials_needed or rule_combos or schematic_api or table_operations)"
 
       - name: Upload pytest test results

README.md

@@ -59,7 +59,7 @@ Please note we have a [code of conduct](CODE_OF_CONDUCT.md), please follow it in
 ```
 git clone https://github.com/Sage-Bionetworks/schematic.git
 ```
-2. Install `poetry` (version 1.2 or later) using either the [official installer](https://python-poetry.org/docs/#installing-with-the-official-installer) or [pipx](https://python-poetry.org/docs/#installing-with-pipx). If you have an older installation of Poetry, we recommend uninstalling it first. 
+2. Install `poetry` (version 1.3.0 or later) using either the [official installer](https://python-poetry.org/docs/#installing-with-the-official-installer) or [pipx](https://python-poetry.org/docs/#installing-with-pipx). If you have an older installation of Poetry, we recommend uninstalling it first. 
 
 3. Start the virtual environment by doing: 
 ```

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "schematicpy"
-version = "22.3.2"
+version = "23.11.1"
 description = "Package for biomedical data model and metadata ingress management"
 authors = [
     "Milen Nikolov <milen.nikolov@sagebase.org>",
@@ -54,7 +54,7 @@ pygsheets = "^2.0.4"
 PyYAML = "^6.0.0"
 rdflib = "^6.0.0"
 setuptools = "^66.0.0"
-synapseclient = "^2.7.0"
+synapseclient = "^3.1.1"
 tenacity = "^8.0.1"
 toml = "^0.10.2"
 Flask = "^2.0.0"
@@ -70,7 +70,7 @@ Flask-Cors = "^3.0.10"
 pdoc = "^12.2.0"
 dateparser = "^1.1.4"
 pandarallel = "^1.6.4"
-schematic-db = {version = "^0.0.29", extras = ["synapse"]}
+schematic-db = {version = "0.0.dev33", extras = ["synapse"]}
 pyopenssl = "^23.0.0"
 typing-extensions = "<4.6.0"
 

pyproject.toml

@@ -54,7 +54,11 @@ def __init__(
         self.creds = services_creds["creds"]
 
         # schema root
-        self.root = root
+        if root:
+            self.root = root
+        # Raise an error if no DataType has been provided
+        else:
+            raise ValueError("No DataType has been provided.")
 
         # alphabetize valid values
         self.alphabetize = alphabetize_valid_values
@@ -79,12 +83,19 @@ def __init__(
 
         # additional metadata to add to manifest
         self.additional_metadata = additional_metadata
+   
+        # Check if the class is in the schema
+        root_in_schema = self.sg.se.is_class_in_schema(self.root)
+        
+        # If the class could not be found, give a notification
+        if not root_in_schema:
+            exception_message = f"The DataType entered ({self.root}) could not be found in the data model schema. " + \
+                                "Please confirm that the datatype is in the data model and that the spelling matches the class label in the .jsonld file."
+            raise LookupError(exception_message) 
 
         # Determine whether current data type is file-based
-        is_file_based = False
-        if self.root:
-            is_file_based = "Filename" in self.sg.get_node_dependencies(self.root)
-        self.is_file_based = is_file_based
+        self.is_file_based = "Filename" in self.sg.get_node_dependencies(self.root)
+
 
     def _attribute_to_letter(self, attribute, manifest_fields):
         """Map attribute to column letter in a google sheet"""

schematic/manifest/generator.py

@@ -186,7 +186,7 @@ def get_component_requirements(
 
     # TODO: abstract validation in its own module
     def validateModelManifest(
-        self, manifestPath: str, rootNode: str, restrict_rules: bool = False, jsonSchema: str = None, project_scope: List = None,
+        self, manifestPath: str, rootNode: str, restrict_rules: bool = False, jsonSchema: str = None, project_scope: List = None, access_token: str = None,
     ) -> List[str]:
         """Check if provided annotations manifest dataframe satisfies all model requirements.
 
@@ -251,7 +251,16 @@ def validateModelManifest(
 
             return errors, warnings
 
-        errors, warnings, manifest = validate_all(self, errors, warnings, manifest, manifestPath, self.sg, jsonSchema, restrict_rules, project_scope)
+        errors, warnings, manifest = validate_all(self, 
+                                                  errors=errors, 
+                                                  warnings=warnings, 
+                                                  manifest=manifest, 
+                                                  manifestPath=manifestPath, 
+                                                  sg=self.sg, 
+                                                  jsonSchema=jsonSchema, 
+                                                  restrict_rules=restrict_rules, 
+                                                  project_scope=project_scope, 
+                                                  access_token=access_token)
         return errors, warnings
 
     def populateModelManifest(self, title, manifestPath: str, rootNode: str, return_excel = False) -> str:
@@ -328,7 +337,7 @@ def submit_metadata_manifest(
 
             # automatic JSON schema generation and validation with that JSON schema
             val_errors, val_warnings = self.validateModelManifest(
-                manifestPath=manifest_path, rootNode=validate_component, restrict_rules=restrict_rules, project_scope=project_scope,
+                manifestPath=manifest_path, rootNode=validate_component, restrict_rules=restrict_rules, project_scope=project_scope, access_token=access_token
             )
 
             # if there are no errors in validation process

schematic/models/metadata.py

@@ -27,6 +27,7 @@
                                             rule_in_rule_list,
                                             )
 
+from synapseclient.core.exceptions import SynapseNoCredentialsError
 
 logger = logging.getLogger(__name__)
 
@@ -564,13 +565,18 @@ class ValidateAttribute(object):
         - Add string length validator
     """
 
-    def get_target_manifests(target_component, project_scope: List):
+    def get_target_manifests(target_component, project_scope: List, access_token: str = None):
         t_manifest_search = perf_counter()
         target_manifest_IDs=[]
         target_dataset_IDs=[]
 
         #login
-        synStore = SynapseStorage(project_scope=project_scope)        
+        try:
+            synStore = SynapseStorage(access_token=access_token, project_scope=project_scope)        
+        except SynapseNoCredentialsError as e:
+            raise ValueError(
+                "No Synapse credentials were provided. Credentials must be provided to utilize cross-manfiest validation functionality."
+                ) from e
 
         #Get list of all projects user has access to
         projects = synStore.getStorageProjects(project_scope=project_scope)
@@ -893,7 +899,7 @@ def url_validation(self, val_rule: str, manifest_col: str, sg: SchemaGenerator,)
         return errors, warnings
 
     def cross_validation(
-        self, val_rule: str, manifest_col: pd.core.series.Series, project_scope: List, sg: SchemaGenerator,
+        self, val_rule: str, manifest_col: pd.core.series.Series, project_scope: List, sg: SchemaGenerator, access_token: str,
     ) -> List[List[str]]:
         """
         Purpose:
@@ -921,7 +927,7 @@ def cross_validation(
 
 
         #Get IDs of manifests with target component
-        synStore, target_manifest_IDs, target_dataset_IDs = ValidateAttribute.get_target_manifests(target_component,project_scope)
+        synStore, target_manifest_IDs, target_dataset_IDs = ValidateAttribute.get_target_manifests(target_component, project_scope, access_token)
 
         t_cross_manifest = perf_counter()
         #Read each manifest