Merge pull request #990 from Sage-Bionetworks/develop

Schematic Release v22.10.3

Author: linglp

api/routes.py

@@ -421,7 +421,10 @@ def get_viz_tangled_tree_text(schema_url, figure_type, text_format):
     return text_df
 
 def get_viz_tangled_tree_layers(schema_url, figure_type):
-  
+
+    # call config_handler()
+    config_handler()
+
     temp_path_to_jsonld = get_temp_jsonld(schema_url)
 
     # Initialize Tangled Tree

run_api.py

@@ -15,4 +15,4 @@
     # Launch app
     app = create_app()
     CORS(app, resources={r"*": {"origins": "*"}})
-    app.run(port=3001, debug=True)
+    app.run(host=host, port=port, debug=True)

schematic/store/synapse.py

@@ -138,7 +138,7 @@ def wrapper(*args, **kwargs):
                 return method(*args, **kwargs)
             except(SynapseHTTPError) as ex:
                 str_message = str(ex).replace("\n","")
-                if 'missing' in str_message:
+                if 'trash' in str_message or 'does not exist' in str_message:
                     logging.warning(str_message)
                     return None
                 else:
@@ -717,16 +717,16 @@ def get_synapse_table(self, synapse_id: str) -> Tuple[pd.DataFrame, CsvFileTable
 
         return df, results
 
-    def _get_tables(self) -> List[Table]:
-        project = self.syn.get(self.project_scope[0])
+    def _get_tables(self, datasetId: str = None) -> List[Table]:
+        project = self.syn.get(self.getDatasetProject(datasetId))
         return list(self.syn.getChildren(project, includeTypes=["table"]))
 
-    def get_table_info(self) -> List[str]:
+    def get_table_info(self, datasetId: str = None) -> List[str]:
         """Gets the names of the tables in the schema
         Returns:
             list[str]: A list of table names
         """
-        tables = self._get_tables()
+        tables = self._get_tables(datasetId)
         if tables:
             return {table["name"]: table["id"] for table in tables}
         else: 
@@ -735,7 +735,7 @@ def get_table_info(self) -> List[str]:
     @missing_entity_handler
     def upload_format_manifest_table(self, se, manifest, datasetId, table_name, restrict, useSchemaLabel):
         # Rename the manifest columns to display names to match fileview
-        table_info = self.get_table_info()
+        table_info = self.get_table_info(datasetId)
 
         blacklist_chars = ['(', ')', '.', ' ']
         manifest_columns = manifest.columns.tolist()

schematic/visualization/attributes_explorer.py

@@ -4,12 +4,10 @@
 import numpy as np
 import os
 import pandas as pd
-import resource
 from typing import Any, Dict, Optional, Text, List
 
 from schematic.schemas import SchemaGenerator
 from schematic.utils.io_utils import load_json
-from schematic import CONFIG
 
 logger = logging.getLogger(__name__)
 
@@ -114,7 +112,16 @@ def parse_attributes(self, save_file=True):
                             # Capitalize attribute if it begins with a lowercase letter, for aesthetics.
                             if attribute[0].islower():
                                 attribute = attribute.capitalize()
-                            conditional_statement = f'{attribute} -is- "{value[0]}"'
+
+                            # Remove "Type" (i.e. turn "Biospecimen Type" to "Biospcimen")
+                            if "Type" in attribute: 
+                                attribute = attribute.split(" ")[0]
+                            
+                            # Remove "Type" (i.e. turn "Tissue Type" to "Tissue")
+                            if "Type" in value[0]:
+                                value[0] = value[0].split(" ")[0]
+
+                            conditional_statement = f'{attribute} is "{value[0]}"'
                             if conditional_statement not in data_dict[key]['Conditional Requirements']:
                                 data_dict[key]['Cond_Req'] = True
                                 data_dict[key]['Conditional Requirements'].extend([conditional_statement])
@@ -123,9 +130,23 @@ def parse_attributes(self, save_file=True):
                             f"There is an error getting conditional requirements related "
                             "to the attribute: {key}. The error is likely caused by naming inconsistencies (e.g. uppercase, camelcase, ...)"
                         )
+
             for key, value in data_dict.items():
                 if 'Conditional Requirements' in value.keys():
-                    data_dict[key]['Conditional Requirements'] = ' || '.join(data_dict[key]['Conditional Requirements'])
+
+                    ## reformat conditional requirement 
+
+                    # get all attributes 
+                    attr_lst = [i.split(" is ")[-1] for i in data_dict[key]['Conditional Requirements']]
+                    
+                    # join a list of attributes by using OR 
+                    attr_str = " OR ".join(attr_lst)
+
+                    # reformat the conditional requirement 
+                    component_name = data_dict[key]['Conditional Requirements'][0].split(' is ')[0]
+                    conditional_statement_str = f" If {component_name} is {attr_str} then {key} is required"
+
+                    data_dict[key]['Conditional Requirements'] = conditional_statement_str
             df = pd.DataFrame(data_dict)
             df = df.T
             cols = ['Attribute', 'Label', 'Description', 'Required', 'Cond_Req', 'Valid Values', 'Conditional Requirements', 'Validation Rules', 'Component']

schematic/visualization/tangled_tree.py

@@ -6,7 +6,6 @@
 import os
 from os import path
 import pandas as pd
-from pathlib import Path
 
 # allows specifying explicit variable types
 from typing import Any, Dict, Optional, Text, List
@@ -159,6 +158,19 @@ def get_topological_generations(self):
 
         return topological_gen, nodes, edges, subg
 
+    def remove_unwanted_characters_from_conditional_statement(self, cond_req: str) -> str:
+        '''Remove unwanted characters from conditional statement
+        Example of conditional requirement: If File Format IS "BAM" OR "CRAM" OR "CSV/TSV" then Genome Build is required
+        Example output: File Format IS "BAM" OR "CRAM" OR "CSV/TSV"
+        '''
+        if "then" in cond_req:
+            # remove everything after "then"
+            cond_req_new = cond_req.split('then')[0]
+
+            # remove "If" and empty space
+            cond_req = cond_req_new.replace("If", "").lstrip().rstrip()
+        return cond_req
+
     def get_ca_alias(self, conditional_requirements: list) -> dict:
         '''Get the alias for each conditional attribute. 
 
@@ -173,13 +185,22 @@ def get_ca_alias(self, conditional_requirements: list) -> dict:
                 value: attribute
         '''
         ca_alias = {}
+
+        # clean up conditional requirements
+        conditional_requirements = [self.remove_unwanted_characters_from_conditional_statement(req) for req in conditional_requirements]
+
         for i, req in enumerate(conditional_requirements):
-            req = req.split(' || ')
-            for r in req:
-                attr, ali = r.split(' -is- ')
+            if "OR" not in req:
+                attr, ali = req.split(' is ')
                 attr = "".join(attr.split())
                 ali = self.strip_double_quotes(ali)
                 ca_alias[ali] = attr
+            else:
+                attr, alias_str = req.split(' is ')
+                alias_lst = alias_str.split(' OR ')
+                for elem in alias_lst:
+                    elem = self.strip_double_quotes(elem)
+                    ca_alias[elem] = attr
         return ca_alias
 
     def gather_component_dependency_info(self, cn, attributes_df):

tests/data/test_config.yml

@@ -1,9 +1,8 @@
 definitions:
   creds_path: "../../credentials.json"
   token_pickle: "token.pickle"
-  #synapse_config: "../../.synapseConfig"
+  synapse_config: "../../.synapseConfig"
   ### Note: this key is required for people who use Synapse token authentication approach. 
-  ### If you run into errors similar to KeyError: 'synpase_config', you might want to consider adding this key. 
 
 synapse:
   master_fileview: "syn23643253"