Merge pull request #1454 from Sage-Bionetworks/develop

Schematic 24.7.1 Release

Author: andrewelamb

.github/workflows/test.yml

@@ -90,7 +90,7 @@ jobs:
         run: |
           # ran only on certain files for now
           # add here when checked
-          poetry run black schematic --check
+          poetry run black schematic tests schematic_api --check
 
       #----------------------------------------------
       #             type checking/enforcement
@@ -101,7 +101,7 @@ jobs:
           # add here when checked
           # poetry run mypy --install-types --non-interactive
           # add here when enforced
-          poetry run mypy --disallow-untyped-defs --install-types --non-interactive schematic/configuration/*.py schematic/exceptions.py schematic/help.py schematic/loader.py schematic/version.py schematic/visualization schematic/utils/
+          poetry run mypy --disallow-untyped-defs --install-types --non-interactive schematic/schemas/ schematic/configuration/ schematic/exceptions.py schematic/help.py schematic/loader.py schematic/version.py schematic/visualization schematic/utils/
 
       #----------------------------------------------
       #             linting

.pre-commit-config.yaml

@@ -10,4 +10,12 @@ repos:
         # pre-commit's default_language_version, see
         # https://pre-commit.com/#top_level-default_language_version
         language_version: python3.10
-        files: schematic/
+        files: ^(tests|schematic|schematic_api)/
+
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)
+        files: ^(tests|schematic|schematic_api)/
+        args: ["--profile", "black", "--filter-files"]

README.md

@@ -141,8 +141,6 @@ model:
 
 # This section is for using google sheets with Schematic
 google_sheets:
-  # The Synapse id of the Google service account credentials.
-  service_acct_creds_synapse_id: "syn25171627"
   # Path to the synapse config file, either absolute or relative to this file
   service_acct_creds: "schematic_service_account_creds.json"
   # When doing google sheet validation (regex match) with the validation rules.

config_example.yml

@@ -35,8 +35,6 @@ model:
 
 # This section is for using google sheets with Schematic
 google_sheets:
-  # The Synapse id of the Google service account credentials.
-  service_acct_creds_synapse_id: "syn25171627"
   # Path to the synapse config file, either absolute or relative to this file
   service_acct_creds: "schematic_service_account_creds.json"
   # When doing google sheet validation (regex match) with the validation rules.

poetry.lock

@@ -47,9 +47,9 @@ graphviz = "^0.20.0"
 inflection = "^0.5.1"
 jsonschema = "^4.0.0"
 networkx = ">=2.2.8"
-numpy = "^1.21.1"
+numpy = "^1.26.4"
 oauth2client = "^4.1.0"  # Specified because of bug in version ^4.0.0
-pandas = "^2.0.0"
+pandas = "^2.2.2"
 pygsheets = "^2.0.4"
 PyYAML = "^6.0.0"
 rdflib = "^6.0.0"
@@ -67,16 +67,18 @@ dateparser = "^1.1.4"
 pandarallel = "^1.6.4"
 schematic-db = {version = "0.0.41", extras = ["synapse"]}
 pyopenssl = {version = "^23.0.0", optional = true}
-typing-extensions = "<4.6.0"
 dataclasses-json = "^0.6.1"
+pydantic = "^1.10.4"
 connexion = {extras = ["swagger-ui"], version = "^2.8.0", optional = true}
 Flask = {version = "2.1.3", optional = true}
 Flask-Cors = {version = "^3.0.10", optional = true}
 uWSGI = {version = "^2.0.21", optional = true}
 Jinja2 = {version = ">2.11.3", optional = true}
+jaeger-client = {version = "^4.8.0", optional = true}
+flask-opentracing = {version="^2.0.0", optional = true}
 
 [tool.poetry.extras]
-api = ["connexion", "Flask", "Flask-Cors", "Jinja2", "pyopenssl"]
+api = ["connexion", "Flask", "Flask-Cors", "Jinja2", "pyopenssl", "jaeger-client", "flask-opentracing"]
 aws = ["uWSGI"]
 
 

pyproject.toml

@@ -164,14 +164,6 @@ def model_location(self) -> str:
         """
         return self._model_config.location
 
-    @property
-    def service_account_credentials_synapse_id(self) -> str:
-        """
-        Returns:
-            str: The Synapse id of the Google service account credentials.
-        """
-        return self._google_sheets_config.service_acct_creds_synapse_id
-
     @property
     def service_account_credentials_path(self) -> str:
         """

schematic/configuration/configuration.py

@@ -124,12 +124,10 @@ class GoogleSheetsConfig:
     strict_validation: When doing google sheet validation (regex match) with the validation rules.
       True is alerting the user and not allowing entry of bad values.
       False is warning but allowing the entry on to the sheet.
-    service_acct_creds_synapse_id: The Synapse id of the Google service account credentials.
     service_acct_creds: Path to the Google service account credentials,
      either absolute or relative to this file
     """
 
-    service_acct_creds_synapse_id: str = "syn25171627"
     service_acct_creds: str = "schematic_service_account_creds.json"
     strict_validation: bool = True
 
@@ -151,7 +149,6 @@ def validate_string_is_not_empty(cls, value: str) -> str:
             raise ValueError(f"{value} is an empty string")
         return value
 
-    @validator("service_acct_creds_synapse_id")
     @classmethod
     def validate_synapse_id(cls, value: str) -> str:
         """Check if string is a valid synapse id

schematic/configuration/dataclasses.py

@@ -32,8 +32,10 @@
 from schematic.configuration.configuration import CONFIG
 from schematic.utils.google_api_utils import export_manifest_drive_service
 
+from opentelemetry import trace
 
 logger = logging.getLogger(__name__)
+tracer = trace.get_tracer("Schematic")
 
 
 class ManifestGenerator(object):
@@ -1289,6 +1291,7 @@ def _gather_all_fields(self, fields, json_schema):
         )
         return required_metadata_fields
 
+    @tracer.start_as_current_span("ManifestGenerator::get_empty_manifest")
     def get_empty_manifest(
         self,
         strict: Optional[bool],
@@ -1334,6 +1337,7 @@ def _get_missing_columns(self, headers_1: list, headers_2: list) -> list:
         """
         return set(headers_1) - set(headers_2)
 
+    @tracer.start_as_current_span("ManifestGenerator::set_dataframe_by_url")
     def set_dataframe_by_url(
         self,
         manifest_url: str,
@@ -1425,6 +1429,7 @@ def map_annotation_names_to_display_names(
         # Use the above dictionary to rename columns in question
         return annotations.rename(columns=label_map)
 
+    @tracer.start_as_current_span("ManifestGenerator::get_manifest_with_annotations")
     def get_manifest_with_annotations(
         self, annotations: pd.DataFrame, strict: Optional[bool] = None
     ) -> Tuple[ps.Spreadsheet, pd.DataFrame]:
@@ -1465,6 +1470,7 @@ def get_manifest_with_annotations(
 
         return manifest_url, manifest_df
 
+    @tracer.start_as_current_span("ManifestGenerator::export_sheet_to_excel")
     def export_sheet_to_excel(
         self, title: str = None, manifest_url: str = None, output_location: str = None
     ) -> str:
@@ -1514,6 +1520,7 @@ def export_sheet_to_excel(
 
         return output_excel_file_path
 
+    @tracer.start_as_current_span("ManifestGenerator::_handle_output_format_logic")
     def _handle_output_format_logic(
         self,
         output_format: str = None,
@@ -1570,6 +1577,7 @@ def _handle_output_format_logic(
             return dataframe
 
     @staticmethod
+    @tracer.start_as_current_span("ManifestGenerator::create_single_manifest")
     def create_single_manifest(
         path_to_data_model: str,
         graph_data_model: nx.MultiDiGraph,
@@ -1623,6 +1631,7 @@ def create_single_manifest(
         return result
 
     @staticmethod
+    @tracer.start_as_current_span("ManifestGenerator::create_manifests")
     def create_manifests(
         path_to_data_model: str,
         data_types: list,
@@ -1751,6 +1760,7 @@ def create_manifests(
 
         return all_results
 
+    @tracer.start_as_current_span("ManifestGenerator::get_manifest")
     def get_manifest(
         self,
         dataset_id: str = None,
@@ -1998,6 +2008,9 @@ def _format_new_excel_column(self, worksheet, new_column_index: int, col: str):
         )
         return worksheet
 
+    @tracer.start_as_current_span(
+        "ManifestGenerator::populate_existing_excel_spreadsheet"
+    )
     def populate_existing_excel_spreadsheet(
         self, existing_excel_path: str = None, additional_df: pd.DataFrame = None
     ):

schematic/manifest/generator.py

@@ -19,9 +19,12 @@
 from schematic.utils.df_utils import load_df
 
 from schematic.models.validate_manifest import validate_all
+from opentelemetry import trace
 
 logger = logging.getLogger(__name__)
 
+tracer = trace.get_tracer("Schematic")
+
 
 class MetadataModel(object):
     """Metadata model wrapper around schema.org specification graph.
@@ -317,6 +320,7 @@ def populateModelManifest(
             manifestPath, emptyManifestURL, return_excel=return_excel, title=title
         )
 
+    @tracer.start_as_current_span("MetadataModel::submit_metadata_manifest")
     def submit_metadata_manifest(  # pylint: disable=too-many-arguments, too-many-locals
         self,
         manifest_path: str,