Merge pull request #877 from Sage-Bionetworks/develop

Release 22.8.1

Author: linglp

.github/workflows/test.yml

@@ -83,13 +83,23 @@ jobs:
       #----------------------------------------------
       #              run test suite   
       #----------------------------------------------
-      - name: Run tests
+      - name: Run regular tests and rule combination tests
         env:
           SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}
+        if: ${{ contains(github.event.head_commit.message, 'runcombos') }}
         run: >
-          source .venv/bin/activate; 
+          source .venv/bin/activate;
           pytest --cov-report=term --cov-report=html:htmlcov --cov=schematic/
           -m "not google_credentials_needed"
+          
+      - name: Run tests
+        env:
+          SYNAPSE_ACCESS_TOKEN: ${{ secrets.SYNAPSE_ACCESS_TOKEN }}          
+        if: ${{ false == contains(github.event.head_commit.message, 'runcombos') }}
+        run: >
+          source .venv/bin/activate;
+          pytest --cov-report=term --cov-report=html:htmlcov --cov=schematic/
+          -m "not (google_credentials_needed or rule_combos)"
     
       - name: Upload pytest test results
         uses: actions/upload-artifact@v2

.gitignore

@@ -168,3 +168,5 @@ great_expectations/expectations/Manifest_test_suite.json
 
 tests/data/example.MockComponent.schema.json
 tests/data/mock_manifests/Invalid_Test_Manifest_censored.csv
+tests/data/mock_manifests/valid_test_manifest_censored.csv
+tests/data/mock_manifests/Rule_Combo_Manifest_censored.csv

RELEASE.md

@@ -5,6 +5,25 @@ Once the code has been merged into the `develop` branch on this repo, there are
 - You should create a GitHub [tag](https://git-scm.com/book/en/v2/Git-Basics-Tagging), with the appropriate version number. Typically, from `v21.06` onwards all tags are created following the Linux Ubuntu versioning convention which is the `YY.MM` format where `Y` is the year and `M` is the month of that year when that release was created.
 - You should push the package to [PyPI](https://pypi.org/). Schematic is on PyPI as [schematicpy](https://pypi.org/project/schematicpy/). You can go through the following two sections for that.
 
+## Steps 
+- Step 1: Open a pull request to merge the `main` branch to the `develop` branch: 
+    - Click on the "Pull Request" tab on Github
+    - Click on the green button "New pull request"
+    - Select `main` as "base" and `develop` as "compare"
+    - Resolve conflicts
+    - Link all PRs and/or issues that are included in the release (example [here](https://github.com/Sage-Bionetworks/data_curator/pull/357))
+
+- Step 2: Create a tag
+`git tag <tag version> -m '<message>'`
+
+- Step 3: Push the tag to main branch (this step assumes that you have checked out the main branch locally)
+`git push origin <tag version>`
+
+This should trigger the PYPI release workflow and release a new version of schematic to PYPI. You could check by cliking on the GitHub action log and login to your PYPI account (and select project `schematicpy`. Please note that you have to obtain access to `schematicpy` to be able to see it.)
+
+>Note: if you make some mistakes and would like to delete a tag, try the following commands: `git push --delete origin <version number>` for deleting a tag remotely and `git tag -d <version number>` for deleting a tag locally. 
+
+
 ## Release to Test PyPI _(optional)_
 
 The purpose of this section is to verify that the package looks and works as intended, by viewing it on [Test PyPI](https://test.pypi.org/) and installing the test version in a separate virtual environment.

api/openapi/api.yaml

@@ -350,6 +350,43 @@ paths:
           description: Check schematic log
       tags:
         - Manifest Operations
+  /get/datatype/manifest:
+    get:
+      summary: Retrieve asset view table as a dataframe.
+      description: Retrieve asset view table as a dataframe.
+      operationId: api.routes.get_manifest_datatype
+      parameters:
+        - in: query
+          name: input_token
+          schema:
+            type: string
+            nullable: false
+          description: Token
+          example: Token
+          required: true
+        - in: query
+          name: asset_view
+          schema:
+            type: string
+            nullable: false
+          description: ID of view listing all project data assets. For example, for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project.(i.e. master_fileview in config.yml)
+          example: syn23643253
+          required: true
+        - in: query
+          name: manifest_id
+          schema:
+            type: string
+            nullable: false
+          description: Manifest ID
+          example: syn27600110
+          required: true
+      responses:
+        "200":
+          description: A list of json
+        "500":
+          description: Check schematic log. 
+      tags:
+        - Manifest Operations
   /storage/projects:
     get:
       summary: Get all storage projects the current user has access to

api/routes.py

@@ -18,6 +18,7 @@
 from schematic.store.synapse import SynapseStorage
 import pandas as pd
 import json
+from schematic.utils.df_utils import load_df
 
 
 # def before_request(var1, var2):
@@ -169,11 +170,13 @@ def validate_manifest_route(schema_url, data_type):
         inputMModelLocation=jsonld, inputMModelLocationType="local"
     )
 
-    errors = metadata_model.validateModelManifest(
+    errors, warnings = metadata_model.validateModelManifest(
         manifestPath=temp_path, rootNode=data_type
     )
+    
+    res_dict = {"errors": errors, "warnings": warnings}
 
-    return errors
+    return res_dict
 
 
 def submit_manifest_route(schema_url, manifest_record_type=None):
@@ -199,7 +202,7 @@ def submit_manifest_route(schema_url, manifest_record_type=None):
         validate_component = data_type
 
     manifest_id = metadata_model.submit_metadata_manifest(
-        manifest_path=temp_path, dataset_id=dataset_id, validate_component=validate_component, input_token=input_token, manifest_record_type = manifest_record_type, restrict_rules = restrict_rules)
+        path_to_json_ld = schema_url, manifest_path=temp_path, dataset_id=dataset_id, validate_component=validate_component, input_token=input_token, manifest_record_type = manifest_record_type, restrict_rules = restrict_rules)
 
     return manifest_id
 
@@ -318,4 +321,19 @@ def get_project_manifests(input_token, project_id, asset_view):
     lst_manifest = store.getProjectManifests(projectId=project_id)
 
     return lst_manifest
-    
+
+def get_manifest_datatype(input_token, manifest_id, asset_view):
+    # use the default asset view from config
+    config_handler(asset_view=asset_view)
+
+    # use Synapse Storage
+    store = SynapseStorage(input_token=input_token)
+
+    # get data types of an existing manifest
+    manifest_dtypes_dict= store.getDataTypeFromManifest(manifest_id)
+
+
+    return manifest_dtypes_dict
+
+
+