feat: move schemas and iwc workflows script to python package (#540)

.github/workflows/run-checks.yml

@@ -33,7 +33,7 @@ jobs:
         run: pip install -r ./catalog/build/py/requirements.txt
       - name: Run linkml-lint
         # Run linting on the LinkML schemas, to enforce conventions such as in naming, and to catch simple errors.
-        run: linkml-lint ./catalog/schema --validate --verbose
+        run: npm run lint-schema
       - name: Test LinkML Python generation
         # Generate Python code from the main LinkML schemas, discarding the output; this will catch more subtle errors such as references to nonexistent elements.
         run: npm run test-gen-python

README.md

@@ -75,7 +75,7 @@ These values will be substituted with assembly-specific values at runtime.
 
 ## Editing the LinkML schemas
 
-If the LinkML schemas in `catalog/schema` are edited, the derived JSON schemas and TypeScript definitions should be
+If the LinkML schemas in `catalog/py_package/catalog_build/schema` are edited, the derived JSON schemas and TypeScript definitions should be
 updated
 as follows:
 

catalog/README.md

@@ -7,5 +7,6 @@ This directory provides the catalog data (information on genome assemblies, orga
   - `py` - Python scripts.
   - `ts` - Typescript scripts.
 - `output` - JSON files output by the catalog build process, to be consumed by the app.
-- `schema` - LinkML schemas for source files.
+- `py_package` - Python package used to share catalog features, such as the schemas and build process, with other projects.
+- `schema` - Schema-related scripts and derived models.
 - `source` - YAML files providing data used as input for building the catalog.

catalog/build/py/build-files-from-ncbi.py → catalog/build/py/build_files_from_ncbi.py

@@ -1,4 +1,4 @@
-from package.catalog_build import build_files
+from ...py_package.catalog_build import build_files
 
 ASSEMBLIES_PATH = "catalog/source/assemblies.yml"
 ORGANISMS_PATH = "catalog/source/organisms.yml"

catalog/build/py/generated_schema/schema.py → catalog/py_package/catalog_build/generated_schema/schema.py

@@ -44,9 +44,9 @@ def __contains__(self, key: str) -> bool:
 
 linkml_meta = LinkMLMeta(
     {
-        "default_prefix": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/schema.yaml#",
+        "default_prefix": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/schema.yaml#",
         "description": "Combined source data schemas.",
-        "id": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/schema.yaml#",
+        "id": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/schema.yaml#",
         "imports": [
             "./assemblies",
             "./organisms",
@@ -61,7 +61,7 @@ def __contains__(self, key: str) -> bool:
                 "prefix_reference": "https://w3id.org/linkml/",
             }
         },
-        "source_file": "./catalog/schema/schema.yaml",
+        "source_file": "/Users/hunter/git-repos/brc-analytics/catalog/py_package/catalog_build/schema_utils/../schema/schema.yaml",
     }
 )
 
@@ -146,7 +146,7 @@ class Assemblies(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/assemblies.yaml#",
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/assemblies.yaml#",
             "tree_root": True,
         }
     )
@@ -167,7 +167,7 @@ class Assembly(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/assemblies.yaml#"
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/assemblies.yaml#"
         }
     )
 
@@ -187,7 +187,7 @@ class Organisms(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/organisms.yaml#",
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/organisms.yaml#",
             "tree_root": True,
         }
     )
@@ -208,7 +208,7 @@ class Organism(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/organisms.yaml#"
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/organisms.yaml#"
         }
     )
 
@@ -238,7 +238,7 @@ class Outbreaks(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/outbreaks.yaml#",
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/outbreaks.yaml#",
             "tree_root": True,
         }
     )
@@ -259,7 +259,7 @@ class Outbreak(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/outbreaks.yaml#"
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/outbreaks.yaml#"
         }
     )
 
@@ -333,7 +333,7 @@ class OutbreakResource(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/outbreaks.yaml#"
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/outbreaks.yaml#"
         }
     )
 
@@ -370,7 +370,7 @@ class MarkdownFileReference(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/outbreaks.yaml#"
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/outbreaks.yaml#"
         }
     )
 
@@ -402,7 +402,7 @@ class WorkflowCategories(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/workflow_categories.yaml#",
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/workflow_categories.yaml#",
             "tree_root": True,
         }
     )
@@ -426,7 +426,7 @@ class WorkflowCategory(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/workflow_categories.yaml#"
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/workflow_categories.yaml#"
         }
     )
 
@@ -476,7 +476,7 @@ class Workflows(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/workflows.yaml#",
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/workflows.yaml#",
             "tree_root": True,
         }
     )
@@ -497,7 +497,7 @@ class Workflow(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/workflows.yaml#"
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/workflows.yaml#"
         }
     )
 
@@ -569,7 +569,7 @@ class WorkflowParameter(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/workflows.yaml#"
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/workflows.yaml#"
         }
     )
 
@@ -610,7 +610,7 @@ class WorkflowUrlSpec(ConfiguredBaseModel):
 
     linkml_meta: ClassVar[LinkMLMeta] = LinkMLMeta(
         {
-            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/workflows.yaml#"
+            "from_schema": "https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/workflows.yaml#"
         }
     )
 

catalog/build/py/iwc_manifest_to_workflows_yaml.py → catalog/py_package/catalog_build/iwc_manifest_to_workflows_yaml.py

@@ -6,7 +6,8 @@
 
 import requests
 import yaml
-from generated_schema.schema import (
+
+from .generated_schema.schema import (
     Workflow,
     WorkflowCategoryId,
     WorkflowParameter,
@@ -15,7 +16,6 @@
 )
 
 URL = "https://iwc.galaxyproject.org/workflow_manifest.json"
-WORKFLOWS_PATH = "catalog/source/workflows.yml"
 DOCKSTORE_COLLECTION_TO_CATEGORY = {
     "Variant Calling": WorkflowCategoryId.VARIANT_CALLING,
     "Transcriptomics": WorkflowCategoryId.TRANSCRIPTOMICS,
@@ -31,9 +31,9 @@
 )
 
 
-def read_existing_yaml():
-    if os.path.exists(WORKFLOWS_PATH):
-        with open(WORKFLOWS_PATH) as fh:
+def read_existing_yaml(workflows_path):
+    if os.path.exists(workflows_path):
+        with open(workflows_path) as fh:
             workflows = Workflows.model_validate(yaml.safe_load(fh)).workflows
     else:
         # start from scratch
@@ -116,8 +116,8 @@ def generate_current_workflows():
     return by_trs_id
 
 
-def merge_into_existing():
-    existing = read_existing_yaml()
+def merge_into_existing(workflows_path):
+    existing = read_existing_yaml(workflows_path)
     current = generate_current_workflows()
     merged: Dict[str, Workflow] = {}
     for versionless_trs_id, current_workflow_input in current.items():
@@ -144,8 +144,8 @@ def merge_into_existing():
     return merged
 
 
-def to_workflows_yaml(exclude_other: bool):
-    by_trs_id = merge_into_existing()
+def to_workflows_yaml(workflows_path: str, exclude_other: bool):
+    by_trs_id = merge_into_existing(workflows_path)
     # sort by trs id, should play nicer with git diffs
     sorted_workflows = list(dict(sorted(by_trs_id.items())).values())
     if exclude_other:
@@ -160,25 +160,28 @@ def to_workflows_yaml(exclude_other: bool):
             final_workflows.append(workflow)
         sorted_workflows = final_workflows
     final_workflows = sorted_workflows
-    with open(WORKFLOWS_PATH, "w") as out:
+    with open(workflows_path, "w") as out:
         yaml.safe_dump(
             Workflows(workflows=final_workflows).model_dump(exclude_none=True),
             out,
             allow_unicode=True,
             sort_keys=False,
         )
     # Turns out the YAML style prettier likes is really hard to create in python ...
-    subprocess.run(["npx", "prettier", "--write", WORKFLOWS_PATH])
+    subprocess.run(["npx", "prettier", "--write", workflows_path])
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Build workflows.yaml file from latest IWC JSON manifest."
     )
+    parser.add_argument(
+        "workflows_path", help="Path of workflows YAML file to read/write."
+    )
     parser.add_argument(
         "--exclude-other",
         action="store_true",
         help="Exclude other items from processing.",
     )
     args = parser.parse_args()
-    to_workflows_yaml(exclude_other=args.exclude_other)
+    to_workflows_yaml(args.workflows_path, exclude_other=args.exclude_other)

catalog/schema/assemblies.yaml → catalog/py_package/catalog_build/schema/assemblies.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/assemblies.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/assemblies.yaml#
 name: assemblies
 description: Schema for defining genomic assemblies available in the BRC Analytics platform.
 

catalog/schema/enums/organism_ploidy.yaml → catalog/py_package/catalog_build/schema/enums/organism_ploidy.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/enums/organism_ploidy.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/enums/organism_ploidy.yaml#
 name: enums_organism_ploidy
 
 enums:

catalog/schema/enums/outbreak_priority.yaml → catalog/py_package/catalog_build/schema/enums/outbreak_priority.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/enums/outbreak_priority.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/enums/outbreak_priority.yaml#
 name: enums_outbreak_priority
 
 enums:

catalog/schema/enums/outbreak_resource_type.yaml → catalog/py_package/catalog_build/schema/enums/outbreak_resource_type.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/enums/outbreak_resource_type.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/enums/outbreak_resource_type.yaml#
 name: enums_outbreak_resource_type
 
 enums:

catalog/schema/enums/workflow_category_id.yaml → catalog/py_package/catalog_build/schema/enums/workflow_category_id.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/enums/workflow_type.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/enums/workflow_type.yaml#
 name: enums_workflow_category_id
 description: Definition of the workflow category ID enum.
 

catalog/schema/enums/workflow_parameter_variable.yaml → catalog/py_package/catalog_build/schema/enums/workflow_parameter_variable.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/enums/workflow_parameter_variable.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/enums/workflow_parameter_variable.yaml#
 name: enums_workflow_parameter_variable
 
 enums:

catalog/schema/enums/workflow_ploidy.yaml → catalog/py_package/catalog_build/schema/enums/workflow_ploidy.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/enums/workflow_ploidy.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/enums/workflow_ploidy.yaml#
 name: enums_workflow_ploidy
 
 enums:

catalog/schema/organisms.yaml → catalog/py_package/catalog_build/schema/organisms.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/organisms.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/organisms.yaml#
 name: organisms
 description: Schema for defining source organism information used in the BRC Analytics platform.
 

catalog/schema/outbreaks.yaml → catalog/py_package/catalog_build/schema/outbreaks.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/outbreaks.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/outbreaks.yaml#
 name: outbreaks
 description: Schema for defining outbreak and pathogen information used in the BRC Analytics platform.
 

catalog/schema/schema.yaml → catalog/py_package/catalog_build/schema/schema.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/schema.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/schema.yaml#
 name: schema
 description: Combined source data schemas.
 

catalog/schema/workflow_categories.yaml → catalog/py_package/catalog_build/schema/workflow_categories.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/workflow_categories.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/workflow_categories.yaml#
 name: workflow_categories
 description: Schema for defining workflow categories used to organize Galaxy workflows in the BRC Analytics platform.
 

catalog/schema/workflows.yaml → catalog/py_package/catalog_build/schema/workflows.yaml

@@ -1,4 +1,4 @@
-id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/schema/workflows.yaml#
+id: https://github.com/galaxyproject/brc-analytics/blob/main/catalog/py_package/catalog_build/schema/workflows.yaml#
 name: workflows
 description: Schema for defining Galaxy workflows available in the BRC Analytics platform.