feat: implement file-based schema binding with project mapping #9
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate and Register JSON Schemas in Synapse | |
| on: | |
| push: | |
| tags: | |
| - 'v*' # Trigger on version tags | |
| workflow_dispatch: # Allow manual triggering | |
| inputs: | |
| version: | |
| description: 'Version to release (e.g., 1.0.0)' | |
| required: true | |
| default: '0.1.0' | |
| env: | |
| PYTHON_VERSION: '3.11' | |
| ORGANIZATION_NAME: 'HTAN2' | |
| jobs: | |
| generate-and-register-schemas: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 # Needed for version detection | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: latest | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }} | |
| - name: Install dependencies | |
| if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' | |
| run: poetry install --no-interaction --no-root | |
| - name: Determine version | |
| id: version | |
| run: | | |
| if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then | |
| echo "version=v${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT | |
| else | |
| # Extract version from git tag | |
| VERSION=${GITHUB_REF#refs/tags/} | |
| echo "version=$VERSION" >> $GITHUB_OUTPUT | |
| fi | |
| echo "Version: ${{ steps.version.outputs.version }}" | |
| - name: Generate JSON Schemas | |
| run: | | |
| # Create output directory | |
| mkdir -p generated-schemas | |
| # Generate schemas for individual Clinical domains | |
| echo "Generating Clinical domain schemas..." | |
| # Demographics | |
| poetry run python scripts/linkml_to_flat_synapse_jsonschema.py \ | |
| "modules/Clinical/domains/demographics.yaml" \ | |
| --class-name "Demographics" \ | |
| --output "HTAN.Demographics-${{ steps.version.outputs.version }}-schema.json" | |
| # Diagnosis | |
| poetry run python scripts/linkml_to_flat_synapse_jsonschema.py \ | |
| "modules/Clinical/domains/diagnosis.yaml" \ | |
| --class-name "Diagnosis" \ | |
| --output "HTAN.Diagnosis-${{ steps.version.outputs.version }}-schema.json" | |
| # Therapy | |
| poetry run python scripts/linkml_to_flat_synapse_jsonschema.py \ | |
| "modules/Clinical/domains/therapy.yaml" \ | |
| --class-name "Therapy" \ | |
| --output "HTAN.Therapy-${{ steps.version.outputs.version }}-schema.json" | |
| # FollowUp | |
| poetry run python scripts/linkml_to_flat_synapse_jsonschema.py \ | |
| "modules/Clinical/domains/followup.yaml" \ | |
| --class-name "FollowUp" \ | |
| --output "HTAN.FollowUp-${{ steps.version.outputs.version }}-schema.json" | |
| # MolecularTest | |
| poetry run python scripts/linkml_to_flat_synapse_jsonschema.py \ | |
| "modules/Clinical/domains/molecular.yaml" \ | |
| --class-name "MolecularTest" \ | |
| --output "HTAN.MolecularTest-${{ steps.version.outputs.version }}-schema.json" | |
| # Exposure | |
| poetry run python scripts/linkml_to_flat_synapse_jsonschema.py \ | |
| "modules/Clinical/domains/exposure.yaml" \ | |
| --class-name "Exposure" \ | |
| --output "HTAN.Exposure-${{ steps.version.outputs.version }}-schema.json" | |
| # FamilyHistory | |
| poetry run python scripts/linkml_to_flat_synapse_jsonschema.py \ | |
| "modules/Clinical/domains/family_history.yaml" \ | |
| --class-name "FamilyHistory" \ | |
| --output "HTAN.FamilyHistory-${{ steps.version.outputs.version }}-schema.json" | |
| # VitalStatus | |
| poetry run python scripts/linkml_to_flat_synapse_jsonschema.py \ | |
| "modules/Clinical/domains/vital_status.yaml" \ | |
| --class-name "VitalStatus" \ | |
| --output "HTAN.VitalStatus-${{ steps.version.outputs.version }}-schema.json" | |
| - name: Register schemas in Synapse | |
| env: | |
| SYNAPSE_USERNAME: ${{ secrets.SYNAPSE_USERNAME }} | |
| SYNAPSE_AUTH_TOKEN: ${{ secrets.SYNAPSE_AUTH_TOKEN }} | |
| run: | | |
| # Register each generated schema | |
| for schema_file in JSON_Schemas/*.json; do | |
| if [ -f "$schema_file" ]; then | |
| schema_name=$(basename "$schema_file" .json) | |
| echo "Processing schema: $schema_name" | |
| poetry run python scripts/synapse_json_schema_bind.py \ | |
| -p "$schema_file" \ | |
| -n "${{ env.ORGANIZATION_NAME }}" \ | |
| --no_bind | |
| fi | |
| done | |
| - name: Bind file-based schemas to project folders | |
| env: | |
| SYNAPSE_USERNAME: ${{ secrets.SYNAPSE_USERNAME }} | |
| SYNAPSE_AUTH_TOKEN: ${{ secrets.SYNAPSE_AUTH_TOKEN }} | |
| run: | | |
| # For now, test with just one project (HTAN2_Ovarian) and WES Level 1 schema | |
| # This will create a WES_Level_1 subfolder and bind the schema with a fileview | |
| # Find WES Level 1 schema | |
| WES_SCHEMA=$(find JSON_Schemas -name "*WES*Level*1*" -o -name "*level_1*" | head -1) | |
| if [ -n "$WES_SCHEMA" ]; then | |
| echo "Found WES Level 1 schema: $WES_SCHEMA" | |
| echo "Binding to HTAN2_Ovarian project..." | |
| poetry run python scripts/bind_file_based_schemas.py \ | |
| --schema-file "$WES_SCHEMA" \ | |
| --project-name "HTAN2_Ovarian" \ | |
| --subfolder-name "WES_Level_1" | |
| else | |
| echo "No WES Level 1 schema found, skipping file-based binding" | |
| fi | |
| # TODO: Later expand to bind all file-based schemas to all projects | |
| # for project in HTAN2_Ovarian HTAN2_Glioma HTAN2_Gastric; do | |
| # poetry run python scripts/bind_file_based_schemas.py \ | |
| # --schema-file "$WES_SCHEMA" \ | |
| # --project-name "$project" \ | |
| # --subfolder-name "WES_Level_1" | |
| # done | |
| - name: Upload generated schemas as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: json-schemas-${{ steps.version.outputs.version }} | |
| path: JSON_Schemas/ | |
| retention-days: 30 | |
| - name: Create Release | |
| if: startsWith(github.ref, 'refs/tags/') | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh release create ${{ github.ref_name }} \ | |
| --title "Release ${{ steps.version.outputs.version }}" \ | |
| --notes "## JSON Schema Release ${{ steps.version.outputs.version }} | |
| This release includes updated JSON schemas for the HTAN2 data model. | |
| ### Generated Schemas: | |
| - Demographics Schema | |
| - Diagnosis Schema | |
| - Therapy Schema | |
| - FollowUp Schema | |
| - MolecularTest Schema | |
| - Exposure Schema | |
| - FamilyHistory Schema | |
| - VitalStatus Schema | |
| All schemas have been registered in Synapse under the ${{ env.ORGANIZATION_NAME }} organization." | |
| - name: Upload schemas as release assets | |
| if: startsWith(github.ref, 'refs/tags/') | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Upload each schema file as a release asset | |
| for schema_file in JSON_Schemas/*.json; do | |
| if [ -f "$schema_file" ]; then | |
| echo "Uploading $schema_file as release asset..." | |
| gh release upload ${{ github.ref_name }} "$schema_file" | |
| fi | |
| done |