Add Validation Layer #13
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Schema Submission | |
| on: | |
| issues: | |
| types: [opened] | |
| jobs: | |
| ingest: | |
| if: startsWith(github.event.issue.title, 'schema-submission:') | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| issues: write | |
| steps: | |
| # ----------------------------------------------------------------------- | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.12" | |
| # ----------------------------------------------------------------------- | |
| - name: Cache pip | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} | |
| restore-keys: ${{ runner.os }}-pip- | |
| - name: Cache sentence-transformers model | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/huggingface/hub | |
| key: hf-minilm-v2 | |
| restore-keys: hf- | |
| # Embeddings cache keyed on all schemas — new schema busts the cache and | |
| # forces a recompute, unchanged schemas reuse their cached vectors. | |
| - name: Cache embeddings | |
| uses: actions/cache@v4 | |
| with: | |
| path: data/embeddings.parquet | |
| key: embeddings-${{ hashFiles('schemas/*.yml') }}-${{ github.run_id }} | |
| restore-keys: embeddings-${{ hashFiles('schemas/*.yml') }}- | |
| embeddings- | |
| # ----------------------------------------------------------------------- | |
| - name: Install dependencies | |
| run: pip install -r requirements.txt | |
| # ----------------------------------------------------------------------- | |
| - name: Parse issue metadata | |
| id: meta | |
| run: | | |
| TITLE="${{ github.event.issue.title }}" | |
| SCHEMA_NAME="${TITLE#schema-submission:}" | |
| SCHEMA_NAME=$(echo "$SCHEMA_NAME" | xargs | tr ' ' '-' | tr '[:upper:]' '[:lower:]') | |
| echo "schema_name=$SCHEMA_NAME" >> $GITHUB_OUTPUT | |
| LABELS='${{ toJson(github.event.issue.labels.*.name) }}' | |
| if echo "$LABELS" | grep -q "breaking"; then | |
| echo "bump=major" >> $GITHUB_OUTPUT | |
| else | |
| echo "bump=minor" >> $GITHUB_OUTPUT | |
| fi | |
| # ----------------------------------------------------------------------- | |
| # Write the submitted schema to schemas/{name}.yml so it's picked up | |
| # by pipeline.py alongside all previously committed schemas. | |
| - name: Extract and write schema file | |
| env: | |
| ISSUE_BODY: ${{ github.event.issue.body }} | |
| SCHEMA_NAME: ${{ steps.meta.outputs.schema_name }} | |
| run: | | |
| python3 - << 'PYEOF' | |
| import os, re, sys, json | |
| import yaml | |
| body = os.environ["ISSUE_BODY"] | |
| name = os.environ["SCHEMA_NAME"] | |
| fenced = re.search(r'```([\w-]*)\s*\n(.*?)```', body, re.DOTALL) | |
| if fenced: | |
| lang = (fenced.group(1) or "yaml").lower() | |
| content = fenced.group(2).strip() | |
| else: | |
| lang = "yaml" | |
| content = body.strip() | |
| json_schema_langs = {"json", "json-schema", "jsonschema"} | |
| os.makedirs("schemas", exist_ok=True) | |
| out_path = f"schemas/{name}.yml" | |
| if lang in json_schema_langs: | |
| print(f"Detected format: JSON Schema ({lang!r}) — converting…") | |
| sys.path.insert(0, ".") | |
| from neuro_ghost.converters.from_jsonschema import convert as js2lm | |
| try: | |
| data = json.loads(content) | |
| except json.JSONDecodeError: | |
| data = yaml.safe_load(content) | |
| linkml_dict = js2lm(data, name) | |
| with open(out_path, "w") as f: | |
| yaml.dump(linkml_dict, f, default_flow_style=False, | |
| allow_unicode=True, sort_keys=False) | |
| print(f"Converted → {out_path} " | |
| f"({len(linkml_dict['classes'])} classes)") | |
| else: | |
| print(f"Detected format: LinkML ({lang!r})") | |
| with open(out_path, "w") as f: | |
| f.write(content) | |
| print(f"Wrote {out_path}") | |
| PYEOF | |
| # ----------------------------------------------------------------------- | |
| - name: Validate LinkML schema | |
| run: | | |
| python neuro_ghost/validate_schema.py \ | |
| schemas/${{ steps.meta.outputs.schema_name }}.yml | |
| # ----------------------------------------------------------------------- | |
| # Full rebuild from all schemas/*.yml in the repo (including the new one | |
| # written above). --skip-converters keeps CI fast — external schemas | |
| # (BIDS/NWB/DANDI/…) are refreshed by the weekly publish_registry.yml. | |
| - name: Rebuild registry (all local schemas) | |
| run: | | |
| python neuro_ghost/pipeline.py \ | |
| --fresh \ | |
| --skip-converters \ | |
| --bump ${{ steps.meta.outputs.bump }} \ | |
| --issue ${{ github.event.issue.number }} \ | |
| --agent "${{ github.event.issue.user.login }}" | |
| # ----------------------------------------------------------------------- | |
| - name: Commit registry + schema file | |
| run: | | |
| git config user.name "github-actions[bot]" | |
| git config user.email "github-actions[bot]@users.noreply.github.com" | |
| git add \ | |
| schemas/${{ steps.meta.outputs.schema_name }}.yml \ | |
| data/registry.json \ | |
| data/versions/ \ | |
| data/provenance.json | |
| git diff --cached --quiet || git commit -m \ | |
| "feat(registry): ingest ${{ steps.meta.outputs.schema_name }} (issue #${{ github.event.issue.number }})" | |
| git push | |
| # ----------------------------------------------------------------------- | |
| - name: Comment success | |
| if: success() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| const name = "${{ steps.meta.outputs.schema_name }}"; | |
| const pagesUrl = "https://sensein.group/NeuroGhost/"; | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: [ | |
| `✅ **\`${name}\` ingested successfully.**`, | |
| ``, | |
| `- All schemas rebuilt and aligned.`, | |
| `- Provenance logged to \`data/provenance.json\`.`, | |
| `- Archived snapshot in \`data/versions/\`.`, | |
| ``, | |
| `Browse: ${pagesUrl}`, | |
| ].join("\n"), | |
| }); | |
| await github.rest.issues.update({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| issue_number: context.issue.number, state: "closed", | |
| }); | |
| - name: Comment failure | |
| if: failure() | |
| uses: actions/github-script@v7 | |
| with: | |
| script: | | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: [ | |
| `❌ **Ingestion failed.**`, | |
| ``, | |
| `Check the run log for validation errors (undefined slot, \`is_a\` parent, or range).`, | |
| ``, | |
| `[View run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})`, | |
| ].join("\n"), | |
| }); |