Merge pull request #12 from sensein/claude/schema-registry-meta-model-2upt56

Claude/schema registry meta model 2upt56

Author: Sulstice

README.md

@@ -100,198 +100,112 @@ the entire registry, or a CSV diff between two schemas.
 
 ## API Reference
 
-**Base URL:** `https://sensein.group/NeuroGhost`
+Static JSON served by GitHub Pages — no auth, no rate limits, CORS open.
 
-The registry exposes two kinds of endpoints:
-
-| Type | Transport | Auth | Status |
-|------|-----------|------|--------|
-| Read (schemas, alignments, provenance) | Static JSON via GitHub Pages | None | ✅ Live |
-| Transform (field mapping between schemas) | Serverless function (planned) | None | 🔜 Planned |
-
-CORS is open on all endpoints. No API key required.
+| Method | URL | Description | Status |
+|--------|-----|-------------|--------|
+| `GET` | [`/data/registry.json`](https://sensein.group/NeuroGhost/data/registry.json) | Full registry: all schemas, classes, properties, alignments | ✅ Live |
+| `GET` | [`/data/versions/{version}.json`](https://sensein.group/NeuroGhost/data/versions/1.7.0.json) | Frozen snapshot at a specific registry version | ✅ Live |
+| `GET` | [`/data/provenance.json`](https://sensein.group/NeuroGhost/data/provenance.json) | Changelog of every schema ingestion | ✅ Live |
+| `GET` | `/api/transform?from={schema}&to={schema}` | Field-to-field mapping between two schemas | 🔜 Planned |
+| `POST` | `/api/transform` | Transform a dataset from one schema format to another | 🔜 Planned |
 
 ---
 
 ### `GET /data/registry.json`
 
-Returns the full registry at the current version: all sources, classes,
-properties, and pre-computed cross-schema alignments.
-
 ```bash
 curl https://sensein.group/NeuroGhost/data/registry.json
 ```
 
-**Response**
 ```json
 {
   "registry_version": "1.7.0",
   "generated_at": "2026-07-23T12:40:24Z",
   "sources": [
-    { "label": "bbqs",  "version": "1.0.0", "class_count": 29 },
-    { "label": "bids",  "version": "1.9.0", "class_count": 1  },
-    { "label": "dandi", "version": "0.6.8", "class_count": 20 },
-    { "label": "nwb",   "version": "2.7.0", "class_count": 53 }
+    { "label": "bbqs", "version": "1.0.0", "class_count": 29 }
   ],
   "classes": [
     {
-      "uid":        "8003dcad-...",
+      "hash_id":    "sha256:abc123...",
       "iri":        "https://registry.sensein.io/obj/Subject",
       "name":       "Subject",
       "definition": "A research participant.",
-      "abstract":   false,
       "source":     "bbqs",
       "properties": [
-        {
-          "uid":        "807f8fec-...",
-          "name":       "age",
-          "definition": "Age in years.",
-          "datatype":   "xsd:integer",
-          "multivalued": false,
-          "required":   false,
-          "source":     "bbqs"
-        }
+        { "hash_id": "sha256:def456...", "name": "age", "value_range": "xsd:integer", "multivalued": false, "required": false }
       ],
       "alignments": [
-        {
-          "target_uid":    "f4f8e4a4-...",
-          "target_name":   "Participant",
-          "target_source": "bids",
-          "distance":      0.12,
-          "method":        "composite",
-          "scores": { "iri": 0.0, "name": 0.08, "desc": 0.19, "slot": 0.0 }
-        }
+        { "target_name": "Participant", "target_source": "bids", "distance": 0.12, "method": "composite" }
       ]
     }
   ]
 }
 ```
 
-`distance` ranges from **0.0** (identical) to **1.0** (unrelated).
+`distance`: **0.0** = identical · **1.0** = unrelated.
 
 ---
 
 ### `GET /data/versions/{version}.json`
 
-Frozen snapshot of the registry at a specific version. Snapshots never change.
+Same shape as `registry.json`. Snapshots are permanent — they never change.
 
 ```bash
 curl https://sensein.group/NeuroGhost/data/versions/1.2.0.json
 ```
 
-Same shape as `registry.json`. To list available versions, check the
-`registry_version` field of the live registry and the `data/versions/`
-directory.
-
 ---
 
 ### `GET /data/provenance.json`
 
-Changelog — every schema ingestion: who submitted it, when, and which registry
-version it produced.
-
 ```bash
 curl https://sensein.group/NeuroGhost/data/provenance.json
 ```
 
 ---
 
-### Client-side filtering
-
-There are no server-side query parameters (static files). Filter in your
-client after fetching:
-
-```js
-const reg = await fetch("https://sensein.group/NeuroGhost/data/registry.json")
-              .then(r => r.json());
-
-// All classes from one schema
-const bbqs = reg.classes.filter(c => c.source === "bbqs");
-
-// Close alignments across any two schemas (distance < 0.35)
-const pairs = reg.classes.flatMap(c =>
-  c.alignments
-    .filter(a => a.distance < 0.35)
-    .map(a => ({ from: `${c.source}/${c.name}`, to: `${a.target_source}/${a.target_name}`, distance: a.distance }))
-);
-```
-
----
-
-### `GET /api/transform` — field mapping *(planned)*
+### `GET /api/transform?from={schema}&to={schema}` *(planned)*
 
-Returns the computed field-to-field mapping between two schemas, derived from
-the alignment graph. No data is sent — this is purely a schema-level lookup.
+Returns the pre-computed field mapping between two schemas. Derived from the
+alignment graph; can be served as a static file with no extra infrastructure.
 
-```
-GET /api/transform?from=bbqs&to=bids
+```bash
+curl "https://sensein.group/NeuroGhost/api/transform?from=bbqs&to=bids"
 ```
 
-**Planned response**
 ```json
 {
-  "from": "bbqs",
-  "to":   "bids",
+  "from": "bbqs", "to": "bids",
   "mappings": [
     {
-      "from_class":    "Subject",
-      "to_class":      "Participant",
-      "distance":      0.12,
+      "from_class": "Subject", "to_class": "Participant", "distance": 0.12,
       "field_mappings": [
-        { "from_field": "age",        "to_field": "age",          "confidence": 0.97 },
-        { "from_field": "species",    "to_field": "species",      "confidence": 0.91 },
-        { "from_field": "subject_id", "to_field": "participant_id","confidence": 0.85 }
+        { "from_field": "subject_id", "to_field": "participant_id", "confidence": 0.85 },
+        { "from_field": "age",        "to_field": "age",            "confidence": 0.97 }
       ]
     }
   ]
 }
 ```
 
-Because the alignment data is already in `registry.json`, this endpoint can be
-pre-computed at export time and served as a static file — no compute layer
-needed.
-
 ---
 
-### `POST /api/transform` — data transform *(planned)*
+### `POST /api/transform` *(planned)*
 
-Send a dataset in one schema's format; receive it mapped to another. Unlike the
-GET above, this requires a live compute layer (a serverless function that reads
-the field-mapping and rewrites the payload).
+Send a record in one schema's format; receive it remapped to another.
+Requires a serverless compute layer (Cloudflare Worker or equivalent).
 
 ```bash
 curl -X POST https://sensein.group/NeuroGhost/api/transform \
   -H "Content-Type: application/json" \
-  -d '{
-    "from": "bbqs",
-    "to":   "bids",
-    "data": {
-      "subject_id": "sub-01",
-      "age": 24,
-      "species": "Homo sapiens"
-    }
-  }'
+  -d '{ "from": "bbqs", "to": "bids", "data": { "subject_id": "sub-01", "age": 24 } }'
 ```
 
-**Planned response**
 ```json
-{
-  "from": "bbqs",
-  "to":   "bids",
-  "result": {
-    "participant_id": "sub-01",
-    "age": 24,
-    "species": "Homo sapiens"
-  },
-  "unmapped_fields": [],
-  "warnings": []
-}
+{ "from": "bbqs", "to": "bids", "result": { "participant_id": "sub-01", "age": 24 }, "unmapped_fields": [], "warnings": [] }
 ```
 
-This will be implemented as a lightweight serverless function (Cloudflare Worker
-or equivalent) that reads the static registry alignment map and applies field
-renaming. Not yet live.
-
 ---
 
 ## Adding your own schema

neuro_ghost/pipeline.py

@@ -0,0 +1,111 @@
+"""
+pipeline.py — Run the full NeuroGhost ingestion pipeline in one command.
+
+Steps
+-----
+1. (Optional) Delete the database file for a clean rebuild
+2. Seed schema.org as the base vocabulary
+3. (Optional) Fetch + convert external schemas via converters/run_all.py
+4. Ingest every .yml file found in the schemas directory
+5. Compute cross-schema alignments
+6. Export registry.json + version snapshot
+
+Usage
+-----
+    # Fresh rebuild — wipe DB, fetch external schemas, ingest everything
+    python neuro_ghost/pipeline.py --fresh
+
+    # Re-ingest local schemas only (skip external fetch)
+    python neuro_ghost/pipeline.py --fresh --skip-converters
+
+    # Incremental — add one schema to an existing DB
+    python neuro_ghost/pipeline.py --skip-converters --schemas schemas/bbqs.yml
+"""
+
+from __future__ import annotations
+import subprocess
+import sys
+from pathlib import Path
+
+import click
+
+HERE     = Path(__file__).parent
+ROOT     = HERE.parent
+SCHEMAS  = ROOT / "schemas"
+DB_PATH  = str(ROOT / "registry.lbug")
+
+
+def _run(cmd: list[str]) -> None:
+    print(f"\n$ {' '.join(cmd)}")
+    result = subprocess.run(cmd, check=False)
+    if result.returncode != 0:
+        print(f"\nERROR: command exited with code {result.returncode}")
+        sys.exit(result.returncode)
+
+
+@click.command()
+@click.option("--db",               default=DB_PATH, show_default=True,
+              help="Path to the LadybugDB file.")
+@click.option("--fresh",            is_flag=True,
+              help="Delete the DB before starting (clean rebuild).")
+@click.option("--skip-converters",  is_flag=True,
+              help="Skip fetching external schemas (BIDS, NWB, DANDI, …).")
+@click.option("--schemas",          default=None, multiple=True,
+              help="Specific .yml files to ingest. Defaults to all files in schemas/.")
+@click.option("--bump",             default="minor", show_default=True,
+              type=click.Choice(["major", "minor", "patch"]),
+              help="Version bump type for the registry export.")
+@click.option("--agent",            default="local", show_default=True,
+              help="Who is running this pipeline (recorded in provenance).")
+def cli(db: str, fresh: bool, skip_converters: bool,
+        schemas: tuple, bump: str, agent: str) -> None:
+    """Run the full NeuroGhost ingestion pipeline."""
+
+    py = sys.executable
+
+    # 1. Fresh wipe
+    if fresh:
+        db_path = Path(db)
+        if db_path.exists():
+            db_path.unlink()
+            print(f"Deleted {db_path}")
+
+    # 2. Seed schema.org
+    _run([py, str(HERE / "seed.py"), "--db", db])
+
+    # 3. External converters
+    if not skip_converters:
+        _run([py, str(HERE / "converters" / "run_all.py")])
+
+    # 4. Ingest schemas
+    targets: list[Path] = (
+        [Path(s) for s in schemas]
+        if schemas
+        else sorted(p for p in SCHEMAS.glob("*.yml")
+                    if p.name != "meta_model.yaml")
+    )
+
+    if not targets:
+        print("No schemas found to ingest.")
+        sys.exit(1)
+
+    for schema_file in targets:
+        _run([py, str(HERE / "ingest_linkml.py"),
+              "--file", str(schema_file),
+              "--db",   db,
+              "--agent", agent])
+
+    # 5. Align
+    _run([py, str(HERE / "align.py"), "--db", db])
+
+    # 6. Export
+    _run([py, str(HERE / "export_json.py"),
+          "--db",    db,
+          "--bump",  bump,
+          "--agent", agent])
+
+    print("\nPipeline complete.")
+
+
+if __name__ == "__main__":
+    cli()