Skip to content

Commit 4cbc212

Browse files
SulsticeSulstice
authored
Update export_json.py
1 parent b00958a commit 4cbc212

1 file changed

Lines changed: 147 additions & 147 deletions

File tree

‎neuro_registry/export_json.py‎

Lines changed: 147 additions & 147 deletions
Original file line numberDiff line numberDiff line change
@@ -1,195 +1,195 @@
11
"""
2-
export_json.py — Export registry snapshot to data/registry.json
2+
export_json.py — Export registry snapshot + provenance to data/
33
---------------------------------------------------------------
4-
Reads the LadybugDB graph and writes a single JSON file that the
5-
GitHub Pages frontend consumes. Runs as the last step in the CI
6-
workflow after every schema ingestion.
7-
8-
Output shape:
9-
{
10-
"generated_at": "2025-...",
11-
"sources": [
12-
{ "label": "schema.org", "version": "1.0.0", "class_count": 38 }
13-
],
14-
"classes": [
15-
{
16-
"uid": "...",
17-
"iri": "https://schema.org/Person",
18-
"uri": "https://registry.sensein.io/obj/Person/v/1.0.0",
19-
"name": "Person",
20-
"definition": "A person...",
21-
"version": "1.0.0",
22-
"abstract": false,
23-
"source": "schema.org",
24-
"properties": [
25-
{
26-
"uid": "...",
27-
"iri": "https://schema.org/name",
28-
"name": "name",
29-
"definition": "...",
30-
"datatype": "xsd:string",
31-
"range_uri": "",
32-
"multivalued": false,
33-
"required": false,
34-
"source": "schema.org"
35-
}
36-
],
37-
"subclass_of": ["https://schema.org/Thing"],
38-
"alignments": [
39-
{
40-
"target_uid": "...",
41-
"target_name": "Investigator",
42-
"target_iri": "https://schema.org/Person",
43-
"target_source": "bbqs",
44-
"distance": 0.0,
45-
"method": "iri"
46-
}
47-
]
48-
}
49-
]
50-
}
4+
Runs after every ingest/align cycle. Produces:
5+
6+
data/registry.json — latest snapshot (frontend reads this)
7+
data/versions/{ver}.json — archived snapshot for this registry version
8+
data/provenance.json — append-only log of every registry version
519
5210
Usage:
5311
python export_json.py
54-
python export_json.py --db ./registry.lbug --out data/registry.json
12+
python export_json.py --db ./registry.lbug --bump minor
13+
python export_json.py --issue 3 --agent sulimansharif --bump minor
5514
"""
5615

5716
from __future__ import annotations
58-
import datetime, json, os
17+
import json, shutil
5918
from pathlib import Path
6019

6120
import click
62-
import ladybug as lb
6321

22+
from db import (
23+
get_connection, now_iso,
24+
current_registry_version, next_registry_version, append_provenance,
25+
PROVENANCE_PATH,
26+
)
27+
28+
DATA_DIR = Path("data")
6429
DB_PATH = "./registry.lbug"
65-
OUT_PATH = "data/registry.json"
6630

6731

68-
def export(conn: lb.Connection) -> dict:
32+
# ---------------------------------------------------------------------------
33+
# Export helpers
34+
# ---------------------------------------------------------------------------
35+
36+
def export_snapshot(conn, registry_version: str) -> dict:
6937
# ---- sources -----------------------------------------------------------
70-
src_result = conn.execute(
38+
src_rows = conn.execute(
7139
"MATCH (s:SchemaSource) RETURN s.uid, s.label, s.version"
72-
)
73-
source_rows = src_result.get_all()
40+
).get_all()
7441

75-
source_class_counts: dict[str, int] = {}
76-
for row in source_rows:
77-
_, label, _ = row
78-
r = conn.execute(
42+
sources = []
43+
for _, label, ver in src_rows:
44+
count = conn.execute(
7945
"MATCH (n:SchemaClass {source_label: $src}) RETURN count(n)",
8046
{"src": label}
81-
)
82-
source_class_counts[label] = r.get_next()[0]
83-
84-
sources = [
85-
{
86-
"label": row[1],
87-
"version": row[2] or "1.0.0",
88-
"class_count": source_class_counts.get(row[1], 0),
89-
}
90-
for row in source_rows
91-
]
47+
).get_next()[0]
48+
sources.append({"label": label, "version": ver or "1.0.0",
49+
"class_count": count})
9250

9351
# ---- classes -----------------------------------------------------------
94-
cls_result = conn.execute("""
52+
cls_rows = conn.execute("""
9553
MATCH (n:SchemaClass)
9654
RETURN n.uid, n.iri, n.uri, n.name, n.definition,
97-
n.version, n.abstract, n.source_label
55+
n.version, n.abstract, n.source_label, n.registry_version
9856
ORDER BY n.source_label, n.name
99-
""")
57+
""").get_all()
10058

10159
classes = []
102-
for row in cls_result.get_all():
103-
uid, iri, uri, name, definition, version, abstract, source = row
60+
for row in cls_rows:
61+
uid, iri, uri, name, defn, ver, abstract, source, reg_ver = row
10462

105-
# properties
106-
prop_result = conn.execute("""
63+
props = conn.execute("""
10764
MATCH (c:SchemaClass {uid: $uid})-[:HAS_PROPERTY]->(p:SchemaProperty)
10865
RETURN p.uid, p.iri, p.name, p.definition,
109-
p.datatype, p.range_uri, p.multivalued,
110-
p.required, p.source_label
66+
p.datatype, p.range_uri, p.multivalued, p.required, p.source_label
11167
ORDER BY p.name
112-
""", {"uid": uid})
113-
114-
properties = [
115-
{
116-
"uid": r[0],
117-
"iri": r[1] or "",
118-
"name": r[2] or "",
119-
"definition": r[3] or "",
120-
"datatype": r[4] or "",
121-
"range_uri": r[5] or "",
122-
"multivalued": bool(r[6]),
123-
"required": bool(r[7]),
124-
"source": r[8] or "",
125-
}
126-
for r in prop_result.get_all()
127-
]
68+
""", {"uid": uid}).get_all()
12869

129-
# subclass_of
130-
sub_result = conn.execute("""
131-
MATCH (c:SchemaClass {uid: $uid})-[:SUBCLASS_OF]->(p:SchemaClass)
132-
RETURN p.iri
133-
""", {"uid": uid})
134-
subclass_of = [r[0] for r in sub_result.get_all() if r[0]]
70+
subclass_of = [
71+
r[0] for r in conn.execute("""
72+
MATCH (c:SchemaClass {uid: $uid})-[:SUBCLASS_OF]->(p:SchemaClass)
73+
RETURN p.iri
74+
""", {"uid": uid}).get_all() if r[0]
75+
]
13576

136-
# alignments
137-
align_result = conn.execute("""
77+
align_rows = conn.execute("""
13878
MATCH (c:SchemaClass {uid: $uid})-[a:ALIGNED_TO]->(t:SchemaClass)
139-
RETURN t.uid, t.name, t.iri, t.source_label, a.distance, a.method
79+
RETURN t.uid, t.name, t.iri, t.source_label,
80+
a.distance, a.method,
81+
a.score_iri, a.score_name, a.score_desc, a.score_slot
14082
ORDER BY a.distance
141-
""", {"uid": uid})
142-
143-
alignments = [
144-
{
145-
"target_uid": r[0],
146-
"target_name": r[1] or "",
147-
"target_iri": r[2] or "",
148-
"target_source": r[3] or "",
149-
"distance": float(r[4]) if r[4] is not None else 1.0,
150-
"method": r[5] or "",
151-
}
152-
for r in align_result.get_all()
153-
]
83+
""", {"uid": uid}).get_all()
15484

15585
classes.append({
156-
"uid": uid,
157-
"iri": iri or "",
158-
"uri": uri or "",
159-
"name": name or "",
160-
"definition": definition or "",
161-
"version": version or "1.0.0",
162-
"abstract": bool(abstract),
163-
"source": source or "",
164-
"properties": properties,
86+
"uid": uid,
87+
"iri": iri or "",
88+
"uri": uri or "",
89+
"name": name or "",
90+
"definition": defn or "",
91+
"version": ver or "1.0.0",
92+
"registry_version": reg_ver or "",
93+
"abstract": bool(abstract),
94+
"source": source or "",
95+
"properties": [
96+
{
97+
"uid": r[0], "iri": r[1] or "",
98+
"name": r[2] or "", "definition": r[3] or "",
99+
"datatype": r[4] or "", "range_uri": r[5] or "",
100+
"multivalued": bool(r[6]), "required": bool(r[7]),
101+
"source": r[8] or "",
102+
}
103+
for r in props
104+
],
165105
"subclass_of": subclass_of,
166-
"alignments": alignments,
106+
"alignments": [
107+
{
108+
"target_uid": r[0], "target_name": r[1] or "",
109+
"target_iri": r[2] or "",
110+
"target_source": r[3] or "",
111+
"distance": float(r[4]) if r[4] is not None else 1.0,
112+
"method": r[5] or "",
113+
"scores": {
114+
"iri": float(r[6]) if r[6] is not None else 0.0,
115+
"name": float(r[7]) if r[7] is not None else 0.0,
116+
"desc": float(r[8]) if r[8] is not None else 0.0,
117+
"slot": float(r[9]) if r[9] is not None else 0.0,
118+
}
119+
}
120+
for r in align_rows
121+
],
167122
})
168123

169124
return {
170-
"generated_at": datetime.datetime.now(datetime.UTC).isoformat(),
171-
"sources": sources,
172-
"classes": classes,
125+
"registry_version": registry_version,
126+
"generated_at": now_iso(),
127+
"sources": sources,
128+
"classes": classes,
173129
}
174130

175131

132+
# ---------------------------------------------------------------------------
133+
# CLI
134+
# ---------------------------------------------------------------------------
135+
176136
@click.command()
177-
@click.option("--db", default=DB_PATH, show_default=True,
178-
help="Path to LadybugDB file.")
179-
@click.option("--out", default=OUT_PATH, show_default=True,
180-
help="Output JSON path.")
181-
def cli(db: str, out: str) -> None:
182-
"""Export registry snapshot to JSON for the GitHub Pages frontend."""
183-
conn = lb.Connection(lb.Database(db))
184-
data = export(conn)
185-
186-
out_path = Path(out)
187-
out_path.parent.mkdir(parents=True, exist_ok=True)
188-
out_path.write_text(json.dumps(data, indent=2))
189-
190-
nc = len(data["classes"])
191-
ns = len(data["sources"])
192-
click.echo(f"Exported {nc} classes from {ns} sources → {out_path}")
137+
@click.option("--db", default=DB_PATH, show_default=True)
138+
@click.option("--bump", default="minor",
139+
type=click.Choice(["major", "minor", "patch"]),
140+
help="Version bump type. major=breaking, minor=new schema, patch=update.")
141+
@click.option("--issue", default="", help="GitHub issue number that triggered this.")
142+
@click.option("--agent", default="github-actions", help="Who triggered this.")
143+
@click.option("--schema", default="", help="Schema name that was ingested.")
144+
def cli(db: str, bump: str, issue: str, agent: str, schema: str) -> None:
145+
"""Export registry snapshot, archive version, append provenance."""
146+
conn = get_connection(db)
147+
148+
# Compute new registry version
149+
current = current_registry_version()
150+
new_ver = next_registry_version(current, bump) if current != "0.0.0" else "1.0.0"
151+
click.echo(f"Registry version: {current} → {new_ver}")
152+
153+
# Build snapshot
154+
snapshot = export_snapshot(conn, new_ver)
155+
nc = len(snapshot["classes"])
156+
ns = len(snapshot["sources"])
157+
158+
# Write data/registry.json (latest)
159+
DATA_DIR.mkdir(parents=True, exist_ok=True)
160+
latest = DATA_DIR / "registry.json"
161+
latest.write_text(json.dumps(snapshot, indent=2))
162+
click.echo(f"Wrote {latest} ({nc} classes, {ns} sources)")
163+
164+
# Archive to data/versions/{ver}.json
165+
versions_dir = DATA_DIR / "versions"
166+
versions_dir.mkdir(exist_ok=True)
167+
archive = versions_dir / f"{new_ver}.json"
168+
shutil.copy(latest, archive)
169+
click.echo(f"Archived → {archive}")
170+
171+
# Count changes vs previous version
172+
prev_path = DATA_DIR / "registry.json"
173+
classes_added = nc # simplified — full diff would compare to previous snapshot
174+
175+
# Append to provenance.json
176+
prov_entry = {
177+
"registry_version": new_ver,
178+
"previous_version": current,
179+
"timestamp": now_iso(),
180+
"bump": bump,
181+
"trigger": "issue" if issue else "manual",
182+
"issue_number": issue,
183+
"agent": agent,
184+
"schema_ingested": schema,
185+
"stats": {
186+
"classes_total": nc,
187+
"sources_total": ns,
188+
},
189+
"archive_path": f"data/versions/{new_ver}.json",
190+
}
191+
append_provenance(prov_entry)
192+
click.echo(f"Appended provenance entry for v{new_ver}")
193193

194194

195195
if __name__ == "__main__":

0 commit comments

Comments
 (0)