Fix autocomplete metadata schema compliance

Author: Copilot

developer/autocomplete_metadata.py

@@ -15,8 +15,10 @@
 
 import json
 import os
+import re
 import sys
 import urllib.request
+from html import unescape
 
 import yaml
 
@@ -142,6 +144,46 @@ def get_chembl_id_from_unichem(sources):
     return None
 
 
+def clean_text(value):
+    if not isinstance(value, str):
+        return value
+    return re.sub(r"<[^>]+>", "", unescape(value)).strip()
+
+
+def safe_float(value):
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def sanitize_sameas(sameas):
+    patterns = {
+        "ChEBI": r"^CHEBI:\d+$",
+        "ChEMBL": r"^CHEMBL\d+$",
+        "lipidmaps": r"^LM(FA|GL|GP|SP|ST|PR|SL|PK)[0-9]{4}([0-9a-zA-Z]{4,6})?$",
+        "metabolights": r"^MTBL[CS]\d+$",
+        "slm": r"^SLM:\d+$",
+        "pdb.ligand": r"^[A-Za-z0-9]+$",
+        "unii": r"^[A-Z0-9]+$",
+        "cas": r"^\d{1,7}-\d{2}-\d$",
+    }
+    sanitized = {}
+    for key, value in sameas.items():
+        if key == "pubchem.compound":
+            if isinstance(value, int):
+                sanitized[key] = value
+            else:
+                try:
+                    sanitized[key] = int(value)
+                except (TypeError, ValueError):
+                    pass
+            continue
+        if isinstance(value, str) and re.match(patterns.get(key, r".+"), value):
+            sanitized[key] = value
+    return sanitized
+
+
 def load_existing_metadata(path):
     if os.path.exists(path):
         with open(path, "r", encoding="utf-8") as f:
@@ -188,7 +230,7 @@ def main():
     chembl = get_chembl(inchikey)
     pubchem = get_pubchem(inchikey)
     sources = get_unichem(inchikey)
-    sameas = extract_sameas(sources)
+    sameas = sanitize_sameas(extract_sameas(sources))
 
     cid = pubchem.get("CID", sameas.get("pubchem.compound"))
     synonyms = get_pubchem_synonyms(cid) if cid else []
@@ -216,6 +258,7 @@ def main():
     # 3. If still no synonyms, try PubChem synonyms
     if not alternate_names and synonyms:
         alternate_names = synonyms
+    alternate_names = [clean_text(name) for name in alternate_names if clean_text(name)]
 
     molecule_props = chembl.get("molecule_properties", {})
     molecule_structures = chembl.get("molecule_structures", {})
@@ -229,14 +272,23 @@ def main():
     else:
         image_url = ""
 
+    nmr_name = (
+        existing.get("NMRlipids", {}).get("name")
+        or clean_text(chembl.get("pref_name", ""))
+        or clean_text(molecule_props.get("iupac_name", ""))
+        or clean_text(pubchem.get("IUPACName", ""))
+        or nmr_id
+    )
+
     bioschema = {
-        "name": molecule_props.get("iupac_name") or pubchem.get("IUPACName", ""),
-        "iupacName": molecule_props.get("iupac_name") or pubchem.get("IUPACName", ""),
+        "name": clean_text(molecule_props.get("iupac_name")) or clean_text(pubchem.get("IUPACName", "")),
+        "iupacName": clean_text(molecule_props.get("iupac_name")) or clean_text(pubchem.get("IUPACName", "")),
         "molecularFormula": molecule_props.get("full_molformula") or pubchem.get("MolecularFormula", ""),
-        "molecularWeight": float(molecule_props.get("full_mwt") or pubchem.get("MolecularWeight", 0)),
-        "inChI": molecule_structures.get("standard_inchi") or pubchem.get("InChI", ""),
-        "inChIKey": molecule_structures.get("standard_inchi_key") or pubchem.get("InChIKey", ""),
-        "smiles": molecule_structures.get("canonical_smiles") or pubchem.get("SMILES", ""),
+        "molecularWeight": safe_float(molecule_props.get("full_mwt") or pubchem.get("MolecularWeight")),
+        "inChI": clean_text(molecule_structures.get("standard_inchi")) or clean_text(pubchem.get("InChI", "")),
+        "inChIKey": clean_text(molecule_structures.get("standard_inchi_key"))
+        or clean_text(pubchem.get("InChIKey", "")),
+        "smiles": clean_text(molecule_structures.get("canonical_smiles")) or clean_text(pubchem.get("SMILES", "")),
         "image": image_url,
         "description": "",
     }
@@ -245,7 +297,7 @@ def main():
         bioschema["alternateName"] = alternate_names
 
     new_data = {
-        "NMRlipids": {"id": nmr_id, "name": "", "charge": ""},
+        "NMRlipids": {"id": nmr_id, "name": nmr_name},
         "sameAs": sameas,
         "bioschema_properties": bioschema,
     }

tests/test_autocomplete_metadata.py

@@ -0,0 +1,81 @@
+import importlib.util
+import json
+import sys
+from pathlib import Path
+
+import yaml
+from jsonschema import Draft7Validator
+
+
+def load_autocomplete_module():
+    module_path = Path(__file__).resolve().parents[1] / "developer" / "autocomplete_metadata.py"
+    spec = importlib.util.spec_from_file_location("autocomplete_metadata", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_autocomplete_output_is_schema_compliant(tmp_path, monkeypatch):
+    mod = load_autocomplete_module()
+
+    metadata_path = tmp_path / "Molecules" / "membrane" / "BOGUS" / "metadata.yaml"
+    metadata_path.parent.mkdir(parents=True)
+    metadata_path.write_text(
+        yaml.safe_dump(
+            {
+                "NMRlipids": {"id": "BOGUS"},
+                "bioschema_properties": {"inChIKey": "HEGSGKPQLMEBJL-RKQHYHRCSA-N"},
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(mod, "get_chembl", lambda _: {"molecule_properties": {}, "molecule_structures": {}})
+    monkeypatch.setattr(
+        mod,
+        "get_pubchem",
+        lambda _: {
+            "CID": 7906,
+            "IUPACName": "(2R,3S)-name",
+            "MolecularFormula": "C14H28O6",
+            "MolecularWeight": 292.37,
+            "InChI": "InChI=1S/...",
+            "InChIKey": "HEGSGKPQLMEBJL-RKQHYHRCSA-N",
+            "SMILES": "CCCCCCCCO<a>C@H]1[C@@H</a>CO)O)O)O",
+        },
+    )
+    monkeypatch.setattr(
+        mod,
+        "get_unichem",
+        lambda _: [
+            {"shortName": "chembl", "compoundId": "CHEMBL446037"},
+            {"shortName": "chebi", "compoundId": "1234"},
+        ],
+    )
+    monkeypatch.setattr(mod, "get_pubchem_synonyms", lambda _: [])
+    monkeypatch.setattr(
+        mod,
+        "get_chebi",
+        lambda _: {"names": {"SYNONYM": [{"type": "SYNONYM", "name": "1-<em>OD&lt;/small&gt;-glucopyranoside"}]}},
+    )
+
+    monkeypatch.setattr(sys, "argv", ["autocomplete_metadata.py", str(metadata_path)])
+    mod.main()
+
+    generated = yaml.safe_load(metadata_path.read_text(encoding="utf-8"))
+    schema_path = (
+        Path(__file__).resolve().parents[1]
+        / "src"
+        / "fairmd"
+        / "lipids"
+        / "schema_validation"
+        / "schema"
+        / "metadata_schema.json"
+    )
+    schema = json.loads(schema_path.read_text(encoding="utf-8"))
+
+    errors = sorted(Draft7Validator(schema).iter_errors(generated), key=lambda e: e.path)
+    assert not errors
+    assert generated["NMRlipids"]["name"] == "(2R,3S)-name"
+    assert generated["bioschema_properties"]["smiles"] == "CCCCCCCCOC@H]1[C@@HCO)O)O)O"
+    assert generated["bioschema_properties"]["alternateName"] == ["1-OD-glucopyranoside"]