Make autocomplete_metadata.py emit schema-compliant molecule metadata and improve CI tests

developer/autocomplete_metadata.py

@@ -8,15 +8,19 @@
 - ChEMBL
 - ChEBI
 - PubChem
+- CAS Common Chemistry (requires the ``CAS_API_KEY`` environment variable)
 
 .. note::
    This file is meant to be used by automated workflows.
 """
 
 import json
 import os
+import re
 import sys
+import urllib.parse
 import urllib.request
+from html import unescape
 
 import yaml
 
@@ -89,6 +93,43 @@ def get_chebi(chebi_id):
     return {}
 
 
+def get_metabolights(chebi_id):
+    # MetaboLights reference compounds use the accession MTBLC<chebi numeric id>.
+    # Return the identifier only if the compound exists in MetaboLights.
+    if not chebi_id:
+        return ""
+    mtbl_id = f"MTBLC{chebi_id}"
+    if check_api(f"https://www.ebi.ac.uk/metabolights/ws/compounds/{mtbl_id}"):
+        return mtbl_id
+    return ""
+
+
+def get_cas(inchikey):
+    # CAS Registry Numbers are not exposed by UniChem. They can be retrieved from
+    # CAS Common Chemistry, which requires an API token supplied via the
+    # CAS_API_KEY environment variable. Returns "" when the token is missing,
+    # the service is unreachable, or no match is found.
+    if not inchikey:
+        return ""
+    api_key = os.environ.get("CAS_API_KEY")
+    if not api_key:
+        return ""
+    # CAS Common Chemistry requires field-qualified queries; a bare InChIKey
+    # does not match, whereas "InChIKey=<value>" does.
+    query = urllib.parse.quote(f"InChIKey={inchikey}")
+    url = f"https://commonchemistry.cas.org/api/search?q={query}"
+    try:
+        req = urllib.request.Request(url, headers={"X-Api-Key": api_key})
+        with urllib.request.urlopen(req, timeout=5) as response:
+            if response.status == 200:
+                results = json.loads(response.read().decode("utf-8")).get("results", [])
+                if results:
+                    return results[0].get("rn", "") or ""
+    except Exception:
+        pass
+    return ""
+
+
 def get_unichem(inchikey):
     url = "https://www.ebi.ac.uk/unichem/api/v1/compounds"
     if check_api("https://www.ebi.ac.uk/unichem/api/v1/sources"):
@@ -115,8 +156,9 @@ def extract_sameas(sources):
         "lipidmaps": "lipidmaps",
         "metabolights": "metabolights",
         "swisslipids": "slm",
-        "pdb": "pdb.ligand",
-        "unii": "unii",
+        "rcsb_pdb": "pdb.ligand",
+        "pdbe": "pdb.ligand",
+        "fdasrs": "unii",
         "cas": "cas",
     }
     result = {}
@@ -125,7 +167,10 @@ def extract_sameas(sources):
         if prefix:
             value = src["compoundId"]
             if prefix == "ChEBI":
-                value = f"CHEBI:{value}" if value else ""
+                if value:
+                    value = value if str(value).startswith("CHEBI:") else f"CHEBI:{value}"
+                else:
+                    value = ""
             elif prefix == "pubchem.compound":
                 try:
                     value = int(value)
@@ -142,6 +187,55 @@ def get_chembl_id_from_unichem(sources):
     return None
 
 
+def clean_text(value):
+    if not isinstance(value, str):
+        return value
+    return re.sub(r"<[^>]+>", "", unescape(value)).strip()
+
+
+def safe_float(value):
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def sanitize_sameas(sameas):
+    patterns = {
+        "ChEBI": r"^CHEBI:\d+$",
+        "ChEMBL": r"^CHEMBL\d+$",
+        "lipidmaps": r"^LM(FA|GL|GP|SP|ST|PR|SL|PK)[0-9]{4}([0-9a-zA-Z]{4,6})?$",
+        "metabolights": r"^MTBL[CS]\d+$",
+        "slm": r"^SLM:\d+$",
+        "pdb.ligand": r"^[A-Za-z0-9]+$",
+        "unii": r"^[A-Z0-9]+$",
+        "cas": r"^\d{1,7}-\d{2}-\d$",
+    }
+    sanitized = {}
+    for key, value in sameas.items():
+        if key == "pubchem.compound":
+            if isinstance(value, int):
+                sanitized[key] = value
+            else:
+                try:
+                    sanitized[key] = int(value)
+                except (TypeError, ValueError):
+                    print(
+                        f"Warning: discarding sameAs '{key}' value {value!r}: not a valid integer.",
+                        file=sys.stderr,
+                    )
+            continue
+        pattern = patterns.get(key, r".+")
+        if isinstance(value, str) and re.match(pattern, value):
+            sanitized[key] = value
+        else:
+            print(
+                f"Warning: discarding sameAs '{key}' value {value!r}: does not match expected pattern {pattern!r}.",
+                file=sys.stderr,
+            )
+    return sanitized
+
+
 def load_existing_metadata(path):
     if os.path.exists(path):
         with open(path, "r", encoding="utf-8") as f:
@@ -188,7 +282,7 @@ def main():
     chembl = get_chembl(inchikey)
     pubchem = get_pubchem(inchikey)
     sources = get_unichem(inchikey)
-    sameas = extract_sameas(sources)
+    sameas = sanitize_sameas(extract_sameas(sources))
 
     cid = pubchem.get("CID", sameas.get("pubchem.compound"))
     synonyms = get_pubchem_synonyms(cid) if cid else []
@@ -197,6 +291,19 @@ def main():
     chebi_id = sameas.get("ChEBI", "").replace("CHEBI:", "")
     chebi_data = get_chebi(chebi_id) if chebi_id else {}
 
+    # MetaboLights is not exposed by UniChem; derive it from the ChEBI id.
+    if chebi_id and "metabolights" not in sameas:
+        metabolights_id = get_metabolights(chebi_id)
+        if metabolights_id:
+            sameas["metabolights"] = metabolights_id
+
+    # CAS Registry Numbers are not exposed by UniChem; fetch them from CAS
+    # Common Chemistry (requires the CAS_API_KEY environment variable).
+    if "cas" not in sameas:
+        cas_rn = get_cas(inchikey)
+        if cas_rn and re.match(r"^\d{1,7}-\d{2}-\d$", cas_rn):
+            sameas["cas"] = cas_rn
+
     # Collect alternate names with priority
     alternate_names = []
 
@@ -216,6 +323,7 @@ def main():
     # 3. If still no synonyms, try PubChem synonyms
     if not alternate_names and synonyms:
         alternate_names = synonyms
+    alternate_names = [clean_text(name) for name in alternate_names if clean_text(name)]
 
     molecule_props = chembl.get("molecule_properties", {})
     molecule_structures = chembl.get("molecule_structures", {})
@@ -229,14 +337,23 @@ def main():
     else:
         image_url = ""
 
+    nmr_name = (
+        existing.get("NMRlipids", {}).get("name")
+        or clean_text(chembl.get("pref_name", ""))
+        or clean_text(molecule_props.get("iupac_name", ""))
+        or clean_text(pubchem.get("IUPACName", ""))
+        or nmr_id
+    )
+
     bioschema = {
-        "name": molecule_props.get("iupac_name") or pubchem.get("IUPACName", ""),
-        "iupacName": molecule_props.get("iupac_name") or pubchem.get("IUPACName", ""),
+        "name": clean_text(molecule_props.get("iupac_name")) or clean_text(pubchem.get("IUPACName", "")),
+        "iupacName": clean_text(molecule_props.get("iupac_name")) or clean_text(pubchem.get("IUPACName", "")),
         "molecularFormula": molecule_props.get("full_molformula") or pubchem.get("MolecularFormula", ""),
-        "molecularWeight": float(molecule_props.get("full_mwt") or pubchem.get("MolecularWeight", 0)),
-        "inChI": molecule_structures.get("standard_inchi") or pubchem.get("InChI", ""),
-        "inChIKey": molecule_structures.get("standard_inchi_key") or pubchem.get("InChIKey", ""),
-        "smiles": molecule_structures.get("canonical_smiles") or pubchem.get("SMILES", ""),
+        "molecularWeight": safe_float(molecule_props.get("full_mwt") or pubchem.get("MolecularWeight")),
+        "inChI": clean_text(molecule_structures.get("standard_inchi")) or clean_text(pubchem.get("InChI", "")),
+        "inChIKey": clean_text(molecule_structures.get("standard_inchi_key"))
+        or clean_text(pubchem.get("InChIKey", "")),
+        "smiles": clean_text(molecule_structures.get("canonical_smiles")) or clean_text(pubchem.get("SMILES", "")),
         "image": image_url,
         "description": "",
     }
@@ -245,7 +362,7 @@ def main():
         bioschema["alternateName"] = alternate_names
 
     new_data = {
-        "NMRlipids": {"id": nmr_id, "name": "", "charge": ""},
+        "NMRlipids": {"id": nmr_id, "name": nmr_name},
         "sameAs": sameas,
         "bioschema_properties": bioschema,
     }

pyproject.toml

@@ -86,6 +86,7 @@ markers = [
     "rdkit: tests features requiring RDKit installation",
     "all: run all tests",
     "min: run tests not requiring GROMACS installation",
+    "network: tests that require live internet access to external APIs",
 ]
 
 [tool.ruff]

tests/test_autocomplete_metadata.py

@@ -0,0 +1,132 @@
+import importlib.util
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+import pytest
+import yaml
+from jsonschema import Draft7Validator
+
+
+def load_autocomplete_module():
+    module_path = Path(__file__).resolve().parents[1] / "developer" / "autocomplete_metadata.py"
+    spec = importlib.util.spec_from_file_location("autocomplete_metadata", module_path)
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_autocomplete_output_is_schema_compliant(tmp_path, monkeypatch):
+    mod = load_autocomplete_module()
+
+    metadata_path = tmp_path / "Molecules" / "membrane" / "BOGUS" / "metadata.yaml"
+    metadata_path.parent.mkdir(parents=True)
+    metadata_path.write_text(
+        yaml.safe_dump(
+            {
+                "NMRlipids": {"id": "BOGUS"},
+                "bioschema_properties": {"inChIKey": "HEGSGKPQLMEBJL-RKQHYHRCSA-N"},
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    monkeypatch.setattr(mod, "get_chembl", lambda _: {"molecule_properties": {}, "molecule_structures": {}})
+    monkeypatch.setattr(
+        mod,
+        "get_pubchem",
+        lambda _: {
+            "CID": 7906,
+            "IUPACName": "(2R,3S)-name",
+            "MolecularFormula": "C14H28O6",
+            "MolecularWeight": 292.37,
+            "InChI": "InChI=1S/...",
+            "InChIKey": "HEGSGKPQLMEBJL-RKQHYHRCSA-N",
+            "SMILES": "CCCCCCCCO<a>C@H]1[C@@H</a>CO)O)O)O",
+        },
+    )
+    monkeypatch.setattr(
+        mod,
+        "get_unichem",
+        lambda _: [
+            {"shortName": "chembl", "compoundId": "CHEMBL446037"},
+            {"shortName": "chebi", "compoundId": "CHEBI:1234"},
+            {"shortName": "rcsb_pdb", "compoundId": "BOG"},
+            {"shortName": "fdasrs", "compoundId": "V109WUT6RL"},
+        ],
+    )
+    monkeypatch.setattr(mod, "get_pubchem_synonyms", lambda _: [])
+    monkeypatch.setattr(
+        mod,
+        "get_chebi",
+        lambda _: {"names": {"SYNONYM": [{"type": "SYNONYM", "name": "1-<em>OD&lt;/small&gt;-glucopyranoside"}]}},
+    )
+    monkeypatch.setattr(mod, "get_metabolights", lambda _: "MTBLC1234")
+    monkeypatch.setattr(mod, "get_cas", lambda _: "29836-26-8")
+
+    monkeypatch.setattr(sys, "argv", ["autocomplete_metadata.py", str(metadata_path)])
+    mod.main()
+
+    generated = yaml.safe_load(metadata_path.read_text(encoding="utf-8"))
+    schema_path = (
+        Path(__file__).resolve().parents[1]
+        / "src"
+        / "fairmd"
+        / "lipids"
+        / "schema_validation"
+        / "schema"
+        / "metadata_schema.json"
+    )
+    schema = json.loads(schema_path.read_text(encoding="utf-8"))
+
+    errors = sorted(Draft7Validator(schema).iter_errors(generated), key=lambda e: e.path)
+    assert not errors
+    assert generated["NMRlipids"]["name"] == "(2R,3S)-name"
+    assert generated["bioschema_properties"]["smiles"] == "CCCCCCCCOC@H]1[C@@HCO)O)O)O"
+    assert generated["bioschema_properties"]["alternateName"] == ["1-OD-glucopyranoside"]
+    assert generated["sameAs"]["ChEBI"] == "CHEBI:1234"
+    assert generated["sameAs"]["pdb.ligand"] == "BOG"
+    assert generated["sameAs"]["unii"] == "V109WUT6RL"
+    assert generated["sameAs"]["metabolights"] == "MTBLC1234"
+    assert generated["sameAs"]["cas"] == "29836-26-8"
+
+
+@pytest.mark.network
+def test_autocomplete_sameas_from_live_apis():
+    """Live end-to-end check that the real APIs yield the expected cross references.
+
+    Uses beta-octyl D-glucopyranoside (BOG). Skipped automatically when the
+    external services are unreachable.
+    """
+    mod = load_autocomplete_module()
+
+    inchikey = "HEGSGKPQLMEBJL-RKQHYHRCSA-N"
+
+    sources = mod.get_unichem(inchikey)
+    if not sources:
+        pytest.skip("UniChem API unreachable; skipping live network test.")
+
+    sameas = mod.sanitize_sameas(mod.extract_sameas(sources))
+    chebi_id = sameas.get("ChEBI", "").replace("CHEBI:", "")
+    if chebi_id and "metabolights" not in sameas:
+        metabolights_id = mod.get_metabolights(chebi_id)
+        if metabolights_id:
+            sameas["metabolights"] = metabolights_id
+
+    expected = {
+        "ChEBI": "CHEBI:41128",
+        "pubchem.compound": 62852,
+        "metabolights": "MTBLC41128",
+        "pdb.ligand": "BOG",
+        "ChEMBL": "CHEMBL446037",
+    }
+    for key, value in expected.items():
+        assert sameas.get(key) == value, f"{key}: expected {value!r}, got {sameas.get(key)!r}"
+
+    # CAS Common Chemistry requires an API token; only verify when CAS_API_KEY is set.
+    if os.environ.get("CAS_API_KEY"):
+        cas_rn = mod.get_cas(inchikey)
+        if cas_rn:
+            assert re.match(r"^\d{1,7}-\d{2}-\d$", cas_rn), f"unexpected CAS format: {cas_rn!r}"