Make autocomplete_metadata.py emit schema-compliant molecule metadata and improve CI tests

.github/workflows/tests-all.yml

@@ -31,14 +31,23 @@ jobs:
 
     - name: run Python tests
       run: |
-        tox -e tests-all
-
-    - name: generate coverage.xml
-      run: coverage xml
+        tox -e tests-package
 
-    - name: upload to codecov.io
-      uses: codecov/codecov-action@v6
+  tests-develop:
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v6
       with:
-        fail_ci_if_error: true
-        files: tests/coverage.xml
-        token: ${{ secrets.CODECOV_TOKEN }}
+        fetch-depth: 0
+
+    - name: setup Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.13"
+        check-latest: true
+
+    - run: python -m pip install tox
+
+    - name: run develop (out-of-package) tests
+      run: |
+        tox -e tests-develop

.github/workflows/tests-develop.yml

@@ -0,0 +1,53 @@
+name: Testing Developmental Scripts
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - "developer/**"
+      - "tests/develop/**"
+      - "tox.ini"
+      - "pyproject.toml"
+      - "MANIFEST.in"
+  pull_request_target:
+    branches: [main]
+    paths:
+      - "developer/**"
+      - "tests/develop/**"
+      - "tox.ini"
+      - "pyproject.toml"
+      - "MANIFEST.in"
+
+permissions:
+  contents: read
+
+jobs:
+  tests:
+    if: github.repository == 'NMRLipids/FAIRMD_lipids' 
+    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.os }} / Python ${{ matrix.python-version }}
+    strategy:
+      matrix:
+        include:
+          - os: ubuntu-24.04
+            python-version: "3.10"
+          - os: ubuntu-24.04
+            python-version: "3.13"
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 0
+        repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+        ref: ${{ github.event.pull_request.head.sha || github.ref }}
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v6
+      with:
+        python-version: ${{ matrix.python-version }}
+        check-latest: true
+
+    - run: python -m pip install tox
+
+    - name: run Python tests
+      run: tox -e tests-develop
+

.github/workflows/tests-min.yml

@@ -1,14 +1,26 @@
-name: Tests
+name: Unit Tests
 
 on:
   push:
     branches: [main]
+    paths:
+      - "src/**"
+      - "tests/package/**"
+      - "tox.ini"
+      - "pyproject.toml"
+      - "MANIFEST.in"
   pull_request_target:
     branches: [main]
+    paths:
+      - "src/**"
+      - "tests/package/**"
+      - "tox.ini"
+      - "pyproject.toml"
+      - "MANIFEST.in"
 
 permissions:
   contents: read
-      
+
 jobs:
   tests:
     if: github.repository == 'NMRLipids/FAIRMD_lipids' 
@@ -41,9 +53,8 @@ jobs:
     - run: python -m pip install tox coverage[toml]
 
     - name: run Python tests
-      run: |
-        tox -e tests-min
-
+      run: tox -e tests-min
+
     - name: generate coverage.xml
       run: coverage xml
 

CONTRIBUTION.md

@@ -99,7 +99,8 @@ If you wish to test only specific functionalities, for example:
 ```bash
   tox -e lint          # code style
   tox -e tests-min     # unit tests of the main library
-  tox -e tests-all     # regression tests
+  tox -e tests-package # full package regression tests (requires GROMACS)
+  tox -e tests-develop # out-of-package tests for developer/ scripts
 ```
 
 You can also use `tox -e format` to use tox to do actual formatting instead of just

developer/autocomplete_metadata.py

@@ -8,15 +8,19 @@
 - ChEMBL
 - ChEBI
 - PubChem
+- CAS Common Chemistry (requires the ``CAS_API_KEY`` environment variable)
 
 .. note::
    This file is meant to be used by automated workflows.
 """
 
 import json
 import os
+import re
 import sys
+import urllib.parse
 import urllib.request
+from html import unescape
 
 import yaml
 
@@ -89,6 +93,43 @@ def get_chebi(chebi_id):
     return {}
 
 
+def get_metabolights(chebi_id):
+    # MetaboLights reference compounds use the accession MTBLC<chebi numeric id>.
+    # Return the identifier only if the compound exists in MetaboLights.
+    if not chebi_id:
+        return ""
+    mtbl_id = f"MTBLC{chebi_id}"
+    if check_api(f"https://www.ebi.ac.uk/metabolights/ws/compounds/{mtbl_id}"):
+        return mtbl_id
+    return ""
+
+
+def get_cas(inchikey):
+    # CAS Registry Numbers are not exposed by UniChem. They can be retrieved from
+    # CAS Common Chemistry, which requires an API token supplied via the
+    # CAS_API_KEY environment variable. Returns "" when the token is missing,
+    # the service is unreachable, or no match is found.
+    if not inchikey:
+        return ""
+    api_key = os.environ.get("CAS_API_KEY")
+    if not api_key:
+        return ""
+    # CAS Common Chemistry requires field-qualified queries; a bare InChIKey
+    # does not match, whereas "InChIKey=<value>" does.
+    query = urllib.parse.quote(f"InChIKey={inchikey}")
+    url = f"https://commonchemistry.cas.org/api/search?q={query}"
+    try:
+        req = urllib.request.Request(url, headers={"X-Api-Key": api_key})
+        with urllib.request.urlopen(req, timeout=5) as response:
+            if response.status == 200:
+                results = json.loads(response.read().decode("utf-8")).get("results", [])
+                if results:
+                    return results[0].get("rn", "") or ""
+    except Exception:
+        pass
+    return ""
+
+
 def get_unichem(inchikey):
     url = "https://www.ebi.ac.uk/unichem/api/v1/compounds"
     if check_api("https://www.ebi.ac.uk/unichem/api/v1/sources"):
@@ -115,8 +156,9 @@ def extract_sameas(sources):
         "lipidmaps": "lipidmaps",
         "metabolights": "metabolights",
         "swisslipids": "slm",
-        "pdb": "pdb.ligand",
-        "unii": "unii",
+        "rcsb_pdb": "pdb.ligand",
+        "pdbe": "pdb.ligand",
+        "fdasrs": "unii",
         "cas": "cas",
     }
     result = {}
@@ -125,7 +167,10 @@ def extract_sameas(sources):
         if prefix:
             value = src["compoundId"]
             if prefix == "ChEBI":
-                value = f"CHEBI:{value}" if value else ""
+                if value:
+                    value = value if str(value).startswith("CHEBI:") else f"CHEBI:{value}"
+                else:
+                    value = ""
             elif prefix == "pubchem.compound":
                 try:
                     value = int(value)
@@ -142,6 +187,55 @@ def get_chembl_id_from_unichem(sources):
     return None
 
 
+def clean_text(value):
+    if not isinstance(value, str):
+        return value
+    return re.sub(r"<[^>]+>", "", unescape(value)).strip()
+
+
+def safe_float(value):
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def sanitize_sameas(sameas):
+    patterns = {
+        "ChEBI": r"^CHEBI:\d+$",
+        "ChEMBL": r"^CHEMBL\d+$",
+        "lipidmaps": r"^LM(FA|GL|GP|SP|ST|PR|SL|PK)[0-9]{4}([0-9a-zA-Z]{4,6})?$",
+        "metabolights": r"^MTBL[CS]\d+$",
+        "slm": r"^SLM:\d+$",
+        "pdb.ligand": r"^[A-Za-z0-9]+$",
+        "unii": r"^[A-Z0-9]+$",
+        "cas": r"^\d{1,7}-\d{2}-\d$",
+    }
+    sanitized = {}
+    for key, value in sameas.items():
+        if key == "pubchem.compound":
+            if isinstance(value, int):
+                sanitized[key] = value
+            else:
+                try:
+                    sanitized[key] = int(value)
+                except (TypeError, ValueError):
+                    print(
+                        f"Warning: discarding sameAs '{key}' value {value!r}: not a valid integer.",
+                        file=sys.stderr,
+                    )
+            continue
+        pattern = patterns.get(key, r".+")
+        if isinstance(value, str) and re.match(pattern, value):
+            sanitized[key] = value
+        else:
+            print(
+                f"Warning: discarding sameAs '{key}' value {value!r}: does not match expected pattern {pattern!r}.",
+                file=sys.stderr,
+            )
+    return sanitized
+
+
 def load_existing_metadata(path):
     if os.path.exists(path):
         with open(path, "r", encoding="utf-8") as f:
@@ -188,7 +282,7 @@ def main():
     chembl = get_chembl(inchikey)
     pubchem = get_pubchem(inchikey)
     sources = get_unichem(inchikey)
-    sameas = extract_sameas(sources)
+    sameas = sanitize_sameas(extract_sameas(sources))
 
     cid = pubchem.get("CID", sameas.get("pubchem.compound"))
     synonyms = get_pubchem_synonyms(cid) if cid else []
@@ -197,6 +291,19 @@ def main():
     chebi_id = sameas.get("ChEBI", "").replace("CHEBI:", "")
     chebi_data = get_chebi(chebi_id) if chebi_id else {}
 
+    # MetaboLights is not exposed by UniChem; derive it from the ChEBI id.
+    if chebi_id and "metabolights" not in sameas:
+        metabolights_id = get_metabolights(chebi_id)
+        if metabolights_id:
+            sameas["metabolights"] = metabolights_id
+
+    # CAS Registry Numbers are not exposed by UniChem; fetch them from CAS
+    # Common Chemistry (requires the CAS_API_KEY environment variable).
+    if "cas" not in sameas:
+        cas_rn = get_cas(inchikey)
+        if cas_rn and re.match(r"^\d{1,7}-\d{2}-\d$", cas_rn):
+            sameas["cas"] = cas_rn
+
     # Collect alternate names with priority
     alternate_names = []
 
@@ -216,6 +323,7 @@ def main():
     # 3. If still no synonyms, try PubChem synonyms
     if not alternate_names and synonyms:
         alternate_names = synonyms
+    alternate_names = [clean_text(name) for name in alternate_names if clean_text(name)]
 
     molecule_props = chembl.get("molecule_properties", {})
     molecule_structures = chembl.get("molecule_structures", {})
@@ -229,14 +337,23 @@ def main():
     else:
         image_url = ""
 
+    nmr_name = (
+        existing.get("NMRlipids", {}).get("name")
+        or clean_text(chembl.get("pref_name", ""))
+        or clean_text(molecule_props.get("iupac_name", ""))
+        or clean_text(pubchem.get("IUPACName", ""))
+        or nmr_id
+    )
+
     bioschema = {
-        "name": molecule_props.get("iupac_name") or pubchem.get("IUPACName", ""),
-        "iupacName": molecule_props.get("iupac_name") or pubchem.get("IUPACName", ""),
+        "name": clean_text(molecule_props.get("iupac_name")) or clean_text(pubchem.get("IUPACName", "")),
+        "iupacName": clean_text(molecule_props.get("iupac_name")) or clean_text(pubchem.get("IUPACName", "")),
         "molecularFormula": molecule_props.get("full_molformula") or pubchem.get("MolecularFormula", ""),
-        "molecularWeight": float(molecule_props.get("full_mwt") or pubchem.get("MolecularWeight", 0)),
-        "inChI": molecule_structures.get("standard_inchi") or pubchem.get("InChI", ""),
-        "inChIKey": molecule_structures.get("standard_inchi_key") or pubchem.get("InChIKey", ""),
-        "smiles": molecule_structures.get("canonical_smiles") or pubchem.get("SMILES", ""),
+        "molecularWeight": safe_float(molecule_props.get("full_mwt") or pubchem.get("MolecularWeight")),
+        "inChI": clean_text(molecule_structures.get("standard_inchi")) or clean_text(pubchem.get("InChI", "")),
+        "inChIKey": clean_text(molecule_structures.get("standard_inchi_key"))
+        or clean_text(pubchem.get("InChIKey", "")),
+        "smiles": clean_text(molecule_structures.get("canonical_smiles")) or clean_text(pubchem.get("SMILES", "")),
         "image": image_url,
         "description": "",
     }
@@ -245,7 +362,7 @@ def main():
         bioschema["alternateName"] = alternate_names
 
     new_data = {
-        "NMRlipids": {"id": nmr_id, "name": "", "charge": ""},
+        "NMRlipids": {"id": nmr_id, "name": nmr_name},
         "sameAs": sameas,
         "bioschema_properties": bioschema,
     }

docs/src/miscsrc/unittesting.rst

@@ -15,7 +15,7 @@ Tests are organized via `tox <https://tox.wiki/>`_.
 
 .. code-block:: bash
 
- tox -e tests-all -- tests/test_load.py
+ tox -e tests-package -- tests/test_load.py
 
 During setting up the environment, tox will replicate ``ToyData`` from ``src/data/ToyData``
 to ``tests`` folder. You can remove ``tests/ToyData`` if you don't need it for debugging.

pyproject.toml

@@ -73,9 +73,6 @@ ignore = ["src/fairmd/lipids/_version.py"]
 testpaths = "tests"
 addopts = [
     "-ra",
-    "--cov=fairmd.lipids",
-    "--cov-append",
-    "--cov-report=",
     "--import-mode=append",
 ]
 markers = [
@@ -86,6 +83,7 @@ markers = [
     "rdkit: tests features requiring RDKit installation",
     "all: run all tests",
     "min: run tests not requiring GROMACS installation",
+    "develop: out-of-package tests for developer/ scripts (not part of the distributed package)",
 ]
 
 [tool.ruff]