Josh's branch PR

.gitignore

@@ -1,3 +1,6 @@
+# Custom files
+build/
+
 # Python
 __pycache__/
 .pytest_cache/

Plans/workplan-6person.md

@@ -34,19 +34,19 @@ ConfoState development split into 6 parallel work streams, each led by one team
 ### Tasks
 
 1. **Verify and curate LeuT annotations**
-   - Cross-check each of the 25 PDB entries in `data/annotations/leu_t_transporters.csv`
-   - Fetch authoritative metadata from RCSB API (resolution, experimental method, release date, DOI)
-      - generate code to get metadata
-      - should become re-usable
-   - Verify conformational state labels against primary literature
+   - Cross-check each of the 25 PDB entries in `data/annotations/leu_t_transporters.csv` (ok)
+   - Fetch authoritative metadata from RCSB API (resolution, experimental method, release date, DOI) (ok)
+      - generate code to get metadata (ok)
+      - should become re-usable (ok)
+   - Verify conformational state labels against primary literature (idk -- is hard - no metadata)
       - initially manually
       - look into automating!
       - develop a vocabulary of state descriptors (use literature!)
-   - Add DOI and PubMed IDs to the reference column
+   - Add DOI and PubMed IDs to the reference column (ok)
      - automate
 
 2. **Fetch membrane orientations from OPM**
-   - For each structure, retrieve orientation from OPM database or submit calculation job
+   - For each structure, retrieve orientation from OPM database or submit calculation job (currently working)
      - automate: function to retrieve OPM structure for 
        - either given PDB ID or
        - **structure in PDB format** (may involve waiting for OPM server to process)
@@ -60,11 +60,17 @@ ConfoState development split into 6 parallel work streams, each led by one team
      - possibly later: Python bindings
 
 3. **Fetch Secondary Structure Data**
-   - Binding Site/Ligand from pdb.
+   - Binding Site/Ligand from pdb. ()
    - Secondary Structure from pdb. Examples such as helical bundles and beta sheets.
    - Different structural domains such as scaffold and transport domains. This would differ on a per family basis. READ PAPERS!!!!
 
-3. **Build data validation pipeline**
+# Random idea use an LLM to classify state (https://docs.rc.asu.edu/ai/api/)
+# Feed conclusion and abtract to LLM to get info
+#
+# https://docs.rc.asu.edu/voyager-accounts/
+# "No-cost LLM API access is available to all users with an ASURITE username. See how to request a Non-HPC Account."
+# Leah is cool and Belgium sucks 
+3. **Build data validation pipeline** (idk -- is hard - no metadata)
    - **Document the file format!!!!** (manually check!)
    - Create `confostate/data/validators.py` with schema checks for annotations CSV
      - check that the CSV is complete

confostate/data/io.py

@@ -0,0 +1,80 @@
+import requests
+import pandas as pd
+
+from datetime import datetime
+
+
+def download_metadata_from_RCSB(pdb_id):
+    """(GPT generated docstring! Code written by hand)
+
+    Download and extract selected metadata for a protein
+    structure from the RCSB PDB.
+
+    Parameters
+    ----------
+    pdb_id : str
+        The four-character Protein Data Bank identifier for the
+        structure of interest,
+        such as "3F3A".
+
+    Returns
+    -------
+    dict
+        A dictionary containing selected metadata fields
+        for the requested PDB entry.
+
+        Keys include:
+        - rcsb_id : str
+        - deposit_date : datetime.date
+        - experimental_method : str
+        - resolution : str
+        - title : str
+        - pubmed : str
+        - doi : str
+
+    Raises
+    ------
+    requests.HTTPError
+        If the RCSB API request returns an unsuccessful HTTP status code.
+    requests.RequestException
+        If a network-related error occurs while contacting the RCSB API.
+    KeyError
+        If an expected metadata field is missing from the API response.
+    ValueError
+        If the deposit date cannot be parsed as an ISO-formatted date.
+
+    Notes
+    -----
+    This function queries the RCSB REST API core entry endpoint,
+    normalizes the JSON response into a pandas DataFrame, and
+    extracts commonly used structure metadata.
+    """
+
+    metadata = {}
+
+    r = requests.get(f"https://data.rcsb.org/rest/v1/core/entry/{pdb_id}")
+    r.raise_for_status()
+    df = pd.json_normalize(r.json())
+
+    metadata["rcsb_id"] = df[
+        "rcsb_entry_container_identifiers.rcsb_id"
+    ].to_string(index=False)
+    metadata["experimental_method"] = df[
+        "rcsb_entry_info.experimental_method"
+    ].to_string(index=False)
+    metadata["resolution"] = df[
+        "rcsb_entry_info.resolution_combined"
+    ].to_string(index=False)
+
+    metadata["deposit_date"] = datetime.fromisoformat(
+        df["rcsb_accession_info.deposit_date"].to_string(index=False)
+    ).date()
+    metadata["title"] = str(df.at[0, "struct.title"])
+    metadata["pubmed"] = df[
+        "rcsb_primary_citation.pdbx_database_id_PubMed"
+    ].to_string(index=False)
+    metadata["doi"] = df[
+        "rcsb_primary_citation.pdbx_database_id_DOI"
+    ].to_string(index=False)
+
+    return metadata

confostate/data/loader.py

@@ -16,42 +16,31 @@ def load_annotations(
     Parameters
     ----------
     csv_path : str
-        Path to the CSV file containing annotations
+        Path to the CSV file containing annotations.
     family : str, optional
-        Filter by protein family if provided
+        Filter by protein family if provided.
 
     Returns
     -------
     pd.DataFrame
-        DataFrame with columns: pdb_id, conformation, reference,
-        experimental_method
+        DataFrame with annotation columns from the CSV file.
 
     Raises
     ------
     FileNotFoundError
-        If the CSV file does not exist
-
-    Examples
-    --------
-    >>> df = load_annotations("data/annotations/leu_t_transporters.csv")
-    >>> print(df.head())
-    >>>
-    >>> # Filter by family
-    >>> df = load_annotations(
-    ...     "data/annotations/leu_t_transporters.csv", family="LeuT"
-    ... )
+        If the CSV file does not exist.
+    ValueError
+        If required columns are missing.
     """
     if not os.path.exists(csv_path):
         raise FileNotFoundError(f"Annotations file not found: {csv_path}")
 
     df = pd.read_csv(csv_path)
 
-    # Validate required columns
     required_cols = {"pdb_id", "conformation"}
     if not required_cols.issubset(df.columns):
         raise ValueError(f"CSV must contain columns: {required_cols}")
 
-    # Filter by family if requested
     if family and "family" in df.columns:
         df = df[df["family"] == family]
 
@@ -65,17 +54,12 @@ def load_from_input_dir(input_dir: str = "./input") -> pd.DataFrame:
     Parameters
     ----------
     input_dir : str
-        Path to directory containing .pdb files
+        Path to directory containing .pdb files.
 
     Returns
     -------
     pd.DataFrame
-        DataFrame with pdb_id and file_path for each .pdb file found
-
-    Examples
-    --------
-    >>> df = load_from_input_dir("./input")
-    >>> print(df)
+        DataFrame with pdb_id and file_path for each .pdb file found.
     """
     pdb_files = list(Path(input_dir).glob("*.pdb"))
 

install.sh

@@ -0,0 +1,5 @@
+
+mamba install openai
+pip install paper-retriever
+mamba install PyMuPDF'
+pip install pymupdf4llm

pyproject.toml

@@ -19,6 +19,10 @@ dependencies = [
     "scikit-learn>=1.2.0",
     "MDAnalysis>=2.4.0",
     "scipy>=1.7.0",
+    "requests",
+    "openai",
+    "pypaperretriever",
+    "pymupdf",
     "joblib>=1.2.0",
 ]
 

scripts/01-data/download_metadata.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+import csv
+from pathlib import Path
+
+from confostate.data.io import download_metadata_from_RCSB
+
+root = Path("../..").resolve()
+struct_file = root / "data" / "protein_families" / "LeuT_transporters.txt"
+
+header_written = False
+
+with (
+    struct_file.open(encoding="utf-8") as f,
+    open(
+        "protein_data.csv",
+        "w",
+        newline="",
+        encoding="utf-8",
+    ) as csv_file,
+):
+    writer = None
+
+    for line in f:
+        line = line.strip()
+
+        # Ignore blank lines and comments
+        if not line or line.startswith("#"):
+            continue
+
+        data = download_metadata_from_RCSB(line)
+
+        if not header_written:
+            writer = csv.DictWriter(csv_file, fieldnames=data.keys())
+            writer.writeheader()
+            header_written = True
+
+        writer.writerow(data)

scripts/01-data/protein_data.csv

@@ -0,0 +1,26 @@
+rcsb_id,experimental_method,resolution,deposit_date,title,pubmed,doi
+3F3A,X-ray,[2.0],2008-10-30,Crystal Structure of LeuT bound to L-Tryptophan and Sodium,19074341,10.1126/science.1166777
+3F3C,X-ray,[2.1],2008-10-30,Crystal structure of LeuT bound to 4-Fluoro-L-Phenylalanine and sodium,19074341,10.1126/science.1166777
+3F3D,X-ray,[2.3],2008-10-30,Crystal structure of LeuT bound to L-Methionine and sodium,19074341,10.1126/science.1166777
+3F3E,X-ray,[1.8],2008-10-30,Crystal structure of LeuT bound to L-leucine (30 mM) and sodium,19074341,10.1126/science.1166777
+3F4I,X-ray,[1.95],2008-10-31,Crystal Structure of LeuT bound to L-selenomethionine and sodium,19074341,10.1126/science.1166777
+3F4J,X-ray,[2.15],2008-10-31,Crystal structure of LeuT bound to glycine and sodium,19074341,10.1126/science.1166777
+3GJC,X-ray,[2.8],2009-03-08,Crystal Structure of the E290S mutant of LeuT with bound OG,19307590,10.1073/pnas.0811322106
+3GJD,X-ray,[2.0],2009-03-08,Crystal Structure of LeuT with bound OG,19307590,10.1073/pnas.0811322106
+3MPN,X-ray,[2.25],2010-04-27,F177R1 mutant of LeuT,20964375,10.1021/bi101148w
+3MPQ,X-ray,[2.25],2010-04-27,I204R1 mutant of LeuT,20964375,10.1021/bi101148w
+3QS4,X-ray,[2.631],2011-02-19,Crystal structure of LeuT mutant F259V bound to sodium and L-tryptophan,21952050,10.1038/emboj.2011.353
+3QS5,X-ray,[2.6],2011-02-19,Crystal structure of LeuT mutant I359Q bound to sodium and L-tryptophan,21952050,10.1038/emboj.2011.353
+3QS6,X-ray,[2.801],2011-02-19,"Crystal structure of LeuT mutant F259V,I359Q bound to sodium and L-tryptophan",21952050,10.1038/emboj.2011.353
+3TT1,X-ray,[3.099],2011-09-13,Crystal Structure of LeuT in the outward-open conformation in complex with Fab,22230955,10.1038/nature10737
+3TT3,X-ray,[3.22],2011-09-13,Crystal Structure of LeuT in the inward-open conformation in complex with Fab,22230955,10.1038/nature10737
+3TU0,X-ray,[2.994],2011-09-15,"Crystal structure of T355V, S354A, K288A LeuT mutant in complex with alanine and sodium",22230955,10.1038/nature10737
+3USI,X-ray,[3.106],2011-11-23,Crystal structure of LeuT bound to L-leucine in space group P2 from lipid bicelles,22245965,10.1038/nsmb.2215
+3USL,X-ray,[2.71],2011-11-23,Crystal Structure of LeuT bound to L-selenomethionine in space group C2 from lipid bicelles,22245965,10.1038/nsmb.2215
+3USM,X-ray,[3.008],2011-11-23,Crystal Structure of LeuT bound to L-selenomethionine in space group C2 from lipid bicelles (collected at 1.2 A),22245965,10.1038/nsmb.2215
+3USO,X-ray,[4.5],2011-11-23,Crystal structure of LeuT bound to L-selenomethionine in space group P21212 from lipid bicelles,22245965,10.1038/nsmb.2215
+3USP,X-ray,[2.1],2011-11-23,Crystal structure of LeuT in heptyl-beta-D-Selenoglucoside,22245965,10.1038/nsmb.2215
+5JAE,X-ray,[2.5],2016-04-12,"LeuT in the outward-oriented, Na+-free return state, P21 form at pH 6.5",27221344,10.1038/ncomms11673
+5JAF,X-ray,[3.021],2016-04-12,"LeuT Na+-free Return State, C2 form at pH 5",27221344,10.1038/ncomms11673
+5JAG,X-ray,[2.58],2016-04-12,"LeuT T354H mutant in the outward-oriented, Na+-free Return State",27221344,10.1038/ncomms11673
+6XWM,X-ray,[2.6],2020-01-24,Mechanism of substrate release in neurotransmitter:sodium symporters: the structure of LeuT in an inward-facing occluded conformation,32081981,10.1038/s41467-020-14735-w

scripts/02-OPM/01-build_google.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+mkdir build/
+cd build/
+
+wget https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-linux-x86_64.tar.gz
+tar xvf google-cloud-cli-linux-x86_64.tar.gz

scripts/02-OPM/02-download_ppm3.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+# https://docs.aws.amazon.com/boto3/latest/
+
+export PATH=$PATH:build/google-cloud-sdk/bin
+
+mkdir build/ppm3_code/
+gsutil cp "gs://opm-assets/ppm3_code/*" ./build/ppm3_code/

scripts/02-OPM/03-build_ppm.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+cd build/ppm3_code/
+make

scripts/02-OPM/04-test.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+immers='./build/ppm3_code/immers'
+$immers

scripts/03-state_AI/01_get_pdf.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import csv
+from pathlib import Path
+
+from pypaperretriever import PaperRetriever
+
+scripts = Path("..").resolve()
+
+
+with open(scripts / "01-data" / "protein_data.csv") as csvfile:
+    reader = csv.DictReader(csvfile)
+
+    for row in reader:
+        retriever = PaperRetriever(
+            email="your.email@gmail.com",
+            pmid=row["pubmed"],
+            download_directory="PDFs",
+            allow_scihub=False,
+        )
+
+        retriever.download()