INITIAL RUN COMPLETE AND 100 CASES MANUALLY VERIFIED

Author: josephrich98

data/radiology_db.csv

@@ -56,8 +56,8 @@ def get_model() -> str:
 DATASET_AVAILABILITY_INSTRUCTIONS = (
     "Determine whether the paper indicates that its dataset is publicly available.\n"
     "Return is_publicly_available = true if there is direct evidence in the provided text such as:\n"
-    "- explicit language like 'open', 'publicly available', 'public', 'available', 'release', or a data availability statement\n"
-    "- a non-DOI URL (e.g., GitHub, Zenodo, institutional repository, challenge site, or dataset website)\n"
+    "- explicit language like 'open', 'publicly available', 'public', 'available', 'release', 'accessible', 'open-access', or a data availability statement\n"
+    "- a non-DOI URL (e.g., GitHub, Zenodo, Kaggle, institutional repository, challenge site, or dataset website)\n"
     "- wording that readers can access or download the dataset\n\n"
     "Return false if:\n"
     "- the text only says data are available on request\n"

notebooks/db_analysis.ipynb

@@ -102,12 +102,12 @@ def compare_dbs(
     right_titles = _unique_title_set(df_right, merge_col)
 
     report = {
-        "unique_titles_left": len(left_titles),
-        "unique_titles_right": len(right_titles),
-        "unique_titles_combined": len(left_titles | right_titles),
+        "titles_left": len(left_titles),
+        "titles_right": len(right_titles),
+        "titles_union": len(left_titles | right_titles),
         "titles_only_left": len(left_titles - right_titles),
         "titles_only_right": len(right_titles - left_titles),
-        "titles_in_both": len(left_titles & right_titles),
+        "titles_intersection": len(left_titles & right_titles),
     }
 
     # merged_df = df_left.merge(
@@ -119,7 +119,6 @@ def compare_dbs(
     # )
     return report
 
-
 def verified_unverified_report(
     df_left: pd.DataFrame,
     df_right: pd.DataFrame,
@@ -137,7 +136,7 @@ def verified_unverified_report(
 
     def build_sets(df: pd.DataFrame) -> Tuple[Set[str], Set[str]]:
         valid = df[[merge_col, verified_col]].copy()
-        valid["_verified_bool"] = valid[verified_col].apply(_to_bool)
+        valid["_verified_bool"] = valid[verified_col].apply(_to_bool).astype("boolean")
         valid = valid.dropna(subset=[merge_col, "_verified_bool"])
         valid[merge_col] = valid[merge_col].astype(str).str.strip()
         valid = valid[valid[merge_col] != ""]
@@ -150,16 +149,22 @@ def build_sets(df: pd.DataFrame) -> Tuple[Set[str], Set[str]]:
     right_verified, right_unverified = build_sets(df_right)
 
     return {
-        "verified_unique_left": len(left_verified),
-        "verified_unique_right": len(right_verified),
-        "verified_unique_combined": len(left_verified | right_verified),
+        "total_left": len(left_verified) + len(left_unverified),
+        "total_right": len(right_verified) + len(right_unverified),
+        "verified_left": len(left_verified),
+        "verified_right": len(right_verified),
+        "verified_union": len(left_verified | right_verified),
         "verified_only_left": len(left_verified - right_verified),
         "verified_only_right": len(right_verified - left_verified),
-        "unverified_unique_left": len(left_unverified),
-        "unverified_unique_right": len(right_unverified),
-        "unverified_unique_combined": len(left_unverified | right_unverified),
+        "verified_intersection": len(left_verified & right_verified),
+        "unverified_left": len(left_unverified),
+        "unverified_right": len(right_unverified),
+        "unverified_union": len(left_unverified | right_unverified),
         "unverified_only_left": len(left_unverified - right_unverified),
         "unverified_only_right": len(right_unverified - left_unverified),
+        "unverified_intersection": len(left_unverified & right_unverified),
+        "fraction_verified_left": len(left_verified) / (len(left_verified) + len(left_unverified)) if (len(left_verified) + len(left_unverified)) > 0 else float("nan"),  # aka PPV
+        "fraction_verified_right": len(right_verified) / (len(right_verified) + len(right_unverified)) if (len(right_verified) + len(right_unverified)) > 0 else float("nan"),  # aka PPV
     }
 
 

radiology_dataset_db/config.py

@@ -51,6 +51,7 @@ class BulkGenomicsDataset(BaseModel):
     pmid: Optional[str] = None
     paper_citation_count: Optional[int] = None
     mesh_terms: List[str] = Field(default_factory=list)
+    keywords: List[str] = Field(default_factory=list)
     pubmed_matches: Optional[List[List[str]]] = None
 
 
@@ -121,6 +122,7 @@ async def extract_bulk_genomics_dataset_info_with_agent(
             output.pmid = publication_metadata.get("pmid")
             output.paper_citation_count = publication_metadata.get("citation_count")
             output.mesh_terms = publication_metadata.get("mesh_terms")
+            output.keywords = publication_metadata.get("keywords")
             output.pubmed_matches = publication_metadata.get("pubmed_matches")
 
             if not output.name:

radiology_dataset_db/db_validation.py

@@ -70,6 +70,7 @@ class RadiologyDataset(BaseModel):
     pmid: Optional[str] = None
     paper_citation_count: Optional[int] = None
     mesh_terms: List[str] = Field(default_factory=list)
+    keywords: List[str] = Field(default_factory=list)
     pubmed_matches: Optional[List[List[str]]] = None
 
 
@@ -193,6 +194,7 @@ async def extract_radiology_dataset_info_with_agent(
             output.pmid = publication_metadata.get("pmid")
             output.paper_citation_count = publication_metadata.get("citation_count")
             output.mesh_terms = publication_metadata.get("mesh_terms")
+            output.keywords = publication_metadata.get("keywords")
             output.pubmed_matches = publication_metadata.get("pubmed_matches")
 
             if not output.name:

radiology_dataset_db/extract_bulk_genomics_dataset_information_llm.py

@@ -60,6 +60,7 @@ class ScRNASeqDataset(BaseModel):
     pmid: Optional[str] = None
     paper_citation_count: Optional[int] = None
     mesh_terms: List[str] = Field(default_factory=list)
+    keywords: List[str] = Field(default_factory=list)
     pubmed_matches: Optional[List[List[str]]] = None
 
 
@@ -128,6 +129,7 @@ async def extract_scrnaseq_dataset_info_with_agent(
             output.pmid = publication_metadata.get("pmid")
             output.paper_citation_count = publication_metadata.get("citation_count")
             output.mesh_terms = publication_metadata.get("mesh_terms")
+            output.keywords = publication_metadata.get("keywords")
             output.pubmed_matches = publication_metadata.get("pubmed_matches")
 
             if not output.name:

radiology_dataset_db/extract_radiology_dataset_information_llm.py

@@ -64,6 +64,7 @@ class SpatialTranscriptomicsDataset(BaseModel):
     pmid: Optional[str] = None
     paper_citation_count: Optional[int] = None
     mesh_terms: List[str] = Field(default_factory=list)
+    keywords: List[str] = Field(default_factory=list)
     pubmed_matches: Optional[List[List[str]]] = None
 
 
@@ -134,6 +135,7 @@ async def extract_spatial_transcriptomics_dataset_info_with_agent(
             output.pmid = publication_metadata.get("pmid")
             output.paper_citation_count = publication_metadata.get("citation_count")
             output.mesh_terms = publication_metadata.get("mesh_terms")
+            output.keywords = publication_metadata.get("keywords")
             output.pubmed_matches = publication_metadata.get("pubmed_matches")
 
             if not output.name:

radiology_dataset_db/extract_scrnaseq_dataset_information_llm.py

@@ -217,6 +217,7 @@ def search_pubmed(pubmed_query: str, max_results: Optional[int] = None, batch_si
 
 def fetch_pubmed_details(id_list, batch_size=200):
     results = []
+    id_list = [str(id) for id in id_list]  # ensure all IDs are strings
     for batch in tqdm(list(chunked(id_list, batch_size)), desc="Fetching PubMed details"):
         handle = Entrez.efetch(
             db="pubmed",
@@ -360,6 +361,16 @@ def _extract_mesh_terms(article) -> List[str]:
     except Exception:
         return []
 
+def _extract_keywords(article) -> List[str]:
+    try:
+        keyword_list = article["MedlineCitation"].get("KeywordList", [])
+        keywords = []
+        for kw_list in keyword_list:
+            keywords.extend([str(k) for k in kw_list])
+        return keywords
+    except Exception:
+        return []
+
 def _extract_pmid(article) -> Optional[str]:
     try:
         pmid = article["MedlineCitation"]["PMID"]
@@ -603,6 +614,7 @@ def extract_pubmed_metadata(article, citation_counts_by_pmid: Optional[Dict[str,
         "authors": _extract_authors(article),
         "journal": _extract_journal(article),
         "mesh_terms": mesh_terms,
+        "keywords": _extract_keywords(article),
         "pmid": pmid,
         "citation_count": citation_counts_by_pmid.get(pmid) if pmid else None,
         "pubmed_matches": pubmed_matches,