enable case insensitive homonym lookups

Author: guuswilmink

src/kotobuki/mapping_updater/cli.py

@@ -39,6 +39,14 @@
     "try to find it through concepts with the same concept name. It is "
     "strongly recommended to index the concept_name column when using this.",
 )
+@click.option(
+    "--case-insensitive-homonyms",
+    is_flag=True,
+    default=False,
+    help="When searching for homonyms, do so in a case-insensitive way. "
+    "It is strongly recommended to create a functional index on the "
+    "concept table for this.",
+)
 @click.option(
     "-m",
     "--write-map-paths",
@@ -73,6 +81,7 @@ def _update_usagi_cli(
     schema: str,
     usagi_file: Path,
     allow_homonyms: bool,
+    case_insensitive_homonyms: bool,
     write_map_paths: bool,
     inspect_only: bool,
     overwrite: bool,
@@ -87,6 +96,7 @@ def _update_usagi_cli(
         schema,
         usagi_file,
         allow_homonyms,
+        case_insensitive_homonyms,
         write_map_paths,
         inspect_only,
         overwrite,

src/kotobuki/mapping_updater/db.py

@@ -1,7 +1,7 @@
 from collections.abc import Sequence
 
 from omop_cdm.regular.cdm54 import Concept, ConceptRelationship
-from sqlalchemy import and_, select
+from sqlalchemy import and_, func, select
 from sqlalchemy.orm import Session
 
 from .relationship import (
@@ -65,8 +65,14 @@ def find_standard_concepts(
     return None
 
 
-def find_all_homonyms(concept_name: str, session: Session) -> Sequence[Concept]:
+def find_all_homonyms(
+    concept_name: str, case_insensitive: bool, session: Session
+) -> Sequence[Concept]:
     """Get Concept ORM objects that match the concept_name."""
+    if case_insensitive:
+        return session.scalars(
+            select(Concept).where(func.lower(Concept.concept_name) == concept_name.lower())
+        ).all()
     return session.scalars(select(Concept).where(Concept.concept_name == concept_name)).all()
 
 
@@ -75,7 +81,9 @@ def find_suitable_homonym(
 ) -> NewMap | None:
     """Return first homonym concept that maps to a standard concept."""
     start_path = [MapLink(concept)]
-    for h in homonyms:
+    # Exclude the concept for which we are trying to find a mapping
+    other_homonyms = [h for h in homonyms if h.concept_id != concept.concept_id]
+    for h in other_homonyms:
         path = start_path.copy()
         path.append(MapLink(h, Relationship.HOMONYM))
         new_map = find_standard_concepts(h.concept_id, session, path)
@@ -84,11 +92,16 @@ def find_suitable_homonym(
     return None
 
 
-def find_new_mapping(concept: Concept, search_homonyms: bool, session: Session) -> NewMap | None:
+def find_new_mapping(
+    concept: Concept,
+    search_homonyms: bool,
+    case_insensitive_homonyms: bool,
+    session: Session,
+) -> NewMap | None:
     # Try to find standard concepts via concept relationships
     new_map = find_standard_concepts(concept.concept_id, session, [MapLink(concept)])
     # Alternatively via concepts with an identical name
     if search_homonyms and new_map is None:
-        homonyms = find_all_homonyms(concept.concept_name, session)
+        homonyms = find_all_homonyms(concept.concept_name, case_insensitive_homonyms, session)
         new_map = find_suitable_homonym(homonyms, session, concept)
     return new_map

src/kotobuki/mapping_updater/update_usagi.py

@@ -30,6 +30,7 @@ def update_usagi_file(
     vocab_schema: str,
     usagi_file: Path,
     allow_homonyms: bool = False,
+    case_insensitive_homonyms: bool = False,
     write_map_paths: bool = False,
     inspect_only: bool = False,
     overwrite: bool = False,
@@ -87,7 +88,7 @@ def update_usagi_file(
 
         logger.info("Querying database for standard concepts...")
         for concept in non_standard:
-            new_map = find_new_mapping(concept, allow_homonyms, session)
+            new_map = find_new_mapping(concept, allow_homonyms, case_insensitive_homonyms, session)
             new_mappings[concept.concept_id] = new_map
         log_remapped_concepts(new_mappings)
 

tests/python/mapping_updater/db_setup/vocab_data/CONCEPT.csv

@@ -18,3 +18,5 @@ concept_id	concept_name	domain_id	vocabulary_id	concept_class_id	standard_concep
 16	val2	0	0	0	S	v2	2017-01-01	2099-12-31	
 17	qwerty	Condition	0	0	S	qwerty	2017-01-01	2099-12-31	
 18	azerty	Condition	0	0		azerty	2017-01-01	2099-12-31	
+19	Qzerty	Condition	0	0		Qzerty	2017-01-01	2099-12-31	
+20	Dvorak	Condition	0	0	S	Dvorak	2017-01-01	2099-12-31	

tests/python/mapping_updater/db_setup/vocab_data/CONCEPT_RELATIONSHIP.csv

@@ -12,3 +12,4 @@ concept_id_1	concept_id_2	relationship_id	valid_start_date	valid_end_date	invali
 13	14	Maps to	1970-01-01	2099-12-31	
 13	15	Maps to value	1970-01-01	2099-12-31	
 13	16	Maps to value	1970-01-01	2099-12-31	
+19	20	Maps to	2017-01-01	2099-12-31	

tests/python/mapping_updater/test_case_insensitive_homonym.py

@@ -0,0 +1,25 @@
+import pytest
+from omop_cdm.regular.cdm54 import Concept
+from sqlalchemy import Engine
+
+from kotobuki.mapping_updater.db import find_new_mapping
+from kotobuki.mapping_updater.relationship import MapLink, Relationship
+from tests.python.mapping_updater.test_usagi_mappings import get_new_map
+
+pytestmark = pytest.mark.usefixtures("create_vocab_tables")
+
+
+def test_find_standard_concept_via_case_insensitive_homonym(pg_db_engine: Engine):
+    """
+    The concept is non-standard and has a 'Maps to value' relationship to a
+    standard concept.
+    """
+    result = get_new_map(
+        concept_id=19,
+        engine=pg_db_engine,
+        homonyms=True,
+        case_insensitive_homonyms=True,
+    )
+    assert len(result.concepts) == 1
+    assert result.concepts[0].concept_id == 20
+    assert not result.value_as_concept

tests/python/mapping_updater/test_usagi_mappings.py

@@ -13,10 +13,20 @@ def get_concept_by_id(session: Session, concept_id: int) -> Concept | None:
     return session.scalars(select(Concept).filter(Concept.concept_id == concept_id)).one_or_none()
 
 
-def get_new_map(concept_id: int, engine: Engine, homonyms: bool = False) -> NewMap | None:
+def get_new_map(
+    concept_id: int,
+    engine: Engine,
+    homonyms: bool = False,
+    case_insensitive_homonyms: bool = False,
+) -> NewMap | None:
     with Session(engine, expire_on_commit=False) as session, session.begin():
         concept = get_concept_by_id(session, concept_id=concept_id)
-        return find_new_mapping(concept=concept, search_homonyms=homonyms, session=session)
+        return find_new_mapping(
+            concept=concept,
+            search_homonyms=homonyms,
+            case_insensitive_homonyms=case_insensitive_homonyms,
+            session=session,
+        )
 
 
 def test_no_map(pg_db_engine: Engine):