Merge pull request #14 from thehyve/issue-11-case-insensitive-homonym-option

enable case insensitive homonym lookups

Author: guuswilmink

docs/kotobuki.md

@@ -40,6 +40,7 @@ update_usagi_file(
     vocab_schema="my_cdm",
     usagi_file=usagi_file,
     allow_homonyms=False,
+    ignore_case=False,
     write_map_paths=False,
     inspect_only=False,
     overwrite=False,
@@ -86,9 +87,16 @@ All available homonyms will be parsed for concept relationships (just like the o
 non-standard concept) until a standard concept is found or all relationship paths have
 been traversed.
 
+For some homonym concepts, the case may be different (e.g. diabetes type 2 vs Diabetes
+Type 2). It needs to be specified within the use of kotobuki whether such homonyms
+should be included.
+
 To include homonyms, add the `--allow-homonyms`/`-h` flag (CLI), or provide
 `allow_homonyms=True` (Python).
 
+To search for homonyms regardless of case, add the `--case-insensitive`/`-i`
+flag (CLI), or provide `case_insensitive=True` (Python).
+
 > ⚠️ **WARNING:**
 > Searching for standard concepts via homonyms is less reliable than via the concept
 > relationships, especially for concepts with a short name.

src/kotobuki/mapping_updater/cli.py

@@ -39,6 +39,15 @@
     "try to find it through concepts with the same concept name. It is "
     "strongly recommended to index the concept_name column when using this.",
 )
+@click.option(
+    "-i",
+    "--ignore-case",
+    is_flag=True,
+    default=False,
+    help="When searching for homonyms, do so in a case-insensitive way. "
+    "It is strongly recommended to create a functional index on the "
+    "concept table for this.",
+)
 @click.option(
     "-m",
     "--write-map-paths",
@@ -73,6 +82,7 @@ def _update_usagi_cli(
     schema: str,
     usagi_file: Path,
     allow_homonyms: bool,
+    ignore_case: bool,
     write_map_paths: bool,
     inspect_only: bool,
     overwrite: bool,
@@ -87,6 +97,7 @@ def _update_usagi_cli(
         schema,
         usagi_file,
         allow_homonyms,
+        ignore_case,
         write_map_paths,
         inspect_only,
         overwrite,

src/kotobuki/mapping_updater/db.py

@@ -1,7 +1,7 @@
 from collections.abc import Sequence
 
 from omop_cdm.regular.cdm54 import Concept, ConceptRelationship
-from sqlalchemy import and_, select
+from sqlalchemy import and_, func, select
 from sqlalchemy.orm import Session
 
 from .relationship import (
@@ -65,9 +65,29 @@ def find_standard_concepts(
     return None
 
 
-def find_all_homonyms(concept_name: str, session: Session) -> Sequence[Concept]:
+def find_all_homonyms(
+    concept: Concept, case_insensitive: bool, session: Session
+) -> Sequence[Concept]:
     """Get Concept ORM objects that match the concept_name."""
-    return session.scalars(select(Concept).where(Concept.concept_name == concept_name)).all()
+    if case_insensitive:
+        homonyms = session.scalars(
+            select(Concept).where(
+                and_(
+                    func.lower(Concept.concept_name) == concept.concept_name.lower(),
+                    Concept.concept_id != concept.concept_id,
+                )
+            )
+        ).all()
+    else:
+        homonyms = session.scalars(
+            select(Concept).where(
+                and_(
+                    Concept.concept_name == concept.concept_name,
+                    Concept.concept_id != concept.concept_id,
+                )
+            )
+        ).all()
+    return homonyms
 
 
 def find_suitable_homonym(
@@ -84,11 +104,16 @@ def find_suitable_homonym(
     return None
 
 
-def find_new_mapping(concept: Concept, search_homonyms: bool, session: Session) -> NewMap | None:
+def find_new_mapping(
+    concept: Concept,
+    search_homonyms: bool,
+    ignore_case: bool,
+    session: Session,
+) -> NewMap | None:
     # Try to find standard concepts via concept relationships
     new_map = find_standard_concepts(concept.concept_id, session, [MapLink(concept)])
     # Alternatively via concepts with an identical name
     if search_homonyms and new_map is None:
-        homonyms = find_all_homonyms(concept.concept_name, session)
+        homonyms = find_all_homonyms(concept, ignore_case, session)
         new_map = find_suitable_homonym(homonyms, session, concept)
     return new_map

src/kotobuki/mapping_updater/update_usagi.py

@@ -30,6 +30,7 @@ def update_usagi_file(
     vocab_schema: str,
     usagi_file: Path,
     allow_homonyms: bool = False,
+    ignore_case: bool = False,
     write_map_paths: bool = False,
     inspect_only: bool = False,
     overwrite: bool = False,
@@ -87,7 +88,7 @@ def update_usagi_file(
 
         logger.info("Querying database for standard concepts...")
         for concept in non_standard:
-            new_map = find_new_mapping(concept, allow_homonyms, session)
+            new_map = find_new_mapping(concept, allow_homonyms, ignore_case, session)
             new_mappings[concept.concept_id] = new_map
         log_remapped_concepts(new_mappings)
 

tests/python/mapping_updater/db_setup/vocab_data/CONCEPT.csv

@@ -18,3 +18,5 @@ concept_id	concept_name	domain_id	vocabulary_id	concept_class_id	standard_concep
 16	val2	0	0	0	S	v2	2017-01-01	2099-12-31	
 17	qwerty	Condition	0	0	S	qwerty	2017-01-01	2099-12-31	
 18	azerty	Condition	0	0		azerty	2017-01-01	2099-12-31	
+19	QzErTy	Condition	0	0		QzErTy	2017-01-01	2099-12-31	
+20	qzerty	Condition	0	0	S	qzerty	2017-01-01	2099-12-31	

tests/python/mapping_updater/db_setup/vocab_data/CONCEPT_RELATIONSHIP.csv

@@ -12,3 +12,4 @@ concept_id_1	concept_id_2	relationship_id	valid_start_date	valid_end_date	invali
 13	14	Maps to	1970-01-01	2099-12-31	
 13	15	Maps to value	1970-01-01	2099-12-31	
 13	16	Maps to value	1970-01-01	2099-12-31	
+20	20	Maps to	2017-01-01	2099-12-31	

tests/python/mapping_updater/test_case_insensitive_homonym.py

@@ -0,0 +1,35 @@
+import pytest
+from sqlalchemy import Engine
+
+from tests.python.mapping_updater.test_usagi_mappings import get_new_map
+
+pytestmark = pytest.mark.usefixtures("create_vocab_tables")
+
+
+def test_find_standard_concept_via_case_insensitive_homonym(pg_db_engine: Engine):
+    """
+    The concept QzErTy is non-standard without a relationship to a standard concept,
+    however, there is the standard homonym "qzerty". Test if this homonym is picked up correctly.
+    """
+    result = get_new_map(
+        concept_id=19,
+        engine=pg_db_engine,
+        homonyms=True,
+        ignore_case=True,
+    )
+    assert len(result.concepts) == 1
+    assert result.concepts[0].concept_id == 20
+    assert not result.value_as_concept
+
+
+def test_ignore_case_false(pg_db_engine: Engine):
+    """
+    Test if "qzerty" is not picked up from "QzErTy" when ignore_case is False.
+    """
+    result = get_new_map(
+        concept_id=19,
+        engine=pg_db_engine,
+        homonyms=True,
+        ignore_case=False,
+    )
+    assert result is None

tests/python/mapping_updater/test_usagi_mappings.py

@@ -13,10 +13,20 @@ def get_concept_by_id(session: Session, concept_id: int) -> Concept | None:
     return session.scalars(select(Concept).filter(Concept.concept_id == concept_id)).one_or_none()
 
 
-def get_new_map(concept_id: int, engine: Engine, homonyms: bool = False) -> NewMap | None:
+def get_new_map(
+    concept_id: int,
+    engine: Engine,
+    homonyms: bool = False,
+    ignore_case: bool = False,
+) -> NewMap | None:
     with Session(engine, expire_on_commit=False) as session, session.begin():
         concept = get_concept_by_id(session, concept_id=concept_id)
-        return find_new_mapping(concept=concept, search_homonyms=homonyms, session=session)
+        return find_new_mapping(
+            concept=concept,
+            search_homonyms=homonyms,
+            ignore_case=ignore_case,
+            session=session,
+        )
 
 
 def test_no_map(pg_db_engine: Engine):