Fix validating R concepts query (#1162)

Author: AndrewThien

app/airflow/dags/libs/auto_mapping/find_R_concepts_to_reuse.py

@@ -99,7 +99,7 @@ def find_matching_value(**kwargs):
             scan_report_table=table_id,
             stage=JobStageType.REUSE_CONCEPTS,
             status=StageStatusType.IN_PROGRESS,
-            details=f"Finding eligible concepts for reuse at the value level",
+            details="Finding eligible concepts for reuse at the value level",
         )
 
         # Create temp table for reuse concepts
@@ -133,7 +133,7 @@ def find_matching_value(**kwargs):
                 },
             )
             logging.info(
-                f"Successfully found M-type concepts for reuse at the value level"
+                "Successfully found M-type concepts for reuse at the value level"
             )
         except Exception as e:
             logging.error(
@@ -181,7 +181,7 @@ def find_matching_field(**kwargs):
             scan_report_table=table_id,
             stage=JobStageType.REUSE_CONCEPTS,
             status=StageStatusType.IN_PROGRESS,
-            details=f"Finding eligible concepts for reuse at the field level",
+            details="Finding eligible concepts for reuse at the field level",
         )
 
         try:
@@ -203,7 +203,7 @@ def find_matching_field(**kwargs):
                 },
             )
             logging.info(
-                f"Successfully found M-type concepts for reuse at the field level"
+                "Successfully found M-type concepts for reuse at the field level"
             )
         except Exception as e:
             logging.error(
@@ -241,7 +241,7 @@ def create_reusing_concepts(**kwargs):
                 find_object_id_query,
                 parameters={"table_id": table_id},
             )
-            logging.info(f"Successfully found object ids for reusing concepts")
+            logging.info("Successfully found object ids for reusing concepts")
         except Exception as e:
             logging.error(f"Failed to find object ids for reusing concepts: {str(e)}")
             update_job_status(
@@ -258,7 +258,7 @@ def create_reusing_concepts(**kwargs):
                 validate_reused_concepts_query,
                 parameters={"table_id": table_id},
             )
-            logging.info(f"Successfully validated reused concepts")
+            logging.info("Successfully validated reused concepts")
         except Exception as e:
             logging.error(f"Failed to validate reused concepts: {str(e)}")
             update_job_status(

app/airflow/dags/libs/queries.py

@@ -409,8 +409,9 @@
 
 
 validate_reused_concepts_query = """
+-- Remove duplicates first (without joining omop.concept)
 DELETE FROM temp_reuse_concepts_%(table_id)s AS temp_table
-USING temp_reuse_concepts_%(table_id)s AS temp_table_duplicate, omop.concept AS omop_concept
+USING temp_reuse_concepts_%(table_id)s AS temp_table_duplicate
 WHERE (
     -- Remove duplicates at the value level, keeping the one with the lowest source_scanreport_id
     (
@@ -432,15 +433,15 @@
         )
         AND temp_table.source_scanreport_id > temp_table_duplicate.source_scanreport_id
     )
-    OR
-    -- Remove concepts whose domain is not in the allowed domains
-    (
-        temp_table.concept_id = omop_concept.concept_id
-        AND omop_concept.domain_id NOT IN (
-            'Condition', 'Drug', 'Procedure', 'Specimen', 'Device',
-            'Measurement', 'Observation', 'Gender', 'Race', 'Ethnicity', 'Spec Anatomic Site'
-        )
-    )
+);
+
+-- Remove concepts whose domain is not in the allowed domains
+DELETE FROM temp_reuse_concepts_%(table_id)s AS temp_table
+USING omop.concept AS omop_concept
+WHERE temp_table.concept_id = omop_concept.concept_id
+AND omop_concept.domain_id NOT IN (
+    'Condition', 'Drug', 'Procedure', 'Specimen', 'Device',
+    'Measurement', 'Observation', 'Gender', 'Race', 'Ethnicity', 'Spec Anatomic Site'
 );
 
 -- Delete concepts that have no object_id