populate biometric_deduplication_enabled during program sync

Author: vitali-yanushchyk-valor

src/country_workspace/admin/program.py

@@ -38,6 +38,7 @@ class ProgramAdmin(SyncAdminMixin, BaseModelAdmin):
         "sector",
         "enabled",
         "beneficiary_group",
+        "biometric_deduplication_enabled",
         "beneficiary_validator",
         "household_checker",
         "individual_checker",

src/country_workspace/contrib/hope/sync/base.py

@@ -7,7 +7,7 @@
 
 from django.contrib.contenttypes.models import ContentType
 from django.core.cache import cache
-from django.db import DatabaseError
+from django.db import DatabaseError, transaction
 from django.db.models import Model
 
 from country_workspace.exceptions import RemoteError
@@ -160,22 +160,22 @@ def sync_entity[T: Model](config: SyncConfig[T], client: HopeClient | None = Non
     stats = stats or Stats(errors=[], add=0, upd=0)
     client = client or HopeClient()
 
-    with cache.lock(f"sync-{model_name}"):
+    with cache.lock(f"hope-sync:{model_name}"):
         logger.info(format_msg("SYNC_START", entity=model_name))
         for record in safe_get(client, config["endpoint"], stats):
             if not (reference_id_val := validated_reference_id(record)):
                 continue
             if should_process and not should_process(record):
                 continue
             try:
-                defaults = prepare_defaults(record) if prepare_defaults else None
-                if defaults is None or not defaults:
+                if not (defaults := prepare_defaults(record)):
                     continue
-                instance, created = model.objects.update_or_create(
-                    defaults=defaults, **{reference_id: reference_id_val}
-                )
-                if post_process:
-                    post_process(instance, created)
+                with transaction.atomic():
+                    instance, created = model.objects.update_or_create(
+                        defaults=defaults, **{reference_id: reference_id_val}
+                    )
+                    if post_process:
+                        post_process(instance, created)
                 stats["add" if created else "upd"] += 1
             except SkipRecordError as e:
                 logger.warning(format_msg("RECORD_SKIPPED", reference_id_val=reference_id_val, error=str(e)))
@@ -184,7 +184,7 @@ def sync_entity[T: Model](config: SyncConfig[T], client: HopeClient | None = Non
                 add_error(stats, error)
                 logger.error(error)
         if stats["errors"]:
-            raise HopeSyncError(stats["errors"])
+            raise HopeSyncError("\n".join(stats["errors"]))
         SyncLog.objects.register_sync(model)
         logger.info(format_msg("SYNC_COMPLETE", entity=model_name, result=stats, errors_count=0))
         return stats

src/country_workspace/contrib/hope/sync/context_programs.py

@@ -33,23 +33,17 @@
     "member_label_plural",
     "master_detail",
 )
-Program_FIELDS = "name", "code", "status", "sector", "country_office", "beneficiary_group"
 
 HOPE_ID = "hope_id"
 BUSINESS_AREAS = "business-areas"
 BENEFICIARY_GROUPS = "beneficiary-groups"
 PROGRAMS = "programs"
 
 
-def get_default_checkers() -> Mapping[str, DataChecker]:
-    return {
-        name: DataChecker.objects.filter(name=const_name).first()
-        for name, const_name in (
-            ("hh", HOUSEHOLD_CHECKER_NAME),
-            ("ind", INDIVIDUAL_CHECKER_NAME),
-            ("ppl", PEOPLE_CHECKER_NAME),
-        )
-    }
+def get_default_checkers() -> Mapping[str, DataChecker | None]:
+    target_to_checker_name = {"hh": HOUSEHOLD_CHECKER_NAME, "ind": INDIVIDUAL_CHECKER_NAME, "ppl": PEOPLE_CHECKER_NAME}
+    checker_by_name = {c.name: c for c in DataChecker.objects.filter(name__in=target_to_checker_name.values())}
+    return {k: checker_by_name.get(v) for k, v in target_to_checker_name.items()}
 
 
 def get_field_extractor(fields: tuple[str, ...]) -> Callable[[dict[str, Any]], dict[str, Any]]:
@@ -109,13 +103,13 @@ def prepare_program_defaults(record: dict[str, Any]) -> dict[str, Any] | None:
         raise SkipRecordError("Beneficiary group not found") from e
 
     return {
-        "name": record["name"],
+        "biometric_deduplication_enabled": record["biometric_deduplication_enabled"],
         "code": record["programme_code"],
-        "status": record["status"],
+        "name": record["name"],
         "sector": record["sector"],
-        "biometric_deduplication_enabled": record["biometric_deduplication_enabled"],
-        "country_office": office,
+        "status": record["status"],
         "beneficiary_group": bg,
+        "country_office": office,
     }
 
 
@@ -151,13 +145,12 @@ def post_process_program(program: Program, created: bool) -> None:
     default_checkers = get_default_checkers()
     update_fields: list[str] = []
 
-    if default_checkers:
-        program.household_checker = default_checkers.get("hh")
-        program.individual_checker = (
-            default_checkers.get("ind") if program.beneficiary_group.master_detail else default_checkers.get("ppl")
-        )
-        if program.household_checker or program.individual_checker:
-            update_fields.extend(["household_checker", "individual_checker"])
+    program.household_checker = default_checkers.get("hh")
+    program.individual_checker = (
+        default_checkers.get("ind") if program.beneficiary_group.master_detail else default_checkers.get("ppl")
+    )
+    if program.household_checker or program.individual_checker:
+        update_fields.extend(["household_checker", "individual_checker"])
 
     hh_ignored = get_default_ignored_fields("hh")
     if hh_ignored:

src/country_workspace/migrations/0043_program_biometric_deduplication_enabled.py

@@ -49,11 +49,15 @@ def get_field_extractor_mock(mocker: MockerFixture) -> Mock:
 
 
 def test_get_default_checkers(mocker: MockerFixture) -> None:
-    data_checker_model_mock = mocker.patch("country_workspace.contrib.hope.sync.context_programs.DataChecker")
+    dc = mocker.patch("country_workspace.contrib.hope.sync.context_programs.DataChecker")
+
     assert get_default_checkers().keys() == {"hh", "ind", "ppl"}
-    data_checker_model_mock.objects.filter.assert_any_call(name=HOUSEHOLD_CHECKER_NAME)
-    data_checker_model_mock.objects.filter.assert_any_call(name=INDIVIDUAL_CHECKER_NAME)
-    data_checker_model_mock.objects.filter.assert_any_call(name=PEOPLE_CHECKER_NAME)
+
+    assert set(dc.objects.filter.call_args.kwargs["name__in"]) == {
+        HOUSEHOLD_CHECKER_NAME,
+        INDIVIDUAL_CHECKER_NAME,
+        PEOPLE_CHECKER_NAME,
+    }
 
 
 def test_get_field_extractor() -> None:
@@ -155,24 +159,26 @@ def test_prepare_program_defaults_all_found(mocker: MockerFixture) -> None:
     get_office_mock = mocker.patch.object(Office.objects, "get")
     get_group_mock = mocker.patch.object(BeneficiaryGroup.objects, "get")
     record = {
+        "beneficiary_group": (bg := "bg"),
+        "biometric_deduplication_enabled": (dedup_enabled := True),
         "business_area_code": (area_code := "area_code"),
-        "beneficiary_group": (group := "group"),
         "name": (name := "name"),
         "programme_code": (programme_code := "programme_code"),
-        "status": (status := "status"),
         "sector": (sector := "sector"),
+        "status": (status := "status"),
     }
 
     assert prepare_program_defaults(record) == {
-        "name": name,
+        "biometric_deduplication_enabled": dedup_enabled,
         "code": programme_code,
-        "status": status,
+        "name": name,
         "sector": sector,
-        "country_office": get_office_mock.return_value,
+        "status": status,
         "beneficiary_group": get_group_mock.return_value,
+        "country_office": get_office_mock.return_value,
     }
     get_office_mock.assert_called_once_with(code=area_code)
-    get_group_mock.assert_called_once_with(hope_id=group)
+    get_group_mock.assert_called_once_with(hope_id=bg)
 
 
 def test_get_default_ignored_fields_collects_from_all_sources() -> None: