chg ! apply mapping at import instead of validation

Author: vitali-yanushchyk-valor

src/country_workspace/contrib/hope/push.py

@@ -14,7 +14,6 @@
 from country_workspace.exceptions import RemoteError
 from country_workspace.models import AsyncJob, Rdp
 from country_workspace.workspaces.models import CountryHousehold, CountryIndividual
-from country_workspace.utils.fields import map_fields
 
 
 type Beneficiary = CountryHousehold | CountryIndividual
@@ -195,9 +194,9 @@ def prepare_batch(self) -> tuple[list[int], list[dict]]:
         for item in self.queryset:
             ids.append(item.id)
             data.append(
-                {**map_fields(item.flex_fields), "members": [map_fields(m.flex_fields) for m in item.members.all()]}
+                {**item.flex_fields, "members": [m.flex_fields for m in item.members.all()]}
                 if self.master_detail
-                else map_fields(item.flex_fields)
+                else item.flex_fields
             )
         return ids, data
 

src/country_workspace/datasources/rdi.py

@@ -13,7 +13,7 @@
 
 from country_workspace.contrib.kobo.api.data.helpers import VALUE_FORMAT
 from country_workspace.datasources.utils import datetime_to_date, date_to_iso_string
-from country_workspace.models import AsyncJob, Batch, Household, Individual, MappingImporter
+from country_workspace.models import AsyncJob, Batch, Household, Individual
 from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
 from country_workspace.utils.fields import Record, clean_field_names
 from country_workspace.utils.functional import compose
@@ -99,14 +99,15 @@ def process_households(sheet: Sheet, job: AsyncJob, batch: Batch, config: Config
     for i, row in enumerate(sheet, 1):
         household_key = get_value(row, config["household_pk_col"])
         label = get_value(row, config["detail_column_label"])
+        flex_fields = job.program.apply_mapping_importer(Household, clean_field_names(row))
 
         try:
             mapping[household_key] = cast(
                 "Household",
                 job.program.households.create(
                     batch=batch,
                     name=label,
-                    flex_fields=clean_field_names(row),
+                    flex_fields=flex_fields,
                 ),
             )
         except Exception as e:
@@ -115,35 +116,35 @@ def process_households(sheet: Sheet, job: AsyncJob, batch: Batch, config: Config
     return mapping
 
 
-def full_name_column(row: Record) -> str | None:
-    for key in row:
-        if key.startswith("full") and "name" in key:
-            return key
-    return None
+def normalize_row_structure(row: Record, people_column_prefix: str | None = None) -> tuple[Record, str | None]:
+    if people_column_prefix:
+        row = {k.removeprefix(people_column_prefix): v for k, v in row.items()}
+    name_column = next((key for key in row if key.startswith("full") and "name" in key), None)
+    return row, name_column
+
+
+def get_hh_for_ind(
+    cleaned_row: dict, master_column_label: str, household_mapping: Mapping[int, Household] | None
+) -> Household | None:
+    if not household_mapping or not master_column_label:
+        return None
+    household_key = get_value(cleaned_row, master_column_label)
+    return household_mapping.get(household_key)
 
 
 def process_beneficiaries(
     sheet: Sheet, job: AsyncJob, batch: Batch, config: Config, household_mapping: Mapping[int, Household] | None = None
 ) -> Mapping[int, Individual]:
-    mapping, name_column = {}, None
-    pp_column_prefix = config.get("people_column_prefix")
-    is_people_only_mode = household_mapping is None
+    mapping = {}
+    people_column_prefix = config.get("people_column_prefix") if household_mapping is None else None
+    master_column_label = config.get("master_column_label") if household_mapping is not None else None
+    sheet_name = PEOPLE if household_mapping is None else INDIVIDUAL
 
     for i, row in enumerate(sheet, 1):
-        cleaned_row = (
-            {k.removeprefix(pp_column_prefix): v for k, v in row.items()}
-            if is_people_only_mode and pp_column_prefix
-            else row
-        )
-
-        if name_column is None:
-            name_column = full_name_column(cleaned_row)
-        name = get_value(cleaned_row, name_column) if name_column else None
-
-        household = None
-        if not is_people_only_mode:
-            household_key = get_value(cleaned_row, config["master_column_label"])
-            household = household_mapping.get(household_key)
+        cleaned_row, name_column = normalize_row_structure(row, people_column_prefix)
+        name = cleaned_row.get(name_column) if name_column else ""
+        household = get_hh_for_ind(cleaned_row, master_column_label, household_mapping)
+        flex_fields = job.program.apply_mapping_importer(Individual, clean_field_names(cleaned_row))
 
         try:
             mapping[i] = cast(
@@ -152,11 +153,10 @@ def process_beneficiaries(
                     batch=batch,
                     name=name,
                     household=household,
-                    flex_fields=clean_field_names(cleaned_row),
+                    flex_fields=flex_fields,
                 ),
             )
         except Exception as e:
-            sheet_name = PEOPLE if is_people_only_mode else INDIVIDUAL
             raise SheetProcessingError(sheet_name, i) from e
 
     return mapping

src/country_workspace/models/base.py

@@ -1,8 +1,6 @@
 from typing import TYPE_CHECKING, Any
 
 from concurrency.fields import IntegerVersionField
-from contextlib import suppress
-from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.urls import reverse
 from django.utils import timezone
@@ -108,11 +106,6 @@ def checker(self) -> "DataChecker":
         raise NotImplementedError
 
     def validate_with_checker(self, fail_if_alien: bool = False) -> bool:
-        # skip if mappingimporter not found
-        with suppress(ObjectDoesNotExist):
-            self.checker.mappingimporter.apply(self.flex_fields)
-            self.save(update_fields=["flex_fields"])
-
         errors = self.checker.validate([self.flex_fields], fail_if_alien=fail_if_alien)
         if errors:
             self.errors = errors[1]

src/country_workspace/models/program.py

@@ -1,5 +1,7 @@
 from typing import TYPE_CHECKING
+from contextlib import suppress
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.utils.translation import gettext as _
 from hope_flex_fields.models import DataChecker
@@ -120,3 +122,11 @@ def get_checker_for(self, m: type[Validable] | Validable) -> DataChecker:
         if isinstance(m, (Individual | CountryIndividual)) or m in (Individual, CountryIndividual):
             return self.individual_checker
         raise ValueError(m)
+
+    def apply_mapping_importer(
+        self, m: type[Validable] | Validable, data: dict[str, str | int | bool]
+    ) -> dict[str, str | int | bool]:
+        # skip if mapping importer not found
+        with suppress(ObjectDoesNotExist):
+            self.get_checker_for(m).mappingimporter.apply(data)
+        return data

src/country_workspace/utils/fields.py

@@ -15,7 +15,6 @@
 
 TO_REMOVE_VALUES = "_h_c", "_h_f", "_i_c", "_i_f"
 TO_UPPERCASE_FIELDS = "relationship", "gender", "residence_status", "consent_sharing"
-TO_MAP_FIELDS = {"gender": "sex"}
 
 
 def clean_field_name(v: str) -> str:
@@ -61,20 +60,6 @@ def uppercase_field_value(k: str, v: Any, fields_to_uppercase: Iterable[str] = T
     return v.upper() if isinstance(v, str) and any(k.startswith(prefix) for prefix in fields_to_uppercase) else v
 
 
-def map_fields(fields: dict[str, str]) -> dict[str, str]:
-    """
-    Map keys in a dictionary to alternative names based on a predefined mapping.
-
-    Args:
-        fields (dict[str, str]): A dictionary containing field names as keys and their values.
-
-    Returns:
-        dict[str, str]: A new dictionary with keys mapped according to the predefined mapping.
-
-    """
-    return {TO_MAP_FIELDS.get(k, k): v for k, v in fields.items() if v is not None}
-
-
 def extract_uuid(value: str, prefix: str | None = None) -> UUID:
     """Extract a UUID from the given string.
 

src/country_workspace/versioning/scripts/0013_create_field_sex_instead_of_gender.py

@@ -0,0 +1,69 @@
+from typing import Self
+from dataclasses import dataclass
+from packaging.version import Version
+from django.db import transaction
+
+from hope_flex_fields.models import FieldDefinition, FlexField, DataChecker
+from country_workspace.models import MappingImporter
+from country_workspace.contrib.hope.constants import INDIVIDUAL_CHECKER_NAME, PEOPLE_CHECKER_NAME
+
+_script_for_version = Version("0.1.0")
+
+
+@dataclass(frozen=True)
+class FieldRename:
+    old_definition: str
+    new_definition: str
+    old_field: str
+    new_field: str
+
+    def reverse(self) -> Self:
+        return FieldRename(self.new_definition, self.old_definition, self.new_field, self.old_field)
+
+
+RENAME = FieldRename("HOPE IND Gender", "HOPE IND Sex", "gender", "sex")
+CHECKERS = (INDIVIDUAL_CHECKER_NAME, PEOPLE_CHECKER_NAME)
+MI_NAME = "gender_to_sex"
+
+
+def _rename_field(rename: FieldRename) -> None:
+    field_def_qs = FieldDefinition.objects.filter(name=rename.old_definition)
+    if field_def_qs.exists():
+        FlexField.objects.filter(definition__in=field_def_qs).update(name=rename.new_field)
+        field_def_qs.update(name=rename.new_definition)
+
+
+def _create_mapping_rules(rename: FieldRename) -> None:
+    for checker_name in CHECKERS:
+        try:
+            dc = DataChecker.objects.get(name=checker_name)
+            MappingImporter.objects.get_or_create(
+                name=MI_NAME,
+                data_checker=dc,
+                defaults={
+                    "rules": f"{rename.old_field}={rename.new_field}",
+                },
+            )
+        except DataChecker.DoesNotExist:
+            continue
+
+
+def _remove_mapping_rules(rename: FieldRename) -> None:
+    MappingImporter.objects.filter(name=MI_NAME).delete()
+
+
+def forward() -> None:
+    with transaction.atomic():
+        _rename_field(RENAME)
+        _create_mapping_rules(RENAME)
+
+
+def backward() -> None:
+    with transaction.atomic():
+        _remove_mapping_rules(RENAME)
+        _rename_field(RENAME.reverse())
+
+
+class Scripts:
+    requires = []
+    operations = [(forward, backward)]

src/country_workspace/versioning/scripts/0014_create_field_sex_instead_of_gender.py

@@ -394,18 +394,19 @@ def test_safe_post_failure(
 
 
 @pytest.mark.django_db
-def test_prepare_batch(mocker: MockerFixture, push_processor: PushProcessor, beneficiary_instance: Beneficiary) -> None:
+def test_prepare_batch(push_processor: PushProcessor, beneficiary_instance: Beneficiary) -> None:
     push_processor.set_queryset([beneficiary_instance.pk])
-    mocker.patch("country_workspace.contrib.hope.push.map_fields", return_value={"field": "value"})
 
     ids, data = push_processor.prepare_batch()
 
     assert ids == [beneficiary_instance.pk]
+    assert len(data) == 1
+
     if push_processor.master_detail:
-        assert len(data) == 1
         assert "members" in data[0]
+        assert data[0]["members"] == [m.flex_fields for m in beneficiary_instance.members.all()]
     else:
-        assert data == [{"field": "value"}]
+        assert data[0] == beneficiary_instance.flex_fields
 
 
 @pytest.mark.parametrize(