chg ! apply mapping at import instead of validation

Author: vitali-yanushchyk-valor

src/country_workspace/contrib/hope/push.py

@@ -14,7 +14,6 @@
 from country_workspace.exceptions import RemoteError
 from country_workspace.models import AsyncJob, Rdp
 from country_workspace.workspaces.models import CountryHousehold, CountryIndividual
-from country_workspace.utils.fields import map_fields
 
 
 type Beneficiary = CountryHousehold | CountryIndividual
@@ -195,9 +194,9 @@ def prepare_batch(self) -> tuple[list[int], list[dict]]:
         for item in self.queryset:
             ids.append(item.id)
             data.append(
-                {**map_fields(item.flex_fields), "members": [map_fields(m.flex_fields) for m in item.members.all()]}
+                {**item.flex_fields, "members": [m.flex_fields for m in item.members.all()]}
                 if self.master_detail
-                else map_fields(item.flex_fields)
+                else item.flex_fields
             )
         return ids, data
 

src/country_workspace/datasources/rdi.py

@@ -100,14 +100,15 @@ def process_households(sheet: Sheet, job: AsyncJob, batch: Batch, config: Config
     for i, row in enumerate(sheet, 1):
         household_key = get_value(row, config["household_pk_col"])
         label = get_value(row, config["detail_column_label"])
+        flex_fields = job.program.apply_mapping_importer(Household, clean_field_names(row))
 
         try:
             mapping[household_key] = cast(
                 "Household",
                 job.program.households.create(
                     batch=batch,
                     name=label,
-                    flex_fields=clean_field_names(row),
+                    flex_fields=flex_fields,
                 ),
             )
         except Exception as e:
@@ -116,35 +117,35 @@ def process_households(sheet: Sheet, job: AsyncJob, batch: Batch, config: Config
     return mapping
 
 
-def full_name_column(row: Record) -> str | None:
-    for key in row:
-        if key.startswith("full") and "name" in key:
-            return key
-    return None
+def normalize_row_structure(row: Record, people_column_prefix: str | None = None) -> tuple[Record, str | None]:
+    if people_column_prefix:
+        row = {k.removeprefix(people_column_prefix): v for k, v in row.items()}
+    name_column = next((key for key in row if key.startswith("full") and "name" in key), None)
+    return row, name_column
+
+
+def get_hh_for_ind(
+    cleaned_row: dict, master_column_label: str, household_mapping: Mapping[int, Household] | None
+) -> Household | None:
+    if not household_mapping or not master_column_label:
+        return None
+    household_key = get_value(cleaned_row, master_column_label)
+    return household_mapping.get(household_key)
 
 
 def process_beneficiaries(
     sheet: Sheet, job: AsyncJob, batch: Batch, config: Config, household_mapping: Mapping[int, Household] | None = None
 ) -> Mapping[int, Individual]:
-    mapping, name_column = {}, None
-    pp_column_prefix = config.get("people_column_prefix")
-    is_people_only_mode = household_mapping is None
+    mapping = {}
+    people_column_prefix = config.get("people_column_prefix") if household_mapping is None else None
+    master_column_label = config.get("master_column_label") if household_mapping is not None else None
+    sheet_name = PEOPLE if household_mapping is None else INDIVIDUAL
 
     for i, row in enumerate(sheet, 1):
-        cleaned_row = (
-            {k.removeprefix(pp_column_prefix): v for k, v in row.items()}
-            if is_people_only_mode and pp_column_prefix
-            else row
-        )
-
-        if name_column is None:
-            name_column = full_name_column(cleaned_row)
-        name = get_value(cleaned_row, name_column) if name_column else None
-
-        household = None
-        if not is_people_only_mode:
-            household_key = get_value(cleaned_row, config["master_column_label"])
-            household = household_mapping.get(household_key)
+        cleaned_row, name_column = normalize_row_structure(row, people_column_prefix)
+        name = cleaned_row.get(name_column) if name_column else None
+        household = get_hh_for_ind(cleaned_row, master_column_label, household_mapping)
+        flex_fields = job.program.apply_mapping_importer(Individual, clean_field_names(cleaned_row))
 
         try:
             mapping[i] = cast(
@@ -153,11 +154,10 @@ def process_beneficiaries(
                     batch=batch,
                     name=name,
                     household=household,
-                    flex_fields=clean_field_names(cleaned_row),
+                    flex_fields=flex_fields,
                 ),
             )
         except Exception as e:
-            sheet_name = PEOPLE if is_people_only_mode else INDIVIDUAL
             raise SheetProcessingError(sheet_name, i) from e
 
     return mapping

src/country_workspace/models/base.py

@@ -1,8 +1,6 @@
 from typing import TYPE_CHECKING, Any
 
 from concurrency.fields import IntegerVersionField
-from contextlib import suppress
-from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.urls import reverse
 from django.utils import timezone
@@ -108,11 +106,6 @@ def checker(self) -> "DataChecker":
         raise NotImplementedError
 
     def validate_with_checker(self, fail_if_alien: bool = False) -> bool:
-        # skip if mappingimporter not found
-        with suppress(ObjectDoesNotExist):
-            self.checker.mappingimporter.apply(self.flex_fields)
-            self.save(update_fields=["flex_fields"])
-
         errors = self.checker.validate([self.flex_fields], fail_if_alien=fail_if_alien)
         if errors:
             self.errors = errors[1]

src/country_workspace/models/program.py

@@ -1,5 +1,7 @@
 from typing import TYPE_CHECKING
+from contextlib import suppress
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.utils.translation import gettext as _
 from hope_flex_fields.models import DataChecker
@@ -120,3 +122,11 @@ def get_checker_for(self, m: type[Validable] | Validable) -> DataChecker:
         if isinstance(m, (Individual | CountryIndividual)) or m in (Individual, CountryIndividual):
             return self.individual_checker
         raise ValueError(m)
+
+    def apply_mapping_importer(
+        self, m: type[Validable] | Validable, data: dict[str, str | int | bool]
+    ) -> dict[str, str | int | bool]:
+        # skip if mapping importer not found
+        with suppress(ObjectDoesNotExist):
+            self.get_checker_for(m).mappingimporter.apply(data)
+        return data

src/country_workspace/utils/fields.py

@@ -15,7 +15,6 @@
 
 TO_REMOVE_VALUES = "_h_c", "_h_f", "_i_c", "_i_f"
 TO_UPPERCASE_FIELDS = "relationship", "gender", "residence_status", "consent_sharing"
-TO_MAP_FIELDS = {"gender": "sex"}
 
 
 def clean_field_name(v: str) -> str:
@@ -61,20 +60,6 @@ def uppercase_field_value(k: str, v: Any, fields_to_uppercase: Iterable[str] = T
     return v.upper() if isinstance(v, str) and any(k.startswith(prefix) for prefix in fields_to_uppercase) else v
 
 
-def map_fields(fields: dict[str, str]) -> dict[str, str]:
-    """
-    Map keys in a dictionary to alternative names based on a predefined mapping.
-
-    Args:
-        fields (dict[str, str]): A dictionary containing field names as keys and their values.
-
-    Returns:
-        dict[str, str]: A new dictionary with keys mapped according to the predefined mapping.
-
-    """
-    return {TO_MAP_FIELDS.get(k, k): v for k, v in fields.items() if v is not None}
-
-
 def extract_uuid(value: str, prefix: str | None = None) -> UUID:
     """Extract a UUID from the given string.
 

tests/utils/test_utils_fields.py

@@ -12,7 +12,6 @@
     clean_field_name,
     TO_REMOVE_VALUES,
     clean_field_names,
-    map_fields,
     extract_uuid,
 )
 from country_workspace.utils.flex_fields import (
@@ -44,20 +43,6 @@ def test_clean_field_names(mocker: MockerFixture) -> None:
     clean_field_name_mock.assert_called_once_with(key)
 
 
-@pytest.mark.parametrize(
-    ("input_fields", "expected_output"),
-    [
-        ({"gender": "male"}, {"sex": "male"}),
-        ({"name": "John"}, {"name": "John"}),
-        ({}, {}),
-        ({"gender": "female", "age": "30"}, {"sex": "female", "age": "30"}),
-    ],
-)
-def test_map_fields(input_fields, expected_output):
-    result = map_fields(input_fields)
-    assert result == expected_output
-
-
 @pytest.mark.parametrize("value", [None, "", "test"])
 def test_base64_image_input(value: str | None) -> None:
     input_ = Base64ImageInput()