chg ! apply mapping at import instead of validation

Author: vitali-yanushchyk-valor

src/country_workspace/contrib/hope/push.py

@@ -14,7 +14,6 @@
 from country_workspace.exceptions import RemoteError
 from country_workspace.models import AsyncJob, Rdp
 from country_workspace.workspaces.models import CountryHousehold, CountryIndividual
-from country_workspace.utils.fields import map_fields
 
 
 type Beneficiary = CountryHousehold | CountryIndividual
@@ -195,9 +194,9 @@ def prepare_batch(self) -> tuple[list[int], list[dict]]:
         for item in self.queryset:
             ids.append(item.id)
             data.append(
-                {**map_fields(item.flex_fields), "members": [map_fields(m.flex_fields) for m in item.members.all()]}
+                {**item.flex_fields, "members": [m.flex_fields for m in item.members.all()]}
                 if self.master_detail
-                else map_fields(item.flex_fields)
+                else item.flex_fields
             )
         return ids, data
 

src/country_workspace/datasources/rdi.py

@@ -100,14 +100,15 @@ def process_households(sheet: Sheet, job: AsyncJob, batch: Batch, config: Config
     for i, row in enumerate(sheet, 1):
         household_key = get_value(row, config["household_pk_col"])
         label = get_value(row, config["detail_column_label"])
+        flex_fields = job.program.apply_mapping_importer(Household, clean_field_names(row))
 
         try:
             mapping[household_key] = cast(
                 "Household",
                 job.program.households.create(
                     batch=batch,
                     name=label,
-                    flex_fields=clean_field_names(row),
+                    flex_fields=flex_fields,
                 ),
             )
         except Exception as e:
@@ -116,35 +117,35 @@ def process_households(sheet: Sheet, job: AsyncJob, batch: Batch, config: Config
     return mapping
 
 
-def full_name_column(row: Record) -> str | None:
-    for key in row:
-        if key.startswith("full") and "name" in key:
-            return key
-    return None
+def normalize_row_structure(row: Record, people_column_prefix: str | None = None) -> tuple[Record, str | None]:
+    if people_column_prefix:
+        row = {k.removeprefix(people_column_prefix): v for k, v in row.items()}
+    name_column = next((key for key in row if key.startswith("full") and "name" in key), None)
+    return row, name_column
+
+
+def get_hh_for_ind(
+    cleaned_row: dict, master_column_label: str, household_mapping: Mapping[int, Household] | None
+) -> Household | None:
+    if not household_mapping or not master_column_label:
+        return None
+    household_key = get_value(cleaned_row, master_column_label)
+    return household_mapping.get(household_key)
 
 
 def process_beneficiaries(
     sheet: Sheet, job: AsyncJob, batch: Batch, config: Config, household_mapping: Mapping[int, Household] | None = None
 ) -> Mapping[int, Individual]:
-    mapping, name_column = {}, None
-    pp_column_prefix = config.get("people_column_prefix")
-    is_people_only_mode = household_mapping is None
+    mapping = {}
+    people_column_prefix = config.get("people_column_prefix") if household_mapping is None else None
+    master_column_label = config.get("master_column_label") if household_mapping is not None else None
+    sheet_name = PEOPLE if household_mapping is None else INDIVIDUAL
 
     for i, row in enumerate(sheet, 1):
-        cleaned_row = (
-            {k.removeprefix(pp_column_prefix): v for k, v in row.items()}
-            if is_people_only_mode and pp_column_prefix
-            else row
-        )
-
-        if name_column is None:
-            name_column = full_name_column(cleaned_row)
-        name = get_value(cleaned_row, name_column) if name_column else None
-
-        household = None
-        if not is_people_only_mode:
-            household_key = get_value(cleaned_row, config["master_column_label"])
-            household = household_mapping.get(household_key)
+        cleaned_row, name_column = normalize_row_structure(row, people_column_prefix)
+        name = cleaned_row.get(name_column) if name_column else ""
+        household = get_hh_for_ind(cleaned_row, master_column_label, household_mapping)
+        flex_fields = job.program.apply_mapping_importer(Individual, clean_field_names(cleaned_row))
 
         try:
             mapping[i] = cast(
@@ -153,11 +154,10 @@ def process_beneficiaries(
                     batch=batch,
                     name=name,
                     household=household,
-                    flex_fields=clean_field_names(cleaned_row),
+                    flex_fields=flex_fields,
                 ),
             )
         except Exception as e:
-            sheet_name = PEOPLE if is_people_only_mode else INDIVIDUAL
             raise SheetProcessingError(sheet_name, i) from e
 
     return mapping

src/country_workspace/models/base.py

@@ -1,8 +1,6 @@
 from typing import TYPE_CHECKING, Any
 
 from concurrency.fields import IntegerVersionField
-from contextlib import suppress
-from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.urls import reverse
 from django.utils import timezone
@@ -108,11 +106,6 @@ def checker(self) -> "DataChecker":
         raise NotImplementedError
 
     def validate_with_checker(self, fail_if_alien: bool = False) -> bool:
-        # skip if mappingimporter not found
-        with suppress(ObjectDoesNotExist):
-            self.checker.mappingimporter.apply(self.flex_fields)
-            self.save(update_fields=["flex_fields"])
-
         errors = self.checker.validate([self.flex_fields], fail_if_alien=fail_if_alien)
         if errors:
             self.errors = errors[1]

src/country_workspace/models/program.py

@@ -1,5 +1,7 @@
 from typing import TYPE_CHECKING
+from contextlib import suppress
 
+from django.core.exceptions import ObjectDoesNotExist
 from django.db import models
 from django.utils.translation import gettext as _
 from hope_flex_fields.models import DataChecker
@@ -120,3 +122,11 @@ def get_checker_for(self, m: type[Validable] | Validable) -> DataChecker:
         if isinstance(m, (Individual | CountryIndividual)) or m in (Individual, CountryIndividual):
             return self.individual_checker
         raise ValueError(m)
+
+    def apply_mapping_importer(
+        self, m: type[Validable] | Validable, data: dict[str, str | int | bool]
+    ) -> dict[str, str | int | bool]:
+        # skip if mapping importer not found
+        with suppress(ObjectDoesNotExist):
+            self.get_checker_for(m).mappingimporter.apply(data)
+        return data

src/country_workspace/utils/fields.py

@@ -15,7 +15,6 @@
 
 TO_REMOVE_VALUES = "_h_c", "_h_f", "_i_c", "_i_f"
 TO_UPPERCASE_FIELDS = "relationship", "gender", "residence_status", "consent_sharing"
-TO_MAP_FIELDS = {"gender": "sex"}
 
 
 def clean_field_name(v: str) -> str:
@@ -61,20 +60,6 @@ def uppercase_field_value(k: str, v: Any, fields_to_uppercase: Iterable[str] = T
     return v.upper() if isinstance(v, str) and any(k.startswith(prefix) for prefix in fields_to_uppercase) else v
 
 
-def map_fields(fields: dict[str, str]) -> dict[str, str]:
-    """
-    Map keys in a dictionary to alternative names based on a predefined mapping.
-
-    Args:
-        fields (dict[str, str]): A dictionary containing field names as keys and their values.
-
-    Returns:
-        dict[str, str]: A new dictionary with keys mapped according to the predefined mapping.
-
-    """
-    return {TO_MAP_FIELDS.get(k, k): v for k, v in fields.items() if v is not None}
-
-
 def extract_uuid(value: str, prefix: str | None = None) -> UUID:
     """Extract a UUID from the given string.
 

tests/contrib/hope/test_push_to_hope_core.py

@@ -394,18 +394,19 @@ def test_safe_post_failure(
 
 
 @pytest.mark.django_db
-def test_prepare_batch(mocker: MockerFixture, push_processor: PushProcessor, beneficiary_instance: Beneficiary) -> None:
+def test_prepare_batch(push_processor: PushProcessor, beneficiary_instance: Beneficiary) -> None:
     push_processor.set_queryset([beneficiary_instance.pk])
-    mocker.patch("country_workspace.contrib.hope.push.map_fields", return_value={"field": "value"})
 
     ids, data = push_processor.prepare_batch()
 
     assert ids == [beneficiary_instance.pk]
+    assert len(data) == 1
+
     if push_processor.master_detail:
-        assert len(data) == 1
         assert "members" in data[0]
+        assert data[0]["members"] == [m.flex_fields for m in beneficiary_instance.members.all()]
     else:
-        assert data == [{"field": "value"}]
+        assert data[0] == beneficiary_instance.flex_fields
 
 
 @pytest.mark.parametrize(

tests/datasources/test_rdi.py

@@ -23,13 +23,13 @@
     image_content,
     extract_images,
     merge_images,
+    normalize_row_structure,
     read_sheets,
-    full_name_column,
     INDIVIDUAL,
     PEOPLE,
 )
 from country_workspace.datasources.utils import datetime_to_date, date_to_iso_string
-from country_workspace.models import Household
+from country_workspace.models import Household, Individual
 from country_workspace.workspaces.exceptions import BeneficiaryValidationError
 from country_workspace.validators.beneficiaries import validate_beneficiaries
 
@@ -169,6 +169,33 @@ def test_filter_rows_with_household_pk(
     )
 
 
+@pytest.mark.parametrize(
+    ("row", "prefix", "expected_row"),
+    [
+        ({"full_name": "John"}, None, {"full_name": "John"}),
+        ({"pp_full_name": "Jane"}, "pp_", {"full_name": "Jane"}),
+        ({"no_prefix": "value"}, "wrong_", {"no_prefix": "value"}),
+    ],
+    ids=["no_prefix", "with_prefix", "prefix_not_found"],
+)
+def test_normalize_row_structure_prefix_handling(row: Record, prefix: str | None, expected_row: Record) -> None:
+    result_row, _ = normalize_row_structure(row, prefix)
+    assert result_row == expected_row
+
+
+@pytest.mark.parametrize(
+    ("row", "expected_name_column"),
+    [
+        ({"full_name": "John"}, "full_name"),
+        ({"age": 30}, None),
+    ],
+    ids=["full_name", "no_name"],
+)
+def test_normalize_row_structure_name_column_detection(row: Record, expected_name_column: str | None) -> None:
+    _, name_column = normalize_row_structure(row)
+    assert name_column == expected_name_column
+
+
 def test_process_households(
     mocker: MockerFixture, config: Config, household_sheet: Sheet, skip_if_not_master_detail
 ) -> None:
@@ -179,9 +206,17 @@ def test_process_households(
     assert result == {
         row[config["household_pk_col"]]: job.program.households.create.return_value for row in household_sheet
     }
+
+    job.program.apply_mapping_importer.assert_has_calls(
+        [call(Household, clean_field_names_mock.return_value) for row in household_sheet]
+    )
     job.program.households.create.assert_has_calls(
         [
-            call(batch=batch, name=row[config["detail_column_label"]], flex_fields=clean_field_names_mock.return_value)
+            call(
+                batch=batch,
+                name=row[config["detail_column_label"]],
+                flex_fields=job.program.apply_mapping_importer.return_value,
+            )
             for row in household_sheet
         ]
     )
@@ -218,13 +253,16 @@ def test_process_beneficiaries_with_households(
     )
 
     assert len(result) == len(list(individual_sheet))
+    job_mock.program.apply_mapping_importer.assert_has_calls(
+        [call(Individual, clean_field_names_mock.return_value) for row in individual_sheet]
+    )
     job_mock.program.individuals.create.assert_has_calls(
         [
             call(
                 batch=batch_mock,
                 name=row[FULL_NAME_COLUMN],
                 household=household_mapping[row[config["master_column_label"]]],
-                flex_fields=clean_field_names_mock.return_value,
+                flex_fields=job_mock.program.apply_mapping_importer.return_value,
             )
             for row in individual_sheet
         ]
@@ -246,19 +284,21 @@ def test_process_beneficiaries_people_only(
     )
 
     assert len(result) == len(list(people_sheet))
-    expected_calls = []
+    expected_calls, expected_apply_mapping_calls = [], []
     for __, row in enumerate(people_sheet, 1):
         prefix = config.get("people_column_prefix", "")
         cleaned_row = {k.removeprefix(prefix): v for k, v in row.items()}
+        expected_apply_mapping_calls.append(call(Individual, clean_field_names_mock.return_value))
         expected_calls.append(
             call(
                 batch=batch_mock,
                 name=cleaned_row[FULL_NAME_COLUMN],
                 household=None,
-                flex_fields=clean_field_names_mock.return_value,
+                flex_fields=job_mock.program.apply_mapping_importer.return_value,
             )
         )
     job_mock.program.individuals.create.assert_has_calls(expected_calls)
+    job_mock.program.apply_mapping_importer.assert_has_calls(expected_apply_mapping_calls)
 
 
 def test_process_beneficiaries_failed_to_create(
@@ -462,18 +502,6 @@ def test_read_sheets_sheet_not_found_error(mocker: MockerFixture, config: Config
     assert str(sheet_index) in str(exc_info.value)
 
 
-@pytest.mark.parametrize(
-    ("record", "expected"),
-    [
-        ({"full_name": "John Smith"}, "full_name"),
-        ({}, None),
-        ({"name_full": "John Smith"}, None),
-    ],
-)
-def test_full_name_column(record: Record, expected: str | None) -> None:
-    assert full_name_column(record) == expected
-
-
 @pytest.mark.parametrize(
     ("inp", "expected"),
     [

tests/utils/test_utils_fields.py

@@ -12,7 +12,6 @@
     clean_field_name,
     TO_REMOVE_VALUES,
     clean_field_names,
-    map_fields,
     extract_uuid,
 )
 from country_workspace.utils.flex_fields import (
@@ -44,20 +43,6 @@ def test_clean_field_names(mocker: MockerFixture) -> None:
     clean_field_name_mock.assert_called_once_with(key)
 
 
-@pytest.mark.parametrize(
-    ("input_fields", "expected_output"),
-    [
-        ({"gender": "male"}, {"sex": "male"}),
-        ({"name": "John"}, {"name": "John"}),
-        ({}, {}),
-        ({"gender": "female", "age": "30"}, {"sex": "female", "age": "30"}),
-    ],
-)
-def test_map_fields(input_fields, expected_output):
-    result = map_fields(input_fields)
-    assert result == expected_output
-
-
 @pytest.mark.parametrize("value", [None, "", "test"])
 def test_base64_image_input(value: str | None) -> None:
     input_ = Base64ImageInput()