Merge pull request #91 from unicef/bugfix/248006-import-data-rdi-detail-column-label-not-working

AB#248006: Import Data RDI Detail column label not working

Author: sergey-misuk-valor

src/country_workspace/datasources/rdi.py

@@ -68,15 +68,15 @@ def process_households(sheet: Sheet, job: AsyncJob, batch: Batch, config: Config
     mapping = {}
 
     for i, row in enumerate(sheet, 1):
-        name = get_value(row, config["master_column_label"])
         household_key = get_value(row, config["household_pk_col"])
+        label = get_value(row, config["detail_column_label"])
 
         try:
             mapping[household_key] = cast(
                 Household,
                 job.program.households.create(
                     batch=batch,
-                    name=name,
+                    name=label,
                     flex_fields=clean_field_names(row),
                 ),
             )
@@ -86,14 +86,22 @@ def process_households(sheet: Sheet, job: AsyncJob, batch: Batch, config: Config
     return mapping
 
 
+def full_name_column(row: Record) -> str | None:
+    for key in row:
+        if key.startswith("full") and "name" in key:
+            return key
+    return None
+
+
 def process_individuals(
     sheet: Sheet, household_mapping: Mapping[int, Household], job: AsyncJob, batch: Batch, config: Config
 ) -> int:
     processed = 0
 
     for i, row in enumerate(sheet, 1):
-        name = get_value(row, config["detail_column_label"])
-        household_key = get_value(row, config["household_pk_col"])
+        name_column = full_name_column(row)
+        name = get_value(row, name_column) if name_column else None
+        household_key = get_value(row, config["master_column_label"])
         household = household_mapping.get(household_key)
 
         try:

src/country_workspace/workspaces/admin/forms.py

@@ -50,7 +50,7 @@ class ImportFileForm(forms.Form):
 
     detail_column_label = forms.CharField(
         required=False,
-        initial="full_name_i_c",
+        initial="household_id",
         help_text="Which column should be used as label for the household. It can use interpolation",
     )
 

tests/datasources/test_rdi.py

@@ -22,6 +22,7 @@
     extract_images,
     merge_images,
     read_sheets,
+    full_name_column,
     # validate_households,
 )
 from country_workspace.models import Household
@@ -33,6 +34,7 @@
 HOUSEHOLD_2_PK = 2
 HOUSEHOLD_1_NAME = "Household 1"
 HOUSEHOLD_2_NAME = "Household 2"
+FULL_NAME_COLUMN = "full_name"
 
 
 @pytest.fixture
@@ -50,21 +52,21 @@ def config() -> Config:
 @pytest.fixture
 def household_sheet(config: Config) -> Sheet:
     return [
-        {config["master_column_label"]: HOUSEHOLD_1_NAME, config["household_pk_col"]: HOUSEHOLD_1_PK},
-        {config["master_column_label"]: HOUSEHOLD_1_NAME, config["household_pk_col"]: HOUSEHOLD_2_PK},
+        {config["detail_column_label"]: HOUSEHOLD_1_NAME, config["household_pk_col"]: HOUSEHOLD_1_PK},
+        {config["detail_column_label"]: HOUSEHOLD_1_NAME, config["household_pk_col"]: HOUSEHOLD_2_PK},
     ]
 
 
 @pytest.fixture
 def individual_sheet(config: Config) -> Sheet:
     return [
         {
-            config["detail_column_label"]: "John Doe",
-            config["household_pk_col"]: HOUSEHOLD_1_PK,
+            FULL_NAME_COLUMN: "John Doe",
+            config["master_column_label"]: HOUSEHOLD_1_PK,
         },
         {
-            config["detail_column_label"]: "Doe John",
-            config["household_pk_col"]: HOUSEHOLD_2_PK,
+            FULL_NAME_COLUMN: "Doe John",
+            config["master_column_label"]: HOUSEHOLD_2_PK,
         },
     ]
 
@@ -135,7 +137,7 @@ def test_process_households(mocker: MockerFixture, config: Config, household_she
     }
     job.program.households.create.assert_has_calls(
         [
-            call(batch=batch, name=row[config["master_column_label"]], flex_fields=clean_field_names_mock.return_value)
+            call(batch=batch, name=row[config["detail_column_label"]], flex_fields=clean_field_names_mock.return_value)
             for row in household_sheet
         ]
     )
@@ -166,8 +168,8 @@ def test_process_individuals(
         [
             call(
                 batch=batch_mock,
-                name=row[config["detail_column_label"]],
-                household_id=household_mapping[row[config["household_pk_col"]]].pk,
+                name=row[FULL_NAME_COLUMN],
+                household_id=household_mapping[row[config["master_column_label"]]].pk,
                 flex_fields=clean_field_names_mock.return_value,
             )
             for row in individual_sheet
@@ -305,3 +307,15 @@ def test_read_sheets(mocker: MockerFixture) -> None:
     extract_images_mock.assert_called_once_with(filepath, sheet_index)
     merge_images_mock.assert_called_once_with(sheet, images)
     filter_rows_with_household_pk_mock.assert_called_once_with(config_mock, merge_images_mock.return_value)
+
+
+@pytest.mark.parametrize(
+    ("record", "expected"),
+    [
+        ({"full_name": "John Smith"}, "full_name"),
+        ({}, None),
+        ({"name_full": "John Smith"}, None),
+    ],
+)
+def test_full_name_column(record: Record, expected: str | None) -> None:
+    assert full_name_column(record) == expected

tests/workspace/test_ws_import.py

@@ -71,7 +71,7 @@ def test_import_data_rdi(force_migrated_records, app, program):
 
     res.forms["import-file"]["_selected_tab"] = "rdi"
     res.forms["import-file"]["rdi-file"] = Upload("rdi_one.xlsx", data)
-    res.forms["import-file"]["rdi-detail_column_label"] = "full_name_i_c"
+    res.forms["import-file"]["rdi-detail_column_label"] = "household_id"
     res = res.forms["import-file"].submit()
     assert res.status_code == 302
     assert program.households.count() == 1