Merge pull request #76 from unicef/bugfix/remove-extra-prefixes-from-kobo-data

Bugfix: Remove extra prefixes from Kobo data

Author: sergey-misuk-valor

src/country_workspace/contrib/kobo/sync.py

@@ -54,12 +54,14 @@ def extract_household_data(submission: Submission, individual_records_field: str
     return {key: value for key, value in submission.items() if key != individual_records_field}
 
 
+def normalize_json(data: dict[str, Any]) -> dict[str, Any]:
+    return {key.split("/")[-1]: value for key, value in data.items()}
+
+
 def create_individuals(batch: Batch, household: Household, submission: Submission, config: Config) -> int:
     individuals = []
     for raw_individual in submission.get(config["individual_records_field"], []):
-        individual = {
-            key.replace(f"{config['individual_records_field']}/", ""): value for key, value in raw_individual.items()
-        }
+        individual = normalize_json(raw_individual)
         fullname = next((key for key in individual if key.startswith("full_name")), None)
         individuals.append(
             Individual(
@@ -74,7 +76,8 @@ def create_individuals(batch: Batch, household: Household, submission: Submissio
 
 
 def create_household(batch: Batch, submission: Submission, config: Config) -> Household:
-    household_fields = extract_household_data(submission, config["individual_records_field"])
+    raw_household_fields = extract_household_data(submission, config["individual_records_field"])
+    household_fields = normalize_json(raw_household_fields)
     return cast(
         Household,
         batch.program.households.create(

tests/contrib/kobo/test_kobo_sync.py

@@ -130,6 +130,7 @@ def test_create_individuals(mocker: MockerFixture, config: Config) -> None:
 def test_create_household(mocker: MockerFixture, config: Config) -> None:
     clean_field_names_mock = mocker.patch("country_workspace.contrib.kobo.sync.clean_field_names")
     extract_household_data_mock = mocker.patch("country_workspace.contrib.kobo.sync.extract_household_data")
+    normalize_json_mock = mocker.patch("country_workspace.contrib.kobo.sync.normalize_json")
 
     household = create_household(
         batch_mock := Mock(name="batch"),
@@ -142,7 +143,8 @@ def test_create_household(mocker: MockerFixture, config: Config) -> None:
     batch_mock.program.households.create.assert_called_once_with(
         batch=batch_mock, flex_fields=clean_field_names_mock.return_value
     )
-    clean_field_names_mock.assert_called_once_with(extract_household_data_mock.return_value)
+    normalize_json_mock.assert_called_once_with(extract_household_data_mock.return_value)
+    clean_field_names_mock.assert_called_once_with(normalize_json_mock.return_value)
 
 
 def test_import_asset(mocker: MockerFixture, config: Config) -> None: