Use value_mapper argument

Author: sergey-misuk-valor

src/country_workspace/contrib/kobo/sync.py

@@ -1,6 +1,6 @@
 import re
 from collections.abc import Callable, Iterable
-from functools import reduce, partial
+from functools import partial
 from typing import Any, Final, TypedDict, cast
 
 from constance import config as constance_config
@@ -16,6 +16,7 @@
 from country_workspace.models import AsyncJob, Batch, Household, Individual
 from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
 from country_workspace.utils.fields import clean_field_names, TO_UPPERCASE_FIELDS
+from country_workspace.utils.functional import compose
 
 
 class Config(BatchNameConfig, FailIfAlienConfig):
@@ -62,27 +63,18 @@ def normalize_json(data: dict[str, Any]) -> dict[str, Any]:
     return {key.split("/")[-1]: value for key, value in data.items()}
 
 
-EntryProcessor = Callable[[dict], dict]
+type RawIndividual = dict[str, Any]
 
 
-def apply_processor(entry: dict, processor: EntryProcessor) -> dict:
-    return processor(entry)
-
-
-def uppercase_fields(fields: tuple[str], entry: dict) -> dict:
-    for field in fields:
-        if field in entry:
-            entry[field] = entry[field].upper()
-
-    return entry
-
-
-def preprocess_individual(individual: dict) -> dict:
-    processors = (
+def preprocess_individual(individual: RawIndividual) -> RawIndividual:
+    clean: Callable[[RawIndividual], RawIndividual] = partial(
+        clean_field_names, fields_to_uppercase=FIELDS_TO_UPPERCASE + TO_UPPERCASE_FIELDS
+    )
+    processor = compose(
         normalize_json,
-        partial(clean_field_names, fields_to_uppercase=FIELDS_TO_UPPERCASE + TO_UPPERCASE_FIELDS),
+        clean,
     )
-    return reduce(apply_processor, processors, individual)
+    return processor(individual)
 
 
 def get_fullname_key(individual: Iterable[str]) -> str | None:
@@ -98,7 +90,7 @@ def create_individuals(batch: Batch, household: Household, submission: Submissio
             Individual(
                 batch=batch,
                 household=household,
-                name=individual.get(fullname, ""),
+                name=individual.get(fullname, "") if fullname else "",
                 flex_fields=individual,
             ),
         )

src/country_workspace/datasources/rdi.py

@@ -68,12 +68,6 @@ def has_household_pk(row: Record) -> bool:
     return filter(has_household_pk, sheet)
 
 
-def postprocess_cell(sheets: MultiSheet) -> MultiSheet:
-    for sheet_idx, rows in sheets:
-        formated_rows = ({k: strip_time_iso(v) for k, v in row.items()} for row in rows)
-        yield sheet_idx, formated_rows
-
-
 def process_households(sheet: Sheet, job: AsyncJob, batch: Batch, config: Config) -> Mapping[int, Household]:
     mapping = {}
 
@@ -162,7 +156,7 @@ def merge_images(sheet: Sheet, sheet_images: Mapping[int, Mapping[int, str]]) ->
 
 
 def read_sheets(config: Config, filepath: str, *sheet_indices: int) -> Generator[Sheet, None, None]:
-    sheets = postprocess_cell(open_xls_multi(filepath, sheets=list(sheet_indices)))
+    sheets = open_xls_multi(filepath, sheets=list(sheet_indices), value_mapper=strip_time_iso)
     sheet_images = extract_images(filepath, *sheet_indices)
     for (_, sheet), images in zip(sheets, sheet_images, strict=False):
         sheet_with_images = merge_images(sheet, images)

src/country_workspace/utils/functional.py

@@ -0,0 +1,27 @@
+from collections.abc import Callable
+from functools import reduce, partial
+from typing import overload, Any
+
+type Function[T, R] = Callable[[T], R]
+
+
+def apply[T, R](obj: T, func: Function[T, R]) -> R:
+    return func(obj)
+
+
+@overload
+def compose[T, R](func0: Function[T, R], /) -> Function[T, R]: ...
+
+
+@overload
+def compose[T, TR0, R](func0: Function[T, TR0], func1: Function[TR0, T], /) -> Function[T, R]: ...
+
+
+@overload
+def compose[T, TR0, TR1, R](
+    func0: Function[T, TR0], func1: Function[TR0, TR1], func2: Function[TR1, R], /
+) -> Function[T, R]: ...
+
+
+def compose(*funcs: Function[Any, Any]) -> Function[Any, Any]:
+    return partial(reduce, apply, funcs)

tests/contrib/kobo/test_kobo_sync.py

@@ -17,8 +17,6 @@
     import_data,
     is_submission_data_url,
     make_client,
-    apply_processor,
-    uppercase_fields,
     FIELDS_TO_UPPERCASE,
     preprocess_individual,
     get_fullname_key,
@@ -215,23 +213,6 @@ def test_import_data(mocker: MockerFixture, config: Config) -> None:
     import_asset_mock.assert_called_once_with(batch_mock, asset_mock, config)
 
 
-def test_apply_processor() -> None:
-    entry_mock = Mock()
-    processor_mock = Mock()
-    assert apply_processor(entry_mock, processor_mock) == processor_mock.return_value
-    processor_mock.assert_called_once_with(entry_mock)
-
-
-def test_uppercase_fields_field_exists() -> None:
-    result = uppercase_fields((field := "test",), {field: "test"})
-    assert result[field].isupper()
-
-
-def test_uppercase_fields_field_does_not_exist() -> None:
-    result = uppercase_fields((field := "test",), {})
-    assert field not in result
-
-
 def test_get_fullname_key_key_exists() -> None:
     assert get_fullname_key((key := "full_name",)) == key
 
@@ -244,14 +225,12 @@ def test_preprocess_individual(mocker: MockerFixture) -> None:
     normalize_json_mock = mocker.patch("country_workspace.contrib.kobo.sync.normalize_json")
     clean_field_names_mock = mocker.patch("country_workspace.contrib.kobo.sync.clean_field_names")
     partial_mock = mocker.patch("country_workspace.contrib.kobo.sync.partial")
-    reduce_mock = mocker.patch("country_workspace.contrib.kobo.sync.reduce")
-    apply_processor_mock = mocker.patch("country_workspace.contrib.kobo.sync.apply_processor")
+    compose_mock = mocker.patch("country_workspace.contrib.kobo.sync.compose")
     individual = Mock()
 
-    assert preprocess_individual(individual) == reduce_mock.return_value
+    assert preprocess_individual(individual) == compose_mock.return_value.return_value
     partial_mock.assert_called_once_with(
         clean_field_names_mock, fields_to_uppercase=FIELDS_TO_UPPERCASE + TO_UPPERCASE_FIELDS
     )
-    reduce_mock.assert_called_once_with(
-        apply_processor_mock, (normalize_json_mock, partial_mock.return_value), individual
-    )
+    compose_mock.assert_called_once_with(normalize_json_mock, partial_mock.return_value)
+    compose_mock.return_value.assert_called_once_with(individual)

tests/datasources/test_rdi.py

@@ -23,7 +23,6 @@
     merge_images,
     read_sheets,
     full_name_column,
-    postprocess_cell,
 )
 from country_workspace.datasources.utils import strip_time_iso
 from country_workspace.models import Household
@@ -294,8 +293,7 @@ def test_merge_images() -> None:
 
 def test_read_sheets(mocker: MockerFixture) -> None:
     fake_sheets = ((Mock(), sheet := Mock()),)
-    postprocess_cell_mock = mocker.patch("country_workspace.datasources.rdi.postprocess_cell")
-    postprocess_cell_mock.return_value = fake_sheets
+    strip_time_iso_mock = mocker.patch("country_workspace.datasources.rdi.strip_time_iso")
     open_xls_multi_mock = mocker.patch("country_workspace.datasources.rdi.open_xls_multi")
     open_xls_multi_mock.return_value = fake_sheets
     extract_images_mock = mocker.patch("country_workspace.datasources.rdi.extract_images")
@@ -307,7 +305,7 @@ def test_read_sheets(mocker: MockerFixture) -> None:
     result = list(read_sheets(config_mock, filepath := "test", sheet_index := 0))
 
     assert result == [filter_rows_with_household_pk_mock.return_value]
-    open_xls_multi_mock.assert_called_once_with(filepath, sheets=[sheet_index])
+    open_xls_multi_mock.assert_called_once_with(filepath, sheets=[sheet_index], value_mapper=strip_time_iso_mock)
     extract_images_mock.assert_called_once_with(filepath, sheet_index)
     merge_images_mock.assert_called_once_with(sheet, images)
     filter_rows_with_household_pk_mock.assert_called_once_with(config_mock, merge_images_mock.return_value)
@@ -339,25 +337,3 @@ def test_full_name_column(record: Record, expected: str | None) -> None:
 )
 def test_strip_time_iso(inp, expected):
     assert strip_time_iso(inp) == expected
-
-
-def test_postprocess_cell_simple():
-    sheets = [
-        (
-            7,
-            [
-                {"a": "2025-05-15 08:00:00", "b": "hello"},
-                {"a": "nope", "b": "2025-05-16 00:00:00"},
-            ],
-        )
-    ]
-    out = list(postprocess_cell(sheets))
-    idx, rows_gen = out[0]
-    assert len(out) == 1
-    assert idx == 7
-
-    rows = list(rows_gen)
-    assert rows == [
-        {"a": "2025-05-15", "b": "hello"},
-        {"a": "nope", "b": "2025-05-16"},
-    ]

tests/utils/test_functional.py

@@ -0,0 +1,16 @@
+from country_workspace.utils.functional import apply, compose
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+
+def test_apply() -> None:
+    f: Callable[[int], int] = lambda x: x + 1
+    assert apply(1, f) == f(1)
+
+
+def test_compose() -> None:
+    f0: Callable[[int], int] = lambda x: x + 1
+    f1: Callable[[int], int] = lambda x: x * 2
+    assert compose(f0, f1)(1) == f1(f0(1))