Add flex fields grouping

Author: datamik

src/country_workspace/contrib/hope/push.py

@@ -1,5 +1,6 @@
 from collections.abc import Callable, Iterator
 from dataclasses import dataclass, field
+from functools import cached_property
 from itertools import batched
 from json import JSONDecodeError
 from typing import Any, TypedDict, ReadOnly
@@ -12,9 +13,8 @@
 
 from country_workspace.contrib.hope.client import HopeClient
 from country_workspace.exceptions import RemoteError
-from country_workspace.models import AsyncJob, Rdp
+from country_workspace.models import AsyncJob, Rdp, Program
 from country_workspace.workspaces.models import CountryHousehold, CountryIndividual
-from country_workspace.utils.fields import map_fields
 
 
 type Beneficiary = CountryHousehold | CountryIndividual
@@ -189,17 +189,24 @@ def safe_post(self, path: str, data: Any, error_msg: str) -> dict[str, Any] | No
             self.total["errors"].append(f"{error_msg}: {e}")
             return None
 
+    @cached_property
+    def program(self) -> Program:
+        return Program.objects.get(hope_id=self.program_hope_id)
+
     def prepare_batch(self) -> tuple[list[int], list[dict]]:
         """Prepare a batch of household/individual|people data for API submission."""
         ids, data = [], []
         for item in self.queryset:
             ids.append(item.id)
-            data.append(
-                {**map_fields(item.flex_fields), "members": [map_fields(m.flex_fields) for m in item.members.all()]}
-                if self.master_detail
-                else map_fields(item.flex_fields)
-            )
-        return ids, data
+            flex_fields = item.apply_grouping()
+            if self.master_detail:
+                flex_fields["members"] = []
+                for member in item.members.all():
+                    flex_fields["members"].append(member.apply_grouping())
+                data.append(flex_fields)
+            else:
+                data.append(flex_fields)
+        return ids, self.program.serialize(data)
 
     def process_batch_response(self, response: dict | None, batch_ids: list[int]) -> list[int]:
         """Process the API response for a batch push operation."""
@@ -279,7 +286,7 @@ def push_to_hope_core(job: AsyncJob) -> dict[str, Any]:
 
     def steps() -> Iterator[Callable[[], None]]:
         """Yield steps for pushing beneficiaries data in batches."""
-        yield processor.rdi_create
+        # yield processor.rdi_create
         for batch_pks in batched(config["pks"], config["batch_size"]):
             processor.set_queryset(batch_pks)
             yield from (processor.check_beneficiaries_validity, processor.rdi_push)

src/country_workspace/models/household.py

@@ -6,6 +6,7 @@
 from django.utils import timezone
 
 from .base import BaseModel, Validable
+from .mixins import FlexFieldGroupingMixin
 
 if TYPE_CHECKING:
     from django.db.models import QuerySet
@@ -24,7 +25,7 @@
 
 
 @pghistory.track(pghistory.UpdateEvent(condition=pghistory.AnyChange("flex_fields", "flex_files", "removed")))
-class Household(Validable, BaseModel):
+class Household(FlexFieldGroupingMixin, Validable, BaseModel):
     system_fields = models.JSONField(default=dict, blank=True)
     members: "QuerySet[Individual]"
 

src/country_workspace/models/individual.py

@@ -6,13 +6,14 @@
 
 from .base import BaseModel, Validable
 from .household import Household
+from .mixins import FlexFieldGroupingMixin
 
 if TYPE_CHECKING:
     from hope_flex_fields.models import DataChecker
 
 
 @pghistory.track(pghistory.UpdateEvent(condition=pghistory.AnyChange("flex_fields", "flex_files", "removed")))
-class Individual(Validable, BaseModel):
+class Individual(FlexFieldGroupingMixin, Validable, BaseModel):
     household = models.ForeignKey(Household, on_delete=models.CASCADE, null=True, blank=True, related_name="members")
     system_fields = models.JSONField(default=dict, blank=True)
 

src/country_workspace/models/mixins.py

@@ -0,0 +1,36 @@
+from collections import defaultdict
+
+
+class FlexFieldGroupingMixin:
+    def get_grouping_info(self) -> dict[str, list[str]]:
+        grouping_info = defaultdict(list)
+        for member in self.checker.members.select_related("fieldset").all():
+            if not member.prefix:
+                continue
+            if member.group == "":
+                continue
+            if member.group is None:
+                grouping_key = member.fieldset.group
+            else:
+                grouping_key = member.group
+
+            if grouping_key:
+                grouping_info[grouping_key].append(member.prefix)
+
+        return grouping_info
+
+    def apply_grouping(self) -> dict:
+        grouping_info = self.get_grouping_info()
+        grouped_data = {}
+        flex_fields = dict(self.flex_fields)
+        for grouping_key, prefixes in grouping_info.items():
+            grouped_data[grouping_key] = []
+            for prefix in prefixes:
+                _data = {}
+                for field_name in self.flex_fields:
+                    if field_name.startswith(prefix):
+                        _data[field_name.removeprefix(prefix)] = flex_fields.pop(field_name)
+                grouped_data[grouping_key].append(_data)
+        grouped_data.update(flex_fields)
+
+        return grouped_data