chg ! push to hope core according to beneficiary groups

Author: vitali-yanushchyk-valor

src/country_workspace/contrib/hope/constants.py

@@ -5,3 +5,4 @@
 # NEVER CHANGE THIS VALUES
 HOUSEHOLD_CHECKER_NAME: Final[str] = "HOPE Household core"
 INDIVIDUAL_CHECKER_NAME: Final[str] = "HOPE Individual core"
+PEOPLE_CHECKER_NAME: Final[str] = "HOPE People core"

src/country_workspace/contrib/hope/push.py

@@ -12,36 +12,48 @@
 from country_workspace.contrib.hope.constants import HOUSEHOLD_PUSH_BATCH_SIZE
 from country_workspace.exceptions import RemoteError
 from country_workspace.models import AsyncJob
-from country_workspace.workspaces.models import CountryHousehold
+from country_workspace.workspaces.models import CountryHousehold, CountryIndividual
 
 
 @dataclass
 class PushProcessor:
-    """Handles pushing household data to an external system through the HopeClient API."""
+    """Handles pushing beneficiaries data to an external system through the HopeClient API."""
 
     co_slug: str
     batch_name: str
     program_id: str
+    master_detail: bool
     queryset: QuerySet[CountryHousehold] = field(default_factory=lambda: CountryHousehold.objects.none())
     client: HopeClient = field(default_factory=HopeClient)
-    total: dict[str, Any] = field(default_factory=lambda: {"households": 0, "errors": []})
+    total: dict[str, Any] = field(default_factory=lambda: {"errors": []})
     rdi_id: str | None = field(default=None, init=False)
+    model: type = field(init=False)
+    push_endpoint: str = field(init=False)
+    has_members: bool = field(init=False)
 
     def __post_init__(self) -> None:
-        """Initialize the base path for API requests."""
         self.base_path = f"{self.co_slug}/rdi/"
+        self.model, self.push_endpoint, self.has_members = (
+            (CountryHousehold, "push/lax/", True) if self.master_detail else (CountryIndividual, "push/people/", False)
+        )
 
-    def check_households_validity(self) -> None:
-        """Check the validity of each household and its members in the queryset.
+    def set_queryset(self, pks: list[int]) -> None:
+        """Set the queryset based on master_detail and provided pks."""
+        qs = self.model.objects.filter(pk__in=pks)
+        self.queryset = qs.prefetch_related("members") if self.master_detail else qs
 
-        Adds errors to `self.total["errors"]` if any household or member is invalid.
+    def check_beneficiaries_validity(self) -> None:
+        """Check the validity of each beneficiaries in the queryset.
+
+        Adds errors to `self.total["errors"]` if any beneficiaries is invalid.
         """
-        for hh in self.queryset:
-            if not hh.is_valid():
-                self.total["errors"].append(f"HH #{hh.pk} invalid.")
-            for ind in hh.members.all():
-                if not ind.is_valid():
-                    self.total["errors"].append(f"Ind #{ind.pk} invalid.")
+        for item in self.queryset:
+            if not item.is_valid():
+                self.total["errors"].append(f"{self.model.__name__} #{item.pk} invalid")
+            if self.has_members:
+                for ind in item.members.all():
+                    if not ind.is_valid():
+                        self.total["errors"].append(f"Ind #{ind.pk} invalid")
 
     def rdi_create(self) -> None:
         """Create a new RDI record in the external system.
@@ -53,9 +65,9 @@ def rdi_create(self) -> None:
         if response := self.safe_post(path, data, "Error creating RDI"):
             self.rdi_id = response.get("id")
 
-    def rdi_push_lax(self) -> None:
+    def rdi_push(self) -> None:
         """
-        Pushes a batch of household data to the external RDI system.
+        Pushes a batch of beneficiaries data to the external RDI system.
 
         Adds errors to `self.total["errors"]` if `rdi_id` is not set.
         Successfully pushed records are marked as removed.
@@ -64,7 +76,7 @@ def rdi_push_lax(self) -> None:
             self.total["errors"].append("Cannot push data: rdi_id is not set")
             return
         batch_ids, batch_data = self.prepare_batch()
-        path = f"{self.base_path}{self.rdi_id}/push/lax/"
+        path = f"{self.base_path}{self.rdi_id}/{self.push_endpoint}"
         if successful_ids := self.process_batch_response(
             self.safe_post(path, batch_data, "Error pushing data"),
             batch_ids,
@@ -103,17 +115,19 @@ def safe_post(self, path: str, data: Any, error_msg: str) -> dict[str, Any] | No
 
     def prepare_batch(self) -> tuple[list[int], list[dict]]:
         """
-        Prepare a batch of household data for API submission.
+        Prepare a batch of household/individual data for API submission.
 
         Returns:
-            tuple[list[int], list[dict]]: A tuple of household IDs and transformed data.
+            tuple[list[int], list[dict]]: A tuple of household/individual IDs and transformed data.
 
         """
         ids, data = [], []
         for item in self.queryset:
             ids.append(item.id)
             data.append(
                 {**map_fields(item.flex_fields), "members": [map_fields(m.flex_fields) for m in item.members.all()]}
+                if self.has_members
+                else map_fields(item.flex_fields)
             )
         return ids, data
 
@@ -133,6 +147,11 @@ def process_batch_response(self, response: dict | None, batch_ids: list[int]) ->
             case {"processed": int(p), "accepted": int(a)} if p == a == len(batch_ids):
                 self.total["households"] = self.total.get("households", 0) + a
                 return batch_ids
+            case {"id": str(_rdi_id), "people": list(_batch_ids)} if _rdi_id == self.rdi_id and len(_batch_ids) == len(
+                batch_ids
+            ):
+                self.total["people"] = self.total.get("people", 0) + len(_batch_ids)
+                return batch_ids
             case {"errors": int(e)} if e > 0:
                 self.total["errors"].append(f"Error pushing data for IDs: {batch_ids} - {response}")
             case None:
@@ -143,27 +162,30 @@ def process_batch_response(self, response: dict | None, batch_ids: list[int]) ->
 
     def mark_batch_removed(self, successful_ids: list[int]) -> None:
         """
-        Mark successfully pushed households and members as removed in the database.
+        Mark successfully pushed beneficiaries as removed in the database.
 
         Args:
-            successful_ids (list[int]): List of successfully pushed household IDs.
+            successful_ids (list[int]): List of successfully pushed beneficiaries IDs.
 
         """
         try:
             with transaction.atomic():
-                households = CountryHousehold.objects.filter(id__in=successful_ids).prefetch_related("members")
-                for hh in households:
-                    if hh.removed:
-                        self.total["errors"].append(f"Household #{hh.id} already marked as removed")
+                items = self.model.objects.filter(id__in=successful_ids)
+                if self.has_members:
+                    items = items.prefetch_related("members")
+                for item in items:
+                    if item.removed:
+                        self.total["errors"].append(f"{self.model.__name__} #{item.id} already marked as removed")
                     else:
-                        hh.removed = True
-                        hh.save(update_fields=["removed"])
-                    for ind in hh.members.all():
-                        if ind.removed:
-                            self.total["errors"].append(f"Individual #{ind.id} already marked as removed")
-                        else:
-                            ind.removed = True
-                            ind.save(update_fields=["removed"])
+                        item.removed = True
+                        item.save(update_fields=["removed"])
+                    if self.has_members:
+                        for ind in item.members.all():
+                            if ind.removed:
+                                self.total["errors"].append(f"Individual #{ind.id} already marked as removed")
+                            else:
+                                ind.removed = True
+                                ind.save(update_fields=["removed"])
         except (DatabaseError, Exception) as e:
             self.total["errors"].append(f"Failed to mark IDs {successful_ids} as removed: {e}")
 
@@ -176,23 +198,28 @@ def push_to_hope_core(job: AsyncJob) -> dict[str, Any]:
         job (AsyncJob): The job configuration containing relevant identifiers and parameters.
 
     Returns:
-        dict[str, Any]: Summary of the operation including processed households and errors.
+        dict[str, Any]: Summary of the operation including processed beneficiaries and errors.
 
     """
 
     def steps() -> Iterator[Callable[[], None]]:
-        """Yield steps for pushing household data in batches."""
+        """Yield steps for pushing beneficiaries data in batches."""
         yield processor.rdi_create
         for batch_pks in batched(job.config["pks"], HOUSEHOLD_PUSH_BATCH_SIZE):
-            processor.queryset = CountryHousehold.objects.filter(pk__in=batch_pks).prefetch_related("members")
-            yield from (processor.check_households_validity, processor.rdi_push_lax)
+            processor.set_queryset(batch_pks)
+            yield from (processor.check_beneficiaries_validity, processor.rdi_push)
         yield processor.rdi_complete
 
+    if job.program.beneficiary_group is None:
+        return {"errors": ["Cannot proceed: beneficiary_group is not set"]}
+
     processor = PushProcessor(
         co_slug=job.program.country_office.slug,
         batch_name=job.config.get("batch_name"),
         program_id=job.program.hope_id,
+        master_detail=job.program.beneficiary_group.master_detail,
     )
+
     for step in steps():
         step()
         if processor.total["errors"]:

src/country_workspace/models/program.py

@@ -92,7 +92,7 @@ class Program(BaseModel):
     household_search = models.TextField(default="name", help_text="Fields to use for searches")
     individual_search = models.TextField(default="name", help_text="Fields to use for searches")
     household_columns = models.TextField(default="name\nid", help_text="Columns to display in the Admin table")
-    individual_columns = models.TextField(default="name\nid", help_text="Columns to display i the Admin table")
+    individual_columns = models.TextField(default="name\nid", help_text="Columns to display in the Admin table")
     extra_fields = models.JSONField(default=dict, blank=True, null=False)
 
     def __str__(self) -> str:

src/country_workspace/versioning/checkers.py

@@ -1,7 +1,11 @@
 from django import forms
 from hope_flex_fields.models import DataChecker, FieldDefinition, Fieldset
 
-from country_workspace.contrib.hope.constants import HOUSEHOLD_CHECKER_NAME, INDIVIDUAL_CHECKER_NAME
+from country_workspace.contrib.hope.constants import (
+    HOUSEHOLD_CHECKER_NAME,
+    INDIVIDUAL_CHECKER_NAME,
+    PEOPLE_CHECKER_NAME,
+)
 
 
 def create_hope_checkers() -> None:  # noqa: PLR0915
@@ -17,6 +21,8 @@ def create_hope_checkers() -> None:  # noqa: PLR0915
     _i_role = FieldDefinition.objects.get(slug="hope-ind-role")
     _i_relationship = FieldDefinition.objects.get(slug="hope-ind-relationship")
 
+    _p_type = FieldDefinition.objects.get(slug="hope-people-type")
+
     hh_fs, __ = Fieldset.objects.get_or_create(name=HOUSEHOLD_CHECKER_NAME)
     hh_fs.fields.get_or_create(name="address", definition=_char)
     hh_fs.fields.get_or_create(name="admin1", definition=_char)
@@ -86,14 +92,28 @@ def create_hope_checkers() -> None:  # noqa: PLR0915
     )
     ind_fs.fields.get_or_create(name="role", attrs={"label": "Role"}, definition=_i_role)
 
+    pp_fs, __ = Fieldset.objects.get_or_create(name=PEOPLE_CHECKER_NAME)
+    pp_fs.fields.get_or_create(name="type", attrs={"label": "People Type", "required": True}, definition=_p_type)
+    pp_fs.fields.get_or_create(name="full_name", attrs={"label": "Full Name", "required": True}, definition=_char)
+    pp_fs.fields.get_or_create(name="country", attrs={"label": "Country", "required": True}, definition=_h_country)
+    pp_fs.fields.get_or_create(
+        name="residence_status", attrs={"label": "Residence Status", "required": True}, definition=_h_residence
+    )
+    pp_fs.fields.get_or_create(name="gender", definition=_i_gender)
+    pp_fs.fields.get_or_create(name="birth_date", attrs={"label": "Birth Date", "required": True}, definition=_date)
+
     hh_dc, __ = DataChecker.objects.get_or_create(name=HOUSEHOLD_CHECKER_NAME)
     hh_dc.fieldsets.add(hh_fs)
     ind_dc, __ = DataChecker.objects.get_or_create(name=INDIVIDUAL_CHECKER_NAME)
     ind_dc.fieldsets.add(ind_fs)
+    pp_dc, __ = DataChecker.objects.get_or_create(name=PEOPLE_CHECKER_NAME)
+    pp_dc.fieldsets.add(pp_fs)
 
 
 def removes_hope_checkers() -> None:
     DataChecker.objects.filter(name=HOUSEHOLD_CHECKER_NAME).delete()
     DataChecker.objects.filter(name=INDIVIDUAL_CHECKER_NAME).delete()
     Fieldset.objects.filter(name=HOUSEHOLD_CHECKER_NAME).delete()
     Fieldset.objects.filter(name=INDIVIDUAL_CHECKER_NAME).delete()
+    Fieldset.objects.filter(name=PEOPLE_CHECKER_NAME).delete()
+    Fieldset.objects.filter(name=PEOPLE_CHECKER_NAME).delete()

src/country_workspace/versioning/hope_fields.py

@@ -69,6 +69,18 @@ def create_hope_field_definitions() -> None:
         },
     )
 
+    FieldDefinition.objects.get_or_create(
+        name="HOPE People Type",
+        slug=slugify("HOPE People Type"),
+        field_type=forms.ChoiceField,
+        attrs={
+            "choices": [
+                ["", ""],
+                ["NON_BENEFICIARY", "Non Beneficiary"],
+            ]
+        },
+    )
+
     SyncLog.objects.create_lookups()
 
 

src/country_workspace/workspaces/admin/cleaners/actions.py

@@ -9,11 +9,12 @@
 
 from country_workspace.contrib.hope.forms import PushToHopeForm
 from country_workspace.contrib.hope.push import push_to_hope_core
-from country_workspace.models import AsyncJob, Household
+from country_workspace.models import AsyncJob
 from country_workspace.state import state
 from country_workspace.utils.fields import rdi_name_default
 from country_workspace.workspaces.admin.forms import BulkUpdateExportForm
 
+from ...models import CountryIndividual, CountryHousehold
 from .bulk_update import bulk_update_export_template
 from .calculate_checksum import calculate_checksum_impl
 from .mass_update import MassUpdateForm, mass_update_impl
@@ -25,7 +26,6 @@
 
     from country_workspace.types import Beneficiary
     from country_workspace.workspaces.admin.hh_ind import BeneficiaryBaseAdmin
-    from country_workspace.workspaces.admin.household import CountryHouseholdAdmin
 
 
 @admin.action(description="Validate selected records", permissions=["validate"])
@@ -189,14 +189,23 @@ def calculate_checksum(
 
 @admin.action(description="Push to HOPE core", permissions=["push_to_hope"])
 def push_to_hope(
-    model_admin: "CountryHouseholdAdmin",
+    model_admin: "BeneficiaryBaseAdmin",
     request: HttpRequest,
-    queryset: "QuerySet[Household]",
+    queryset: "QuerySet[Beneficiary]",
 ) -> HttpResponse:
     ctx = model_admin.get_common_context(request, title=_(push_to_hope.short_description))
     form = PushToHopeForm(request.POST)
     ctx["form"] = form
     if "_push" in request.POST and form.is_valid():
+        if (program := model_admin.get_selected_program(request)) and program.beneficiary_group:
+            expected_model = CountryHousehold if program.beneficiary_group.master_detail else CountryIndividual
+            if not isinstance(queryset.model, expected_model.__class__):
+                model_admin.message_user(
+                    request,
+                    "Program, beneficiary group not set, or action unavailable for this model.",
+                    messages.ERROR,
+                )
+                return redirect(request.path)
         job = AsyncJob.objects.create(
             description=push_to_hope.short_description,
             type=AsyncJob.JobType.TASK,
@@ -205,6 +214,7 @@ def push_to_hope(
             program=state.program,
             config={
                 "batch_name": form.cleaned_data["batch_name"] or rdi_name_default(),
+                "master_detail": program.beneficiary_group.master_detail,
                 "pks": list(queryset.values_list("pk", flat=True)),
             },
         )