chg ! RDI for people

Author: vitali-yanushchyk-valor

src/country_workspace/datasources/rdi.py

@@ -2,7 +2,7 @@
 from base64 import b64encode
 from collections import defaultdict
 from collections.abc import Iterable, Generator
-from typing import Any, Mapping, cast
+from typing import Any, Mapping, cast, NotRequired
 
 import openpyxl
 from PIL import Image
@@ -28,9 +28,12 @@
 
 
 class Config(BatchNameConfig, FailIfAlienConfig):
-    household_pk_col: str
-    master_column_label: str
-    detail_column_label: str
+    master_detail: bool
+    household_pk_col: NotRequired[str]
+    master_column_label: NotRequired[str]
+    detail_column_label: NotRequired[str]
+    people_column_prefix: NotRequired[str]
+    first_line: int
 
 
 class ColumnConfigurationError(Exception):
@@ -129,6 +132,30 @@ def process_individuals(
     return processed
 
 
+def process_people_only(sheet: Sheet, job: AsyncJob, batch: Batch, config: Config) -> int:
+    processed = 0
+
+    for i, row in enumerate(sheet, 1):
+        cleaned_row = {k.removeprefix(config["people_column_prefix"]): v for k, v in row.items()}
+        name_column = full_name_column(cleaned_row)
+        name = get_value(cleaned_row, name_column) if name_column else None
+        flex_fields = clean_field_names(cleaned_row)
+
+        try:
+            job.program.individuals.create(
+                batch=batch,
+                name=name,
+                household_id=None,  # No household association in People scenario
+                flex_fields=flex_fields,
+            )
+        except Exception as e:
+            raise SheetProcessingError(INDIVIDUAL, i) from e
+
+        processed += 1
+
+    return processed
+
+
 def image_location(image: RDIImage) -> tuple[int, int]:
     return image.anchor._from.row, image.anchor._from.col
 
@@ -166,13 +193,15 @@ def read_sheets(config: Config, filepath: str, *sheet_indices: int) -> Generator
     sheet_images = extract_images(filepath, *sheet_indices)
     for (_, sheet), images in zip(sheets, sheet_images, strict=False):
         sheet_with_images = merge_images(sheet, images)
-        yield filter_rows_with_household_pk(config, sheet_with_images)
+        if config["master_detail"]:
+            yield filter_rows_with_household_pk(config, sheet_with_images)
+        else:
+            yield sheet_with_images
 
 
 def import_from_rdi(job: AsyncJob) -> dict[str, int]:
     with atomic():
-        config: Config = job.config
-        rdi = job.file
+        config = job.config
         batch = Batch.objects.create(
             name=config["batch_name"],
             program=job.program,
@@ -181,14 +210,12 @@ def import_from_rdi(job: AsyncJob) -> dict[str, int]:
             source=Batch.BatchSource.RDI,
         )
 
-        household_sheet, individual_sheet = read_sheets(config, rdi, 0, 1)
-
-        household_mapping = process_households(household_sheet, job, batch, config)
-        individuals_number = process_individuals(individual_sheet, household_mapping, job, batch, config)
-
-        validate_beneficiaries(config, household_mapping)
+        if config["master_detail"]:
+            household_sheet, individual_sheet = read_sheets(config, job.file, 0, 1)
+            household_mapping = process_households(household_sheet, job, batch, config)
+            individuals_count = process_individuals(individual_sheet, household_mapping, job, batch, config)
+            validate_beneficiaries(config, household_mapping)
+            return {"household": len(household_mapping), "individual": individuals_count}
 
-        return {
-            "household": len(household_mapping),
-            "individual": individuals_number,
-        }
+        (people_sheet,) = read_sheets(config, job.file, 0)
+        return {"people": process_people_only(people_sheet, job, batch, config)}

src/country_workspace/workspaces/admin/forms.py

@@ -4,6 +4,7 @@
 
 from country_workspace.workspaces.admin.cleaners.base import BaseActionForm
 from country_workspace.workspaces.validators import ValidatableFileValidator
+from country_workspace.models import BeneficiaryGroup
 
 if TYPE_CHECKING:
     from hope_flex_fields.models import DataChecker
@@ -54,6 +55,12 @@ class ImportFileForm(forms.Form):
         help_text="Which column should be used as label for the household. It can use interpolation",
     )
 
+    people_column_prefix = forms.CharField(
+        required=False,
+        initial="pp_",
+        help_text="People' column group prefix",
+    )
+
     first_line = forms.IntegerField(required=True, initial=0, help_text="First line to process")
 
     check_before = forms.BooleanField(
@@ -63,3 +70,14 @@ class ImportFileForm(forms.Form):
         required=False, help_text="Fails if it finds fields which do not exists in data checker."
     )
     file = forms.FileField(validators=[ValidatableFileValidator()])
+
+    def __init__(self, *args: Any, beneficiary_group: BeneficiaryGroup | None = None, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        if beneficiary_group:
+            exclude_fields = (
+                ("people_column_prefix",)
+                if beneficiary_group.master_detail
+                else ("pk_column_name", "master_column_label", "detail_column_label")
+            )
+            for field_name in exclude_fields:
+                self.fields.pop(field_name, None)

src/country_workspace/workspaces/admin/program.py

@@ -324,7 +324,7 @@ def import_data(self, request: HttpRequest, pk: str) -> "HttpResponse":
                     if not (form_kobo := self.import_kobo(request, program)):
                         return HttpResponseRedirect(reverse("workspace:workspaces_countryasyncjob_changelist"))
         else:
-            form_rdi = ImportFileForm(prefix="rdi")
+            form_rdi = ImportFileForm(prefix="rdi", beneficiary_group=program.beneficiary_group)
             form_aurora = ImportAuroraForm(prefix="aurora", program=program)
             form_kobo = ImportKoboForm(prefix="kobo", kobo_country_code=program.country_office.kobo_country_code)
 
@@ -335,15 +335,27 @@ def import_data(self, request: HttpRequest, pk: str) -> "HttpResponse":
         return render(request, "workspace/program/import.html", context)
 
     def import_rdi(self, request: HttpRequest, program: CountryProgram) -> "ImportFileForm | None":
-        form = ImportFileForm(request.POST, request.FILES, prefix="rdi")
+        form = ImportFileForm(request.POST, request.FILES, prefix="rdi", beneficiary_group=program.beneficiary_group)
         if form.is_valid():
             config: RDIConfig = {
+                "master_detail": (
+                    master_detail := (program.beneficiary_group.master_detail if program.beneficiary_group else False)
+                ),
                 "batch_name": form.cleaned_data["batch_name"] or batch_name_default(),
-                "household_pk_col": form.cleaned_data["pk_column_name"],
-                "master_column_label": form.cleaned_data["master_column_label"],
-                "detail_column_label": form.cleaned_data["detail_column_label"],
+                "first_line": form.cleaned_data["first_line"],
                 "check_before": (check_before := form.cleaned_data.get("check_before", False)),
                 "fail_if_alien": form.cleaned_data.get("fail_if_alien", False) if check_before else False,
+                **(
+                    {
+                        "household_pk_col": form.cleaned_data.get("pk_column_name"),
+                        "master_column_label": form.cleaned_data.get("master_column_label"),
+                        "detail_column_label": form.cleaned_data.get("detail_column_label"),
+                    }
+                    if master_detail
+                    else {
+                        "people_column_prefix": form.cleaned_data.get("people_column_prefix"),
+                    }
+                ),
             }
             job: AsyncJob = AsyncJob.objects.create(
                 description="RDI import",