Merge pull request #161 from unicef/feature/262502-fail_if_alien-default-as-true

use validate with choices when import instead of checkboxes

Author: domdinicola

src/country_workspace/contrib/aurora/forms.py

@@ -4,9 +4,10 @@
 
 from country_workspace.contrib.aurora.models import Registration
 from country_workspace.models import Program
+from country_workspace.workspaces.admin.forms import BaseImportForm
 
 
-class ImportAuroraForm(forms.Form):
+class ImportAuroraForm(BaseImportForm):
     batch_name = forms.CharField(required=False, help_text="Label for this batch.")
     registration = forms.ModelChoiceField(
         queryset=Registration.objects.none(),
@@ -24,12 +25,6 @@ class ImportAuroraForm(forms.Form):
         initial="family_name",
         help_text="Which Individual's column should be used as label for the household.",
     )
-    check_before = forms.BooleanField(
-        required=False, help_text="Prevent import if errors if data is not valid against data checker."
-    )
-    fail_if_alien = forms.BooleanField(
-        required=False, help_text="Fails if it finds fields which do not exists in data checker."
-    )
 
     def __init__(self, *args: Any, program: Program | None = None, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)

src/country_workspace/contrib/aurora/pipeline.py

@@ -6,12 +6,13 @@
 from country_workspace.contrib.aurora.exceptions import TooManyBeneficiaryError
 from country_workspace.models import AsyncJob, Batch, Household, Individual
 from country_workspace.models.household import RELATIONSHIP_HEAD, RELATIONSHIP_FIELDNAME
-from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
+from country_workspace.utils.config import BatchNameConfig, ValidateModeConfig
 from country_workspace.utils.fields import clean_field_names
+from country_workspace.utils.types import BeneficiaryMapping
 from country_workspace.validators.beneficiaries import validate_beneficiaries
 
 
-class Config(BatchNameConfig, FailIfAlienConfig):
+class Config(BatchNameConfig, ValidateModeConfig):
     registration_reference_pk: str | None
     master_detail: bool
     household_column_prefix: NotRequired[str]
@@ -61,12 +62,13 @@ def import_from_aurora(job: AsyncJob) -> dict[str, int]:
                 total["households"] += 1
             records_data.append((record_id, individuals))
 
-        validate_records(records_data, cfg)
+        if mapping := validate_records(records_data, cfg):
+            validate_beneficiaries(mapping, cfg, job.program.country_office)
 
     return total
 
 
-def validate_records(records_data: list[tuple[int, list[Individual]]], cfg: Config) -> None:
+def validate_records(records_data: list[tuple[int, list[Individual]]], cfg: Config) -> BeneficiaryMapping:
     """Validate beneficiaries based on configuration and record data.
 
     Args:
@@ -87,9 +89,7 @@ def validate_records(records_data: list[tuple[int, list[Individual]]], cfg: Conf
                 raise TooManyBeneficiaryError("Individual", record_id=record_id, count=len(individuals))
             if individuals:
                 mapping[record_id] = individuals[0]
-
-    if mapping:
-        validate_beneficiaries(cfg, mapping)
+    return mapping
 
 
 def create_household(batch: Batch, data: dict[str, Any], prefix: str) -> Household:

src/country_workspace/contrib/kobo/forms.py

@@ -3,22 +3,17 @@
 from django import forms
 
 from country_workspace.contrib.kobo.sync import make_client
+from country_workspace.workspaces.admin.forms import BaseImportForm
 
 
-class ImportKoboForm(forms.Form):
+class ImportKoboForm(BaseImportForm):
     batch_name = forms.CharField(required=False, help_text="Label for this batch")
     project_id = forms.ChoiceField(required=True, choices=(), help_text="Select a project")
     individual_records_field = forms.CharField(
         required=False,
         initial="individual_questions",
         help_text="Which field contains individual records",
     )
-    check_before = forms.BooleanField(
-        required=False, help_text="Prevent import if errors if data is not valid against data checker."
-    )
-    fail_if_alien = forms.BooleanField(
-        required=False, help_text="Fails if it finds fields which do not exists in data checker."
-    )
 
     def __init__(self, *args: Any, kobo_country_code: str | None, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)

src/country_workspace/contrib/kobo/sync.py

@@ -14,12 +14,12 @@
 from country_workspace.contrib.kobo.api.data.submission import Submission
 from country_workspace.contrib.kobo.models import KoboSubmission
 from country_workspace.models import AsyncJob, Batch, Household, Individual
-from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
+from country_workspace.utils.config import BatchNameConfig, ValidateModeConfig
 from country_workspace.utils.fields import clean_field_names, TO_UPPERCASE_FIELDS
 from country_workspace.utils.functional import compose
 
 
-class Config(BatchNameConfig, FailIfAlienConfig):
+class Config(BatchNameConfig, ValidateModeConfig):
     project_id: str
     individual_records_field: str
 

src/country_workspace/datasources/rdi.py

@@ -4,6 +4,7 @@
 from collections.abc import Iterable, Generator
 from enum import StrEnum
 from typing import Any, Mapping, cast, NotRequired
+from functools import partial
 
 import openpyxl
 from PIL import Image
@@ -14,9 +15,10 @@
 from country_workspace.contrib.kobo.api.data.helpers import VALUE_FORMAT
 from country_workspace.datasources.utils import datetime_to_date, date_to_iso_string
 from country_workspace.models import AsyncJob, Batch, Household, Individual
-from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
+from country_workspace.utils.config import BatchNameConfig, ValidateModeConfig
 from country_workspace.utils.fields import Record, clean_field_names
 from country_workspace.utils.functional import compose
+from country_workspace.utils.types import ValidateBeneficiaries
 from country_workspace.validators.beneficiaries import validate_beneficiaries
 
 RDI = str | io.BytesIO
@@ -29,7 +31,7 @@
 PEOPLE = "people"
 
 
-class Config(BatchNameConfig, FailIfAlienConfig):
+class Config(BatchNameConfig, ValidateModeConfig):
     master_detail: bool
     household_pk_col: NotRequired[str]
     master_column_label: NotRequired[str]
@@ -219,20 +221,24 @@ def import_from_rdi(job: AsyncJob) -> dict[str, int]:
             imported_by=job.owner,
             source=Batch.BatchSource.RDI,
         )
+        validate = partial(validate_beneficiaries, config=config, office=job.program.country_office)
         if config["master_detail"]:
-            return _import_master_detail(job, batch, config)
-        return _import_people_only(job, batch, config)
+            return _import_master_detail(job, batch, config, validate)
+        return _import_people_only(job, batch, config, validate)
 
 
-def _import_master_detail(job: AsyncJob, batch: Batch, config: dict) -> dict[str, int]:
+def _import_master_detail(
+    job: AsyncJob, batch: Batch, config: Config, validate: ValidateBeneficiaries
+) -> dict[str, int]:
     household_sheet, individual_sheet = read_sheets(config, job.file, SheetName.HOUSEHOLDS, SheetName.INDIVIDUALS)
     household_mapping = process_households(household_sheet, job, batch, config)
     individuals_mapping = process_beneficiaries(individual_sheet, job, batch, config, household_mapping)
-    validate_beneficiaries(config, household_mapping)
+    validate(household_mapping)
     return {"household": len(household_mapping), "individual": len(individuals_mapping)}
 
 
-def _import_people_only(job: AsyncJob, batch: Batch, config: dict) -> dict[str, int]:
+def _import_people_only(job: AsyncJob, batch: Batch, config: Config, validate: ValidateBeneficiaries) -> dict[str, int]:
     (people_sheet,) = read_sheets(config, job.file, SheetName.PEOPLE)
-    validate_beneficiaries(config, people_mapping := process_beneficiaries(people_sheet, job, batch, config))
+    people_mapping = process_beneficiaries(people_sheet, job, batch, config)
+    validate(people_mapping)
     return {"people": len(people_mapping)}

src/country_workspace/migrations/0020_alter_rdp_unique_together.py

@@ -0,0 +1,16 @@
+# Generated by Django 5.2.3 on 2025-07-10 13:56
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("country_workspace", "0019_mappingimporter"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="rdp",
+            unique_together={("push_date", "name")},
+        ),
+    ]

src/country_workspace/models/batch.py

@@ -14,7 +14,7 @@ class BatchSource(models.TextChoices):
     country_office = models.ForeignKey("Office", on_delete=models.CASCADE, related_name="%(class)ss")
     program = models.ForeignKey("Program", on_delete=models.CASCADE, related_name="%(class)ss")
     name = models.CharField(max_length=255, blank=True, null=True)
-    import_date = models.DateTimeField(auto_now=True, db_index=True)
+    import_date = models.DateTimeField(auto_now=True)
     imported_by = models.ForeignKey(User, on_delete=models.CASCADE)
     source = models.CharField(max_length=255, blank=True, null=True, choices=BatchSource.choices)
 

src/country_workspace/models/rdp.py

@@ -20,11 +20,11 @@ class PushStatus(models.TextChoices):
     hope_rdi_id = models.CharField(
         max_length=200, null=True, editable=False, help_text=_("RDI unique ID within the HOPE core.")
     )
-    push_date = models.DateTimeField(auto_now=True, db_index=True)
+    push_date = models.DateTimeField(auto_now=True)
     pushed_by = models.ForeignKey(User, on_delete=models.CASCADE)
 
     class Meta:
-        unique_together = (("program", "name"),)
+        unique_together = (("push_date", "name"),)
         verbose_name = _("Registration Data Push")
         verbose_name_plural = _("Registration Data Pushes")
 

src/country_workspace/utils/config.py

@@ -1,13 +1,10 @@
 from typing import TypedDict
+from country_workspace.workspaces.admin.forms import ValidateMode
 
 
 class BatchNameConfig(TypedDict):
     batch_name: str
 
 
-class CheckBeforeConfig(TypedDict):
-    check_before: bool
-
-
-class FailIfAlienConfig(CheckBeforeConfig):
-    fail_if_alien: bool
+class ValidateModeConfig(TypedDict):
+    validate_mode: ValidateMode

src/country_workspace/utils/types.py

@@ -1,4 +1,10 @@
-from typing import TypeVar
+from typing import Mapping, Protocol, TypeVar
 from country_workspace.models import Household, Individual
 
 T_Beneficiary = TypeVar("T_Beneficiary", bound=Individual | Household)
+
+type BeneficiaryMapping = Mapping[int, T_Beneficiary]
+
+
+class ValidateBeneficiaries(Protocol):
+    def __call__(self, mapping: BeneficiaryMapping) -> None: ...