use validate with choices when import instead of checkboxes

src/country_workspace/contrib/aurora/forms.py

@@ -4,9 +4,10 @@
 
 from country_workspace.contrib.aurora.models import Registration
 from country_workspace.models import Program
+from country_workspace.workspaces.admin.forms import BaseImportForm
 
 
-class ImportAuroraForm(forms.Form):
+class ImportAuroraForm(BaseImportForm):
     batch_name = forms.CharField(required=False, help_text="Label for this batch.")
     registration = forms.ModelChoiceField(
         queryset=Registration.objects.none(),
@@ -24,12 +25,6 @@ class ImportAuroraForm(forms.Form):
         initial="family_name",
         help_text="Which Individual's column should be used as label for the household.",
     )
-    check_before = forms.BooleanField(
-        required=False, help_text="Prevent import if errors if data is not valid against data checker."
-    )
-    fail_if_alien = forms.BooleanField(
-        required=False, help_text="Fails if it finds fields which do not exists in data checker."
-    )
 
     def __init__(self, *args: Any, program: Program | None = None, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)

src/country_workspace/contrib/aurora/pipeline.py

@@ -6,12 +6,13 @@
 from country_workspace.contrib.aurora.exceptions import TooManyBeneficiaryError
 from country_workspace.models import AsyncJob, Batch, Household, Individual
 from country_workspace.models.household import RELATIONSHIP_HEAD, RELATIONSHIP_FIELDNAME
-from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
+from country_workspace.utils.config import BatchNameConfig, ValidateModeConfig
 from country_workspace.utils.fields import clean_field_names
+from country_workspace.utils.types import BeneficiaryMapping
 from country_workspace.validators.beneficiaries import validate_beneficiaries
 
 
-class Config(BatchNameConfig, FailIfAlienConfig):
+class Config(BatchNameConfig, ValidateModeConfig):
     registration_reference_pk: str | None
     master_detail: bool
     household_column_prefix: NotRequired[str]
@@ -61,12 +62,13 @@ def import_from_aurora(job: AsyncJob) -> dict[str, int]:
                 total["households"] += 1
             records_data.append((record_id, individuals))
 
-        validate_records(records_data, cfg)
+        if mapping := validate_records(records_data, cfg):
+            validate_beneficiaries(mapping, cfg, job.program.country_office)
 
     return total
 
 
-def validate_records(records_data: list[tuple[int, list[Individual]]], cfg: Config) -> None:
+def validate_records(records_data: list[tuple[int, list[Individual]]], cfg: Config) -> BeneficiaryMapping:
     """Validate beneficiaries based on configuration and record data.
 
     Args:
@@ -87,9 +89,7 @@ def validate_records(records_data: list[tuple[int, list[Individual]]], cfg: Conf
                 raise TooManyBeneficiaryError("Individual", record_id=record_id, count=len(individuals))
             if individuals:
                 mapping[record_id] = individuals[0]
-
-    if mapping:
-        validate_beneficiaries(cfg, mapping)
+    return mapping
 
 
 def create_household(batch: Batch, data: dict[str, Any], prefix: str) -> Household:

src/country_workspace/contrib/kobo/forms.py

@@ -3,22 +3,17 @@
 from django import forms
 
 from country_workspace.contrib.kobo.sync import make_client
+from country_workspace.workspaces.admin.forms import BaseImportForm
 
 
-class ImportKoboForm(forms.Form):
+class ImportKoboForm(BaseImportForm):
     batch_name = forms.CharField(required=False, help_text="Label for this batch")
     project_id = forms.ChoiceField(required=True, choices=(), help_text="Select a project")
     individual_records_field = forms.CharField(
         required=False,
         initial="individual_questions",
         help_text="Which field contains individual records",
     )
-    check_before = forms.BooleanField(
-        required=False, help_text="Prevent import if errors if data is not valid against data checker."
-    )
-    fail_if_alien = forms.BooleanField(
-        required=False, help_text="Fails if it finds fields which do not exists in data checker."
-    )
 
     def __init__(self, *args: Any, kobo_country_code: str | None, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)

src/country_workspace/contrib/kobo/sync.py

@@ -14,12 +14,12 @@
 from country_workspace.contrib.kobo.api.data.submission import Submission
 from country_workspace.contrib.kobo.models import KoboSubmission
 from country_workspace.models import AsyncJob, Batch, Household, Individual
-from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
+from country_workspace.utils.config import BatchNameConfig, ValidateModeConfig
 from country_workspace.utils.fields import clean_field_names, TO_UPPERCASE_FIELDS
 from country_workspace.utils.functional import compose
 
 
-class Config(BatchNameConfig, FailIfAlienConfig):
+class Config(BatchNameConfig, ValidateModeConfig):
     project_id: str
     individual_records_field: str
 

src/country_workspace/datasources/rdi.py

@@ -4,6 +4,7 @@
 from collections.abc import Iterable, Generator
 from enum import StrEnum
 from typing import Any, Mapping, cast, NotRequired
+from functools import partial
 
 import openpyxl
 from PIL import Image
@@ -14,9 +15,10 @@
 from country_workspace.contrib.kobo.api.data.helpers import VALUE_FORMAT
 from country_workspace.datasources.utils import datetime_to_date, date_to_iso_string
 from country_workspace.models import AsyncJob, Batch, Household, Individual
-from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
+from country_workspace.utils.config import BatchNameConfig, ValidateModeConfig
 from country_workspace.utils.fields import Record, clean_field_names
 from country_workspace.utils.functional import compose
+from country_workspace.utils.types import ValidateBeneficiaries
 from country_workspace.validators.beneficiaries import validate_beneficiaries
 
 RDI = str | io.BytesIO
@@ -29,7 +31,7 @@
 PEOPLE = "people"
 
 
-class Config(BatchNameConfig, FailIfAlienConfig):
+class Config(BatchNameConfig, ValidateModeConfig):
     master_detail: bool
     household_pk_col: NotRequired[str]
     master_column_label: NotRequired[str]
@@ -219,20 +221,24 @@ def import_from_rdi(job: AsyncJob) -> dict[str, int]:
             imported_by=job.owner,
             source=Batch.BatchSource.RDI,
         )
+        validate = partial(validate_beneficiaries, config=config, office=job.program.country_office)
         if config["master_detail"]:
-            return _import_master_detail(job, batch, config)
-        return _import_people_only(job, batch, config)
+            return _import_master_detail(job, batch, config, validate)
+        return _import_people_only(job, batch, config, validate)
 
 
-def _import_master_detail(job: AsyncJob, batch: Batch, config: dict) -> dict[str, int]:
+def _import_master_detail(
+    job: AsyncJob, batch: Batch, config: Config, validate: ValidateBeneficiaries
+) -> dict[str, int]:
     household_sheet, individual_sheet = read_sheets(config, job.file, SheetName.HOUSEHOLDS, SheetName.INDIVIDUALS)
     household_mapping = process_households(household_sheet, job, batch, config)
     individuals_mapping = process_beneficiaries(individual_sheet, job, batch, config, household_mapping)
-    validate_beneficiaries(config, household_mapping)
+    validate(household_mapping)
     return {"household": len(household_mapping), "individual": len(individuals_mapping)}
 
 
-def _import_people_only(job: AsyncJob, batch: Batch, config: dict) -> dict[str, int]:
+def _import_people_only(job: AsyncJob, batch: Batch, config: Config, validate: ValidateBeneficiaries) -> dict[str, int]:
     (people_sheet,) = read_sheets(config, job.file, SheetName.PEOPLE)
-    validate_beneficiaries(config, people_mapping := process_beneficiaries(people_sheet, job, batch, config))
+    people_mapping = process_beneficiaries(people_sheet, job, batch, config)
+    validate(people_mapping)
     return {"people": len(people_mapping)}

src/country_workspace/migrations/0020_alter_rdp_unique_together.py

@@ -0,0 +1,16 @@
+# Generated by Django 5.2.3 on 2025-07-10 13:56
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("country_workspace", "0019_mappingimporter"),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name="rdp",
+            unique_together={("push_date", "name")},
+        ),
+    ]

src/country_workspace/models/batch.py

@@ -14,7 +14,7 @@ class BatchSource(models.TextChoices):
     country_office = models.ForeignKey("Office", on_delete=models.CASCADE, related_name="%(class)ss")
     program = models.ForeignKey("Program", on_delete=models.CASCADE, related_name="%(class)ss")
     name = models.CharField(max_length=255, blank=True, null=True)
-    import_date = models.DateTimeField(auto_now=True, db_index=True)
+    import_date = models.DateTimeField(auto_now=True)
     imported_by = models.ForeignKey(User, on_delete=models.CASCADE)
     source = models.CharField(max_length=255, blank=True, null=True, choices=BatchSource.choices)
 

src/country_workspace/models/rdp.py

@@ -20,11 +20,11 @@ class PushStatus(models.TextChoices):
     hope_rdi_id = models.CharField(
         max_length=200, null=True, editable=False, help_text=_("RDI unique ID within the HOPE core.")
     )
-    push_date = models.DateTimeField(auto_now=True, db_index=True)
+    push_date = models.DateTimeField(auto_now=True)
     pushed_by = models.ForeignKey(User, on_delete=models.CASCADE)
 
     class Meta:
-        unique_together = (("program", "name"),)
+        unique_together = (("push_date", "name"),)
         verbose_name = _("Registration Data Push")
         verbose_name_plural = _("Registration Data Pushes")
 

src/country_workspace/utils/config.py

@@ -1,13 +1,10 @@
 from typing import TypedDict
+from country_workspace.workspaces.admin.forms import ValidateMode
 
 
 class BatchNameConfig(TypedDict):
     batch_name: str
 
 
-class CheckBeforeConfig(TypedDict):
-    check_before: bool
-
-
-class FailIfAlienConfig(CheckBeforeConfig):
-    fail_if_alien: bool
+class ValidateModeConfig(TypedDict):
+    validate_mode: ValidateMode

src/country_workspace/utils/types.py

@@ -1,4 +1,10 @@
-from typing import TypeVar
+from typing import Mapping, Protocol, TypeVar
 from country_workspace.models import Household, Individual
 
 T_Beneficiary = TypeVar("T_Beneficiary", bound=Individual | Household)
+
+type BeneficiaryMapping = Mapping[int, T_Beneficiary]
+
+
+class ValidateBeneficiaries(Protocol):
+    def __call__(self, mapping: BeneficiaryMapping) -> None: ...

src/country_workspace/validators/beneficiaries.py

@@ -1,12 +1,19 @@
-from typing import Mapping
 from country_workspace.workspaces.exceptions import BeneficiaryValidationError
-from country_workspace.utils.types import T_Beneficiary
 
-from country_workspace.utils.config import FailIfAlienConfig
+from country_workspace.models import Office
+from country_workspace.state import state
+from country_workspace.utils.config import ValidateModeConfig
+from country_workspace.utils.types import BeneficiaryMapping
+from country_workspace.workspaces.admin.forms import ValidateMode
 
 
-def validate_beneficiaries(config: FailIfAlienConfig, beneficiary_mapping: Mapping[int, T_Beneficiary]) -> None:
-    if config.get("check_before", False):
+def validate_beneficiaries(beneficiary_mapping: BeneficiaryMapping, config: ValidateModeConfig, office: Office) -> None:
+    mode = ValidateMode(config["validate_mode"])
+    if mode is ValidateMode.NONE:
+        return
+
+    fail_if_alien = mode is ValidateMode.CHECK_AND_FAIL_IF_ALIEN
+    with state.set(tenant=office):
         for key, beneficiary in beneficiary_mapping.items():
-            if not beneficiary.validate_with_checker(fail_if_alien=config.get("fail_if_alien", False)):
+            if not beneficiary.validate_with_checker(fail_if_alien=fail_if_alien):
                 raise BeneficiaryValidationError(beneficiary._meta.object_name, key)

src/country_workspace/workspaces/admin/forms.py

@@ -1,6 +1,8 @@
 from typing import TYPE_CHECKING, Any
 
 from django import forms
+from django.db.models import TextChoices
+from django.utils.translation import gettext_lazy as _
 
 from country_workspace.workspaces.admin.cleaners.base import BaseActionForm
 from country_workspace.workspaces.validators import ValidatableFileValidator
@@ -34,41 +36,54 @@ class BulkUpdateImportForm(forms.Form):
     )
 
 
-class ImportFileForm(forms.Form):
+class ValidateMode(TextChoices):
+    NONE = "none", _("Skip validation — import data as is.")
+    CHECK_BEFORE = "check_before", _("Prevent import if data is not valid against data checker.")
+    CHECK_AND_FAIL_IF_ALIEN = (
+        "check_and_fail_if_alien",
+        _("Prevent import if data is invalid AND fail if an alien field is found."),
+    )
+
+
+class BaseImportForm(forms.Form):
     batch_name = forms.CharField(required=False, help_text="Label for this batch")
+    validate_mode = forms.TypedChoiceField(
+        choices=ValidateMode.choices,
+        coerce=ValidateMode,
+        empty_value=ValidateMode.CHECK_AND_FAIL_IF_ALIEN,
+        initial=ValidateMode.CHECK_AND_FAIL_IF_ALIEN,
+        required=True,
+        help_text=_("How to validate data before import"),
+    )
+
 
+class ImportFileForm(BaseImportForm):
     pk_column_name = forms.CharField(
         required=True,
         initial="household_id",
-        help_text="Which column contains the unique identifier of the record.It is mandatory from Master/detail",
+        help_text=_("Which column contains the unique identifier of the record. It is mandatory from Master/detail"),
     )
 
     master_column_label = forms.CharField(
         required=False,
         initial="household_id",
-        help_text="Which column contains the 'link' to the household record.",
+        help_text=_("Which column contains the 'link' to the household record."),
     )
 
     detail_column_label = forms.CharField(
         required=False,
         initial="household_id",
-        help_text="Which column should be used as label for the household. It can use interpolation",
+        help_text=_("Which column should be used as label for the household. It can use interpolation"),
     )
 
     people_column_prefix = forms.CharField(
         required=False,
         initial="pp_",
-        help_text="People' column group prefix",
+        help_text=_("People' column group prefix"),
     )
 
     first_line = forms.IntegerField(required=True, initial=0, help_text="First line to process")
 
-    check_before = forms.BooleanField(
-        required=False, help_text="Prevent import if errors if data is not valid against data checker."
-    )
-    fail_if_alien = forms.BooleanField(
-        required=False, help_text="Fails if it finds fields which do not exists in data checker."
-    )
     file = forms.FileField(validators=[ValidatableFileValidator()])
 
     def __init__(self, *args: Any, beneficiary_group: BeneficiaryGroup | None = None, **kwargs: Any) -> None:

src/country_workspace/workspaces/admin/program.py

@@ -342,9 +342,8 @@ def import_rdi(self, request: HttpRequest, program: CountryProgram) -> "ImportFi
                     master_detail := (program.beneficiary_group.master_detail if program.beneficiary_group else False)
                 ),
                 "batch_name": form.cleaned_data["batch_name"] or batch_name_default(),
+                "validate_mode": form.cleaned_data["validate_mode"],
                 "first_line": form.cleaned_data["first_line"],
-                "check_before": (check_before := form.cleaned_data.get("check_before", False)),
-                "fail_if_alien": form.cleaned_data.get("fail_if_alien", False) if check_before else False,
                 **(
                     {
                         "household_pk_col": form.cleaned_data.get("pk_column_name"),
@@ -376,13 +375,12 @@ def import_aurora(self, request: HttpRequest, program: "CountryProgram") -> "Imp
         if form.is_valid():
             config: AuroraConfig = {
                 "batch_name": form.cleaned_data["batch_name"] or batch_name_default(),
+                "validate_mode": form.cleaned_data["validate_mode"],
                 "registration_reference_pk": getattr(form.cleaned_data.get("registration"), "reference_pk", None),
                 "individuals_column_prefix": form.cleaned_data["individuals_column_prefix"],
                 "master_detail": (
                     master_detail := (program.beneficiary_group.master_detail if program.beneficiary_group else False)
                 ),
-                "check_before": (check_before := form.cleaned_data.get("check_before", False)),
-                "fail_if_alien": form.cleaned_data.get("fail_if_alien", False) if check_before else False,
                 **(
                     {
                         "household_column_prefix": form.cleaned_data.get("household_column_prefix"),
@@ -411,10 +409,9 @@ def import_kobo(self, request: HttpRequest, program: "CountryProgram") -> Import
         if form.is_valid():
             config: KoboConfig = {
                 "batch_name": form.cleaned_data["batch_name"] or batch_name_default(),
+                "validate_mode": form.cleaned_data["validate_mode"],
                 "project_id": form.cleaned_data["project_id"],
                 "individual_records_field": form.cleaned_data["individual_records_field"],
-                "check_before": (check_before := form.cleaned_data.get("check_before", False)),
-                "fail_if_alien": form.cleaned_data.get("fail_if_alien", False) if check_before else False,
             }
             job: AsyncJob = AsyncJob.objects.create(
                 description=KOBO_IMPORT_JOB_DESCRIPTION.format(program_name=program.name),