chg ! use validate with choices when import instead of checkboxes

Author: vitali-yanushchyk-valor

src/country_workspace/contrib/aurora/forms.py

@@ -4,9 +4,10 @@
 
 from country_workspace.contrib.aurora.models import Registration
 from country_workspace.models import Program
+from country_workspace.workspaces.admin.forms import BaseImportForm
 
 
-class ImportAuroraForm(forms.Form):
+class ImportAuroraForm(BaseImportForm):
     batch_name = forms.CharField(required=False, help_text="Label for this batch.")
     registration = forms.ModelChoiceField(
         queryset=Registration.objects.none(),
@@ -24,12 +25,6 @@ class ImportAuroraForm(forms.Form):
         initial="family_name",
         help_text="Which Individual's column should be used as label for the household.",
     )
-    check_before = forms.BooleanField(
-        required=False, help_text="Prevent import if errors if data is not valid against data checker."
-    )
-    fail_if_alien = forms.BooleanField(
-        required=False, help_text="Fails if it finds fields which do not exists in data checker."
-    )
 
     def __init__(self, *args: Any, program: Program | None = None, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)

src/country_workspace/contrib/aurora/pipeline.py

@@ -6,12 +6,12 @@
 from country_workspace.contrib.aurora.exceptions import TooManyBeneficiaryError
 from country_workspace.models import AsyncJob, Batch, Household, Individual
 from country_workspace.models.household import RELATIONSHIP_HEAD, RELATIONSHIP_FIELDNAME
-from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
+from country_workspace.utils.config import BatchNameConfig, ValidateModeConfig
 from country_workspace.utils.fields import clean_field_names
 from country_workspace.validators.beneficiaries import validate_beneficiaries
 
 
-class Config(BatchNameConfig, FailIfAlienConfig):
+class Config(BatchNameConfig, ValidateModeConfig):
     registration_reference_pk: str | None
     master_detail: bool
     household_column_prefix: NotRequired[str]

src/country_workspace/contrib/kobo/forms.py

@@ -3,22 +3,17 @@
 from django import forms
 
 from country_workspace.contrib.kobo.sync import make_client
+from country_workspace.workspaces.admin.forms import BaseImportForm
 
 
-class ImportKoboForm(forms.Form):
+class ImportKoboForm(BaseImportForm):
     batch_name = forms.CharField(required=False, help_text="Label for this batch")
     project_id = forms.ChoiceField(required=True, choices=(), help_text="Select a project")
     individual_records_field = forms.CharField(
         required=False,
         initial="individual_questions",
         help_text="Which field contains individual records",
     )
-    check_before = forms.BooleanField(
-        required=False, help_text="Prevent import if errors if data is not valid against data checker."
-    )
-    fail_if_alien = forms.BooleanField(
-        required=False, help_text="Fails if it finds fields which do not exists in data checker."
-    )
 
     def __init__(self, *args: Any, kobo_country_code: str | None, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)

src/country_workspace/contrib/kobo/sync.py

@@ -14,12 +14,12 @@
 from country_workspace.contrib.kobo.api.data.submission import Submission
 from country_workspace.contrib.kobo.models import KoboSubmission
 from country_workspace.models import AsyncJob, Batch, Household, Individual
-from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
+from country_workspace.utils.config import BatchNameConfig, ValidateModeConfig
 from country_workspace.utils.fields import clean_field_names, TO_UPPERCASE_FIELDS
 from country_workspace.utils.functional import compose
 
 
-class Config(BatchNameConfig, FailIfAlienConfig):
+class Config(BatchNameConfig, ValidateModeConfig):
     project_id: str
     individual_records_field: str
 

src/country_workspace/datasources/rdi.py

@@ -13,7 +13,7 @@
 
 from country_workspace.contrib.kobo.api.data.helpers import VALUE_FORMAT
 from country_workspace.models import AsyncJob, Batch, Household, Individual
-from country_workspace.utils.config import BatchNameConfig, FailIfAlienConfig
+from country_workspace.utils.config import BatchNameConfig, ValidateModeConfig
 from country_workspace.utils.fields import Record, clean_field_names
 from country_workspace.utils.functional import compose
 from country_workspace.validators.beneficiaries import validate_beneficiaries
@@ -30,7 +30,7 @@
 PEOPLE = "people"
 
 
-class Config(BatchNameConfig, FailIfAlienConfig):
+class Config(BatchNameConfig, ValidateModeConfig):
     master_detail: bool
     household_pk_col: NotRequired[str]
     master_column_label: NotRequired[str]

src/country_workspace/utils/config.py

@@ -1,13 +1,10 @@
 from typing import TypedDict
+from country_workspace.workspaces.admin.forms import ValidateMode
 
 
 class BatchNameConfig(TypedDict):
     batch_name: str
 
 
-class CheckBeforeConfig(TypedDict):
-    check_before: bool
-
-
-class FailIfAlienConfig(CheckBeforeConfig):
-    fail_if_alien: bool
+class ValidateModeConfig(TypedDict):
+    validate_mode: ValidateMode

src/country_workspace/validators/beneficiaries.py

@@ -2,11 +2,16 @@
 from country_workspace.workspaces.exceptions import BeneficiaryValidationError
 from country_workspace.utils.types import T_Beneficiary
 
-from country_workspace.utils.config import FailIfAlienConfig
+from country_workspace.utils.config import ValidateModeConfig
+from country_workspace.workspaces.admin.forms import ValidateMode
 
 
-def validate_beneficiaries(config: FailIfAlienConfig, beneficiary_mapping: Mapping[int, T_Beneficiary]) -> None:
-    if config.get("check_before", False):
-        for key, beneficiary in beneficiary_mapping.items():
-            if not beneficiary.validate_with_checker(fail_if_alien=config.get("fail_if_alien", False)):
-                raise BeneficiaryValidationError(beneficiary._meta.object_name, key)
+def validate_beneficiaries(config: ValidateModeConfig, beneficiary_mapping: Mapping[int, T_Beneficiary]) -> None:
+    mode = ValidateMode(config["validate_mode"])
+    if mode is ValidateMode.NONE:
+        return
+
+    fail_if_alien = mode is ValidateMode.CHECK_AND_FAIL_IF_ALIEN
+    for key, beneficiary in beneficiary_mapping.items():
+        if not beneficiary.validate_with_checker(fail_if_alien=fail_if_alien):
+            raise BeneficiaryValidationError(beneficiary._meta.object_name, key)

src/country_workspace/workspaces/admin/forms.py

@@ -1,6 +1,8 @@
 from typing import TYPE_CHECKING, Any
 
 from django import forms
+from django.db.models import TextChoices
+from django.utils.translation import gettext_lazy as _
 
 from country_workspace.workspaces.admin.cleaners.base import BaseActionForm
 from country_workspace.workspaces.validators import ValidatableFileValidator
@@ -34,41 +36,54 @@ class BulkUpdateImportForm(forms.Form):
     )
 
 
-class ImportFileForm(forms.Form):
+class ValidateMode(TextChoices):
+    NONE = "none", _("Skip validation — import data as is")
+    CHECK_BEFORE = "check_before", _("Prevent import if data is not valid against data checker.")
+    CHECK_AND_FAIL_IF_ALIEN = (
+        "check_and_fail_if_alien",
+        _("Prevent import if data is invalid AND fail if an alien field is found."),
+    )
+
+
+class BaseImportForm(forms.Form):
     batch_name = forms.CharField(required=False, help_text="Label for this batch")
+    validate_mode = forms.TypedChoiceField(
+        choices=ValidateMode.choices,
+        coerce=ValidateMode,
+        empty_value=ValidateMode.CHECK_AND_FAIL_IF_ALIEN,
+        initial=ValidateMode.CHECK_AND_FAIL_IF_ALIEN,
+        required=True,
+        help_text=_("How to validate data before import"),
+    )
+
 
+class ImportFileForm(BaseImportForm):
     pk_column_name = forms.CharField(
         required=True,
         initial="household_id",
-        help_text="Which column contains the unique identifier of the record.It is mandatory from Master/detail",
+        help_text=_("Which column contains the unique identifier of the record. It is mandatory from Master/detail"),
     )
 
     master_column_label = forms.CharField(
         required=False,
         initial="household_id",
-        help_text="Which column contains the 'link' to the household record.",
+        help_text=_("Which column contains the 'link' to the household record."),
     )
 
     detail_column_label = forms.CharField(
         required=False,
         initial="household_id",
-        help_text="Which column should be used as label for the household. It can use interpolation",
+        help_text=_("Which column should be used as label for the household. It can use interpolation"),
     )
 
     people_column_prefix = forms.CharField(
         required=False,
         initial="pp_",
-        help_text="People' column group prefix",
+        help_text=_("People' column group prefix"),
     )
 
     first_line = forms.IntegerField(required=True, initial=0, help_text="First line to process")
 
-    check_before = forms.BooleanField(
-        required=False, help_text="Prevent import if errors if data is not valid against data checker."
-    )
-    fail_if_alien = forms.BooleanField(
-        required=False, help_text="Fails if it finds fields which do not exists in data checker."
-    )
     file = forms.FileField(validators=[ValidatableFileValidator()])
 
     def __init__(self, *args: Any, beneficiary_group: BeneficiaryGroup | None = None, **kwargs: Any) -> None:

src/country_workspace/workspaces/admin/program.py

@@ -342,9 +342,8 @@ def import_rdi(self, request: HttpRequest, program: CountryProgram) -> "ImportFi
                     master_detail := (program.beneficiary_group.master_detail if program.beneficiary_group else False)
                 ),
                 "batch_name": form.cleaned_data["batch_name"] or batch_name_default(),
+                "validate_mode": form.cleaned_data["validate_mode"],
                 "first_line": form.cleaned_data["first_line"],
-                "check_before": (check_before := form.cleaned_data.get("check_before", False)),
-                "fail_if_alien": form.cleaned_data.get("fail_if_alien", False) if check_before else False,
                 **(
                     {
                         "household_pk_col": form.cleaned_data.get("pk_column_name"),
@@ -376,13 +375,12 @@ def import_aurora(self, request: HttpRequest, program: "CountryProgram") -> "Imp
         if form.is_valid():
             config: AuroraConfig = {
                 "batch_name": form.cleaned_data["batch_name"] or batch_name_default(),
+                "validate_mode": form.cleaned_data["validate_mode"],
                 "registration_reference_pk": getattr(form.cleaned_data.get("registration"), "reference_pk", None),
                 "individuals_column_prefix": form.cleaned_data["individuals_column_prefix"],
                 "master_detail": (
                     master_detail := (program.beneficiary_group.master_detail if program.beneficiary_group else False)
                 ),
-                "check_before": (check_before := form.cleaned_data.get("check_before", False)),
-                "fail_if_alien": form.cleaned_data.get("fail_if_alien", False) if check_before else False,
                 **(
                     {
                         "household_column_prefix": form.cleaned_data.get("household_column_prefix"),
@@ -411,10 +409,9 @@ def import_kobo(self, request: HttpRequest, program: "CountryProgram") -> Import
         if form.is_valid():
             config: KoboConfig = {
                 "batch_name": form.cleaned_data["batch_name"] or batch_name_default(),
+                "validate_mode": form.cleaned_data["validate_mode"],
                 "project_id": form.cleaned_data["project_id"],
                 "individual_records_field": form.cleaned_data["individual_records_field"],
-                "check_before": (check_before := form.cleaned_data.get("check_before", False)),
-                "fail_if_alien": form.cleaned_data.get("fail_if_alien", False) if check_before else False,
             }
             job: AsyncJob = AsyncJob.objects.create(
                 description=KOBO_IMPORT_JOB_DESCRIPTION.format(program_name=program.name),