chg ! tests

Author: vitali-yanushchyk-valor

src/country_workspace/contrib/aurora/pipeline.py

@@ -4,10 +4,11 @@
 
 from country_workspace.contrib.aurora.client import AuroraClient
 from country_workspace.contrib.aurora.exceptions import TooManyBeneficiaryError
-from country_workspace.models import AsyncJob, Batch, Household, Individual, Office
+from country_workspace.models import AsyncJob, Batch, Household, Individual
 from country_workspace.models.household import RELATIONSHIP_HEAD, RELATIONSHIP_FIELDNAME
 from country_workspace.utils.config import BatchNameConfig, ValidateModeConfig
 from country_workspace.utils.fields import clean_field_names
+from country_workspace.utils.types import BeneficiaryMapping
 from country_workspace.validators.beneficiaries import validate_beneficiaries
 
 
@@ -61,18 +62,18 @@ def import_from_aurora(job: AsyncJob) -> dict[str, int]:
                 total["households"] += 1
             records_data.append((record_id, individuals))
 
-        validate_records(records_data, cfg, job.program.country_office)
+        if mapping := validate_records(records_data, cfg):
+            validate_beneficiaries(mapping, cfg, job.program.country_office)
 
     return total
 
 
-def validate_records(records_data: list[tuple[int, list[Individual]]], cfg: Config, office: Office) -> None:
+def validate_records(records_data: list[tuple[int, list[Individual]]], cfg: Config) -> BeneficiaryMapping:
     """Validate beneficiaries based on configuration and record data.
 
     Args:
         records_data: List of tuples containing record ID and created individuals.
         cfg: Configuration for validation and mapping.
-        office: The office context for validation.
 
     Raises:
         TooManyBeneficiaryError: If more than one Individual is created when master_detail is False.
@@ -88,9 +89,7 @@ def validate_records(records_data: list[tuple[int, list[Individual]]], cfg: Conf
                 raise TooManyBeneficiaryError("Individual", record_id=record_id, count=len(individuals))
             if individuals:
                 mapping[record_id] = individuals[0]
-
-    if mapping:
-        validate_beneficiaries(cfg, mapping, office)
+    return mapping
 
 
 def create_household(batch: Batch, data: dict[str, Any], prefix: str) -> Household:

src/country_workspace/datasources/rdi.py

@@ -230,17 +230,15 @@ def import_from_rdi(job: AsyncJob) -> dict[str, int]:
 def _import_master_detail(
     job: AsyncJob, batch: Batch, config: Config, validate: ValidateBeneficiaries
 ) -> dict[str, int]:
-    household_sheet, individual_sheet = read_sheets(
-        config, job.file, SheetName.HOUSEHOLDS.value, SheetName.INDIVIDUALS.value
-    )
+    household_sheet, individual_sheet = read_sheets(config, job.file, SheetName.HOUSEHOLDS, SheetName.INDIVIDUALS)
     household_mapping = process_households(household_sheet, job, batch, config)
     individuals_mapping = process_beneficiaries(individual_sheet, job, batch, config, household_mapping)
     validate(household_mapping)
     return {"household": len(household_mapping), "individual": len(individuals_mapping)}
 
 
 def _import_people_only(job: AsyncJob, batch: Batch, config: Config, validate: ValidateBeneficiaries) -> dict[str, int]:
-    (people_sheet,) = read_sheets(config, job.file, SheetName.PEOPLE.value)
+    (people_sheet,) = read_sheets(config, job.file, SheetName.PEOPLE)
     people_mapping = process_beneficiaries(people_sheet, job, batch, config)
     validate(people_mapping)
     return {"people": len(people_mapping)}

src/country_workspace/validators/beneficiaries.py

@@ -7,13 +7,13 @@
 from country_workspace.workspaces.admin.forms import ValidateMode
 
 
-def validate_beneficiaries(config: ValidateModeConfig, beneficiary_mapping: BeneficiaryMapping, office: Office) -> None:
+def validate_beneficiaries(beneficiary_mapping: BeneficiaryMapping, config: ValidateModeConfig, office: Office) -> None:
     mode = ValidateMode(config["validate_mode"])
     if mode is ValidateMode.NONE:
         return
 
     fail_if_alien = mode is ValidateMode.CHECK_AND_FAIL_IF_ALIEN
-    with state.set(tenant=office, program=office.program):
+    with state.set(tenant=office):
         for key, beneficiary in beneficiary_mapping.items():
             if not beneficiary.validate_with_checker(fail_if_alien=fail_if_alien):
                 raise BeneficiaryValidationError(beneficiary._meta.object_name, key)

src/country_workspace/workspaces/admin/forms.py

@@ -37,7 +37,7 @@ class BulkUpdateImportForm(forms.Form):
 
 
 class ValidateMode(TextChoices):
-    NONE = "none", _("Skip validation — import data as is")
+    NONE = "none", _("Skip validation — import data as is.")
     CHECK_BEFORE = "check_before", _("Prevent import if data is not valid against data checker.")
     CHECK_AND_FAIL_IF_ALIEN = (
         "check_and_fail_if_alien",

tests/datasources/test_rdi.py

@@ -31,7 +31,6 @@
 from country_workspace.datasources.utils import datetime_to_date, date_to_iso_string
 from country_workspace.models import Household, Individual
 from country_workspace.workspaces.exceptions import BeneficiaryValidationError
-from country_workspace.validators.beneficiaries import validate_beneficiaries
 
 
 HOUSEHOLD_1_PK = 1
@@ -45,13 +44,12 @@
 def config(request) -> Config:
     return {
         "batch_name": "batch_name",
+        "validate_mode": "none",
         "master_detail": request.param,
         "household_pk_col": "household_pk",
         "master_column_label": "master_column",
         "detail_column_label": "detail_column",
         "people_column_prefix": "pp_",
-        "check_before": False,
-        "fail_if_alien": False,
     }
 
 
@@ -110,6 +108,11 @@ def household_mapping() -> Mapping[int, Mock]:
     }
 
 
+@pytest.fixture
+def people_mapping(people_sheet: Sheet) -> Mapping[int, Mock]:
+    return {i: Mock() for i in range(len(list(people_sheet)))}
+
+
 def test_column_configuration_error_format() -> None:
     error = ColumnConfigurationError(column_name := "test_column")
     assert column_name in str(error)
@@ -317,60 +320,34 @@ def test_process_beneficiaries_failed_to_create(
     assert exc_info.value.sheet_name == expected_sheet_name
 
 
-def test_validate_beneficiaries(config: Config, household_mapping: Mapping[int, Mock]) -> None:
-    config["check_before"] = True
-
-    validate_beneficiaries(config, household_mapping)
-
-    for household in household_mapping.values():
-        household.validate_with_checker.assert_called_once()
-
-
-def test_validate_beneficiaries_raises_exception_on_failed_validation(
-    config: Config, household_mapping: Mapping[int, Mock]
-) -> None:
-    config["check_before"] = True
-    household_mapping[HOUSEHOLD_1_PK].validate_with_checker.return_value = False
-
-    with pytest.raises(BeneficiaryValidationError):
-        validate_beneficiaries(config, household_mapping)
-
-
-def test_validate_beneficiaries_check_before_is_false(config: Config, household_mapping: Mapping[int, Mock]) -> None:
-    config["check_before"] = False
-
-    validate_beneficiaries(config, household_mapping)
-
-    for household in household_mapping.values():
-        household.validate_with_checker.assert_not_called()
-
-
 def test_import_from_rdi(
     mocker: MockerFixture,
     config: Config,
     household_sheet: Sheet,
     individual_sheet: Sheet,
     people_sheet: Sheet,
     household_mapping: Mapping[int, Mock],
+    people_mapping: Mapping[int, Mock],
 ) -> None:
     job = Mock()
     job.config = config
     batch_class_mock = mocker.patch("country_workspace.datasources.rdi.Batch")
     read_sheets_mock = mocker.patch("country_workspace.datasources.rdi.read_sheets")
     process_beneficiaries_mock = mocker.patch("country_workspace.datasources.rdi.process_beneficiaries")
     validate_beneficiaries_mock = mocker.patch("country_workspace.datasources.rdi.validate_beneficiaries")
+    partial_mock = mocker.patch("country_workspace.datasources.rdi.partial")
     if config["master_detail"]:
         read_sheets_mock.return_value = household_sheet, individual_sheet
         process_households_mock = mocker.patch("country_workspace.datasources.rdi.process_households")
         process_households_mock.return_value = household_mapping
         process_beneficiaries_mock.return_value = (processed_individuals := list(individual_sheet))
     else:
         read_sheets_mock.return_value = (people_sheet,)
-        people_mapping = {i: Mock() for i in range(len(list(people_sheet)))}
         process_beneficiaries_mock.return_value = people_mapping
 
     result = import_from_rdi(job)
 
+    partial_mock.assert_called_once_with(validate_beneficiaries_mock, config=config, office=job.program.country_office)
     if config["master_detail"]:
         assert result == {"household": len(household_mapping), "individual": len(processed_individuals)}
         process_households_mock.assert_called_once_with(
@@ -383,7 +360,7 @@ def test_import_from_rdi(
             config,
             household_mapping,
         )
-        validate_beneficiaries_mock.assert_called_once_with(config, household_mapping)
+        partial_mock.return_value.assert_called_once_with(household_mapping)
     else:
         assert result == {"people": len(people_mapping)}
         process_beneficiaries_mock.assert_called_once_with(
@@ -392,7 +369,7 @@ def test_import_from_rdi(
             batch_class_mock.objects.create.return_value,
             config,
         )
-        validate_beneficiaries_mock.assert_called_once_with(config, people_mapping)
+        partial_mock.return_value.assert_called_once_with(people_mapping)
 
     batch_class_mock.objects.create.assert_called_once_with(
         name=config["batch_name"],

tests/functional/test_f_import_data.py

@@ -66,10 +66,9 @@ def test_rdi_import_tab(browser_program):
     browser_program.assert_element_present('button[data-input-value="rdi"].selected')
 
     browser_program.assert_element_visible("#id_rdi-batch_name")
+    browser_program.assert_element_visible("#id_rdi-validate_mode")
     browser_program.assert_element_visible("#id_rdi-first_line")
     browser_program.assert_element_visible("#id_rdi-file")
-    browser_program.assert_element_visible("#id_rdi-check_before")
-    browser_program.assert_element_visible("#id_rdi-fail_if_alien")
 
 
 @pytest.mark.selenium
@@ -80,10 +79,9 @@ def test_rdi_import_tab_with_beneficiary(browser_program_beneficiary, program_be
     browser_program_beneficiary.assert_element_present('button[data-input-value="rdi"].selected')
 
     browser_program_beneficiary.assert_element_visible("#id_rdi-batch_name")
+    browser_program_beneficiary.assert_element_visible("#id_rdi-validate_mode")
     browser_program_beneficiary.assert_element_visible("#id_rdi-first_line")
     browser_program_beneficiary.assert_element_visible("#id_rdi-file")
-    browser_program_beneficiary.assert_element_visible("#id_rdi-check_before")
-    browser_program_beneficiary.assert_element_visible("#id_rdi-fail_if_alien")
 
     if program_beneficiary.beneficiary_group.master_detail:
         browser_program_beneficiary.assert_element_visible("#id_rdi-pk_column_name")
@@ -107,10 +105,9 @@ def test_aurora_import_tab(browser_program):
 
     aurora_input_ids = [
         "#id_aurora-batch_name",
+        "#id_aurora-validate_mode",
         "#id_aurora-registration",
         "#id_aurora-individuals_column_prefix",
-        "#id_aurora-check_before",
-        "#id_aurora-fail_if_alien",
     ]
 
     for input_id in aurora_input_ids:
@@ -129,10 +126,9 @@ def test_aurora_import_tab_with_beneficiary(browser_program_beneficiary, program
 
     common_input_ids = [
         "#id_aurora-batch_name",
+        "#id_aurora-validate_mode",
         "#id_aurora-registration",
         "#id_aurora-individuals_column_prefix",
-        "#id_aurora-check_before",
-        "#id_aurora-fail_if_alien",
     ]
 
     for input_id in common_input_ids:
@@ -159,7 +155,6 @@ def test_kobo_import_tab(browser_program):
     browser_program.assert_element_present('button[data-input-value="kobo"].selected')
 
     browser_program.assert_element_visible("#id_kobo-batch_name")
+    browser_program.assert_element_visible("#id_kobo-validate_mode")
     browser_program.assert_element_visible("#id_kobo-project_id")
     browser_program.assert_element_visible("#id_kobo-individual_records_field")
-    browser_program.assert_element_visible("#id_kobo-check_before")
-    browser_program.assert_element_visible("#id_kobo-fail_if_alien")

tests/functional/test_f_rdi_import.py

@@ -4,6 +4,7 @@
 from django.core.management import call_command
 
 from country_workspace.models import Household, Individual
+from country_workspace.workspaces.admin.forms import ValidateMode
 from testutils.factories import OfficeFactory
 from testutils.factories.program import BeneficiaryGroupFactory
 from testutils.factories import CountryProgramFactory
@@ -49,6 +50,7 @@ def test_rdi_import_household(browser, program):
     browser.click_link("Programme")
     browser.click("#btn-import_data")
     browser.fill('input[name="rdi-batch_name"]', "Test Batch")
+    browser.select_option_by_value('select[name="rdi-validate_mode"]', ValidateMode.NONE.value)
     test_file_path = Path(__file__).parent.parent / "data/rdi_one.xlsx"
     browser.choose_file('input[type="file"]', str(test_file_path))
     browser.click('input[type="submit"][value="Import"]')

tests/validators/test_validate_beneficiaries.py

@@ -0,0 +1,59 @@
+import pytest
+from unittest.mock import Mock
+
+from country_workspace.validators.beneficiaries import validate_beneficiaries
+from country_workspace.workspaces.admin.forms import ValidateMode
+from country_workspace.utils.types import BeneficiaryMapping
+from country_workspace.utils.config import ValidateModeConfig
+from country_workspace.models.office import Office
+from country_workspace.workspaces.exceptions import BeneficiaryValidationError
+
+
+@pytest.fixture
+def mapping_and_office() -> tuple[BeneficiaryMapping, Office]:
+    from testutils.factories import CountryHouseholdFactory, ProgramFactory
+
+    program = ProgramFactory()
+    mapping: BeneficiaryMapping = {}
+    for i in range(1, 3):
+        beneficiary = CountryHouseholdFactory(batch__program=program, batch__country_office=program.country_office)
+        beneficiary.validate_with_checker = Mock(return_value=True)
+        mapping[i] = beneficiary
+    return mapping, program.country_office
+
+
+@pytest.fixture(params=ValidateMode.__members__.values())
+def config(request) -> ValidateModeConfig:
+    return {
+        "validate_mode": request.param,
+    }
+
+
+@pytest.fixture(params=[mode for mode in ValidateMode if mode != ValidateMode.NONE])
+def failing_config(request) -> ValidateModeConfig:
+    return {
+        "validate_mode": request.param,
+    }
+
+
+def test_validate_beneficiaries(
+    config: ValidateModeConfig, mapping_and_office: tuple[BeneficiaryMapping, Office]
+) -> None:
+    beneficiary_mapping, office = mapping_and_office
+    validate_beneficiaries(beneficiary_mapping, config, office)
+    for beneficiary in beneficiary_mapping.values():
+        if config["validate_mode"] == ValidateMode.NONE:
+            beneficiary.validate_with_checker.assert_not_called()
+        else:
+            fail_if_alien = config["validate_mode"] == ValidateMode.CHECK_AND_FAIL_IF_ALIEN
+            beneficiary.validate_with_checker.assert_called_once_with(fail_if_alien=fail_if_alien)
+
+
+def test_validate_beneficiaries_raises_exception_on_failed_validation(
+    failing_config: ValidateModeConfig, mapping_and_office: tuple[BeneficiaryMapping, Office]
+) -> None:
+    beneficiary_mapping, office = mapping_and_office
+    beneficiary_mapping[1].validate_with_checker.return_value = False
+
+    with pytest.raises(BeneficiaryValidationError):
+        validate_beneficiaries(beneficiary_mapping, failing_config, office)

tests/workspace/admin/test_program.py

@@ -11,6 +11,7 @@
 from country_workspace.workspaces.admin import CountryProgramAdmin
 from country_workspace.workspaces.admin.program import KOBO_IMPORT_JOB_DESCRIPTION
 from country_workspace.workspaces.models import CountryProgram
+from country_workspace.workspaces.admin.forms import ValidateMode
 
 
 def test__country_program_admin__import_kobo__job_description(mocker: MockerFixture) -> None:
@@ -106,6 +107,7 @@ def test_import_kobo_valid_form(program_admin, mock_request, mock_program):
 
         mock_request.POST = {
             "kobo-batch_name": "Test Import Batch",
+            "kobo-validate_mode": ValidateMode.NONE.value,
             "kobo-project_id": "test_project_123",
             "kobo-individual_records_field": "individual_questions",
             "_selected_tab": "kobo",

tests/workspace/test_ws_import.py

@@ -11,6 +11,7 @@
 
 from country_workspace.models import Office, Individual, Household
 from country_workspace.state import state
+from country_workspace.workspaces.admin.forms import ValidateMode
 from country_workspace.contrib.aurora.exceptions import TooManyBeneficiaryError
 from tests.contrib.aurora import stub
 
@@ -89,6 +90,7 @@ def form_import_rdi(app: "DjangoTestApp", program: "CountryProgram") -> forms.Fo
     data = (Path(__file__).parent.parent / "data/rdi_one.xlsx").read_bytes()
     res = app.get(url)
 
+    res.forms["import-file"]["rdi-validate_mode"] = ValidateMode.NONE.value
     res.forms["import-file"]["_selected_tab"] = "rdi"
     res.forms["import-file"]["rdi-file"] = Upload("rdi_one.xlsx", data)
 
@@ -161,6 +163,7 @@ def form_aurora(
 
     res = app.get(url)
     res.forms["import-aurora"]["_selected_tab"] = "aurora"
+    res.forms["import-aurora"]["aurora-validate_mode"] = ValidateMode.NONE.value
     res.forms["import-aurora"]["aurora-registration"] = program.projects.registrations.first().pk
 
     return res.forms["import-aurora"]