Feature/change export logic (#272)

* Add two new columns is_valid and errors to export excel file

* Alter 'export_bulk_update_template' function

Add annotated column is_valid

* Minor logic improvements

Move qs generation to a dedicated function

* Add tests

Add a new test case 'test_create_bulk_update_template_with_errors'

* Add a new field "include_errors" to "BulkUpdateExportForm"

* Add columns "is_valid" and "errors" conditionally on include_errors flag

* Alter test cases

---------

Co-authored-by: patsatsia <giorgi.patsatsia@valor-software.com>
Co-authored-by: Domenico <ddinicola@unicef.org>

Author: 3 people

src/country_workspace/workspaces/admin/cleaners/actions.py

@@ -12,7 +12,6 @@
 from country_workspace.state import state
 from country_workspace.utils.fields import rdi_name_default
 from country_workspace.workspaces.admin.forms import BulkUpdateExportForm
-
 from .bulk_update import export_bulk_update_template
 from .calculate_checksum import calculate_checksum_impl
 from .mass_update import MassUpdateForm, mass_update_impl
@@ -143,6 +142,8 @@ def bulk_update_export(
     ctx["form"] = form
     if "_export" in request.POST and form.is_valid():
         columns = ["id", "version"] + form.cleaned_data["fields"]
+        if form.cleaned_data.get("include_errors", False):
+            columns += ["is_valid", "errors"]
         opts = queryset.model._meta
         job = AsyncJob.objects.create(
             description=bulk_update_export.short_description,

src/country_workspace/workspaces/admin/cleaners/bulk_update.py

@@ -3,6 +3,7 @@
 from datetime import datetime
 from io import BytesIO
 from typing import TYPE_CHECKING, Any
+from django.db.models import Case, When, Value, Q, CharField
 
 from constance import config as constance_config
 from django import forms
@@ -20,13 +21,13 @@
 from xlsxwriter.format import Format
 from xlsxwriter.worksheet import Worksheet
 from country_workspace.models import AsyncJob, Program, Country
+from country_workspace.models.base import Validable
 from country_workspace.state import state
 from country_workspace.storages import MEDIA_STORAGE
 from country_workspace.workspaces.admin.cleaners.exceptions import BulkImportError, BulkImportFileProcessingError
+from django.db.models import QuerySet
 
 if TYPE_CHECKING:
-    from django.db.models import QuerySet
-
     from country_workspace.types import Beneficiary
 
 """
@@ -322,7 +323,7 @@ def _send_template_email(job: AsyncJob, out: BytesIO, filename: str) -> None:
 def export_bulk_update_template(job: AsyncJob) -> str:
     with state.set(tenant=job.program.country_office, program=job.program):
         model = apps.get_model(job.config["model_name"])
-        queryset = model.objects.filter(pk__in=job.config["pks"])
+        queryset = _get_queryset_with_is_valid_annotation(model, job)
 
         out = create_bulk_update_template(queryset, job.program, job.config["columns"])
         filename = f"bulk_update_template/{job.program.pk}/{job.owner.pk}/{job.config['model_name']}.xlsx"
@@ -336,6 +337,21 @@ def export_bulk_update_template(job: AsyncJob) -> str:
         return filepath
 
 
+def _get_queryset_with_is_valid_annotation(model: Validable, job: AsyncJob) -> QuerySet:
+    qs = model.objects.filter(pk__in=job.config["pks"])
+    if (columns := job.config.get("columns")) and ("is_valid" in columns and "errors" in columns):
+        qs = qs.annotate(
+            is_valid=Case(
+                When(last_checked__isnull=True, then=Value("Not Checked")),
+                When(Q(last_checked__isnull=False) & Q(errors={}), then=Value("True")),
+                default=Value("False"),
+                output_field=CharField(),
+            )
+        )
+
+    return qs
+
+
 def import_bulk_update_file(job: AsyncJob, entity_getter: Callable[[int], Any]) -> dict[str, Any]:  # noqa: C901
     total = {"processed": 0, "not_found": [], "errors": {}}
     version_check_enabled = constance_config.CONCURRENCY_GUARD

src/country_workspace/workspaces/admin/forms.py

@@ -14,6 +14,11 @@
 
 class BulkUpdateExportForm(BaseActionForm):
     fields = forms.MultipleChoiceField(choices=[], widget=forms.CheckboxSelectMultiple())
+    include_errors = forms.BooleanField(
+        label=_("Include errors and validity status"),
+        required=False,
+        initial=False,
+    )
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
         checker: "DataChecker" = kwargs.pop("checker")

tests/workspace/actions/stub.py

@@ -13,6 +13,7 @@
         "role",
     ],
 }
+header_last: Final[tuple[str, ...]] = ["is_valid", "errors"]
 
 batch_name: Final[str] = "TestBatch"
 batch_size: Final[int] = 10

tests/workspace/actions/test_ws_bulk.py

@@ -1,17 +1,25 @@
+import datetime
 import io
 from typing import TYPE_CHECKING, Any
 from unittest import mock
+from unittest.mock import Mock
 
 import openpyxl
 import pytest
 import xlsxwriter
 from django.urls import reverse
+from django.utils import timezone
 from testutils.factories import FlexFieldFactory
 from testutils.utils import select_office
 from webtest import Upload
 
 from country_workspace.state import state
-from country_workspace.workspaces.admin.cleaners.bulk_update import TYPES, create_bulk_update_template
+from country_workspace.workspaces.admin.cleaners.bulk_update import (
+    TYPES,
+    create_bulk_update_template,
+    _get_queryset_with_is_valid_annotation,
+)
+from tests.extras.testutils.factories import CountryHouseholdFactory
 from tests.workspace.actions import stub
 
 if TYPE_CHECKING:
@@ -138,6 +146,54 @@ def test_create_bulk_update_template(household: "CountryHousehold", force_migrat
     assert headers == selected_fields
 
 
+def test_create_bulk_update_template_with_errors(program):
+    CountryHouseholdFactory.create_batch(2, batch__program=program)
+    CountryHouseholdFactory.create_batch(
+        3, batch__program=program, last_checked=(timezone.now() - datetime.timedelta(days=3)), errors={}
+    )
+    CountryHouseholdFactory.create_batch(
+        4,
+        batch__program=program,
+        last_checked=(timezone.now() - datetime.timedelta(days=4)),
+        errors={"cool_error": "cooler error text"},
+    )
+    selected_fields = stub.header_base + stub.header_add["ind"] + stub.header_last
+
+    job = Mock()
+    job.config = {
+        "pks": CountryHouseholdFactory._meta.model.objects.values_list("id", flat=True),
+        "columns": selected_fields,
+    }
+    qs = _get_queryset_with_is_valid_annotation(CountryHouseholdFactory._meta.model, job)
+
+    ret = create_bulk_update_template(
+        qs,
+        program,
+        selected_fields,
+    )
+
+    workbook = openpyxl.load_workbook(io.BytesIO(ret.getvalue()))
+    sheet = workbook.worksheets[0]
+    headers = [cell.value for cell in next(sheet.iter_rows(min_row=1, max_row=1))]
+    assert headers == selected_fields
+
+    valid_count = 0
+    invalid_count = 0
+    not_checked_count = 0
+    for row in sheet.iter_rows(min_row=2):
+        is_valid = row[-2].value
+        if is_valid == "True":
+            valid_count += 1
+        elif is_valid == "False":
+            invalid_count += 1
+        elif is_valid == "Not Checked":
+            not_checked_count += 1
+
+    assert valid_count == 3
+    assert invalid_count == 4
+    assert not_checked_count == 2
+
+
 def test_bulk_update_export(
     app: "DjangoTestApp",
     force_migrated_records,