286744: Add concatenate field and generate full name actions (#275)

* 286744: Add concatenate field and generate full name actions

- Introduced two new admin actions: `concatenate_field` and `generate_full_name` for the BeneficiaryBaseAdmin.
- Implemented a form for concatenating fields based on a specified pattern.
- Added a new template for the concatenate action to preview and apply changes.
- Created a utility module for handling concatenation logic and updating records.

* Enhance admin actions with new tests and implementations

- Added unit tests for `concatenate_field`, `generate_full_name`, and related actions in the admin workspace.
- Introduced mock fixtures to simulate request and queryset behaviors for testing.
- Implemented logic for handling concatenation and full name generation, ensuring proper job scheduling and context management.
- Improved test coverage for various scenarios, including form validation and checksum updates.

* Add comprehensive tests for admin actions in `test_actions.py`

- Implemented unit tests for various admin actions including `concatenate_field`, `generate_full_name`, `regex_update`, and `push_to_hope`.
- Enhanced test coverage for scenarios involving empty querysets, invalid forms, and job scheduling.
- Utilized mock objects to simulate request and queryset behaviors, ensuring robust validation and rendering logic.
- Verified context management and response handling for different action outcomes.

* Refactor queryset ordering in create_validation_jobs function to improve readability

* Remove generate_full_name action from BeneficiaryBaseAdmin and associated tests to streamline admin actions.

* Remove multiple test cases from `test_actions.py` to streamline the test suite. The removed tests include those for mass update, regex update, bulk update export, checksum calculation, and push to hope actions. This cleanup aims to enhance maintainability and focus on essential test coverage.

* Add tests for handling None flex_fields and save=False in concatenate_field_impl

- Implemented tests to verify that concatenate_field_impl correctly processes records with None flex_fields, ensuring proper initialization and updates.
- Added a test case to confirm that no updates occur when save=False is specified, maintaining the integrity of the record's flex_fields.

* lint

---------

Signed-off-by: Domenico <ddinicola@unicef.org>
Co-authored-by: Arsen P <arsen.pidhoretskyi@valor.com>
Co-authored-by: Domenico <ddinicola@unicef.org>
Co-authored-by: Domenico DiNicola <dom.dinicola@gmail.com>

Author: 3 people

src/country_workspace/workspaces/admin/cleaners/actions.py

@@ -14,6 +14,7 @@
 from country_workspace.workspaces.admin.forms import BulkUpdateExportForm
 from .bulk_update import export_bulk_update_template
 from .calculate_checksum import calculate_checksum_impl
+from .concatenate import ConcatenateFieldForm, concatenate_field_impl
 from .mass_update import MassUpdateForm, mass_update_impl
 from .name_parser import NameParserForm, name_parser_impl
 from .regex import RegexUpdateForm, regex_update_impl
@@ -281,3 +282,53 @@ def push_to_hope(
         )
     ctx = model_admin.get_common_context(request, title=push_to_hope.short_description, form=form)
     return render(request, "workspace/actions/push_to_hope.html", ctx)
+
+
+@admin.action(description="Concatenate field action", permissions=["mass_update"])
+def concatenate_field(
+    model_admin: "BeneficiaryBaseAdmin",
+    request: "HttpRequest",
+    queryset: "QuerySet[Beneficiary]",
+) -> HttpResponse:
+    if model_admin._check_empty_queryset(request, queryset):
+        return redirect(".")
+    ctx = model_admin.get_common_context(request, title=_(concatenate_field.short_description))
+    ctx["checker"] = checker = model_admin.get_checker(request)
+    ctx["queryset"] = queryset
+    ctx["opts"] = model_admin.model._meta
+    ctx["preserved_filters"] = model_admin.get_preserved_filters(request)
+    if "_preview" in request.POST:
+        form = ConcatenateFieldForm(request.POST, checker=checker)
+        if form.is_valid():
+            changes = concatenate_field_impl(queryset.all()[:10], form.cleaned_data, save=False)
+            ctx["changes"] = changes
+    elif "_apply" in request.POST:
+        form = ConcatenateFieldForm(request.POST, checker=checker)
+        if form.is_valid():
+            opts = queryset.model._meta
+            job = AsyncJob.objects.create(
+                description=concatenate_field.short_description,
+                type=AsyncJob.JobType.ACTION,
+                owner=state.request.user,
+                action=fqn(concatenate_field_impl),
+                program=state.program,
+                config={
+                    "pks": list(queryset.values_list("pk", flat=True)),
+                    "model_name": opts.label,
+                    "kwargs": {"config": form.cleaned_data},
+                },
+            )
+            job.queue()
+            model_admin.message_user(request, "Task scheduled", messages.SUCCESS)
+    else:
+        form = ConcatenateFieldForm(
+            checker=checker,
+            initial={
+                "action": request.POST["action"],
+                "select_across": request.POST["select_across"],
+                "_selected_action": request.POST.getlist("_selected_action"),
+            },
+        )
+
+    ctx["form"] = form
+    return render(request, "workspace/actions/concatenate.html", ctx)

src/country_workspace/workspaces/admin/cleaners/concatenate.py

@@ -0,0 +1,131 @@
+import re
+from collections.abc import Iterable
+from typing import TYPE_CHECKING, Any, NamedTuple
+
+from django import forms
+from django.db import transaction
+
+from country_workspace.utils.flex_fields import get_checker_fields
+
+from .base import BaseActionForm
+
+if TYPE_CHECKING:
+    from django.db.models import QuerySet
+    from hope_flex_fields.models import DataChecker
+
+    from country_workspace.types import Beneficiary
+
+
+class ConcatenateFieldForm(BaseActionForm):
+    destination_field = forms.ChoiceField(choices=[], help_text="Field to update with concatenated value")
+    pattern = forms.CharField(
+        help_text='Pattern with field placeholders, e.g., "{first_name} {middle_name} {last_name}"'
+    )
+    replace_only_empty = forms.BooleanField(
+        required=False,
+        initial=False,
+        help_text="Only update the destination field if it is currently empty",
+    )
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        checker: "DataChecker" = kwargs.pop("checker")
+        super().__init__(*args, **kwargs)
+        choices = list(get_checker_fields(checker, with_fs_prefix=True))
+        self.fields["destination_field"].choices = choices
+
+    def clean_pattern(self) -> str:
+        pattern = self.cleaned_data.get("pattern", "")
+        # Validate that pattern contains at least one field placeholder
+        if not re.search(r"\{[^}]+\}", pattern):
+            raise forms.ValidationError("Pattern must contain at least one field placeholder like {field_name}")
+        return pattern
+
+
+class ConcatenateResult(NamedTuple):
+    record_id: str
+    original: Any
+    updated: str
+
+
+def normalize_whitespace(value: str) -> str:
+    """Trim value and remove multiple spaces."""
+    if not isinstance(value, str):
+        return value
+    # Replace multiple spaces with single space and strip
+    return re.sub(r"\s+", " ", value.strip())
+
+
+def extract_field_names(pattern: str) -> list[str]:
+    """Extract field names from pattern like {field_name}."""
+    return re.findall(r"\{([^}]+)\}", pattern)
+
+
+def concatenate_value(record: "Beneficiary", pattern: str) -> str:
+    """Concatenate fields based on pattern."""
+    field_names = extract_field_names(pattern)
+    flex_fields = record.flex_fields or {}
+
+    # Build the result by replacing placeholders
+    result = pattern
+    for field_name in field_names:
+        # Get value from flex_fields, default to empty string if not found
+        value = flex_fields.get(field_name, "")
+        if value is None:
+            value = ""
+        else:
+            value = str(value)
+        # Replace placeholder with actual value
+        result = result.replace(f"{{{field_name}}}", value)
+
+    return normalize_whitespace(result)
+
+
+def update_checksum(records: "QuerySet[Beneficiary]", initial_fields: set[str]) -> Iterable[str]:
+    """Update checksum for all records in the queryset."""
+    fields = initial_fields.copy()
+    for record in records:
+        if update_fields := record.update_checksum(initial_fields):
+            fields.update(update_fields)
+    return fields
+
+
+def concatenate_field_impl(
+    records: "QuerySet[Beneficiary]",
+    config: dict[str, Any],
+    save: bool = True,
+) -> list[ConcatenateResult]:
+    destination_field = config["destination_field"]
+    pattern = config["pattern"]
+    replace_only_empty = config.get("replace_only_empty", False)
+
+    ret: list[ConcatenateResult] = []
+    records_to_save: list["Beneficiary"] = []
+    with transaction.atomic():
+        for record in records:
+            flex_fields = record.flex_fields or {}
+            original_value = flex_fields.get(destination_field)
+
+            # Check if we should skip (only replace empty values)
+            if replace_only_empty and original_value:
+                ret.append(ConcatenateResult(record.id, original_value, str(original_value)))
+                continue
+
+            # Concatenate the value
+            concatenated_value = concatenate_value(record, pattern)
+
+            # Update the flex_fields
+            if flex_fields is None:
+                flex_fields = {}
+            flex_fields[destination_field] = concatenated_value
+            record.flex_fields = flex_fields
+
+            ret.append(ConcatenateResult(record.id, original_value, concatenated_value))
+
+            if save:
+                records_to_save.append(record)
+
+        if save and records_to_save:
+            fields = update_checksum(records_to_save, {"flex_fields"})
+            records.model.objects.bulk_update(records_to_save, list(fields), batch_size=1000)
+
+    return ret

src/country_workspace/workspaces/admin/cleaners/validate.py

@@ -66,7 +66,7 @@ def _validate_and_count(objs: Iterable[Model]) -> tuple[int, int]:
 
 def create_validation_jobs(description: str, owner: str, program: Program, queryset: QuerySet) -> AsyncJob:
     opts = queryset.model._meta
-    queryset = queryset.values_list("pk", flat=True).order_by("pk")
+    queryset = queryset.order_by("pk").values_list("pk", flat=True)
     for chunk in batched(queryset, config.CHUNK_SIZE_FOR_VALIDATION_TASK):
         job = AsyncJob.objects.create(
             description=f"{description} (PKs {chunk[0]} - {chunk[-1]})",

src/country_workspace/workspaces/admin/hh_ind.py

@@ -57,6 +57,7 @@ class BeneficiaryBaseAdmin(AdminAutoCompleteSearchMixin, SelectedProgramMixin, W
     actions = [
         actions.bulk_update_export,
         actions.calculate_checksum,
+        actions.concatenate_field,
         actions.mass_update,
         actions.regex_update,
         actions.validate_records,

src/country_workspace/workspaces/templates/workspace/actions/concatenate.html

@@ -0,0 +1,44 @@
+{% extends "workspace/actions/base.html" %}{% load i18n %}
+{% block action-content %}
+    <div class="min-w-full py-5">
+        All selected {{ opts.verbose_name_plural }} will be updated by concatenating fields based on the pattern.<br/>
+        Use field placeholders like <code>{first_name}</code>, <code>{middle_name}</code>, <code>{last_name}</code> in the pattern.
+    </div>
+    <div class="m-2 bg-white">
+        <form method="post" id="concatenate-field-form">
+            {% csrf_token %}
+            {{ form.non_fields_errors }}
+            <table class="min-w-full">
+                {{ form.as_table }}
+            </table>
+            <div></div>
+            <div class="submit-row">
+                <input type="submit" value="{% translate 'Preview' %}" class="default" name="_preview">
+            </div>
+            {% if changes %}
+                <h3>Preview (showing first 10 records):</h3>
+                <table class="min-w-full">
+                    <thead>
+                        <tr>
+                            <th>Record ID</th>
+                            <th>Original Value</th>
+                            <th>New Value</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+                        {% for result in changes %}
+                            <tr>
+                                <td>{{ result.record_id }}</td>
+                                <td>{{ result.original|default:"(empty)" }}</td>
+                                <td>{{ result.updated }}</td>
+                            </tr>
+                        {% endfor %}
+                    </tbody>
+                </table>
+                <div class="submit-row">
+                    <input type="submit" value="{% translate 'Apply' %}" class="default" name="_apply">
+                </div>
+            {% endif %}
+        </form>
+    </div>
+{% endblock action-content %}