Merge branch 'develop' into feature/routestream

Author: srugano

.github/workflows/sdlc-version-create.yml

@@ -92,7 +92,7 @@ jobs:
         uses: actions/checkout@v6
 
       - name: Run Trivy vulnerability scanner
-        uses: aquasecurity/trivy-action@0.33.1
+        uses: aquasecurity/trivy-action@0.34.1
         with:
           image-ref: ${{ vars.DOCKERHUB_ORGANIZATION }}/${{ vars.DOCKERHUB_REPOSITORY }}:${{ needs.prepare-version.outputs.version }}
           format: 'sarif'

.pre-commit-config.yaml

@@ -9,7 +9,7 @@ repos:
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.36.0
+    rev: 0.37.0
     hooks:
       - id: check-github-workflows
         args: [ "--verbose" ]
@@ -19,24 +19,28 @@ repos:
       - id: tox-ini-fmt
         args: [ "-p", "lint" ]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.14.10"
+    rev: "v0.15.4"
     hooks:
       - id: ruff-format
       - id: ruff
         args: [ "--fix", "--unsafe-fixes", "--exit-non-zero-on-fix" ]
   - repo: https://github.com/adamchainz/djade-pre-commit
-    rev: "1.7.0"
+    rev: "1.9.0"
     hooks:
       - id: djade
         args: [ --target-version, "5.1" ]  # Replace with Django version
-
+  - repo: https://github.com/adamchainz/django-upgrade
+    rev: "1.30.0"  # replace with latest tag on GitHub
+    hooks:
+      - id: django-upgrade
+        args: [--target-version, "5.2"]
   - repo: meta
     hooks:
       - id: check-hooks-apply
       - id: check-useless-excludes
 
   - repo: https://github.com/saxix/pch
-    rev: '79b0c0e'
+    rev: '0.5'
     hooks:
 
       - id: check-missed-migrations
@@ -63,8 +67,8 @@ repos:
         name: check tailwind CSS ready for production
         files: src/hope_live/theme/static_src/src/styles.scss
         args:
-          - -o
-          - src/hope_country_report/web/theme/static/css/styles.css
+          - -d
+          - src/hope_country_report/web/theme/static/css
 
       - id: check-minimized
         name: check missing minimized javascript

pyproject.toml

@@ -21,7 +21,7 @@ dynamic = [ "version" ]
 dependencies = [
   "azure-devops",
   "celery",
-  "django",
+  "django>=5.1,<5.3",
   "django-admin-cursor-paginator",
   "django-admin-extra-buttons",
   "django-adminactions",

pytest.ini

@@ -36,5 +36,5 @@ markers =
 python_files =test_*.py
 filterwarnings =
     ignore::DeprecationWarning
-    ignore::django.utils.deprecation.RemovedInDjango60Warning
+    ignore::django.utils.deprecation.RemovedInDjango61Warning
     ignore::coverage.exceptions.CoverageWarning

src/hope_country_report/apps/core/models.py

@@ -1,6 +1,5 @@
 import datetime
 from typing import TYPE_CHECKING, Any
-from zoneinfo import ZoneInfo
 
 from django.conf import settings
 from django.contrib.auth.models import Group
@@ -156,7 +155,6 @@ def get_absolute_url(self) -> str:
 
 
 class User(TimeStampedModel, SecurityMixin, AbstractUser):  # type: ignore
-    timezone: ZoneInfo
     timezone = TimeZoneField(verbose_name=_("Timezone"), default="UTC")
     language = models.CharField(verbose_name=_("Language"), max_length=10, choices=settings.LANGUAGES, default="en")
     date_format = models.CharField(

src/hope_country_report/apps/hope/models/_inspect.py

@@ -1298,8 +1298,8 @@ class Household(HopeModel):
     unknown_sex_group_count = models.IntegerField(blank=True, null=True)
     latitude = models.FloatField(blank=True, null=True)
     longitude = models.FloatField(blank=True, null=True)
-    collision_flag = models.BooleanField(null=True)
     identification_key = models.CharField(max_length=255, blank=True, null=True)
+    originating_id = models.CharField(unique=True, max_length=150, blank=True, null=True)
 
     class Meta:
         managed = False
@@ -1439,6 +1439,7 @@ class Individual(HopeModel):
     biometric_deduplication_golden_record_results = models.JSONField(null=True)
     biometric_deduplication_golden_record_status = models.CharField(max_length=50, null=True)
     identification_key = models.CharField(max_length=255, blank=True, null=True)
+    originating_id = models.CharField(unique=True, max_length=150, blank=True, null=True)
 
     class Meta:
         managed = False
@@ -1868,7 +1869,7 @@ class Payment(HopeModel):
     signature_hash = models.CharField(max_length=40, null=True)
     status = models.CharField(max_length=255, null=True)
     status_date = models.DateTimeField(null=True)
-    currency = models.CharField(max_length=4, blank=True, null=True)
+    currency = models.CharField(max_length=5, blank=True, null=True)
     entitlement_quantity = models.DecimalField(max_digits=15, decimal_places=2, blank=True, null=True)
     entitlement_quantity_usd = models.DecimalField(max_digits=15, decimal_places=2, blank=True, null=True)
     delivered_quantity = models.DecimalField(max_digits=15, decimal_places=2, blank=True, null=True)
@@ -1919,6 +1920,8 @@ class Payment(HopeModel):
     parent_split = models.ForeignKey(
         "Paymentplansplit", on_delete=models.DO_NOTHING, related_name="payment_parent_split", blank=True, null=True
     )
+    extras = models.JSONField(null=True)
+    sent_to_fsp_date = models.DateTimeField(blank=True, null=True)
 
     class Meta:
         managed = False
@@ -1967,7 +1970,7 @@ class PaymentPlan(HopeModel):
     total_undelivered_quantity_usd = models.DecimalField(max_digits=15, decimal_places=2, blank=True, null=True)
     status = models.CharField(max_length=50, null=True)
     background_action_status = models.CharField(max_length=50, blank=True, null=True)
-    currency = models.CharField(max_length=4, blank=True, null=True)
+    currency = models.CharField(max_length=5, blank=True, null=True)
     dispersion_start_date = models.DateField(blank=True, null=True)
     dispersion_end_date = models.DateField(blank=True, null=True)
     female_children_count = models.IntegerField(null=True)
@@ -2016,6 +2019,8 @@ class PaymentPlan(HopeModel):
     flag_exclude_if_active_adjudication_ticket = models.BooleanField(null=True)
     flag_exclude_if_on_sanction_list = models.BooleanField(null=True)
     abort_comment = models.CharField(max_length=255, null=True)
+    flat_amount_value = models.DecimalField(max_digits=15, decimal_places=2, blank=True, null=True)
+    custom_exchange_rate = models.BooleanField(null=True)
 
     class Meta:
         managed = False
@@ -2292,6 +2297,7 @@ class Program(HopeModel):
     slug = models.CharField(max_length=4, null=True)
     reconciliation_window_in_days = models.IntegerField(null=True)
     send_reconciliation_window_expiry_notifications = models.BooleanField(null=True)
+    identification_key_individual_label = models.CharField(max_length=255, blank=True, null=True)
 
     class Meta:
         managed = False
@@ -2513,34 +2519,6 @@ def __str__(self) -> str:
         return str(self.name)
 
 
-class DataRegistrationdataimportdatahub(HopeModel):
-    id = models.UUIDField(primary_key=True)
-    created_at = models.DateTimeField(null=True)
-    updated_at = models.DateTimeField(null=True)
-    name = models.CharField(max_length=255, null=True)
-    import_date = models.DateTimeField(null=True)
-    hct_id = models.UUIDField(blank=True, null=True)
-    import_done = models.CharField(max_length=15, null=True)
-    business_area_slug = models.CharField(max_length=250, null=True)
-    import_data = models.OneToOneField(
-        DataImportdata,
-        on_delete=models.DO_NOTHING,
-        related_name="dataregistrationdataimportdatahub_import_data",
-        blank=True,
-        null=True,
-    )
-
-    class Meta:
-        managed = False
-        db_table = "registration_data_registrationdataimportdatahub"
-
-    class Tenant:
-        tenant_filter_field: str = "__all__"
-
-    def __str__(self) -> str:
-        return str(self.name)
-
-
 class ListSanctionlist(HopeModel):
     id = models.BigAutoField(primary_key=True)
     created_at = models.DateTimeField(null=True)

src/hope_country_report/apps/power_query/admin.py

@@ -110,17 +110,20 @@ def get_queryset(self, request):
             return qs.filter(country_office=state.tenant)
         return qs
 
+    @admin.display(boolean=True)
     def success(self, obj: Query) -> bool:
         return not bool(obj.error_message)
 
-    success.boolean = True
-
     def change_view(
         self, request: "HttpRequest", object_id: str, form_url: str = "", extra_context: "dict[str, Any] | None" = None
     ) -> "HttpResponse":
         return super().change_view(request, object_id, form_url, extra_context)
 
-    @button(label="Inspect", icon="search")
+    @button(
+        label="Inspect",
+        icon="search",
+        permission=lambda r, o, handler: handler.model_admin.has_queue_permission("inspect", r, o),
+    )
     def celery_inspect(self, request: HttpRequest, pk: int) -> HttpResponse:
         self.object = self.get_object(request, pk)
         ctx = self.get_common_context(request, pk=pk, object=self.object)
@@ -132,8 +135,15 @@ def celery_inspect(self, request: HttpRequest, pk: int) -> HttpResponse:
         )
 
     def has_change_permission(self, request: HttpRequest, obj: "Any|None" = None) -> bool:
+        if request.user.is_superuser:
+            return True
+        if obj and obj.owner == request.user:
+            return True
         return super().has_change_permission(request, obj)
 
+    def has_queue_permission(self, perm, request: HttpRequest, o: "Query | None") -> bool:
+        return self.has_change_permission(request, o)
+
     @button()
     def notification(self, request: HttpRequest, pk: str) -> HttpResponse:
         obj = self.get_object(request, pk)
@@ -182,10 +192,15 @@ def datasets(self, request: HttpRequest, pk: int) -> HttpResponseRedirect:
         except Exception as e:  # pragma: no cover
             self.message_user(request, f"{e.__class__.__name__}: {e}", messages.ERROR)
 
-    @button(visible=settings.DEBUG)
+    @button(
+        visible=settings.DEBUG,
+        permission=lambda r, o, handler: handler.model_admin.has_queue_permission("run", r, o),
+    )
     def run(self, request: HttpRequest, pk: int) -> HttpResponse:
         ctx = self.get_common_context(request, pk, title="Run results")
         query = self.get_object(request, str(pk))
+        if query is None:
+            return self._get_obj_does_not_exist_redirect(request, self.opts, str(pk))
         results = query.execute_matrix(persist=True)
         self.message_user(request, "Done", messages.SUCCESS)
         ctx["results"] = results
@@ -452,7 +467,14 @@ def get_queryset(self, request):
         return qs
 
     def has_change_permission(self, request: HttpRequest, obj: "Any|None" = None) -> bool:
-        return request.user.is_superuser or bool(obj and obj.owner == request.user)
+        if request.user.is_superuser:
+            return True
+        if obj and obj.owner == request.user:
+            return True
+        return super().has_change_permission(request, obj)
+
+    def has_queue_permission(self, perm, request: HttpRequest, o: "ReportConfiguration | None") -> bool:
+        return self.has_change_permission(request, o)
 
     def get_changeform_initial_data(self, request: HttpRequest) -> "dict[str, Any]":
         kwargs: dict[str, Any] = {"owner": request.user}
@@ -463,7 +485,11 @@ def get_changeform_initial_data(self, request: HttpRequest) -> "dict[str, Any]":
             kwargs["notify_to"] = [request.user]
         return kwargs
 
-    @button(label="Inspect", icon="search")
+    @button(
+        label="Inspect",
+        icon="search",
+        permission=lambda r, o, handler: handler.model_admin.has_queue_permission("inspect", r, o),
+    )
     def celery_inspect(self, request: HttpRequest, pk: int) -> HttpResponse:
         self.object = self.get_object(request, pk)
         ctx = self.get_common_context(request, pk=pk, object=self.object)

src/hope_country_report/apps/power_query/migrations/0006_query_valid_query.py

@@ -12,7 +12,7 @@ class Migration(migrations.Migration):
         migrations.AddConstraint(
             model_name="query",
             constraint=models.CheckConstraint(
-                check=models.Q(
+                condition=models.Q(
                     models.Q(
                         ("code__isnull", True),
                         ("country_office__isnull", False),

src/hope_country_report/apps/power_query/models/arguments.py

@@ -74,7 +74,12 @@ def save(
     ) -> None:
         if not self.code:
             self.code = slugify(self.name)
-        super().save(force_insert, force_update, using, update_fields)
+        super().save(
+            force_insert=force_insert,
+            force_update=force_update,
+            using=using,
+            update_fields=update_fields,
+        )
 
     def refresh(self) -> None:
         if self.source:

src/hope_country_report/apps/power_query/models/formatter.py

@@ -6,9 +6,8 @@
 from django.db import models
 from strategy_field.fields import StrategyField
 from strategy_field.utils import fqn
-
 from ...core.models import CountryOffice
-from ..processors import TYPE_DETAIL, TYPE_LIST, TYPES, ProcessorStrategy, ToHTML, mimetype_map, registry
+from ..processors import TYPE_DETAIL, TYPE_LIST, TYPES, ToHTML, mimetype_map, registry
 from ._base import MIMETYPES
 from .report_template import ReportTemplate
 
@@ -30,16 +29,14 @@ def batched(iterable, n):
 
 
 class Formatter(models.Model):
-    processor: "ProcessorStrategy"
-
+    processor = StrategyField(registry=registry, default=fqn(ToHTML))
     country_office = models.ForeignKey(CountryOffice, on_delete=models.CASCADE, blank=True, null=True)
 
     name = models.CharField(max_length=255, unique=True)
     code = models.TextField(blank=True, null=True)
     template = models.ForeignKey(ReportTemplate, on_delete=models.CASCADE, blank=True, null=True)
 
     file_suffix = models.CharField(max_length=10, choices=MIMETYPES)
-    processor = StrategyField(registry=registry, default=fqn(ToHTML))
     type = models.IntegerField(choices=TYPES, default=TYPE_LIST)
 
     compress = models.BooleanField(default=False, blank=True)
@@ -88,4 +85,9 @@ def save(
     ) -> None:
         if not self.file_suffix:
             self.file_suffix = self.processor.file_suffix
-        super().save(force_insert, force_update, using, update_fields)
+        super().save(
+            force_insert=force_insert,
+            force_update=force_update,
+            using=using,
+            update_fields=update_fields,
+        )