refactor: change the admin stats

- create a stats model to cache all stats
- create schedulers and tasks to update stats
- refactor schedulers to use standard template

Author: tudoramariei

backend/donations/admin/donors.py

@@ -1,6 +1,7 @@
 import logging
 
 from django.contrib import admin, messages
+from django.core.management import call_command
 from django.core.validators import EMPTY_VALUES
 from django.db.models import QuerySet
 from django.http import HttpRequest
@@ -140,6 +141,7 @@ class DonorAdmin(ModelAdmin):
     )
 
     actions = ("remove_donations",)
+    actions_list = ("run_redirections_stats_generator", "run_redirections_stats_generator_force")
 
     def has_change_permission(self, request, obj=None):
         return False
@@ -179,3 +181,11 @@ def remove_donations(self, request, queryset: QuerySet[Donor]):
         ) % {"failure": task_results[REMOVE_DONATIONS_FAILURE_FLAG]}
 
         self.message_user(request, ", ".join([part_1, part_2, part_3 + "."]))
+
+    @action(description=_("Schedule redirections stats"), url_path="schedule-redirections-stats-generator")
+    def run_redirections_stats_generator(self, request, queryset: QuerySet[Donor]):
+        call_command("generate_redirections_stats")
+
+    @action(description=_("Schedule redirections stats [FORCE]"), url_path="schedule-redirections-stats-generator-f")
+    def run_redirections_stats_generator_force(self, request, queryset: QuerySet[Donor]):
+        call_command("generate_redirections_stats", "--force")

backend/donations/management/commands/generate_redirections_stats.py

@@ -0,0 +1,55 @@
+import logging
+from datetime import date
+from typing import Set
+
+from django.core.management import BaseCommand
+from django_q.tasks import async_task
+
+from donations.models import Donor
+from donations.models.stat_configs import StatsChoices, create_stat
+from stats.models import Stat
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Run a task that "
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--force",
+            action="store_true",
+            help="Force regeneration of stats even if they already exist for certain dates.",
+            default=False,
+        )
+
+    def handle(self, *args, **kwargs):
+        """
+        Generate redirection statistics for donors.
+        `force` argument forces regeneration of stats even if they already exist.
+        If the `force` argument is not provided, the command will only generate stats for dates
+        that do not already have stats recorded and expired stats will be cleaned up first.
+        """
+        force: bool = kwargs.get("force", False)
+
+        target_set: Set[date] = {
+            dt.date() for dt in (Donor.available.all().values_list("date_created", flat=True).distinct())
+        }
+
+        if not force:
+            # Get existing stats dates that are not expired
+            existing_stats_dates: Set[date] = set(
+                Stat.objects.filter(name=StatsChoices.REDIRECTIONS_PER_DAY)
+                .exclude(expires_at__lte=date.today())
+                .values_list("date", flat=True)
+                .distinct()
+            )
+
+            target_set: Set[date] = target_set - existing_stats_dates
+
+        for single_date in target_set:
+            async_task(
+                create_stat,
+                stat_choice=StatsChoices.REDIRECTIONS_PER_DAY,
+                for_date=single_date,
+            )

backend/donations/management/commands/generate_stats.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+
+from django.core.management import BaseCommand
+
+from donations.models.stat_configs import StatsChoices, create_stat
+
+
+class Command(BaseCommand):
+    help = "Generate the statistics for the dashboard."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "statistic",
+            type=str,
+            help="What type of statistic to generate",
+            choices=StatsChoices.values,
+        )
+        parser.add_argument(
+            "--date",
+            type=str,
+            help="Date for which to generate the statistic (YYYY-MM-DD). Required for REDIRECTIONS_PER_DAY.",
+        )
+
+    def handle(self, *args, **options):
+        statistic_type: str = options["statistic"]
+        for_date_str: str = options.get("date")
+
+        if statistic_type == StatsChoices.REDIRECTIONS_PER_DAY and not for_date_str:
+            self.stderr.write("Error: --date argument is required for REDIRECTIONS_PER_DAY statistic.")
+            return
+
+        self.stdout.write(f"Generating statistics for: {statistic_type}")
+
+        if for_date_str:
+            for_date = datetime.strptime(for_date_str, "%Y-%m-%d").date()
+            create_stat(stat_choice=StatsChoices(statistic_type), for_date=for_date)
+        else:
+            create_stat(stat_choice=StatsChoices(statistic_type))

backend/donations/management/commands/schedule_stats_generator_ngos.py

@@ -0,0 +1,55 @@
+import logging
+from datetime import timedelta
+
+from django.utils import timezone
+from django_q.models import Schedule
+
+from donations.models.stat_configs import StatsChoices
+from utils.common.commands import SchedulerCommand
+
+logger = logging.getLogger(__name__)
+
+
+class Command(SchedulerCommand):
+    help = "Schedule a recurring task to generate ngos statistics."
+
+    command_name: str = "generate_stats"
+
+    schedule_prefix: str = "GENERATE_STATS_NGOS"
+    schedule_details = {
+        "schedule_type": Schedule.MINUTES,
+        "minutes": 15,
+        "repeats": -1,
+        "next_run": timezone.now() + timedelta(minutes=0),
+    }
+
+    choices = [StatsChoices.NGOS_REGISTERED, StatsChoices.NGOS_ACTIVE, StatsChoices.NGOS_WITH_NGOHUB]
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "statistic",
+            type=str,
+            help="What type of statistic to generate",
+            choices=self.choices,
+        )
+
+    def handle(self, *args, **kwargs):
+        statistic: str = kwargs["statistic"]
+
+        if statistic not in self.choices:
+            self.stderr.write("Error: Invalid statistic type provided.")
+            return
+
+        if statistic == StatsChoices.NGOS_REGISTERED:
+            self.schedule_name = f"{self.schedule_prefix}_REGISTERED"
+            self.function_args = (StatsChoices.NGOS_REGISTERED,)
+
+        if statistic == StatsChoices.NGOS_ACTIVE:
+            self.schedule_name = f"{self.schedule_prefix}_ACTIVE"
+            self.function_args = (StatsChoices.NGOS_ACTIVE,)
+
+        if statistic == StatsChoices.NGOS_WITH_NGOHUB:
+            self.schedule_name = f"{self.schedule_prefix}_WITH_NGOHUB"
+            self.function_args = (StatsChoices.NGOS_WITH_NGOHUB,)
+
+        super().handle(*args, **kwargs)

backend/donations/management/commands/schedule_stats_generator_redirections.py

@@ -0,0 +1,23 @@
+import logging
+from datetime import timedelta
+
+from django.utils import timezone
+from django_q.models import Schedule
+
+from utils.common.commands import SchedulerCommand
+
+logger = logging.getLogger(__name__)
+
+
+class Command(SchedulerCommand):
+    help = "Schedule a recurring task to generate donation statistics."
+
+    command_name: str = "generate_redirections_stats"
+
+    schedule_name: str = "GENERATE_STATS_REDIRECTIONS"
+    schedule_details = {
+        "schedule_type": Schedule.MINUTES,
+        "minutes": 5,
+        "repeats": -1,
+        "next_run": timezone.now() + timedelta(minutes=0),
+    }

backend/donations/models/stat_configs.py

@@ -0,0 +1,84 @@
+from datetime import date, datetime, timedelta
+from decimal import Decimal
+from typing import Optional
+
+from django.db import models
+from django.utils.timezone import now
+
+from donations.models import Donor, Ngo
+from stats.models import Stat
+
+
+class StatsChoices(models.TextChoices):
+    REDIRECTIONS_PER_DAY = "donations_per_day", "Donations per Year and Month"
+    NGOS_REGISTERED = "ngos_registered", "Registered NGOs"
+    NGOS_ACTIVE = "ngos_active", "Active NGOs"
+    NGOS_WITH_NGOHUB = "ngos_with_ngohub", "NGOs with NGO Hub"
+
+
+def _create_stat_redirections_per_day(for_date: date):
+    metric: Decimal = Donor.available.filter(date_created__date=for_date).count()
+    expiration: Optional[datetime] = now() + timedelta(days=5)
+    if for_date < now().date():
+        expiration = None
+
+    Stat.objects.update_or_create(
+        name=StatsChoices.REDIRECTIONS_PER_DAY,
+        date=for_date,
+        defaults={
+            "value": metric,
+            "expires_at": expiration,
+        },
+    )
+
+
+def _create_stat_ngos_registered():
+    metric: Decimal = Ngo.active.count()
+    expiration: datetime = now() + timedelta(minutes=5)
+
+    Stat.objects.update_or_create(
+        name=StatsChoices.NGOS_REGISTERED,
+        defaults={
+            "value": metric,
+            "expires_at": expiration,
+        },
+    )
+
+
+def _create_stat_ngos_active():
+    metric: Decimal = Ngo.with_forms_this_year.count()
+    expiration: datetime = now() + timedelta(minutes=5)
+
+    Stat.objects.update_or_create(
+        name=StatsChoices.NGOS_ACTIVE,
+        defaults={
+            "value": metric,
+            "expires_at": expiration,
+        },
+    )
+
+
+def _create_stat_ngos_with_ngohub():
+    metric: Decimal = Ngo.ngo_hub.count()
+    expiration: datetime = now() + timedelta(minutes=5)
+
+    Stat.objects.update_or_create(
+        name=StatsChoices.NGOS_WITH_NGOHUB,
+        defaults={
+            "value": metric,
+            "expires_at": expiration,
+        },
+    )
+
+
+def create_stat(*, stat_choice: StatsChoices, for_date: date = None) -> None:
+    if stat_choice == StatsChoices.REDIRECTIONS_PER_DAY:
+        if for_date is None:
+            raise ValueError("for_date must be provided for REDIRECTIONS_PER_DAY statistic.")
+        _create_stat_redirections_per_day(for_date=for_date)
+    elif stat_choice == StatsChoices.NGOS_REGISTERED:
+        _create_stat_ngos_registered()
+    elif stat_choice == StatsChoices.NGOS_ACTIVE:
+        _create_stat_ngos_active()
+    elif stat_choice == StatsChoices.NGOS_WITH_NGOHUB:
+        _create_stat_ngos_with_ngohub()