refactor: change the admin stats

- create a stats model to cache all stats
- create schedulers and tasks to update stats
- refactor schedulers to use standard template

Author: tudoramariei

backend/donations/admin/donors.py

@@ -1,6 +1,7 @@
 import logging
 
 from django.contrib import admin, messages
+from django.core.management import call_command
 from django.core.validators import EMPTY_VALUES
 from django.db.models import QuerySet
 from django.http import HttpRequest
@@ -140,6 +141,7 @@ class DonorAdmin(ModelAdmin):
     )
 
     actions = ("remove_donations",)
+    actions_list = ("run_redirections_stats_generator", "run_redirections_stats_generator_force")
 
     def has_change_permission(self, request, obj=None):
         return False
@@ -179,3 +181,11 @@ def remove_donations(self, request, queryset: QuerySet[Donor]):
         ) % {"failure": task_results[REMOVE_DONATIONS_FAILURE_FLAG]}
 
         self.message_user(request, ", ".join([part_1, part_2, part_3 + "."]))
+
+    @action(description=_("Schedule redirections stats"), url_path="schedule-redirections-stats-generator")
+    def run_redirections_stats_generator(self, request, queryset: QuerySet[Donor]):
+        call_command("generate_redirections_stats")
+
+    @action(description=_("Schedule redirections stats [FORCE]"), url_path="schedule-redirections-stats-generator-f")
+    def run_redirections_stats_generator_force(self, request, queryset: QuerySet[Donor]):
+        call_command("generate_redirections_stats", "--force")

backend/donations/management/commands/generate_redirections_stats.py

@@ -0,0 +1,45 @@
+import logging
+from datetime import date
+from typing import Set
+
+from django.core.management import BaseCommand
+from django_q.tasks import async_task
+
+from donations.models import Donor
+from donations.models.stat_configs import StatsChoices, create_stat
+from stats.models import Stat
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Run a generator that looks over all of the dates that are supposed to have donation stats and generates them."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--force",
+            action="store_true",
+            help="Force regeneration of stats even if they already exist for certain dates.",
+            default=False,
+        )
+
+    def handle(self, *args, **kwargs):
+        force: bool = kwargs.get("force", False)
+
+        target_set: Set[date] = {
+            dt.date() for dt in (Donor.available.all().values_list("date_created", flat=True).distinct())
+        }
+
+        if not force:
+            existing_stats_dates: Set[date] = set(
+                Stat.objects.filter(name=StatsChoices.REDIRECTIONS_PER_DAY).values_list("date", flat=True).distinct()
+            )
+
+            target_set: Set[date] = target_set - existing_stats_dates
+
+        for single_date in target_set:
+            async_task(
+                create_stat,
+                stat_choice=StatsChoices.REDIRECTIONS_PER_DAY,
+                for_date=single_date,
+            )

backend/donations/management/commands/generate_stats.py

@@ -0,0 +1,38 @@
+from datetime import datetime
+
+from django.core.management import BaseCommand
+
+from donations.models.stat_configs import StatsChoices, create_stat
+
+
+class Command(BaseCommand):
+    help = "Generate the statistics for the dashboard."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "statistic",
+            type=str,
+            help="What type of statistic to generate",
+            choices=StatsChoices.values,
+        )
+        parser.add_argument(
+            "--date",
+            type=str,
+            help="Date for which to generate the statistic (YYYY-MM-DD). Required for REDIRECTIONS_PER_DAY.",
+        )
+
+    def handle(self, *args, **options):
+        statistic_type: str = options["statistic"]
+        for_date_str: str = options.get("date")
+
+        if statistic_type == StatsChoices.REDIRECTIONS_PER_DAY and not for_date_str:
+            self.stderr.write("Error: --date argument is required for REDIRECTIONS_PER_DAY statistic.")
+            return
+
+        self.stdout.write(f"Generating statistics for: {statistic_type}")
+
+        if for_date_str:
+            for_date = datetime.strptime(for_date_str, "%Y-%m-%d").date()
+            create_stat(stat_choice=StatsChoices(statistic_type), for_date=for_date)
+        else:
+            create_stat(stat_choice=StatsChoices(statistic_type))

backend/donations/management/commands/schedule_stats_generator.py

@@ -0,0 +1,23 @@
+import logging
+from datetime import timedelta
+
+from django.utils import timezone
+from django_q.models import Schedule
+
+from utils.common.commands import SchedulerCommand
+
+logger = logging.getLogger(__name__)
+
+
+class Command(SchedulerCommand):
+    help = "Schedule a recurring task to generate donation statistics."
+
+    command_name: str = "generate_redirections_stats"
+
+    schedule_name: str = "REDIRECTION_STATS"
+    schedule_details = {
+        "schedule_type": Schedule.MINUTES,
+        "minutes": 5,
+        "repeats": -1,
+        "next_run": timezone.now() + timedelta(minutes=0),
+    }

backend/donations/models/stat_configs.py

@@ -0,0 +1,76 @@
+from datetime import date, datetime, timedelta
+from decimal import Decimal
+from typing import Optional
+
+from django.db import models
+from django.utils.timezone import now
+
+from donations.models import Donor, Ngo
+from stats.models import Stat
+
+
+class StatsChoices(models.TextChoices):
+    REDIRECTIONS_PER_DAY = "donations_per_day", "Donations per Year and Month"
+    NGOS_REGISTERED = "ngos_registered", "Registered NGOs"
+    NGOS_ACTIVE = "ngos_active", "Active NGOs"
+    NGOS_WITH_NGOHUB = "ngos_with_ngohub", "NGOs with NGO Hub"
+
+
+def _create_stat_redirections_per_day(for_date: date):
+    metric: Decimal = Donor.available.filter(date_created__date=for_date).count()
+    expiration: Optional[datetime] = now() + timedelta(days=5)
+    if for_date < now().date():
+        expiration = None
+
+    Stat.objects.update_or_create(
+        name=StatsChoices.REDIRECTIONS_PER_DAY,
+        date=for_date,
+        value=metric,
+        expires_at=expiration,
+    )
+
+
+def _create_stat_ngos_registered():
+    metric: Decimal = Ngo.active.count()
+    expiration: datetime = now() + timedelta(minutes=5)
+
+    Stat.objects.update_or_create(
+        name=StatsChoices.NGOS_REGISTERED,
+        value=metric,
+        expires_at=expiration,
+    )
+
+
+def _create_stat_ngos_active():
+    metric: Decimal = Ngo.with_forms_this_year.count()
+    expiration: datetime = now() + timedelta(minutes=5)
+
+    Stat.objects.update_or_create(
+        name=StatsChoices.NGOS_ACTIVE,
+        value=metric,
+        expires_at=expiration,
+    )
+
+
+def _create_stat_ngos_with_ngohub():
+    metric: Decimal = Ngo.ngo_hub.count()
+    expiration: datetime = now() + timedelta(minutes=5)
+
+    Stat.objects.update_or_create(
+        name=StatsChoices.NGOS_WITH_NGOHUB,
+        value=metric,
+        expires_at=expiration,
+    )
+
+
+def create_stat(*, stat_choice: StatsChoices, for_date: date = None) -> None:
+    if stat_choice == StatsChoices.REDIRECTIONS_PER_DAY:
+        if for_date is None:
+            raise ValueError("for_date must be provided for REDIRECTIONS_PER_DAY statistic.")
+        _create_stat_redirections_per_day(for_date=for_date)
+    elif stat_choice == StatsChoices.NGOS_REGISTERED:
+        _create_stat_ngos_registered()
+    elif stat_choice == StatsChoices.NGOS_ACTIVE:
+        _create_stat_ngos_active()
+    elif stat_choice == StatsChoices.NGOS_WITH_NGOHUB:
+        _create_stat_ngos_with_ngohub()

backend/donations/views/dashboard/admin_dashboard.py

@@ -1,4 +1,4 @@
-from datetime import datetime
+from datetime import datetime, tzinfo
 from typing import Dict, List, Union
 
 from django.conf import settings
@@ -7,10 +7,17 @@
 from django.utils.timezone import now
 from django.utils.translation import gettext_lazy as _
 
-from donations.models.ngos import Ngo
+from donations.views.dashboard.stats_helpers.chart import donors_for_month
+from donations.views.dashboard.stats_helpers.metrics import (
+    all_redirections,
+    all_registered_ngos,
+    current_year_redirections,
+    ngos_active_in_current_year,
+    ngos_with_ngo_hub,
+)
+from donations.views.dashboard.stats_helpers.yearly import get_stats_for_year
 from redirectioneaza.common.cache import cache_decorator
 
-from ...models.donors import Donor
 from .helpers import (
     generate_donations_per_month_chart,
     get_current_year_range,
@@ -19,20 +26,18 @@
 
 ADMIN_DASHBOARD_CACHE_KEY = "ADMIN_DASHBOARD"
 ADMIN_DASHBOARD_STATS_CACHE_KEY = "ADMIN_DASHBOARD_STATS"
+ADMIN_DASHBOARD_HEADER_CACHE_KEY = "ADMIN_DASHBOARD_HEADER"
 
 
-def callback(request, context) -> Dict:
+def callback(_, context) -> Dict:
     context.update(_get_admin_stats())
-
     return context
 
 
-@cache_decorator(timeout=settings.TIMEOUT_CACHE_NORMAL, cache_key=ADMIN_DASHBOARD_STATS_CACHE_KEY)
 def _get_admin_stats() -> Dict:
-    today = now()
     years_range_ascending = get_current_year_range()
 
-    header_stats: List[List[Dict[str, Union[str, int]]]] = _get_header_stats(today)
+    header_stats: List[List[Dict[str, Union[str, int]]]] = _get_header_stats()
 
     yearly_stats: List[Dict] = _get_yearly_stats(years_range_ascending)
 
@@ -45,72 +50,79 @@ def _get_admin_stats() -> Dict:
     }
 
 
-def _get_header_stats(today) -> List[List[Dict[str, Union[str, int | datetime]]]]:
-    current_year = today.year
+@cache_decorator(timeout=settings.TIMEOUT_CACHE_SHORT, cache_key=ADMIN_DASHBOARD_HEADER_CACHE_KEY)
+def _get_header_stats() -> List[List[Dict[str, Union[str, int | datetime]]]]:
+    today: datetime = now()
 
-    current_year_range = get_encoded_current_year_range(current_year, today.tzinfo)
+    current_year: int = today.year
+    tz_info: tzinfo = today.tzinfo
+
+    current_year_range = get_encoded_current_year_range(current_year, tz_info)
 
     return [
         [
             {
                 "title": _("Donations this year"),
                 "icon": "edit_document",
-                "metric": Donor.available.filter(date_created__year=current_year).count(),
+                "metric": current_year_redirections(),
                 "footer": _create_stat_link(
-                    url=f"{reverse('admin:donations_donor_changelist')}?{current_year_range}", text=_("View all")
+                    url=f"{reverse('admin:donations_donor_changelist')}?{current_year_range}",
+                    text=_("View all"),
                 ),
-                "timestamp": now(),
             },
             {
                 "title": _("Donations all-time"),
                 "icon": "edit_document",
-                "metric": Donor.available.count(),
-                "footer": _create_stat_link(url=reverse("admin:donations_donor_changelist"), text=_("View all")),
-                "timestamp": now(),
+                "metric": all_redirections(),
+                "footer": _create_stat_link(
+                    url=reverse("admin:donations_donor_changelist"),
+                    text=_("View all"),
+                ),
             },
             {
                 "title": _("NGOs registered"),
                 "icon": "foundation",
-                "metric": Ngo.active.count(),
+                "metric": all_registered_ngos(),
                 "footer": _create_stat_link(
-                    url=f"{reverse('admin:donations_ngo_changelist')}?is_active=1", text=_("View all")
+                    url=f"{reverse('admin:donations_ngo_changelist')}?is_active=1",
+                    text=_("View all"),
                 ),
-                "timestamp": now(),
             },
             {
                 "title": _("Functioning NGOs"),
                 "icon": "foundation",
-                "metric": Ngo.with_forms_this_year.count(),
-                "footer": _create_stat_link(url=f"{reverse('admin:donations_ngo_changelist')}", text=_("View all")),
-                "timestamp": now(),
+                "metric": ngos_active_in_current_year(),
+                "footer": _create_stat_link(
+                    url=f"{reverse('admin:donations_ngo_changelist')}",
+                    text=_("View all"),
+                ),
             },
             {
                 "title": _("NGOs from NGO Hub"),
                 "icon": "foundation",
-                "metric": Ngo.ngo_hub.count(),
+                "metric": ngos_with_ngo_hub(),
                 "footer": _create_stat_link(
-                    url=f"{reverse('admin:donations_ngo_changelist')}?is_active=1&has_ngohub=1", text=_("View all")
+                    url=f"{reverse('admin:donations_ngo_changelist')}?is_active=1&has_ngohub=1",
+                    text=_("View all"),
                 ),
-                "timestamp": now(),
             },
         ]
     ]
 
 
 def _create_chart_statistics() -> Dict[str, str]:
     default_border_width: int = 3
+    current_year = now().year
 
-    donations_per_month_queryset = [
-        Donor.available.filter(date_created__month=month) for month in range(1, settings.DONATIONS_LIMIT.month + 1)
+    donations_per_month_queryset: List[int] = [
+        donors_for_month(month, current_year)["metric"] for month in range(1, settings.DONATIONS_LIMIT.month + 1)
     ]
 
-    forms_per_month_chart = generate_donations_per_month_chart(default_border_width, donations_per_month_queryset)
-
-    return forms_per_month_chart
+    return generate_donations_per_month_chart(default_border_width, donations_per_month_queryset)
 
 
 def _get_yearly_stats(years_range_ascending) -> List[Dict[str, Union[int, List[Dict]]]]:
-    statistics = [_get_stats_for_year(year) for year in years_range_ascending]
+    statistics = [get_stats_for_year(year) for year in years_range_ascending]
 
     for index, statistic in enumerate(statistics):
         if index == 0:
@@ -129,24 +141,6 @@ def _get_yearly_stats(years_range_ascending) -> List[Dict[str, Union[int, List[D
     return sorted(final_statistics, key=lambda x: x["year"], reverse=True)
 
 
-# TODO: This cache seems useless because we already cache the entire dashboard stats
-@cache_decorator(timeout=settings.TIMEOUT_CACHE_NORMAL, cache_key_prefix=ADMIN_DASHBOARD_CACHE_KEY)
-def _get_stats_for_year(year: int) -> Dict[str, int | datetime]:
-    donations: int = Donor.available.filter(date_created__year=year).count()
-    ngos_registered: int = Ngo.objects.filter(date_created__year=year).count()
-    ngos_with_forms: int = Donor.available.filter(date_created__year=year).values("ngo_id").distinct().count()
-
-    statistic = {
-        "year": year,
-        "donations": donations,
-        "ngos_registered": ngos_registered,
-        "ngos_with_forms": ngos_with_forms,
-        "timestamp": now(),
-    }
-
-    return statistic
-
-
 def _format_yearly_stats(statistics) -> List[Dict[str, Union[int, List[Dict]]]]:
     return [
         {
@@ -191,5 +185,5 @@ def _format_yearly_stats(statistics) -> List[Dict[str, Union[int, List[Dict]]]]:
     ]
 
 
-def _create_stat_link(url: str, text: str) -> str:
+def _create_stat_link(url: str, text) -> str:
     return mark_safe(f'<a href="{url}" class="text-orange-700 font-semibold">{text}</a>')