fix: precompute global stats for Ion Wrapped

Author: shrysjain

docs/source/reference_index/apps/wrapped.rst

@@ -8,6 +8,7 @@ wrapped
    :toctree: ../../reference
 
    apps
+   management.commands.warm_wrapped_cache
    stats
    urls
    views

intranet/apps/wrapped/management/__init__.py

@@ -0,0 +1 @@
+

intranet/apps/wrapped/management/commands/__init__.py

@@ -0,0 +1 @@
+

intranet/apps/wrapped/management/commands/warm_wrapped_cache.py

@@ -0,0 +1,68 @@
+import time
+
+from django.core.management.base import BaseCommand
+
+from intranet.apps.wrapped.stats import (
+    compute_signup_counts,
+    compute_unique_activity_counts,
+    compute_visit_counts,
+    set_cached_cohort_counts,
+)
+from intranet.utils.date import get_date_range_this_year, get_school_year_label
+
+
+class Command(BaseCommand):
+    help = "Warm Ion Wrapped cohort caches used for percentile/rank calculations."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--skip-visits",
+            action="store_true",
+            default=False,
+            help="Warm only 8th period cohort caches and skip the request-log visit distribution.",
+        )
+
+    def warm_distribution(self, label, cache_name, start, end, compute_counts):
+        self.stdout.write(f"{label}: computing...")
+        started = time.monotonic()
+        counts = set_cached_cohort_counts(cache_name, start, end, compute_counts())
+        elapsed = time.monotonic() - started
+        self.stdout.write(self.style.SUCCESS(f"{label}: cached {len(counts)} values in {elapsed:.1f}s."))
+
+    def handle(self, *args, **options):
+        start, end = get_date_range_this_year()
+        start_date = start.date()
+        end_date = end.date()
+        year_label = get_school_year_label()
+
+        self.stdout.write(f"Warming Ion Wrapped cohort caches for {year_label}.")
+        self.stdout.write(f"Date range: {start.isoformat()} through {end.isoformat()}")
+
+        self.warm_distribution(
+            "8th period signup counts",
+            "signup-counts",
+            start_date,
+            end_date,
+            lambda: compute_signup_counts(start_date, end_date),
+        )
+        self.warm_distribution(
+            "Unique activity counts",
+            "unique-activity-counts",
+            start_date,
+            end_date,
+            lambda: compute_unique_activity_counts(start_date, end_date),
+        )
+
+        if options["skip_visits"]:
+            self.stdout.write(self.style.WARNING("Skipping Ion visit counts. Visit percentile labels will be omitted until this cache is warmed."))
+        else:
+            self.stdout.write("Ion visit counts use request logs and may take noticeably longer than the 8th period caches.")
+            self.warm_distribution(
+                "Ion visit counts",
+                "visit-counts",
+                start,
+                end,
+                lambda: compute_visit_counts(start, end),
+            )
+
+        self.stdout.write(self.style.SUCCESS("Ion Wrapped cohort cache warm complete."))

intranet/apps/wrapped/stats.py

@@ -5,7 +5,7 @@
 from django.conf import settings
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
-from django.db.models import Count, Q
+from django.db.models import Count
 from django.utils import timezone
 
 from ...utils.date import get_date_range_this_year, get_school_year_label
@@ -16,7 +16,7 @@
 from ..polls.models import Answer, Poll
 
 LAST_MINUTE_WINDOW = datetime.timedelta(minutes=10)
-COHORT_CACHE_TIMEOUT = 60 * 60
+COHORT_CACHE_TIMEOUT = None
 RANK_BUCKETS = (0.01, 0.1, 1, 5, 20, 50)
 
 
@@ -47,15 +47,74 @@ def rank_label(top_percent):
     return None
 
 
-def cached_cohort_counts(name, start, end, compute_counts):
-    cache_key = f"wrapped:cohort:v1:{name}:{start.isoformat()}:{end.isoformat()}"
-    counts = cache.get(cache_key)
-    if counts is None:
-        counts = list(compute_counts())
-        cache.set(cache_key, counts, COHORT_CACHE_TIMEOUT)
+def cohort_cache_key(name, start, end):
+    return f"wrapped:cohort:v1:{name}:{start.isoformat()}:{end.isoformat()}"
+
+
+def get_cached_cohort_counts(name, start, end):
+    return cache.get(cohort_cache_key(name, start, end))
+
+
+def set_cached_cohort_counts(name, start, end, counts):
+    counts = list(counts)
+    cache.set(cohort_cache_key(name, start, end), counts, COHORT_CACHE_TIMEOUT)
     return counts
 
 
+def student_ids_for_cohort():
+    return list(get_user_model().objects.get_students().values_list("id", flat=True))
+
+
+def compute_signup_counts(start_date, end_date):
+    student_ids = student_ids_for_cohort()
+    counts_by_user = dict(
+        EighthSignup.objects.filter(
+            user_id__in=student_ids,
+            scheduled_activity__block__date__gte=start_date,
+            scheduled_activity__block__date__lte=end_date,
+        )
+        .values("user_id")
+        .annotate(
+            wrapped_count=Count("id"),
+        )
+        .values_list("user_id", "wrapped_count")
+    )
+    return [counts_by_user.get(student_id, 0) for student_id in student_ids]
+
+
+def compute_unique_activity_counts(start_date, end_date):
+    student_ids = student_ids_for_cohort()
+    counts_by_user = dict(
+        EighthSignup.objects.filter(
+            user_id__in=student_ids,
+            scheduled_activity__block__date__gte=start_date,
+            scheduled_activity__block__date__lte=end_date,
+        )
+        .values("user_id")
+        .annotate(
+            wrapped_count=Count(
+                "scheduled_activity__activity",
+                distinct=True,
+            )
+        )
+        .values_list("user_id", "wrapped_count")
+    )
+    return [counts_by_user.get(student_id, 0) for student_id in student_ids]
+
+
+def compute_visit_counts(start, end):
+    student_ids = student_ids_for_cohort()
+    counts_by_user = dict(
+        Request.objects.filter(user_id__in=student_ids, timestamp__gte=start, timestamp__lte=end, method="GET")
+        .exclude(path__startswith="/wrapped")
+        .exclude(path__startswith="/api")
+        .values("user_id")
+        .annotate(wrapped_count=Count("id"))
+        .values_list("user_id", "wrapped_count")
+    )
+    return [counts_by_user.get(student_id, 0) for student_id in student_ids]
+
+
 def path_area(path):
     clean_path = urlsplit(path).path
     if clean_path in ("", "/"):
@@ -177,30 +236,8 @@ def build_eighth_stats(user, start_date, end_date):
         for sponsor in sponsors:
             sponsor_counts[sponsor.name] += 1
 
-    students = get_user_model().objects.get_students()
-    signup_counts = cached_cohort_counts(
-        "signup-counts",
-        start_date,
-        end_date,
-        lambda: students.annotate(
-            wrapped_count=Count(
-                "eighthsignup",
-                filter=Q(eighthsignup__scheduled_activity__block__date__gte=start_date, eighthsignup__scheduled_activity__block__date__lte=end_date),
-            )
-        ).values_list("wrapped_count", flat=True),
-    )
-    unique_counts = cached_cohort_counts(
-        "unique-activity-counts",
-        start_date,
-        end_date,
-        lambda: students.annotate(
-            wrapped_count=Count(
-                "eighthsignup__scheduled_activity__activity",
-                filter=Q(eighthsignup__scheduled_activity__block__date__gte=start_date, eighthsignup__scheduled_activity__block__date__lte=end_date),
-                distinct=True,
-            )
-        ).values_list("wrapped_count", flat=True),
-    )
+    signup_counts = get_cached_cohort_counts("signup-counts", start_date, end_date)
+    unique_counts = get_cached_cohort_counts("unique-activity-counts", start_date, end_date)
 
     return {
         "total": total,
@@ -243,20 +280,7 @@ def build_usage_stats(user, start, end):
         month_counts[local_time.strftime("%B")] += 1
         hour_counts[local_time.hour] += 1
 
-    visit_filter = (
-        Q(request__timestamp__gte=start, request__timestamp__lte=end, request__method="GET")
-        & ~Q(request__path__startswith="/wrapped")
-        & ~Q(request__path__startswith="/api")
-    )
-    visit_counts = cached_cohort_counts(
-        "visit-counts",
-        start,
-        end,
-        lambda: get_user_model()
-        .objects.get_students()
-        .annotate(wrapped_count=Count("request", filter=visit_filter))
-        .values_list("wrapped_count", flat=True),
-    )
+    visit_counts = get_cached_cohort_counts("visit-counts", start, end)
 
     busiest_hour = None
     if hour_counts:

intranet/settings/__init__.py

@@ -726,6 +726,7 @@ def get_oidc_private_key():
     "intranet.apps.features",
     "intranet.apps.oauth",
     "intranet.apps.logs",
+    "intranet.apps.wrapped",
     # Django plugins
     "widget_tweaks",
     "oauth2_provider",  # django-oauth-toolkit