Grace/menu fix (#409)

* load_next_menu loads 7 days in advance

* updated threading to only be with API loading

* makes api_wrapper thread only API calls

* formatting fixes

* PLEASE LINTER

* now it passes the test case

* updated imports

Author: gracech1

backend/dining/api_wrapper.py

@@ -1,11 +1,12 @@
 import datetime
 import json
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 import requests
 from django.conf import settings
 from django.utils import timezone
 from django.utils.timezone import make_aware
-from requests.exceptions import ConnectTimeout, ReadTimeout
+from requests.exceptions import ConnectionError, ConnectTimeout, ReadTimeout
 
 from dining.models import DiningItem, DiningMenu, DiningStation, Venue
 from utils.errors import APIError
@@ -59,7 +60,7 @@ def get_venues(self):
         venues_route = OPEN_DATA_ENDPOINTS["VENUES"]
         response = self.request("GET", venues_route)
         if response.status_code != 200:
-            raise APIError("Dining: Error connecting to API")
+            raise APIError("Dining: error connecting to API " + response.text)
         venues = response.json()["result_data"]["campuses"]["203"]["cafes"]
         for key, value in venues.items():
             # Cleaning up json response
@@ -107,21 +108,47 @@ def get_venues(self):
             results.append(value)
         return results
 
+    def fetch_menu(self, venue_id, date):
+        """
+        Calls API to fetch menu for a given venue and date
+        """
+        worker = DiningAPIWrapper()  # avoid shared mutable token state across threads
+        menu_base = OPEN_DATA_ENDPOINTS["MENUS"]
+        response = worker.request("GET", f"{menu_base}?cafe={venue_id}&date={date}")
+        if response.status_code != 200:
+            raise APIError("Dining: error connecting to API " + response.text)
+        return (
+            venue_id,
+            response.json(),
+        )  # also storing venue_id to later access in fetched_menus list
+
     def load_menu(self, date=timezone.now().date()):
         """
-        Loads the weeks menu starting from today
+        Loads today's menu
         NOTE: This method should only be used in load_next_menu.py, which is
         run based on a cron job every day
         """
-
         # Venues without a menu should not be parsed
         skipped_venues = [747, 1163, 1731, 1732, 1733, 1464004, 1464009]
 
         # TODO: Handle API responses during empty menus (holidays)
-        menu_base = OPEN_DATA_ENDPOINTS["MENUS"]
         venues = [v for v in Venue.objects.all() if v.venue_id not in skipped_venues]
-        for venue in venues:
-            response = self.request("GET", f"{menu_base}?cafe={venue.venue_id}&date={date}").json()
+        venue_map = {venue.venue_id: venue for venue in venues}
+
+        # Fetch all menus in parallel to speed up loading time.
+        fetched_menus = []
+        with ThreadPoolExecutor(max_workers=8) as executor:  # 8 can be tuned
+            futures = [executor.submit(self.fetch_menu, venue.venue_id, date) for venue in venues]
+            for future in as_completed(futures):
+                try:
+                    venue_id, response_json = future.result()
+                    fetched_menus.append((venue_id, response_json))
+                except Exception as e:
+                    print(f"Error fetching menu: {e}")
+
+        # Process the fetched menus and load them into the database
+        for venue_id, response in fetched_menus:
+            venue = venue_map[venue_id]
             # Load new items into database
             # TODO: There is something called a "goitem" for venues like English House.
             # We are currently not loading them in
@@ -144,6 +171,7 @@ def load_menu(self, date=timezone.now().date()):
                     end_time=daypart["endtime"],
                     service=daypart["label"],
                 )
+
                 # Append stations to dining menu
                 self.load_stations(daypart["stations"], dining_menu)
 

backend/dining/management/commands/load_next_menu.py

@@ -8,12 +8,25 @@
 
 class Command(BaseCommand):
     """
-    Loads Menu for 1 week in advance.
+    Loads menu for the next 7 days, starting from today. Takes about 3 minutes to run.
     Invariant: For every date, the database should contain the menus for
     the next 7 days, including the original date.
     """
 
-    def handle(self, *args, **kwargs):
+    def load_one_menu(self, delta, *args, **kwargs):
+        """
+        Loads menu for a single day
+        """
         d = DiningAPIWrapper()
-        d.load_menu(timezone.now().date() + datetime.timedelta(days=6))
-        self.stdout.write("Loaded new Dining Menu!")
+        d.load_menu(timezone.now().date() + datetime.timedelta(days=delta))
+        self.stdout.write(
+            "Loaded new Dining Menu for "
+            + str(timezone.now().date() + datetime.timedelta(days=delta))
+        )
+
+    def handle(self, *args, **kwargs):
+        """
+        Load menu for the next 7 days
+        """
+        for i in range(7):
+            self.load_one_menu(i)

backend/dining/views.py

@@ -2,7 +2,8 @@
 
 from analytics.entries import FuncEntry, ViewEntry
 from django.core.cache import cache
-from django.db.models import Count
+from django.db.models import Count, F, Window
+from django.db.models.functions.window import RowNumber
 from django.shortcuts import get_object_or_404
 from django.utils import timezone
 from django.utils.timezone import make_aware
@@ -54,11 +55,27 @@ def get_queryset(self):
         # if date_param is out of bounds
         if date_param := self.kwargs.get("date"):
             date = make_aware(datetime.datetime.strptime(date_param, "%Y-%m-%d"))
-            return DiningMenu.objects.filter(date=date)
+            base = DiningMenu.objects.filter(date=date)
         else:
             start_date = timezone.now().date()
             end_date = start_date + datetime.timedelta(days=6)
-            return DiningMenu.objects.filter(date__gte=start_date, date__lte=end_date)
+            base = DiningMenu.objects.filter(date__gte=start_date, date__lte=end_date)
+        # Only returns most recently loaded menus
+        latest = base.annotate(
+            rn=Window(
+                expression=RowNumber(),
+                partition_by=[
+                    F("venue_id"),
+                    F("date"),
+                    F("service"),
+                    F("start_time"),
+                    F("end_time"),
+                ],
+                order_by=[F("id").desc()],
+            )
+        ).filter(rn=1)
+
+        return latest
 
 
 class Preferences(APIView):

backend/tests/dining/test_views.py

@@ -6,6 +6,7 @@
 from django.core.management import call_command
 from django.test import TestCase
 from django.urls import reverse
+from requests.exceptions import ConnectionError
 from rest_framework.test import APIClient
 
 from dining.api_wrapper import APIError, DiningAPIWrapper