add event categories (update events model to use JSONField, add to OpenAI prompt for category assignment)

Author: ericahan22

.github/workflows/update-events-data.yml

@@ -120,10 +120,9 @@ jobs:
         id: generate_static
         working-directory: backend/scraping
         run: python generate_static_data.py
-        if: github.event_name == 'schedule'
 
       - name: Commit and push changes
-        if: github.event_name == 'schedule' && steps.generate_static.outcome == 'success'
+        if: steps.generate_static.outcome == 'success'
         run: |
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'

backend/apps/events/migrations/0020_alter_events_categories.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2025-11-03 20:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('events', '0019_remove_events_events_school_status_dtstart_utc_dtend_utc_added_at_idx_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='events',
+            name='categories',
+            field=models.JSONField(blank=True, default=list, help_text="List of event categories (e.g. ['Academic', 'Cultural'])"),
+        ),
+    ]

backend/apps/events/models.py

@@ -38,11 +38,10 @@ class Events(models.Model):
     duration = models.DurationField(blank=True, null=True, help_text="'8:00:00'")
 
     # Event categorization
-    categories = models.CharField(
-        max_length=255,
-        null=True,
+    categories = models.JSONField(
+        default=list,
         blank=True,
-        help_text="'Career, Networking, Professional Development'",
+        help_text="List of event categories (e.g. ['Academic', 'Cultural'])",
     )
 
     # Timezone information

backend/scraping/instagram_feed.py

@@ -135,6 +135,7 @@ def append_event_to_csv(
     school = event_data.get("school", "")
     source_image_url = event_data.get("source_image_url", "")
     title = event_data.get("title", "")
+    categories = event_data.get("categories", [])
 
     fieldnames = [
         "ig_handle",
@@ -158,6 +159,7 @@ def append_event_to_csv(
         "source_image_url",
         "all_day",
         "club_type",
+        "categories",
         "raw_json",
         "added_to_db",
         "status",
@@ -191,6 +193,7 @@ def append_event_to_csv(
                 "source_image_url": source_image_url,
                 "all_day": all_day,
                 "club_type": club_type or event_data.get("club_type") or "",
+                "categories": json.dumps(categories, ensure_ascii=False),
                 "raw_json": json.dumps(event_data, ensure_ascii=False),
                 "added_to_db": added_to_db,
                 "status": "CONFIRMED",
@@ -223,6 +226,11 @@ def insert_event_to_db(event_data, ig_handle, source_url):
     latitude = event_data.get("latitude", None)
     longitude = event_data.get("longitude", None)
     school = event_data.get("school", "")
+    categories = event_data.get("categories", [])
+
+    if not categories or not isinstance(categories, list):
+        logger.warning(f"Event '{title}' missing categories, assigning 'Uncategorized'")
+        categories = ["Uncategorized"]
 
     if is_duplicate_event(event_data):
         try:
@@ -232,6 +240,7 @@ def insert_event_to_db(event_data, ig_handle, source_url):
                 source_url,
                 added_to_db="duplicate",
                 embedding=event_data.get("embedding"),
+                club_type=None,
             )
         except Exception as csv_e:
             logger.error(f"Error writing duplicate event to CSV: {csv_e}")
@@ -301,6 +310,7 @@ def insert_event_to_db(event_data, ig_handle, source_url):
         "longitude": longitude,
         "school": school,
         "rrule": event_data.get("rrule", ""),
+        "categories": categories,
     }
 
     try:

backend/scripts/backfill_event_categories.py

@@ -0,0 +1,91 @@
+import os
+import sys
+import django
+import openai
+import concurrent.futures
+import threading
+
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.development")
+django.setup()
+
+from apps.events.models import Events
+from shared.constants.event_categories import EVENT_CATEGORIES
+
+DEFAULT_CATEGORY = "Uncategorized"
+OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY")
+
+CATEGORY_LIST = "\n".join(f"- {cat}" for cat in EVENT_CATEGORIES)
+lock = threading.Lock()  # For Django ORM thread safety
+
+
+def get_categories_from_openai(title, description, event_id=None):
+    prompt = (
+        f"Given the following event title and description, select all applicable categories from this list (output as a JSON array of strings, case-sensitive, must use only these):\n"
+        f"{CATEGORY_LIST}\n\n"
+        f"Title: {title}\n"
+        f"Description: {description}\n\n"
+        f"Return a JSON array of one or more categories. If none fit, return ['{DEFAULT_CATEGORY}']."
+    )
+    try:
+        print(f"[{event_id}] Requesting categories from OpenAI...")
+        client = openai.OpenAI(api_key=OPENAI_API_KEY)
+        response = client.chat.completions.create(
+            model="gpt-3.5-turbo",
+            messages=[
+                {"role": "system", "content": "You are an expert event classifier."},
+                {"role": "user", "content": prompt},
+            ],
+            max_tokens=64,
+            temperature=0,
+        )
+        content = response.choices[0].message.content.strip()
+        print(f"[{event_id}] OpenAI response: {content}")
+        if content.startswith("```"):
+            content = content.split("```")[-2].strip() if "```" in content[3:] else content.replace("```json", "").replace("```", "").strip()
+        import json
+        cats = json.loads(content)
+        if isinstance(cats, list) and all(isinstance(c, str) for c in cats):
+            valid_cats = [c for c in cats if c in EVENT_CATEGORIES]
+            if valid_cats:
+                print(f"[{event_id}] Valid categories: {valid_cats}")
+                return valid_cats
+            else:
+                print(f"[{event_id}] No valid categories found in response, using default.")
+    except Exception as e:
+        print(f"[{event_id}] OpenAI error: {e}")
+    return [DEFAULT_CATEGORY]
+
+
+def process_event(event):
+    title = event.title or ""
+    description = event.description or ""
+    print(f"\nProcessing event (ID: {event.id})")
+    print(f"[{event.id}] Title: {title}")
+    print(f"[{event.id}] Description: {description[:100]}{'...' if len(description) > 100 else ''}")
+    cats = get_categories_from_openai(title, description, event_id=event.id)
+    with lock:
+        if event.categories != cats:
+            print(f"[{event.id}] Updating categories from {event.categories} to {cats}")
+            event.categories = cats
+            event.save(update_fields=["categories"])
+            print(f"[{event.id}] Updated successfully.")
+            return 1
+        else:
+            print(f"[{event.id}] Categories already up to date.")
+            return 0
+
+
+def main():
+    print("Starting event category backfill (only uncategorized events)...")
+    events = list(Events.objects.filter(categories=[DEFAULT_CATEGORY]))
+    print(f"Found {len(events)} uncategorized events to process.")
+    updated = 0
+    with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor:
+        results = list(executor.map(process_event, events))
+        updated = sum(results)
+    print(f"\nDone. Updated {updated} out of {len(events)} uncategorized events with OpenAI-categorized categories.")
+
+
+if __name__ == "__main__":
+    main()

backend/services/openai_service.py

@@ -19,6 +19,7 @@
 
 from scraping.logging_config import logger
 from shared.constants.emojis import EMOJI_CATEGORIES
+from shared.constants.event_categories import EVENT_CATEGORIES
 from utils.events_utils import clean_datetime
 from utils.date_utils import get_current_semester_end_time
 from datetime import timezone as pytimezone
@@ -120,9 +121,10 @@ def extract_events_from_caption(
         context_date = context_datetime.strftime("%Y-%m-%d")
         context_day = context_datetime.strftime("%A")
         context_time = context_datetime.strftime("%H:%M")
-        
+
         # Get current semester end time for inferring RRULE UNTIL dates
         semester_end_time = get_current_semester_end_time(school)
+        categories_str = "\n".join(f"- {cat}" for cat in EVENT_CATEGORIES)
 
         prompt = f"""
     Analyze the following Instagram caption and image and extract event information if it's an event post.
@@ -165,7 +167,8 @@ def extract_events_from_caption(
         "rdate": string,              // comma-separated datetime strings in format "YYYYMMDDTHHMMSS,YYYYMMDDTHHMMSS,..." (e.g., "20251113T170000,20251204T170000,20251218T170000")
         "school": string,
         "source_image_url": string,
-        "description": string
+        "description": string,
+        "categories": list            // one or more of the following, as a JSON array of strings: {categories_str}
     }}
 
     CONSOLIDATION RULES (VERY IMPORTANT):

backend/shared/constants/event_categories.py

@@ -0,0 +1,11 @@
+EVENT_CATEGORIES = [
+    "Academic",
+    "Career & Networking",
+    "Social & Games",
+    "Athletics",
+    "Creative Arts",
+    "Cultural",
+    "Religious",
+    "Advocacy & Causes",
+    "Sales & Fundraising"
+]