updated find_fimilar_events to only show similar upcoming events

Author: Tony Qiu

backend/example/embedding_utils.py

@@ -8,32 +8,35 @@
 
 
 def find_similar_events(
-    embedding: list[float], threshold: float = 0.985, limit: int = None
+    embedding: list[float], threshold: float = 0.985, limit: int = None, min_date: str = None
 ) -> list[dict]:
     with connection.cursor() as cursor:
+        # Base query with date filtering first for performance
+        base_query = """
+            SELECT id, 1 - (embedding <=> %s::vector) as similarity
+            FROM events 
+            WHERE embedding IS NOT NULL
+        """
+        
+        params = [embedding]
+        
+        # Add date filter if provided to reduce search space
+        if min_date:
+            base_query += " AND date >= %s"
+            params.append(min_date)
+        
+        # Add similarity threshold
+        base_query += " AND 1 - (embedding <=> %s::vector) > %s"
+        params.extend([embedding, threshold])
+        
+        # Order and limit
+        base_query += " ORDER BY similarity DESC"
+        
         if limit is not None:
-            cursor.execute(
-                """
-                SELECT id, 1 - (embedding <=> %s::vector) as similarity
-                FROM events 
-                WHERE embedding IS NOT NULL
-                AND 1 - (embedding <=> %s::vector) > %s
-                ORDER BY similarity DESC
-                LIMIT %s
-            """,
-                [embedding, embedding, threshold, limit],
-            )
-        else:
-            cursor.execute(
-                """
-                SELECT id, 1 - (embedding <=> %s::vector) as similarity
-                FROM events 
-                WHERE embedding IS NOT NULL
-                AND 1 - (embedding <=> %s::vector) > %s
-                ORDER BY similarity DESC
-            """,
-                [embedding, embedding, threshold],
-            )
+            base_query += " LIMIT %s"
+            params.append(limit)
+        
+        cursor.execute(base_query, params)
 
         return [
             {"id": row[0], "similarity": float(row[1])} for row in cursor.fetchall()

backend/scraping/instagram_feed.py

@@ -19,9 +19,10 @@
 from dotenv import load_dotenv
 from instaloader import Instaloader
 
-from example.embedding_utils import is_duplicate_event
+from example.embedding_utils import is_duplicate_event, find_similar_events
 from example.models import Clubs, Events
 from services.openai_service import extract_events_from_caption, generate_embedding
+from django.db import connection
 from services.storage_service import upload_image_from_url
 
 USER_AGENTS = [
@@ -179,16 +180,31 @@ def insert_event_to_db(event_data, club_ig, post_url):
                 f"Club with handle {club_ig} not found, inserting event with null club_type"
             )
 
-        # Check duplicates using vector sim
-        logger.debug(f"Checking duplicates for event with data: {event_data}")
-        if is_duplicate_event(event_data):
-            logger.debug(f"Duplicate event found: {event_name} at {event_location}")
-            return False
-
-        # Generate embedding
         embedding = generate_embedding(event_data["description"])
 
-        # Create event using Django ORM
+        try:
+            # Pass event date as min_date to filter out past events first for performance
+            similar_events = find_similar_events(
+                embedding, threshold=0.90, limit=10, min_date=event_date
+            )
+            candidate_ids = [row["id"] for row in similar_events]
+            if candidate_ids:
+                for existing in Events.objects.filter(id__in=candidate_ids, date=event_date):
+                    if (
+                        (existing.location or "") == (event_location or "")
+                        and (existing.start_time or "") == (event_data.get("start_time") or "")
+                        and (
+                            (existing.end_time or None)
+                            == (event_data.get("end_time") or None)
+                        )
+                    ):
+                        logger.info(
+                            f"Deleting older duplicate event id={existing.id} before inserting refreshed version"
+                        )
+                        existing.delete()
+        except Exception as dedup_err:
+            logger.error(f"Duplicate check via utility failed: {dedup_err}")
+
         Events.objects.create(
             club_handle=club_ig,
             url=post_url,

frontend/src/hooks/useEvents.ts

@@ -69,17 +69,12 @@ const fetchEvents = async ({
   queryKey: string[];
 }): Promise<EventsResponse> => {
   const searchTerm = queryKey[1] || "";
-  const startDate = queryKey[2] || "";
 
   const params = new URLSearchParams();
 
   if (searchTerm) {
     params.append("search", searchTerm);
   }
-  
-  if (startDate) {
-    params.append("start_date", startDate);
-  }
 
   const queryString = params.toString() ? `?${params.toString()}` : "";
   const response = await fetch(
@@ -96,15 +91,10 @@ export function useEvents(view: "grid" | "calendar") {
   const [searchParams, setSearchParams] = useSearchParams();
   const searchTerm = searchParams.get("search") || "";
 
-  // Calculate start date based on view
-  const startDate = view === "grid" 
-    ? new Date().toISOString().split('T')[0] // YYYY-MM-DD format for today
-    : "";  
-
   const hasActiveFilters = searchTerm !== "";
 
   const { data, isLoading, error } = useQuery({
-    queryKey: ["events", searchTerm, startDate],
+    queryKey: ["events", searchTerm],
     queryFn: fetchEvents,
     refetchOnWindowFocus: false,
     enabled: hasActiveFilters,
@@ -136,9 +126,7 @@ export function useEvents(view: "grid" | "calendar") {
     if (searchTerm) {
       title = `${events.length} Found Events - Wat2Do`;
     } else {
-      title = view === "grid" 
-        ? `${events.length} Upcoming Events - Wat2Do`
-        : `${events.length} Total Events - Wat2Do`;
+      title = `${events.length} Upcoming Events - Wat2Do`;
     }
 
     if (!isLoadingData) {