fix datetime handling (django > built-in), small improvements to logging in feed processing

Author: ericahan22

.gitignore

@@ -26,6 +26,7 @@ node_modules/
 # IDE/editor settings
 .vscode/
 chrome_profile/
+.flake8
 
 # SQLite database (for local dev only)
 db.sqlite3

backend/scraping/instagram_feed.py

@@ -14,14 +14,15 @@
 import re
 import time
 import traceback
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 from pathlib import Path
 
 import requests
 from dotenv import load_dotenv
 from instaloader import Instaloader
 from logging_config import logger
 from zyte_setup import setup_zyte
+from django.utils import timezone
 
 from apps.clubs.models import Clubs
 from apps.events.models import Events
@@ -124,58 +125,85 @@ def append_event_to_csv(
 
     dtstart = event_data.get("dtstart", "")
     dtend = event_data.get("dtend", "")
-    dtstart_utc, dtend_utc, duration, all_day = tz_compute(dtstart, dtend)
+    dtstart_utc = event_data.get("dtstart_utc", "")
+    dtend_utc = event_data.get("dtend_utc", "")
+    duration = event_data.get("duration", "")
+    all_day = event_data.get("all_day", False)
+    location = event_data.get("location", "")
+    food = event_data.get("food", "")
+    price = event_data.get("price", "")
+    registration = bool(event_data.get("registration", False))
+    description = event_data.get("description", "")
+    rrule = event_data.get("rrule", "")
+    latitude = event_data.get("latitude", None)
+    longitude = event_data.get("longitude", None)
+    tz = event_data.get("tz", "")
+    school = event_data.get("school", "")
+    source_image_url = event_data.get("source_image_url", "")
+    title = event_data.get("title", "")
+
+    fieldnames = [
+        "ig_handle",
+        "title",
+        "source_url",
+        "dtstart",
+        "dtstart_utc",
+        "dtend",
+        "dtend_utc",
+        "duration",
+        "location",
+        "food",
+        "price",
+        "registration",
+        "description",
+        "rrule",
+        "latitude",
+        "longitude",
+        "tz",
+        "school",
+        "source_image_url",
+        "all_day",
+        "club_type",
+        "raw_json",
+        "added_to_db",
+        "status",
+        "embedding",
+    ]
 
     with open(csv_file, "a", newline="", encoding="utf-8") as csvfile:
-        fieldnames = [
-            "ig_handle",
-            "title",
-            "source_url",
-            "dtstart",
-            "dtstart_utc",
-            "dtend",
-            "dtend_utc",
-            "duration",
-            "location",
-            "food",
-            "price",
-            "registration",
-            "description",
-            "reactions",
-            "embedding",
-            "source_image_url",
-            "all_day",
-            "club_type",
-            "raw_json",
-            "added_to_db",
-        ]
-        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
+        writer = csv.DictWriter(csvfile, fieldnames=fieldnames, lineterminator="\n")
         if not file_exists:
             writer.writeheader()
         writer.writerow(
             {
                 "ig_handle": ig_handle,
-                "title": event_data.get("title"),
+                "title": title,
                 "source_url": source_url,
                 "dtstart": dtstart,
                 "dtstart_utc": dtstart_utc,
                 "dtend": dtend,
                 "dtend_utc": dtend_utc,
                 "duration": duration,
-                "location": event_data.get("location"),
-                "food": event_data.get("food", ""),
-                "price": event_data.get("price", ""),
-                "registration": bool(event_data.get("registration", False)),
-                "description": event_data.get("description", ""),
-                "reactions": json.dumps(event_data.get("reactions") or {}),
-                "embedding": embedding or "",
-                "source_image_url": event_data.get("source_image_url") or "",
+                "location": location,
+                "food": food,
+                "price": price,
+                "registration": registration,
+                "description": description,
+                "rrule": rrule,
+                "latitude": latitude,
+                "longitude": longitude,
+                "tz": tz,
+                "school": school,
+                "source_image_url": source_image_url,
                 "all_day": all_day,
                 "club_type": club_type or event_data.get("club_type") or "",
                 "raw_json": json.dumps(event_data, ensure_ascii=False),
                 "added_to_db": added_to_db,
+                "status": "CONFIRMED",
+                "embedding": embedding or "",
             }
         )
+        logger.info(f"Event written to CSV with status: {added_to_db}")
 
 
 def insert_event_to_db(event_data, ig_handle, source_url):
@@ -271,17 +299,21 @@ def insert_event_to_db(event_data, ig_handle, source_url):
             embedding=embedding,
             club_type=club_type,
         )
+        logger.info("Event added successfully")
         return True
     except Exception as e:
-        logger.error(f"Error inserting event to db: {e}")
-        append_event_to_csv(
-            event_data,
-            ig_handle,
-            source_url,
-            added_to_db="failed",
-            embedding=embedding,
-            club_type=club_type,
-        )
+        logger.error(f"Error inserting event to DB: {e}")
+        try:
+            append_event_to_csv(
+                event_data,
+                ig_handle,
+                source_url,
+                added_to_db="failed",
+                embedding=embedding,
+                club_type=club_type,
+            )
+        except Exception as csv_e:
+            logger.error(f"Error writing event to CSV after DB failure: {csv_e}")
         return False
 
 
@@ -327,7 +359,7 @@ def process_recent_feed(
                 if post.shortcode in seen_shortcodes or post_time < cutoff:
                     consec_old_posts += 1
                     logger.debug(
-                        f"Skipping post {post.shortcode}; consec_old_posts={consec_old_posts}"
+                        f"[{post.shortcode}] [{post.owner_username}] Skipping post; consec_old_posts={consec_old_posts}"
                     )
                     if consec_old_posts >= max_consec_old_posts:
                         termination_reason = (
@@ -342,19 +374,17 @@ def process_recent_feed(
                 consec_old_posts = 0
                 posts_processed += 1
                 logger.info("-" * 100)
-                logger.info(
-                    f"Processing post: {post.shortcode} by {post.owner_username}"
-                )
+                logger.info(f"[{post.shortcode}] [{post.owner_username}] Processing post")
 
                 # Safely get image URL and upload to S3
                 raw_image_url = get_post_image_url(post)
                 if raw_image_url:
                     time.sleep(random.uniform(1, 3))
                     source_image_url = upload_image_from_url(raw_image_url)
-                    logger.info(f"Uploaded image to S3: {source_image_url}")
+                    logger.info(f"[{post.shortcode}] [{post.owner_username}] Uploaded image to S3: {source_image_url}")
                 else:
                     logger.warning(
-                        f"No image URL found for post {post.shortcode}, skipping image upload"
+                        f"[{post.shortcode}] [{post.owner_username}] No image URL found for post, skipping image upload"
                     )
                     source_image_url = None
 
@@ -363,14 +393,15 @@ def process_recent_feed(
                 )
                 if not events_data or len(events_data) == 0:
                     logger.warning(
-                        f"AI client returned no events for post {post.shortcode}"
+                        f"[{post.shortcode}] [{post.owner_username}] AI client returned no events for post"
                     )
                     if posts_processed >= max_posts:
                         termination_reason = f"reached_max_posts={max_posts}"
                         logger.info(f"Reached max post limit of {max_posts}, stopping")
                         break
                     continue
 
+                logger.debug(f"[{post.shortcode}] [{post.owner_username}] Event data: {json.dumps(events_data, ensure_ascii=False, separators=(',', ':'))}")
                 source_url = f"https://www.instagram.com/p/{post.shortcode}/"
                 today = datetime.now(timezone.utc).date()
 
@@ -388,7 +419,7 @@ def process_recent_feed(
                             if not event_data.get(key)
                         ]
                         logger.warning(
-                            f"Missing required fields for event '{event_data.get('title', 'Unknown')}': {missing_fields}, skipping event"
+                            f"[{post.shortcode}] [{post.owner_username}] Missing required fields for event '{event_data.get('title', 'Unknown')}': {missing_fields}, skipping event"
                         )
                         embedding = generate_event_embedding(event_data)
                         append_event_to_csv(
@@ -403,18 +434,18 @@ def process_recent_feed(
                     date = datetime.fromisoformat(event_data.get("dtstart")).date()
                     if date < today:
                         logger.info(
-                            f"Skipping event '{event_data.get('title')}' with past date {date}"
+                            f"[{post.shortcode}] [{post.owner_username}] Skipping event '{event_data.get('title')}' with past date {date}"
                         )
                         continue
 
                     if insert_event_to_db(event_data, post.owner_username, source_url):
                         events_added += 1
                         logger.info(
-                            f"Successfully added event '{event_data.get('title')}' from {post.owner_username}"
+                            f"[{post.shortcode}] [{post.owner_username}] Successfully added event '{event_data.get('title')}'"
                         )
                     else:
                         logger.error(
-                            f"Failed to add event '{event_data.get('title')}' from {post.owner_username}"
+                            f"[{post.shortcode}] [{post.owner_username}] Failed to add event '{event_data.get('title')}'"
                         )
 
                 if posts_processed >= max_posts:
@@ -426,9 +457,9 @@ def process_recent_feed(
 
             except Exception as e:
                 logger.error(
-                    f"Error processing post {post.shortcode} by {post.owner_username}: {e!s}"
+                    f"[{post.shortcode}] [{post.owner_username}] Error processing post: {e!s}"
                 )
-                logger.error(f"Traceback: {traceback.format_exc()}")
+                logger.error(f"[{post.shortcode}] [{post.owner_username}] Traceback: {traceback.format_exc()}")
                 time.sleep(random.uniform(3, 8))
                 continue
 

backend/scraping/logging_config.py

@@ -12,8 +12,9 @@
     logging.getLogger("requests").setLevel(logging.WARNING)
     logging.getLogger("botocore").setLevel(logging.WARNING)
     logging.getLogger("httpcore").setLevel(logging.WARNING)
+    logging.getLogger("openai").setLevel(logging.WARNING)
 
-    fmt = "%(asctime)s - pid=%(process)d - thread=%(threadName)s - %(name)s - %(levelname)s - %(message)s"
+    fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     handlers = [
         logging.StreamHandler(sys.stdout),
         logging.FileHandler(LOG_FILE, encoding="utf-8"),

backend/services/openai_service.py

@@ -136,7 +136,7 @@ def extract_events_from_caption(
     - Title-case event titles (e.g., "...talk" -> "...Talk", "COFFEE CRAWL" -> "Coffee Crawl")
     - If multiple dates are mentioned (e.g., "Friday and Saturday"), create separate events for each date
     - If recurring events are mentioned (e.g., "every Friday"), just create one event
-    - For dtstart and dtend, if year not found, assume {now.year}
+    - For dtstart and dtend, if year not found, assume {now.year}. If an event's end time is earlier than its start time (e.g., "from 7 pm - 12 am"), interpret end time as being on the NEXT calendar day
     - For dtstart_utc and dtend_utc: convert the local time (dtstart/dtend) to UTC using the timezone (tz). Format as YYYY-MM-DD HH:MM:SSZ. If dtstart/dtend are empty, then dtstart_utc/dtend_utc should also be empty
     - For duration: calculate the duration between dtstart and dtend in HH:MM:SS format (e.g., "02:30:00" for 2 hours 30 minutes). If dtend is empty or not available, use empty string ""
     - When interpreting relative terms like "tonight", "tomorrow", "weekly", "every Friday", use the current date context above and the date the post was made. If an explicit date is found in the image, use that date
@@ -365,7 +365,7 @@ def generate_recommended_filters(self, events_data: list[dict]) -> list[list[str
 Generate filter keywords that:
 1. Capture the most common themes in the events titles data above that are actually found as a string within the events data above. if you do something that's 2 words, it better be fully included in the event data above.
 2. Are SHORT (1-3 words max) and SPECIFIC
-3. Reflect actual themes in the event data above  
+3. Reflect actual themes in the event data above
 4. The list MUST start with ["Food%20and%20Drink", "Pizza", "free food"]
 5. The Filter string MUST exist in atleast 3 event titles above.