small improvements to scraping, processing, logging, workflows

Author: ericahan22

.github/workflows/process-single-user.yml

@@ -1,4 +1,5 @@
 name: Process Single User
+run-name: Process @${{ github.event.client_payload.username }}
 
 on:
   repository_dispatch:

backend/requirements.txt

@@ -9,7 +9,6 @@ pgvector==0.4.1
 django-ratelimit==3.0.1
 
 # Scraping and web utilities
---find-links=./wheels
 setuptools>=65.5.0
 apify_client
 python-dotenv

backend/scraping/event_processor.py

@@ -128,11 +128,16 @@ async def process(self, posts_data, cutoff_date):
         # 2. Upload all images for each post (with carousel support)
         all_image_tasks = []
         for post in valid_posts:
+            ig_handle = post.get("ownerUsername")
+            shortcode = post.get("url", "").strip("/").split("/")[-1]
+            logger.info(f"[{ig_handle}] [{shortcode}] Uploading images...")
             image_urls = _get_all_images(post)
             post["all_image_urls"] = image_urls
             all_image_tasks.append([self._upload_image(img_url) for img_url in image_urls])
+        
         flat_tasks = [task for sublist in all_image_tasks for task in sublist]
         flat_results = await asyncio.gather(*flat_tasks)
+        
         # Map uploaded S3 URLs back to posts
         idx = 0
         for post in valid_posts:
@@ -141,9 +146,11 @@ async def process(self, posts_data, cutoff_date):
             idx += n_imgs
 
         # 3. Extract Events
-        logger.info(f"[{ig_handle}] [{shortcode}] Extracting event data...")
         extract_tasks = []
         for post in valid_posts:
+            ig_handle = post.get("ownerUsername")
+            shortcode = post.get("url", "").strip("/").split("/")[-1]
+            logger.info(f"[{ig_handle}] [{shortcode}] Extracting event data...")
             extract_tasks.append(self._process_single_post_extraction({
                 **post,
                 "all_s3_urls": post["all_s3_urls"]

backend/scraping/logging_config.py

@@ -26,7 +26,7 @@ def setup_logging():
     logging.getLogger("httpx").setLevel(logging.WARNING)
     logging.getLogger("httpcore").setLevel(logging.WARNING)
     logging.getLogger("urllib3").setLevel(logging.WARNING)
-    fmt = "%(asctime)s - %(levelname)s - %(message)s"
+    fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
     handlers = [
         logging.StreamHandler(sys.stderr),
         logging.FileHandler(LOG_FILE, encoding="utf-8"),

backend/scraping/main.py

@@ -78,7 +78,7 @@ def main():
             sys.exit(0)
         else:
             logger.info("No new events were added")
-            sys.exit(2)
+            sys.exit(0)
     except Exception as e:
         logger.error(f"Critical error in processing: {e}", exc_info=True)
         sys.exit(1)

backend/utils/scraping_utils.py

@@ -150,10 +150,20 @@ def is_duplicate_event(event_data):
     if not target_start:
         return False
 
+    from datetime import timedelta
+    day_start = target_start.replace(hour=0, minute=0, second=0, microsecond=0)
+    day_end = day_start + timedelta(days=1)
+
     try:
         candidates = EventDates.objects.select_related("event").filter(
-            dtstart_utc__date=target_start.date()
+            dtstart_utc__gte=day_start,
+            dtstart_utc__lt=day_end
         )
+        if candidates:
+            logger.debug(f"Found {len(candidates)} existing events on {day_start.date()} for duplicate check.")
+            for i, cand in enumerate(candidates[:3]):
+                evt = cand.event
+                logger.debug(f"  Candidate #{i+1}: '{evt.title}' @ {cand.dtstart_utc}")
 
         for candidate in candidates:
             existing_event = candidate.event
@@ -168,10 +178,6 @@ def is_duplicate_event(event_data):
             if not c_start:
                 continue
 
-            # Compare same-day occurrences with fuzzy matching on title/location/description.
-            if c_start.date() != target_start.date():
-                continue
-
             norm_title = normalize(title)
             norm_c_title = normalize(c_title)
             substring_match = norm_c_title in norm_title or norm_title in norm_c_title