Improved image download and event update processes

Author: ericahan22

backend/scraping/event_processor.py

@@ -127,13 +127,18 @@ async def process(self, posts_data, cutoff_date):
 
         # 2. Upload all images for each post (with carousel support)
         all_image_tasks = []
+        
+        async def _upload_image_bounded(url):
+            async with self.semaphore:
+                return await self._upload_image(url)
+
         for post in valid_posts:
             ig_handle = post.get("ownerUsername")
             shortcode = post.get("url", "").strip("/").split("/")[-1]
             logger.info(f"[{ig_handle}] [{shortcode}] Uploading images...")
             image_urls = _get_all_images(post)
             post["all_image_urls"] = image_urls
-            all_image_tasks.append([self._upload_image(img_url) for img_url in image_urls])
+            all_image_tasks.append([_upload_image_bounded(img_url) for img_url in image_urls])
 
         flat_tasks = [task for sublist in all_image_tasks for task in sublist]
         flat_results = await asyncio.gather(*flat_tasks, return_exceptions=True)
@@ -216,15 +221,23 @@ async def process(self, posts_data, cutoff_date):
 
                 club_type = await self._get_club_type(ig_handle)
                 try:
-                    success = await self._save_event(event_data, ig_handle, source_url, club_type)
+                    result = await self._save_event(event_data, ig_handle, source_url, club_type)
                 except Exception as e:
                     append_event_to_csv(event_data, ig_handle, source_url, added_to_db="error", club_type=club_type)
                     logger.error(f"[{ig_handle}] [{shortcode}] Error saving event: {e}")
                     continue
-                if success:
+
+                if result is True:
                     append_event_to_csv(event_data, ig_handle, source_url, added_to_db="success", club_type=club_type)
                     logger.info(f"[{ig_handle}] [{shortcode}] Saved event: '{event_data.get('title', '')}'")
                     saved_count += 1
+                elif result == "updated":
+                    append_event_to_csv(event_data, ig_handle, source_url, added_to_db="updated", club_type=club_type)
+                    logger.info(f"[{ig_handle}] [{shortcode}] Updated event: '{event_data.get('title', '')}'")
+                    saved_count += 1
+                elif result == "duplicate":
+                    append_event_to_csv(event_data, ig_handle, source_url, added_to_db="duplicate_post", club_type=club_type)
+                    logger.info(f"[{ig_handle}] [{shortcode}] Duplicate event (no changes): '{event_data.get('title', '')}'")
 
         logger.info(f"Processing complete. Saved {saved_count} new events.")
         return saved_count

backend/services/storage_service.py

@@ -53,13 +53,21 @@ def _validate_image(self, image_data: bytes) -> bool:
 
     def _download_image_from_url(self, image_url: str) -> bytes | None:
         """Download image from URL"""
-        try:
-            response = requests.get(image_url, timeout=60)
-            response.raise_for_status()
-            return response.content
-        except Exception as e:
-            logger.error(f"Failed to download image from {image_url}: {e}")
-            return None
+        import time
+        max_retries = 3
+        for attempt in range(max_retries):
+            try:
+                response = requests.get(image_url, timeout=60)
+                response.raise_for_status()
+                return response.content
+            except Exception as e:
+                if attempt == max_retries - 1:
+                    logger.error(f"Failed to download image from {image_url} after {max_retries} attempts: {e}")
+                    return None
+                wait_time = 2 ** attempt
+                logger.warning(f"Download failed (attempt {attempt + 1}/{max_retries}): {e}. Retrying in {wait_time}s...")
+                time.sleep(wait_time)
+        return None
 
     def upload_image_from_url(
         self, image_url: str, filename: str | None = None

backend/utils/scraping_utils.py

@@ -5,6 +5,7 @@
 from pathlib import Path
 
 from django.db import transaction
+from django.utils import timezone
 
 from utils.date_utils import parse_utc_datetime
 
@@ -92,6 +93,7 @@ def insert_event_to_db(event_data, ig_handle, source_url, club_type=None):
                 matched_event.source_image_url = source_image_url or None
                 matched_event.categories = categories
                 matched_event.source_url = source_url
+                matched_event.added_at = timezone.now() # Bump to top
                 matched_event.save()
 
                 # Delete old event dates and create new ones
@@ -201,7 +203,8 @@ def find_match(self, event_data, ig_handle=None, source_url=None):
         description = event_data.get("description") or ""
         occurrences = event_data.get("occurrences")
 
-        log_prefix = f"[{ig_handle}] [{source_url.split('/')[-1] if source_url else 'UNKNOWN'}]"
+        shortcode = source_url.strip("/").split("/")[-1] if source_url else "UNKNOWN"
+        log_prefix = f"[{ig_handle}] [{shortcode}]"
 
         if not occurrences:
             return False, None
@@ -418,4 +421,5 @@ def append_event_to_csv(
                 "status": "CONFIRMED",
             }
         )
-        logger.info(f"[{ig_handle}] [{source_url.split('/')[-1] if source_url else 'UNKNOWN'}] Event written to CSV: '{title}' - Status: {added_to_db}")
+        shortcode = source_url.strip("/").split("/")[-1] if source_url else "UNKNOWN"
+        logger.info(f"[{ig_handle}] [{shortcode}] Event written to CSV - Status: {added_to_db}")