delete s3 file in scraper if not event

Author: tonyqiu123

backend/scraping/instagram_feed.py

@@ -22,7 +22,7 @@
 
 from example.embedding_utils import generate_event_embedding, is_duplicate_event
 from services.openai_service import extract_event_from_caption
-from services.storage_service import upload_image_from_url
+from services.storage_service import upload_image_from_url, delete_image
 
 
 USER_AGENTS = [
@@ -110,6 +110,12 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
+def extract_s3_filename_from_url(image_url: str) -> str:
+    if not image_url:
+        return None
+    filename = image_url.split('/')[-1]
+    return f"events/{filename}"
+
 def append_event_to_csv(event_data, club_ig, post_url, status="success", embedding=None):
     csv_file = Path(__file__).resolve().parent / "events_scraped.csv"
     csv_file.parent.mkdir(parents=True, exist_ok=True)
@@ -301,6 +307,11 @@ def process_recent_feed(
             event_data = extract_event_from_caption(post.caption, image_url)
             if event_data is None:
                 logger.warning(f"AI client returned None for post {post.shortcode}")
+                # Delete uploaded S3 file if event extraction failed
+                if image_url:
+                    s3_filename = extract_s3_filename_from_url(image_url)
+                    if s3_filename and delete_image(s3_filename):
+                        logger.info(f"Deleted S3 file for failed event extraction: {s3_filename}")
                 continue
 
             post_url = f"https://www.instagram.com/p/{post.shortcode}/"
@@ -313,6 +324,12 @@ def process_recent_feed(
                 if insert_event_to_db(event_data, post.owner_username, post_url):
                     events_added += 1
                     logger.info(f"Successfully added event from {post.owner_username}")
+                else:
+                    # Event failed to insert to DB, delete S3 file
+                    if image_url:
+                        s3_filename = extract_s3_filename_from_url(image_url)
+                        if s3_filename and delete_image(s3_filename):
+                            logger.info(f"Deleted S3 file for failed DB insert: {s3_filename}")
             else:
                 missing_fields = [
                     key
@@ -323,6 +340,11 @@ def process_recent_feed(
                     f"Missing required fields: {missing_fields}, skipping event"
                 )
                 embedding = generate_event_embedding(event_data)
+                # Delete S3 file for events with missing required fields
+                if image_url:
+                    s3_filename = extract_s3_filename_from_url(image_url)
+                    if s3_filename and delete_image(s3_filename):
+                        logger.info(f"Deleted S3 file for event with missing fields: {s3_filename}")
                 append_event_to_csv(
                     event_data, post.owner_username, post_url, status="missing_fields", embedding=embedding
                 )