fix events_scraped.csv path again + other minor fixes

Author: ericahan22

backend/scraping/instagram_feed.py

@@ -1,4 +1,4 @@
-from instaloader import *
+from instaloader import Instaloader
 from dotenv import load_dotenv
 import os
 import csv
@@ -8,10 +8,11 @@
 import psycopg2
 import logging
 import traceback
-from datetime import datetime
 import sys
 from fuzzywuzzy import fuzz
 import time
+from pathlib import Path
+
 
 logging.basicConfig(
     level=logging.DEBUG,
@@ -23,6 +24,21 @@
 )
 logger = logging.getLogger(__name__)
 
+
+# Load environment variables from .env file
+load_dotenv()
+
+# Get credentials from environment variables
+USERNAME = os.getenv("USERNAME")
+PASSWORD = os.getenv("PASSWORD")
+CSRFTOKEN = os.getenv("CSRFTOKEN")
+SESSIONID = os.getenv("SESSIONID")
+DS_USER_ID = os.getenv("DS_USER_ID")
+MID = os.getenv("MID")
+IG_DID = os.getenv("IG_DID")
+SUPABASE_DB_URL = os.getenv("SUPABASE_DB_URL")
+
+
 def get_post_image_url(post):
     try:
         if "image_versions2" in post._node and post._node["image_versions2"]:
@@ -56,23 +72,11 @@ def wrapper(*args, **kwargs):
     return wrapper
 
 
-# Load environment variables from .env file
-load_dotenv()
-
-# Get credentials from environment variables
-USERNAME = os.getenv("USERNAME")
-PASSWORD = os.getenv("PASSWORD")
-CSRFTOKEN = os.getenv("CSRFTOKEN")
-SESSIONID = os.getenv("SESSIONID")
-DS_USER_ID = os.getenv("DS_USER_ID")
-MID = os.getenv("MID")
-IG_DID = os.getenv("IG_DID")
-
-
 def append_event_to_csv(event_data, club_ig, post_url, status="success"):
-    csv_file = "backend/scraping/events_scraped.csv"
-    os.makedirs(os.path.dirname(csv_file), exist_ok=True)
-    file_exists = os.path.isfile(csv_file)
+    csv_file = Path(__file__).resolve().parent / "events_scraped.csv"
+    csv_file.parent.mkdir(parents=True, exist_ok=True)
+    file_exists = csv_file.exists()
+    
     with open(csv_file, "a", newline="", encoding="utf-8") as csvfile:
         fieldnames = [
             "club_handle", "url", "name", "date", "start_time", "end_time",
@@ -104,7 +108,7 @@ def insert_event_to_db(event_data, club_ig, post_url, sim_threshold=80):
     event_location = event_data.get("location") #.title()
     conn = None
     try:
-        conn = psycopg2.connect(os.getenv("SUPABASE_DB_URL"))
+        conn = psycopg2.connect(SUPABASE_DB_URL)
         cur = conn.cursor()
 
         # Check duplicates
@@ -146,6 +150,7 @@ def insert_event_to_db(event_data, club_ig, post_url, sim_threshold=80):
 
         try:
             append_event_to_csv(event_data, club_ig, post_url, status="success")
+            logger.info(f"Appended event to CSV: {event_data.get('name')}")
         except Exception as csv_err:
             logger.error(f"Database insert succeeded, but failed to append to CSV: {csv_err}")
             logger.error(f"Traceback: {traceback.format_exc()}")
@@ -159,6 +164,7 @@ def insert_event_to_db(event_data, club_ig, post_url, sim_threshold=80):
 
         try:
             append_event_to_csv(event_data, club_ig, post_url, status="failed")
+            logger.info(f"Appended event to CSV: {event_data.get('name')}")
         except Exception as csv_err:
             logger.error(f"Database insert failed, and failed to append to CSV: {csv_err}")
             logger.error(f"Traceback: {traceback.format_exc()}")
@@ -169,79 +175,72 @@ def insert_event_to_db(event_data, club_ig, post_url, sim_threshold=80):
             conn.close()
 
 
-def process_recent_feed(cutoff=datetime.now(timezone.utc) - timedelta(days=2), max_posts=100, max_consec_old_posts=3):
+def process_recent_feed(loader, cutoff=datetime.now(timezone.utc) - timedelta(days=2), max_posts=100, max_consec_old_posts=3):
     # Process Instagram feed posts and extract event info. Stops
     #   scraping once posts become older than cutoff.
-    try:
-        logger.info(f"Starting feed processing with cutoff: {cutoff}")
-        events_added = 0
-        posts_processed = 0
-        consec_old_posts = 0
-        s3_uploader = S3ImageUploader()  # Initialize S3 uploader
-
-        
-        for post in L.get_feed_posts():
-            try:
-                posts_processed += 1
-                logger.info("\n" + "-" * 50)
-                logger.info(f"Processing post: {post.shortcode} by {post.owner_username}")
-                
-                post_time = post.date_utc.replace(tzinfo=timezone.utc)
-                if post_time < cutoff:
-                    consec_old_posts += 1
-                    logger.debug(f"Post {post.shortcode} is older than cutoff ({post_time}), consecutive old posts: {consec_old_posts}")
-                    if consec_old_posts >= max_consec_old_posts:
-                        logger.info(f"Reached {max_consec_old_posts} consecutive old posts, stopping.")
-                        break
-                    continue # to next post
-                consec_old_posts = 0
-
-                if posts_processed >= max_posts:
-                    logger.info(f"Reached max post limit of {max_posts}, stopping.")
+    events_added = 0
+    posts_processed = 0
+    consec_old_posts = 0
+    s3_uploader = S3ImageUploader()  # Initialize S3 uploader
+    logger.info(f"Starting feed processing with cutoff: {cutoff}")
+    
+    for post in loader.get_feed_posts():
+        try:
+            posts_processed += 1
+            logger.info("\n" + "-" * 50)
+            logger.info(f"Processing post: {post.shortcode} by {post.owner_username}")
+            
+            post_time = post.date_utc.replace(tzinfo=timezone.utc)
+            if post_time < cutoff:
+                consec_old_posts += 1
+                if consec_old_posts >= max_consec_old_posts:
+                    logger.info(f"Reached {max_consec_old_posts} consecutive old posts, stopping.")
                     break
+                continue # to next post
+            consec_old_posts = 0
 
-                # Safely get image URL and upload to S3
-                raw_image_url = get_post_image_url(post)
-                if raw_image_url:
-                    image_url = s3_uploader.upload_image(raw_image_url)
-                    print(f"Uploaded image to S3: {image_url}")
-                else:
-                    logger.warning(f"No image URL found for post {post.shortcode}, skipping image upload")
-                    image_url = None
-                
-                event_data = parse_caption_for_event(post.caption, image_url)
-                
-                if event_data is None:
-                    logger.warning(f"AI client returned None for post {post.shortcode}")
-                    continue
-                
-                post_url = f"https://www.instagram.com/p/{post.shortcode}/"
-                if event_data.get("name") and event_data.get("date") and event_data.get("location") and event_data.get("start_time"):
-                    if insert_event_to_db(event_data, post.owner_username, post_url):
-                        events_added += 1
-                        logger.info(f"Successfully added event from {post.owner_username}")
-                else:
-                    missing_fields = [key for key in ['name', 'date', 'location', 'start_time'] if not event_data.get(key)]
-                    logger.warning(f"Missing required fields: {missing_fields}, skipping event")
-                time.sleep(5)
-            except Exception as e:
-                logger.error(f"Error processing post {post.shortcode} by {post.owner_username}: {str(e)}")
-                logger.error(f"Traceback: {traceback.format_exc()}")
-                continue # with next post
-        print(f"\n--- Summary ---")
-        print(f"Added {events_added} event(s) to Supabase")
-        logger.info(f"Feed processing completed. Processed {posts_processed} posts, added {events_added} events")
-    except Exception as e:
-        logger.error(f"Error in process_recent_feed: {str(e)}")
-        logger.error(f"Traceback: {traceback.format_exc()}")
-        raise
+            if posts_processed >= max_posts:
+                logger.info(f"Reached max post limit of {max_posts}, stopping.")
+                break
+
+            # Safely get image URL and upload to S3
+            raw_image_url = get_post_image_url(post)
+            if raw_image_url:
+                image_url = s3_uploader.upload_image(raw_image_url)
+                logger.info(f"Uploaded image to S3: {image_url}")
+            else:
+                logger.warning(f"No image URL found for post {post.shortcode}, skipping image upload")
+                image_url = None
+            
+            event_data = parse_caption_for_event(post.caption, image_url)
+            
+            if event_data is None:
+                logger.warning(f"AI client returned None for post {post.shortcode}")
+                continue
+            
+            post_url = f"https://www.instagram.com/p/{post.shortcode}/"
+            if event_data.get("name") and event_data.get("date") and event_data.get("location") and event_data.get("start_time"):
+                if insert_event_to_db(event_data, post.owner_username, post_url):
+                    events_added += 1
+                    logger.info(f"Successfully added event from {post.owner_username}")
+            else:
+                missing_fields = [key for key in ['name', 'date', 'location', 'start_time'] if not event_data.get(key)]
+                logger.warning(f"Missing required fields: {missing_fields}, skipping event")
+            time.sleep(5)
+        except Exception as e:
+            logger.error(f"Error processing post {post.shortcode} by {post.owner_username}: {str(e)}")
+            logger.error(f"Traceback: {traceback.format_exc()}")
+            continue # with next post
+    logger.info(f"Feed processing completed. Processed {posts_processed} posts, added {events_added} events")
+    logger.info(f"\n--- Summary ---")
+    logger.info(f"Added {events_added} event(s) to Supabase")
 
 
 @handle_instagram_errors
 def session():
     L = Instaloader()
+    logger.info("Attemping to load Instagram session...")
     try:
-        logger.info("Attemping to load Instagram session...")
         L.load_session(
             USERNAME,
             {
@@ -262,4 +261,4 @@ def session():
 
 if __name__ == "__main__":
     L = session()
-    process_recent_feed()
+    process_recent_feed(L)