Refactor scraping for daily and live ingestion:

- Created a new `main.py` script to orchestrate the scraping process and handle both single and batch modes
- Deleted `process_single_user.py` and `instagram_feed.py` to use `main.py` instead as entry point
- Moved Apify-related functions to `apify_client.py`
- Added `post_processing.py` to manage the processing of scraped posts (replaces `instagram_feed.py`)
- Updated `urls_to_scrape.py` to only include most "important" clubs

Author: ericahan22

.github/workflows/process-single-user.yml

@@ -66,7 +66,7 @@ jobs:
       - name: Run Scraper for Single User
         working-directory: backend/scraping
         run: |
-          python -u process_single_user.py 2>&1 | tee logs/scraping.log
+          python -u main.py 2>&1 | tee logs/scraping.log
 
       - name: Upload logs as artifacts
         if: always()

.github/workflows/update-events-data.yml

@@ -64,7 +64,7 @@ jobs:
       - name: Run scraper
         working-directory: backend/scraping
         run: |
-          python -u instagram_feed.py 2>&1 | tee logs/scraping.log
+          python -u main.py 2>&1 | tee logs/scraping.log
         continue-on-error: false
 
       - name: Upload logs as artifacts

backend/scraping/apify_client.py

@@ -0,0 +1,50 @@
+import os
+import logging
+from datetime import timedelta
+from django.utils import timezone
+from apify_client import ApifyClient
+
+logger = logging.getLogger(__name__)
+
+APIFY_TOKEN = os.getenv("APIFY_API_TOKEN")
+APIFY_ACTOR = "apify/instagram-post-scraper"
+
+def run_apify_scraper(usernames=None, results_limit=None):
+    """
+    Run Apify actor for one or more usernames and return the results.
+    """
+    if not APIFY_TOKEN:
+        logger.critical("APIFY_API_TOKEN not set")
+        raise RuntimeError("APIFY_API_TOKEN not set")
+
+    cutoff_date = timezone.now() - timedelta(days=1)
+    cutoff_str = cutoff_date.strftime("%Y-%m-%d")
+
+    client = ApifyClient(APIFY_TOKEN)
+
+    input_data = {
+        "skipPinnedPosts": True,
+        "onlyPostsNewerThan": cutoff_str,
+    }
+
+    if usernames:
+        if isinstance(usernames, str):
+            usernames = [usernames]
+        input_data["username"] = usernames
+    
+    if results_limit is not None:
+        input_data["resultsLimit"] = results_limit
+
+    logger.info(f"Starting Apify actor with cutoff {cutoff_str} for: {usernames}")
+
+    run = client.actor(APIFY_ACTOR).call(run_input=input_data)
+
+    if not run:
+        logger.error("Apify run failed or returned no data.")
+        return []
+
+    logger.info(f"Apify run finished (ID: {run.get('id')}). Fetching dataset...")
+    dataset_items = client.dataset(run["defaultDatasetId"]).list_items().items
+    
+    logger.info(f"Retrieved {len(dataset_items)} items from Apify.")
+    return dataset_items