implement single user scraping as separate workflow

Author: ericahan22

.github/workflows/process-single-user.yml

@@ -0,0 +1,80 @@
+name: Process Single User
+
+on:
+  repository_dispatch:
+    types: [new_instagram_post]
+
+jobs:
+  process-post:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    env:
+      # --- Django & App Config ---
+      PRODUCTION: '1'
+      DJANGO_SETTINGS_MODULE: 'config.settings.development'
+      SECRET_KEY: ${{ secrets.SECRET_KEY }}
+      CLERK_SECRET_KEY: ${{ secrets.CLERK_SECRET_KEY }}
+      # --- Database Config ---
+      DATABASE_URL: ${{ secrets.SUPABASE_DB_URL }}
+      SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }}
+      POSTGRES_DB: ${{ secrets.POSTGRES_DB }}
+      POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
+      POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+      POSTGRES_HOST: ${{ secrets.POSTGRES_HOST }}
+      POSTGRES_PORT: ${{ secrets.POSTGRES_PORT }}
+      # --- Service Keys ---
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_S3_BUCKET_NAME: ${{ secrets.AWS_S3_BUCKET_NAME }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
+      RESEND_FROM_EMAIL: ${{ secrets.RESEND_FROM_EMAIL }}
+      EMAIL_ENCRYPTION_KEY: ${{ secrets.EMAIL_ENCRYPTION_KEY }}
+      EMAIL_HASH_KEY: ${{ secrets.EMAIL_HASH_KEY }}
+      # --- Apify Token ---
+      APIFY_API_TOKEN: ${{ secrets.APIFY_API_TOKEN }}
+      TARGET_USERNAME: ${{ github.event.client_payload.username }}
+      MAX_CONCURRENT_TASKS: 1
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+
+      - name: Create logs directory
+        working-directory: backend/scraping
+        run: mkdir -p logs
+
+      - name: Cache pip
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('backend/requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        working-directory: backend
+        run: |
+          pip install --prefer-binary -r requirements.txt
+
+      - name: Run Scraper for Single User
+        working-directory: backend/scraping
+        run: |
+          python -u process_single_user.py 2>&1 | tee logs/scraping.log
+
+      - name: Upload logs as artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-${{ github.run_number }}
+          path: |
+            backend/scraping/logs/events_scraped.csv
+            backend/scraping/logs/scraping.log
+            backend/scraping/apify_raw_results.json
+          if-no-files-found: 'ignore'

.github/workflows/update-events-data.yml

@@ -3,12 +3,6 @@ name: Scrape Instagram, Update Events DB, Update Static Data
 on:
   schedule:
     - cron: '0 13 * * *'  # 8am EST (UTC-5)
-  workflow_dispatch:
-    inputs:
-      username:
-        description: 'Instagram username to scrape'
-        required: true
-        type: string
 
 jobs:
   instagram_feed:
@@ -69,8 +63,6 @@ jobs:
 
       - name: Run scraper
         working-directory: backend/scraping
-        env:
-          SCRAPE_USERNAME: ${{ github.event.inputs.username }}
         run: |
           python -u instagram_feed.py 2>&1 | tee logs/scraping.log
         continue-on-error: false

backend/scraping/instagram_feed.py

@@ -305,28 +305,22 @@ def get_seen_shortcodes():
         return set()
 
 
-def get_apify_input():
-    """
-    Builds the Apify actor input JSON for apify/instagram-post-scraper.
-    """
+def get_apify_input(username=None):
     cutoff_date = timezone.now() - timedelta(days=CUTOFF_DAYS)
     cutoff_str = cutoff_date.strftime("%Y-%m-%d")
     logger.info(f"Setting post cutoff date to {cutoff_str} ({CUTOFF_DAYS} day ago)")
 
-    # Scrape single username if provided via env
-    single_username = os.getenv("SCRAPE_USERNAME")
-    if single_username:
-        usernames = [single_username]
-        logger.info(f"Scraping @{single_username}")
+    if username:
+        usernames = [username]
+        logger.info(f"Scraping @{username}")
     else:
-        # Parse usernames from URLs
         usernames = []
         for url in FULL_URLS:
             try:
                 clean_url = url.split("instagram.com/")[1]
-                username = clean_url.split("/")[0]
-                if username and username not in usernames:
-                    usernames.append(username)
+                uname = clean_url.split("/")[0]
+                if uname and uname not in usernames:
+                    usernames.append(uname)
             except Exception:
                 logger.warning(f"Could not parse username from URL: {url}")
 
@@ -557,10 +551,11 @@ async def process_scraped_posts(posts_data, cutoff_date):
     logger.info(f"Added {total_events_added} event(s) to Supabase")
 
 
-def run_apify_scraper():
+def run_apify_scraper(username=None):
     """
     Initializes Apify client, runs the Instagram scraper,
     saves the raw results, and processes them.
+    If username is provided, only scrape that user.
     """
     if not APIFY_API_TOKEN:
         logger.critical("APIFY_API_TOKEN not found in environment. Aborting.")
@@ -569,7 +564,7 @@ def run_apify_scraper():
     posts_data = []
     try:
         client = ApifyClient(APIFY_API_TOKEN)
-        actor_input = get_apify_input()
+        actor_input = get_apify_input(username)
         logger.info("Starting Apify actor 'apify/instagram-post-scraper'...")
         run = client.actor("apify/instagram-post-scraper").call(run_input=actor_input)
         logger.info(f"Apify run started (ID: {run['id']}). Waiting for results...")

backend/scraping/process_single_user.py

@@ -0,0 +1,39 @@
+import os
+import sys
+import django
+
+# Setup Django 
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.development")
+django.setup()
+
+from scraping.logging_config import logger
+from scraping.instagram_feed import run_apify_scraper, process_scraped_posts
+from django.utils import timezone
+from datetime import timedelta
+
+
+def main():
+    target_user = os.environ.get("TARGET_USERNAME")
+    if not target_user:
+        logger.error("No TARGET_USERNAME provided.")
+        sys.exit(1)
+
+    logger.info(f"Scraping @{target_user}...")
+    posts_data = run_apify_scraper(username=target_user)
+    if not posts_data:
+        logger.warning("No posts found.")
+        return
+
+    cutoff_date = timezone.now() - timedelta(days=2)
+    import asyncio
+    try:
+        asyncio.run(process_scraped_posts(posts_data, cutoff_date))
+        logger.info("Done.")
+    except Exception as e:
+        logger.error(f"Error during processing: {e}")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()