Merge pull request #99 from ericahan22/zyte

Implement Zyte proxy setup

Author: ericahan22

.github/workflows/update-events-data.yml

@@ -13,13 +13,18 @@ on:
         required: true
         type: boolean
         default: false
+      MAX_POSTS:
+        required: false
+        type: number
+        default: 100
 
 jobs:
   instagram_feed:
     runs-on: ubuntu-latest
     permissions:
       contents: write
     env:
+      MAX_POSTS: ${{ github.event.inputs.MAX_POSTS || '100' }}
       SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }}
       POSTGRES_DB: ${{ secrets.POSTGRES_DB }}
       POSTGRES_USER: ${{ secrets.POSTGRES_USER }}
@@ -43,6 +48,7 @@ jobs:
       AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
       RESEND_FROM_EMAIL: ${{ secrets.RESEND_FROM_EMAIL }}
+      ZYTE_PROXY: ${{ secrets.ZYTE_PROXY }}
       EMAIL_ENCRYPTION_KEY: ${{ secrets.EMAIL_ENCRYPTION_KEY }}
       EMAIL_HASH_KEY: ${{ secrets.EMAIL_HASH_KEY }}
 
@@ -68,6 +74,14 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-pip-
 
+      - name: Cache Instaloader session
+        uses: actions/cache@v3
+        with:
+          path: .insta_cache
+          key: insta-session-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            insta-session-
+            
       - name: Install dependencies
         working-directory: backend
         run: |

backend/scraping/certs/zyte-ca.crt

@@ -0,0 +1,26 @@
+-----BEGIN CERTIFICATE-----
+MIIEUjCCAzqgAwIBAgIUTSEKJT9gFqPek0yW4Yq+QewypCgwDQYJKoZIhvcNAQEL
+BQAwgbkxCzAJBgNVBAYTAklFMRAwDgYDVQQIDAdNdW5zdGVyMQ0wCwYDVQQHDARD
+b3JrMRQwEgYDVQQKDAtTY3JhcGluZ0h1YjE1MDMGA1UECwwsTGVhZGluZyBUZWNo
+bm9sb2d5IGFuZCBQcm9mZXNzaW9uYWwgU2VydmljZXMxFDASBgNVBAMMC0NyYXds
+ZXJhIENBMSYwJAYJKoZIhvcNAQkBFhdzdXBwb3J0QHNjcmFwaW5naHViLmNvbTAe
+Fw0yNTA1MTYxODE3MTVaFw0zNTA1MTQxODE3MTVaMIG5MQswCQYDVQQGEwJJRTEQ
+MA4GA1UECAwHTXVuc3RlcjENMAsGA1UEBwwEQ29yazEUMBIGA1UECgwLU2NyYXBp
+bmdIdWIxNTAzBgNVBAsMLExlYWRpbmcgVGVjaG5vbG9neSBhbmQgUHJvZmVzc2lv
+bmFsIFNlcnZpY2VzMRQwEgYDVQQDDAtDcmF3bGVyYSBDQTEmMCQGCSqGSIb3DQEJ
+ARYXc3VwcG9ydEBzY3JhcGluZ2h1Yi5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQDcjecMfrYzsUdPocblk1L1wF5eYxloprFh2v7uKhKJVfOu4OQf
+e5foJBkuS9HYqkaJwc6d3QUzhLisdhChZzzZv9QaDCNB2LjsQPx+oI+6wRyCj/t5
+jfLZj8mK8RYqTp/P6AGW8QeMqPYTLNnsVoTFMQxBNN7bwSuhdM3JJRy1j2Omu0qu
+zyOMZd3axHoAcXUzo77kJP/mReG4M739D1N/KbvDEGxT+T7qGE0f5GRiU0S+S/R3
+y92S79kpMIG4v0M1OqBcW8CY6PQw0SN/mQWsX3mD6Tn4/E9TThM9S07BTfQZhsKM
+T55xiyhM4ODr6O5NCg2TdEW3QDTBlLeXLWudAgMBAAGjUDBOMB0GA1UdDgQWBBSf
+p9cc6kMixqzBMysrYPBw4cu1nTAfBgNVHSMEGDAWgBSfp9cc6kMixqzBMysrYPBw
+4cu1nTAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBmkvLIUf5O165u
+Sf5NqBhI5yrUe3xCYrNEyvYcaLb0KiJKaGQZLa+cyJZ1gDL+iVuX209TglbyLwXL
+M4Xl6YTERPF1KXrqmDMbsPJRX9nl/Ok1VdmNyY201fL1E8fZ1LFf8lUmgVhhtpwJ
+MZttrBmmovS9x7nQO+ITzZXG//VPewT3NasY1rvMWizfwtYiyALzCj5DAoEypY5V
+oP/DmHGOdpy2zy4uy00gMJot6KyLv0U4H9t8soe2+1PMZUSDlsB9CfSa2ORUXbcg
+hxy7ujs8hx4Zb+3f5AvpoydCHxlKsq9v1a6lHXLKD1UnsxPnlZ4N2P2VBkT7n+gp
+xwhGbUVi
+-----END CERTIFICATE-----

backend/scraping/instagram_feed.py

@@ -12,9 +12,10 @@
 import csv
 import logging
 import random
-import re
 import time
 import traceback
+import re
+import requests
 import json
 from datetime import datetime, timedelta, timezone
 from pathlib import Path
@@ -26,6 +27,8 @@
 from apps.events.models import Events
 from services.openai_service import extract_events_from_caption, generate_embedding
 from services.storage_service import upload_image_from_url
+from zyte_setup import setup_zyte
+from logging_config import logger
 from utils.embedding_utils import find_similar_events
 
 USER_AGENTS = [
@@ -36,23 +39,7 @@
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36",
 ]
 
-LOG_DIR = Path("logs")
-LOG_DIR.mkdir(exist_ok=True)
-LOG_FILE = LOG_DIR / "scraping.log"
-
-logging.getLogger("urllib3").setLevel(logging.WARNING)
-logging.getLogger("requests").setLevel(logging.WARNING)
-logging.basicConfig(
-    level=logging.DEBUG,
-    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
-    handlers=[
-        logging.StreamHandler(sys.stdout),
-        logging.FileHandler(LOG_FILE, encoding="utf-8"),
-    ],
-)
-logger = logging.getLogger(__name__)
-
-MAX_POSTS = 50
+MAX_POSTS = int(os.getenv("MAX_POSTS", "100"))
 MAX_CONSEC_OLD_POSTS = 10
 CUTOFF_DAYS = 2
 
@@ -429,10 +416,53 @@ def process_recent_feed(
     logger.info(f"Added {events_added} event(s) to Supabase")
 
 
+def test_zyte_proxy(country="CA"):
+    """
+    Patch requests.Session to route through Zyte with geolocation,
+    test Zyte proxy routing and geolocation
+    """
+    zyte_cert_path = setup_zyte()
+    zyte_proxy = os.getenv("ZYTE_PROXY")
+    os.environ['https_proxy'] = zyte_proxy
+    
+    old_request = requests.Session.request
+
+    def zyte_request(self, method, url, **kwargs):
+        headers = kwargs.get("headers", {})
+        headers["Zyte-Geolocation"] = country
+        kwargs["headers"] = headers
+        kwargs["verify"] = zyte_cert_path
+        kwargs["proxies"] = {"http": zyte_proxy, "https": zyte_proxy}
+        kwargs["timeout"] = kwargs.get("timeout", 60)
+        return old_request(self, method, url, **kwargs)
+
+    requests.Session.request = zyte_request
+    
+    logging.debug(f"Testing Zyte proxy geolocation: {country}")
+    try:
+        resp = requests.get(
+            "https://ipapi.co/json/",
+            timeout=15,
+            verify=zyte_cert_path)
+        resp.raise_for_status()
+        data = resp.json()
+        logging.debug(f"Connected via Zyte proxy")
+        logging.debug(f"Public IP: {data.get('ip')}")
+        logging.debug(f"Country: {data.get('country_name')} ({data.get('country')})")
+        logging.debug(f"City: {data.get('city')}")
+    except Exception as e:
+        print(f"Proxy geolocation test failed: {e}")
+        
+        
 @handle_instagram_errors
 def session():
     L = Instaloader(user_agent=random.choice(USER_AGENTS))
-    session_file = Path(__file__).resolve().parent.parent / ("session-" + USERNAME)
+    try:
+        SESSION_CACHE_DIR = Path(os.getenv("GITHUB_WORKSPACE", ".")) / ".insta_cache"
+        SESSION_CACHE_DIR.mkdir(exist_ok=True)
+        session_file = SESSION_CACHE_DIR / f"session-{USERNAME}"
+    except Exception as e:
+        session_file = Path(__file__).resolve().parent.parent / ("session-" + USERNAME)
     try:
         if session_file.exists():
             L.load_session_from_file(USERNAME, filename=str(session_file))
@@ -457,6 +487,7 @@ def session():
 
 
 if __name__ == "__main__":
+    test_zyte_proxy("CA")  
     logger.info("Attemping to load Instagram session...")
     L = session()
     if L:

backend/scraping/logging_config.py

@@ -0,0 +1,19 @@
+import logging
+import sys
+from pathlib import Path
+
+LOG_DIR = Path("logs")
+LOG_DIR.mkdir(exist_ok=True)
+LOG_FILE = LOG_DIR / "scraping.log"
+
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("requests").setLevel(logging.WARNING)
+logging.basicConfig(
+    level=logging.DEBUG,
+    format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler(LOG_FILE, encoding="utf-8"),
+    ],
+)
+logger = logging.getLogger(__name__)

backend/scraping/zyte_setup.py

@@ -0,0 +1,18 @@
+import os
+from pathlib import Path
+from logging_config import logger
+
+
+def setup_zyte():
+    """
+    Sets up Zyte CA certificate, sets environment variables for requests,
+    returns cert path
+    """
+    cert_path = Path(__file__).parent / "certs" / "zyte-ca.crt"
+    try:
+        os.environ["REQUESTS_CA_BUNDLE"] = str(cert_path)
+        os.environ["CURL_CA_BUNDLE"] = str(cert_path)
+        os.environ["SSL_CERT_FILE"] = str(cert_path)
+    except Exception as e:
+        logger.error(f"Failed to set Zyte cert env vars: {e}")
+    return cert_path