Add proxy support and SSL handling in scraper.py

- Introduce proxy configuration in scraper.py to enhance web scraping capabilities
- Implement SSL context to bypass certificate verification for secure connections
- Update GitHub Actions workflow to include proxy URL as an environment variable
- Ensure compatibility with gzipped responses during HTML content retrieval

Author: Adriano Sanges

.github/workflows/run-project.yml

@@ -36,6 +36,7 @@ jobs:
           echo "telegram_bot_api_key=${{ secrets.telegram_bot_api_key }}" >> .env
           echo "chat_id=${{ secrets.chat_id }}" >> .env
           echo "chat_tag=${{ secrets.chat_tag }}" >> .env
+          echo "proxy_url=${{ secrets.proxy_url }}" >> .env
       - name: Run Python Script
         env:
           warehouse_name: ${{ secrets.warehouse_name }}
@@ -44,6 +45,7 @@ jobs:
           telegram_bot_api_key: ${{ secrets.telegram_bot_api_key }}
           chat_id: ${{ secrets.chat_id }}
           chat_tag: ${{ secrets.chat_tag }}
+          proxy_url: ${{ secrets.proxy_url }}
         run: |
           cd real-estate-etl
           uv run scan_properties.py

real-estate-etl/scraper.py

@@ -3,7 +3,10 @@
 import requests
 from bs4 import BeautifulSoup
 from typing import Optional, Dict, List
-
+import urllib.request
+import ssl
+import gzip
+import os
 
 def parse_price(price_str):
     if not price_str:
@@ -14,22 +17,45 @@ def parse_price(price_str):
 
 
 def parse_page(url: str) -> Dict[str, Optional[any]]:
-    headers = {
-        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
-        'Accept-Language': 'en-US,en;q=0.9',
-        'Accept-Encoding': 'gzip, deflate, br',
-        'Connection': 'keep-alive',
-        'Upgrade-Insecure-Requests': '1',
-        'Sec-Fetch-Dest': 'document',
-        'Sec-Fetch-Mode': 'navigate',
-        'Sec-Fetch-Site': 'none',
-        'Sec-Fetch-User': '?1',
-    }
     logging.debug("Parsing page: %s", url)
-    response = requests.get(url, headers=headers)
+
+    proxy_url = os.getenv('proxy_url')
 
-    soup = BeautifulSoup(response.text, 'html.parser')
+    # Create a context that doesn't verify certificates
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    
+    opener = urllib.request.build_opener(
+        urllib.request.ProxyHandler({
+            'http': proxy_url,
+            'https': proxy_url
+        }),
+        urllib.request.HTTPSHandler(context=ctx)  # Add the SSL context
+    )
+    opener.addheaders = [
+        ('User-Agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'),
+        ('Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8'),
+        ('Accept-Language', 'en-US,en;q=0.9'),
+        ('Accept-Encoding', 'gzip, deflate, br'),
+        ('Connection', 'keep-alive'),
+        ('Upgrade-Insecure-Requests', '1'),
+        ('Sec-Fetch-Dest', 'document'),
+        ('Sec-Fetch-Mode', 'navigate'),
+        ('Sec-Fetch-Site', 'none'),
+        ('Sec-Fetch-User', '?1'),
+    ]
+    
+    response = opener.open(url)
+    
+    # Check if response is gzipped
+    if response.info().get('Content-Encoding') == 'gzip':
+        import gzip
+        html_content = gzip.decompress(response.read()).decode('utf-8', errors='replace')
+    else:
+        html_content = response.read().decode('utf-8', errors='replace')
+    
+    soup = BeautifulSoup(html_content, 'html.parser')
 
     listings = soup.select('section.ld-layoutContentCenter')
     for listing in listings: