fix: add user-agent header to prevent WAF blocking (#382)

Author: reybahl

ferry/crawler/cas_request.py

@@ -9,6 +9,9 @@
 import httpx
 import requests
 
+# User-Agent header to avoid AWS WAF challenges
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"
+
 
 class RateLimitError(Exception):
     pass

ferry/crawler/evals/__init__.py

@@ -6,7 +6,7 @@
 from tqdm import tqdm
 
 from ferry.crawler.cache import load_cache_json, save_cache_json
-from ferry.crawler.cas_request import CASClient
+from ferry.crawler.cas_request import CASClient, USER_AGENT
 from ferry.crawler.classes.parse import ParsedCourse
 
 from .fetch import FetchError, fetch_course_evals
@@ -49,7 +49,7 @@ async def crawl_evals(
     print(f"Fetching course evals for valid seasons: {seasons}...")
 
     # initiate Yale client session to access evals
-    client = httpx.AsyncClient()
+    client = httpx.AsyncClient(headers={"User-Agent": USER_AGENT})
     cas_client = CASClient(cas_cookie=cas_cookie)
 
     # Season level is synchronous, following same logic as class fetcher
@@ -92,5 +92,6 @@ async def crawl_evals(
             data.sort(key=lambda x: x["crn"])
             save_cache_json(data_dir / "parsed_evaluations" / f"{season}.json", data)
 
+    await client.aclose()
     print("\033[F", end="")
     print(f"Fetching course evals for valid seasons: {seasons}... ✔")

main.py

@@ -8,6 +8,7 @@
 
 from ferry.args_parser import Args, get_args, parse_seasons_arg
 from ferry.crawler.cache import load_cache_json
+from ferry.crawler.cas_request import USER_AGENT
 from ferry.crawler.classes import crawl_classes
 from ferry.crawler.evals import crawl_evals
 from ferry.crawler.seasons import fetch_seasons
@@ -26,7 +27,7 @@ async def start_crawl(args: Args):
     classes = None
     # Initialize HTTPX client, only used for fetching classes (evals fetch
     # initializes its own client with CAS auth)
-    client = AsyncClient(timeout=None)
+    client = AsyncClient(timeout=None, headers={"User-Agent": USER_AGENT})
     if args.crawl_seasons:
         course_seasons = await fetch_seasons(
             data_dir=args.data_dir, client=client, use_cache=args.use_cache