refactor logging paths, fix django async issue

Author: ericahan22

.github/workflows/process-single-user.yml

@@ -75,7 +75,7 @@ jobs:
         with:
           name: logs-${{ github.run_number }}
           path: |
-            backend/scraping/logs/events_scraped.csv
-            backend/scraping/logs/scraping.log
+            backend/scraping/events_scraped.csv
+            backend/scraping/scraping.log
             backend/scraping/apify_raw_results.json
           if-no-files-found: 'ignore'

.github/workflows/update-events-data.yml

@@ -73,7 +73,7 @@ jobs:
         with:
           name: logs-${{ github.run_number }}
           path: |
-            backend/scraping/logs/events_scraped.csv
-            backend/scraping/logs/scraping.log
+            backend/scraping/events_scraped.csv
+            backend/scraping/scraping.log
             backend/scraping/apify_raw_results.json
           if-no-files-found: 'ignore'

backend/scraping/event_processor.py

@@ -1,7 +1,6 @@
 import asyncio
 import os
 import sys
-from functools import lru_cache
 
 # Set up Django
 if "django" not in sys.modules:
@@ -35,9 +34,8 @@ def __init__(self, concurrency=5):
         self.concurrency = concurrency
         self.semaphore = asyncio.Semaphore(concurrency)
 
-    @staticmethod
-    @lru_cache(maxsize=512)
-    def _get_club_type(ig_handle):
+    @sync_to_async(thread_sensitive=True)
+    def _get_club_type(self, ig_handle):
         try:
             return Clubs.objects.get(ig=ig_handle).club_type
         except Clubs.DoesNotExist:

backend/scraping/logging_config.py

@@ -2,7 +2,7 @@
 import sys
 from pathlib import Path
 
-LOG_DIR = Path("logs")
+LOG_DIR = Path(".")
 LOG_DIR.mkdir(exist_ok=True)
 LOG_FILE = LOG_DIR / "scraping.log"
 

backend/scraping/main.py

@@ -1,7 +1,9 @@
 import asyncio
+import json
 import os
 import sys
 from datetime import timedelta
+from pathlib import Path
 
 # 1. Setup Django
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -45,6 +47,10 @@ def main():
     else:
         posts = scraper.scrape(targets)
 
+    raw_path = Path(__file__).parent / "apify_raw_results.json"
+    with raw_path.open("w", encoding="utf-8") as f:
+        json.dump(posts, f, ensure_ascii=False, indent=2)
+
     if not posts:
         logger.info("No posts retrieved. Exiting.")
         sys.exit(0)

backend/utils/scraping_utils.py

@@ -207,10 +207,8 @@ def append_event_to_csv(
     added_to_db="success",
     club_type=None,
 ):
-    logs_dir = Path(__file__).parent / "logs"
-    logs_dir.mkdir(parents=True, exist_ok=True)
-    csv_file = logs_dir / "events_scraped.csv"
-
+    csv_file = Path(__file__).parent.parent / "scraping" / "events_scraped.csv"
+    csv_file.parent.mkdir(parents=True, exist_ok=True)
     file_exists = csv_file.exists()
 
     occurrences = event_data.get("occurrences", []) or []