Merge pull request #456 from Center-for-AI-Innovation/fix-download-convo-larger-than-500

Fix download conversations larger than 500

Author: rohan-uiuc

.env.local

@@ -106,3 +106,11 @@ NEXT_PUBLIC_KEYCLOAK_CLIENT_ID=illinois_chat
 # NOMIC_API_KEY=
 # SENTRY_DSN=
 
+# =============================================================================
+# SMTP EMAIL CONFIGURATION
+# =============================================================================
+SES_HOST=
+EMAIL_SENDER=
+PASSWORD_SMTP=
+SES_PORT=
+USERNAME_SMTP=

.env.template

@@ -36,3 +36,10 @@ NUMEXPR_MAX_THREADS=2
 # Embeddings
 EMBEDDING_MODEL=Qwen/Qwen3-Embedding-8B
 EMBEDDING_API_BASE=
+
+# Email
+SES_HOST=
+EMAIL_SENDER=
+PASSWORD_SMTP=
+SES_PORT=
+USERNAME_SMTP=

ai_ta_backend/service/export_service.py

@@ -6,7 +6,7 @@
 import zipfile
 
 import pandas as pd
-import requests
+import redis
 from injector import inject
 
 from ai_ta_backend.database.aws import AWSStorage
@@ -89,7 +89,7 @@ def export_documents_json(self, course_name: str, from_date='', to_date=''):
 
         curr_doc_count = 0
         # create a temporary directory
-        temp_dir = tempfile.mkdtemp(prefix="export_")
+        temp_dir = tempfile.mkdtemp(prefix="export_", dir=tempfile.gettempdir())
 
         filename = course_name + '_' + str(uuid.uuid4()) + '_documents.jsonl'
         file_path = os.path.join(temp_dir, filename)
@@ -160,7 +160,7 @@ def export_convo_history_json(self, course_name: str, from_date='', to_date=''):
       last_id = response["data"][-1]['id']
       total_count = count
 
-      temp_dir = tempfile.mkdtemp(prefix="export_")
+      temp_dir = tempfile.mkdtemp(prefix="export_", dir=tempfile.gettempdir())
 
       filename = course_name[0:10] + '-convos.jsonl'
       file_path = os.path.join(temp_dir, filename)
@@ -231,7 +231,7 @@ def export_conversations(self, course_name: str, from_date: str, to_date: str, e
       last_id = response["data"][-1]['id']
       total_count = count
 
-      temp_dir = tempfile.mkdtemp(prefix="export_")
+      temp_dir = tempfile.mkdtemp(prefix="export_", dir=tempfile.gettempdir())
 
       filename = course_name[0:10] + '-convos.jsonl'
       file_path = os.path.join(temp_dir, filename)
@@ -538,13 +538,15 @@ def export_data_in_bg_extended(response, download_type, course_name, s3_path):
     os.remove(zip_file_path)
     s3_url = s3.generatePresignedUrl('get_object', os.environ['S3_BUCKET_NAME'], s3_path, 172800)
 
-    # Fetch course metadata to get admin emails
-    headers = {"Authorization": f"Bearer {os.environ['VERCEL_READ_ONLY_API_KEY']}", "Content-Type": "application/json"}
-    hget_url = str(os.environ['VERCEL_BASE_URL']) + "course_metadatas/" + course_name
-    response = requests.get(hget_url, headers=headers)
-    course_metadata = response.json()
-    course_metadata = json.loads(course_metadata['result'])
-    admin_emails = course_metadata['course_admins']
+    # get admin email IDs from Redis
+    print("Connecting to Redis... with url: ", os.environ['REDIS_URL'])
+    redis_client = redis.Redis.from_url(os.environ['REDIS_URL'], db=0)
+    course_metadata_json = redis_client.hget('course_metadatas', key=course_name)
+    if not course_metadata_json:
+      raise ValueError(f"No course metadata found in Redis for project '{course_name}'")
+    course_metadata = json.loads(course_metadata_json)
+    admin_emails = course_metadata.get('course_admins', [])
+
     bcc_emails = []
 
     # Handle specific email cases
@@ -584,7 +586,7 @@ def export_data_in_bg(response, download_type, course_name, s3_path):
 		response (dict): The response from the Supabase query.
 		download_type (str): The type of download - 'documents' or 'conversations'.
 		course_name (str): The name of the course.
-	  s3_path (str): The S3 path where the file will be uploaded.
+	    s3_path (str): The S3 path where the file will be uploaded.
 	"""
   s3 = AWSStorage()
   sql = SQLDatabase()
@@ -595,7 +597,7 @@ def export_data_in_bg(response, download_type, course_name, s3_path):
   print("pre-defined s3_path: ", s3_path)
 
   curr_doc_count = 0
-  temp_dir = tempfile.mkdtemp(prefix="export_")
+  temp_dir = tempfile.mkdtemp(prefix="export_", dir=tempfile.gettempdir())
 
   filename = s3_path.split('/')[-1].split('.')[0] + '.jsonl'
   file_path = os.path.join(temp_dir, filename)
@@ -641,17 +643,18 @@ def export_data_in_bg(response, download_type, course_name, s3_path):
     # generate presigned URL
     s3_url = s3.generatePresignedUrl('get_object', os.environ['S3_BUCKET_NAME'], s3_path, 172800)
 
-    # get admin email IDs
-    headers = {"Authorization": f"Bearer {os.environ['VERCEL_READ_ONLY_API_KEY']}", "Content-Type": "application/json"}
+    # get admin email IDs from Redis
+    print("Connecting to Redis... with url: ", os.environ['REDIS_URL'])
+    redis_client = redis.Redis.from_url(os.environ['REDIS_URL'], db=0)
+    course_metadata_json = redis_client.hget('course_metadatas', key=course_name)
+    if not course_metadata_json:
+      raise ValueError(f"No course metadata found in Redis for project '{course_name}'")
+    course_metadata = json.loads(course_metadata_json)
+    admin_emails = course_metadata.get('course_admins', [])
 
-    hget_url = str(os.environ['VERCEL_BASE_URL']) + "course_metadatas/" + course_name
-    response = requests.get(hget_url, headers=headers)
-    course_metadata = response.json()
-    course_metadata = json.loads(course_metadata['result'])
-    admin_emails = course_metadata['course_admins']
     bcc_emails = []
 
-    # check for Kastan's email and move to bcc
+    # check for dev's email and move to bcc
     if 'rohan13@illinois.edu' in admin_emails:
       admin_emails.remove('rohan13@illinois.edu')
       bcc_emails.append('rohan13@illinois.edu')
@@ -707,7 +710,7 @@ def export_data_in_bg_emails(response, download_type, course_name, s3_path, emai
 
   curr_doc_count = 0
 
-  temp_dir = tempfile.mkdtemp(prefix="export_")
+  temp_dir = tempfile.mkdtemp(prefix="export_", dir=tempfile.gettempdir())
 
   filename = s3_path.split('/')[-1].split('.')[0] + '.jsonl'
   file_path = os.path.join(temp_dir, filename)

ai_ta_backend/utils/export_utils.py

@@ -16,7 +16,7 @@ def _initialize_file_paths(course_name: str):
     base_name = _initialize_base_name(course_name)
 
     # Create a unique temp directory for this course_name
-    temp_dir = tempfile.mkdtemp(prefix=f"{course_name}_")
+    temp_dir = tempfile.mkdtemp(prefix=f"{course_name}_", dir=tempfile.gettempdir())
 
     file_paths = {
         "zip": os.path.join(temp_dir, base_name + ".zip"),
@@ -258,7 +258,7 @@ def _append_to_jsonl(convo_data, jsonl_file_path, error_log):
 
 
 def _create_zip(file_paths, error_log):
-  temp_dir = tempfile.mkdtemp(prefix="export_")
+  temp_dir = tempfile.mkdtemp(prefix="export_", dir=tempfile.gettempdir())
   zip_file_path = os.path.join(temp_dir, os.path.basename(file_paths['zip']))
   error_log_path = os.path.join(temp_dir, 'error.log')
   with open(error_log_path, 'w') as log_file:
@@ -286,7 +286,7 @@ def _create_zip(file_paths, error_log):
 
 
 def _create_zip_for_user_convo_export(markdown_dir, media_dir, error_log):
-  temp_dir = tempfile.mkdtemp(prefix="user_convo_export_")
+  temp_dir = tempfile.mkdtemp(prefix="user_convo_export_", dir=tempfile.gettempdir())
   zip_file_path = os.path.join(temp_dir, 'user_convo_export.zip')
   error_log_path = os.path.join(temp_dir, 'error.log')