Merge pull request #423 from Center-for-AI-Innovation/canvas-fixes

Canvas fixes

Author: max-zilla

ai_ta_backend/beam/canvas_ingest.py

@@ -313,7 +313,6 @@ def ingest_course_content(self,
     4. Delete extracted files from local directory
     """
 
-    print("In ingest_course_content")
     try:
       # a dictionary of all contents we want to ingest - files, pages, modules, syllabus, assignments, discussions.
       if content_ingest_dict is None:

ai_ta_backend/database/aws.py

@@ -11,7 +11,9 @@ def __init__(self):
     # S3
     self.s3_client = boto3.client(
         's3',
-        # endpoint_url=os.environ.get('MINIO_API_URL'),  # for Self hosted MinIO bucket
+        # aws_access_key_id=os.getenv('AWS_ACCESS_KEY_ID'),
+        # aws_secret_access_key=os.getenv('AWS_SECRET_ACCESS_KEY'),
+        # endpoint_url=os.environ.get('MINIO_URL'),  # for Self hosted MinIO bucket
     )
 
   def upload_file(self, file_path: str, bucket_name: str, object_name: str):

ai_ta_backend/main.py

@@ -228,8 +228,8 @@ def delete(service: RetrievalService, flaskExecutor: ExecutorInterface):
   start_time = time.monotonic()
   # background execution of tasks!!
   flaskExecutor.submit(service.delete_data, course_name, s3_path, source_url)
-  print(f"From {course_name}, deleted file: {s3_path}")
-  print(f"⏰ Runtime of FULL delete func: {(time.monotonic() - start_time):.2f} seconds")
+  logging.info(f"From {course_name}, deleted file: {s3_path}")
+  logging.debug(f"⏰ Runtime of FULL delete func: {(time.monotonic() - start_time):.2f} seconds")
   # we need instant return. Delets are "best effort" assume always successful... sigh :(
   response = jsonify({"outcome": 'success'})
   response.headers.add('Access-Control-Allow-Origin', '*')
@@ -744,28 +744,31 @@ def ingest() -> Response:
 
   # TODO: Authentication?
 
-  result = active_queue.addJobToIngestQueue(data)
-  logging.info("Result from addJobToIngestQueue:  %s", result)
+  job_id = active_queue.addJobToIngestQueue(data)
+  logging.info("Result from addJobToIngestQueue:  %s", job_id)
 
   response = jsonify(
     {
       "outcome": f'Queued Ingest task',
-      "task_id": result
+      "task_id": job_id
     }
   )
   response.headers.add('Access-Control-Allow-Origin', '*')
   return response
 
 @app.route('/canvas_ingest', methods=['POST'])
-def canvas_ingest(request) -> Response:
-  course_name: request.data.get('course_name', '')
-  canvas_url: request.data.get('canvas_url', None)
-  files: bool = request.data.get('files', True)
-  pages: bool = request.data.get('pages', True)
-  modules: bool = request.data.get('modules', True)
-  syllabus: bool = request.data.get('syllabus', True)
-  assignments: bool = request.data.get('assignments', True)
-  discussions: bool = request.data.get('discussions', True)
+def canvas_ingest() -> Response:
+  data = request.get_json()
+  logging.info("Canvas ingest data: %s", data)
+
+  course_name: str = data.get('course_name', '')
+  canvas_url: str = data.get('canvas_url', None)
+  files: bool = data.get('files', True)
+  pages: bool = data.get('pages', True)
+  modules: bool = data.get('modules', True)
+  syllabus: bool = data.get('syllabus', True)
+  assignments: bool = data.get('assignments', True)
+  discussions: bool = data.get('discussions', True)
   options = {
     'files': str(files).lower() == 'true',
     'pages': str(pages).lower() == 'true',
@@ -779,34 +782,37 @@ def canvas_ingest(request) -> Response:
   print("Canvas URL: ", canvas_url)
   print("Download Options: ", options)
 
-  try:
-    # canvas.illinois.edu/courses/COURSE_CODE
-    match = re.search(r'canvas\.illinois\.edu/courses/([^/]+)', canvas_url)
-    canvas_course_id = match.group(1) if match else None
-
-    ingester = IngestCanvas()
-    ingester.ingest_course_content(canvas_course_id=canvas_course_id,
-                                   course_name=course_name,
-                                   content_ingest_dict=options)
-  except:
-    print("Error ingesting course content from Canvas.")
-    abort(500, description="Error ingesting course content from Canvas.")
-
-  active_queue = Queue()
-  data = request.get_json()
-  logging.info("Canvas ingest data: %s", data)
+  # canvas.illinois.edu/courses/COURSE_CODE
+  match = re.search(r'canvas\.illinois\.edu/courses/([^/]+)', canvas_url)
+  canvas_course_id = match.group(1) if match else None
 
-  result = active_queue.addJobToIngestQueue(data, queue_name='uiuc-chat-canvas')
-  logging.info("Result from addJobToIngestQueue:  %s", result)
+  canvas_id = os.getenv("CANVAS_ACCESS_TOKEN", default="")
+  if len(canvas_id) == 0:
+      response = jsonify(message=f"CANVAS_ACCESS_TOKEN is not configured.")
+      response.status_code = 500
+      response.headers.add('Access-Control-Allow-Origin', '*')
+      return response
 
-  response = jsonify(
-    {
-      "outcome": f'Queued Canvas Ingest task',
-      "ingest_task_id": result
-    }
-  )
-  response.headers.add('Access-Control-Allow-Origin', '*')
-  return response
+  try:
+      ingester = IngestCanvas()
+      accept_status = ingester.auto_accept_enrollments(canvas_course_id)
+      job_ids = ingester.ingest_course_content(canvas_course_id=canvas_course_id,
+                                       course_name=course_name,
+                                       content_ingest_dict=options)
+
+      response = jsonify(
+        {
+          "outcome": f'Queued Canvas Ingest task',
+          "ingest_task_ids": job_ids
+        }
+      )
+      response.headers.add('Access-Control-Allow-Origin', '*')
+      return response
+  except Exception as e:
+      response = jsonify(error=str(e), message=f"Internal Server Error {e}")
+      response.status_code = 500
+      response.headers.add('Access-Control-Allow-Origin', '*')
+      return response
 
 @app.route('/createProject', methods=['POST'])
 def createProject(service: ProjectService, flaskExecutor: ExecutorInterface) -> Response: