V3.0.1 chg patch lms (#85)

* FIX remove git hook container escape

* FIX remove git hook container escape

* FIX migration issue, theia options for course and superuser course context

* CHG improve caching issues

* CHG take theia images out of deploy script

* ADD app context aware with_context wrapper

Author: wabscale

api/anubis/models/__init__.py

@@ -56,11 +56,12 @@ class User(db.Model):
     def data(self):
         professor_for = [pf.data for pf in self.professor_for]
         ta_for = [taf.data for taf in self.ta_for]
-        extra_for = []
+        super_for = None
         if self.is_superuser:
+            super_for = []
             courses = Course.query.all()
             for course in courses:
-                extra_for.append({'id': course.id, 'name': course.name})
+                super_for.append({'id': course.id, 'name': course.name})
         return {
             "id": self.id,
             "netid": self.netid,
@@ -70,7 +71,7 @@ def data(self):
             "is_admin": len(professor_for) > 0 or len(ta_for) > 0 or self.is_superuser,
             "professor_for": professor_for,
             "ta_for": ta_for,
-            "admin_for": professor_for + ta_for + extra_for,
+            "admin_for": super_for or (professor_for + ta_for),
         }
 
     def __repr__(self):
@@ -93,7 +94,7 @@ class Course(db.Model):
     section = db.Column(db.TEXT, nullable=True)
     professor = db.Column(db.TEXT, nullable=False)
     theia_default_image = db.Column(db.TEXT, nullable=False, default='registry.osiris.services/anubis/xv6')
-    theia_default_options = db.Column(MutableJson, default=lambda: {"limits": {"cpu": "4", "memory": "4Gi"}})
+    theia_default_options = db.Column(MutableJson, default=lambda: {"limits": {"cpu": "2", "memory": "500Mi"}})
 
     @property
     def total_assignments(self):

api/anubis/rpc/theia.py

@@ -19,6 +19,12 @@ def create_theia_pod_obj(theia_session: TheiaSession):
     name = get_theia_pod_name(theia_session)
     containers = []
 
+    # Get the theia session options
+    limits = theia_session.options.get('limits', {"cpu": "2", "memory": "500Mi"})
+    requests = theia_session.options.get('requests', {"cpu": "250m", "memory": "100Mi"})
+    autosave = theia_session.options.get('autosave', True)
+    credentials = theia_session.options.get('credentials', False)
+
     # PVC
     volume_name = name + "-volume"
     pvc = client.V1PersistentVolumeClaim(
@@ -66,12 +72,8 @@ def create_theia_pod_obj(theia_session: TheiaSession):
         ],
     )
 
-    limits = theia_session.options.get('limits', {"cpu": "2", "memory": "500Mi"})
-    requests = theia_session.options.get('requests', {"cpu": "250m", "memory": "100Mi"})
-    autosave = theia_session.options.get('autosave', True)
-
     extra_env = []
-    if theia_session.options.get('credentials', False):
+    if credentials:
         extra_env.append(client.V1EnvVar(
             name='INCLUSTER',
             value=base64.b64encode(create_token(theia_session.owner.netid).encode()).decode(),

api/anubis/utils/data.py

@@ -7,7 +7,7 @@
 from smtplib import SMTP
 from typing import Union, Tuple
 
-from flask import Response
+from flask import Response, has_app_context, has_request_context
 
 from anubis.config import config
 
@@ -302,11 +302,15 @@ def with_context(function):
 
     @functools.wraps(function)
     def wrapper(*args, **kwargs):
-
         # Do the import here to avoid circular
         # import issues.
         from anubis.app import create_app
 
+        # Only create an app context if
+        # there is not already one
+        if has_app_context() or has_request_context():
+            return function(*args, **kwargs)
+
         # Create a fresh app
         app = create_app()
 

api/anubis/utils/lms/assignments.py

@@ -3,7 +3,7 @@
 from typing import Union, List, Dict, Tuple
 
 from dateutil.parser import parse as date_parse, ParserError
-from sqlalchemy import or_, and_
+from sqlalchemy import or_
 
 from anubis.models import (
     db,
@@ -25,7 +25,7 @@
 from anubis.utils.services.logger import logger
 
 
-@cache.memoize(timeout=5, unless=is_debug)
+@cache.memoize(timeout=60, unless=is_debug)
 def get_courses(netid: str):
     """
     Get all classes a given netid is in
@@ -122,48 +122,6 @@ def get_assignments(netid: str, course_id=None) -> Union[List[Dict[str, str]], N
     return response
 
 
-@cache.memoize(timeout=3, unless=is_debug)
-def get_submissions(
-        user_id=None, course_id=None, assignment_id=None
-) -> Union[List[Dict[str, str]], None]:
-    """
-    Get all submissions for a given netid. Cache the results. Optionally specify
-    a class_name and / or assignment_name for additional filtering.
-
-    :param user_id:
-    :param course_id:
-    :param assignment_id: id of assignment
-    :return:
-    """
-
-    # Load user
-    owner = User.query.filter(User.id == user_id).first()
-
-    # Verify user exists
-    if owner is None:
-        return None
-
-    # Build filters
-    filters = []
-    if course_id is not None and course_id != "":
-        filters.append(Course.id == course_id)
-    if user_id is not None and user_id != "":
-        filters.append(User.id == user_id)
-    if assignment_id is not None:
-        filters.append(Assignment.id == assignment_id)
-
-    submissions = (
-        Submission.query.join(Assignment)
-            .join(Course)
-            .join(InCourse)
-            .join(User)
-            .filter(Submission.owner_id == owner.id, *filters)
-            .all()
-    )
-
-    return [s.full_data for s in submissions]
-
-
 def assignment_sync(assignment_data: dict) -> Tuple[Union[dict, str], bool]:
     """
     Take an assignment_data dictionary from a assignment meta.yaml
@@ -181,24 +139,24 @@ def assignment_sync(assignment_data: dict) -> Tuple[Union[dict, str], bool]:
 
     # Attempt to find the class
     course_name = assignment_data.get('class', None) or assignment_data.get('course', None)
-    c: Course = Course.query.filter(
+    course: Course = Course.query.filter(
         or_(
             Course.name == course_name,
             Course.course_code == course_name,
         )
     ).first()
-    if c is None:
+    if course is None:
         return "Unable to find class", False
 
-    assert_course_admin(c.id)
+    assert_course_admin(course.id)
 
     # Check if it exists
     if assignment is None:
         assignment = Assignment(
-            theia_image=c.theia_default_image,
-            theia_options=c.theia_default_options,
+            theia_image=course.theia_default_image,
+            theia_options=course.theia_default_options,
             unique_code=assignment_data["unique_code"],
-            course=c,
+            course=course,
         )
 
     # Update fields
@@ -220,10 +178,8 @@ def assignment_sync(assignment_data: dict) -> Tuple[Union[dict, str], bool]:
     # Go through assignment tests, and delete those that are now
     # not in the assignment data.
     for assignment_test in AssignmentTest.query.filter(
-            and_(
-                AssignmentTest.assignment_id == assignment.id,
-                AssignmentTest.name.notin_(assignment_data["tests"]),
-            )
+            AssignmentTest.assignment_id == assignment.id,
+            AssignmentTest.name.notin_(assignment_data["tests"]),
     ).all():
         # Delete any and all submission test results that are still outstanding
         # for an assignment test that will be deleted.
@@ -257,7 +213,7 @@ def assignment_sync(assignment_data: dict) -> Tuple[Union[dict, str], bool]:
 
     # Sync the questions in the assignment data
     question_message = None
-    if 'questions' in assignment_data:
+    if 'questions' in assignment_data and isinstance(assignment_data['questions'], list):
         accepted, ignored, rejected = ingest_questions(
             assignment_data["questions"], assignment
         )
@@ -266,3 +222,5 @@ def assignment_sync(assignment_data: dict) -> Tuple[Union[dict, str], bool]:
     db.session.commit()
 
     return {"assignment": assignment.data, "questions": question_message}, True
+
+

api/anubis/utils/lms/repos.py

@@ -0,0 +1,17 @@
+from typing import List
+
+from anubis.models import AssignmentRepo, Assignment
+from anubis.utils.services.cache import cache
+
+
+@cache.memoize(timeout=3600)
+def get_repos(user_id: str):
+    repos: List[AssignmentRepo] = (
+        AssignmentRepo.query.join(Assignment)
+            .filter(AssignmentRepo.owner_id == user_id)
+            .distinct(AssignmentRepo.repo_url)
+            .order_by(Assignment.release_date.desc())
+            .all()
+    )
+
+    return [repo.data for repo in repos]

api/anubis/utils/lms/submissions.py

@@ -1,8 +1,10 @@
 from datetime import datetime
-from typing import List, Union
+from typing import List, Union, Dict
 
-from anubis.models import Submission, AssignmentRepo, User, db
+from anubis.models import Submission, AssignmentRepo, User, db, Course, Assignment, InCourse
+from anubis.utils.data import is_debug
 from anubis.utils.http.https import error_response, success_response
+from anubis.utils.services.cache import cache
 from anubis.utils.services.rpc import enqueue_autograde_pipeline
 
 
@@ -146,3 +148,45 @@ def fix_dangling():
             enqueue_autograde_pipeline(s.id)
 
     return fixed
+
+
+@cache.memoize(timeout=3600, unless=is_debug)
+def get_submissions(
+        user_id=None, course_id=None, assignment_id=None
+) -> Union[List[Dict[str, str]], None]:
+    """
+    Get all submissions for a given netid. Cache the results. Optionally specify
+    a class_name and / or assignment_name for additional filtering.
+
+    :param user_id:
+    :param course_id:
+    :param assignment_id: id of assignment
+    :return:
+    """
+
+    # Load user
+    owner = User.query.filter(User.id == user_id).first()
+
+    # Verify user exists
+    if owner is None:
+        return None
+
+    # Build filters
+    filters = []
+    if course_id is not None and course_id != "":
+        filters.append(Course.id == course_id)
+    if user_id is not None and user_id != "":
+        filters.append(User.id == user_id)
+    if assignment_id is not None:
+        filters.append(Assignment.id == assignment_id)
+
+    submissions = (
+        Submission.query.join(Assignment)
+            .join(Course)
+            .join(InCourse)
+            .join(User)
+            .filter(Submission.owner_id == owner.id, *filters)
+            .all()
+    )
+
+    return [s.full_data for s in submissions]

api/anubis/utils/lms/webhook.py

@@ -1,4 +1,6 @@
 from anubis.models import db, User, AssignmentRepo
+from anubis.utils.services.cache import cache
+from anubis.utils.lms.repos import get_repos
 
 
 def parse_webhook(webhook):
@@ -99,5 +101,8 @@ def check_repo(assignment, repo_url, github_username, user=None) -> AssignmentRe
         db.session.add(repo)
         db.session.commit()
 
+        if user is not None:
+            cache.delete_memoized(get_repos, user.id)
+
     # Return the repo object
     return repo

api/anubis/utils/visuals/usage.py

@@ -81,7 +81,7 @@ def get_theia_sessions() -> pd.DataFrame:
     return theia_sessions
 
 
-@cache.memoize(timeout=10)
+@cache.memoize(timeout=360)
 def get_raw_submissions() -> List[Dict[str, Any]]:
     submissions_df = get_submissions()
     data = submissions_df.groupby(['assignment_id', 'created'])['id'].count() \

api/anubis/views/admin/autograde.py

@@ -8,14 +8,16 @@
 from anubis.utils.lms.course import assert_course_admin, assert_course_context, get_course_context
 from anubis.utils.lms.questions import get_assigned_questions
 from anubis.utils.services.elastic import log_endpoint
+from anubis.utils.services.cache import cache
 
 autograde_ = Blueprint("admin-autograde", __name__, url_prefix="/admin/autograde")
 
 
 @autograde_.route("/assignment/<assignment_id>")
 @require_admin()
+@cache.memoize(timeout=60)
 @json_response
-def admin_autograde_assignment_assignment_id(assignment_id, netid=None):
+def admin_autograde_assignment_assignment_id(assignment_id):
     """
     Calculate result statistics for an assignment. This endpoint is
     potentially very IO and computationally expensive. We basically
@@ -29,7 +31,6 @@ def admin_autograde_assignment_assignment_id(assignment_id, netid=None):
     timeout. *
 
     :param assignment_id:
-    :param netid:
     :return:
     """
 
@@ -58,6 +59,7 @@ def admin_autograde_assignment_assignment_id(assignment_id, netid=None):
 
 @autograde_.route("/for/<assignment_id>/<user_id>")
 @require_admin()
+@cache.memoize(timeout=60)
 @json_response
 def admin_autograde_for_assignment_id_user_id(assignment_id, user_id):
     """
@@ -68,9 +70,6 @@ def admin_autograde_for_assignment_id_user_id(assignment_id, user_id):
     :return:
     """
 
-    # Get the course context
-    course = get_course_context()
-
     # Pull the assignment object
     assignment = Assignment.query.filter(
         Assignment.id == assignment_id
@@ -101,6 +100,7 @@ def admin_autograde_for_assignment_id_user_id(assignment_id, user_id):
 @autograde_.route("/submission/<string:assignment_id>/<string:netid>")
 @require_admin()
 @log_endpoint("cli", lambda: "submission-stats")
+@cache.memoize(timeout=60)
 @json_response
 def private_submission_stats_id(assignment_id: str, netid: str):
     """

api/anubis/views/admin/regrade.py

@@ -11,7 +11,7 @@
 from anubis.utils.http.decorators import json_response
 from anubis.utils.http.decorators import load_from_id
 from anubis.utils.http.https import error_response, success_response, get_number_arg
-from anubis.utils.lms.course import assert_course_admin, get_course_context, assert_course_context
+from anubis.utils.lms.course import assert_course_context
 from anubis.utils.services.elastic import log_endpoint
 from anubis.utils.services.rpc import enqueue_autograde_pipeline, rpc_enqueue
 
@@ -62,7 +62,7 @@ def admin_regrade_status(assignment: Assignment):
 @require_admin()
 @log_endpoint("cli", lambda: "regrade-commit")
 @json_response
-def admin_regrade_submission_commit(commit):
+def admin_regrade_submission_commit(commit: str):
     """
     Regrade a specific submission via the unique commit hash.