V2.2.9 fix assignment 3 sched (#81)

* WIP midterm post

* ADD midterm retro post

* ADD new rpc services to mindebug

* FIX mobile content shift

* FIX header top padding

* FIX add onclick wrapper to non public nav items

* ADD docs to some utils functions

* ADD docs to more utils functions and views

* CHG reorganize minikube debug

* FIX cache missing issue

* FIX weird reaper crashing issue

* ADD timestamp to usage plot

* ADD regrade button to admin panel

* ADD regrade queue

* ADD rpc-regrade to restart script

* CHG add check to reaper for user Nonetype

Author: wabscale

Makefile

@@ -1,5 +1,5 @@
 PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master logstash adminer
-RESTART_ALWAYS_SERVICES := api web-dev rpc-default rpc-theia
+RESTART_ALWAYS_SERVICES := api web-dev rpc-default rpc-theia rpc-regrade
 PUSH_SERVICES := api web logstash theia-init theia-proxy theia-admin theia-xv6
 
 
@@ -55,20 +55,38 @@ debug:
 	sleep 3
 	@echo 'running migrations'
 	make -C api migrations
+	@echo ''
 	@echo 'seed: http://localhost/api/admin/seed/'
 	@echo 'auth: http://localhost/api/admin/auth/token/jmc1283'
 	@echo 'site: http://localhost/'
 
 .PHONY: mindebug     # Start the minimal cluster in debug mode
-mindebug: build
+mindebug:
 	docker-compose up -d traefik db redis-master logstash
 	docker-compose up \
 		-d --force-recreate \
-		api web rpc-worker
+		api web rpc-default rpc-theia
 	@echo 'Waiting a moment before running migrations'
 	sleep 3
 	@echo 'running migrations'
 	make -C api migrations
+	@echo ''
+	@echo 'seed: http://localhost/api/admin/seed/'
+	@echo 'auth: http://localhost/api/admin/auth/token/jmc1283'
+	@echo 'site: http://localhost/'
+
+.PHONY: mkdebug      # Start minikube debug
+mkdebug:
+	./kube/debug/provision.sh
+	@echo ''
+	@echo 'seed: http://localhost/api/admin/seed/'
+	@echo 'auth: http://localhost/api/admin/auth/token/jmc1283'
+	@echo 'site: http://localhost/'
+
+.PHONY: mkrestart    # Restart minikube debug
+mkrestart:
+	./kube/debug/restart.sh
+	@echo ''
 	@echo 'seed: http://localhost/api/admin/seed/'
 	@echo 'auth: http://localhost/api/admin/auth/token/jmc1283'
 	@echo 'site: http://localhost/'

api/anubis/rpc/pipeline.py

@@ -100,15 +100,15 @@ def cleanup_jobs(batch_v1) -> int:
     return active_count
 
 
-def test_repo(submission_id: str):
+def create_submission_pipeline(submission_id: str):
     """
     This function should launch the appropriate testing container
     for the assignment, passing along the function arguments.
 
     :param submission_id: submission.id of to test
     """
     from anubis.app import create_app
-    from anubis.utils.rpc import enqueue_webhook
+    from anubis.utils.rpc import enqueue_autograde_pipeline
 
     app = create_app()
 
@@ -153,8 +153,7 @@ def test_repo(submission_id: str):
                 "TOO many jobs - re-enqueue {}".format(submission_id),
                 extra={"submission_id": submission_id},
             )
-            enqueue_webhook(submission_id)
-            time.sleep(1)
+            enqueue_autograde_pipeline(submission_id)
             exit(0)
 
         # Create job object

api/anubis/rpc/seed.py

@@ -16,6 +16,7 @@
     TheiaSession,
     AssignmentQuestion,
     AssignedStudentQuestion,
+    AssignedQuestionResponse,
 )
 from anubis.utils.data import rand
 from anubis.utils.questions import assign_questions
@@ -149,6 +150,7 @@ def create_course(users):
 def seed_main():
     # Yeet
     TheiaSession.query.delete()
+    AssignedQuestionResponse.query.delete()
     AssignedStudentQuestion.query.delete()
     AssignmentQuestion.query.delete()
     SubmissionTestResult.query.delete()

api/anubis/utils/assignments.py

@@ -1,6 +1,6 @@
 import traceback
 from datetime import datetime
-from typing import Union, List, Dict
+from typing import Union, List, Dict, Tuple
 
 from dateutil.parser import parse as date_parse, ParserError
 from sqlalchemy import or_, and_
@@ -64,11 +64,11 @@ def get_assignments(netid: str, course_id=None) -> Union[List[Dict[str, str]], N
         filters.append(Assignment.release_date <= datetime.now())
         filters.append(Assignment.hidden == False)
 
-    assignments = Assignment.query\
+    assignments = Assignment.query \
         .join(Course).join(InCourse).join(User).filter(
-            User.netid == netid,
-            *filters
-        ).order_by(Assignment.due_date.desc()).all()
+        User.netid == netid,
+        *filters
+    ).order_by(Assignment.due_date.desc()).all()
 
     a = [a.data for a in assignments]
     for assignment_data in a:
@@ -135,7 +135,17 @@ def get_submissions(
     return [s.full_data for s in submissions]
 
 
-def assignment_sync(assignment_data):
+def assignment_sync(assignment_data: dict) -> Tuple[Union[dict, str], bool]:
+    """
+    Take an assignment_data dictionary from a assignment meta.yaml
+    and update any and all existing data about the assignment.
+
+    * This includes the assignment object fields, assignment tests,
+    and assignment questions. *
+
+    :param assignment_data:
+    :return:
+    """
     assignment = Assignment.query.filter(
         Assignment.unique_code == assignment_data["unique_code"]
     ).first()
@@ -170,24 +180,31 @@ def assignment_sync(assignment_data):
         return "Unable to parse datetime", 406
 
     db.session.add(assignment)
-    db.session.commit()
 
+    # Go through assignment tests, and delete those that are now
+    # not in the assignment data.
     for assignment_test in AssignmentTest.query.filter(
             and_(
                 AssignmentTest.assignment_id == assignment.id,
                 AssignmentTest.name.notin_(assignment_data["tests"]),
             )
     ).all():
+        # Delete any and all submission test results that are still outstanding
+        # for an assignment test that will be deleted.
         SubmissionTestResult.query.filter(
             SubmissionTestResult.assignment_test_id == assignment_test.id,
         ).delete()
+
+        # Delete the assignment test
         AssignmentTest.query.filter(
             AssignmentTest.assignment_id == assignment.id,
             AssignmentTest.name == assignment_test.name,
         ).delete()
-    db.session.commit()
 
+    # Run though the tests in the assignment data
     for test_name in assignment_data["tests"]:
+
+        # Find if the assignment test exists
         assignment_test = (
             AssignmentTest.query.filter(
                 Assignment.id == assignment.id,
@@ -197,14 +214,19 @@ def assignment_sync(assignment_data):
                 .first()
         )
 
+        # Create the assignment test if it did not already exist
         if assignment_test is None:
             assignment_test = AssignmentTest(assignment=assignment, name=test_name)
             db.session.add(assignment_test)
-            db.session.commit()
 
-    accepted, ignored, rejected = ingest_questions(
-        assignment_data["questions"], assignment
-    )
-    question_message = {"accepted": accepted, "ignored": ignored, "rejected": rejected}
+    # Sync the questions in the assignment data
+    question_message = None
+    if 'questions' in assignment_data:
+        accepted, ignored, rejected = ingest_questions(
+            assignment_data["questions"], assignment
+        )
+        question_message = {"accepted": accepted, "ignored": ignored, "rejected": rejected}
+
+    db.session.commit()
 
     return {"assignment": assignment.data, "questions": question_message}, True

api/anubis/utils/cache.py

@@ -3,6 +3,6 @@
 cache = Cache(config={"CACHE_TYPE": "redis"})
 
 
-@cache.cached(timeout=1)
+@cache.memoize(timeout=1)
 def cache_health():
-    pass
+    return None

api/anubis/utils/data.py

@@ -218,15 +218,32 @@ def split_chunks(lst, n):
     return _chunks
 
 
-def rand(max_len=None):
+def rand(max_len: int = None):
+    """
+    Get a relatively random hex string of up
+    to max_len.
+
+    :param max_len:
+    :return:
+    """
     rand_hash = sha256(urandom(32)).hexdigest()
     if max_len is not None:
         return rand_hash[:max_len]
     return rand_hash
 
 
 def human_readable_to_bytes(size: str) -> int:
-    size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+    """
+    Convert a string in the form of 5GB and get an integer value
+    for the number of bytes in that data size.
+
+    >>> human_readable_to_bytes('1 GiB')
+    >>> 1073741824
+
+    :param size:
+    :return:
+    """
+    size_name = ("B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB", "ZiB", "YiB")
     size = size.split()  # divide '1 GB' into ['1', 'GB']
     num, unit = int(size[0]), size[1]
     # index in list of sizes determines power to raise it to
@@ -236,16 +253,25 @@ def human_readable_to_bytes(size: str) -> int:
     return num * factor
 
 
-def row2dict(row):
-    d = {}
+def row2dict(row) -> dict:
+    """
+    Convert an sqlalchemy object to a dictionary from its column
+    values. This function looks at internal sqlalchemy fields
+    to create a raw dictionary from the columns in the table.
+
+    :param row:
+    :return:
+    """
+
+    raw = {}
 
     for column in row.__table__.columns:
         value = getattr(row, column.name)
 
         if isinstance(value, datetime):
-            d[column.name] = str(value)
+            raw[column.name] = str(value)
             continue
 
-        d[column.name] = value
+        raw[column.name] = value
 
-    return d
+    return raw

api/anubis/utils/rpc.py

@@ -2,7 +2,7 @@
 from rq import Queue
 
 from anubis.config import config
-from anubis.rpc.pipeline import test_repo
+from anubis.rpc.pipeline import create_submission_pipeline
 from anubis.rpc.seed import seed_debug
 from anubis.rpc.theia import (
     initialize_theia_session,
@@ -29,20 +29,18 @@ def rpc_enqueue(func, queue=None, args=None):
         conn.close()
 
 
-def enqueue_webhook(*args):
+def enqueue_autograde_pipeline(*args, queue: str = 'default'):
     """Enqueues a test job"""
-    rpc_enqueue(test_repo, args=args)
+    rpc_enqueue(create_submission_pipeline, queue=queue, args=args)
 
 
 def enqueue_ide_initialize(*args):
     """Enqueue an ide initialization job"""
-
-    rpc_enqueue(initialize_theia_session, 'theia', args=args)
+    rpc_enqueue(initialize_theia_session, queue='theia', args=args)
 
 
 def enqueue_ide_stop(*args):
     """Reap theia session kube resources"""
-
     rpc_enqueue(reap_theia_session, queue='theia', args=args)
 
 
@@ -51,9 +49,11 @@ def enqueue_ide_reap_stale(*args):
     rpc_enqueue(reap_stale_theia_sessions, queue='theia', args=args)
 
 
-def seed():
-    rpc_enqueue(seed_debug)
+def enqueue_seed():
+    """Enqueue debug seed data"""
+    rpc_enqueue(seed_debug, queue='default')
 
 
-def create_visuals(*_):
-    rpc_enqueue(create_visuals_)
+def enqueue_create_visuals(*_):
+    """Enqueue create visuals"""
+    rpc_enqueue(create_visuals_, queue='default')