V2.2.7 chg midterm spring2021 (#79)

* FIX repeat background image

* switch pipeline healthcheck to http request

* CHG python 3.8 -> 3.9

* fixes for misterm 2021

* ADD question editor

* ADD warning to question reset

* CHG minor word edits

* ADD question add and delete to editor

* ADD logging to admin questions endpoint

* LINT

Author: wabscale

Makefile

@@ -1,7 +1,6 @@
-PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master logstash adminer 
-RESTART_ALWAYS_SERVICES := api web rpc-worker
+PERSISTENT_SERVICES := db traefik kibana elasticsearch-coordinating redis-master logstash adminer
+RESTART_ALWAYS_SERVICES := api web-dev rpc-worker
 PUSH_SERVICES := api web logstash theia-init theia-proxy theia-admin theia-xv6
-BUILD_ALLWAYS := api web
 
 
 
@@ -32,17 +31,22 @@ help:
 	@echo 'Available make targets:'
 	@grep PHONY: Makefile | cut -d: -f2 | sed '1d;s/^/make/'
 
+.PHONY: deploy       # Deploy Anubis to cluster
+deploy:
+	./kube/deploy.sh
+	./kube/restart.sh
+
 .PHONY: build        # Build all docker images
 build:
-	docker-compose build --parallel --pull $(BUILD_ALLWAYS)
+	docker-compose build --parallel --pull
 
 .PHONY: push         # Push images to registry.osiris.services (requires vpn)
 push:
 	docker-compose build --parallel --pull $(PUSH_SERVICES)
 	docker-compose push $(PUSH_SERVICES)
 
 .PHONY: debug        # Start the cluster in debug mode
-debug: build
+debug:
 	docker-compose up -d $(PERSISTENT_SERVICES)
 	docker-compose up \
 		-d --force-recreate \
@@ -69,36 +73,12 @@ mindebug: build
 	@echo 'auth: http://localhost/api/admin/auth/token/jmc1283'
 	@echo 'site: http://localhost/'
 
-
-.PHONY: jupyter      # Start he jupyterhub container
-jupyter:
-	docker-compose up --force-recreate --build api-dev
-
-.PHONY: deploy       # Start the cluster in production mode
-deploy: check build restart
-
-.PHONY: backup       # Backup database to file
-backup:
-	./scripts/backup.sh
-
-.PHONY: restore      # Restore to most recent backup
-restore:
-	./scripts/restore.sh
-
 yeetdb:
 	docker-compose kill db
 	docker-compose rm -f
 	docker volume rm anubis_db_data
 	docker-compose up -d --force-recreate db
 
-.PHONY: cleandata    # yeet data
-cleandata:
-	docker-compose kill
-	docker-compose rm -f
-	if [ -n "${VOLUMES}" ]; then \
-		docker volume rm $(VOLUMES); \
-	fi
-
 .PHONY: clean        # Clean up volumes, images and data
 clean:
 	docker-compose kill

api/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-alpine
+FROM python:3.9-alpine
 
 ENV SECRET_KEY=DEFAULT
 

api/anubis/app.py

@@ -1,5 +1,4 @@
 from flask import Flask
-from anubis.utils.logger import logger
 
 
 def init_services(app):

api/anubis/models/__init__.py

@@ -1,6 +1,6 @@
 import base64
-import os
 import json
+import os
 from datetime import datetime, timedelta
 
 from flask_sqlalchemy import SQLAlchemy
@@ -251,9 +251,13 @@ class AssignmentQuestion(db.Model):
     solution = db.Column(db.Text, nullable=True)
     sequence = db.Column(db.Integer, index=True, nullable=False)
     code_question = db.Column(db.Boolean, default=False)
-    code_language = db.Column(db.String(128), nullable=True, default=None)
+    code_language = db.Column(db.String(128), nullable=True, default='')
     placeholder = db.Column(db.Text, nullable=True, default="")
 
+    # Timestamps
+    created = db.Column(db.DateTime, default=datetime.now)
+    last_updated = db.Column(db.DateTime, default=datetime.now, onupdate=datetime.now)
+
     # Relationships
     assignment = db.relationship(Assignment, cascade="all,delete", backref="questions")
 
@@ -630,7 +634,7 @@ class StaticFile(db.Model):
     filename = db.Column(db.String(256))
     path = db.Column(db.String(256))
     content_type = db.Column(db.String(128))
-    blob = db.Column(db.LargeBinary(length=(2**32)-1))
+    blob = db.Column(db.LargeBinary(length=(2 ** 32) - 1))
     hidden = db.Column(db.Boolean)
 
     # Timestamps

api/anubis/rpc/pipeline.py

@@ -6,8 +6,9 @@
 from kubernetes import config, client
 
 from anubis.models import Config, Submission
-from anubis.utils.logger import logger
 from anubis.utils.data import is_debug
+from anubis.utils.logger import logger
+
 
 def create_pipeline_job_obj(client, submission):
     """
@@ -31,6 +32,7 @@ def create_pipeline_job_obj(client, submission):
         image=submission.assignment.pipeline_image,
         image_pull_policy=os.environ.get("IMAGE_PULL_POLICY", default="Always"),
         env=[
+            client.V1EnvVar(name="NETID", value=submission.owner.netid),
             client.V1EnvVar(name="TOKEN", value=submission.token),
             client.V1EnvVar(name="COMMIT", value=submission.commit),
             client.V1EnvVar(name="GIT_REPO", value=submission.repo.repo_url),

api/anubis/rpc/theia.py

@@ -1,12 +1,12 @@
 import os
 import time
-from datetime import datetime, timedelta
+from datetime import datetime
 
 from kubernetes import config, client
 
 from anubis.models import db, Config, TheiaSession
-from anubis.utils.logger import logger
 from anubis.utils.elastic import esindex
+from anubis.utils.logger import logger
 
 
 def get_theia_pod_name(theia_session: TheiaSession) -> str:

api/anubis/utils/assignments.py

@@ -60,44 +60,44 @@ def get_assignments(netid: str, course_id=None) -> Union[List[Dict[str, str]], N
 
     assignments = (
         Assignment.query.join(Course)
-        .join(InCourse)
-        .join(User)
-        .filter(
+            .join(InCourse)
+            .join(User)
+            .filter(
             User.netid == netid,
             Assignment.hidden == False,
             Assignment.release_date <= datetime.now(),
             *filters
         )
-        .order_by(Assignment.due_date.desc())
-        .all()
+            .order_by(Assignment.due_date.desc())
+            .all()
     )
 
     a = [a.data for a in assignments]
     for assignment_data in a:
         assignment_data["has_submission"] = (
-            Submission.query.join(User)
-            .join(Assignment)
-            .filter(
-                Assignment.id == assignment_data["id"],
-                User.netid == netid,
-            )
-            .first()
-            is not None
+                Submission.query.join(User)
+                .join(Assignment)
+                .filter(
+                    Assignment.id == assignment_data["id"],
+                    User.netid == netid,
+                )
+                .first()
+                is not None
         )
         assignment_data["has_repo"] = (
-            AssignmentRepo.query.filter(
-                AssignmentRepo.owner_id == user.id,
-                AssignmentRepo.assignment_id == assignment_data['id'],
-            ).first()
-            is not None
+                AssignmentRepo.query.filter(
+                    AssignmentRepo.owner_id == user.id,
+                    AssignmentRepo.assignment_id == assignment_data['id'],
+                ).first()
+                is not None
         )
 
     return a
 
 
 @cache.memoize(timeout=3, unless=is_debug)
 def get_submissions(
-    user_id=None, course_id=None, assignment_id=None
+        user_id=None, course_id=None, assignment_id=None
 ) -> Union[List[Dict[str, str]], None]:
     """
     Get all submissions for a given netid. Cache the results. Optionally specify
@@ -127,11 +127,11 @@ def get_submissions(
 
     submissions = (
         Submission.query.join(Assignment)
-        .join(Course)
-        .join(InCourse)
-        .join(User)
-        .filter(Submission.owner_id == owner.id, *filters)
-        .all()
+            .join(Course)
+            .join(InCourse)
+            .join(User)
+            .filter(Submission.owner_id == owner.id, *filters)
+            .all()
     )
 
     return [s.full_data for s in submissions]
@@ -175,10 +175,10 @@ def assignment_sync(assignment_data):
     db.session.commit()
 
     for i in AssignmentTest.query.filter(
-        and_(
-            AssignmentTest.assignment_id == assignment.id,
-            AssignmentTest.name.notin_(assignment_data["tests"]),
-        )
+            and_(
+                AssignmentTest.assignment_id == assignment.id,
+                AssignmentTest.name.notin_(assignment_data["tests"]),
+            )
     ).all():
         db.session.delete(i)
     db.session.commit()
@@ -189,8 +189,8 @@ def assignment_sync(assignment_data):
                 Assignment.id == assignment.id,
                 AssignmentTest.name == test_name,
             )
-            .join(Assignment)
-            .first()
+                .join(Assignment)
+                .first()
         )
 
         if assignment_test is None:

api/anubis/utils/data.py

@@ -1,10 +1,10 @@
+from datetime import datetime
 from email.mime.text import MIMEText
 from hashlib import sha256
 from json import dumps
 from os import environ, urandom
 from smtplib import SMTP
 from typing import Union, Tuple
-from datetime import datetime
 
 from flask import Response
 
@@ -214,7 +214,7 @@ def split_chunks(lst, n):
     """
     _chunks = []
     for i in range(0, len(lst), n):
-        _chunks.append(lst[i : i + n])
+        _chunks.append(lst[i: i + n])
     return _chunks
 
 

api/anubis/utils/decorators.py

@@ -50,8 +50,8 @@ def json_wrap(*args, **kwargs):
 
 
 def json_endpoint(
-    required_fields: Union[List[str], List[Tuple], None] = None,
-    only_required: bool = False,
+        required_fields: Union[List[str], List[Tuple], None] = None,
+        only_required: bool = False,
 ):
     """
     Wrap a route so that it always converts data
@@ -69,13 +69,13 @@ def wrapper(func):
         @wraps(func)
         def json_wrap(*args, **kwargs):
             if not request.headers.get("Content-Type", default=None).startswith(
-                "application/json"
+                    "application/json"
             ):
                 return {
-                    "success": False,
-                    "error": "Content-Type header is not application/json",
-                    "data": None,
-                }, 406  # Not Acceptable
+                           "success": False,
+                           "error": "Content-Type header is not application/json",
+                           "data": None,
+                       }, 406  # Not Acceptable
             json_data: dict = request.json
 
             if required_fields is not None:

api/anubis/utils/elastic.py

@@ -34,8 +34,8 @@ def wrapper(*args, **kwargs):
             ip = get_request_ip()
             location = geolite2.lookup(ip)
             longitude, latitude = location.location[::-1] \
-                            if location is not None \
-                            else [0, 0]
+                if location is not None \
+                else [0, 0]
 
             # Skip indexing if the app has ELK disabled
             if not config.DISABLE_ELK:

api/anubis/utils/questions.py

@@ -14,7 +14,7 @@
 
 
 def get_question_sequence_mapping(
-    questions: List[AssignmentQuestion],
+        questions: List[AssignmentQuestion],
 ) -> Dict[int, List[AssignmentQuestion]]:
     """
     Get mapping of sequence to question mapping from list of questions
@@ -91,8 +91,8 @@ def assign_questions(assignment: Assignment):
     assigned_questions = []
     students = (
         User.query.join(InCourse)
-        .filter(InCourse.course_id == assignment.course_id)
-        .all()
+            .filter(InCourse.course_id == assignment.course_id)
+            .all()
     )
     for student in students:
         for sequence, qs in questions.items():

api/anubis/utils/rpc.py

@@ -1,14 +1,14 @@
 from redis import Redis
 from rq import Queue
 
+from anubis.config import config
 from anubis.rpc.pipeline import test_repo
+from anubis.rpc.seed import seed_debug
 from anubis.rpc.theia import (
     initialize_theia_session,
     reap_theia_session,
     reap_stale_theia_sessions,
 )
-from anubis.rpc.seed import seed_debug
-from anubis.config import config
 
 
 def rpc_enqueue(func, *args):

api/anubis/utils/stats.py

@@ -12,13 +12,13 @@ def stats_for(student_id, assignment_id):
     best = None
     best_count = -1
     for submission in (
-        Submission.query.filter(
-            Submission.assignment_id == assignment_id,
-            Submission.owner_id == student_id,
-            Submission.processed,
-        )
-        .order_by(Submission.created.desc())
-        .all()
+            Submission.query.filter(
+                Submission.assignment_id == assignment_id,
+                Submission.owner_id == student_id,
+                Submission.processed,
+            )
+                    .order_by(Submission.created.desc())
+                    .all()
     ):
         correct_count = sum(
             map(lambda result: 1 if result.passed else 0, submission.test_results)
@@ -80,8 +80,8 @@ def bulk_stats(assignment_id, netids=None, offset=0, limit=20):
     bests = []
 
     assignment = (
-        Assignment.query.filter_by(name=assignment_id).first()
-        or Assignment.query.filter_by(id=assignment_id).first()
+            Assignment.query.filter_by(name=assignment_id).first()
+            or Assignment.query.filter_by(id=assignment_id).first()
     )
     if assignment is None:
         return error_response("assignment does not exist")