V2.0.1 fixes (#65)

* fix ontime icon and multiple repo handling

* graceful rolling update

* INIT design.md (no images yet)

* ADD more to design doc (at WIP state)

* "made version differences more concise "

* fix the annoying auth.nyu.edu sdc issue

* add dagling reset and no regrade dangling

* typos in reset-danling

* FIX fix-dangling submissions

* UPDATE stats endpoint, and add stats to cli

* CHG improve stats caching

Co-authored-by: Sejinkonye <[email protected]>

Author: wabscale

api/anubis/config.py

@@ -1,9 +1,10 @@
 import logging
 import os
+import hashlib
 
 
 class Config:
-    SECRET_KEY = os.environ.get('SECRET_KEY')
+    SECRET_KEY = os.environ.get('SECRET_KEY', default=hashlib.sha512(os.urandom(10)).hexdigest())
 
     # sqlalchemy
     SQLALCHEMY_DATABASE_URI = None
@@ -31,6 +32,9 @@ def __init__(self):
         self.OAUTH_CONSUMER_KEY = os.environ.get('OAUTH_CONSUMER_KEY', default='DEBUG')
         self.OAUTH_CONSUMER_SECRET = os.environ.get('OAUTH_CONSUMER_SECRET', default='DEBUG')
 
+        # Redis
+        self.CACHE_REDIS_HOST = os.environ.get('CACHE_REDIS_HOST', default='redis')
+
         logging.info('Starting with DATABASE_URI: {}'.format(
             self.SQLALCHEMY_DATABASE_URI))
         logging.info('Starting with SECRET_KEY: {}'.format(self.SECRET_KEY))

api/anubis/models/__init__.py

@@ -1,6 +1,7 @@
 import base64
 import logging
 import os
+from copy import deepcopy
 from datetime import datetime
 
 from flask_sqlalchemy import SQLAlchemy
@@ -307,6 +308,10 @@ def init_submission_models(self):
         sb = SubmissionBuild(submission=self)
         db.session.add(sb)
 
+        self.processed = False
+        self.state = 'Reset'
+        db.session.add(self)
+
         # Commit new models
         db.session.commit()
 
@@ -355,13 +360,19 @@ def data(self):
             'state': self.state,
             'created': str(self.created),
             'last_updated': str(self.last_updated),
-
-            # Connected models
-            'repo': self.repo.repo_url,
-            'tests': self.tests,
-            'build': self.build.data if self.build is not None else None,
         }
 
+    @property
+    def full_data(self):
+        data = self.data
+
+        # Add connected models
+        data['repo'] = self.repo.repo_url
+        data['tests'] = self.tests
+        data['build'] = self.build.data if self.build is not None else None
+
+        return data
+
 
 class SubmissionTestResult(db.Model):
     __tablename__ = 'submission_test_result'
@@ -398,6 +409,12 @@ def data(self):
             'last_updated': str(self.last_updated),
         }
 
+    @property
+    def stat_data(self):
+        data = self.data
+        del data['stdout']
+        return data
+
     def __str__(self):
         return 'testname: {}\nerrors: {}\npassed: {}\n'.format(
             self.testname,
@@ -432,3 +449,9 @@ def data(self):
             'stdout': self.stdout,
             'passed': self.passed,
         }
+
+    @property
+    def stat_data(self):
+        data = self.data
+        del data['stdout']
+        return data

api/anubis/routes/pipeline.py

@@ -28,7 +28,7 @@ def pipeline_report_panic(submission: Submission):
     :return:
     """
 
-    logger.info('submission panic reported', extra={
+    logger.error('submission panic reported', extra={
         'type': 'panic_report',
         'submission_id': submission.id, 'assignment_id': submission.assignment_id,
         'owner_id': submission.owner_id, 'data': json.dumps(request.json)})

api/anubis/routes/private.py

@@ -18,36 +18,25 @@
 from anubis.utils.elastic import log_endpoint
 from anubis.utils.redis_queue import enqueue_webhook_rpc
 from anubis.utils.logger import logger
-from anubis.utils.data import fix_dangling
+from anubis.utils.data import fix_dangling, bulk_stats, get_students
 
 private = Blueprint('private', __name__, url_prefix='/private')
 
 
-@cache.memoize(timeout=30)
-def stats_for(student_id, assignment_id):
-    # TODO rewrite this
-    raise
-
-
-@cache.cached(timeout=30)
-def get_students():
-    return [s.data for s in User.query.all()]
-
-
 @private.route('/')
 def private_index():
     return 'super duper secret'
 
 
-if is_debug():
-    @private.route('/token/<netid>')
-    def private_token_netid(netid):
-        user = User.query.filter_by(netid=netid).first()
-        if user is None:
-            return error_response('User does not exist')
-        res = Response(json.dumps(success_response(get_token(user.netid))), headers={'Content-Type': 'application/json'})
-        res.set_cookie('token', get_token(user.netid), httponly=True)
-        return res
+@private.route('/token/<netid>')
+def private_token_netid(netid):
+    user = User.query.filter_by(netid=netid).first()
+    if user is None:
+        return error_response('User does not exist')
+    token = get_token(user.netid)
+    res = Response(json.dumps(success_response(token)), headers={'Content-Type': 'application/json'})
+    res.set_cookie('token', token, httponly=True)
+    return res
 
 
 @private.route('/assignment/sync', methods=['POST'])
@@ -102,7 +91,7 @@ def private_assignment_sync(assignment_data: dict, tests: List[str]):
             Assignment.id == a.id,
             AssignmentTest.name == test_name,
         ).join(Assignment).first()
-        
+
         if at is None:
             at = AssignmentTest(assignment=a, name=test_name)
             db.session.add(at)
@@ -124,7 +113,7 @@ def private_dangling():
     """
 
     dangling = Submission.query.filter(
-        Submission.student_id == None,
+        Submission.owner_id == None,
     ).all()
     dangling = [a.data for a in dangling]
 
@@ -134,6 +123,38 @@ def private_dangling():
     })
 
 
+@private.route('/reset-dangling')
+@log_endpoint('reset-dangling', lambda: 'reset-dangling')
+@json_response
+def private_reset_dangling():
+    resets = []
+    for s in Submission.query.filter_by(owner_id=None).all():
+        s.init_submission_models()
+        resets.append(s.data)
+    return success_response({'reset': resets})
+
+
+@private.route('/regrade-submission/<commit>')
+@log_endpoint('cli', lambda: 'regrade-commit')
+@json_response
+def private_regrade_submission(commit):
+    s = Submission.query.filter(
+        Submission.commit == commit,
+        Submission.owner_id != None,
+    ).first()
+
+    if s is None:
+        return error_response('not found')
+
+    s.init_submission_models()
+    enqueue_webhook_rpc(s.id)
+
+    return success_response({
+        'submission': s.data,
+        'user': s.owner.data
+    })
+
+
 @private.route('/regrade/<assignment_name>')
 @log_endpoint('cli', lambda: 'regrade')
 @json_response
@@ -159,8 +180,9 @@ def private_regrade_assignment(assignment_name):
     if assignment is None:
         return error_response('cant find assignment')
 
-    submission = Submission.query.filter_by(
-        assignment=assignment
+    submission = Submission.query.filter(
+        Submission.assignment_id == assignment.id,
+        Submission.owner_id != None
     ).all()
 
     response = []
@@ -184,12 +206,11 @@ def private_fix_dangling():
     return fix_dangling()
 
 
-@private.route('/stats/<assignment_name>')
-@private.route('/stats/<assignment_name>/<netid>')
+@private.route('/stats/<assignment_id>')
+@private.route('/stats/<assignment_id>/<netid>')
 @log_endpoint('cli', lambda: 'stats')
-@cache.memoize(timeout=60, unless=lambda: request.args.get('netids', None) is not None)
 @json_response
-def private_stats_assignment(assignment_name, netid=None):
+def private_stats_assignment(assignment_id, netid=None):
     netids = request.args.get('netids', None)
 
     if netids is not None:
@@ -199,47 +220,7 @@ def private_stats_assignment(assignment_name, netid=None):
     else:
         netids = list(map(lambda x: x['netid'], get_students()))
 
-    students = get_students()
-    students = filter(
-        lambda x: x['netid'] in netids,
-        students
-    )
-
-    bests = {}
-
-    assignment = Assignment.query.filter_by(name=assignment_name).first()
-    if assignment is None:
-        return error_response('assignment does not exist')
-
-    for student in students:
-        submissionid = stats_for(student['id'], assignment.id)
-        netid = student['netid']
-        if submissionid is None:
-            # no submission
-            bests[netid] = None
-        else:
-            submission = Submission.query.filter_by(
-                id=submissionid
-            ).first()
-            build = len(submission.builds) > 0
-            best_count = sum(map(lambda x: 1 if x.passed else 0, submission.reports))
-            late = 'past due' if assignment.due_date < submission.timestamp else False
-            late = 'past grace' if assignment.grace_date < submission.timestamp else late
-            bests[netid] = {
-                'submission': submission.data,
-                'builds': build,
-                'reports': [rep.data for rep in submission.reports],
-                'total_tests_passed': best_count,
-                'repo_url': submission.repo,
-                'master': 'https://github.com/{}'.format(
-                    submission.repo[submission.repo.index(':') + 1:-len('.git')],
-                ),
-                'commit_tree': 'https://github.com/{}/tree/{}'.format(
-                    submission.repo[submission.repo.index(':') + 1:-len('.git')],
-                    submission.commit
-                ),
-                'late': late
-            }
+    bests = bulk_stats(assignment_id, netids)
     return success_response({'stats': bests})
 
 

api/anubis/routes/public.py

@@ -207,7 +207,7 @@ def public_submission(commit: str):
         return error_response('Commit does not exist'), 406
 
     # Hand back submission
-    return success_response({'submission': s.data})
+    return success_response({'submission': s.full_data})
 
 
 def webhook_log_msg():
@@ -308,6 +308,7 @@ def public_webhook():
     repo = AssignmentRepo.query.join(Assignment).join(Class_).join(InClass).join(User).filter(
         User.github_username == github_username,
         Assignment.unique_code == assignment.unique_code,
+        AssignmentRepo.repo_url == repo_url,
     ).first()
 
     logger.debug('webhook data', extra={