diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 923b30c51..117c03d85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,7 @@ jobs: DEBIAN_FRONTEND: noninteractive run: | sudo apt-get install gettext + docker pull davis68/relate-octave - name: Run test suite env: RL_CI_TEST: ${{ matrix.suite }} diff --git a/.gitignore b/.gitignore index 6685f8c7e..353c1d553 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,7 @@ bulk-storage /coverage.xml /.vscode/ /.venv/ + +my-relate-venv +package-lock.json +.gitignore diff --git a/course/page/__init__.py b/course/page/__init__.py index e268f9d69..d564f2883 100644 --- a/course/page/__init__.py +++ b/course/page/__init__.py @@ -37,6 +37,8 @@ ChoiceQuestion, MultipleChoiceQuestion, SurveyChoiceQuestion) from course.page.code import ( PythonCodeQuestion, PythonCodeQuestionWithHumanTextFeedback) +from course.page.code import ( + OctaveCodeQuestion) from course.page.upload import FileUploadQuestion __all__ = ( @@ -51,6 +53,7 @@ "ChoiceQuestion", "SurveyChoiceQuestion", "MultipleChoiceQuestion", "PythonCodeQuestion", "PythonCodeQuestionWithHumanTextFeedback", + "OctaveCodeQuestion", "FileUploadQuestion", ) diff --git a/course/page/code.py b/course/page/code.py index 6ace0324f..b9a889156 100644 --- a/course/page/code.py +++ b/course/page/code.py @@ -245,8 +245,10 @@ def debug_print(s): if container_id is not None: docker_cnx.start(container_id) + print(docker_cnx) container_props = docker_cnx.inspect_container(container_id) + print(container_props) (port_info,) = (container_props ["NetworkSettings"]["Ports"]["%d/tcp" % CODE_QUESTION_CONTAINER_PORT]) @@ -259,6 +261,8 @@ def debug_print(s): else: port = CODE_QUESTION_CONTAINER_PORT + #print(container_id,CODE_QUESTION_CONTAINER_PORT) + from time import time, sleep start_time = time() @@ -281,8 +285,9 @@ def check_timeout(): while True: try: connection = http_client.HTTPConnection(connect_host_ip, port) + print("attempting connection", connect_host_ip, port) - connection.request("GET", "/ping") + connection.request("GET", "/ping") # XXX here's the trouble response = connection.getresponse() response_data = response.read().decode() @@ -323,7 +328,7 @@ def check_timeout(): start_time = time() debug_print("BEFPOST") - connection.request("POST", "/run-python", json_run_req, headers) + connection.request("POST", "/run-code", json_run_req, headers) debug_print("AFTPOST") http_response = connection.getresponse() @@ -349,6 +354,7 @@ def check_timeout(): "exec_host": connect_host_ip, } finally: + print(container_id) if container_id is not None: debug_print("-----------BEGIN DOCKER LOGS for %s" % container_id) debug_print(docker_cnx.logs(container_id)) @@ -484,6 +490,7 @@ class CodeQuestion(PageBaseWithTitle, PageBaseWithValue): Optional. Symbols that the participant's code is expected to define. These will be made available to the :attr:`test_code`. + Some remapping of types will be made between Octave and Python classes. .. attribute:: test_code @@ -543,7 +550,7 @@ class CodeQuestion(PageBaseWithTitle, PageBaseWithValue): answer. This overrides the image set in the `local_settings.py` configuration. The Docker image should provide two files; these are supplied in RELATE's standard Python Docker image by `course/page/ - code_run_backend_python.py` and `course/page/code_feedback.py`, for + code_run_backend_py.py` and `course/page/code_feedback.py`, for instance. Consult `docker-image-run-py/docker-build.sh` for one example of a local build. The Docker image should already be loaded on the system (RELATE does not pull the image automatically). @@ -716,7 +723,14 @@ def get_test_code(self): if correct_code is None: correct_code = "" - from .code_run_backend import substitute_correct_code_into_test_code + if self.page_desc.type in [ + "OctaveCodeQuestion"]: + from .code_run_backend_octave \ + import substitute_correct_code_into_test_code + else: + from .code_run_backend_py \ + import substitute_correct_code_into_test_code + return substitute_correct_code_into_test_code(test_code, correct_code) @staticmethod @@ -1503,4 +1517,201 @@ def grade(self, page_context, page_data, answer_data, grade_data): # }}} + +# {{{ octave code question + +class OctaveCodeQuestion(CodeQuestion): + """ + An auto-graded question allowing an answer consisting of Octave code. + All user code as well as all code specified as part of the problem + is in Octave 4.2 or higher. + + If you are not including the + :attr:`course.constants.flow_permission.change_answer` + permission for your entire flow, you likely want to + include this snippet in your question definition: + + .. code-block:: yaml + + access_rules: + add_permissions: + - change_answer + + This will allow participants multiple attempts at getting + the right answer. + + .. attribute:: id + + |id-page-attr| + + .. attribute:: type + + ``OctaveCodeQuestion`` + + .. attribute:: is_optional_page + + |is-optional-page-attr| + + .. attribute:: access_rules + + |access-rules-page-attr| + + .. attribute:: title + + |title-page-attr| + + .. attribute:: value + + |value-page-attr| + + .. attribute:: prompt + + The page's prompt, written in :ref:`markup`. + + .. attribute:: timeout + + A number, giving the number of seconds for which setup code, + the given answer code, and the test code (combined) will be + allowed to run. + + .. attribute:: setup_code + + Optional. + Octave code to prepare the environment for the participants + answer. + + .. attribute:: show_setup_code + + Optional. ``True`` or ``False``. If true, the :attr:`setup_code` + will be shown to the participant. + + .. attribute:: names_for_user + + Optional. + Symbols defined at the end of the :attr:`setup_code` that will be + made available to the participant's code. + + A deep copy (using the standard library function :func:`copy.deepcopy`) + of these values is made, to prevent the user from modifying trusted + state of the grading code. + + .. attribute:: names_from_user + + Optional. + Symbols that the participant's code is expected to define. + These will be made available to the :attr:`test_code`. + + .. attribute:: test_code + + Optional. + Code that will be run to determine the correctness of a + student-provided solution. Will have access to variables in + :attr:`names_from_user` (which will be *None*) if not provided. Should + never raise an exception. + + This may contain the marker "###CORRECT_CODE###", which will + be replaced with the contents of :attr:`correct_code`, with + each line indented to the same depth as where the marker + is found. The line with this marker is only allowed to have + white space and the marker on it. + + .. attribute:: show_test_code + + Optional. ``True`` or ``False``. If true, the :attr:`test_code` + will be shown to the participant. + + .. attribute:: correct_code_explanation + + Optional. + Code that is revealed when answers are visible + (see :ref:`flow-permissions`). This is shown before + :attr:`correct_code` as an explanation. + + .. attribute:: correct_code + + Optional. + Code that is revealed when answers are visible + (see :ref:`flow-permissions`). + + .. attribute:: initial_code + + Optional. + Code present in the code input field when the participant first starts + working on their solution. + + .. attribute:: data_files + + Optional. + A list of file names in the :ref:`git-repo` whose contents will be made + available to :attr:`setup_code` and :attr:`test_code` through the + ``data_files`` dictionary. (see below) + + .. attribute:: single_submission + + Optional, a Boolean. If the question does not allow multiple submissions + based on its :attr:`access_rules` (not the ones of the flow), a warning + is shown. Setting this attribute to True will silence the warning. + + The following symbols are available in :attr:`setup_code` and :attr:`test_code`: + + * ``GradingComplete``: An exception class that can be raised to indicated + that the grading code has concluded. + + * ``feedback``: A class instance with the following interface:: + + feedback.set_points(0.5) # 0<=points<=1 (usually) + feedback.add_feedback("This was wrong") + + # combines the above two and raises GradingComplete + feedback.finish(0, "This was wrong") + + feedback.check_numpy_array_sanity(name, num_axes, data) + + feedback.check_numpy_array_features(name, ref, data, report_failure=True) + + feedback.check_numpy_array_allclose(name, ref, data, + accuracy_critical=True, rtol=1e-5, atol=1e-8, + report_success=True, report_failure=True) + # If report_failure is True, this function will only return + # if *data* passes the tests. It will return *True* in this + # case. + # + # If report_failure is False, this function will always return, + # and the return value will indicate whether *data* passed the + # accuracy/shape/kind checks. + + feedback.check_list(name, ref, data, entry_type=None) + + feedback.check_scalar(name, ref, data, accuracy_critical=True, + rtol=1e-5, atol=1e-8, report_success=True, report_failure=True) + # returns True if accurate + + feedback.call_user(f, *args, **kwargs) + # Calls a user-supplied function and prints an appropriate + # feedback message in case of failure. + + * ``data_files``: A dictionary mapping file names from :attr:`data_files` + to :class:`bytes` instances with that file's contents. + + * ``user_code``: The user code being tested, as a string. + """ + + @property + def language_mode(self): + return "octave" + + @property + def container_image(self): + return settings.RELATE_DOCKER_RUNOC_IMAGE + + @property + def suffix(self): + return ".m" + + def __init__(self, vctx, location, page_desc, language_mode="octave"): + super(OctaveCodeQuestion, self).__init__(vctx, location, page_desc, + language_mode) + +# }}} + # vim: foldmethod=marker diff --git a/course/page/code_run_backend_octave.py b/course/page/code_run_backend_octave.py new file mode 100644 index 000000000..a05d8f57c --- /dev/null +++ b/course/page/code_run_backend_octave.py @@ -0,0 +1,303 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import + +__copyright__ = "Copyright (C) 2014 Andreas Kloeckner" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import sys +import traceback + +try: + from .code_feedback import Feedback, GradingComplete +except SystemError: + from code_feedback import Feedback, GradingComplete # type: ignore +except ImportError: + from code_feedback import Feedback, GradingComplete # type: ignore + + +__doc__ = """ +PROTOCOL +======== + +.. class:: Request + + .. attribute:: setup_code + + .. attribute:: names_for_user + + .. attribute:: user_code + + .. attribute:: names_from_user + + .. attribute:: test_code + + .. attribute:: data_files + + A dictionary from data file names to their + base64-cencoded contents. + Optional. + + .. attribute:: compile_only + + :class:`bool` + +.. class Response:: + .. attribute:: result + + One of + + * ``success`` + * ``timeout`` + * ``uncaught_error`` + * ``setup_compile_error`` + * ``setup_error``, + * ``user_compile_error`` + * ``user_error`` + * ``test_compile_error`` + * ``test_error`` + + Always present. + + .. attribute:: message + + Optional. + + .. attribute:: traceback + + Optional. + + .. attribute:: stdout + + Whatever came out of stdout. + + Optional. + + .. attribute:: stderr + + Whatever came out of stderr. + + Optional. + + .. attribute:: figures + + A list of ``(index, mime_type, string)``, where *string* is a + base64-encoded representation of the figure. *index* will usually + correspond to the matplotlib figure number. + + Optional. + + .. attribute:: html + + A list of HTML strings generated. These are aggressively sanitized + before being rendered. + + .. attribute:: points + + A number between 0 and 1 (inclusive). + + Present on ``success`` if :attr:`Request.compile_only` is *False*. + + .. attribute:: feedback + + A list of strings. + + Present on ``success`` if :attr:`Request.compile_only` is *False*. +""" + + +# {{{ tools + +class Struct(object): + def __init__(self, entries): + for name, val in entries.items(): + self.__dict__[name] = val + + def __repr__(self): + return repr(self.__dict__) + +# }}} + + +def substitute_correct_code_into_test_code(test_code, correct_code): + import re + CORRECT_CODE_TAG = re.compile(r"^(\s*)###CORRECT_CODE###\s*$") # noqa + + new_test_code_lines = [] + for line in test_code.split("\n"): + match = CORRECT_CODE_TAG.match(line) + if match is not None: + prefix = match.group(1) + for cc_l in correct_code.split("\n"): + new_test_code_lines.append(prefix+cc_l) + else: + new_test_code_lines.append(line) + + return "\n".join(new_test_code_lines) + + +def package_exception(result, what): + tp, val, tb = sys.exc_info() + result["result"] = what + result["message"] = "%s: %s" % (tp.__name__, str(val)) + result["traceback"] = "".join( + traceback.format_exception(tp, val, tb)) + + +def run_code(result, run_req): + # {{{ set up octave process + + import oct2py + + oc = oct2py.Oct2Py() + + # }}} + + # {{{ run code + + data_files = {} + if hasattr(run_req, "data_files"): + from base64 import b64decode + for name, contents in run_req.data_files.items(): + # This part "cheats" a litle, since Octave lets us evaluate functions + # in the same context as the main code. (MATLAB segregates these.) + data_files[name] = b64decode(contents.encode()) + oc.eval(b64decode(contents.encode()).decode("utf-8")) + + generated_html = [] + result["html"] = generated_html + + def output_html(s): + generated_html.append(s) + + feedback = Feedback() + maint_ctx = { + "feedback": feedback, + "user_code": run_req.user_code, + "data_files": data_files, + "output_html": output_html, + "GradingComplete": GradingComplete, + } + + if getattr(run_req, "setup_code", None): + try: + oc.eval(run_req.setup_code) + # put variables from user in main context + for name in run_req.names_for_user: + try: + maint_ctx[name] = oc.pull(name) + except oct2py.Oct2PyError: + maint_ctx[name] = None + except Exception: + package_exception(result, "setup_error") + return + + if getattr(run_req, "test_code", None): + try: + test_code = compile( + run_req.test_code, "[test code]", "exec") + except Exception: + package_exception(result, "test_compile_error") + return + else: + test_code = None + + if hasattr(run_req, "compile_only") and run_req.compile_only: + result["result"] = "success" + return + + """ + user_ctx = {} + if hasattr(run_req, "names_for_user"): #XXX unused for Octave context currently + for name in run_req.names_for_user: + if name not in maint_ctx: + result["result"] = "setup_error" + result["message"] = "Setup code did not define \'%s\'." % name + + user_ctx[name] = maint_ctx[name] + + from copy import deepcopy + user_ctx = deepcopy(user_ctx) + """ + + try: + #user_ctx["_MODULE_SOURCE_CODE"] = run_req.user_code + oc.eval(run_req.user_code) + except Exception: + package_exception(result, "user_error") + return + + # {{{ export plots + + """ + if 'matplotlib' in sys.modules: + import matplotlib.pyplot as pt + from io import BytesIO + from base64 import b64encode + + format = 'png' + mime = 'image/png' + figures = [] + + for fignum in pt.get_fignums(): + pt.figure(fignum) + bio = BytesIO() + try: + pt.savefig(bio, format=format) + except Exception: + pass + else: + figures.append( + (fignum, mime, b64encode(bio.getvalue()).decode())) + + result['figures'] = figures + """ + # }}} + + if hasattr(run_req, "names_from_user"): + for name in run_req.names_from_user: + try: + maint_ctx[name] = oc.pull(name) + except oct2py.Oct2PyError: + feedback.add_feedback( + "Required answer variable '%s' is not defined." + % name) + maint_ctx[name] = None + + if test_code is not None: # XXX test code is written in Python + try: + maint_ctx["_MODULE_SOURCE_CODE"] = run_req.test_code + exec(test_code, maint_ctx) + except GradingComplete: + pass + except Exception: + package_exception(result, "test_error") + return + + result["points"] = feedback.points + result["feedback"] = feedback.feedback_items + + # }}} + + result["result"] = "success" + +# vim: foldmethod=marker diff --git a/course/page/code_run_backend.py b/course/page/code_run_backend_py.py similarity index 100% rename from course/page/code_run_backend.py rename to course/page/code_run_backend_py.py diff --git a/doc/misc.rst b/doc/misc.rst index d1a3d3706..aff943024 100644 --- a/doc/misc.rst +++ b/doc/misc.rst @@ -134,6 +134,8 @@ You should also pull the default container image:: docker pull inducer/relate-runpy-amd64 +(or `docker pull davis68/relate-octave` if using Octave). + Add to kernel command line, if needed:: [...] cgroup_enable=memory swapaccount=1 diff --git a/docker-image-run-octave/Dockerfile b/docker-image-run-octave/Dockerfile new file mode 100644 index 000000000..920898c46 --- /dev/null +++ b/docker-image-run-octave/Dockerfile @@ -0,0 +1,61 @@ +FROM inducer/debian-amd64-minbase +MAINTAINER Neal Davis +EXPOSE 9941 +RUN useradd runcode + +RUN echo 'APT::Default-Release "testing";' >> /etc/apt/apt.conf + +# Based on `compdatasci/octave-desktop` Docker image +ARG OCTAVE_VERSION=5.1.0 + +# Install system packages and Octave +RUN apt-get update +RUN apt-get install -y --no-install-recommends \ + wget \ + curl \ + build-essential \ + gfortran \ + cmake \ + libarchive-tools \ + rsync \ + imagemagick \ + \ + gnuplot-x11 \ + libopenblas-base +RUN apt-get install -y --no-install-recommends \ + octave \ + liboctave-dev \ + octave-parallel \ + octave-struct \ + octave-statistics +RUN apt-get install -y --no-install-recommends \ + python3-dev \ + python3-setuptools \ + python3-pip \ + python3-numpy \ + python3-scipy \ + python3-matplotlib \ + pandoc +RUN apt-get clean && \ + apt-get autoremove && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN apt-get update +RUN pip3 install --upgrade pip +RUN pip3 install oct2py + +RUN apt-get clean +RUN fc-cache + +RUN mkdir -p /opt/runcode +ADD runcode /opt/runcode/ +COPY code_feedback.py /opt/runcode/ +COPY code_run_backend.py /opt/runcode/ +RUN ls /opt/runcode + +# currently no graphics support +#TODO + +RUN rm -Rf /root/.cache + +# may use ./flatten-container.sh to reduce disk space diff --git a/docker-image-run-octave/docker-build.sh b/docker-image-run-octave/docker-build.sh new file mode 100755 index 000000000..6321ae18c --- /dev/null +++ b/docker-image-run-octave/docker-build.sh @@ -0,0 +1,5 @@ +#! /bin/sh +cp ../course/page/code_feedback.py . +cp ../course/page/code_run_backend_octave.py ./code_run_backend.py +docker build --no-cache . -t davis68/relate-octave +rm code_feedback.py code_run_backend.py diff --git a/docker-image-run-octave/flatten-container.sh b/docker-image-run-octave/flatten-container.sh new file mode 100755 index 000000000..d18bb6434 --- /dev/null +++ b/docker-image-run-octave/flatten-container.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +if test "$1" = ""; then + echo "$0 imagename" + exit 1 +fi +CONTAINER=$(docker create "$1") +docker export "$CONTAINER" | \ + docker import \ + -c "EXPOSE 9941" \ + - +docker rm -f $CONTAINER diff --git a/docker-image-run-octave/runcode b/docker-image-run-octave/runcode new file mode 100755 index 000000000..2b130aa4d --- /dev/null +++ b/docker-image-run-octave/runcode @@ -0,0 +1,156 @@ +#! /usr/bin/env python3 + +# placate flake8 +from __future__ import print_function + +__copyright__ = "Copyright (C) 2014 Andreas Kloeckner" + +__license__ = """ +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +""" + +import socketserver +import json +import sys +import io +try: + from code_run_backend import Struct, run_code, package_exception +except ImportError: + try: + # When faking a container for unittest + from course.page.code_run_backend import ( + Struct, run_code, package_exception) + except ImportError: + # When debugging, i.e., run "octave-cli runoc" command line + import os + sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir))) + from course.page.code_run_backend import ( + Struct, run_code, package_exception) + +from http.server import BaseHTTPRequestHandler + +PORT = 9941 +OUTPUT_LENGTH_LIMIT = 16*1024 + +TEST_COUNT = 0 + + +def truncate_if_long(s): + if len(s) > OUTPUT_LENGTH_LIMIT: + s = s[:OUTPUT_LENGTH_LIMIT] + "[TRUNCATED... TOO MUCH OUTPUT]" + return s + + +class RunRequestHandler(BaseHTTPRequestHandler): + def do_GET(self): + print("GET RECEIVED", file=sys.stderr) + if self.path != "/ping": + raise RuntimeError("unrecognized path in GET") + + self.send_response(200) + self.send_header("Content-type", "text/plain") + self.end_headers() + + self.wfile.write(b"OK") + print("PING RESPONSE DONE", file=sys.stderr) + + def do_POST(self): + global TEST_COUNT + TEST_COUNT += 1 + + response = {} + + prev_stdout = sys.stdout # noqa + prev_stderr = sys.stderr # noqa + + try: + print("POST RECEIVED", file=prev_stderr) + if self.path != "/run-code": + raise RuntimeError("unrecognized path in POST") + + clength = int(self.headers['content-length']) + recv_data = self.rfile.read(clength) + + print("RUNPY RECEIVED %d bytes" % len(recv_data), + file=prev_stderr) + run_req = Struct(json.loads(recv_data.decode("utf-8"))) + print("REQUEST: %r" % run_req, file=prev_stderr) + + stdout = io.StringIO() + stderr = io.StringIO() + + sys.stdin = None + sys.stdout = stdout + sys.stderr = stderr + + run_code(response, run_req) + + response["stdout"] = truncate_if_long(stdout.getvalue()) + response["stderr"] = truncate_if_long(stderr.getvalue()) + + print("REQUEST SERVICED: %r" % response, file=prev_stderr) + + json_result = json.dumps(response).encode("utf-8") + + self.send_response(200) + self.send_header("Content-type", "application/json") + self.end_headers() + + print("WRITING RESPONSE", file=prev_stderr) + self.wfile.write(json_result) + print("WROTE RESPONSE", file=prev_stderr) + except: + print("ERROR RESPONSE", file=prev_stderr) + response = {} + package_exception(response, "uncaught_error") + json_result = json.dumps(response).encode("utf-8") + + self.send_response(500) + self.send_header("Content-type", "application/json") + self.end_headers() + + self.wfile.write(json_result) + finally: + sys.stdout = prev_stdout + sys.stderr = prev_stderr + + +def main(): + print("STARTING, LISTENING ON %d" % PORT, file=sys.stderr) + server = socketserver.TCPServer(("", PORT), RunRequestHandler) + + serve_single_test = len(sys.argv) > 1 and sys.argv[1] == "-1" + + while True: + server.handle_request() + print("SERVED REQUEST", file=sys.stderr) + if TEST_COUNT > 0 and serve_single_test: + break + + server.server_close() + print("FINISHED server_close()", file=sys.stderr) + + print("EXITING", file=sys.stderr) + + +if __name__ == "__main__": + main() + +# vim: foldmethod=marker diff --git a/docker-image-run-py/docker-build.sh b/docker-image-run-py/docker-build.sh index a53f0528f..b309ae7a3 100755 --- a/docker-image-run-py/docker-build.sh +++ b/docker-image-run-py/docker-build.sh @@ -1,5 +1,5 @@ #! /bin/sh cp ../course/page/code_feedback.py . -cp ../course/page/code_run_backend.py . -docker build --no-cache . -t inducer/relate-runcode-python +cp ../course/page/code_run_backend_py.py ./code_run_backend.py +docker build --no-cache . -t inducer/relate-runpy-amd64 rm code_feedback.py code_run_backend.py diff --git a/docker-image-run-py/runcode b/docker-image-run-py/runcode index 0e124466c..c0bfe4bc9 100755 --- a/docker-image-run-py/runcode +++ b/docker-image-run-py/runcode @@ -84,7 +84,7 @@ class RunRequestHandler(BaseHTTPRequestHandler): try: print("POST RECEIVED", file=prev_stderr) - if self.path != "/run-python": + if self.path != "/run-code": raise RuntimeError("unrecognized path in POST") clength = int(self.headers['content-length']) diff --git a/local_settings_example.py b/local_settings_example.py index 5bccadcea..ac827ea2a 100644 --- a/local_settings_example.py +++ b/local_settings_example.py @@ -376,7 +376,8 @@ # A string containing the image ID of the docker image to be used to run # student Python code. Docker should download the image on first run. -RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runcode-python" +RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runpy-amd64" +RELATE_DOCKER_RUNOC_IMAGE = "davis68/relate-octave" # RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runpy-amd64-tensorflow" # (bigger, but includes TensorFlow) diff --git a/relate/bin/relate.py b/relate/bin/relate.py index 64c190e4e..2de524111 100644 --- a/relate/bin/relate.py +++ b/relate/bin/relate.py @@ -50,11 +50,6 @@ def expand_yaml(yml_file, repo_root): # {{{ code test def test_code_question(page_desc, repo_root): - if page_desc.type not in [ - "PythonCodeQuestion", - "PythonCodeQuestionWithHumanTextFeedback"]: - return - print(75*"-") print("TESTING", page_desc.id, "...", end=" ") sys.stdout.flush() diff --git a/setup.cfg b/setup.cfg index 68b4dcf26..5317924e3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,8 @@ omit = setup.py local_settings_example.py course/page/code_feedback.py - course/page/code_run_backend.py + course/page/code_run_backend_py.py + course/page/code_run_backend_octave.py */wsgi.py */tests/* */tests.py diff --git a/tests/test_pages/markdowns.py b/tests/test_pages/markdowns.py index dc0039d84..e321f55e9 100644 --- a/tests/test_pages/markdowns.py +++ b/tests/test_pages/markdowns.py @@ -418,4 +418,358 @@ # }}} +# {{{ octave code questions + +OCTAVE_CODE_MARKDWON = """ +type: OctaveCodeQuestion +access_rules: + add_permissions: + - change_answer +id: addition +value: 1 +timeout: 10 +prompt: | + + # Adding 1 and 2, and assign it to c + +names_from_user: [c] + +initial_code: | + c = + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = 3 + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + + c = 2 + 1 + +correct_code_explanation: This is the [explanation](http://example.com/1). +""" + +OCTAVE_CODE_MARKDWON_PATTERN_WITH_DATAFILES = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +data_files: + - question-data/random-data.m + %(extra_data_file)s +prompt: | + + # Adding two numbers in Octave + +setup_code: | + pkg load statistics + a = unifrnd(-10,10) + b = unifrnd(-10,10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" + +OCTAVE_CODE_MARKDWON_WITH_DATAFILES_BAD_FORMAT = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +data_files: + - question-data/random-data.m + - - foo + - bar +prompt: | + + # Adding two numbers in Octave + +setup_code: | + pkg load statistics + a = unifrnd(-10,10) + b = unifrnd(-10,10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" + +OCTAVE_CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT1 = """ +type: OctaveCodeQuestion +access_rules: + add_permissions: + - see_answer_after_submission +id: addition +value: 1 +timeout: 10 +prompt: | + + # Adding two numbers in Octave + +setup_code: | + pkg load statistics + a = unifrnd(-10,10) + b = unifrnd(-10,10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" + +OCTAVE_CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT2 = """ +type: OctaveCodeQuestion +access_rules: + remove_permissions: + - see_answer_after_submission +id: addition +value: 1 +timeout: 10 +prompt: | + + # Adding two numbers in Octave + +setup_code: | + pkg load statistics + a = unifrnd(-10,10) + b = unifrnd(-10,10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" + +OCTAVE_CODE_MARKDWON_PATTERN_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +single_submission: True +prompt: | + + # Adding two numbers in Octave + +setup_code: | + pkg load statistics + a = unifrnd(-10,10) + b = unifrnd(-10,10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +correct_code: | + c = a + b +""" + +OCTAVE_CODE_MARKDWON_PATTERN_WITHOUT_TEST_CODE = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +single_submission: True +prompt: | + + # Adding two numbers in Octave + +setup_code: | + pkg load statistics + a = unifrnd(-10,10) + b = unifrnd(-10,10) + +names_for_user: [a, b] + +names_from_user: [c] + +correct_code: | + c = a + b +""" + +OCTAVE_CODE_MARKDWON_PATTERN_WITHOUT_CORRECT_CODE = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +single_submission: True +prompt: | + + # Adding two numbers in Octave + +setup_code: | + pkg load statistics + a = unifrnd(-10,10) + b = unifrnd(-10,10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(1, "Your computed c was correct.") + else: + feedback.finish(0, "Your computed c was incorrect.") + +""" + +OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +prompt: | + + # Adding two numbers in Octave + +setup_code: | + pkg load statistics + a = unifrnd(-10,10) + b = unifrnd(-10,10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(%(full_points)s, "Your computed c was correct.") + else: + feedback.finish(%(min_points)s, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" # noqa + +OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN = """ +type: OctaveCodeQuestion +id: addition +value: 1 +timeout: 10 +prompt: | + + # Adding two numbers in Octave + +setup_code: | + pkg load statistics + a = unifrnd(-10,10) + b = unifrnd(-10,10) + +names_for_user: [a, b] + +names_from_user: [c] + +test_code: | + if not isinstance(c, float): + feedback.finish(0, "Your computed c is not a float.") + + correct_c = a + b + rel_err = abs(correct_c-c)/abs(correct_c) + + if rel_err < 1e-7: + feedback.finish(%(full_points)s, "Your computed c was correct.") + else: + feedback.finish(%(min_points)s, "Your computed c was incorrect.") + +correct_code: | + + c = a + b +""" # noqa + +# }}} + # vim: fdm=marker diff --git a/tests/test_pages/test_code.py b/tests/test_pages/test_code.py index f9711e8a9..a8843eafa 100644 --- a/tests/test_pages/test_code.py +++ b/tests/test_pages/test_code.py @@ -244,7 +244,8 @@ def test_not_multiple_submit_warning(self): ) def test_not_multiple_submit_warning2(self): - markdown = markdowns.CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT1 + markdown = \ + markdowns.CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT1 resp = self.get_page_sandbox_preview_response(markdown) self.assertEqual(resp.status_code, 200) self.assertSandboxHasValidPage(resp) @@ -254,7 +255,8 @@ def test_not_multiple_submit_warning2(self): ) def test_not_multiple_submit_warning3(self): - markdown = markdowns.CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT2 + markdown = \ + markdowns.CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT2 resp = self.get_page_sandbox_preview_response(markdown) self.assertEqual(resp.status_code, 200) self.assertSandboxHasValidPage(resp) @@ -271,8 +273,8 @@ def test_allow_multiple_submit(self): self.assertSandboxWarningTextContain(resp, None) def test_explicity_not_allow_multiple_submit(self): - markdown = ( - markdowns.CODE_MARKDWON_PATTERN_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT + markdown = (markdowns. + CODE_MARKDWON_PATTERN_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT % {"extra_data_file": ""} ) resp = self.get_page_sandbox_preview_response(markdown) @@ -308,6 +310,106 @@ def test_question_without_correct_code(self): self.assertEqual(resp.status_code, 200) self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 1) + def test_data_files_missing_random_question_data_file_octave(self): + file_name = "question-data/random-data.m" + markdown = ( + markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITH_DATAFILES + % {"extra_data_file": "- %s" % file_name} + ) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxNotHasValidPage(resp) + self.assertResponseContextContains( + resp, PAGE_ERRORS, "data file '%s' not found" % file_name) + + def test_data_files_missing_random_question_data_file_bad_format_octave(self): + markdown = markdowns.OCTAVE_CODE_MARKDWON_WITH_DATAFILES_BAD_FORMAT + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxNotHasValidPage(resp) + self.assertResponseContextContains( + resp, PAGE_ERRORS, + "data file '%s' not found" % "question-data/random-data.m") + + def test_not_multiple_submit_warning_octave(self): + markdown = ( + markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITH_DATAFILES + % {"extra_data_file": ""} + ) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain( + resp, + NOT_ALLOW_MULTIPLE_SUBMISSION_WARNING + ) + + def test_not_multiple_submit_warning2_octave(self): + markdown = markdowns. \ + OCTAVE_CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT1 + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain( + resp, + NOT_ALLOW_MULTIPLE_SUBMISSION_WARNING + ) + + def test_not_multiple_submit_warning3_octave(self): + markdown = markdowns. \ + OCTAVE_CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT2 + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain( + resp, + NOT_ALLOW_MULTIPLE_SUBMISSION_WARNING + ) + + def test_allow_multiple_submit_octave(self): + markdown = markdowns.OCTAVE_CODE_MARKDWON + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain(resp, None) + + def test_explicity_not_allow_multiple_submit_octave(self): + markdown = (markdowns. + OCTAVE_CODE_MARKDWON_PATTERN_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT + % {"extra_data_file": ""}) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain(resp, None) + + def test_question_without_test_code_octave(self): + markdown = markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITHOUT_TEST_CODE + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain(resp, None) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, None) + self.assertResponseContextAnswerFeedbackContainsFeedback( + resp, NO_CORRECTNESS_INFO_MSG) + + def test_question_without_correct_code_octave(self): + markdown = markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITHOUT_CORRECT_CODE + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + self.assertSandboxWarningTextContain(resp, None) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 1) + def test_question_with_human_feedback_both_feedback_value_feedback_percentage_present(self): # noqa markdown = (markdowns.CODE_WITH_HUMAN_FEEDBACK_MARKDWON_PATTERN % {"value": 3, @@ -538,7 +640,7 @@ def assert_runpy_result_and_response(self, result_type, expected_msgs=None, self.assertEqual(resp.status_code, 200) self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, - correctness) + correctness) self.assertEqual(len(mail.outbox), mail_count) def test_request_run_with_retries_timed_out(self): @@ -979,7 +1081,312 @@ def test_feedback_code_error_exceed_max_extra_credit_factor_email(self): markdown, answer_data={"answer": ['c = b + a\r']}) self.assertEqual(resp.status_code, 200) - self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, None) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, + None) + error_msg = (AUTO_FEEDBACK_POINTS_OUT_OF_RANGE_ERROR_MSG_PATTERN + % (MAX_EXTRA_CREDIT_FACTOR, invalid_feedback_points)) + + self.assertResponseContextAnswerFeedbackNotContainsFeedback( + resp, error_msg) + + self.assertResponseContextAnswerFeedbackContainsFeedback( + resp, GRADE_CODE_FAILING_MSG) + self.assertEqual(len(mail.outbox), 1) + + self.assertIn(error_msg, mail.outbox[0].body) + + # }}} + + # {{{ Octave code tests patterned after Python tests + + # def test_data_files_missing_random_question_data_file(self): + # file_name = "foo" + # markdown = ( + # markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITH_DATAFILES + # % {"extra_data_file": "- %s" % file_name} + # ) + # resp = self.get_page_sandbox_preview_response(markdown) + # self.assertEqual(resp.status_code, 200) + # self.assertSandboxNotHasValidPage(resp) + # self.assertResponseContextContains( + # resp, PAGE_ERRORS, "data file '%s' not found" % file_name) + + # def test_data_files_missing_random_question_data_file_bad_format(self): + # markdown = markdowns.OCTAVE_CODE_MARKDWON_WITH_DATAFILES_BAD_FORMAT + # resp = self.get_page_sandbox_preview_response(markdown) + # self.assertEqual(resp.status_code, 200) + # self.assertSandboxNotHasValidPage(resp) + # self.assertResponseContextContains( + # resp, PAGE_ERRORS, "data file '%s' not found" % "['foo', 'bar']") + + # def test_not_multiple_submit_warning(self): + # markdown = ( + # markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITH_DATAFILES + # % {"extra_data_file": ""} + # ) + # resp = self.get_page_sandbox_preview_response(markdown) + # self.assertEqual(resp.status_code, 200) + # self.assertSandboxHasValidPage(resp) + # self.assertSandboxWarningTextContain( + # resp, + # NOT_ALLOW_MULTIPLE_SUBMISSION_WARNING + # ) + + # def test_not_multiple_submit_warning2(self): + # markdown = markdowns. \ + # OCTAVE_CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT1 + # resp = self.get_page_sandbox_preview_response(markdown) + # self.assertEqual(resp.status_code, 200) + # self.assertSandboxHasValidPage(resp) + # self.assertSandboxWarningTextContain( + # resp, + # NOT_ALLOW_MULTIPLE_SUBMISSION_WARNING + # ) + + # def test_not_multiple_submit_warning3(self): + # markdown = markdowns. \ + # OCTAVE_CODE_MARKDWON_NOT_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT2 + # resp = self.get_page_sandbox_preview_response(markdown) + # self.assertEqual(resp.status_code, 200) + # self.assertSandboxHasValidPage(resp) + # self.assertSandboxWarningTextContain( + # resp, + # NOT_ALLOW_MULTIPLE_SUBMISSION_WARNING + # ) + + # def test_allow_multiple_submit(self): + # markdown = markdowns.OCTAVE_CODE_MARKDWON + # resp = self.get_page_sandbox_preview_response(markdown) + # self.assertEqual(resp.status_code, 200) + # self.assertSandboxHasValidPage(resp) + # self.assertSandboxWarningTextContain(resp, None) + + # def test_explicity_not_allow_multiple_submit(self): + # markdown = (markdowns. \ + # OCTAVE_CODE_MARKDWON_PATTERN_EXPLICITLY_NOT_ALLOW_MULTI_SUBMIT + # % {"extra_data_file": ""} + # ) + # resp = self.get_page_sandbox_preview_response(markdown) + # self.assertEqual(resp.status_code, 200) + # self.assertSandboxHasValidPage(resp) + # self.assertSandboxWarningTextContain(resp, None) + + # def test_question_without_test_code(self): + # markdown = markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITHOUT_TEST_CODE + # resp = self.get_page_sandbox_preview_response(markdown) + # self.assertEqual(resp.status_code, 200) + # self.assertSandboxHasValidPage(resp) + # self.assertSandboxWarningTextContain(resp, None) + + # resp = self.get_page_sandbox_submit_answer_response( + # markdown, + # answer_data={"answer": ['c = b + a\r']}) + # self.assertEqual(resp.status_code, 200) + # self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, None) + # self.assertResponseContextAnswerFeedbackContainsFeedback( + # resp, NO_CORRECTNESS_INFO_MSG) + + # def test_question_without_correct_code(self): + # markdown = markdowns.OCTAVE_CODE_MARKDWON_PATTERN_WITHOUT_CORRECT_CODE + # resp = self.get_page_sandbox_preview_response(markdown) + # self.assertEqual(resp.status_code, 200) + # self.assertSandboxHasValidPage(resp) + # self.assertSandboxWarningTextContain(resp, None) + + # resp = self.get_page_sandbox_submit_answer_response( + # markdown, + # answer_data={"answer": ['c = b + a\r']}) + # self.assertEqual(resp.status_code, 200) + # self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 1) + + def test_feedback_points_close_to_1_octave(self): + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": 1.000000000002, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 1) + + def test_feedback_code_exceed_1_octave(self): + feedback_points = 1.1 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": feedback_points, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 1.1) + + expected_feedback = "Your answer is correct and earned bonus points." + + self.assertResponseContextAnswerFeedbackContainsFeedback( + resp, expected_feedback) + + def test_feedback_code_positive_close_to_0_octave(self): + # https://github.com/inducer/relate/pull/448#issuecomment-363655132 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": 1, + "min_points": 0.00000000001 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + # Post a wrong answer + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b - a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 0) + + def test_feedback_code_negative_close_to_0_octave(self): + # https://github.com/inducer/relate/pull/448#issuecomment-363655132 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": 1, + "min_points": -0.00000000001 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + # Post a wrong answer + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b - a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, 0) + + def test_feedback_code_error_close_below_max_auto_feedback_points_octave(self): + feedback_points = MAX_EXTRA_CREDIT_FACTOR - 1e-6 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": feedback_points, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals( + resp, MAX_EXTRA_CREDIT_FACTOR) + + def test_feedback_code_error_close_above_max_auto_feedback_points_octave(self): + feedback_points = MAX_EXTRA_CREDIT_FACTOR + 1e-6 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": feedback_points, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals( + resp, MAX_EXTRA_CREDIT_FACTOR) + + def test_feedback_code_error_negative_feedback_points_octave(self): + invalid_feedback_points = -0.1 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": 1, + "min_points": invalid_feedback_points + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + # Post a wrong answer + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b - a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, None) + + error_msg = (AUTO_FEEDBACK_POINTS_OUT_OF_RANGE_ERROR_MSG_PATTERN + % (MAX_EXTRA_CREDIT_FACTOR, invalid_feedback_points)) + + self.assertResponseContextAnswerFeedbackNotContainsFeedback( + resp, error_msg) + + self.assertResponseContextAnswerFeedbackContainsFeedback( + resp, GRADE_CODE_FAILING_MSG) + + def test_feedback_code_error_exceed_max_extra_credit_factor_octave(self): + invalid_feedback_points = 10.1 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": invalid_feedback_points, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, None) + error_msg = (AUTO_FEEDBACK_POINTS_OUT_OF_RANGE_ERROR_MSG_PATTERN + % (MAX_EXTRA_CREDIT_FACTOR, invalid_feedback_points)) + + self.assertResponseContextAnswerFeedbackNotContainsFeedback( + resp, error_msg) + + self.assertResponseContextAnswerFeedbackContainsFeedback( + resp, GRADE_CODE_FAILING_MSG) + + def test_feedback_code_error_exceed_max_extra_credit_factor_email_octave(self): + invalid_feedback_points = 10.1 + markdown = (markdowns.OCTAVE_FEEDBACK_POINTS_CODE_MARKDWON_PATTERN + % { + "full_points": invalid_feedback_points, + "min_points": 0 + }) + resp = self.get_page_sandbox_preview_response(markdown) + self.assertEqual(resp.status_code, 200) + self.assertSandboxHasValidPage(resp) + + with mock.patch("course.page.PageContext") as mock_page_context: + mock_page_context.return_value.in_sandbox = False + mock_page_context.return_value.course = self.course + + # This remove the warning caused by mocked commit_sha value + # "CacheKeyWarning: Cache key contains characters that + # will cause errors ..." + mock_page_context.return_value.commit_sha = b"1234" + + resp = self.get_page_sandbox_submit_answer_response( + markdown, + answer_data={"answer": ['c = b + a\r']}) + self.assertEqual(resp.status_code, 200) + self.assertResponseContextAnswerFeedbackCorrectnessEquals(resp, + None) error_msg = (AUTO_FEEDBACK_POINTS_OUT_OF_RANGE_ERROR_MSG_PATTERN % (MAX_EXTRA_CREDIT_FACTOR, invalid_feedback_points)) @@ -1031,12 +1438,12 @@ def test_image_not_none(self): def test_docker_container_ping_failure(self): with ( - mock.patch("docker.client.Client.create_container")) as mock_create_ctn, ( # noqa - mock.patch("docker.client.Client.start")) as mock_ctn_start, ( - mock.patch("docker.client.Client.logs")) as mock_ctn_logs, ( - mock.patch("docker.client.Client.remove_container")) as mock_remove_ctn, ( # noqa - mock.patch("docker.client.Client.inspect_container")) as mock_inpect_ctn, ( # noqa - mock.patch("http.client.HTTPConnection.request")) as mock_ctn_request: # noqa + mock.patch("docker.client.Client.create_container")) as mock_create_ctn, ( # noqa + mock.patch("docker.client.Client.start")) as mock_ctn_start, ( + mock.patch("docker.client.Client.logs")) as mock_ctn_logs, ( + mock.patch("docker.client.Client.remove_container")) as mock_remove_ctn, ( # noqa + mock.patch("docker.client.Client.inspect_container")) as mock_inpect_ctn, ( # noqa + mock.patch("http.client.HTTPConnection.request")) as mock_ctn_request: # noqa mock_create_ctn.return_value = {"Id": "someid"} mock_ctn_start.side_effect = lambda x: None diff --git a/tests/test_pages/utils.py b/tests/test_pages/utils.py index b4cd67705..68ecc1379 100644 --- a/tests/test_pages/utils.py +++ b/tests/test_pages/utils.py @@ -69,6 +69,7 @@ def _skip_real_docker_test(): REAL_RELATE_DOCKER_URL = "unix:///var/run/docker.sock" REAL_RELATE_DOCKER_TLS_CONFIG = None REAL_RELATE_DOCKER_RUNPY_IMAGE = "inducer/relate-runcode-python" +REAL_RELATE_DOCKER_RUNOC_IMAGE = "davis68/relate-octave" class RealDockerTestMixin: @@ -109,3 +110,7 @@ def make_sure_docker_image_pulled(cls): if not bool(cli.images(REAL_RELATE_DOCKER_RUNPY_IMAGE)): # This should run only once and get cached on Travis-CI cli.pull(REAL_RELATE_DOCKER_RUNPY_IMAGE) + + if not bool(cli.images(REAL_RELATE_DOCKER_RUNOC_IMAGE)): + # This should run only once and get cached on Travis-CI + cli.pull(REAL_RELATE_DOCKER_RUNOC_IMAGE)