From d81ac40b7381b75b81e36986e8161d7f34657cc4 Mon Sep 17 00:00:00 2001 From: Stefan Kraus Date: Wed, 25 Jul 2018 12:29:11 +0200 Subject: [PATCH 1/5] Handle invalid input files which should be declined by at least one input validator Place them into a folder input_format_validators/bad_inputs, the input format validators should decline them. Signed-off-by: Stefan Kraus --- problemtools/verifyproblem.py | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 80a477b1..29ecb7d6 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -795,8 +795,44 @@ def modified_input_validates(applicable, modifier): os.unlink(file_name) + self._verify_invalid_inputs() + return self._check_res + def _verify_invalid_inputs(self): + """Check that input format validators decline invalid input files + given in input_format_validators/bad_inputs + """ + + path_to_invalid_inputs = os.path.join(self._problem.probdir, 'input_format_validators', 'bad_inputs') + + # verify only if invalid inputs are given, otherwise nothing to check in this function + if not os.path.exists(path_to_invalid_inputs): + return + + # verify that invalid inputs are given in a directory, not a file + if not os.path.isdir(path_to_invalid_inputs): + self.error("%s should be a directory containing invalid inputs, not a file" % path_to_invalid_inputs) + return + + for invalid_input_filename in os.listdir(path_to_invalid_inputs): + invalid_infile = os.path.join(path_to_invalid_inputs, invalid_input_filename) + if not invalid_infile.endswith('.in'): + self.warning('Input file %s is not an input file' % invalid_input_filename) + continue + + for val in self._validators: + status, _ = val.run(invalid_infile, args=None) + if not os.WIFEXITED(status): + self.error('Input format validator %s crashed on input %s' % (val, invalid_infile)) + + # If an input validator declines the input file, everything is fine and we break. + if os.WEXITSTATUS(status) == 42: + break + else: + # Will only be executed if loop wasn't ended by break. + # No input validator declined the invalid input file, this is an error. + self.error('Input format validators accepted invalid input %s' % invalid_infile) def validate(self, testcase): flags = testcase.testcasegroup.config['input_validator_flags'].split() From 6641e548c4367c86ce8d47dbc682de49cfa16422 Mon Sep 17 00:00:00 2001 From: Stefan Kraus Date: Wed, 25 Jul 2018 12:30:53 +0200 Subject: [PATCH 2/5] Implement wrong-answer outputfiles Files placed at data/{sample,secret,whatever}/{testcasename}.ans.wrong-{something} are files which should be declined by the output checker for inputfile {testcasename}.in and result in a wrong answer judgement. Signed-off-by: Stefan Kraus --- problemtools/verifyproblem.py | 50 +++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 29ecb7d6..8a7b86c9 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -323,6 +323,7 @@ def check(self, args): infiles = glob.glob(os.path.join(self._datadir, '*.in')) ansfiles = glob.glob(os.path.join(self._datadir, '*.ans')) + wafiles = glob.glob(os.path.join(self._datadir, '*.ans.wrong-*')) if self._parent is None: seen_secret = False @@ -358,13 +359,21 @@ def check(self, args): if len(files) > 1: self.warning("Identical input files: '%s'" % str(files)) + wafiles_with_input = [] + for f in infiles: if not f[:-3] + '.ans' in ansfiles: self.error("No matching answer file for input '%s'" % f) + wafiles_with_input.extend(glob.glob(OutputValidators.WA_GLOB % (f[:-3] + '.ans'))) + for f in ansfiles: if not f[:-4] + '.in' in infiles: self.error("No matching input file for answer '%s'" % f) + for f in wafiles: + if f not in wafiles_with_input: + self.error("No matching input file for wrong answer '%s'" % f) + for subdata in self._items: if subdata.matches_filter(args.data_filter): subdata.check(args) @@ -930,6 +939,8 @@ def grade(self, sub_results, testcasegroup, shadow_result=False): class OutputValidators(ProblemAspect): _default_validator = run.get_tool('default_validator') + WA_GLOB = "%s.wrong-*" # %s is the answerfile to a test case, i.e. 1.ans + def __init__(self, problem): self._problem = problem @@ -983,8 +994,47 @@ def check(self, args): self.warning('%s gets AC' % (desc)) os.unlink(file_name) + self._verify_invalid_outputs() + return self._check_res + def _verify_invalid_outputs(self): + """Check that output validators decline invalid answer files""" + + val_timelim = self._problem.config.get('limits')['validation_time'] + val_memlim = self._problem.config.get('limits')['validation_memory'] + flags = self._problem.config.get('validator_flags').split() + + for testcase in self._problem.testdata.get_all_testcases(): + if not os.path.exists(testcase.infile): + # .in-file doesn't exist, already reported by testcase-check, so 'ignore' here + continue + + wrong_answers = glob.glob(OutputValidators.WA_GLOB % testcase.ansfile) + if wrong_answers: + if self._problem.is_interactive: + self.warning('Output validator check with wrong answers for interactive problems is currently not supported (skipping)') + return + + if self._problem.config.get('validation') == 'default': + self.warning('Output validator check with wrong answers and default validator is unnecessary (skipping)') + return + + for wafile in wrong_answers: + for val in self._validators: + feedbackdir = tempfile.mkdtemp(prefix='feedback', dir=self._problem.tmpdir) + status, runtime = val.run(wafile, + args=[testcase.infile, testcase.ansfile, feedbackdir] + flags, + timelim=val_timelim, memlim=val_memlim) + res = self._parse_validator_results(val, status, feedbackdir, testcase) + shutil.rmtree(feedbackdir) + if res.verdict != 'AC': + # One output validator declined this wrong answer, so stop validating + break + else: + # Will only be executed if loop wasn't ended by break + # No output validator declined wrong answer, this is an error. + self.error("Wrong answer file %s was not declined by output validators" % wafile) def _parse_validator_results(self, val, status, feedbackdir, testcase): custom_score = self._problem.config.get('grading')['custom_scoring'] From c951e760d78e3657f5e21640f22ea0abc1ba402b Mon Sep 17 00:00:00 2001 From: Stefan Kraus Date: Thu, 26 Jul 2018 19:06:15 +0200 Subject: [PATCH 3/5] Decline output at 43, not 42 --- problemtools/verifyproblem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 8a7b86c9..ab59751a 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -836,7 +836,7 @@ def _verify_invalid_inputs(self): self.error('Input format validator %s crashed on input %s' % (val, invalid_infile)) # If an input validator declines the input file, everything is fine and we break. - if os.WEXITSTATUS(status) == 42: + if os.WEXITSTATUS(status) == 43: break else: # Will only be executed if loop wasn't ended by break. From 0fa96f76bf3901aab626318d42c5e21bc941cdc7 Mon Sep 17 00:00:00 2001 From: Stefan Kraus Date: Fri, 27 Jul 2018 10:57:40 +0200 Subject: [PATCH 4/5] Make check for existence of inputfile clearer This way, no globbing is needed, it's shorter and easier to understand. Signed-off-by: Stefan Kraus --- problemtools/verifyproblem.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index ab59751a..97dbfb08 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -359,19 +359,16 @@ def check(self, args): if len(files) > 1: self.warning("Identical input files: '%s'" % str(files)) - wafiles_with_input = [] - for f in infiles: if not f[:-3] + '.ans' in ansfiles: self.error("No matching answer file for input '%s'" % f) - wafiles_with_input.extend(glob.glob(OutputValidators.WA_GLOB % (f[:-3] + '.ans'))) for f in ansfiles: if not f[:-4] + '.in' in infiles: self.error("No matching input file for answer '%s'" % f) for f in wafiles: - if f not in wafiles_with_input: + if f.split('.ans.wrong-')[0] + '.in' not in infiles: self.error("No matching input file for wrong answer '%s'" % f) for subdata in self._items: From c30fd011f7a9555a0a41ea684637abb0571fefd7 Mon Sep 17 00:00:00 2001 From: Stefan Kraus Date: Fri, 27 Jul 2018 14:01:58 +0200 Subject: [PATCH 5/5] Support wrong answer files for default validation. The result of _actual_validators() also contains the default validator which declines wrong answers in validation mode "default" Signed-off-by: Stefan Kraus --- problemtools/verifyproblem.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/problemtools/verifyproblem.py b/problemtools/verifyproblem.py index 97dbfb08..27befc01 100644 --- a/problemtools/verifyproblem.py +++ b/problemtools/verifyproblem.py @@ -1013,12 +1013,8 @@ def _verify_invalid_outputs(self): self.warning('Output validator check with wrong answers for interactive problems is currently not supported (skipping)') return - if self._problem.config.get('validation') == 'default': - self.warning('Output validator check with wrong answers and default validator is unnecessary (skipping)') - return - for wafile in wrong_answers: - for val in self._validators: + for val in self._actual_validators(): feedbackdir = tempfile.mkdtemp(prefix='feedback', dir=self._problem.tmpdir) status, runtime = val.run(wafile, args=[testcase.infile, testcase.ansfile, feedbackdir] + flags,