diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7e9ceca75..f1f0a1946 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -51,5 +51,6 @@ jobs: texlive-science latexmk texlive-lang-german + asymptote - shell: wsl-bash {0} run: pytest diff --git a/bin/config.py b/bin/config.py index eba8a1341..1d5660633 100644 --- a/bin/config.py +++ b/bin/config.py @@ -108,6 +108,8 @@ "jobs": (os.cpu_count() or 1) // 2, "time": 600, # Used for `bt fuzz` "verbose": 0, + "action": None, + "no_visualizer": True, } diff --git a/bin/contest.py b/bin/contest.py index db00d2884..241301dd8 100644 --- a/bin/contest.py +++ b/bin/contest.py @@ -41,7 +41,7 @@ def problems_yaml() -> Optional[list[dict[str, Any]]]: def get_api() -> str: - api = config.args.api or contest_yaml().get("api") + api = config.args.api or cast(str, contest_yaml().get("api")) if not api: fatal( "Could not find key `api` in contest.yaml and it was not specified on the command line." diff --git a/bin/export.py b/bin/export.py index a336ed2b1..219c01a36 100644 --- a/bin/export.py +++ b/bin/export.py @@ -13,6 +13,7 @@ from latex import PdfType from problem import Problem from validate import InputValidator, AnswerValidator, OutputValidator +from visualize import TestCaseVisualizer, OutputVisualizer def select_languages(problems: list[Problem]) -> list[str]: @@ -125,6 +126,8 @@ def build_problem_zip(problem: Problem, output: Path) -> bool: ("submissions/accepted/**/*", True), ("submissions/*/**/*", False), ("attachments/**/*", problem.interactive or problem.multi_pass), + (f"{TestCaseVisualizer.source_dir}/**/*", False), + (f"{OutputVisualizer.source_dir}/**/*", False), ] # Do not include PDFs for kattis. @@ -212,6 +215,8 @@ def add_testcase(in_file: Path) -> None: f"{OutputValidator.source_dir}/**/*", # "statement/*", "solution/*", "problem_slide/*", use \constant{} commands # "submissions/*/**/*", removed support? + f"{TestCaseVisualizer.source_dir}/**/*", + f"{OutputVisualizer.source_dir}/**/*", ] for pattern in constants_supported: for f in export_dir.glob(pattern): @@ -292,7 +297,7 @@ def add_testcase(in_file: Path) -> None: validator_flags = " ".join( problem.get_testdata_yaml( problem.path / "data", - "output_validator_args", + OutputValidator.args_key, PrintBar("Getting validator_flags for legacy export"), ) ) diff --git a/bin/generate.py b/bin/generate.py index 321774bb9..e1d5465d1 100644 --- a/bin/generate.py +++ b/bin/generate.py @@ -9,13 +9,14 @@ from collections.abc import Callable, Sequence from colorama import Fore, Style from pathlib import Path, PurePosixPath -from typing import Any, Final, Optional, overload +from typing import Any, Final, Iterable, Optional, overload import config import parallel import program import run import validate +import visualize from testcase import Testcase from verdicts import Verdict from problem import Problem @@ -88,7 +89,6 @@ def resolve_path(path, *, allow_absolute, allow_relative): # The following classes inherit from Invocation: # - GeneratorInvocation # - SolutionInvocation -# - VisualizerInvocation class Invocation: SEED_REGEX: Final[re.Pattern[str]] = re.compile(r"\{seed(:[0-9]+)?\}") NAME_REGEX: Final[re.Pattern[str]] = re.compile(r"\{name\}") @@ -121,7 +121,7 @@ def __init__(self, problem: Problem, string: str, *, allow_absolute: bool, allow raise ParseException("{seed(:[0-9]+)} may appear at most once.") # Automatically set self.program when that program has been built. - self.program: Optional[program.Generator | program.Visualizer | run.Submission] = None + self.program: Optional[program.Generator | run.Submission] = None def callback(program): self.program = program @@ -172,7 +172,7 @@ def run(self, bar, cwd, name, seed, retries=1): ) if result.status: break - if not result.retry: + if result.status == ExecStatus.TIMEOUT: break if not result.status: @@ -189,30 +189,6 @@ def run(self, bar, cwd, name, seed, retries=1): return result -class VisualizerInvocation(Invocation): - def __init__(self, problem, string): - super().__init__(problem, string, allow_absolute=True, allow_relative=False) - - # Run the visualizer, taking {name} as a command line argument. - # Stdin and stdout are not used. - # {name} is no longer used and hardcoded to `testcase` (see #273), and {seed} is also not used. - def run(self, bar, cwd): - assert isinstance(self.program, program.Visualizer), "Visualizer program must be built!" - - result = self.program.run(cwd, args=self._sub_args()) - - if result.status == ExecStatus.TIMEOUT: - bar.debug(f"{Style.RESET_ALL}-> {shorten_path(self.problem, cwd)}") - bar.error(f"Visualizer TIMEOUT after {result.duration}s") - elif not result.status: - bar.debug(f"{Style.RESET_ALL}-> {shorten_path(self.problem, cwd)}") - bar.error("Visualizer failed", result.err) - - if result.status and config.args.error and result.err: - bar.log("stderr", result.err) - return result - - class SolutionInvocation(Invocation): def __init__(self, problem, string): super().__init__(problem, string, allow_absolute=True, allow_relative=False) @@ -244,8 +220,7 @@ def run(self, bar, cwd): def run_interaction(self, bar, cwd, t): in_path = cwd / "testcase.in" interaction_path = cwd / "testcase.interaction" - if interaction_path.is_file(): - return True + interaction_path.unlink(missing_ok=True) testcase = Testcase(self.problem, in_path, short_path=(t.path.parent / (t.name + ".in"))) assert isinstance(self.program, run.Submission) @@ -327,7 +302,6 @@ def __init__(self, generator_config): "generate", "copy", "solution", - "visualizer", "random_salt", "retries", "count", @@ -339,18 +313,16 @@ def __init__(self, generator_config): "testdata.yaml", "include", "solution", - "visualizer", "random_salt", "retries", ] RESERVED_DIRECTORY_KEYS: Final[Sequence[str]] = ["command"] KNOWN_ROOT_KEYS: Final[Sequence[str]] = ["generators", "parallel", "version"] -DEPRECATED_ROOT_KEYS: Final[Sequence[str]] = ["gitignore_generated"] +DEPRECATED_ROOT_KEYS: Final[Sequence[str]] = ["gitignore_generated", "visualizer"] # Holds all inheritable configuration options. Currently: # - config.solution -# - config.visualizer # - config.random_salt class Config: # Used at each directory or testcase level. @@ -362,13 +334,6 @@ def parse_solution(p, x, path): return None return SolutionInvocation(p, x) - @staticmethod - def parse_visualizer(p, x, path): - assert_type("Visualizer", x, [type(None), str], path) - if x is None: - return None - return VisualizerInvocation(p, x) - @staticmethod def parse_random_salt(p, x, path): assert_type("Random_salt", x, [type(None), str], path) @@ -379,7 +344,6 @@ def parse_random_salt(p, x, path): INHERITABLE_KEYS: Final[Sequence] = [ # True: use an AC submission by default when the solution: key is not present. ("solution", True, parse_solution), - ("visualizer", None, parse_visualizer), ("random_salt", "", parse_random_salt), # Non-portable keys only used by BAPCtools: # The number of retries to run a generator when it fails, each time incrementing the {seed} @@ -388,7 +352,6 @@ def parse_random_salt(p, x, path): ] solution: SolutionInvocation - visualizer: Optional[VisualizerInvocation] random_salt: str retries: int @@ -725,7 +688,6 @@ def validate_in(t, problem: Problem, testcase: Testcase, meta_yaml: dict, bar: P ) return True - # we assume .ans is a valid output and validate it as such def validate_ans_and_out( t, problem: Problem, testcase: Testcase, meta_yaml: dict, bar: ProgressBar ): @@ -745,18 +707,22 @@ def validate_ans_and_out( bar.error("No .out file was generated!") return False - answer_validator_hashes = {**testcase.validator_hashes(validate.AnswerValidator, bar)} - if all(h in meta_yaml["answer_validator_hashes"] for h in answer_validator_hashes): - return True + ans_out_validator_hashes = testcase.validator_hashes(validate.AnswerValidator, bar).copy() + output_validator_hashes = testcase.validator_hashes(validate.OutputValidator, bar) mode = validate.Mode.ANSWER - if testcase.root in config.INVALID_CASE_DIRECTORIES: + if testcase.root == "invalid_answer": mode = validate.Mode.INVALID - elif testcase.root == "valid_output": - mode = validate.Mode.VALID_OUTPUT - elif outfile.is_file(): + elif testcase.root == "invalid_output": + ans_out_validator_hashes.update(output_validator_hashes) + mode = validate.Mode.INVALID + elif testcase.root == "valid_output" or outfile.is_file(): + ans_out_validator_hashes.update(output_validator_hashes) mode = validate.Mode.VALID_OUTPUT + if all(h in meta_yaml["ans_out_validator_hashes"] for h in ans_out_validator_hashes): + return True + if not testcase.validate_format( mode, bar=bar, @@ -767,8 +733,9 @@ def validate_ans_and_out( bar.done(False) return False else: - for h in answer_validator_hashes: - meta_yaml["answer_validator_hashes"][h] = answer_validator_hashes[h] + for h in ans_out_validator_hashes: + meta_yaml["ans_out_validator_hashes"][h] = ans_out_validator_hashes[h] + meta_yaml["visualizer_hash"] = dict() write_yaml( meta_yaml, problem.tmpdir / "data" / t.hash / "meta_.yaml", @@ -820,7 +787,7 @@ def init_meta(): "generated_extensions": [], "input_validator_hashes": dict(), "solution_hash": dict(), - "answer_validator_hashes": dict(), + "ans_out_validator_hashes": dict(), "visualizer_hash": dict(), } meta_yaml["rule"] = t.rule @@ -904,7 +871,7 @@ def generate_from_rule(): if not infile.is_file() or meta_yaml.get("rule_hashes") != rule_hashes: # clear all generated files - shutil.rmtree(cwd) + shutil.rmtree(cwd, ignore_errors=True) cwd.mkdir(parents=True, exist_ok=True) meta_yaml = init_meta() @@ -957,7 +924,7 @@ def generate_from_rule(): assert t._has_required_in(infile), f"Failed to generate in file: {infile.name}" return True - def generate_from_solution(): + def generate_from_solution(testcase: Testcase): nonlocal meta_yaml if testcase.root in [*config.INVALID_CASE_DIRECTORIES, "valid_output"]: @@ -1022,7 +989,7 @@ def needed(ext): if used_solution: meta_yaml["solution_hash"] = solution_hash if changed_ans: - meta_yaml["answer_validator_hashes"] = dict() + meta_yaml["ans_out_validator_hashes"] = dict() meta_yaml["visualizer_hash"] = dict() if changed_ans or used_solution: write_yaml(meta_yaml, meta_path, allow_yamllib=True) @@ -1030,43 +997,118 @@ def needed(ext): assert ansfile.is_file(), f"Failed to generate ans file: {ansfile}" return True - def generate_empty_interactive_sample_ans(): - if not t.sample: - return True - if not problem.interactive and not problem.multi_pass: - return True - for ext in ["", ".statement", ".download"]: - ans_ext_file = infile.with_suffix(f".ans{ext}") - if ans_ext_file.exists(): - return True - if infile.with_suffix(f".in{ext}").exists(): - ans_ext_file.write_text("") - return True - return True - - def generate_visualization(): + def generate_visualization(testcase: Testcase, bar: ProgressBar): nonlocal meta_yaml - if testcase.root in [*config.INVALID_CASE_DIRECTORIES, "valid_output"]: - return True - if not t.config.visualizer: + if testcase.root in config.INVALID_CASE_DIRECTORIES: return True if config.args.no_visualizer: return True + # Generate visualization + in_path = cwd / "testcase.in" + ans_path = cwd / "testcase.ans" + out_path = cwd / "testcase.out" + assert in_path.is_file() + assert ans_path.is_file() + + feedbackdir = in_path.with_suffix(".feedbackdir") + image_files = [f"judgeimage{ext}" for ext in config.KNOWN_VISUALIZER_EXTENSIONS] + [ + f"teamimage{ext}" for ext in config.KNOWN_VISUALIZER_EXTENSIONS + ] + + def use_feedback_image(feedbackdir: Path, source: str) -> None: + for name in image_files: + path = feedbackdir / name + if path.exists(): + ensure_symlink(in_path.with_suffix(path.suffix), path) + bar.log(f"Using {name} from {source} as visualization") + return + + visualizer: Optional[visualize.AnyVisualizer] = problem.visualizer( + visualize.TestCaseVisualizer + ) + output_visualizer = problem.visualizer(visualize.OutputVisualizer) + if output_visualizer is not None: + if out_path.is_file() or problem.settings.ans_is_output: + if visualizer is None or out_path.is_file(): + visualizer = output_visualizer + if not out_path.is_file(): + assert problem.settings.ans_is_output + out_path = ans_path + + if visualizer is None: + for ext in config.KNOWN_VISUALIZER_EXTENSIONS: + in_path.with_suffix(ext).unlink(True) + use_feedback_image(feedbackdir, "validator") + return True + + visualizer_args = testcase.testdata_yaml_args(visualizer, bar) visualizer_hash = { - "visualizer_hash": t.config.visualizer.hash(), - "visualizer": t.config.visualizer.cache_command(), + "visualizer_hash": visualizer.hash, + "visualizer_args": visualizer_args, } if meta_yaml.get("visualizer_hash") == visualizer_hash: return True - # Generate visualization - t.config.visualizer.run(bar, cwd) + for ext in config.KNOWN_VISUALIZER_EXTENSIONS: + in_path.with_suffix(ext).unlink(True) + + if isinstance(visualizer, visualize.TestCaseVisualizer): + result = visualizer.run(in_path, ans_path, cwd, visualizer_args) + else: + feedbackcopy = in_path.with_suffix(".feedbackcopy") + shutil.rmtree(feedbackcopy) + + def skip_images(src: str, content: list[str]) -> list[str]: + return [] if src != str(feedbackdir) else image_files + + shutil.copytree(feedbackdir, feedbackcopy, ignore=skip_images) + + result = visualizer.run( + in_path, + ans_path, + out_path if not problem.interactive else None, + feedbackcopy, + visualizer_args, + ) + if result.status: + use_feedback_image(feedbackdir, "output_visualizer") + + if result.status == ExecStatus.TIMEOUT: + bar.debug(f"{Style.RESET_ALL}-> {shorten_path(problem, cwd)}") + bar.error( + f"{type(visualizer).visualizer_type.capitalize()} Visualizer TIMEOUT after {result.duration}s" + ) + elif not result.status: + bar.debug(f"{Style.RESET_ALL}-> {shorten_path(problem, cwd)}") + bar.error( + f"{type(visualizer).visualizer_type.capitalize()} Visualizer failed", result.err + ) - meta_yaml["visualizer_hash"] = visualizer_hash - write_yaml(meta_yaml, meta_path, allow_yamllib=True) + if result.status and config.args.error and result.err: + bar.log("stderr", result.err) + + if result.status: + meta_yaml["visualizer_hash"] = visualizer_hash + write_yaml(meta_yaml, meta_path, allow_yamllib=True) + + # errors in the visualizer are not critical + return True + + def generate_empty_interactive_sample_ans(): + if not t.sample: + return True + if not problem.interactive and not problem.multi_pass: + return True + for ext in ["", ".statement", ".download"]: + ans_ext_file = infile.with_suffix(f".ans{ext}") + if ans_ext_file.exists(): + return True + if infile.with_suffix(f".in{ext}").exists(): + ans_ext_file.write_text("") + return True return True def copy_generated(): @@ -1170,7 +1212,7 @@ def add_testdata_to_cache(): return # Step 4: generate .ans and .interaction if needed - if not generate_from_solution(): + if not generate_from_solution(testcase): return # Step 5: validate .ans (and .out if it exists) @@ -1178,7 +1220,7 @@ def add_testdata_to_cache(): return # Step 6: generate visualization if needed - if not generate_visualization(): + if not generate_visualization(testcase, bar): return # Step 7: for interactive and/or multi-pass samples, generate empty .ans if it does not exist @@ -1829,7 +1871,6 @@ def add_included_case(t: TestcaseRule): def build(self, build_visualizers=True, skip_double_build_warning=False): generators_used: set[Path] = set() solutions_used: set[Path] = set() - visualizers_used: set[Path] = set() # Collect all programs that need building. # Also, convert the default submission into an actual Invocation. @@ -1850,16 +1891,14 @@ def collect_programs(t): default_solution = DefaultSolutionInvocation(self) t.config.solution = default_solution solutions_used.add(t.config.solution.program_path) - if build_visualizers and t.config.visualizer: - visualizers_used.add(t.config.visualizer.program_path) self.root_dir.walk(collect_programs, dir_f=None) def build_programs( - program_type: type[program.Generator | program.Visualizer | run.Submission], - program_paths: set[Path], + program_type: type[program.Generator | run.Submission], + program_paths: Iterable[Path], ): - programs = list[program.Generator | program.Visualizer | run.Submission]() + programs = list[program.Generator | run.Submission]() for program_path in program_paths: path = self.problem.path / program_path if program_type is program.Generator and program_path in self.generators: @@ -1893,7 +1932,9 @@ def build_program(p): # TODO: Consider building all types of programs in parallel as well. build_programs(program.Generator, generators_used) build_programs(run.Submission, solutions_used) - build_programs(program.Visualizer, visualizers_used) + if build_visualizers: + self.problem.visualizer(visualize.TestCaseVisualizer) + self.problem.visualizer(visualize.OutputVisualizer) self.problem.validators(validate.InputValidator) self.problem.validators(validate.AnswerValidator) @@ -1902,10 +1943,6 @@ def build_program(p): def cleanup_build_failures(t): if t.config.solution and t.config.solution.program is None: t.config.solution = None - if not build_visualizers or ( - t.config.visualizer and t.config.visualizer.program is None - ): - t.config.visualizer = None self.root_dir.walk(cleanup_build_failures, dir_f=None) diff --git a/bin/interactive.py b/bin/interactive.py index 8936afb1e..8f587b80f 100644 --- a/bin/interactive.py +++ b/bin/interactive.py @@ -56,7 +56,7 @@ def get_validator_command(): run.testcase.ans_path.resolve(), run.feedbackdir.resolve(), ] - + run.testcase.testdata_yaml_validator_args( + + run.testcase.testdata_yaml_args( output_validator, bar or PrintBar("Run interactive test case"), ) @@ -167,6 +167,8 @@ def get_validator_command(): verdict = Verdict.VALIDATOR_CRASH break + run._visualize_output(bar or PrintBar("Visualize interaction")) + if tle_result is None: # Set result.err to validator error and result.out to team error. return ExecResult( @@ -431,6 +433,8 @@ def kill_handler_function(): if interaction_file is not None: interaction_file.close() + run._visualize_output(bar or PrintBar("Visualize interaction")) + if tle_result is None: return ExecResult( None, diff --git a/bin/problem.py b/bin/problem.py index db8524dcf..1b98b0343 100644 --- a/bin/problem.py +++ b/bin/problem.py @@ -6,7 +6,7 @@ from collections.abc import Callable, Sequence from pathlib import Path -from typing import Any, Final, Literal, Optional, TYPE_CHECKING +from typing import Any, Final, Literal, Optional, overload, TYPE_CHECKING if TYPE_CHECKING: # Prevent circular import: https://stackoverflow.com/a/39757388 from program import Program @@ -20,6 +20,7 @@ import validate import validator_tests import verdicts +import visualize from util import * from colorama import Fore, Style @@ -266,7 +267,7 @@ def __init__( self.limits = ProblemLimits(parse_setting(yaml_data, "limits", {}), problem, self) parse_deprecated_setting( - yaml_data, "validator_flags", "output_validator_args' in 'testdata.yaml" + yaml_data, "validator_flags", f"{validate.OutputValidator.args_key}' in 'testdata.yaml" ) self.keywords: list[str] = parse_optional_list_setting(yaml_data, "keywords", str) @@ -337,6 +338,9 @@ def __init__(self, path: Path, tmpdir: Path, label: Optional[str] = None): tuple[type[validate.AnyValidator], bool], list[validate.AnyValidator] ]() self._validators_warn_cache = set[tuple[type[validate.AnyValidator], bool]]() + self._visualizer_cache = dict[ + type[visualize.AnyVisualizer], Optional[visualize.AnyVisualizer] + ]() self._programs = dict[Path, "Program"]() self._program_callbacks = dict[Path, list[Callable[["Program"], None]]]() # Dictionary from path to parsed file contents. @@ -383,7 +387,7 @@ def _determine_statement_languages(self): unnormalised_yamlname = self.settings.name[lang] yamlname = " ".join(unnormalised_yamlname.split()) texpath = self.path / latex.PdfType.PROBLEM.path(lang) - with open(texpath) as texfile: + with texpath.open() as texfile: match texname := latex.get_argument_for_command(texfile, "problemname"): case None: error(rf"No \problemname found in {texpath.name}") @@ -456,20 +460,31 @@ def _parse_testdata_yaml(p, path, bar): p._testdata_yamls[f] = flags = parse_yaml(raw, path=f, plain=True) parse_deprecated_setting( - flags, "output_validator_flags", "output_validator_args" + flags, "output_validator_flags", validate.OutputValidator.args_key + ) + parse_deprecated_setting( + flags, "input_validator_flags", validate.InputValidator.args_key ) - parse_deprecated_setting(flags, "input_validator_flags", "input_validator_args") # Verify testdata.yaml for k in flags: match k: - case "output_validator_args": - if not isinstance(flags[k], str): - bar.error(f"{k} must be string", resume=True, print_item=False) - case "input_validator_args": - if not isinstance(flags[k], (str, dict)): + case ( + validate.OutputValidator.args_key + | validate.AnswerValidator.args_key + | visualize.TestCaseVisualizer.args_key + | visualize.OutputVisualizer.args_key + ): + if not isinstance(flags[k], list): + bar.error( + f"{k} must be a list of strings", + resume=True, + print_item=False, + ) + case validate.InputValidator.args_key: + if not isinstance(flags[k], (list, dict)): bar.error( - f"{k} must be string or map", + f"{k} must be list or map", resume=True, print_item=False, ) @@ -504,8 +519,8 @@ def _parse_testdata_yaml(p, path, bar): def get_testdata_yaml( p, path: Path, - key: Literal["input_validator_args"] | Literal["output_validator_args"], - bar: ProgressBar | PrintBar, + key: str, + bar: BAR_TYPE, name: Optional[str] = None, ) -> list[str]: """ @@ -517,8 +532,7 @@ def get_testdata_yaml( Arguments --------- path: absolute path (a file or a directory) - key: The testdata.yaml key to look for, either of 'input_validator_args', 'output_validator_args', or 'grading'. - TODO: 'grading' is not yet implemented. + key: The testdata.yaml key to look for (TODO: 'grading' is not yet implemented) name: If key == 'input_validator_args', optionally the name of the input validator. Returns: @@ -526,9 +540,16 @@ def get_testdata_yaml( A list of string arguments, which is empty if no testdata.yaml is found. TODO: when 'grading' is supported, it also can return dict """ - if key not in ["input_validator_args", "output_validator_args"]: + known_args_keys = [ + validate.InputValidator.args_key, + validate.OutputValidator.args_key, + validate.AnswerValidator.args_key, + visualize.TestCaseVisualizer.args_key, + visualize.OutputVisualizer.args_key, + ] + if key not in known_args_keys: raise NotImplementedError(key) - if key != "input_validator_args" and name is not None: + if key != validate.InputValidator.args_key and name is not None: raise ValueError( f"Only input validators support flags by validator name, got {key} and {name}" ) @@ -547,20 +568,28 @@ def get_testdata_yaml( continue flags = p._testdata_yamls[f] if key in flags: - if key == "output_validator_args": - if not isinstance(flags[key], str): - bar.error("output_validator_args must be string") + args = flags[key] + if key == validate.InputValidator.args_key: + if not isinstance(args, (list, dict)): + bar.error(f"{key} must be list of strings or map of lists") return [] - return flags[key].split() - - if key == "input_validator_args": - if not isinstance(flags[key], (str, dict)): - bar.error("input_validator_args must be string or map") + if isinstance(args, list): + if any(not isinstance(arg, str) for arg in args): + bar.error(f"{key} must be list of strings or map of lists") + return [] + return args + elif name in args: + if not isinstance(args[name], list) or any( + not isinstance(arg, str) for arg in args[name] + ): + bar.error(f"{key} must be list of strings or map of lists") + return [] + return args[name] + elif key in known_args_keys: + if not isinstance(args, list) or any(not isinstance(arg, str) for arg in args): + bar.error(f"{key} must be a list of strings") return [] - if isinstance(flags[key], str): - return flags[key].split() - elif name in flags[key]: - return flags[key][name].split() + return args return [] @@ -611,7 +640,7 @@ def testcases( testcases = [] for f in in_paths: - t = testcase.Testcase(p, f, print_warn=True) + t = testcase.Testcase(p, f) if ( (p.interactive or p.multi_pass) and mode in [validate.Mode.INVALID, validate.Mode.VALID_OUTPUT] @@ -857,6 +886,30 @@ def build_program(p): assert isinstance(problem._submissions, list) return problem._submissions.copy() + @overload + def visualizer( + problem, cls: type[visualize.TestCaseVisualizer] + ) -> Optional[visualize.TestCaseVisualizer]: ... + @overload + def visualizer( + problem, cls: type[visualize.OutputVisualizer] + ) -> Optional[visualize.OutputVisualizer]: ... + def visualizer( + problem, cls: type[visualize.AnyVisualizer] + ) -> Optional[visualize.AnyVisualizer]: + path = problem.path / cls.source_dir + if not path.is_dir(): + return None + if cls not in problem._visualizer_cache: + visualizer = cls(problem, path) + bar = ProgressBar(f"Building {cls.visualizer_type} visualizer", items=[visualizer]) + localbar = bar.start(visualizer) + visualizer.build(localbar) + localbar.done() + bar.finalize(print_done=False) + problem._visualizer_cache[cls] = visualizer if visualizer.ok else None + return problem._visualizer_cache[cls] + def validators( problem, cls: type[validate.AnyValidator], @@ -960,7 +1013,7 @@ def build_program(p): problem._validators_cache[key] = validators return validators - # get all testcses and submissions and prepare the output validator + # get all testcases and submissions and prepare the output validator and visualizer def prepare_run(problem): testcases = problem.testcases() if not testcases: @@ -970,6 +1023,10 @@ def prepare_run(problem): if not problem.validators(validate.OutputValidator): return False + # Pre build the output visualizer to prevent nested ProgressBars. + if not config.args.no_visualizer: + problem.visualizer(visualize.OutputVisualizer) + submissions = problem.submissions() if not submissions: return False @@ -1003,6 +1060,10 @@ def run_submissions(problem): testcases, submissions = ts_pair ok, verdict_table = Problem.run_some(testcases, submissions) + if len(testcases) * len(submissions) > 1: + if not config.args.verbose and not config.args.no_visualizer: + log("use -v with --visualize to see the paths to the generated images") + if config.args.table: Problem._print_table(verdict_table.results, testcases) elif config.args.overview and not config.args.tree: @@ -1291,7 +1352,7 @@ def validate_valid_extra_data(p) -> bool: def _validate_data( problem, mode: validate.Mode, - constraints: dict | Literal[True] | None, + constraints: validate.ConstraintsDict | Literal[True] | None, action: str, testcases: Sequence[testcase.Testcase], extra: bool = False, @@ -1300,13 +1361,11 @@ def _validate_data( if not testcases: return True - if constraints is True: - constraints = {} - assert constraints is None or isinstance(constraints, dict) + constraints_dict = {} if constraints is True else constraints + check_constraints = constraints_dict is not None # Pre-build the relevant Validators so as to avoid clash with ProgressBar bar below # Also, pick the relevant testcases - check_constraints = constraints is not None match mode: case validate.Mode.INPUT: problem.validators(validate.InputValidator, check_constraints=check_constraints) @@ -1352,7 +1411,7 @@ def process_testcase(testcase: testcase.Testcase): return ok = testcase.validate_format( - mode, bar=localbar, constraints=constraints, warn_instead_of_error=extra + mode, bar=localbar, constraints=constraints_dict, warn_instead_of_error=extra ) success &= ok localbar.done(ok) @@ -1362,8 +1421,8 @@ def process_testcase(testcase: testcase.Testcase): bar.finalize(print_done=True) # Make sure all constraints are satisfied. - if constraints: - for loc, value in sorted(constraints.items()): + if constraints_dict: + for loc, value in sorted(constraints_dict.items()): loc = Path(loc).name name, has_low, has_high, vmin, vmax, low, high = value if not has_low: diff --git a/bin/program.py b/bin/program.py index fd38bd3d8..40278acd7 100644 --- a/bin/program.py +++ b/bin/program.py @@ -141,18 +141,16 @@ def __init__( # Set self.name and self.tmpdir. # Ideally they are the same as the path inside the problem, but fallback to just the name. - try: - # Only resolve the parent of the program. This preserves programs that are symlinks to other directories. - relpath = (path.parent.resolve() / path.name).relative_to( - problem.path.resolve() / self.subdir - ) - self.short_path = relpath - self.name: str = str(relpath) - self.tmpdir = problem.tmpdir / self.subdir / relpath - except ValueError: - self.short_path = Path(path.name) - self.name = str(path.name) - self.tmpdir = problem.tmpdir / self.subdir / path.name + relpath = Path(path.name) + if path.absolute().parent != problem.path.absolute(): + try: + relpath = path.absolute().relative_to(problem.path.absolute() / subdir) + except ValueError: + pass + + self.short_path = relpath + self.name: str = str(relpath) + self.tmpdir = problem.tmpdir / self.subdir / self.name self.compile_command: Optional[list[str]] = None self.run_command: Optional[list[str]] = None @@ -515,7 +513,7 @@ def build(self, bar: ProgressBar): return True - def _exec_command(self, *args, **kwargs): + def _exec_command(self, *args, **kwargs) -> ExecResult: if "timeout" not in kwargs and "timeout" in self.limits: kwargs["timeout"] = self.limits["timeout"] if "memory" not in kwargs and "memory" in self.limits: @@ -567,16 +565,12 @@ def run(self, bar, cwd, name, args=[]): cwd=cwd, ) - result.retry = False - if result.status == ExecStatus.TIMEOUT: # Timeout -> stop retrying and fail. bar.log(f"TIMEOUT after {timeout}s", color=Fore.RED) return result if not result.status: - # Other error -> try again. - result.retry = True return result if stdout_path.read_text(): @@ -591,24 +585,3 @@ def run(self, bar, cwd, name, args=[]): return result return result - - -class Visualizer(Program): - def __init__(self, problem: "Problem", path: Path, **kwargs): - super().__init__( - problem, - path, - "visualizers", - limits={"timeout": problem.limits.visualizer_time}, - substitute_constants=True, - **kwargs, - ) - - # Run the visualizer. - # Stdin and stdout are not used. - def run(self, cwd, args=[]): - assert self.run_command is not None - return self._exec_command( - self.run_command + args, - cwd=cwd, - ) diff --git a/bin/run.py b/bin/run.py index 297cb1fc5..51c05fe6f 100644 --- a/bin/run.py +++ b/bin/run.py @@ -5,7 +5,7 @@ from colorama import Fore, Style from pathlib import Path -from typing import cast +from typing import Optional import config import interactive @@ -13,8 +13,10 @@ import problem import program import validate +import visualize from testcase import Testcase from util import ( + BAR_TYPE, crop_output, ensure_symlink, error, @@ -23,6 +25,7 @@ is_bsd, is_windows, ProgressBar, + shorten_path, warn, ) from verdicts import from_string, from_string_domjudge, RunUntil, Verdict, Verdicts @@ -40,7 +43,7 @@ def __init__(self, problem: "problem.Problem", submission: "Submission", testcas self.problem.tmpdir / "runs" / self.submission.short_path - / cast(Path, self.testcase.short_path).with_suffix("") + / self.testcase.short_path.with_suffix("") ) self.in_path: Path = self.tmpdir / "testcase.in" @@ -174,7 +177,9 @@ def run(self, bar, *, interaction=None, submission_args=None): result.duration = max_duration - # Delete .out files larger than 1MB. + self._visualize_output(bar) + + # Delete .out files larger than 1GB. if ( not config.args.error and self.out_path.is_file() @@ -215,7 +220,7 @@ def _prepare_nextpass(self, nextpass): shutil.move(nextpass, self.in_path) return True - def _validate_output(self, bar): + def _validate_output(self, bar: BAR_TYPE) -> Optional[ExecResult]: output_validators = self.problem.validators(validate.OutputValidator) if not output_validators: return None @@ -224,7 +229,21 @@ def _validate_output(self, bar): return output_validator.run( self.testcase, self, - args=self.testcase.testdata_yaml_validator_args(output_validator, bar), + args=self.testcase.testdata_yaml_args(output_validator, bar), + ) + + def _visualize_output(self, bar: BAR_TYPE) -> Optional[ExecResult]: + if config.args.no_visualizer: + return None + output_visualizer = self.problem.visualizer(visualize.OutputVisualizer) + if output_visualizer is None: + return None + return output_visualizer.run( + self.in_path, + self.testcase.ans_path.resolve(), + self.out_path if not self.problem.interactive else None, + self.feedbackdir, + args=self.testcase.testdata_yaml_args(output_visualizer, bar), ) @@ -420,14 +439,16 @@ def process_run(run: Run): if result.out: data = crop_output(result.out) - judgemessage = run.feedbackdir / "judgemessage.txt" - judgeerror = run.feedbackdir / "judgeerror.txt" # Add data from feedbackdir. for f in run.feedbackdir.iterdir(): - if f in [judgemessage, judgeerror]: - continue if f.name.startswith("."): continue # skip "hidden" files + if f.name in ["judgemessage.txt", "judgeerror.txt"]: + continue + if f.name.startswith("judgeimage.") or f.name.startswith("teamimage."): + data += f"{f.name}: {shorten_path(self.problem, f.parent) / f.name}\n" + ensure_symlink(run.problem.path / f.name, f, output=True, relative=False) + continue if not f.is_file(): localbar.warn(f"Validator wrote to {f} but it's not a file.") continue diff --git a/bin/skel.py b/bin/skel.py index 4427a69c8..ba0c2a6ff 100644 --- a/bin/skel.py +++ b/bin/skel.py @@ -93,7 +93,7 @@ def new_problem() -> None: ) author = config.args.author if config.args.author else ask_variable_string("author") - output_validator_args = "#output_validator_args:" + output_validator_args = f"#{OutputValidator.args_key}:" custom_output = False if config.args.type: problem_type = config.args.type @@ -105,7 +105,7 @@ def new_problem() -> None: # The validation type `float` is not official, it only helps setting the `output_validator_args`. if problem_type == "float": problem_type = "pass-fail" - output_validator_args = "output_validator_args: float_tolerance 1e-6" + output_validator_args = f"{OutputValidator.args_key}: float_tolerance 1e-6" log("Using default float tolerance of 1e-6") # Since version 2023-07-draft of the spec, the `custom` validation type is no longer explicit. # The mere existence of the output_validator(s)/ folder signals non-default output validation. @@ -121,7 +121,7 @@ def new_problem() -> None: "dirname": dirname, "author": author, "type": problem_type, - "output_validator_args": output_validator_args, + OutputValidator.args_key: output_validator_args, "testdata_yaml_comment": "#" if output_validator_args[0] == "#" else "", } @@ -180,13 +180,17 @@ def new_problem() -> None: else: error("ruamel.yaml library not found. Please update problems.yaml manually.") + skip = [] + if custom_output: + skip.append(skeldir / OutputValidator.source_dir) + copytree_and_substitute( skeldir, target_dir / dirname, variables, exist_ok=True, preserve_symlinks=preserve_symlinks, - skip=[skeldir / OutputValidator.source_dir] if not custom_output else None, + skip=skip, ) # Warn about missing problem statement skeletons for non-en languages diff --git a/bin/testcase.py b/bin/testcase.py index d07f2af79..26f061566 100644 --- a/bin/testcase.py +++ b/bin/testcase.py @@ -1,19 +1,26 @@ """Test case""" +from collections.abc import Sequence from colorama import Fore, Style from pathlib import Path -from typing import cast, Literal, Optional +from typing import Optional, TYPE_CHECKING from util import ( + BAR_TYPE, ExecStatus, combine_hashes_dict, fatal, print_name, + ProgressBar, shorten_path, ) import config import validate +if TYPE_CHECKING: # Prevent circular import: https://stackoverflow.com/a/39757388 + import visualize + import problem + class Testcase: """ @@ -57,7 +64,13 @@ class Testcase: """ - def __init__(self, base_problem, path, *, short_path=None, print_warn=False): + def __init__( + self, + base_problem: "problem.Problem", + path: Path, + *, + short_path: Optional[Path] = None, + ): """ Arguments --------- @@ -76,17 +89,17 @@ def __init__(self, base_problem, path, *, short_path=None, print_warn=False): # TODO add self.out_path if short_path is None: try: - self.short_path = path.relative_to(self.problem.path / "data") + self.short_path: Path = path.relative_to(self.problem.path / "data") except ValueError: fatal(f"Testcase {path} is not inside {self.problem.path / 'data'}.") else: self.short_path = short_path - self.root = self.short_path.parts[0] + self.root: str = self.short_path.parts[0] - self.in_path = path - self.ans_path = self.in_path.with_suffix(".ans") - self.out_path = ( + self.in_path: Path = path + self.ans_path: Path = self.in_path.with_suffix(".ans") + self.out_path: Optional[Path] = ( self.in_path.with_suffix(".out") if self.root in ["valid_output", "invalid_output"] or self.in_path.with_suffix(".out").is_file() @@ -94,18 +107,18 @@ def __init__(self, base_problem, path, *, short_path=None, print_warn=False): ) # Display name: everything after data/. - self.name = str(self.short_path.with_suffix("")) + self.name: str = str(self.short_path.with_suffix("")) - def __repr__(self): + def __repr__(self) -> str: return self.name - def with_suffix(self, ext): + def with_suffix(self, ext: str) -> Path: return self.in_path.with_suffix(ext) - def testdata_yaml_validator_args( + def testdata_yaml_args( self, - validator, # TODO #102: Fix circular import when setting type to validate.AnyValidator - bar, # TODO #102: Type should probably be ProgressBar | PrintBar or something + program: "validate.AnyValidator | visualize.AnyVisualizer", + bar: BAR_TYPE, ) -> list[str]: """ The flags specified in testdata.yaml for the given validator applying to this testcase. @@ -116,21 +129,18 @@ def testdata_yaml_validator_args( A nonempty list of strings, such as ["space_change_sensitive", "case_sensitive"] or ["--max_N", "50"] or even [""]. """ - key, name = ( - ("input_validator_args", validator.name) - if isinstance(validator, validate.InputValidator) - else ("output_validator_args", None) - ) path = self.problem.path / "data" / self.short_path return self.problem.get_testdata_yaml( path, - cast(Literal["input_validator_args", "output_validator_args"], key), + type(program).args_key, bar, - name=name, + name=program.name if isinstance(program, validate.InputValidator) else None, ) - def validator_hashes(self, cls: type["validate.AnyValidator"], bar): + def validator_hashes( + self, cls: type[validate.AnyValidator], bar: BAR_TYPE + ) -> dict[str, dict[str, str]]: """ Returns ------- @@ -138,7 +148,6 @@ def validator_hashes(self, cls: type["validate.AnyValidator"], bar): hash => - name - flags - - hash indicating which validators will be run for this testcase. """ assert cls in [validate.InputValidator, validate.AnswerValidator, validate.OutputValidator] @@ -147,29 +156,29 @@ def validator_hashes(self, cls: type["validate.AnyValidator"], bar): d = dict() for validator in validators: - flags = self.testdata_yaml_validator_args(validator, bar) - if not flags: - continue - flags_string = " ".join(flags) if flags is not None else None - o = { + flags = self.testdata_yaml_args(validator, bar) + flags_string = " ".join(flags) + h = combine_hashes_dict( + { + "name": validator.name, + "flags": flags_string, + "hash": validator.hash, + } + ) + d[h] = { "name": validator.name, "flags": flags_string, - "hash": validator.hash, } - h = combine_hashes_dict(o) - # Don't actually store the somewhat useless validator hash. - del o["hash"] - d[h] = o return d def validate_format( self, - mode: "validate.Mode", + mode: validate.Mode, *, - bar, - constraints=None, - warn_instead_of_error=False, + bar: ProgressBar, + constraints: Optional[validate.ConstraintsDict] = None, + warn_instead_of_error: bool = False, ) -> bool: check_constraints = constraints is not None @@ -263,13 +272,13 @@ def validate_format( def _run_validators( self, - mode: "validate.Mode", - validators, - expect_rejection, + mode: validate.Mode, + validators: Sequence[validate.AnyValidator], + expect_rejection: bool, *, - bar, - constraints=None, - warn_instead_of_error=False, + bar: ProgressBar, + constraints: Optional[validate.ConstraintsDict] = None, + warn_instead_of_error: bool = False, ) -> bool: args = [] results = [] @@ -278,10 +287,8 @@ def _run_validators( if isinstance(validator, validate.OutputValidator) and mode == validate.Mode.ANSWER: args += ["case_sensitive", "space_change_sensitive"] name = f"{name} (ans)" - flags = self.testdata_yaml_validator_args(validator, bar) - if flags is False: - continue - flags = args if flags is None else flags + args + flags = self.testdata_yaml_args(validator, bar) + flags = flags + args ret = validator.run(self, mode=mode, constraints=constraints, args=flags) results.append(ret.status) @@ -325,7 +332,7 @@ def _run_validators( data += ( f"{Style.RESET_ALL}-> {shorten_path(self.problem, file.parent) / file.name}\n" ) - else: + elif ret.err: data = ret.err if expect_rejection: @@ -343,7 +350,7 @@ def _run_validators( ) else: bar.part_done( - ret.status, + bool(ret.status), message, data=data, warn_instead_of_error=warn_instead_of_error, diff --git a/bin/tools.py b/bin/tools.py index 50a6c1ea6..5edac5b23 100755 --- a/bin/tools.py +++ b/bin/tools.py @@ -637,21 +637,25 @@ def build_parser(): ) genparser.add_argument( "--no-validators", + default=False, action="store_true", help="Ignore results of input and answer validation. Validators are still run.", ) genparser.add_argument( "--no-solution", + default=False, action="store_true", help="Skip generating .ans/.interaction files with the solution.", ) genparser.add_argument( "--no-visualizer", + default=False, action="store_true", help="Skip generating graphics with the visualizer.", ) genparser.add_argument( "--no-testcase-sanity-checks", + default=False, action="store_true", help="Skip sanity checks on testcases.", ) @@ -693,6 +697,12 @@ def build_parser(): action="store_true", help="Do not run `generate` before running submissions.", ) + runparser.add_argument( + "--visualizer", + dest="no_visualizer", + action="store_false", + help="Also run the output visualizer.", + ) runparser.add_argument( "--all", "-a", diff --git a/bin/upgrade.py b/bin/upgrade.py index e9b6bff37..cbf2a5413 100644 --- a/bin/upgrade.py +++ b/bin/upgrade.py @@ -13,6 +13,64 @@ from ruamel.yaml.comments import CommentedMap, CommentedSeq +# src_base must be a dir (or symlink to dir) +# dst_base must not exists +# the parents of dst_base must exist +def _move_dir(src_base: Path, dst_base: Path) -> None: + assert src_base.is_dir() + assert not dst_base.exists() + + src_base = src_base.absolute() + dst_base = dst_base.absolute() + base = [a for a, b in zip(reversed(src_base.parents), reversed(dst_base.parents)) if a == b][-1] + + def resolve_up(parts: tuple[str, ...]) -> Path: + resolved: list[str] = [] + for part in parts: + if part == ".": + continue + if part == ".." and len(resolved) and resolved[-1] != "..": + resolved.pop() + else: + resolved.append(part) + return Path(*resolved) + + def movetree(src: Path, dst: Path) -> None: + if src.is_symlink(): + # create a new symlink and make sure that the destination is handled properly + destination = src.readlink() + if destination.is_absolute(): + # absolute links should stay absolute + # if their destination is inside the dir we move we have to change it + if destination.is_relative_to(src_base): + destination = dst_base / destination.relative_to(src_base) + dst.symlink_to(destination) + src.unlink() + else: + if resolve_up(src.parent.parts + destination.parts).is_relative_to(src_base): + # the link is relative and points to another file we move + src.rename(dst) + else: + # the link is relative but points to a fixed place + src_rel = src.parent.relative_to(base) + dst_rel = dst.parent.relative_to(base) + parts = (("..",) * len(dst_rel.parts)) + src_rel.parts + destination.parts + dst.symlink_to(resolve_up(parts)) + src.unlink() + elif src.is_dir(): + # recursively move stuff inside dirs + dst.mkdir() + for file in [*src.iterdir()]: + movetree(file, dst / file.name) + # delete now empty dir + src.rmdir() + else: + # move file + src.rename(dst) + + movetree(src_base, dst_base) + + def upgrade_data(problem_path: Path, bar: ProgressBar) -> None: rename = [ ("data/invalid_inputs", "data/invalid_input"), @@ -61,8 +119,8 @@ def rename_testcase(old_base: Path, new_dir: Path) -> None: def upgrade_testdata_yaml(problem_path: Path, bar: ProgressBar) -> None: rename = [ - ("output_validator_flags", "output_validator_args"), - ("input_validator_flags", "input_validator_args"), + ("output_validator_flags", OutputValidator.args_key), + ("input_validator_flags", InputValidator.args_key), ] for f in (problem_path / "data").rglob("testdata.yaml"): @@ -91,6 +149,11 @@ def upgrade_generators_yaml(problem_path: Path, bar: ProgressBar) -> None: changed = False + if "visualizer" in yaml_data: + warn( + "Cannot automatically upgrade 'visualizer'.\n - move visualizer to 'test_case_visualizer/'\n - first argument is the in_file\n - second argument is the ans_file" + ) + if "data" in yaml_data and isinstance(yaml_data["data"], dict): data = cast(CommentedMap, yaml_data["data"]) @@ -153,8 +216,8 @@ def upgrade_generated_testdata_yaml(data: dict[str, Any], path: str) -> bool: print_path = f" ({path[1:]})" if len(path) > 1 else "" rename = [ - ("output_validator_flags", "output_validator_args"), - ("input_validator_flags", "input_validator_args"), + ("output_validator_flags", OutputValidator.args_key), + ("input_validator_flags", InputValidator.args_key), ] for old, new in rename: if old in testdata: @@ -244,31 +307,7 @@ def upgrade_output_validators(problem_path: Path, bar: ProgressBar) -> None: bar.log( f"renaming 'output_validators/{content[0].name}' to '{OutputValidator.source_dir}/'" ) - - def move(src: str, dst: str) -> None: - if Path(src).is_symlink(): - src_dst = Path(src).resolve() - if src_dst.is_relative_to(content[0]): # local symlink - Path(src).rename(dst) - else: # link outside output_validators/ - dst_pos = Path(dst).resolve() - common = [ - a - for a, b in zip(reversed(src_dst.parents), reversed(dst_pos.parents)) - if a == b - ][-1] - link = Path( - "../" * (len(dst_pos.parents) - len(common.parts)) - ) / src_dst.relative_to(common) - Path(dst).symlink_to(link) - Path(src).unlink() - else: - Path(src).rename(dst) - - shutil.copytree( - content[0], problem_path / OutputValidator.source_dir, copy_function=move - ) - shutil.rmtree(problem_path / "output_validators") + _move_dir(content[0], problem_path / OutputValidator.source_dir) else: bar.log(f"renaming 'output_validators/' to '{OutputValidator.source_dir}/'") (problem_path / "output_validators").rename(problem_path / OutputValidator.source_dir) @@ -365,14 +404,17 @@ def upgrade_problem_yaml(problem_path: Path, bar: ProgressBar) -> None: ryaml_filter(data, "limits") def add_args(new_data: dict[str, Any]) -> bool: - if "output_validator_args" in new_data: + if OutputValidator.args_key in new_data: bar.error( - "can't change 'validator_flags', 'output_validator_args' already exists in testdata.yaml", + f"can't change 'validator_flags', '{OutputValidator.args_key}' already exists in testdata.yaml", resume=True, ) return False - bar.log("change 'validator_flags' to 'output_validator_args' in testdata.yaml") - new_data["output_validator_args"] = data["validator_flags"] + bar.log(f"change 'validator_flags' to '{OutputValidator.args_key}' in testdata.yaml") + validator_flags = data["validator_flags"] + new_data[OutputValidator.args_key] = ( + validator_flags.split() if isinstance(validator_flags, str) else validator_flags + ) ryaml_filter(data, "validator_flags") return True diff --git a/bin/util.py b/bin/util.py index f86911338..7475fd4e4 100644 --- a/bin/util.py +++ b/bin/util.py @@ -791,7 +791,7 @@ def write_yaml( exit(1) if path is None: return yamllib.dump(data) - with open(path, "w") as stream: + with path.open("w") as stream: yamllib.dump(data, stream) return None with write_yaml_lock: @@ -993,41 +993,46 @@ def strip_newline(s: str) -> str: # When output is True, copy the file when args.cp is true. -def ensure_symlink(link: Path, target: Path, output: bool = False, relative: bool = False) -> None: - # on windows copy if necessary - if is_windows() and not windows_can_symlink: - if link.exists() or link.is_symlink(): - link.unlink() - shutil.copyfile(target, link) - return +def ensure_symlink(link: Path, target: Path, output: bool = False, relative: bool = False) -> bool: + try: + # on windows copy if necessary + if is_windows() and not windows_can_symlink: + if link.exists() or link.is_symlink(): + link.unlink() + shutil.copyfile(target, link) + return True - # For output files: copy them on Windows, or when --cp is passed. - if output and config.args.cp: - if link.exists() or link.is_symlink(): - link.unlink() - shutil.copyfile(target, link) - return + # For output files: copy them on Windows, or when --cp is passed. + if output and config.args.cp: + if link.exists() or link.is_symlink(): + link.unlink() + shutil.copyfile(target, link) + return True - # Do nothing if link already points to the right target. - if link.is_symlink() and link.resolve() == target.resolve(): - is_absolute = os.readlink(link) - if not relative and is_absolute: - return - # if relative and not is_absolute: return + # Do nothing if link already points to the right target. + if link.is_symlink() and link.resolve() == target.resolve(): + is_absolute = os.readlink(link) + if not relative and is_absolute: + return True + # if relative and not is_absolute: return - if link.is_symlink() or link.exists(): - if link.is_dir() and not link.is_symlink(): - shutil.rmtree(link) - else: - link.unlink() + if link.is_symlink() or link.exists(): + if link.is_dir() and not link.is_symlink(): + shutil.rmtree(link) + else: + link.unlink() - # for windows the symlink needs to know if it points to a directory or file - if relative: - # Rewrite target to be relative to link. - # Use os.path.relpath instead of Path.relative_to for non-subdirectories. - link.symlink_to(os.path.relpath(target, link.parent), target.is_dir()) - else: - link.symlink_to(target.resolve(), target.is_dir()) + # for windows the symlink needs to know if it points to a directory or file + if relative: + # Rewrite target to be relative to link. + # Use os.path.relpath instead of Path.relative_to for non-subdirectories. + link.symlink_to(os.path.relpath(target, link.parent), target.is_dir()) + else: + link.symlink_to(target.resolve(), target.is_dir()) + return True + except (FileNotFoundError, FileExistsError): + # this must be a race condition + return False def has_substitute( @@ -1478,7 +1483,7 @@ def hash_file_content(file: Path, buffer_size: int = 65536) -> str: raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), str(file)) sha = hashlib.sha512(usedforsecurity=False) - with open(file, "rb") as f: + with file.open("rb") as f: while True: data = f.read(buffer_size) if not data: @@ -1496,7 +1501,7 @@ def hash_file(file: Path, buffer_size: int = 65536) -> str: sha.update(len(name).to_bytes(8, "big")) sha.update(name) - with open(file, "rb") as f: + with file.open("rb") as f: while True: data = f.read(buffer_size) if not data: diff --git a/bin/validate.py b/bin/validate.py index 0eda6c1f7..6ca13e3f8 100644 --- a/bin/validate.py +++ b/bin/validate.py @@ -7,10 +7,10 @@ import config import program -import testcase if TYPE_CHECKING: # Prevent circular import: https://stackoverflow.com/a/39757388 import run + import testcase class Mode(Enum): @@ -174,7 +174,7 @@ def format_exec_code_map(returncode): return ExecStatus.ERROR if self.language == "checktestdata": - with main_path.open() as main_file: + with main_path.open("rb") as main_file: return self._exec_command( self.run_command, exec_code_map=format_exec_code_map, @@ -209,7 +209,7 @@ def _exec_helper(self, *args, cwd, **kwargs): def run( self, - testcase: testcase.Testcase, + testcase: "testcase.Testcase", mode: Mode, constraints: Optional[ConstraintsDict] = None, args: Optional[list[str]] = None, @@ -230,12 +230,14 @@ class InputValidator(Validator): source_dir: Final[str] = "input_validators" + args_key: Final[str] = "input_validator_args" + def __init__(self, problem, path, **kwargs): super().__init__(problem, path, InputValidator.source_dir, **kwargs) def run( self, - testcase: testcase.Testcase, + testcase: "testcase.Testcase", mode: Mode = Mode.INPUT, constraints: Optional[ConstraintsDict] = None, args: Optional[list[str]] = None, @@ -263,7 +265,7 @@ def run( invocation = self.run_command.copy() - with testcase.in_path.open() as in_file: + with testcase.in_path.open("rb") as in_file: ret = self._exec_helper( invocation + arglist, exec_code_map=validator_exec_code_map, @@ -290,12 +292,15 @@ class AnswerValidator(Validator): source_dir: Final[str] = "answer_validators" + # use output_validator_args as well + args_key: Final[str] = "output_validator_args" + def __init__(self, problem, path, **kwargs): super().__init__(problem, path, AnswerValidator.source_dir, **kwargs) def run( self, - testcase: testcase.Testcase, + testcase: "testcase.Testcase", mode: Mode = Mode.ANSWER, constraints: Optional[ConstraintsDict] = None, args: Optional[list[str]] = None, @@ -316,7 +321,7 @@ def run( invocation = self.run_command + [testcase.in_path.resolve()] - with testcase.ans_path.open() as ans_file: + with testcase.ans_path.open("rb") as ans_file: ret = self._exec_helper( invocation + arglist, exec_code_map=validator_exec_code_map, @@ -341,12 +346,14 @@ class OutputValidator(Validator): source_dir: Final[str] = "output_validator" + args_key: Final[str] = "output_validator_args" + def __init__(self, problem, path, **kwargs): super().__init__(problem, path, OutputValidator.source_dir, **kwargs) def run( self, - testcase: testcase.Testcase, + testcase: "testcase.Testcase", mode: "Mode | run.Run", constraints: Optional[ConstraintsDict] = None, args: Optional[list[str]] = None, @@ -358,7 +365,7 @@ def run( --------- mode: either a run.Run (namely, when validating submission output) or a Mode - (namely, when validation a testcase) + (namely, when validating a testcase) Returns ------- @@ -368,7 +375,7 @@ def run( assert self.run_command is not None, "Validator should be built before running it" if mode == Mode.INPUT: - raise ValueError("OutputValidator do not support Mode.INPUT") + raise ValueError("OutputValidator does not support Mode.INPUT") in_path = testcase.in_path.resolve() ans_path = testcase.ans_path.resolve() @@ -388,7 +395,7 @@ def run( assert mode != Mode.INPUT # mode is actually a Run path = mode.out_path - in_path = mode.in_path + in_path = mode.in_path # relevant for multipass if self.language in Validator.FORMAT_VALIDATOR_LANGUAGES: raise ValueError("Invalid output validator language") @@ -398,7 +405,7 @@ def run( cwd = mode.feedbackdir invocation = self.run_command + [in_path, ans_path, cwd] - with path.open() as file: + with path.open("rb") as file: ret = self._exec_helper( invocation + arglist, exec_code_map=validator_exec_code_map, @@ -460,7 +467,7 @@ def sanity_check(problem, path, bar, strict_whitespace=True): if not path.exists(): fatal(f"{path} not found during sanity check") - with open(path, "rb") as file: + with path.open("rb") as file: name = { ".in": "Input", ".ans": "Answer", @@ -475,7 +482,9 @@ def sanity_check(problem, path, bar, strict_whitespace=True): if _has_invalid_byte(file_bytes, other_whitespaces=not strict_whitespace): bar.warn(f"{name} contains unexpected characters but was accepted!") - elif len(file_bytes) == 0: + elif len(file_bytes) == 0 and not ( + path.suffix == ".ans" and problem.multi_pass + ): # explicitly allow empty .ans files for multipass bar.warn(f"{name} is empty but was accepted!") elif len(file_bytes) > 20_000_000: bar.warn(f"{name} is larger than 20MB!") @@ -491,7 +500,7 @@ def sanity_check(problem, path, bar, strict_whitespace=True): and 2 * len(file_bytes) > problem.limits.output * 1024 * 1024 ): bar.warn(f"{name} is close to output limit") - elif strict_whitespace: + elif strict_whitespace and len(file_bytes) > 0: if file_bytes[0] in [ord(" "), ord("\n")]: bar.warn(f"{name} starts with whitespace but was accepted!") elif file_bytes[-1] != ord("\n"): diff --git a/bin/visualize.py b/bin/visualize.py new file mode 100644 index 000000000..074248cfd --- /dev/null +++ b/bin/visualize.py @@ -0,0 +1,97 @@ +from pathlib import Path +from typing import Any, Final, Optional, TYPE_CHECKING + +import program + +from util import * + +if TYPE_CHECKING: # Prevent circular import: https://stackoverflow.com/a/39757388 + from problem import Problem + + +class TestCaseVisualizer(program.Program): + """ + Visualizes a test case, called as: + + ./visualizer input answer [args] + + """ + + visualizer_type: Final[str] = "test case" + + source_dir: Final[str] = "test_case_visualizer" + + args_key: Final[str] = "test_case_visualizer_args" + + def __init__(self, problem: "Problem", path: Path, **kwargs: Any): + super().__init__( + problem, + path, + TestCaseVisualizer.source_dir, + limits={"timeout": problem.limits.visualizer_time}, + substitute_constants=True, + **kwargs, + ) + + # Run the visualizer (should create a testcase. file). + def run( + self, in_path: Path, ans_path: Path, cwd: Path, args: Optional[list[str]] = None + ) -> ExecResult: + assert self.run_command is not None, ( + "Test Case Visualizer should be built before running it" + ) + + return self._exec_command( + self.run_command + [in_path, ans_path] + (args or []), + cwd=cwd, + ) + + +class OutputVisualizer(program.Program): + """ + Visualizes the output of a submission + + ./visualizer input answer feedbackdir [args] < output + + """ + + visualizer_type: Final[str] = "output" + + source_dir: Final[str] = "output_visualizer" + + args_key: Final[str] = "output_visualizer_args" + + def __init__(self, problem: "Problem", path: Path, **kwargs: Any): + super().__init__( + problem, + path, + OutputVisualizer.source_dir, + limits={"timeout": problem.limits.visualizer_time}, + substitute_constants=True, + **kwargs, + ) + + # Run the visualizer. + # should write to feedbackdir/judgeimage. and/or feedbackdir/teamimage. + def run( + self, + in_path: Path, + ans_path: Path, + out_path: Optional[Path], + cwd: Path, + args: Optional[list[str]] = None, + ) -> ExecResult: + assert self.run_command is not None, "Output Visualizer should be built before running it" + assert (out_path is None) == self.problem.interactive, ( + "out_path should be None if and only if problem is interactive" + ) + + command = self.run_command + [in_path, ans_path, cwd] + (args or []) + if out_path is not None: + with out_path.open("rb") as out_file: + return self._exec_command(command, stdin=out_file, cwd=cwd) + else: + return self._exec_command(command, cwd=cwd) + + +AnyVisualizer = TestCaseVisualizer | OutputVisualizer diff --git a/doc/commands.md b/doc/commands.md index 8ae849ed4..102f03603 100644 --- a/doc/commands.md +++ b/doc/commands.md @@ -112,6 +112,7 @@ Use `bt run -v` to show results for all testcases. - `--overview`/`-o`: Print a live overview of the received verdicts for all submissions and testcases. If combined with `--no-bar` only the final table is printed. - `--no-testcase-sanity-checks`: when passed, all sanity checks on the testcases are skipped. You might want to set this in `.bapctools.yaml`. - `--sanitizer`: when passed, run submissions with additional sanitizer flags (currently only C++). Note that this removes all memory limits for submissions. +- `--visualizer`: when passed, run the output visualizer. ## `test` diff --git a/doc/generators.md b/doc/generators.md index edf424ecb..572ed2700 100644 --- a/doc/generators.md +++ b/doc/generators.md @@ -28,8 +28,6 @@ The two main object types are `directory` and `generator`. The root of `generato - `testdata.yaml`: Optional yaml configuration that will be copied to `testdata.yaml` in this directory. - `solution`: Optional invocation of a solution to be used to generate `.ans` files. Set to empty to disable generating `.ans`. (Useful for e.g. the `data/samples/` directory.) This must be an absolute path relative to the problem root. -- `visualizer`: Optional invocation of a visualizer to generate visualizations for each test case in this directory. - This must be an absolute path relative to the problem root. Set to empty to disable. - `random_salt`: Optional string that will be prepended to each command before computing its `{seed}`. May be used to regenerate all random cases and to prevent predictable seeds. - `data`: The test cases / test groups contained in this directory. This may take two forms: - A dictionary, each key is the name of a test case/test group, and each value must be a `directory` or `generator` object. diff --git a/doc/generators.yaml b/doc/generators.yaml index b6e13a7bd..fb78d06d0 100644 --- a/doc/generators.yaml +++ b/doc/generators.yaml @@ -11,24 +11,13 @@ # TOOLING: may pick a default if not specified, but should raise an error. solution: /submissions/accepted/sol.py -# The visualizer is used when no suitable image was generated already. -# This should read `testcase.in` and/or `testcase.ans` from the current working -# directory, and write `testcase.ext` for an extension in: -# .png, .jpg, .svg -# -# This must be the absolute path, starting in the problem root. -# -# TOOLING: may provide a flag to make running this optional, as it can be slow -# and usually isn't required. -visualizer: /visualizers/vis.py - # Optionally, a salt for generating the {seed} variables. Will be prepended to # the command being run. random_salt: abcd # The top level may contain a testdata.yaml that will be written to data/ as specified. testdata.yaml: - output_validator_args: "" + output_validator_args: [] # We support three types of generators: # - Standalone files, like generators/a.cpp, generators/b.py, ..., which will @@ -116,7 +105,7 @@ data: secret: include: # You can include other testcroups by there yaml name - - 'sample' + - "sample" # This will include "1", "2", "3", "4", and "5" from sample data: # Types of generator programs. @@ -147,8 +136,8 @@ data: 11-random-3: graph seed={seed:2} # Different seed, because command isn't the same. #11-random-4: graph {seed} {seed:2} # Not allowed because the regex matches twice. 12-counted: - generate: graph {seed:3} {count} - count: 2 # generate two testcases at once + generate: graph {seed:3} {count} + count: 2 # generate two testcases at once # No key (testcase or testgroup) may be a prefix of another key. #01-second: graph 6 # Collision with rule 01 above. @@ -169,12 +158,11 @@ data: "13": write_in_and_ans.py # To override the global/testgroup configuration on a per-testcase basis, - # a dictionary may be used. This allows the solution: and visualizer: keys, + # a dictionary may be used. This allows the solution: key, # as well as the generate: key which contains the command to execute. - 14_no_visualizer: + 14_override: generate: large_case_generator.py 1000000 solution: /generators/gnu_multi_precision.cpp - visualizer: # Empty to disable the visualizer here. random_salt: "123" # An entry must include *some* key that produces an in-file, @@ -189,7 +177,7 @@ data: hard_cases_group: # Directories may contain a testdata.yaml that will be written as specified. testdata.yaml: - output_validator_args: space_change_sensitive + output_validator_args: [space_change_sensitive] # To enable automatic numbering of testcases, data: may also contain a list of # single-element dictionaries instead of a single dictionary. In this case, diff --git a/skel/problem/generators/example.py b/skel/problem/generators/example_generator.py similarity index 100% rename from skel/problem/generators/example.py rename to skel/problem/generators/example_generator.py diff --git a/skel/problem/generators/generators.yaml b/skel/problem/generators/generators.yaml index 2c3a3fe72..fa2e7209f 100644 --- a/skel/problem/generators/generators.yaml +++ b/skel/problem/generators/generators.yaml @@ -1,6 +1,5 @@ #solution: /submissions/accepted/submission.py -#visualizer: /visualizers/asy.sh -version: 2025-02 # use this version of the generators framework +version: 2025-04 # use this version of the generators framework {%testdata_yaml_comment%}testdata.yaml: # One or more of: diff --git a/skel/problem/output_visualizer/example_output_visualizer.py b/skel/problem/output_visualizer/example_output_visualizer.py new file mode 100644 index 000000000..0d27bcb5e --- /dev/null +++ b/skel/problem/output_visualizer/example_output_visualizer.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +import sys + +input_file = open(sys.argv[1]).read().strip() +answer_file = open(sys.argv[2]).read().strip() +# input yields the team output +args = sys.argv[4:] +with open(f"{sys.argv[3]}/judgeimage.svg", "w") as f: + # this is unsafe since args could contain svg tags + print(f"args: {args}", file=f) diff --git a/skel/problem/test_case_visualizer/example_test_case_visualizer.py b/skel/problem/test_case_visualizer/example_test_case_visualizer.py new file mode 100644 index 000000000..df1162d22 --- /dev/null +++ b/skel/problem/test_case_visualizer/example_test_case_visualizer.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +import sys + +input_file = open(sys.argv[1]).read().strip() +answer_file = open(sys.argv[2]).read().strip() +args = sys.argv[3:] +with open("testcase.svg", "w") as f: + # this is unsafe since args could contain svg tags + print(f"args: {args}", file=f) diff --git a/skel/problem/test_case_visualizer/readme.md b/skel/problem/test_case_visualizer/readme.md new file mode 100644 index 000000000..88516a9c0 --- /dev/null +++ b/skel/problem/test_case_visualizer/readme.md @@ -0,0 +1,2 @@ +This test case visualizer is intended for use with BAPCtools' `bt generate`. +The visualizer should be invoked as `./visualizer <...test_case_visualizer_args>` and should write a `testcase.` file. diff --git a/support/schemas/generators.cue b/support/schemas/generators.cue index fa5f20b32..5dce093ee 100644 --- a/support/schemas/generators.cue +++ b/support/schemas/generators.cue @@ -20,12 +20,10 @@ import "strings" _parts: [#path, ...#command_args] } -// Test cases and test groups allow configuration of solution, visualiser, and random salt. +// Test cases and test groups allow configuration of solution, and random salt. #config: { // Path to solution starts with slash, such as "/submissions/accepted/foo.py" solution?: #filepath & =~"^/" - // Visualiser can be omitted to disable visualisation, may not use {count} - visualizer?: #command & =~"^/" & !~"\\{count" | null random_salt?: string } @@ -73,7 +71,7 @@ import "strings" valid_output?: #testgroup }) #testgroup_config - version: =~"^[0-9]{4}-[0-9]{2}$" | *"2025-02" + version: =~"^[0-9]{4}-[0-9]{2}$" | *"2025-04" ... // Do allow unknown_key at top level for tooling } diff --git a/support/schemas/generators_yaml_schema.json b/support/schemas/generators_yaml_schema.json index f75b294bb..3377488a3 100644 --- a/support/schemas/generators_yaml_schema.json +++ b/support/schemas/generators_yaml_schema.json @@ -43,9 +43,6 @@ "solution": { "$ref": "#/$defs/solution" }, - "visualizer": { - "$ref": "#/$defs/visualizer" - } }, "additionalProperties": false }, @@ -67,9 +64,6 @@ }, "input_validator_args": { "oneOf": [ - { - "type": "string" - }, { "type": "array", "items": { @@ -80,7 +74,10 @@ "type": "object", "patternProperties": { "^([A-Za-z0-9][A-Za-z0-9_-]*[A-Za-z0-9]|[A-Za-z0-9])$":{ - "type": "string" + "type": "array", + "items": { + "type": "string" + } } } } @@ -88,19 +85,26 @@ "description": "Defines arguments passed to each input validator for the test case/group. If a sequence of strings, then those are the arguments that will be passed to each input validator for this the case/group. If a map, then each key is the name of the input validator and the value is the arguments to pass to that input validator for the test case/group. Validators not present in the map are run without any arguments." }, "output_validator_args": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ], + "type": "array", + "items": { + "type": "string" + }, "description": "Defines arguments passed to the output validator for the test case/group." }, + "test_case_visualizer_args": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Defines arguments passed to the test case visualizer for the test case/group." + }, + "output_visualizer_args": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Defines arguments passed to the output visualizer for the test case/group." + }, "input_validator_flags": { "type": "string", "deprecated": true, @@ -109,7 +113,7 @@ "output_validator_flags": { "type": "string", "deprecated": true, - "description": "With 'problem_format_version: 2023-07-draft' in problem.yaml, use input_validator_args instead." + "description": "With 'problem_format_version: 2023-07-draft' in problem.yaml, use output_validator_args instead." }, "accept_score": { "type": "string" @@ -221,9 +225,6 @@ "title": "Hint", "description": "Feedback shown to the solver about this test case given as a string" }, - "visualizer": { - "$ref": "#/$defs/visualizer" - }, "random_salt": { "$ref": "#/$defs/random_salt" }, @@ -235,24 +236,6 @@ } ] }, - "visualizer": { - "title": "Visualizer", - "description": "Absolute path to and arguments for a visualizer. Leave empty to disable visualizion.", - "examples": [ - "/visualizer", - "/visualizers/asy.py", - "/visualizers/vis --large" - ], - "oneOf": [ - { - "type": "string", - "pattern": "^\/([^{}]|\\{name\\})*(\\{seed(:[0-9]+)?\\})?([^{}]|\\{name\\})*$" - }, - { - "type": "null" - } - ] - }, "random_salt": { "title": "Random Salt", "type": "string", @@ -297,14 +280,11 @@ } }, "additionalProperties": true, - "description": "Generate test data for this problem. Version 2025-02.", + "description": "Generate test data for this problem. Version 2025-04.", "properties": { "solution": { "$ref": "#/$defs/solution" }, - "visualizer": { - "$ref": "#/$defs/visualizer" - }, "random_salt": { "$ref": "#/$defs/random_salt" }, diff --git a/support/schemas/problemformat.cue b/support/schemas/problemformat.cue index e063dd16a..8f5fca3d5 100644 --- a/support/schemas/problemformat.cue +++ b/support/schemas/problemformat.cue @@ -23,8 +23,10 @@ let filename = "[A-Za-z0-9][A-Za-z0-9_.-]{0,253}[A-Za-z0-9]" // Test data settings #testdata_settings: { - input_validator_args?: *"" | string | {[string]: string} - output_validator_args?: *"" | string + input_validator_args?: *[] | [string] | {[string]: [string]} + output_validator_args?: *[] | [string] + test_case_visualizer_args?: *[] | [string] + output_visualizer_args?: *[] | [string] grading?: { score?: >0 max_score?: >0 diff --git a/test/problems/divsort/generators/generators.yaml b/test/problems/divsort/generators/generators.yaml index 5f44912f6..dd1d0f5b1 100644 --- a/test/problems/divsort/generators/generators.yaml +++ b/test/problems/divsort/generators/generators.yaml @@ -8,11 +8,11 @@ data: secret: testdata.yaml: input_validator_args: - integers: small + integers: [small] data: integers: testdata.yaml: - input_validator_args: --integer + input_validator_args: [--integer] #grading: foo data: - unsorted-integer: @@ -21,7 +21,7 @@ data: sorted: testdata.yaml: input_validator_args: - strings: --sorted + strings: [--sorted] data: - sorted-integer: in: 10.0 1.0 ab cd @@ -30,14 +30,14 @@ data: data: nested_1: testdata.yaml: - input_validator_args: --small + input_validator_args: [--small] data: small_floats: in: 10 3.5 ab cd nested_2: testdata.yaml: input_validator_args: - integers: "" # hides the input_validator_args in secret/testdata.yaml + integers: [] # hides the input_validator_args in secret/testdata.yaml data: - tiny_floats: in: 10.0 3.5 ab dc @@ -46,7 +46,7 @@ data: desc: Must validate, because `secret/testdata.yaml` hidden by `secret/general/nested_2/testdata.yaml` tolerant: testdata.yaml: - output_validator_args: float_tolerance 1e-2 + output_validator_args: [float_tolerance, "1e-2"] data: - tiny_floats: in: 10.0 3.0 ab dc @@ -62,14 +62,14 @@ data: too_many_tokens: {in: 10.0 2.5 ab cd ef} integers: testdata.yaml: - input_validator_args: --integer + input_validator_args: [--integer] data: ints_expected: {in: 10.0 2.5 ab cd} include: - small_floats sorted: testdata.yaml: - input_validator_args: --sorted + input_validator_args: [--sorted] include: - unsorted # invalid here because of --sorted flag (valid input in invalid_answers/no_output_validator_args) invalid_answer: @@ -84,7 +84,7 @@ data: ans: 5.0 Abccd with_output_validator_args: testdata.yaml: - output_validator_args: --forbid_abcd + output_validator_args: [--forbid_abcd] include: - imprecise # must reject because its ans includes abcd invalid_output: @@ -94,6 +94,8 @@ data: ans: 3.333333333 abcd out: 3.33 abcd valid_output: + testdata.yaml: + output_validator_args: [float_tolerance, "1e-2"] data: valid: in: 10.0 3.0 ab cd diff --git a/test/problems/fltcmp/data/testdata.yaml b/test/problems/fltcmp/data/testdata.yaml index af7ce1f19..ded323389 100644 --- a/test/problems/fltcmp/data/testdata.yaml +++ b/test/problems/fltcmp/data/testdata.yaml @@ -1 +1 @@ -output_validator_args: float_tolerance 1E-6 +output_validator_args: [float_tolerance, "1E-6"] diff --git a/test/problems/generatorincludes/generators/generators.yaml b/test/problems/generatorincludes/generators/generators.yaml index c82e8b97e..139deb187 100644 --- a/test/problems/generatorincludes/generators/generators.yaml +++ b/test/problems/generatorincludes/generators/generators.yaml @@ -11,10 +11,10 @@ data: data: - small: testdata.yaml: - output_validator_args: space_change_sensitive + output_validator_args: [space_change_sensitive] input_validator_args: - connected: --small - strongly-connected: --small + connected: [--small] + strongly-connected: [--small] data: - positive: data: diff --git a/test/problems/guess/output_visualizer_disabled/guess-visualizer.py b/test/problems/guess/output_visualizer_disabled/guess-visualizer.py new file mode 100644 index 000000000..4f201edf2 --- /dev/null +++ b/test/problems/guess/output_visualizer_disabled/guess-visualizer.py @@ -0,0 +1,32 @@ +import sys +from pathlib import Path + + +with open(sys.argv[1]) as in_file, open(sys.argv[3] / Path("judgemessage.txt"), "r") as msg_file: + mode = in_file.read().split()[0] + assert mode in ("random", "fixed", "adaptive"), mode + judgemessages = iter(msg_file) + + print(r"""\documentclass[varwidth]{standalone} +\usepackage{tikz} +\usetikzlibrary{patterns} +\tikzset{every node/.style={font=\sffamily}} +\begin{document} +\begin{tikzpicture} + """) + if not mode == "adaptive": + secret = int(next(judgemessages).split()[-1]) + print(rf"\node at ({secret / 100},-1.5) {{ {secret} ({mode}) }};") + else: + next(judgemessages) + print(r"\node at (5,-.5) { adaptive };") + for line in judgemessages: + rnd, guess = int(line.split()[1]), int(line.split()[3]) + y = -1 - rnd + print(rf"\draw [very thick, blue!20] (0, {y}) -- (10, {y});") + print(rf"\node at ({guess / 100}, {y})[anchor=north]", r"{$\uparrow$};") + print(rf"\node at ({guess / 100}, {y - 0.5})[anchor=north] {{ {guess} }};") + if not mode == "adaptive": + print(rf"\draw [red] ({secret / 100}, {-rnd - 1}) -- ({secret / 100}, 0);") + + print(r"\end{tikzpicture}\end{document}") diff --git a/test/problems/guess/output_visualizer_disabled/run b/test/problems/guess/output_visualizer_disabled/run new file mode 100755 index 000000000..62229c730 --- /dev/null +++ b/test/problems/guess/output_visualizer_disabled/run @@ -0,0 +1,41 @@ +#!/bin/bash + +# Set script directory +SCRIPT_DIR="$(dirname "$0")" + +# Check if visualize.py exists +if [[ ! -f "$SCRIPT_DIR/guess-visualizer.py" ]]; then + echo "Error: guess-visualizer.py not found in $SCRIPT_DIR" >&2 + exit 1 +fi + +tmptexdir=$(mktemp -d) # Create a unique temporary directory +OUTPUT_FILE="$tmptexdir/judgeimage.tex" + +# Run visualize.py +python3 "$SCRIPT_DIR/guess-visualizer.py" $1 $2 $3 > "$OUTPUT_FILE" +if [[ $? -ne 0 ]]; then + echo "Error: guess-visualizer.py failed" >&2 + exit 1 +fi + +# Check if judgeimage.tex exists +if [[ ! -f "$OUTPUT_FILE" ]]; then + echo "Error: texfile not found in $SCRIPT_DIR" >&2 + exit 1 +fi + +# Run pdflatex +( + cd "$tmptexdir" && pdflatex judgeimage.tex +) +if [[ $? -ne 0 ]]; then + echo "Error: pdflatex failed" >&2 + exit 1 +fi + +mv "$tmptexdir/judgeimage.pdf" $3 +rm -r "$tmptexdir" + +echo "Script completed successfully." +exit 0 diff --git a/test/problems/identity/data/sample/testdata.yaml b/test/problems/identity/data/sample/testdata.yaml new file mode 100644 index 000000000..cb6f96a79 --- /dev/null +++ b/test/problems/identity/data/sample/testdata.yaml @@ -0,0 +1 @@ +output_visualizer_args: [--draw-please] diff --git a/test/problems/identity/generators/generators.yaml b/test/problems/identity/generators/generators.yaml index d8e5c0f20..e6090d53c 100644 --- a/test/problems/identity/generators/generators.yaml +++ b/test/problems/identity/generators/generators.yaml @@ -1,6 +1,4 @@ solution: /submissions/accepted/author.py -# The visualizer is disabled to speed up testing. -#visualizer: /visualizers random_salt: "abc" generators: @@ -74,6 +72,8 @@ data: "6": in.statement: "6" ans.statement: "6" + testdata.yaml: + output_visualizer_args: [--draw-please] secret: data: @@ -133,7 +133,6 @@ data: solution: /generators/solution.c generate: random_gen.py {seed:7} testcase_dict_3: - visualizer: generate: random_gen.py {seed:8} unused_args_1: > # Spread arguments over multiple lines. random_gen.py diff --git a/test/problems/identity/output_visualizer_disabled/run b/test/problems/identity/output_visualizer_disabled/run new file mode 100755 index 000000000..d8d0ead16 --- /dev/null +++ b/test/problems/identity/output_visualizer_disabled/run @@ -0,0 +1,13 @@ +#!/usr/bin/env sh + +set -e + +draw=false +for var in "$@" +do + [ "$var" = "--draw-please" ] && draw=true +done + +if [ "$draw" = true ]; then + asy -f png $(dirname $0)/visualize.asy -u infilename="'${1}'" -u ansfilename="'${2}'" -o $3/judgeimage.png +fi diff --git a/test/problems/identity/output_visualizer_disabled/visualize.asy b/test/problems/identity/output_visualizer_disabled/visualize.asy new file mode 100644 index 000000000..dc5bda3b0 --- /dev/null +++ b/test/problems/identity/output_visualizer_disabled/visualize.asy @@ -0,0 +1,20 @@ +defaultpen(1); + +string outvalue = stdin; + +string infilename; +string ansfilename; +usersetting(); +file fin=input(infilename); +file fans=input(infilename); +string invalue = fin; +string ansvalue = fans; + +string label = "\texttt{in}: " + invalue ; +label(scale(5)*label, (0,200)); +string label = "\texttt{ans}: " + ansvalue ; +label(scale(5)*label, (0,100)); +pen labelPen = (invalue == outvalue) ? green : red; +string label = "\texttt{out}: " + outvalue ; +label(scale(5)*label, (0,0), p=labelPen); +shipout(bbox(xmargin=5, white, Fill)); diff --git a/test/problems/identity/test_case_visualizer_disabled/run b/test/problems/identity/test_case_visualizer_disabled/run new file mode 100755 index 000000000..ecc84b0c4 --- /dev/null +++ b/test/problems/identity/test_case_visualizer_disabled/run @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +set -e + +cat $1 $2 | asy -f png $(dirname $0)/visualize.asy -o testcase.png diff --git a/test/problems/identity/visualizers/visualize.asy b/test/problems/identity/test_case_visualizer_disabled/visualize.asy similarity index 100% rename from test/problems/identity/visualizers/visualize.asy rename to test/problems/identity/test_case_visualizer_disabled/visualize.asy diff --git a/test/problems/identity/visualizers/run b/test/problems/identity/visualizers/run deleted file mode 100755 index b82da4c8d..000000000 --- a/test/problems/identity/visualizers/run +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env sh - -set -e - -name=$1 -cat $name.in $name.ans | asy -f png $(dirname $0)/visualize.asy -o $name.png diff --git a/test/yaml/generators/invalid_yaml/bad_generators.yaml b/test/yaml/generators/invalid_yaml/bad_generators.yaml index 7c1f24525..2ce4851af 100644 --- a/test/yaml/generators/invalid_yaml/bad_generators.yaml +++ b/test/yaml/generators/invalid_yaml/bad_generators.yaml @@ -23,9 +23,6 @@ solution: true --- solution: false --- -# visualizer must be null or string -visualizer: 0 ---- # random_salt must be null or string random_salt: 0 --- @@ -164,21 +161,13 @@ data: data: ab: /generators/dir/gen.py --- -# Solution ans visualizer must have an absolute path: +# Solution must have an absolute path: solution: a --- solution: a/b --- solution: a 1 2 --- -visualizer: a ---- -visualizer: a/b ---- -visualizer: a 1 2 ---- -visualizer: a {name} ---- # Directories may not have generate:. generate: xyz --- diff --git a/test/yaml/generators/invalid_yaml/invalid.generators.yaml b/test/yaml/generators/invalid_yaml/invalid.generators.yaml index b78f8e92a..d4618cc0c 100644 --- a/test/yaml/generators/invalid_yaml/invalid.generators.yaml +++ b/test/yaml/generators/invalid_yaml/invalid.generators.yaml @@ -29,10 +29,6 @@ data: {sample: {data: []}, secret: {data: []}} solution: false data: {sample: {data: []}, secret: {data: []}} --- -# visualizer must be null or string -visualizer: 0 -data: {sample: {data: []}, secret: {data: []}} ---- # random_salt must be null or string random_salt: 0 data: {sample: {data: []}, secret: {data: []}} @@ -266,7 +262,7 @@ data: a: generate: /generators/gen.py --- -# Solution and visualizer must have an absolute path: +# Solution must have an absolute path: solution: a data: {sample: {data: []}, secret: {data: []}} --- @@ -276,18 +272,6 @@ data: {sample: {data: []}, secret: {data: []}} solution: a 1 2 data: {sample: {data: []}, secret: {data: []}} --- -visualizer: a -data: {sample: {data: []}, secret: {data: []}} ---- -visualizer: a/b -data: {sample: {data: []}, secret: {data: []}} ---- -visualizer: a 1 2 -data: {sample: {data: []}, secret: {data: []}} ---- -visualizer: a {name} -data: {sample: {data: []}, secret: {data: []}} ---- ## No toplevel generate TODO #generate: xyz #data: {sample: {data: []}, secret: {data: []}} @@ -405,6 +389,5 @@ data: data: - '': in: '1 2' - visualizer: "/ab/c" # this is fine - testdata.yaml: # this is not - input_validator_args: "connected" + testdata.yaml: # this is not ok + input_validator_args: [connected] diff --git a/test/yaml/generators/valid_yaml/rich-generators.yaml b/test/yaml/generators/valid_yaml/rich-generators.yaml index b2f5d743f..4e32ed274 100644 --- a/test/yaml/generators/valid_yaml/rich-generators.yaml +++ b/test/yaml/generators/valid_yaml/rich-generators.yaml @@ -29,8 +29,7 @@ data: count: 5 'group_with_testdata': testdata.yaml: - input_validator_args: "--connected --max_n 2000" - visualizer: "/foo/bar/baz" + input_validator_args: [--connected, --max_n, "2000"] data: 'a': my_generator invalid_input: