From 45e23172995ee8a4c597370093989edfe1dfdcb8 Mon Sep 17 00:00:00 2001 From: F-park <1527093924@qq.com> Date: Sat, 20 Jan 2024 00:26:41 +0800 Subject: [PATCH 1/3] support test_code to add pyright-config comment --- challenges/advanced-forward/question.py | 3 + docs/Contribute.md | 1 + .../basic-foo-pyright-config/question.py | 14 ++++ tests/conftest.py | 6 ++ tests/test_challenge.py | 24 +++--- tests/test_questions.py | 2 - views/challenge.py | 76 +++++++++++++++---- views/views.py | 2 +- 8 files changed, 99 insertions(+), 29 deletions(-) create mode 100644 tests/assets/challenges/basic-foo-pyright-config/question.py diff --git a/challenges/advanced-forward/question.py b/challenges/advanced-forward/question.py index 3f953e7..e2c92cf 100644 --- a/challenges/advanced-forward/question.py +++ b/challenges/advanced-forward/question.py @@ -14,3 +14,6 @@ def copy(self) -> MyClass: inst = MyClass(x=1) assert_type(inst.copy(), MyClass) + +## End of test code ## +# pyright: analyzeUnannotatedFunctions=false diff --git a/docs/Contribute.md b/docs/Contribute.md index bcceb6b..b6a1de8 100644 --- a/docs/Contribute.md +++ b/docs/Contribute.md @@ -42,6 +42,7 @@ Once you come up with an idea, go to the next steps. - Describe the challenge, make sure people understand what they need to accomplish (i.e. the `TODO:` part) - A comment `## End of your code ##`. This is mandatory, just copy and paste it. - Several test cases. Add a comment `# expect-type-error` after the lines where type errors should be thrown. + - (Optional) Add a comment `## End of test code ##`.Several [pyright-config](https://github.com/microsoft/pyright/blob/main/docs/configuration.md#type-check-diagnostics-settings) with the format `# pyright: =` `solution.py` contains the right solution, with everything else unchanged. diff --git a/tests/assets/challenges/basic-foo-pyright-config/question.py b/tests/assets/challenges/basic-foo-pyright-config/question.py new file mode 100644 index 0000000..dbb2c0d --- /dev/null +++ b/tests/assets/challenges/basic-foo-pyright-config/question.py @@ -0,0 +1,14 @@ +"""A simple question, only for running tests. +""" + + +def foo(): + pass + + +## End of your code ## +foo(1) +foo(1, 2) # expect-type-error + +## End of test code ## +# pyright: reportGeneralTypeIssues=error diff --git a/tests/conftest.py b/tests/conftest.py index 6035b3f..c47312f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,7 @@ from flask.testing import FlaskClient from app import app +from views.challenge import ChallengeManager CHALLENGES_DIR = Path(__file__).parent.parent / "challenges" ALL_QUESTIONS = list(CHALLENGES_DIR.glob("**/question.py")) @@ -22,6 +23,11 @@ def assets_dir() -> Path: return Path(__file__).parent / "assets" +@pytest.fixture() +def mgr(assets_dir: Path): + return ChallengeManager(assets_dir / "challenges") + + @pytest.fixture() def test_client() -> FlaskClient: return app.test_client() diff --git a/tests/test_challenge.py b/tests/test_challenge.py index 22877bf..df91471 100644 --- a/tests/test_challenge.py +++ b/tests/test_challenge.py @@ -1,6 +1,5 @@ from pathlib import Path -import pytest from views.challenge import ChallengeKey, ChallengeManager @@ -11,23 +10,24 @@ def test_load_empty_dir(self, tmpdir): def test_defaults(self): assert ChallengeManager().challenge_count > 0 - def test_load_tests_assets(self, assets_dir): - mgr = ChallengeManager(assets_dir / "challenges") + def test_load_tests_assets(self, mgr: ChallengeManager): assert mgr.challenge_count > 0 + def test_partition_test_code(self, mgr: ChallengeManager): + pyright_config_test = mgr.get_challenge( + ChallengeKey.from_str("basic-foo-pyright-config") + ).test_code + + _, pyright_basic_config = mgr._partition_test_code(pyright_config_test) + assert pyright_basic_config.endswith("pyright: reportGeneralTypeIssues=error\n") -class TestChallengeWithHints: - @pytest.fixture() - def challenge_mgr(self, assets_dir): - return ChallengeManager(assets_dir / "challenges") - def test_misc(self, challenge_mgr): - c_foo = challenge_mgr.get_challenge(ChallengeKey.from_str("basic-foo")) +class TestChallengeWithHints: + def test_misc(self, mgr: ChallengeManager): + c_foo = mgr.get_challenge(ChallengeKey.from_str("basic-foo")) assert c_foo.hints is None # Get the challenge with hints - c_foo_hints = challenge_mgr.get_challenge( - ChallengeKey.from_str("basic-foo-hints") - ) + c_foo_hints = mgr.get_challenge(ChallengeKey.from_str("basic-foo-hints")) assert c_foo_hints.hints assert isinstance(c_foo_hints.hints, str) diff --git a/tests/test_questions.py b/tests/test_questions.py index a843d72..dd70aec 100644 --- a/tests/test_questions.py +++ b/tests/test_questions.py @@ -4,8 +4,6 @@ from pathlib import Path -import pytest - from views.challenge import ChallengeManager diff --git a/views/challenge.py b/views/challenge.py index ba4a14e..0dfc125 100644 --- a/views/challenge.py +++ b/views/challenge.py @@ -11,6 +11,29 @@ from typing import ClassVar, Optional, TypeAlias ROOT_DIR = Path(__file__).parent.parent +PYRIGHT_BASIC_CONFIG = """ +# pyright: analyzeUnannotatedFunctions=true +# pyright: strictParameterNoneValue=true +# pyright: disableBytesTypePromotions=false +# pyright: strictListInference=false +# pyright: strictDictionaryInference=false +# pyright: strictSetInference=false +# pyright: deprecateTypingAliases=false +# pyright: enableExperimentalFeatures=false +# pyright: reportMissingImports=error +# pyright: reportUndefinedVariable=error +# pyright: reportGeneralTypeIssues=error +# pyright: reportOptionalSubscript=error +# pyright: reportOptionalMemberAccess=error +# pyright: reportOptionalCall=error +# pyright: reportOptionalIterable=error +# pyright: reportOptionalContextManager=error +# pyright: reportOptionalOperand=error +# pyright: reportTypedDictNotRequiredAccess=error +# pyright: reportPrivateImportUsage=error +# pyright: reportUnboundVariable=error +# pyright: reportUnusedCoroutine=error +""" class Level(StrEnum): @@ -145,13 +168,30 @@ def _get_challenges_groupby_level(self) -> dict[Level, list[ChallengeName]]: # Pyright error messages look like: # `:: - : ` # Here we only capture the error messages and line numbers - PYRIGHT_MESSAGE_REGEX = r"^(?:.+?):(\d+):[\s\-\d]+(error:.+)$" + PYRIGHT_MESSAGE_REGEX = ( + r"^(?:.+?):(?P\d+):[\s\-\d]+(?Perror:.+)$" + ) + + @staticmethod + def _partition_test_code(test_code: str): + TEST_SPLITTER = "\n## End of test code ##\n" + + # PYRIGHT_BASIC_CONFIG aim to limit user to modify the config + test_code, end_test_comment, pyright_config = test_code.partition(TEST_SPLITTER) + pyright_basic_config = PYRIGHT_BASIC_CONFIG + + # Replace `## End of test code ##` with PYRIGHT_BASIC_CONFIG + if end_test_comment: + pyright_basic_config += pyright_config + return test_code, pyright_basic_config @classmethod def _type_check_with_pyright( cls, user_code: str, test_code: str ) -> TypeCheckResult: - code = f"{user_code}{test_code}" + test_code, pyright_basic_config = cls._partition_test_code(test_code) + + code = f"{user_code}{test_code}{pyright_basic_config}" buffer = io.StringIO(code) # This produces a stream of TokenInfos, example: @@ -187,37 +227,45 @@ def _type_check_with_pyright( return TypeCheckResult(message=stderr, passed=False) error_lines: list[str] = [] - # Substract lineno in merged code by lineno_delta, so that the lineno in + # Substract lineno in merged code by user_code_line_len, so that the lineno in # error message matches those in the test code editor. Fixed #20. - lineno_delta = len(user_code.splitlines()) + user_code_lines_len = len(user_code.splitlines()) for line in stdout.splitlines(): m = re.match(cls.PYRIGHT_MESSAGE_REGEX, line) if m is None: continue - line_number, message = int(m.group(1)), m.group(2) + line_number, message = int(m["line_number"]), m["message"] if line_number in error_line_seen_in_err_msg: # Each reported error should be attached to a specific line, # If it is commented with # expect-type-error, let it pass. error_line_seen_in_err_msg[line_number] = True continue # Error could be thrown from user code too, in which case delta shouldn't be applied. - error_lines.append( - f"{line_number if line_number <= lineno_delta else line_number - lineno_delta}:{message}" - ) + error_line = f"%s:{message}" + + if line_number <= user_code_lines_len: + error_lines.append(error_line % line_number) + elif line_number <= user_code_lines_len + len(test_code.splitlines()): + error_lines.append(error_line % (line_number - user_code_lines_len)) + else: + error_lines.append(error_line % "[pyright-config]") # If there are any lines that are expected to fail but not reported by pyright, # they should be considered as errors. for line_number, seen in error_line_seen_in_err_msg.items(): if not seen: error_lines.append( - f"{line_number - lineno_delta}: error: Expected type error but instead passed" + f"{line_number - user_code_lines_len}: error: Expected type error but instead passed" ) - passed = len(error_lines) == 0 - if passed: - error_lines.append("\nAll tests passed") - else: - error_lines.append(f"\nFound {len(error_lines)} errors") + # Error for pyright-config will not fail the challenge + passed = True + for error_line in error_lines: + if error_line.startswith("[pyright-config]"): + continue + passed = False + + error_lines.append(f"\nFound {len(error_lines)} errors") return TypeCheckResult(message="\n".join(error_lines), passed=passed) diff --git a/views/views.py b/views/views.py index cf54522..5197def 100644 --- a/views/views.py +++ b/views/views.py @@ -59,7 +59,7 @@ def get_challenge(level: str, name: str): "level": challenge.level, "challenges_groupby_level": challenge_manager.challenges_groupby_level, "code_under_test": challenge.user_code, - "test_code": challenge.test_code, + "test_code": challenge.test_code.partition("\n## End of test code ##\n")[0], "hints_for_display": render_hints(challenge.hints) if challenge.hints else None, "python_info": platform.python_version(), } From 2053e13fd94a0577137e27cfe6a35a44f2dec114 Mon Sep 17 00:00:00 2001 From: F-park <1527093924@qq.com> Date: Sat, 20 Jan 2024 00:32:10 +0800 Subject: [PATCH 2/3] fix text --- docs/Contribute.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/Contribute.md b/docs/Contribute.md index b6a1de8..99e64c9 100644 --- a/docs/Contribute.md +++ b/docs/Contribute.md @@ -42,7 +42,7 @@ Once you come up with an idea, go to the next steps. - Describe the challenge, make sure people understand what they need to accomplish (i.e. the `TODO:` part) - A comment `## End of your code ##`. This is mandatory, just copy and paste it. - Several test cases. Add a comment `# expect-type-error` after the lines where type errors should be thrown. - - (Optional) Add a comment `## End of test code ##`.Several [pyright-config](https://github.com/microsoft/pyright/blob/main/docs/configuration.md#type-check-diagnostics-settings) with the format `# pyright: =` + - (Optional) Add a comment `## End of test code ##`. Several [pyright-config](https://github.com/microsoft/pyright/blob/main/docs/configuration.md#type-check-diagnostics-settings) with the format `# pyright: =` `solution.py` contains the right solution, with everything else unchanged. From 8676543050659855f72d2c0453d3070d48a7681a Mon Sep 17 00:00:00 2001 From: F-park <1527093924@qq.com> Date: Sat, 20 Jan 2024 09:19:07 +0800 Subject: [PATCH 3/3] fix test --- tests/test_identical.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/tests/test_identical.py b/tests/test_identical.py index e53b59f..c35924a 100644 --- a/tests/test_identical.py +++ b/tests/test_identical.py @@ -8,17 +8,19 @@ def test_identical(solution_file: Path): - level, challenge_name = solution_file.parent.name.split("-", maxsplit=1) - with solution_file.open() as f: - solution_code = f.read() - solution_test = Challenge( - name=challenge_name, level=Level(level), code=solution_code - ).test_code - - question_file = solution_file.parent / "question.py" - with question_file.open() as f: - question_code = f.read() - question_test = Challenge( - name=challenge_name, level=Level(level), code=question_code - ).test_code + def get_test_code(path: Path): + TEST_SPLITTER = "\n## End of test code ##\n" + level, challenge_name = path.parent.name.split("-", maxsplit=1) + + with solution_file.open() as f: + challenge_code = f.read() + challenge = Challenge( + name=challenge_name, level=Level(level), code=challenge_code + ) + + return challenge.test_code.partition(TEST_SPLITTER)[0] + + solution_test = get_test_code(solution_file) + question_test = get_test_code(solution_file.parent / "question.py") + assert solution_test.strip() == question_test.strip()