run_all.py

#!/usr/bin/env python3.10

""" Runner for CPython 3.10 test suite comparing against Nuitka.

Not every test of CPython has to pass, but instead it should fail just
the same with Nuitka.

"""

import os
import sys

# Find nuitka package relative to us.
sys.path.insert(
    0,
    os.path.normpath(
        os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")
    ),
)

# isort:start

from nuitka.tools.testing.Common import (
    addToPythonPath,
    compareWithCPython,
    createSearchMode,
    reportSkip,
    setup,
    setupCacheHashSalt,
)


def checkPath(dirname, filename):
    # Complex stuff, pylint: disable=too-many-branches,too-many-return-statements

    extra_flags = [
        "remove_output",
        # Import test_support which won't be included and potentially others.
        "binary_python_path",
        # We mean to compile only that one module
        "recurse_none",
        # Keep us informed about timing, even if concurrent runs spoil the
        # accuracy.
        "timing",
        # Use the original __file__ value, at least one case warns about things
        # with filename included.
        "original_file",
        # Cache the CPython results for reuse, they will normally not change.
        "cpython_cache",
        # Need to provide version, to know if to disable frozen modules.
        "--python-version=%s" % ".".join(str(d) for d in python_version),
    ]

    # TODO: This deadlocks, likely a threading problem.
    if python_version >= (3, 4) and filename == "test_concurrent_futures.py":
        reportSkip("Skipping (due to threading issue)", dirname, filename)
        return

    if dirname == "doctest_generated":
        if python_version >= (3, 10):
            extra_flags.append("expect_success")

        if filename == "test_generators.py":
            extra_flags.append("ignore_stderr")

            # On Windows with 32 bit, the MemoryError breaks the test
            if os.name == "nt":
                reportSkip(
                    "not enough memory with 32 bits on Windows", dirname, filename
                )

                return

    if filename in ("test_platform.py", "test_dataclasses.py", "test_asyncgen.py"):
        extra_flags.append("ignore_stderr")

    # TODO: Delayed, get it to work.
    if filename == "test_exceptions.py":
        reportSkip("KNOWN BUGGY", dirname, filename)
        return

    if filename == "test_datetime.py":
        extra_flags.append("recurse_to:test.datetimetester")

    if filename == "test_contextlib_async.py":
        # Task warnings
        extra_flags.append("ignore_stderr")

    if python_version < (3, 10):
        if filename in (
            "test_abc.py",
            "test_os.py",
            "test_ast.py",
            "test_fstring.py",
            "test_functools.py",
            "test_grammar.py",
            "test_named_expressions.py",
            "test_positional_only_arg.py",
            "test_inspect.py",
            "test_call.py",
            "test_dataclasses.py",
            "test_types.py",
        ):
            reportSkip("Not useful with older Python", dirname, filename)
            return

        # Traceback differences
        if filename == "test_dict.py":
            extra_flags.append("ignore_stderr")
        if filename == "test_dictcomps.py":
            extra_flags.append("ignore_stderr")

        # Outputs dict versions which are different.
        if filename == "test_dict_version.py":
            extra_flags.append("ignore_stderr")
    elif python_version >= (3, 11):
        if filename in (
            "test_ast.py",
            "test_capi.py",
            "test_code.py",
            "test_descrtut.py",
            "test_netrc.py",
            "test_ntpath.py",
            "test_reprlib.py",
            "test_tarfile.py",
            "test_tempfile.py",
            "test_threading.py",
            "test_traceback.py",
            "test_tracemalloc.py",
            "test_shelve.py",
            "test_sqlite.py",
            "test_wsgiref.py",
            "test_sys.py",
        ):
            reportSkip("Not useful with newer Python", dirname, filename)
            return

        if filename == "test_gc.py":
            reportSkip(
                "Nuitka passes more tests with compiled 3.11 than CPython 3.11",
                dirname,
                filename,
            )
            return
    else:
        # Put 3.10 specific stuff here.
        if filename == "test_capi.py":
            reportSkip(
                "Many details mismatch due to CPython 3.9 checking C stacks",
                dirname,
                filename,
            )
            return

        if filename in ("test_grammar.py",):
            reportSkip(
                "Many details mismatch due to CPython 3.9 checking C stacks",
                dirname,
                filename,
            )
            return



    compareWithCPython(
        dirname=dirname,
        filename=filename,
        extra_flags=extra_flags,
        search_mode=search_mode,
    )


def checkDir(dirname):
    for filename in sorted(os.listdir(dirname)):
        if not filename.endswith(".py") or not filename.startswith("test_"):
            continue

        if filename == "test_support.py":
            continue

        active = search_mode.consider(dirname, filename)

        if active:
            checkPath(dirname, filename)


python_version = setup(suite="CPython310", needs_io_encoding=True)
setupCacheHashSalt(".")

search_mode = createSearchMode()

addToPythonPath(os.path.abspath("."), in_front=True)

checkDir("test")
checkDir("doctest_generated")

search_mode.finish()