Experiment: removing setuptools._distutils

lib/ts_utils/metadata.py

@@ -166,6 +166,7 @@ def is_obsolete(self) -> bool:
         "tool",
         "partial_stub",
         "requires_python",
+        "mypy-tests",
     }
 )
 _KNOWN_METADATA_TOOL_FIELDS: Final = {

lib/ts_utils/mypy.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import os
+import sys
+import tempfile
+from collections.abc import Generator, Iterable
+from contextlib import contextmanager
+from enum import Enum
+from typing import Any, NamedTuple
+
+import tomli
+
+from ts_utils.metadata import metadata_path
+
+
+class MypyDistConf(NamedTuple):
+    module_name: str
+    values: dict[str, dict[str, Any]]
+
+
+class MypyResult(Enum):
+    SUCCESS = 0
+    FAILURE = 1
+    CRASH = 2
+
+
+# The configuration section in the metadata file looks like the following, with multiple module sections possible
+# [mypy-tests]
+# [mypy-tests.yaml]
+# module_name = "yaml"
+# [mypy-tests.yaml.values]
+# disallow_incomplete_defs = true
+# disallow_untyped_defs = true
+
+
+def mypy_configuration_from_distribution(distribution: str) -> list[MypyDistConf]:
+    with metadata_path(distribution).open("rb") as f:
+        data = tomli.load(f)
+
+    # TODO: This could be added to ts_utils.metadata
+    mypy_tests_conf: dict[str, dict[str, Any]] = data.get("mypy-tests", {})
+    if not mypy_tests_conf:
+        return []
+
+    def validate_configuration(section_name: str, mypy_section: dict[str, Any]) -> MypyDistConf:
+        assert isinstance(mypy_section, dict), f"{section_name} should be a section"
+        module_name = mypy_section.get("module_name")
+
+        assert module_name is not None, f"{section_name} should have a module_name key"
+        assert isinstance(module_name, str), f"{section_name} should be a key-value pair"
+
+        assert "values" in mypy_section, f"{section_name} should have a values section"
+        values: dict[str, dict[str, Any]] = mypy_section["values"]
+        assert isinstance(values, dict), "values should be a section"
+        return MypyDistConf(module_name, values.copy())
+
+    assert isinstance(mypy_tests_conf, dict), "mypy-tests should be a section"
+    return [validate_configuration(section_name, mypy_section) for section_name, mypy_section in mypy_tests_conf.items()]
+
+
+@contextmanager
+def temporary_mypy_config_file(
+    configurations: Iterable[MypyDistConf],
+) -> Generator[tempfile._TemporaryFileWrapper[str]]:  # pyright: ignore[reportPrivateUsage]
+    # We need to work around a limitation of tempfile.NamedTemporaryFile on Windows
+    # For details, see https://github.com/python/typeshed/pull/13620#discussion_r1990185997
+    # Python 3.12 added a workaround with `tempfile.NamedTemporaryFile("w+", delete_on_close=False)`
+    temp = tempfile.NamedTemporaryFile("w+", delete=sys.platform != "win32")  # noqa: SIM115
+    try:
+        for dist_conf in configurations:
+            temp.write(f"[mypy-{dist_conf.module_name}]\n")
+            for k, v in dist_conf.values.items():
+                temp.write(f"{k} = {v}\n")
+        temp.write("[mypy]\n")
+        temp.flush()
+        yield temp
+    finally:
+        temp.close()
+        if sys.platform == "win32":
+            os.remove(temp.name)

pyrightconfig.json

@@ -47,4 +47,6 @@
     "reportSelfClsParameterName": "none",
     // Not actionable in typeshed
     "reportDeprecated": "none",
+    // DO NOT MERGE! Temporary workaround until https://github.com/pypa/distutils/pull/338 is merged upstream
+    "reportMissingTypeStubs": "none",
 }

pyrightconfig.stricter.json

@@ -130,4 +130,6 @@
     "reportSelfClsParameterName": "none",
     // Not actionable in typeshed
     "reportDeprecated": "none",
+    // DO NOT MERGE! Temporary workaround until https://github.com/pypa/distutils/pull/338 is merged upstream
+    "reportMissingTypeStubs": "none",
 }

pyrightconfig.testcases.json

@@ -29,4 +29,6 @@
     "reportUnnecessaryIsInstance": "none",
     // The name of the self/cls parameter is out of typeshed's control.
     "reportSelfClsParameterName": "none",
+    // DO NOT MERGE! Temporary workaround until https://github.com/pypa/distutils/pull/338 is merged upstream
+    "reportMissingTypeStubs": "none",
 }

stubs/cffi/METADATA.toml

@@ -2,6 +2,12 @@ version = "1.16.*"
 upstream_repository = "https://foss.heptapod.net/pypy/cffi"
 requires = ["types-setuptools"]
 
+# When ran directly on our stubs, mypy won't understand that they are partial
+# This is not an issue with pyright because it understands per-module py.typed (setuptools/_distutils/py.typed)
+[mypy-tests.distutils]
+module_name = "setuptools._distutils.*"
+values = { "follow_untyped_imports" = true }
+
 [tool.stubtest]
 # linux and darwin are mostly equivalent, except for a single `RTLD_DEEPBIND` variable
 platforms = ["linux", "win32"]

stubs/fanstatic/METADATA.toml

@@ -1,3 +1,9 @@
 version = "1.5.*"
 upstream_repository = "https://github.com/zopefoundation/fanstatic"
-requires = ["types-setuptools", "types-WebOb"]
+requires = ["types-WebOb", "types-setuptools"]
+
+# When ran directly on our stubs, mypy won't understand that they are partial
+# This is not an issue with pyright because it understands per-module py.typed (setuptools/_distutils/py.typed)
+[mypy-tests.distutils]
+module_name = "setuptools._distutils.*"
+values = { "follow_untyped_imports" = true }

stubs/libsass/METADATA.toml

@@ -2,6 +2,12 @@ version = "0.23.*"
 requires = ["types-setuptools"]
 upstream_repository = "https://github.com/sass/libsass-python"
 
+# When ran directly on our stubs, mypy won't understand that they are partial
+# This is not an issue with pyright because it understands per-module py.typed (setuptools/_distutils/py.typed)
+[mypy-tests.distutils]
+module_name = "setuptools._distutils.*"
+values = { "follow_untyped_imports" = true }
+
 [tool.stubtest]
 # The runtime package has an undeclared dependency on setuptools
 stubtest_requirements = ["setuptools"]

stubs/pygit2/METADATA.toml

@@ -3,5 +3,11 @@ upstream_repository = "https://github.com/libgit2/pygit2"
 requires = ["types-cffi"]
 obsolete_since = "1.16.0" # Released on 2024-10-11
 
+# When ran directly on our stubs, mypy won't understand that they are partial
+# This is not an issue with pyright because it understands per-module py.typed (setuptools/_distutils/py.typed)
+[mypy-tests.distutils]
+module_name = "setuptools._distutils.*"
+values = { "follow_untyped_imports" = true }
+
 [tool.stubtest]
 platforms = ["darwin", "linux", "win32"]

stubs/setuptools/@tests/stubtest_allowlist.txt

@@ -1,99 +1,27 @@
-# Is a functools.partial, so stubtest says "is not a function"
-setuptools.modified.newer_pairwise_group
-setuptools._distutils._modified.newer_pairwise_group
-
-# Runtime initializes to None, but this really should never be None when used
-setuptools._distutils.compilers.C.base.Compiler.compiler_type
-
-# Dynamically created in __init__
-setuptools._distutils.dist.Distribution.get_name
-setuptools._distutils.dist.Distribution.get_version
-setuptools._distutils.dist.Distribution.get_fullname
-setuptools._distutils.dist.Distribution.get_author
-setuptools._distutils.dist.Distribution.get_author_email
-setuptools._distutils.dist.Distribution.get_maintainer
-setuptools._distutils.dist.Distribution.get_maintainer_email
-setuptools._distutils.dist.Distribution.get_contact
-setuptools._distutils.dist.Distribution.get_contact_email
-setuptools._distutils.dist.Distribution.get_url
-setuptools._distutils.dist.Distribution.get_license
-setuptools._distutils.dist.Distribution.get_licence
-setuptools._distutils.dist.Distribution.get_description
-setuptools._distutils.dist.Distribution.get_long_description
-setuptools._distutils.dist.Distribution.get_keywords
-setuptools._distutils.dist.Distribution.get_platforms
-setuptools._distutils.dist.Distribution.get_classifiers
-setuptools._distutils.dist.Distribution.get_download_url
-setuptools._distutils.dist.Distribution.get_requires
-setuptools._distutils.dist.Distribution.get_provides
-setuptools._distutils.dist.Distribution.get_obsoletes
-
-# Missing objects from setuptools._distutils
-setuptools._distutils.archive_util.ARCHIVE_FORMATS
-setuptools._distutils.archive_util.check_archive_formats
-setuptools._distutils.cmd.Command.dump_options
-setuptools._distutils.command.build_clib.show_compilers
-setuptools._distutils.command.build_ext.extension_name_re
-setuptools._distutils.command.build_ext.show_compilers
-setuptools._distutils.command.build_scripts
-setuptools._distutils.command.check
-setuptools._distutils.command.clean
-setuptools._distutils.command.install_data
-setuptools._distutils.command.install_headers
-setuptools._distutils.command.install.HAS_USER_SITE
-setuptools._distutils.command.install.INSTALL_SCHEMES
-setuptools._distutils.command.install.SCHEME_KEYS
-setuptools._distutils.command.install.WINDOWS_SCHEME
-setuptools._distutils.command.install_lib.PYTHON_SOURCE_EXTENSION
-setuptools._distutils.dist.fix_help_options
-setuptools._distutils.extension.read_setup_file
-setuptools._distutils.filelist.findall
-setuptools._distutils.filelist.glob_to_re
-setuptools._distutils.filelist.translate_pattern
-setuptools._distutils.sysconfig.BASE_EXEC_PREFIX
-setuptools._distutils.sysconfig.BASE_PREFIX
-setuptools._distutils.sysconfig.IS_PYPY
-setuptools._distutils.sysconfig.build_flags
-setuptools._distutils.sysconfig.expand_makefile_vars
-setuptools._distutils.sysconfig.get_python_version
-setuptools._distutils.sysconfig.parse_config_h
-setuptools._distutils.sysconfig.parse_makefile
-setuptools._distutils.sysconfig.project_base
-setuptools._distutils.sysconfig.python_build
-setuptools._distutils.util.is_freethreaded
-setuptools._distutils.util.MACOSX_VERSION_VAR
-
-# Missing submodules from setuptools._distutils
-# (Many of these may be implementation details,
-# but they can be added if people ask for them)
-setuptools._distutils.command.__all__
-setuptools._distutils.command.bdist_dumb
-setuptools._distutils.command.build_scripts
-setuptools._distutils.command.check
-setuptools._distutils.command.clean
-setuptools._distutils.command.config
-setuptools._distutils.command.install_data
-setuptools._distutils.command.install_egg_info
-setuptools._distutils.command.install_headers
-setuptools._distutils.compat.py39
-setuptools._distutils.core
-setuptools._distutils.cygwinccompiler
-setuptools._distutils.debug
-setuptools._distutils.dir_util
-setuptools._distutils.fancy_getopt
-setuptools._distutils.file_util
-setuptools._distutils.log
-setuptools._distutils.text_file
-setuptools._distutils.unixccompiler
-setuptools._distutils.version
-setuptools._distutils.versionpredicate
-setuptools._distutils.zosccompiler
-
-# Reexported from setuptools._distutils; problems should be fixed there
+# Reexported from https://github.com/pypa/distutils; problems should be fixed there
 distutils\..+
+setuptools._distutils\..+
 
 # Private APIs, tests and other vendored code
 setuptools.config._validate_pyproject.*
 setuptools.compat.*
-setuptools.command.build_py.build_py.existing_egg_info_dir
 .+?\.tests.*
+
+# DO NOT MERGE! Temporary workaround until https://github.com/pypa/distutils/pull/345 is merged upstream
+# stubtest fails on implicit submodule imports for __all__ (in setuptools/_distutils/command/__init__.py)
+setuptools.command.bdist_rpm
+setuptools.command.bdist_wheel
+setuptools.command.build_clib
+setuptools.command.build_py
+setuptools.command.develop
+setuptools.command.dist_info
+setuptools.command.easy_install
+setuptools.command.editable_wheel
+setuptools.command.egg_info
+setuptools.command.install
+setuptools.command.install_lib
+setuptools.command.install_scripts
+setuptools.command.sdist
+setuptools.installer
+setuptools.package_index
+setuptools.wheel

stubs/setuptools/METADATA.toml

@@ -4,7 +4,15 @@ extra_description = """\
 Given that `pkg_resources` is typed since `setuptools >= 71.1`, \
 it is no longer included with `types-setuptools`.
 """
-requires = ["setuptools"] # For pkg_resources
+requires = ["setuptools"] # For pkg_resources and setuptools._distutils
+# setuptools/_distutils is a vendor of pypa's distutils. Which is now as typed as typeshed.
+partial_stub = true
+
+# When ran directly on our stubs, mypy won't understand that they are partial
+# This is not an issue with pyright because it understands per-module py.typed (setuptools/_distutils/py.typed)
+[mypy-tests.distutils]
+module_name = "setuptools._distutils.*"
+values = { "follow_untyped_imports" = true }
 
 [tool.stubtest]
 # darwin is equivalent to linux for OS-specific methods

stubs/setuptools/distutils/command/bdist_dumb.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.command.bdist_dumb import *

stubs/setuptools/distutils/command/build_scripts.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.command.build_scripts import *

stubs/setuptools/distutils/command/check.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.command.check import *

stubs/setuptools/distutils/command/clean.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.command.clean import *

stubs/setuptools/distutils/command/config.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.command.config import *

stubs/setuptools/distutils/command/install_egg_info.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.command.install_egg_info import *

stubs/setuptools/distutils/command/install_headers.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.command.install_headers import *

stubs/setuptools/distutils/compilers/C/cygwin.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.compilers.C.cygwin import *

stubs/setuptools/distutils/compilers/C/unix.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.compilers.C.unix import *

stubs/setuptools/distutils/compilers/C/zos.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.compilers.C.zos import *

stubs/setuptools/distutils/core.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.core import *

stubs/setuptools/distutils/cygwinccompiler.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.cygwinccompiler import *

stubs/setuptools/distutils/debug.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.debug import *

stubs/setuptools/distutils/dir_util.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.dir_util import *

stubs/setuptools/distutils/fancy_getopt.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.fancy_getopt import *

stubs/setuptools/distutils/file_util.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.file_util import *

stubs/setuptools/distutils/log.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.log import *

stubs/setuptools/distutils/text_file.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.text_file import *

stubs/setuptools/distutils/unixccompiler.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.unixccompiler import *

stubs/setuptools/distutils/version.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.version import *

stubs/setuptools/distutils/versionpredicate.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.versionpredicate import *

stubs/setuptools/distutils/zosccompiler.pyi

@@ -0,0 +1 @@
+from setuptools._distutils.zosccompiler import *

stubs/setuptools/setuptools/__init__.pyi

@@ -4,7 +4,8 @@ from collections.abc import Mapping, Sequence
 from typing import Any, Literal, TypedDict, TypeVar, overload, type_check_only
 from typing_extensions import NotRequired
 
-from ._distutils.cmd import Command as _Command
+from setuptools._distutils.cmd import Command as _Command
+
 from .command.alias import alias
 from .command.bdist_egg import bdist_egg
 from .command.bdist_rpm import bdist_rpm