Use standard extension configuration for CPython 3.12+

cpython-unix/Makefile

@@ -186,6 +186,9 @@ $(OUTDIR)/libedit-$(LIBEDIT_VERSION)-$(PACKAGE_SUFFIX).tar: $(LIBEDIT_DEPENDS)
 $(OUTDIR)/patchelf-$(PATCHELF_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-patchelf.sh
 	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) patchelf
 
+$(OUTDIR)/pkgconf-$(PKGCONF_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-pkgconf.sh
+	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) pkgconf
+
 $(OUTDIR)/sqlite-$(SQLITE_VERSION)-$(PACKAGE_SUFFIX).tar: $(PYTHON_DEP_DEPENDS) $(HERE)/build-sqlite.sh
 	$(RUN_BUILD) --docker-image $(DOCKER_IMAGE_BUILD) sqlite
 
@@ -265,6 +268,7 @@ PYTHON_DEPENDS_$(1) := \
     $$(if $$(NEED_OPENSSL_1_1),$$(OUTDIR)/openssl-1.1-$$(OPENSSL_1.1_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_OPENSSL_3_5),$$(OUTDIR)/openssl-3.5-$$(OPENSSL_3.5_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_PATCHELF),$$(OUTDIR)/patchelf-$$(PATCHELF_VERSION)-$$(PACKAGE_SUFFIX).tar) \
+    $$(if $$(NEED_PKGCONF),$$(OUTDIR)/pkgconf-$$(PKGCONF_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_SQLITE),$$(OUTDIR)/sqlite-$$(SQLITE_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_TCL),$$(OUTDIR)/tcl-$$(TCL_VERSION)-$$(PACKAGE_SUFFIX).tar) \
     $$(if $$(NEED_TK),$$(OUTDIR)/tk-$$(TK_VERSION)-$$(PACKAGE_SUFFIX).tar) \

cpython-unix/build-cpython.sh

@@ -15,6 +15,13 @@ export PKG_CONFIG_PATH=${TOOLS_PATH}/deps/share/pkgconfig:${TOOLS_PATH}/deps/lib
 # Ensure that `pkg-config` invocations include the static libraries
 export PKG_CONFIG="pkg-config --static"
 
+# Dependency pkg-config files use /tools/deps as their prefix. macOS builds
+# extract dependencies into a temporary directory, so have pkgconf relocate
+# their prefix based on the location of each .pc file.
+if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
+    export PKG_CONFIG="${PKG_CONFIG} --define-prefix"
+fi
+
 # configure somehow has problems locating llvm-profdata even though it is in
 # PATH. The macro it is using allows us to specify its path via an
 # environment variable.
@@ -203,23 +210,12 @@ if [[ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_10}" ]]; then
     patch -p1 -i "${ROOT}/patch-posixmodule-remove-system.patch"
 fi
 
-# Python 3.11 has configure support for configuring extension modules. We really,
-# really, really want to use this feature because it looks promising. But at the
-# time we added this code the functionality didn't support all extension modules
-# nor did it easily support static linking, including static linking of extra
-# libraries (which appears to be a limitation of `makesetup`). So for now we
-# disable the functionality and require our auto-generated Setup.local to provide
-# everything.
-if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_11}" ]; then
-    if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
-        # This sets MODULE_<NAME>_STATE=disabled in the Makefile for all extension
-        # modules that are not unavailable (n/a) based on the platform.
-        # Valid STATE variables are needed to create the _missing_stdlib_info.py
-        # file during the build in Python 3.15+
-        patch -p1 -i "${ROOT}/patch-configure-disable-stdlib-mod-3.12.patch"
-    else
-        patch -p1 -i "${ROOT}/patch-configure-disable-stdlib-mod.patch"
-    fi
+# CPython 3.11 does not configure every standard-library extension, so disable
+# its incomplete generated rules and supply all rules through Setup.local.
+# CPython 3.12+ uses Setup.stdlib with a small Setup.local for per-module
+# link-type overrides.
+if [ "${PYTHON_MAJMIN_VERSION}" = "3.11" ]; then
+    patch -p1 -i "${ROOT}/patch-configure-disable-stdlib-mod.patch"
 
     # This hack also prevents the conditional definition of the pwd module in
     # Setup.bootstrap.in from working. So we remove that conditional.
@@ -247,17 +243,6 @@ else
     patch -p1 -i "${ROOT}/patch-pgo-file-pool-3.11.patch"
 fi
 
-# There's a post-build Python script that verifies modules were
-# built correctly. Ideally we'd invoke this. But our nerfing of
-# the configure-based module building and replacing it with our
-# own Setup-derived version completely breaks assumptions in this
-# script. So leave it off for now... at our own peril.
-if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_15}" ]; then
-    patch -p1 -i "${ROOT}/patch-checksharedmods-disable-3.15.patch"
-elif [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
-    patch -p1 -i "${ROOT}/patch-checksharedmods-disable.patch"
-fi
-
 # CPython < 3.11 always linked against libcrypt. We backport part of
 # upstream commit be21706f3760bec8bd11f85ce02ed6792b07f51f to avoid this
 # behavior.
@@ -364,13 +349,6 @@ if [[ "${CC}" = "clang" && -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then
     EXTRA_CONFIGURE_FLAGS="${EXTRA_CONFIGURE_FLAGS} --with-tail-call-interp"
 fi
 
-# On Python 3.12+ we need to link the special hacl library provided some SHA-256
-# implementations. Since we hack up the regular extension building mechanism, we
-# need to reinvent this wheel.
-if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
-    LDFLAGS="${LDFLAGS} -LModules/_hacl"
-fi
-
 # On PPC we need to prevent the glibc 2.22 __tls_get_addr_opt symbol
 # from being introduced to preserve runtime compatibility with older
 # glibc.
@@ -400,6 +378,12 @@ CONFIGURE_FLAGS="
     --without-ensurepip
     ${EXTRA_CONFIGURE_FLAGS}"
 
+if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
+    # Build standard-library extensions as built-ins by default. Setup.local
+    # overrides only the extensions that must remain shared or disabled.
+    CONFIGURE_FLAGS="${CONFIGURE_FLAGS} MODULE_BUILDTYPE=static --enable-loadable-sqlite-extensions"
+fi
+
 
 # Build a libpython3.x.so, but statically link the interpreter against
 # libpython.
@@ -637,13 +621,6 @@ if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_13}" && "${TARGET_TRIPLE}" == x86_64*
     PROFILE_TASK="${PROFILE_TASK} --ignore test.test_bytes.BytesTest.test_from_format"
 fi
 
-# ./configure tries to auto-detect whether it can build 128-bit and 256-bit SIMD helpers for HACL,
-# but on x86-64 that requires v2 and v3 respectively, and on arm64 the performance is bad as noted
-# in the comments, so just don't even try. (We should check if we can make this conditional)
-if [[ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_14}" ]]; then
-    patch -p1 -i "${ROOT}/patch-python-configure-hacl-no-simd.patch"
-fi
-
 # We use ndbm on macOS and BerkeleyDB elsewhere.
 if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
     CONFIGURE_FLAGS="${CONFIGURE_FLAGS} --with-dbmliborder=ndbm"
@@ -781,6 +758,12 @@ BOLT_COMMON_FLAGS="${BOLT_COMMON_FLAGS:-}" BOLT_APPLY_FLAGS="${BOLT_APPLY_FLAGS:
 # Supplement produced Makefile with our modifications.
 cat ../Makefile.extra >> Makefile
 
+if [[ "${PYBUILD_PLATFORM}" = macos* && -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]]; then
+    # configure-derived MODLIBS uses plain -l flags. Hide symbols from bundled
+    # static dependency libraries in both libpython and the interpreter.
+    printf '\nMODLIBS := $(subst -l,-Xlinker -hidden-l,$(MODLIBS))\n' >> Makefile
+fi
+
 # Debian's PPC64LE GCC 6 defaults to PIE but selects the non-PIE startup object unless -pie is
 # passed explicitly. Add it only to LINKFORSHARED, which CPython uses when linking executables.
 # See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=81170.
@@ -1116,9 +1099,6 @@ if linux_uapi_include_arch:
         "",
     )
 replace_in_all("-isystem %s/deps/linux-uapi/usr/include" % tools_path, "")
-# See https://github.com/python/cpython/issues/145810#issuecomment-4068139183
-replace_in_all("-LModules/_hacl", "")
-
 EOF
 
 ${BUILD_PYTHON} "${ROOT}/hack_sysconfig.py" "${ROOT}/out/python"

cpython-unix/build-pkgconf.sh

@@ -0,0 +1,29 @@
+#!/usr/bin/env bash
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+set -ex
+
+ROOT=$(pwd)
+
+export PATH=${TOOLS_PATH}/${TOOLCHAIN}/bin:${TOOLS_PATH}/host/bin:$PATH
+
+tar -xf "pkgconf-${PKGCONF_VERSION}.tar.xz"
+
+pushd "pkgconf-${PKGCONF_VERSION}"
+
+CC="${HOST_CC}" CXX="${HOST_CXX}" CFLAGS="${EXTRA_HOST_CFLAGS} -fPIC" CPPFLAGS="${EXTRA_HOST_CFLAGS} -fPIC" LDFLAGS="${EXTRA_HOST_LDFLAGS}" ./configure \
+    --build="${BUILD_TRIPLE}" \
+    --prefix=/tools/host \
+    --disable-shared \
+    --disable-dependency-tracking \
+    --with-system-libdir=/usr/lib \
+    --with-system-includedir=/usr/include
+
+make -j "${NUM_CPUS}"
+make -j "${NUM_CPUS}" install DESTDIR="${ROOT}/out"
+
+ln -s pkgconf "${ROOT}/out/tools/host/bin/pkg-config"
+
+"${ROOT}/out/tools/host/bin/pkg-config" --version

cpython-unix/build-tcl.sh

@@ -80,6 +80,12 @@ fi
 export CFLAGS LDFLAGS
 export CPPFLAGS="${CFLAGS}"
 
+# CPython uses the macOS SDK's zlib without a zlib.pc file. Keep the -lz in
+# Libs.private, but do not make pkg-config require the missing metadata.
+if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
+    sed -i '' -e 's/ zlib >= 1.2.3//' tcl.pc.in
+fi
+
 ./configure \
     --build="${BUILD_TRIPLE}" \
     --host="${TARGET_TRIPLE}" \

cpython-unix/build-tk.sh

@@ -31,6 +31,10 @@ if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then
 else
     LDFLAGS="${LDFLAGS} -Wl,--exclude-libs,ALL"
     EXTRA_CONFIGURE_FLAGS="--x-includes=${TOOLS_PATH}/deps/include --x-libraries=${TOOLS_PATH}/deps/lib"
+
+    # Make pkg-config --static resolve libX11's private xcb and Xau dependencies.
+    sed -i '/^Requires: /a\
+Requires.private: x11' tk.pc.in
 fi
 
 CFLAGS="${CFLAGS}" CPPFLAGS="${CFLAGS}" LDFLAGS="${LDFLAGS}" ./configure \

cpython-unix/build.py

@@ -1154,6 +1154,7 @@ def main():
             "ncurses",
             "openssl-3.5",
             "patchelf",
+            "pkgconf",
             "sqlite",
             "tcl",
             "uuid",
@@ -1164,7 +1165,7 @@ def main():
             "zlib",
             "zstd",
         ):
-            tools_path = "host" if action in ("m4", "patchelf") else "deps"
+            tools_path = "host" if action in ("m4", "patchelf", "pkgconf") else "deps"
             extra_archives = {
                 "tcl": {"zlib"},
             }.get(action)

cpython-unix/extension-modules.yml

@@ -725,6 +725,20 @@ _testcapi:
     - source: _testcapi/weakref.c
       minimum-python-version: "3.13"
 
+_testclinic:
+  minimum-python-version: '3.12'
+  disabled-targets:
+    - .*
+  sources:
+    - _testclinic.c
+
+_testclinic_limited:
+  minimum-python-version: '3.13'
+  disabled-targets:
+    - .*
+  sources:
+    - _testclinic_limited.c
+
 _testexternalinspection:
   minimum-python-version: '3.13'
   maximum-python-version: '3.13'
@@ -1102,6 +1116,13 @@ xxlimited_35:
   disabled-targets:
     - .*
 
+xxlimited_3_13:
+  minimum-python-version: '3.15'
+  disabled-targets:
+    - .*
+  sources:
+    - xxlimited_3_13.c
+
 xxsubtype:
   setup-enabled-conditional:
     - enabled: true

cpython-unix/targets.yml

@@ -108,6 +108,7 @@ aarch64-apple-darwin:
     - m4
     - mpdecimal
     - openssl-3.5
+    - pkgconf
     - sqlite
     - tcl
     - tk
@@ -659,6 +660,7 @@ x86_64-apple-darwin:
     - m4
     - mpdecimal
     - openssl-3.5
+    - pkgconf
     - sqlite
     - tcl
     - tk