refactor: use tomlkit instead of manual parsing

Signed-off-by: Henry Schreiner <[email protected]>

Author: henryiii

CMakeLists.txt

@@ -330,8 +330,10 @@ if(PYBIND11_INSTALL)
   if(DEFINED SKBUILD_PROJECT_NAME AND SKBUILD_PROJECT_NAME STREQUAL "pybind11")
     file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/empty")
     file(TOUCH "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py")
-    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py" DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/")
-    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py" DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/")
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py"
+            DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/")
+    install(FILES "${CMAKE_CURRENT_BINARY_DIR}/empty/__init__.py"
+            DESTINATION "${CMAKE_INSTALL_DATAROOTDIR}/pkgconfig/")
   endif()
 
   # Uninstall target

noxfile.py

@@ -3,7 +3,6 @@
 import argparse
 import contextlib
 import os
-import re
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -16,10 +15,6 @@
 nox.options.sessions = ["lint", "tests", "tests_packaging"]
 nox.options.default_venv_backend = "uv|virtualenv"
 
-MARKER_PATTERN = re.compile(
-    r"# not-in-global-start.*?# not-in-global-end\n?", re.DOTALL
-)
-
 
 @nox.session(reuse_venv=True)
 def lint(session: nox.Session) -> None:
@@ -133,22 +128,17 @@ def build_global(session: nox.Session) -> None:
     """
 
     installer = ["--installer=uv"] if session.venv_backend == "uv" else []
-    session.install("build")
+    session.install("build", "tomlkit")
     session.log("Building pybind11-global files")
     pyproject = Path("pyproject.toml")
-    with preserve_file(pyproject) as txt:
-        new_txt = txt.replace('name = "pybind11"', 'name = "pybind11-global"')
-        assert txt != new_txt
-        newer_txt = MARKER_PATTERN.sub("", new_txt)
-        assert new_txt != newer_txt
-
+    with preserve_file(pyproject):
+        newer_txt = session.run("python", "tools/make_global.py", silent=True)
+        assert isinstance(newer_txt, str)
         pyproject.write_text(newer_txt, encoding="utf-8")
         session.run(
             "python",
             "-m",
             "build",
             *installer,
-            "-Cskbuild.wheel.install-dir=/data",
-            "-Cskbuild.experimental=true",
             *session.posargs,
         )

pyproject.toml

@@ -42,9 +42,9 @@ Discussions = "https://github.com/pybind/pybind11/discussions"
 Changelog = "https://pybind11.readthedocs.io/en/latest/changelog.html"
 Chat = "https://gitter.im/pybind/Lobby"
 
-# not-in-global-start
 [project.optional-dependencies]
 global = ["pybind11-global"]  # TODO: pin
+
 [project.scripts]
 pybind11-config = "pybind11.__main__:main"
 
@@ -54,24 +54,6 @@ pybind11 = "pybind11.__main__:main"
 [project.entry-points.pkg_config]
 pybind11 = "pybind11.share.pkgconfig"
 
-[[tool.scikit-build.generate]]
-path = "pybind11/_version.py"
-template = '''
-from __future__ import annotations
-
-
-def _to_int(s: str) -> int | str:
-    try:
-        return int(s)
-    except ValueError:
-        return s
-
-
-__version__ = "$version"
-version_info = tuple(_to_int(s) for s in __version__.split("."))
-'''
-# not-in-global-end
-
 
 [tool.scikit-build]
 minimum-version = "build-system.requires"
@@ -81,11 +63,6 @@ sdist.exclude = [
 ]
 wheel.install-dir = "pybind11"
 wheel.platlib = false
-wheel.packages = [
-# not-in-global-start
-  "pybind11",
-# not-in-global-end
-]
 
 [tool.scikit-build.cmake.define]
 BUILD_TESTING = false
@@ -103,6 +80,24 @@ regex = '''(?sx)
 result = "{major}.{minor}.{patch}"
 
 
+[[tool.scikit-build.generate]]
+path = "pybind11/_version.py"
+template = '''
+from __future__ import annotations
+
+
+def _to_int(s: str) -> int | str:
+    try:
+        return int(s)
+    except ValueError:
+        return s
+
+
+__version__ = "$version"
+version_info = tuple(_to_int(s) for s in __version__.split("."))
+'''
+
+
 [tool.mypy]
 files = ["pybind11"]
 python_version = "3.8"
@@ -111,7 +106,7 @@ enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
 warn_unreachable = true
 
 [[tool.mypy.overrides]]
-module = ["ghapi.*"]
+module = ["ghapi.*", "tomlkit"]  # tomlkit has types, but not very helpful
 ignore_missing_imports = true
 
 

tests/extra_python_package/test_files.py

@@ -2,23 +2,23 @@
 
 import contextlib
 import os
-import re
 import subprocess
 import sys
 import tarfile
 import zipfile
 from pathlib import Path
 from typing import Generator
-from pprint import pprint
 
 # These tests must be run explicitly
 
 DIR = Path(__file__).parent.resolve()
 MAIN_DIR = DIR.parent.parent
 
-MARKER_PATTERN = re.compile(
-    r"# not-in-global-start.*?# not-in-global-end\n?", re.DOTALL
-)
+
+# Newer pytest has global path setting, but keeping old pytest for now
+sys.path.append(str(MAIN_DIR / "tools"))
+
+from make_global import get_global  # noqa: E402
 
 PKGCONFIG = """\
 prefix=${{pcfiledir}}/../../
@@ -151,12 +151,8 @@ def build_global() -> Generator[None, None, None]:
     """
 
     pyproject = MAIN_DIR / "pyproject.toml"
-    with preserve_file(pyproject) as txt:
-        new_txt = txt.replace('name = "pybind11"', 'name = "pybind11-global"')
-        assert txt != new_txt
-        newer_txt = MARKER_PATTERN.sub("", new_txt)
-        assert new_txt != newer_txt
-
+    with preserve_file(pyproject):
+        newer_txt = get_global()
         pyproject.write_text(newer_txt, encoding="utf-8")
         yield
 
@@ -183,16 +179,13 @@ def test_build_sdist(monkeypatch, tmpdir):
     (sdist,) = tmpdir.visit("*.tar.gz")
 
     with tarfile.open(str(sdist), "r:gz") as tar:
-        start = tar.getnames()[0] + "/"
-        version = start[9:-1]
         simpler = {n.split("/", 1)[-1] for n in tar.getnames()[1:]}
 
         pyproject_toml = read_tz_file(tar, "pyproject.toml")
 
     files = headers | sdist_files
     assert files <= simpler
 
-    simple_version = ".".join(version.split(".")[:3])
     assert b'name = "pybind11"' in pyproject_toml
 
 
@@ -207,8 +200,6 @@ def test_build_global_dist(monkeypatch, tmpdir):
     (sdist,) = tmpdir.visit("*.tar.gz")
 
     with tarfile.open(str(sdist), "r:gz") as tar:
-        start = tar.getnames()[0] + "/"
-        version = start[16:-1]
         simpler = {n.split("/", 1)[-1] for n in tar.getnames()[1:]}
 
         pyproject_toml = read_tz_file(tar, "pyproject.toml")
@@ -249,11 +240,8 @@ def tests_build_global_wheel(monkeypatch, tmpdir):
     monkeypatch.chdir(MAIN_DIR)
     with build_global():
         subprocess.run(
-            [sys.executable, "-m", "pip", "wheel", ".",
-            "-Cskbuild.wheel.install-dir=/data",
-            "-Cskbuild.experimental=true",
-             "-w", 
-             str(tmpdir)], check=True
+            [sys.executable, "-m", "pip", "wheel", ".", "-w", str(tmpdir)],
+            check=True,
         )
 
     (wheel,) = tmpdir.visit("*.whl")
@@ -272,7 +260,6 @@ def tests_build_global_wheel(monkeypatch, tmpdir):
         names = z.namelist()
 
     beginning = names[0].split("/", 1)[0].rsplit(".", 1)[0]
-    pprint(names)
     trimmed = {n[len(beginning) + 1 :] for n in names}
 
     assert files == trimmed

tests/requirements.txt

@@ -11,3 +11,4 @@ pytest-timeout
 scipy~=1.5.4; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version<"3.10"
 scipy~=1.8.0; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version=="3.10" and sys_platform!='win32'
 scipy~=1.11.1; platform_python_implementation!="PyPy" and platform_python_implementation!="GraalVM" and python_version>="3.11" and python_version<"3.13" and sys_platform!='win32'
+tomlkit

tools/make_global.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env -S uv run -q
+
+# /// script
+# dependencies = ["tomlkit"]
+# ///
+from __future__ import annotations
+
+from pathlib import Path
+
+import tomlkit
+
+DIR = Path(__file__).parent.resolve()
+PYPROJECT = DIR.parent / "pyproject.toml"
+
+
+def get_global() -> str:
+    pyproject = tomlkit.parse(PYPROJECT.read_text())
+    del pyproject["tool"]["scikit-build"]["generate"]
+    del pyproject["project"]["optional-dependencies"]
+    del pyproject["project"]["entry-points"]
+    del pyproject["project"]["scripts"]
+    pyproject["project"]["name"] = "pybind11-global"
+    pyproject["tool"]["scikit-build"]["experimental"] = True
+    pyproject["tool"]["scikit-build"]["wheel"]["install-dir"] = "/data"
+    pyproject["tool"]["scikit-build"]["wheel"]["packages"] = []
+
+    result = tomlkit.dumps(pyproject)
+    assert isinstance(result, str)
+    return result
+
+
+if __name__ == "__main__":
+    print(get_global())