Revert "Revert to leak internals"

This reverts commit c5ec1cf.
This reverts commit 72c2e0a.

Author: XuehaiPan

include/pybind11/detail/internals.h

@@ -39,7 +39,7 @@
 /// further ABI-incompatible changes may be made before the ABI is officially
 /// changed to the new version.
 #ifndef PYBIND11_INTERNALS_VERSION
-#    define PYBIND11_INTERNALS_VERSION 11
+#    define PYBIND11_INTERNALS_VERSION 12
 #endif
 
 #if PYBIND11_INTERNALS_VERSION < 11
@@ -337,7 +337,20 @@ struct internals {
     internals(internals &&other) = delete;
     internals &operator=(const internals &other) = delete;
     internals &operator=(internals &&other) = delete;
-    ~internals() = default;
+    ~internals() {
+        // Normally this destructor runs during interpreter finalization and it may DECREF things.
+        // In odd finalization scenarios it might end up running after the interpreter has
+        // completely shut down, In that case, we should not decref these objects because pymalloc
+        // is gone.  This also applies across sub-interpreters, we should only DECREF when the
+        // original owning interpreter is active.
+        auto *cur_istate = get_interpreter_state_unchecked();
+        if (cur_istate && cur_istate == istate) {
+            gil_scoped_acquire_simple gil;
+            Py_CLEAR(instance_base);
+            Py_CLEAR(default_metaclass);
+            Py_CLEAR(static_property_type);
+        }
+    }
 };
 
 // the internals struct (above) is shared between all the modules. local_internals are only
@@ -347,13 +360,29 @@ struct internals {
 // impact any other modules, because the only things accessing the local internals is the
 // module that contains them.
 struct local_internals {
+    local_internals() : istate(get_interpreter_state_unchecked()) {}
+
     // It should be safe to use fast_type_map here because this entire
     // data structure is scoped to our single module, and thus a single
     // DSO and single instance of type_info for any particular type.
     fast_type_map<type_info *> registered_types_cpp;
 
     std::forward_list<ExceptionTranslator> registered_exception_translators;
     PyTypeObject *function_record_py_type = nullptr;
+    PyInterpreterState *istate = nullptr;
+
+    ~local_internals() {
+        // Normally this destructor runs during interpreter finalization and it may DECREF things.
+        // In odd finalization scenarios it might end up running after the interpreter has
+        // completely shut down, In that case, we should not decref these objects because pymalloc
+        // is gone.  This also applies across sub-interpreters, we should only DECREF when the
+        // original owning interpreter is active.
+        auto *cur_istate = get_interpreter_state_unchecked();
+        if (cur_istate && cur_istate == istate) {
+            gil_scoped_acquire_simple gil;
+            Py_CLEAR(function_record_py_type);
+        }
+    }
 };
 
 enum class holder_enum_t : uint8_t {
@@ -712,16 +741,27 @@ class internals_pp_manager {
     internals_pp_manager(char const *id, on_fetch_function *on_fetch)
         : holder_id_(id), on_fetch_(on_fetch) {}
 
+    static void internals_shutdown(PyObject *capsule) {
+        auto *pp = static_cast<std::unique_ptr<InternalsType> *>(
+            PyCapsule_GetPointer(capsule, nullptr));
+        if (pp) {
+            pp->reset();
+        }
+        // We reset the unique_ptr's contents but cannot delete the unique_ptr itself here.
+        // The pp_manager in this module (and possibly other modules sharing internals) holds
+        // a raw pointer to this unique_ptr, and that pointer would dangle if we deleted it now.
+        //
+        // For pybind11-owned interpreters (via embed.h or subinterpreter.h), destroy() is
+        // called after Py_Finalize/Py_EndInterpreter completes, which safely deletes the
+        // unique_ptr. For interpreters not owned by pybind11 (e.g., a pybind11 extension
+        // loaded into an external interpreter), destroy() is never called and the unique_ptr
+        // shell (8 bytes, not its contents) is leaked.
+        // (See PR #5958 for ideas to eliminate this leak.)
+    }
+
     std::unique_ptr<InternalsType> *get_or_create_pp_in_state_dict() {
-        // The `unique_ptr<InternalsType>` is intentionally leaked on interpreter shutdown.
-        // Once an instance is created, it will never be deleted until the process exits (compare
-        // to interpreter shutdown in multiple-interpreter scenarios).
-        // We cannot guarantee the destruction order of capsules in the interpreter state dict on
-        // interpreter shutdown, so deleting internals too early could cause undefined behavior
-        // when other pybind11 objects access `get_internals()` during finalization (which would
-        // recreate empty internals).
         auto result = atomic_get_or_create_in_state_dict<std::unique_ptr<InternalsType>>(
-            holder_id_, /*dtor=*/nullptr /* leak the capsule content */);
+            holder_id_, &internals_shutdown);
         auto *pp = result.first;
         bool created = result.second;
         // Only call on_fetch_ when fetching existing internals, not when creating new ones.
@@ -801,6 +841,8 @@ PYBIND11_NOINLINE internals &get_internals() {
 
 /// Return the PyObject* for the internals capsule (borrowed reference).
 /// Returns nullptr if the capsule doesn't exist yet.
+/// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types
+/// to hold a reference to the capsule (see comments in generic_type::initialize).
 inline PyObject *get_internals_capsule() {
     auto state_dict = reinterpret_borrow<dict>(get_python_state_dict());
     return dict_getitemstring(state_dict.ptr(), PYBIND11_INTERNALS_ID);
@@ -818,6 +860,8 @@ inline const std::string &get_local_internals_key() {
 
 /// Return the PyObject* for the local_internals capsule (borrowed reference).
 /// Returns nullptr if the capsule doesn't exist yet.
+/// This is used to prevent use-after-free during interpreter shutdown by allowing pybind11 types
+/// to hold a reference to the capsule (see comments in generic_type::initialize).
 inline PyObject *get_local_internals_capsule() {
     const auto &key = get_local_internals_key();
     auto state_dict = reinterpret_borrow<dict>(get_python_state_dict());

include/pybind11/pybind11.h

@@ -1733,6 +1733,31 @@ class generic_type : public object {
             PYBIND11_WARNING_POP
         });
 
+        // Prevent use-after-free during interpreter shutdown. GC order is not guaranteed, so the
+        // internals capsule may be destroyed (resetting internals via internals_shutdown) before
+        // all pybind11 types are destroyed. If a type's tp_traverse/tp_clear then calls py::cast,
+        // it would recreate an empty internals and fail because the type registry is gone.
+        //
+        // By holding references to the capsules, we ensure they outlive all pybind11 types. We use
+        // weakrefs on the type with a cpp_function callback. When the type is destroyed, Python
+        // will call the callback which releases the capsule reference and the weakref.
+        if (PyObject *capsule = get_internals_capsule()) {
+            Py_INCREF(capsule);
+            (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void {
+                               Py_XDECREF(get_internals_capsule());
+                               wr.dec_ref();
+                           }))
+                .release();
+        }
+        if (PyObject *capsule = get_local_internals_capsule()) {
+            Py_INCREF(capsule);
+            (void) weakref(handle(m_ptr), cpp_function([](handle wr) -> void {
+                               Py_XDECREF(get_local_internals_capsule());
+                               wr.dec_ref();
+                           }))
+                .release();
+        }
+
         if (rec.bases.size() > 1 || rec.multiple_inheritance) {
             mark_parents_nonsimple(tinfo->type);
             tinfo->simple_ancestors = false;