Refactoring and studying leaks reported by asan/lsan in pybind11

Apparently, CPython intentionally does not free all memory at shutdown.
The line defining the scoped_interpreter triggers Py_InitializeFromConfig,
where CPython allocates internal state.

Author: iglesias

adventofcode/2023/day/21/CMakeLists.txt

@@ -1,8 +1,8 @@
-cmake_minimum_required(VERSION 3.15..3.27)
+cmake_minimum_required(VERSION 3.15..4.2.3)
 project(aoc-2023-21 LANGUAGES CXX)
 set(CMAKE_CXX_STANDARD 23)
-#set(CMAKE_CXX_FLAGS "-Wall -Wconversion -Wextra -pedantic -fsanitize=address,pointer-overflow,signed-integer-overflow,undefined")
+set(CMAKE_CXX_FLAGS "-ggdb -Wall -Wconversion -Wextra -pedantic -fsanitize=address,pointer-overflow,signed-integer-overflow,undefined -fsanitize-address-use-after-scope -DGLIBCXX_DEBUG")
 find_package(pybind11 REQUIRED)
 add_executable(step_counter step_counter.cpp)
 target_link_libraries(step_counter PRIVATE pybind11::embed)
-set(CMAKE_BUILD_TYPE Release)
+set(CMAKE_BUILD_TYPE Debug)

adventofcode/2023/day/21/lsan.supp

@@ -0,0 +1,6 @@
+leak:libpython
+leak:_PyObject_Malloc
+leak:_PyMem_RawMalloc
+leak:Py_InitializeFromConfig
+leak:PyRun_
+leak:_PyRuntime

adventofcode/2023/day/21/run

@@ -0,0 +1 @@
+PYTHONMALLOC=malloc LSAN_OPTIONS=suppressions=lsan.supp ./build/step_counter < step_counter_sample_input.txt

adventofcode/2023/day/21/step_counter.cpp

@@ -44,6 +44,7 @@ void print(grid_t const& g)
 int main()
 {
   read_input();
+  pybind11::scoped_interpreter const guard{};
   std::pair const ans = solve();
   std::cout << "Part one: " << ans.first << "\nPart two: " << ans.second << '\n';
 }
@@ -113,17 +114,21 @@ ii get_start_position(grid_t const& G)
 int64_t fit_polynomial_and_extrapolate(std::vector<int64_t> const& x, std::vector<int64_t> const& y)
 {
   namespace py = pybind11;
-  py::scoped_interpreter const guard{};
-  py::object const scipy_interpolate = py::module::import("scipy.interpolate");
+
+  py::gil_scoped_acquire gil;
+
+  py::object const scipy_interpolate = py::module_::import("scipy.interpolate");
+  py::object const numpy_polynomial = py::module_::import("numpy.polynomial");
+
   py::object const poly =
       scipy_interpolate.attr("lagrange")(py::array_t<int64_t>(x.size(), x.data()),
                                          py::array_t<int64_t>(y.size(), y.data()));
-  //TODO does the call chain leak?
-  py::buffer_info const coef_buf = poly.attr("coef").cast<py::array_t<double>>().request();
+
+  py::array_t<double> const coef_arr = poly.attr("coef").cast<py::array_t<double>>();
+  py::buffer_info const coef_buf = coef_arr.request();
   std::vector<double> coef_vec(static_cast<double*>(coef_buf.ptr),
                                static_cast<double*>(coef_buf.ptr) + coef_buf.size);
   std::ranges::reverse(coef_vec);
-  py::object const numpy_polynomial = py::module::import("numpy.polynomial");
   return std::llround(numpy_polynomial.attr("Polynomial")
                       (py::array_t<double>(coef_vec.size(), coef_vec.data()))
                       (26501365).cast<double>());