diff --git a/ci/check_module_allocators.sh b/ci/check_module_allocators.sh new file mode 100755 index 000000000..396d1fe4d --- /dev/null +++ b/ci/check_module_allocators.sh @@ -0,0 +1,277 @@ +#!/bin/bash -e +# +# Copyright (c) 2025, valkey-search contributors +# All rights reserved. +# SPDX-License-Identifier: BSD 3-Clause +# +# Guards the invariants that keep the module's heap on ValkeyModule_Alloc: +# 1. static initializers are deferred until ValkeyModule_Alloc exists +# 2. the module defines the C allocator itself and imports none of it +# 3. no C++ heap object can cross a DSO boundary (libstdc++ is static) +# 4. nothing exported can collide with libstdc++.so.6 +# 5. no unhandled libc function hands us memory allocated by libc +# 6. no new shared library dependency appears unreviewed +# +# Each is verified to fail on a build that violates it; a silent pass here means +# the module would crash at load or corrupt the heap. +# +# Usage: check_module_allocators.sh + +MODULE_SO="$1" +BUILD_DIR="$2" + +if [ ! -f "${MODULE_SO}" ]; then + echo "check_module_allocators: no such file: ${MODULE_SO}" >&2 + exit 1 +fi + +FAILED=0 + +# Symbols the module defines for itself in vmsdk/src/memory_allocation_c_api.cc. +ALLOCATORS="malloc free calloc realloc aligned_alloc posix_memalign valloc + malloc_usable_size strdup realpath getcwd" + +undefined_syms() { + nm -D --undefined-only "$1" 2>/dev/null | sed 's/@.*//' | awk '{print $2}' | + sort -u +} + +# +# Check 1: static initializers are deferred. +# +# vmsdk/deferred_init.lds moves the module's C++ static initializers out of +# .init_array into .vmsdk_init_array, so the dynamic loader does not run them at +# dlopen() -- before ValkeyModule_Alloc exists. What is left in .init_array is +# crtbegin's frame_dummy, a single entry, which must keep running at load. +# +INIT_ARRAY_SZ=$(readelf -d "${MODULE_SO}" | awk '/\(INIT_ARRAYSZ\)/ {print $3}') +: "${INIT_ARRAY_SZ:=0}" +DEFERRED_HEX=$(readelf -S -W "${MODULE_SO}" | awk ' + { for (i = 1; i <= NF; i++) + if ($i == ".vmsdk_init_array") { print $(i + 4); exit } }') +if [ -n "${DEFERRED_HEX}" ]; then + DEFERRED_SZ=$((16#${DEFERRED_HEX})) +else + DEFERRED_SZ=0 +fi + +if [ "${INIT_ARRAY_SZ}" -gt 8 ]; then + echo "FAIL: ${MODULE_SO} has DT_INIT_ARRAYSZ=${INIT_ARRAY_SZ} (> 8 bytes)." >&2 + echo " Static initializers would run at dlopen(), before" >&2 + echo " ValkeyModule_Alloc is established. Is vmsdk/deferred_init.lds" >&2 + echo " still being passed to the linker?" >&2 + FAILED=1 +fi +if [ "${DEFERRED_SZ}" -eq 0 ]; then + echo "FAIL: ${MODULE_SO} has no .vmsdk_init_array section, or it is empty." >&2 + echo " vmsdk/deferred_init.lds did not take effect." >&2 + FAILED=1 +fi +if [ "${FAILED}" -eq 0 ]; then + echo "check_module_allocators: $((DEFERRED_SZ / 8)) static initializers deferred, \ +$((INIT_ARRAY_SZ / 8)) left at load time" +fi + +# +# Check 2: the module owns its allocator. +# +# memory_allocation_c_api.cc defines malloc and friends, and versionscript.lds +# marks them local so that every reference from inside the module -- libstdc++'s +# operator new, abseil, protobuf, gRPC, ICU, hdrhistogram, rax -- binds to them +# rather than to libc.so.6. Two things must hold, and neither is visible without +# checking: each name is defined here, and none is still imported from libc. An +# import means something is allocating outside ValkeyModule_Alloc, whose memory +# Valkey never accounts for and which crashes if it later reaches our free(). +# +UNDEF=$(undefined_syms "${MODULE_SO}") +# Names exported with GLOBAL or WEAK binding, i.e. the ones the dynamic linker +# will happily resolve somewhere else. +DYN_GLOBAL=$(readelf --dyn-syms -W "${MODULE_SO}" 2>/dev/null | + awk '$5 == "GLOBAL" || $5 == "WEAK" {sub(/@@?.*$/, "", $8); + print $8}' | sort -u) +for sym in ${ALLOCATORS}; do + if echo "${UNDEF}" | grep -qx "${sym}"; then + echo "FAIL: ${MODULE_SO} imports ${sym} from libc instead of using its" >&2 + echo " own definition. Is memory_allocation_c_api.cc still linked" >&2 + echo " into the module?" >&2 + FAILED=1 + elif ! nm "${MODULE_SO}" 2>/dev/null | grep -qE "^[0-9a-f]+ [Tt] ${sym}$"; then + echo "FAIL: ${MODULE_SO} does not define ${sym}." >&2 + echo " memory_allocation_c_api.cc is not being linked in." >&2 + FAILED=1 + elif echo "${DYN_GLOBAL}" | grep -qx "${sym}"; then + # Defined, but exported with global binding -- which means preemptible. + # References from inside the module then resolve through the global + # scope, find libc.so.6's definition first, and this one is silently + # bypassed: the module keeps running on the system allocator and Valkey + # accounts for none of it. Nothing crashes, so only this check catches + # it. + echo "FAIL: ${MODULE_SO} defines ${sym} but exports it with global" >&2 + echo " binding, so it is preemptible and will be bypassed in" >&2 + echo " favour of libc's. Is ${sym} still listed under local: in" >&2 + echo " vmsdk/versionscript.lds?" >&2 + FAILED=1 + fi +done + +# +# Check 3: no C++ heap object can cross a DSO boundary. +# +# The allocator above only covers code linked into this .so. If the module still +# called into libstdc++.so.6, an object allocated there (by std::getline, +# std::filesystem::path, std::locale::name, ...) would be freed here with +# ValkeyModule_Free -- a jemalloc free of a libc malloc pointer. Linking +# libstdc++ statically removes the boundary; this confirms it stayed removed. +# +GLIBCXX_UNDEF=$(nm -D --undefined-only "${MODULE_SO}" 2>/dev/null | + grep -c "GLIBCXX" || true) +if [ "${GLIBCXX_UNDEF}" -ne 0 ]; then + echo "FAIL: ${MODULE_SO} has ${GLIBCXX_UNDEF} undefined GLIBCXX symbols," >&2 + echo " so it is calling into libstdc++.so.6. Objects allocated there" >&2 + echo " would be freed here with ValkeyModule_Free and crash. Is" >&2 + echo " -static-libstdc++ still being passed to the linker?" >&2 + nm -D --undefined-only "${MODULE_SO}" | grep "GLIBCXX" | head -5 >&2 + FAILED=1 +fi + +# +# Check 4: nothing the module exports can collide with libstdc++.so.6. +# +# libstdc++ is linked statically, but exported symbols still take part in +# dynamic symbol resolution. The dangerous ones are the locale facet ids +# (std::num_put::id and friends): they are STB_GNU_UNIQUE, which the +# dynamic linker unifies process-wide even for an RTLD_LOCAL dlopen. If +# libstdc++.so.6 is also present -- it arrives with any other C++ module, and +# valkey-json is loaded before search in the integration tests -- our facet ids +# and its become one object while the facet arrays stay separate, so the first +# ostream insertion dereferences the wrong facet and segfaults at module load. +# +# -Wl,--exclude-libs,ALL is what keeps this list empty. nm prints +# libstdc++.so.6's names with an @@GLIBCXX version suffix and ours without, so +# the suffix is stripped before comparing. Only GLOBAL/WEAK exports matter, so +# the local entries the version script produces are filtered out. +# +LIBSTDCXX=$(gcc -print-file-name=libstdc++.so.6 2>/dev/null || true) +if [ -n "${LIBSTDCXX}" ] && [ -f "${LIBSTDCXX}" ]; then + CLASHES=$(comm -12 \ + <(nm -D --defined-only "${MODULE_SO}" | + awk '$2 ~ /^[TDWVBRi]$/ {sub(/@@?.*$/, "", $3); print $3}' | + sort -u) \ + <(nm -D --defined-only "${LIBSTDCXX}" | + awk '{sub(/@@?.*$/, "", $3); print $3}' | sort -u)) + if [ -n "${CLASHES}" ]; then + NCLASH=$(echo "${CLASHES}" | wc -l) + echo "FAIL: ${MODULE_SO} exports ${NCLASH} symbol(s) that" >&2 + echo " libstdc++.so.6 also defines. The locale facet ids among" >&2 + echo " them are STB_GNU_UNIQUE and get merged across the two" >&2 + echo " libstdc++ copies as soon as another C++ module is loaded," >&2 + echo " crashing at module load. Is -Wl,--exclude-libs,ALL still" >&2 + echo " being passed to the linker?" >&2 + echo "${CLASHES}" | head -5 | sed 's/^/ /' >&2 + FAILED=1 + fi +fi + +# +# Check 5: no unhandled libc function hands us libc-allocated memory. +# +# The module's allocator is local, so libc.so.6 never sees it and keeps using +# its own. Memory allocated inside libc and freed inside libc is therefore +# self-consistent. The one way a pointer crosses is a libc function that +# allocates a result and returns it to us: we would later release it through our +# free(), handing a libc pointer to ValkeyModule_Free. +# +# memory_allocation_c_api.cc handles every such function the module references +# today -- strdup is reimplemented, realpath and getcwd abort. If a new one +# appears, it must be handled there before this check will pass. +# +# Note that __realpath_chk is absent from this list on purpose: it is the +# fortified form taking a caller-provided buffer, which does not allocate. ICU's +# uprv_tzname uses it legitimately. +# +# Both the public names and the glibc-internal aliases the compiler actually +# emits: turns getline() into __getdelim(), for instance. +ALLOCATING_LIBC="strndup __strdup __strndup + getline getdelim __getdelim + asprintf vasprintf __asprintf __vasprintf + canonicalize_file_name get_current_dir_name + scandir scandir64 tempnam wcsdup __wcsdup open_memstream" +for sym in ${ALLOCATING_LIBC}; do + if echo "${UNDEF}" | grep -qx "${sym}"; then + echo "FAIL: ${MODULE_SO} references ${sym}(), which allocates its" >&2 + echo " result with libc's malloc. Freeing that pointer inside the" >&2 + echo " module passes it to ValkeyModule_Free and corrupts the" >&2 + echo " heap. Handle it in vmsdk/src/memory_allocation_c_api.cc" >&2 + echo " alongside strdup/realpath/getcwd." >&2 + FAILED=1 + fi +done + +# +# Check 6: the set of shared libraries the module depends on is pinned. +# +# Every DSO the module links against is another allocator boundary: memory it +# allocates and hands back is allocated by its allocator, not ours, and freeing +# it here would pass it to ValkeyModule_Free. Check 5 covers libc, which is the +# only one whose allocate-and-return functions the module calls directly. The +# rest have been reviewed and are safe: +# +# libsystemd Only sd_is_socket_inet, sd_is_socket_sockaddr, +# sd_is_socket_unix and sd_listen_fds are imported. All return +# int; nothing crosses. +# libssl, The constructors and duplicators imported (BIO_new, SSL_new, +# libcrypto EVP_*_CTX_new, X509_NAME_dup, SSL_get1_peer_certificate, ...) +# are each paired with the matching free function, which is +# imported too. The raw-buffer cases (ASN1_STRING_to_UTF8, the +# i2d_* family with a null output pointer) must be released with +# OPENSSL_free, which is CRYPTO_free inside libcrypto -- so both +# the allocation and the free happen on the far side of the +# boundary, as with getaddrinfo/freeaddrinfo. +# libm, No allocation. +# libmvec +# libgcc_s Unwinder only. +# ld-linux dlopen/dlsym only. +# +# A new entry here means a boundary nobody has looked at, so it fails the build +# until someone does. Note that OpenSSL is deliberately dynamic: linking it +# statically would mean rebuilding the module for every OpenSSL CVE rather than +# picking up a distribution update. +# +# Only additions fail. A dependency disappearing is not a memory-safety problem +# -- and it is how this list last changed, when -static-libstdc++ removed +# libstdc++.so.6. +# +ALLOWED_NEEDED="libc.so.6 libm.so.6 libmvec.so.1 libgcc_s.so.1 + libssl.so.3 libcrypto.so.3 libsystemd.so.0" + +# Unquoted, so that the newlines and indentation above collapse to single +# spaces; the match below relies on every entry being space-delimited. +ALLOWED_NEEDED=$(echo ${ALLOWED_NEEDED}) + +NEEDED=$(readelf -d "${MODULE_SO}" 2>/dev/null | + sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p') +for lib in ${NEEDED}; do + # The dynamic loader's own name is architecture-specific. + case "${lib}" in + ld-linux-*.so.*) continue ;; + esac + case " ${ALLOWED_NEEDED} " in + *" ${lib} "*) continue ;; + esac + echo "FAIL: ${MODULE_SO} has a new shared library dependency: ${lib}" >&2 + echo " Each DSO is another allocator boundary. Check whether it has" >&2 + echo " functions that allocate memory and hand it to the caller: if" >&2 + echo " the module frees such a pointer, it goes to ValkeyModule_Free" >&2 + echo " and corrupts the heap. Record the finding next to check 6 in" >&2 + echo " this script and add it to ALLOWED_NEEDED." >&2 + FAILED=1 +done + +if [ "${FAILED}" -ne 0 ]; then + echo "" >&2 + echo "See vmsdk/src/memory_allocation_c_api.cc for how module memory is" >&2 + echo "expected to be allocated." >&2 + exit 1 +fi + +echo "check_module_allocators: OK" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6ccd6cee6..cbb5e5d98 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -5,15 +5,102 @@ add_subdirectory(expr) add_subdirectory(query) add_subdirectory(utils) -set(SRCS_MODULE_LOADER ${CMAKE_CURRENT_LIST_DIR}/module_loader.cc) +# The module's C allocator entry points are a direct object of the shared +# library rather than a member of vmsdklib. Two reasons: a static archive member +# is only pulled in to satisfy an existing undefined reference, which makes +# whether malloc gets defined depend on link order; and vmsdklib is also linked +# into the unit test executables, where defining malloc would override the +# allocator for the whole test process, including libc's own startup +# allocations. Tests therefore run entirely on the system allocator. +set(SRCS_MODULE_LOADER + ${CMAKE_CURRENT_LIST_DIR}/module_loader.cc + ${CMAKE_SOURCE_DIR}/vmsdk/src/memory_allocation_c_api.cc) # This is the module target -valkey_search_add_shared_library(libsearch ${SRCS_MODULE_LOADER}) +valkey_search_add_shared_library(libsearch "${SRCS_MODULE_LOADER}") target_include_directories(libsearch PUBLIC ${CMAKE_CURRENT_LIST_DIR}) if(UNIX AND NOT APPLE) target_link_options( libsearch PRIVATE "LINKER:--version-script=${CMAKE_SOURCE_DIR}/vmsdk/versionscript.lds") + set_property( + TARGET libsearch APPEND PROPERTY LINK_DEPENDS + ${CMAKE_SOURCE_DIR}/vmsdk/versionscript.lds) +endif() + +# Defer the module's C++ static initializers until ValkeyModule_Alloc has been +# established, so that they allocate from Valkey rather than the system +# allocator. See vmsdk/deferred_init.lds and vmsdk/src/deferred_init.cc. +# +# Sanitizer builds are excluded: the sanitizer emits its own constructor into +# .init_array to register global redzones, and that must run at dlopen(). They +# do not use the Valkey allocator at all, so there is nothing to defer for. +# macOS is excluded because Mach-O has no linker scripts; it is a build-only +# target and likewise does not use the Valkey allocator. +# Link libstdc++ into the module rather than using libstdc++.so. +# +# REQUIRED for correctness, not an optimization. The module replaces global +# operator new/delete so that C++ allocation goes to ValkeyModule_Alloc, but a +# replacement only covers code linked into this .so -- the module is dlopen'd +# with local scope and versionscript.lds hides _Znwm/_ZdlPv, so libstdc++.so +# keeps using libc malloc. Any heap object allocated inside libstdc++.so and +# destroyed here would then be passed to ValkeyModule_Free, which is a jemalloc +# free of a non-jemalloc pointer -- an immediate crash. +# +# That is not hypothetical: std::getline lives in libstdc++.so and grows the +# caller's std::string with libc malloc. vmsdk::helper::ParseCPUInfo, reached +# from a static initializer, does exactly this and segfaults at module load +# without this flag. It only appeared once the allocator switch moved to the top +# of ValkeyModule_OnLoad; previously these paths ran before the switch. +# +# Linking libstdc++ statically puts that code inside the module, where the +# operator new/delete replacements apply, and leaves the module with zero +# undefined GLIBCXX symbols -- so no C++ heap object can cross a DSO boundary. +# ci/check_module_allocators.sh enforces that. +if(UNIX AND NOT APPLE AND NOT SAN_BUILD) + target_link_options(libsearch PRIVATE -static-libstdc++) + + # ...and hide everything that came from a static archive, libstdc++.a + # included. Static linking alone is not enough, because versionscript.lds is + # "global: *" and so exports the statically linked libstdc++ symbols -- among + # them the locale facet ids, such as std::num_put::id. + # + # Those ids are STB_GNU_UNIQUE ("u" in nm output), which the dynamic linker + # unifies to a single instance process-wide, RTLD_LOCAL notwithstanding. As + # soon as any other module that links libstdc++.so.6 dynamically is loaded -- + # valkey-json does, and the integration tests load it before search -- our + # facet id object and libstdc++.so.6's become the same object. std::locale:: + # id::_M_id() then hands us an index into that library's facet numbering while + # our own facet array is a separate one, and the first ostream insertion + # dereferences the wrong facet and segfaults. It reproduces as a crash inside + # std::ostream::_M_insert during module load. + # + # Hiding the archive symbols keeps them out of .dynsym, so they are no longer + # unique and each library keeps a self-consistent set of facet ids. Only + # ValkeyModule_OnLoad/OnUnload need to be exported, and they come from + # module_loader.cc.o rather than an archive, so they are unaffected. The + # module consumes JSON's shared API through ValkeyModule_GetSharedAPI (a + # function pointer from the server) and exports none of its own, so nothing + # depends on these symbols being visible. + # + # ci/check_module_allocators.sh enforces this. + target_link_options(libsearch PRIVATE "LINKER:--exclude-libs,ALL") +endif() + +if(UNIX AND NOT APPLE AND NOT SAN_BUILD) + target_link_options( + libsearch PRIVATE + "LINKER:-T,${CMAKE_SOURCE_DIR}/vmsdk/deferred_init.lds") + set_property( + TARGET libsearch APPEND PROPERTY LINK_DEPENDS + ${CMAKE_SOURCE_DIR}/vmsdk/deferred_init.lds) + + add_custom_command( + TARGET libsearch + POST_BUILD + COMMAND ${CMAKE_SOURCE_DIR}/ci/check_module_allocators.sh + $ ${CMAKE_BINARY_DIR} + COMMENT "Verifying static-init deferral and allocator usage") endif() target_link_libraries(libsearch PUBLIC keyspace_event_manager) diff --git a/src/indexes/text/rax/rax_malloc.h b/src/indexes/text/rax/rax_malloc.h index 4c9c6a929..1f155679a 100644 --- a/src/indexes/text/rax/rax_malloc.h +++ b/src/indexes/text/rax/rax_malloc.h @@ -32,15 +32,20 @@ #define RAX_ALLOC_H #include -/* Override with the wrappers provided by VMSDK. */ -extern void* __wrap_malloc(size_t size); -extern void __wrap_free(void* ptr); -extern void* __wrap_realloc(void* ptr, size_t size); -extern int __wrap_malloc_usable_size(void* ptr); +/* Plain libc names: inside the module these bind to the allocator defined in + * vmsdk/src/memory_allocation_c_api.cc, which routes to ValkeyModule_Alloc. */ +#include -#define rax_malloc __wrap_malloc -#define rax_realloc __wrap_realloc -#define rax_free __wrap_free -#define rax_ptr_alloc_size(ptr) ((size_t)__wrap_malloc_usable_size(ptr)) +#ifdef __APPLE__ +#include +#define rax_ptr_alloc_size(ptr) malloc_size(ptr) +#else +#include +#define rax_ptr_alloc_size(ptr) malloc_usable_size(ptr) +#endif + +#define rax_malloc malloc +#define rax_realloc realloc +#define rax_free free #endif diff --git a/src/module_loader.cc b/src/module_loader.cc index afaa0c81a..883d1af43 100644 --- a/src/module_loader.cc +++ b/src/module_loader.cc @@ -31,7 +31,7 @@ inline std::list ACLPermissionFormatter( } // namespace vmsdk::module::Options options = { - .name = "search", + .name = kModuleName, .acl_categories = ACLPermissionFormatter({ valkey_search::kSearchCategory, }), @@ -126,4 +126,4 @@ vmsdk::module::Options options = { valkey_search::ValkeySearch::Instance().OnUnload(ctx); }, }; -VALKEY_MODULE(options); +VALKEY_MODULE(options, kModuleName, kModuleVersion); diff --git a/src/version.h b/src/version.h index a2213a7f5..a596849ce 100644 --- a/src/version.h +++ b/src/version.h @@ -10,6 +10,13 @@ #include "utils.h" +// +// The module name. Passed to VALKEY_MODULE, which reads it before the module's +// static initializers have run, so it must be constant-initialized -- it cannot +// be sourced from vmsdk::module::Options. See the VALKEY_MODULE comment. +// +inline constexpr char kModuleName[] = "search"; + // // Set the module version to the current release // diff --git a/testing/rax_wrapper_test.cc b/testing/rax_wrapper_test.cc index 84a2bb02e..3a670086b 100644 --- a/testing/rax_wrapper_test.cc +++ b/testing/rax_wrapper_test.cc @@ -24,16 +24,8 @@ #include #include "gtest/gtest.h" -#include "vmsdk/src/memory_allocation.h" #include "vmsdk/src/testing_infra/utils.h" -// Override the weak symbol empty_usable_size (defined in -// memory_allocation_overrides.cc) with actual memory tracking for -// RaxMallocMemoryTracking. -extern "C" size_t empty_usable_size(void *ptr) noexcept { - return malloc_usable_size(ptr); -} - namespace valkey_search::indexes::text { namespace { @@ -589,25 +581,25 @@ TEST_F(RaxTest, FindTarget) { nullptr); // extension of existing word } -TEST_F(RaxTest, RaxMallocMemoryTracking) { - // Validates that rax_malloc.h correctly routes allocations through - // the VMSDK memory tracking system. - - uint64_t initial_memory = vmsdk::GetUsedMemoryCnt(); - { - // Create empty Rax. The only heap allocations are from raxNew(). - Rax empty_rax{nullptr}; - uint64_t after_create_memory = vmsdk::GetUsedMemoryCnt(); - std::cout << "Memory increased by " - << (after_create_memory - initial_memory) << " bytes" - << std::endl; - EXPECT_GT(after_create_memory, initial_memory) - << "Creating Rax should increase the tracked allocated memory"; - EXPECT_EQ(empty_rax.GetAllocSize(), after_create_memory - initial_memory); - } - // The memory should return to zero after falling out of scope. - EXPECT_EQ(initial_memory, vmsdk::GetUsedMemoryCnt()) - << "Destroying Rax should free all rax allocations"; +TEST_F(RaxTest, RaxAllocSizeReporting) { + // Validates rax_malloc.h's rax_ptr_alloc_size wiring: rax must be able to + // report the usable size of its own allocations. + // + // This deliberately does not assert against vmsdk::GetUsedMemoryCnt(). + // rax_malloc.h now uses the plain allocator names, which inside the module + // bind to the definitions in vmsdk/src/memory_allocation_c_api.cc and so feed + // the VMSDK accounting. That file is linked only into the module, not into + // test executables -- see src/CMakeLists.txt -- so here the names resolve to + // libc and nothing is accounted. That the module binds them correctly is + // enforced at link time by ci/check_module_allocators.sh. + // The fixture's rax_ starts empty; its only heap allocation is from raxNew(). + size_t empty_size = rax_.GetAllocSize(); + EXPECT_GT(empty_size, 0u) + << "Rax should report the usable size of its allocations"; + + AddWords({{"hello", 1}, {"world", 2}}); + EXPECT_GT(rax_.GetAllocSize(), empty_size) + << "Adding entries should increase the reported allocation size"; } } // namespace diff --git a/testing/valkey_search_test.cc b/testing/valkey_search_test.cc index a1d5d4909..87aa7561e 100644 --- a/testing/valkey_search_test.cc +++ b/testing/valkey_search_test.cc @@ -338,13 +338,13 @@ TEST_P(LoadTest, load) { .WillRepeatedly(testing::Return(0)); } vmsdk::module::Options options = { + .name = kModuleName, .version = kModuleVersion, .minimum_valkey_server_version = kMinimumServerVersion, }; auto load_res = vmsdk::module::OnLoadDone( ValkeySearch::Instance().OnLoad(&fake_ctx_, args.data(), args.size()), &fake_ctx_, options); - vmsdk::ResetValkeyAlloc(); EXPECT_EQ(load_res, test_case.expected_load_ret); auto writer_thread_pool = ValkeySearch::Instance().GetWriterThreadPool(); auto reader_thread_pool = ValkeySearch::Instance().GetReaderThreadPool(); diff --git a/vmsdk/deferred_init.lds b/vmsdk/deferred_init.lds new file mode 100644 index 000000000..208afa03a --- /dev/null +++ b/vmsdk/deferred_init.lds @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2025, valkey-search contributors + * All rights reserved. + * SPDX-License-Identifier: BSD 3-Clause + * + * Defer the module's C++ static initializers until after ValkeyModule_Alloc is + * known. + * + * All memory allocated by the module must come from ValkeyModule_Alloc/Free, + * but their addresses are not known until ValkeyModule_OnLoad runs. Static + * initializers normally run at dlopen(), i.e. before that. This script renames + * the output section holding their function pointers, which removes the + * DT_INIT_ARRAY entry the dynamic loader would have walked. The array is + * instead walked explicitly from ValkeyModule_OnLoad once the allocator is + * established -- see vmsdk/src/deferred_init.cc. + * + * This is an addition to the default linker script (INSERT), not a replacement. + * The input-section rules mirror the default .init_array rules exactly, so + * initialization priority ordering is preserved. crtbegin/crtend are excluded + * so that frame_dummy stays in the real .init_array and still runs at dlopen(). + */ +SECTIONS +{ + .vmsdk_init_array : + { + PROVIDE_HIDDEN (__vmsdk_init_array_start = .); + KEEP (*(SORT_BY_INIT_PRIORITY(.init_array.*) SORT_BY_INIT_PRIORITY(.ctors.*))) + KEEP (*(EXCLUDE_FILE (*crtbegin.o *crtbegin?.o *crtend.o *crtend?.o) .init_array .ctors)) + PROVIDE_HIDDEN (__vmsdk_init_array_end = .); + } +} INSERT BEFORE .init_array; diff --git a/vmsdk/src/CMakeLists.txt b/vmsdk/src/CMakeLists.txt index 2ba7b9bfb..8613b803a 100644 --- a/vmsdk/src/CMakeLists.txt +++ b/vmsdk/src/CMakeLists.txt @@ -30,10 +30,14 @@ set(VMSDKLIB_SRCS ${CMAKE_CURRENT_LIST_DIR}/module.h ${CMAKE_CURRENT_LIST_DIR}/module_config.cc ${CMAKE_CURRENT_LIST_DIR}/module_config.h - ${CMAKE_CURRENT_LIST_DIR}/memory_allocation_overrides.cc + # NOTE: memory_allocation_c_api.cc, which defines malloc/free/etc for the + # module, is deliberately NOT part of vmsdklib -- it is linked directly into + # the libsearch target. See src/CMakeLists.txt. ${CMAKE_CURRENT_LIST_DIR}/memory_allocation_overrides.h ${CMAKE_CURRENT_LIST_DIR}/memory_allocation.cc ${CMAKE_CURRENT_LIST_DIR}/memory_allocation.h + ${CMAKE_CURRENT_LIST_DIR}/deferred_init.cc + ${CMAKE_CURRENT_LIST_DIR}/deferred_init.h ${CMAKE_CURRENT_LIST_DIR}/type_conversions.h ${CMAKE_CURRENT_LIST_DIR}/managed_pointers.h ${CMAKE_CURRENT_LIST_DIR}/blocked_client.cc diff --git a/vmsdk/src/deferred_init.cc b/vmsdk/src/deferred_init.cc new file mode 100644 index 000000000..a23830e2a --- /dev/null +++ b/vmsdk/src/deferred_init.cc @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2025, valkey-search contributors + * All rights reserved. + * SPDX-License-Identifier: BSD 3-Clause + * + */ + +// This translation unit runs before the module's static initializers have run. +// It must therefore not define, or depend on, anything that is dynamically +// initialized -- no absl logging, no std::string, no function-local statics +// with non-trivial types. Everything here is zero-initialized or constant. + +#include "vmsdk/src/deferred_init.h" + +#include + +#ifndef __APPLE__ +extern "C" { +// Bounds of the relocated .init_array, provided by vmsdk/deferred_init.lds. +// Weak: builds that do not apply the linker script (sanitizer builds) leave +// these undefined, and static initialization happens at dlopen() as usual. +extern void (*__vmsdk_init_array_start[])(int, char **, char **) + __attribute__((weak)); +extern void (*__vmsdk_init_array_end[])(int, char **, char **) + __attribute__((weak)); +} // extern "C" +#endif // !__APPLE__ + +namespace vmsdk { +namespace { +// Zero-initialized, so safe to read before any initializer has run. +size_t initializers_run = 0; +} // namespace + +size_t GetDeferredInitializerCount() { return initializers_run; } + +size_t RunDeferredStaticInitializers() { +#ifdef __APPLE__ + // Static initialization is never deferred here: the relocation is done by a + // GNU linker script and Mach-O has no equivalent, so the initializers already + // ran at dlopen() time using the system allocator. + // + // The bounds symbols cannot even be declared on this platform. ELF resolves a + // weak undefined symbol to a null address, which is what the check below + // relies on; Mach-O has no such thing, and a plain weak declaration of a + // missing symbol is a link error. + return 0; +#else + if (__vmsdk_init_array_start == nullptr || + __vmsdk_init_array_end == nullptr) { + // Static initialization was not deferred on this build (sanitizer builds); + // it already ran at dlopen() time, legitimately using the system allocator. + return 0; + } + + // Guard against a second module load re-running initializers. + if (initializers_run != 0) { + return initializers_run; + } + + const size_t count = __vmsdk_init_array_end - __vmsdk_init_array_start; + for (size_t i = 0; i < count; ++i) { + __vmsdk_init_array_start[i](0, nullptr, nullptr); + } + initializers_run = count; + return count; +#endif // __APPLE__ +} + +} // namespace vmsdk diff --git a/vmsdk/src/deferred_init.h b/vmsdk/src/deferred_init.h new file mode 100644 index 000000000..be272555f --- /dev/null +++ b/vmsdk/src/deferred_init.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2025, valkey-search contributors + * All rights reserved. + * SPDX-License-Identifier: BSD 3-Clause + * + */ + +#ifndef VMSDK_SRC_DEFERRED_INIT_H_ +#define VMSDK_SRC_DEFERRED_INIT_H_ + +#include + +namespace vmsdk { + +// Runs the module's C++ static initializers, which vmsdk/deferred_init.lds has +// relocated out of .init_array so that they do NOT run at dlopen() time. +// +// Must be called from ValkeyModule_OnLoad after ValkeyModule_Init has +// established ValkeyModule_Alloc/Free, and before anything touches a +// dynamically-initialized global. Until it returns, every such global is still +// zero-initialized. +// +// The initializers allocate, so the ordering against ValkeyModule_Init is what +// keeps the module off the system allocator entirely. Get it wrong and +// ValkeyModule_Alloc is still null, so the first allocation faults at its call +// site rather than quietly succeeding. +// +// Returns the number of initializers run. Returns 0 without doing anything on +// builds that do not apply the linker script (see GetDeferredInitializerCount). +size_t RunDeferredStaticInitializers(); + +// Number of initializers RunDeferredStaticInitializers() ran, for logging once +// logging is available. Zero on builds that do not defer static initialization. +size_t GetDeferredInitializerCount(); + +} // namespace vmsdk + +#endif // VMSDK_SRC_DEFERRED_INIT_H_ diff --git a/vmsdk/src/info.cc b/vmsdk/src/info.cc index 041811a83..777d2f614 100644 --- a/vmsdk/src/info.cc +++ b/vmsdk/src/info.cc @@ -12,6 +12,7 @@ #include "absl/container/btree_map.h" #include "absl/container/flat_hash_set.h" +#include "absl/strings/str_cat.h" #include "vmsdk/src/log.h" #include "vmsdk/src/module_config.h" #include "vmsdk/src/utils.h" @@ -333,7 +334,7 @@ static size_t DumpNames(ValkeyModuleCtx* ctx, << "/" << field->GetName(); ValkeyModule_ReplyWithCString(ctx, "Section"); ValkeyModule_ReplyWithCString(ctx, field->GetSection().data()); - std::string external_name = options.name + "." + field->GetName(); + std::string external_name = absl::StrCat(options.name, ".", field->GetName()); ValkeyModule_ReplyWithCString(ctx, "Name"); ValkeyModule_ReplyWithCString(ctx, external_name.data()); return 4; diff --git a/vmsdk/src/memory_allocation.cc b/vmsdk/src/memory_allocation.cc index e1b857589..fc2849729 100644 --- a/vmsdk/src/memory_allocation.cc +++ b/vmsdk/src/memory_allocation.cc @@ -26,11 +26,6 @@ thread_local static int64_t memory_delta = 0; ShardedAtomic used_memory_bytes; -void ResetValkeyAllocStats() { - used_memory_bytes.Reset(); - memory_delta = 0; -} - uint64_t GetUsedMemoryCnt() { return used_memory_bytes.GetTotal(); } void ReportAllocMemorySize(uint64_t size) { diff --git a/vmsdk/src/memory_allocation.h b/vmsdk/src/memory_allocation.h index df08c64cb..d2dc1738d 100644 --- a/vmsdk/src/memory_allocation.h +++ b/vmsdk/src/memory_allocation.h @@ -11,7 +11,7 @@ #include namespace vmsdk { -void ResetValkeyAllocStats(); + // Report used memory counter. uint64_t GetUsedMemoryCnt(); diff --git a/vmsdk/src/memory_allocation_c_api.cc b/vmsdk/src/memory_allocation_c_api.cc new file mode 100644 index 000000000..c56698540 --- /dev/null +++ b/vmsdk/src/memory_allocation_c_api.cc @@ -0,0 +1,240 @@ +/* + * Copyright (c) 2025, valkey-search contributors + * All rights reserved. + * SPDX-License-Identifier: BSD 3-Clause + * + */ + +// The module's C allocator entry points. +// +// Everything the module allocates must come from ValkeyModule_Alloc, so that +// Valkey accounts for it and it lives in the server's jemalloc arena. Rather +// than redirecting call sites -- which only ever covered the translation units +// that included a particular header, plus C++ via replaced operator new/delete +// -- this file simply *defines* malloc and friends inside the module. +// +// That works because of two link options on the module (see src/CMakeLists.txt +// and vmsdk/versionscript.lds): +// +// -static-libstdc++ puts libstdc++, including operator new/delete, inside +// the module, so C++ allocation reaches these functions. +// version script lists these symbols as local, so they are not exported +// + --exclude-libs and, being non-preemptible, every reference from +// within the module binds here at link time. +// +// The second point is what makes this work at all: a dlopened library's symbol +// lookups search the global scope first, and libc.so.6 defines malloc, so a +// module-defined malloc with default visibility is simply ignored -- even by +// the module's own operator new. Made local, it captures everything linked into +// the module: libstdc++, abseil, protobuf, gRPC, ICU, hdrhistogram and rax. +// +// Because these are local to the module, libc.so.6 never sees them and keeps +// using its own allocator. Memory allocated inside libc and freed inside libc +// therefore stays self-consistent; the only hazard is a pointer that crosses +// that boundary, which is what the strdup/realpath/getcwd definitions at the +// bottom of this file address. +// +// There is deliberately no fallback for the window before ValkeyModule_Alloc +// is established. Nothing in the module allocates then -- static initializers +// are deferred until after it is set, see vmsdk/src/deferred_init.cc -- and if +// something ever did, ValkeyModule_Alloc is still a null function pointer, so +// the call faults immediately at the offending call site. Valkey's crash +// handler prints the backtrace, which localises the problem better than any +// bookkeeping we could carry on every allocation to detect it after the fact. + +// Defines VMSDK_USE_VALKEY_ALLOC_OVERRIDES. +#include "vmsdk/src/memory_allocation_overrides.h" + +#ifdef VMSDK_USE_VALKEY_ALLOC_OVERRIDES + +// Deliberately inside the guard: is glibc-only, and this file +// compiles to nothing on the platforms that do not define the guard. +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "absl/base/optimization.h" +#include "vmsdk/src/memory_allocation.h" +#include "vmsdk/src/valkey_module_api/valkey_module.h" + +extern "C" { +// glibc's fortified realpath. Declared here because only exposes it +// when _FORTIFY_SOURCE is on. Distinct from the realpath defined below, so +// calling it does not recurse. +char* __realpath_chk(const char* path, char* resolved, size_t resolved_len); +} // extern "C" + +namespace { + +// Valkey exposes no aligned allocation entry point. jemalloc returns memory +// aligned to the size class, so rounding the request up to a multiple of the +// alignment gets us the alignment we need. +// +// See https://linux.die.net/man/3/jemalloc: "... Chunks are always aligned to +// multiples of the chunk size..." +size_t AlignSize(size_t size, size_t alignment = 16) { + return (size + alignment - 1) & ~(alignment - 1); +} + +} // namespace + +extern "C" { + +void* malloc(size_t size) noexcept { + // Force 16-byte alignment; Valkey may otherwise return 8-byte aligned memory. + void* ptr = ValkeyModule_Alloc(AlignSize(size)); + if (ABSL_PREDICT_TRUE(ptr != nullptr)) { + vmsdk::ReportAllocMemorySize(ValkeyModule_MallocUsableSize(ptr)); + } + return ptr; +} + +void free(void* ptr) noexcept { + if (ptr == nullptr) { + return; + } + vmsdk::ReportFreeMemorySize(ValkeyModule_MallocUsableSize(ptr)); + ValkeyModule_Free(ptr); +} + +void* calloc(size_t nmemb, size_t size) noexcept { + void* ptr = ValkeyModule_Calloc(nmemb, AlignSize(size)); + if (ABSL_PREDICT_TRUE(ptr != nullptr)) { + vmsdk::ReportAllocMemorySize(ValkeyModule_MallocUsableSize(ptr)); + } + return ptr; +} + +void* realloc(void* ptr, size_t size) noexcept { + if (ABSL_PREDICT_FALSE(ptr == nullptr)) { + return malloc(size); + } + size_t old_size = ValkeyModule_MallocUsableSize(ptr); + void* new_ptr = ValkeyModule_Realloc(ptr, AlignSize(size)); + if (ABSL_PREDICT_TRUE(new_ptr != nullptr)) { + vmsdk::ReportFreeMemorySize(old_size); + vmsdk::ReportAllocMemorySize(ValkeyModule_MallocUsableSize(new_ptr)); + } + return new_ptr; +} + +void* aligned_alloc(size_t alignment, size_t size) noexcept { + void* ptr = ValkeyModule_Alloc(AlignSize(size, alignment)); + if (ABSL_PREDICT_TRUE(ptr != nullptr)) { + vmsdk::ReportAllocMemorySize(ValkeyModule_MallocUsableSize(ptr)); + } + return ptr; +} + +int posix_memalign(void** memptr, size_t alignment, size_t size) noexcept { + *memptr = aligned_alloc(alignment, size); + return *memptr == nullptr ? ENOMEM : 0; +} + +void* valloc(size_t size) noexcept { + return aligned_alloc(sysconf(_SC_PAGESIZE), size); +} + +size_t malloc_usable_size(void* ptr) noexcept { + if (ABSL_PREDICT_FALSE(ptr == nullptr)) { + return 0; + } + return ValkeyModule_MallocUsableSize(ptr); +} + +// +// libc functions that allocate and hand the result to the caller. +// +// These are the only way a pointer can cross between libc's allocator and ours: +// glibc would allocate the result with its own malloc, and the caller -- inside +// this module -- would release it through the free() above, handing a libc +// pointer to ValkeyModule_Free. Defining them here keeps both halves on the +// same allocator. +// +// ci/check_module_allocators.sh fails the build if the module ever references +// an allocate-and-return libc function that is not handled here. +// + +// Reached from absl::InitializeSymbolizer and libstdc++'s message catalogs. +char* strdup(const char* s) noexcept { + size_t size = strlen(s) + 1; + char* copy = static_cast(malloc(size)); + if (ABSL_PREDICT_FALSE(copy == nullptr)) { + return nullptr; + } + memcpy(copy, s, size); + return copy; +} + +// realpath(path, nullptr) and getcwd(nullptr, 0) return a buffer glibc +// allocated with its own malloc, which free() above would hand to +// ValkeyModule_Free. Both are reimplemented so the result comes from our +// allocator instead. +// +// Neither may call the libc function of the same name: that name binds to the +// definition here and would recurse. getcwd goes straight to the kernel, and +// realpath delegates to glibc's fortified entry point, which is a distinct +// symbol this file does not define. ICU's uprv_tzname already calls +// __realpath_chk directly with its own buffer, which allocates nothing. +char* realpath(const char* path, char* resolved_path) noexcept { + // __realpath_chk resolves into a caller-provided buffer and __chk_fail()s if + // it is smaller than PATH_MAX, which is also what POSIX requires callers of + // realpath() to supply. Resolve into our own buffer either way, so a failure + // leaves the caller's untouched. + char resolved[PATH_MAX]; + if (__realpath_chk(path, resolved, sizeof(resolved)) == nullptr) { + return nullptr; + } + if (resolved_path != nullptr) { + return strcpy(resolved_path, resolved); + } + return strdup(resolved); +} + +char* getcwd(char* buf, size_t size) noexcept { + if (buf != nullptr) { + // Nothing is allocated on this path. + if (size == 0) { + errno = EINVAL; + return nullptr; + } + if (syscall(SYS_getcwd, buf, size) < 0) { + return nullptr; + } + return buf; + } + // GNU extension: allocate the result. A non-zero size is a hard limit; a zero + // size means "however much it takes", so grow until it fits. + size_t capacity = (size != 0) ? size : PATH_MAX; + for (;;) { + char* cwd = static_cast(malloc(capacity)); + if (cwd == nullptr) { + errno = ENOMEM; + return nullptr; + } + if (syscall(SYS_getcwd, cwd, capacity) >= 0) { + return cwd; + } + const int saved_errno = errno; + free(cwd); + if (saved_errno != ERANGE || size != 0) { + errno = saved_errno; + return nullptr; + } + if (capacity > (1u << 20)) { + errno = ENAMETOOLONG; + return nullptr; + } + capacity *= 2; + } +} + +} // extern "C" + +#endif // VMSDK_USE_VALKEY_ALLOC_OVERRIDES diff --git a/vmsdk/src/memory_allocation_overrides.cc b/vmsdk/src/memory_allocation_overrides.cc deleted file mode 100644 index aec5e8f49..000000000 --- a/vmsdk/src/memory_allocation_overrides.cc +++ /dev/null @@ -1,405 +0,0 @@ -/* - * Copyright (c) 2025, valkey-search contributors - * All rights reserved. - * SPDX-License-Identifier: BSD 3-Clause - * - */ - -#include -#include -#include -#include - -#include "absl/base/no_destructor.h" -#include "absl/base/thread_annotations.h" -#include "absl/container/flat_hash_set.h" -#include "absl/hash/hash.h" -#include "absl/synchronization/mutex.h" -#include "vmsdk/src/memory_allocation.h" -#include "vmsdk/src/valkey_module_api/valkey_module.h" - -// clang-format off -// We put this at the end since it will otherwise mangle the malloc symbols in -// the dependencies. -#include "vmsdk/src/memory_allocation_overrides.h" - -namespace vmsdk { - // We use a combination of a thread local static variable and a global atomic -// variable to perform the switch to the new allocator. The global is only -// accessed during the initial loading phase, and once we switch allocators the -// thread local variable is exclusively used. This should guarantee that the -// switch is done atomically while not having performance impact during steady -// state. -thread_local static bool thread_using_valkey_module_alloc = false; -static std::atomic use_valkey_module_alloc_switch = false; - -bool IsUsingValkeyAlloc() { - if (ABSL_PREDICT_FALSE(!thread_using_valkey_module_alloc && - use_valkey_module_alloc_switch.load(std::memory_order_relaxed))) { - thread_using_valkey_module_alloc = true; - return true; - } - return thread_using_valkey_module_alloc; -} - -absl::Mutex switch_allocator_mutex_; -// SystemAllocTracker tracks memory allocations to the system allocator, so that -// subsequent free calls can be redirected to the appropriate allocator. -class SystemAllocTracker { - public: - static SystemAllocTracker& GetInstance() { - static absl::NoDestructor instance; - return *instance; - } - SystemAllocTracker() = default; - SystemAllocTracker(const SystemAllocTracker&) = delete; - SystemAllocTracker& operator=(const SystemAllocTracker&) = delete; - ~SystemAllocTracker() = default; - - void TrackPointer(void* ptr) { - if (ABSL_PREDICT_FALSE(ptr == nullptr)) { - return; - } - absl::MutexLock lock(&mutex_); - tracked_ptrs_.insert(ptr); - } - - bool IsTracked(void* ptr) const { - if (ABSL_PREDICT_TRUE(IsUsingValkeyAlloc() && !tracked_ptrs_snapshot_.contains(ptr))) { - return false; - } - absl::MutexLock lock(&mutex_); - return tracked_ptrs_.contains(ptr); - } - - bool UntrackPointer(void* ptr) { - if (ABSL_PREDICT_TRUE(IsUsingValkeyAlloc() && !tracked_ptrs_snapshot_.contains(ptr))) { - return false; - } - absl::MutexLock lock(&mutex_); - return tracked_ptrs_.erase(ptr); - } - - size_t GetTrackedPointersCnt() const { - absl::MutexLock lock(&mutex_); - return tracked_ptrs_.size(); - } - - void CreateTrackedSnapshot() { - absl::MutexLock lock(&mutex_); - tracked_ptrs_snapshot_ = tracked_ptrs_; - } - // Used for testing - void ClearTrackedAddresses() { - absl::MutexLock lock(&mutex_); - tracked_ptrs_.clear(); - tracked_ptrs_snapshot_.clear(); - } - - private: - - mutable absl::Mutex mutex_; - absl::flat_hash_set, std::equal_to, - RawSystemAllocator> - tracked_ptrs_ ABSL_GUARDED_BY(mutex_); - // `tracked_ptrs_snapshot_` provides a lock-free fast path to check if an address is tracked. - // It is initialized as a read-only snapshot of `tracked_ptrs_` when switching to the - // Valkey allocator. - // - // Notes: - // 1. False positives are possible. Any positive match MUST be verified against the - // `tracked_ptrs_` address tracker. - // 2. Tests indicate this snapshot typically tracks ~1K addresses. - absl::flat_hash_set, std::equal_to, - RawSystemAllocator> - tracked_ptrs_snapshot_; -}; - -void* PerformAndTrackMalloc(size_t size, void* (*malloc_fn)(size_t), - size_t (*malloc_size_fn)(void*)) { - void* ptr = malloc_fn(size); - if (ABSL_PREDICT_TRUE(ptr != nullptr)) { - ReportAllocMemorySize(malloc_size_fn(ptr)); - } - return ptr; -} -void* PerformAndTrackCalloc(size_t n, size_t size, - void* (*calloc_fn)(size_t, size_t), - size_t (*malloc_size_fn)(void*)) { - void* ptr = calloc_fn(n, size); - if (ABSL_PREDICT_TRUE(ptr != nullptr)) { - ReportAllocMemorySize(malloc_size_fn(ptr)); - } - return ptr; -} -void PerformAndTrackFree(void* ptr, void (*free_fn)(void*), - size_t (*malloc_size_fn)(void*)) { - ReportFreeMemorySize(malloc_size_fn(ptr)); - free_fn(ptr); -} -void* PerformAndTrackRealloc(void* ptr, size_t size, - void* (*realloc_fn)(void*, size_t), - size_t (*malloc_size_fn)(void*)) { - size_t old_size = 0; - if (ABSL_PREDICT_TRUE(ptr != nullptr)) { - old_size = malloc_size_fn(ptr); - } - void* new_ptr = realloc_fn(ptr, size); - if (ABSL_PREDICT_TRUE(new_ptr != nullptr)) { - if (ABSL_PREDICT_TRUE(ptr != nullptr)) { - ReportFreeMemorySize(old_size); - } - ReportAllocMemorySize(malloc_size_fn(new_ptr)); - } - return new_ptr; -} -void* PerformAndTrackAlignedAlloc(size_t align, size_t size, - void*(aligned_alloc_fn)(size_t, size_t), - size_t (*malloc_size_fn)(void*)) { - void* ptr = aligned_alloc_fn(align, size); - if (ABSL_PREDICT_TRUE(ptr != nullptr)) { - ReportAllocMemorySize(malloc_size_fn(ptr)); - } - return ptr; -} - -void UseValkeyAlloc() { - absl::WriterMutexLock switch_allocator_lock(&switch_allocator_mutex_); - SystemAllocTracker::GetInstance().CreateTrackedSnapshot(); - use_valkey_module_alloc_switch.store(true, std::memory_order_relaxed); -} - -void ResetValkeyAlloc() { - absl::WriterMutexLock switch_allocator_lock(&switch_allocator_mutex_); - use_valkey_module_alloc_switch.store(false, std::memory_order_relaxed); - thread_using_valkey_module_alloc = false; - SystemAllocTracker::GetInstance().ClearTrackedAddresses(); - ResetValkeyAllocStats(); -} - -} // namespace vmsdk - -extern "C" { -// Our allocator doesn't support tracking system memory size, so we just -// return 0. -// NOLINTNEXTLINE -__attribute__((weak)) size_t empty_usable_size(void* ptr) noexcept { return 0; } - -// For Valkey allocation - we need to ensure alignment by taking advantage of -// jemalloc alignment properties, as there is no aligned malloc module -// function. -// -// "... Chunks are always aligned to multiples of the chunk size..." -// -// See https://linux.die.net/man/3/jemalloc -size_t AlignSize(size_t size, int alignment = 16) { - return (size + alignment - 1) & ~(alignment - 1); -} - -void* __wrap_malloc(size_t size) noexcept { - if (ABSL_PREDICT_FALSE(!vmsdk::IsUsingValkeyAlloc())) { - absl::ReaderMutexLock switch_allocator_lock(&vmsdk::switch_allocator_mutex_); - if (!vmsdk::IsUsingValkeyAlloc()) { - auto ptr = - vmsdk::PerformAndTrackMalloc(size, __real_malloc, empty_usable_size); - vmsdk::SystemAllocTracker::GetInstance().TrackPointer(ptr); - return ptr; - } - } - // Forcing 16-byte alignment in Valkey, which may otherwise return 8-byte - // aligned memory. - return vmsdk::PerformAndTrackMalloc(AlignSize(size), ValkeyModule_Alloc, - ValkeyModule_MallocUsableSize); -} -void __wrap_free(void* ptr) noexcept { - if (ptr == nullptr) { - return; - } - bool was_tracked = - vmsdk::SystemAllocTracker::GetInstance().UntrackPointer(ptr); - // During bootstrap - there are some cases where memory is still allocated - // outside of our wrapper functions - for example if a library calls into - // another DSO which doesn't have our wrapped symbols (namely libc.so). For - // this reason, we bypass the tracking during the bootstrap phase. - if (was_tracked || !vmsdk::IsUsingValkeyAlloc()) { - vmsdk::PerformAndTrackFree(ptr, __real_free, empty_usable_size); - } else { - vmsdk::PerformAndTrackFree(ptr, ValkeyModule_Free, - ValkeyModule_MallocUsableSize); - } -} -// NOLINTNEXTLINE -void* __wrap_calloc(size_t __nmemb, size_t size) noexcept { - if (ABSL_PREDICT_FALSE(!vmsdk::IsUsingValkeyAlloc())) { - absl::ReaderMutexLock switch_allocator_lock(&vmsdk::switch_allocator_mutex_); - if (!vmsdk::IsUsingValkeyAlloc()) { - auto ptr = vmsdk::PerformAndTrackCalloc(__nmemb, size, __real_calloc, - empty_usable_size); - vmsdk::SystemAllocTracker::GetInstance().TrackPointer(ptr); - return ptr; - } - } - return vmsdk::PerformAndTrackCalloc(__nmemb, AlignSize(size), ValkeyModule_Calloc, - ValkeyModule_MallocUsableSize); -} - -void* __wrap_realloc(void* ptr, size_t size) noexcept { - if (ABSL_PREDICT_FALSE(ptr == nullptr)) { - return __wrap_malloc(size); - } - if (ABSL_PREDICT_FALSE(!vmsdk::IsUsingValkeyAlloc())) { - absl::ReaderMutexLock switch_allocator_lock(&vmsdk::switch_allocator_mutex_); - if (!vmsdk::IsUsingValkeyAlloc()) { - // Bootstrap path: still using system allocator - auto new_ptr = vmsdk::PerformAndTrackRealloc(ptr, size, __real_realloc, - empty_usable_size); - vmsdk::SystemAllocTracker::GetInstance().TrackPointer(new_ptr); - return new_ptr; - } - } - bool was_tracked = - vmsdk::SystemAllocTracker::GetInstance().UntrackPointer(ptr); - - // Fast path: using Valkey allocator and pointer already in Valkey allocator - if (ABSL_PREDICT_TRUE(!was_tracked)) { - return vmsdk::PerformAndTrackRealloc(ptr, AlignSize(size), - ValkeyModule_Realloc, - ValkeyModule_MallocUsableSize); - } - // Migration path: system allocator → Valkey allocator (when was_tracked=true) - - // Step 1: Allocate from Valkey allocator - void* new_ptr = vmsdk::PerformAndTrackMalloc(AlignSize(size), - ValkeyModule_Alloc, - ValkeyModule_MallocUsableSize); - if (ABSL_PREDICT_FALSE(new_ptr == nullptr)) { - // Valkey allocation failed, keep system buffer and restore tracking - vmsdk::SystemAllocTracker::GetInstance().TrackPointer(ptr); - return nullptr; - } - // Bootstrap path: still using system allocator - auto tmp_ptr = vmsdk::PerformAndTrackRealloc(ptr, size, __real_realloc, empty_usable_size); - if (ABSL_PREDICT_FALSE(tmp_ptr == nullptr)) { - // Valkey allocation failed, keep system buffer and restore tracking - vmsdk::SystemAllocTracker::GetInstance().TrackPointer(ptr); - vmsdk::PerformAndTrackFree(new_ptr, ValkeyModule_Free, ValkeyModule_MallocUsableSize); - return nullptr; - } - memcpy(new_ptr, tmp_ptr, size); - vmsdk::PerformAndTrackFree(tmp_ptr, __real_free, empty_usable_size); - return new_ptr; -} -// NOLINTNEXTLINE -void* __wrap_aligned_alloc(size_t __alignment, size_t __size) noexcept { - if (ABSL_PREDICT_FALSE(!vmsdk::IsUsingValkeyAlloc())) { - absl::ReaderMutexLock switch_allocator_lock(&vmsdk::switch_allocator_mutex_); - if (!vmsdk::IsUsingValkeyAlloc()) { - auto ptr = vmsdk::PerformAndTrackAlignedAlloc( - __alignment, __size, __real_aligned_alloc, empty_usable_size); - vmsdk::SystemAllocTracker::GetInstance().TrackPointer(ptr); - return ptr; - } - } - - return vmsdk::PerformAndTrackMalloc(AlignSize(__size, __alignment), - ValkeyModule_Alloc, - ValkeyModule_MallocUsableSize); -} - -int __wrap_malloc_usable_size(void* ptr) noexcept { - if (vmsdk::SystemAllocTracker::GetInstance().IsTracked(ptr)) { - return empty_usable_size(ptr); - } - return ValkeyModule_MallocUsableSize(ptr); -} - -// NOLINTNEXTLINE -int __wrap_posix_memalign(void** r, size_t __alignment, size_t __size) PMES { - *r = __wrap_aligned_alloc(__alignment, __size); - return 0; -} - -void* __wrap_valloc(size_t size) noexcept { - return __wrap_aligned_alloc(sysconf(_SC_PAGESIZE), size); -} - -} // extern "C" - -size_t GetNewAllocSize(size_t size) { - if (size == 0) { - return 1; - } - return size; -} - -#ifndef SAN_BUILD -void* operator new(size_t size) noexcept(false) { - return __wrap_malloc(GetNewAllocSize(size)); -} -void operator delete(void* p) noexcept { __wrap_free(p); } -void operator delete(void* p, size_t size) noexcept { __wrap_free(p); } -void* operator new[](size_t size) noexcept(false) { - // A non-null pointer is expected to be returned even if size = 0. - if (size == 0) { - size++; - } - return __wrap_malloc(size); -} -void operator delete[](void* p) noexcept { __wrap_free(p); } -// NOLINTNEXTLINE -void operator delete[](void* p, size_t size) noexcept { __wrap_free(p); } -// NOLINTNEXTLINE -void* operator new(size_t size, const std::nothrow_t& nt) noexcept { - return __wrap_malloc(GetNewAllocSize(size)); -} -// NOLINTNEXTLINE -void* operator new[](size_t size, const std::nothrow_t& nt) noexcept { - return __wrap_malloc(GetNewAllocSize(size)); -} -void operator delete(void* p, const std::nothrow_t& nt) noexcept { - __wrap_free(p); -} -void operator delete[](void* p, const std::nothrow_t& nt) noexcept { - __wrap_free(p); -} -void* operator new(size_t size, std::align_val_t alignment) noexcept(false) { - return __wrap_aligned_alloc(static_cast(alignment), - GetNewAllocSize(size)); -} -void* operator new(size_t size, std::align_val_t alignment, - const std::nothrow_t&) noexcept { - return __wrap_aligned_alloc(static_cast(alignment), - GetNewAllocSize(size)); -} -void operator delete(void* p, std::align_val_t alignment) noexcept { - __wrap_free(p); -} -void operator delete(void* p, std::align_val_t alignment, - const std::nothrow_t&) noexcept { - __wrap_free(p); -} -void operator delete(void* p, size_t size, - std::align_val_t alignment) noexcept { - __wrap_free(p); -} -void* operator new[](size_t size, std::align_val_t alignment) noexcept(false) { - return __wrap_aligned_alloc(static_cast(alignment), - GetNewAllocSize(size)); -} -void* operator new[](size_t size, std::align_val_t alignment, - const std::nothrow_t&) noexcept { - return __wrap_aligned_alloc(static_cast(alignment), - GetNewAllocSize(size)); -} -void operator delete[](void* p, std::align_val_t alignment) noexcept { - __wrap_free(p); -} -void operator delete[](void* p, std::align_val_t alignment, - const std::nothrow_t&) noexcept { - __wrap_free(p); -} -void operator delete[](void* p, size_t size, - std::align_val_t alignment) noexcept { - __wrap_free(p); -} -#endif // !SAN_BUILD diff --git a/vmsdk/src/memory_allocation_overrides.h b/vmsdk/src/memory_allocation_overrides.h index 49fdac210..636e6f242 100644 --- a/vmsdk/src/memory_allocation_overrides.h +++ b/vmsdk/src/memory_allocation_overrides.h @@ -9,123 +9,95 @@ #define VMSDK_SRC_MEMORY_ALLOCATION_OVERRIDES_H_ #include -#include #include -#include #include #include "vmsdk/src/memory_allocation.h" -#if defined(__clang__) -#define WEAK_SYMBOL __attribute__((weak)) -#else -#define WEAK_SYMBOL +// VMSDK_USE_VALKEY_ALLOC_OVERRIDES is defined when this build routes the +// module's heap through ValkeyModule_Alloc/Free. When it is not defined, the +// allocator definitions in memory_allocation_c_api.cc are compiled out and +// everything runs on the system allocator. +// +// Sanitizer builds opt out so that the sanitizer's own allocator sees every +// allocation -- defining malloc here would fight its interceptors. +// +// macOS opts out as well. It is a build-only target: +// .github/workflows/macos.yml runs build.sh with no tests, and the module is +// never executed there. That matters because the deferral of static +// initializers that makes the Valkey allocator usable from the very start of +// module load (see vmsdk/deferred_init.lds and vmsdk/src/deferred_init.cc) is +// implemented with a GNU linker script, and Mach-O has no equivalent. +// +// IF macOS EVER BECOMES A PRODUCTION TARGET, this problem must be solved for +// that platform before the overrides can be enabled there. The Mach-O analogue +// of the .init_array rename is the __DATA,__mod_init_func section, which would +// need to be renamed at link time (ld64 -rename_section) and walked explicitly +// from ValkeyModule_OnLoad the same way deferred_init.cc does. Simply defining +// VMSDK_USE_VALKEY_ALLOC_OVERRIDES on macOS without that would reintroduce the +// bug this design removes: static initializers allocating from the system +// allocator and later being freed with ValkeyModule_Free. +#if !defined(SAN_BUILD) && !defined(__APPLE__) +#define VMSDK_USE_VALKEY_ALLOC_OVERRIDES 1 #endif +#ifdef VMSDK_USE_VALKEY_ALLOC_OVERRIDES extern "C" { -// NOLINTNEXTLINE -WEAK_SYMBOL void* (*__real_malloc)(size_t) = malloc; -// NOLINTNEXTLINE -WEAK_SYMBOL void (*__real_free)(void*) = free; -// NOLINTNEXTLINE -WEAK_SYMBOL void* (*__real_calloc)(size_t, size_t) = calloc; -// NOLINTNEXTLINE -WEAK_SYMBOL void* (*__real_realloc)(void*, size_t) = realloc; -// NOLINTNEXTLINE -WEAK_SYMBOL void* (*__real_aligned_alloc)(size_t, size_t) = aligned_alloc; -// NOLINTNEXTLINE -WEAK_SYMBOL int (*__real_posix_memalign)(void**, size_t, - size_t) = posix_memalign; -// NOLINTNEXTLINE -WEAK_SYMBOL void* (*__real_valloc)(size_t) = valloc; -// NOLINTNEXTLINE -__attribute__((weak)) size_t empty_usable_size(void* ptr) noexcept; +// glibc's allocator, reached by name so that it is not captured by the module's +// own malloc/free (see memory_allocation_c_api.cc). Used by RawSystemAllocator +// below. +void* __libc_malloc(size_t size); +void __libc_free(void* ptr); } // extern "C" +#endif // VMSDK_USE_VALKEY_ALLOC_OVERRIDES + +namespace vmsdk { -// Different exception specifier between CLANG & GCC -#ifdef __clang__ -#define PMES +// The system allocator, named so that the module's own malloc/free cannot +// capture it. Where the module does not define those (sanitizer builds, macOS) +// the plain names already are the system allocator, and __libc_malloc does not +// exist outside glibc. +inline void* RawSystemMalloc(std::size_t size) { +#ifdef VMSDK_USE_VALKEY_ALLOC_OVERRIDES + return __libc_malloc(size); #else -#define PMES noexcept + return std::malloc(size); #endif +} -extern "C" { -// See https://www.gnu.org/software/libc/manual/html_node/Replacing-malloc.html -// NOLINTNEXTLINE -void* __wrap_malloc(size_t size) noexcept; -// NOLINTNEXTLINE -void __wrap_free(void* ptr) noexcept; -// NOLINTNEXTLINE -void* __wrap_calloc(size_t __nmemb, size_t size) noexcept; -// NOLINTNEXTLINE -void* __wrap_realloc(void* ptr, size_t size) noexcept; -// NOLINTNEXTLINE -void* __wrap_aligned_alloc(size_t __alignment, size_t __size) noexcept; -// NOLINTNEXTLINE -int __wrap_malloc_usable_size(void* ptr) noexcept; -// NOLINTNEXTLINE -int __wrap_posix_memalign(void** r, size_t __alignment, size_t __size) PMES; -// NOLINTNEXTLINE -void* __wrap_valloc(size_t size) noexcept; -} // extern "C" - -#ifndef SAN_BUILD -// NOLINTNEXTLINE -#define malloc(...) __wrap_malloc(__VA_ARGS__) -// NOLINTNEXTLINE -#define calloc(...) __wrap_calloc(__VA_ARGS__) -// NOLINTNEXTLINE -#define realloc(...) __wrap_realloc(__VA_ARGS__) -// NOLINTNEXTLINE -#define free(...) __wrap_free(__VA_ARGS__) -// NOLINTNEXTLINE -#define aligned_alloc(...) __wrap_aligned_alloc(__VA_ARGS__) -// NOLINTNEXTLINE -#define posix_memalign(...) __wrap_posix_memalign(__VA_ARGS__) -// NOLINTNEXTLINE -#define valloc(...) __wrap_valloc(__VA_ARGS__) +inline void RawSystemFree(void* ptr) { +#ifdef VMSDK_USE_VALKEY_ALLOC_OVERRIDES + __libc_free(ptr); +#else + std::free(ptr); +#endif +} -void* operator new(size_t size) noexcept(false); -void operator delete(void* p) noexcept; -void operator delete(void* p, size_t size) noexcept; -void* operator new[](size_t size) noexcept(false); -void operator delete[](void* p) noexcept; -void operator delete[](void* p, size_t size) noexcept; -void* operator new(size_t size, const std::nothrow_t& nt) noexcept; -void* operator new[](size_t size, const std::nothrow_t& nt) noexcept; -void operator delete(void* p, const std::nothrow_t& nt) noexcept; -void operator delete[](void* p, const std::nothrow_t& nt) noexcept; -void* operator new(size_t size, std::align_val_t alignment) noexcept(false); -void* operator new(size_t size, std::align_val_t alignment, - const std::nothrow_t&) noexcept; -void operator delete(void* p, std::align_val_t alignment) noexcept; -void operator delete(void* p, std::align_val_t alignment, - const std::nothrow_t&) noexcept; -void operator delete(void* p, size_t size, std::align_val_t alignment) noexcept; -void* operator new[](size_t size, std::align_val_t alignment) noexcept(false); -void* operator new[](size_t size, std::align_val_t alignment, - const std::nothrow_t&) noexcept; -void operator delete[](void* p, std::align_val_t alignment) noexcept; -void operator delete[](void* p, std::align_val_t alignment, - const std::nothrow_t&) noexcept; -void operator delete[](void* p, size_t size, - std::align_val_t alignment) noexcept; -#endif // !SAN_BUILD +} // namespace vmsdk namespace vmsdk { -// Updates the custom allocator to perform any future allocations using the -// Valkey allocator. -void UseValkeyAlloc(); - -// Switch back to the default allocator. No guarantees around atomicity. Only -// safe in single-threaded or testing environments. -void ResetValkeyAlloc(); struct DisableRawSystemAllocatorReporting { }; // Pass this (or void) to DISABLE reporting -// RawSystemAllocator implements an allocator that will not go through -// the SystemAllocTracker, for use by the SystemAllocTracker to prevent -// infinite recursion when tracking pointers. + +// RawSystemAllocator allocates straight from glibc, bypassing both the Valkey +// allocator and the memory accounting. +// +// This is not an optimization and it cannot be replaced with std::allocator. +// The accounting counters are themselves ShardedAtomics, so +// ReportAllocMemorySize -> ShardedAtomic::Add allocates: it constructs a +// thread_local ThreadLocalNode, whose constructor registers it in a vector, and +// it grows that node's value array under resize_mutex. Route those allocations +// through the module allocator and each one calls ReportAllocMemorySize again, +// re-entering either a thread_local's own initialization or a non-reentrant +// absl::Mutex. Tried it: the module hangs on a futex during load, accumulating +// no CPU time, before the server ever accepts connections. +// +// Allocating from Valkey but skipping the accounting would break the cycle too, +// but ShardedAtomic is also linked into the unit test executables, where +// ValkeyModule_Alloc is a mock that is unset until a fixture installs it. Going +// straight to glibc is what keeps this allocator independent of everything it +// underpins. template struct RawSystemAllocator { // NOLINTNEXTLINE @@ -139,14 +111,14 @@ struct RawSystemAllocator { if constexpr (!std::is_same_v) { ReportAllocMemorySize(n * sizeof(T)); } - return static_cast(__real_malloc(n * sizeof(T))); + return static_cast(RawSystemMalloc(n * sizeof(T))); } // NOLINTNEXTLINE void deallocate(T* p, std::size_t) { if constexpr (!std::is_same_v) { ReportFreeMemorySize(sizeof(T)); } - __real_free(p); + RawSystemFree(p); } }; diff --git a/vmsdk/src/module.cc b/vmsdk/src/module.cc index e038f612e..7a3233f41 100644 --- a/vmsdk/src/module.cc +++ b/vmsdk/src/module.cc @@ -7,6 +7,7 @@ #include "vmsdk/src/module.h" +#include #include #include #include @@ -16,6 +17,7 @@ #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/string_view.h" +#include "vmsdk/src/deferred_init.h" #include "vmsdk/src/log.h" #include "vmsdk/src/managed_pointers.h" #include "vmsdk/src/memory_allocation_overrides.h" @@ -70,19 +72,31 @@ absl::Status RegisterCommands(ValkeyModuleCtx *ctx, } int OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc, - const Options &options) { - if (ValkeyModule_Init(ctx, options.name.c_str(), options.version, - VALKEYMODULE_APIVER_1) == VALKEYMODULE_ERR) { - ValkeyModule_Log(ctx, VALKEYMODULE_LOGLEVEL_WARNING, - "Failed to init module"); - return VALKEYMODULE_ERR; - } + const Options &options, const char *module_name, + vmsdk::ValkeyVersion module_version) { + // ValkeyModule_Init already ran, from the VALKEY_MODULE macro, before the + // deferred static initializers -- it is what establishes ValkeyModule_Alloc. auto status = vmsdk::InitLogging(ctx); if (!status.ok()) { ValkeyModule_Log(ctx, VALKEYMODULE_LOGLEVEL_WARNING, "Failed to init logging, %s", status.message().data()); return VALKEYMODULE_ERR; } + // `options` was still zero-initialized when ValkeyModule_Init was given + // module_name/module_version. Now that it is constructed, confirm the module + // was registered under the name and version it describes itself with. + if (options.name == nullptr || std::strcmp(options.name, module_name) != 0 || + options.version != module_version) { + VMSDK_LOG(WARNING, ctx) + << "Module registered as '" << module_name << "' v" << module_version + << " but describes itself as '" + << (options.name == nullptr ? "(null)" : options.name) << "' v" + << options.version + << ". The VALKEY_MODULE arguments must match the Options fields."; + return VALKEYMODULE_ERR; + } + VMSDK_LOG(NOTICE, ctx) << "Ran " << vmsdk::GetDeferredInitializerCount() + << " deferred static initializers"; if (ValkeyModule_GetServerVersion == nullptr) { VMSDK_LOG(WARNING, ctx) << "ValkeyModule_GetServerVersion function is not available"; @@ -147,7 +161,8 @@ int OnLoadDone(absl::Status status, ValkeyModuleCtx *ctx, if (status.ok()) { VMSDK_LOG(NOTICE, ctx) << options.name << " module was successfully loaded!"; - vmsdk::UseValkeyAlloc(); + // The switch to the Valkey allocator happened at the top of + // ValkeyModule_OnLoad, before static initialization -- see VALKEY_MODULE. return VALKEYMODULE_OK; } VMSDK_LOG(WARNING, ctx) << status.message().data(); diff --git a/vmsdk/src/module.h b/vmsdk/src/module.h index 3e40204d7..0a9fd79df 100644 --- a/vmsdk/src/module.h +++ b/vmsdk/src/module.h @@ -15,20 +15,45 @@ #include "absl/functional/any_invocable.h" #include "absl/status/status.h" #include "absl/strings/string_view.h" +#include "vmsdk/src/deferred_init.h" #include "vmsdk/src/utils.h" // IWYU pragma: keep #include "vmsdk/src/valkey_module_api/valkey_module.h" -#define VALKEY_MODULE(options) \ +// Defines the module entry points. +// +// `module_name` and `module_version` must be constant-initialized (a constexpr +// name array and version), NOT members of `options`. They are read before the +// module's static initializers have run, at which point `options` -- which +// holds std::list and absl::AnyInvocable members and therefore requires dynamic +// initialization -- is still entirely zero. GCC happens to emit the +// constant-computable members of such an object statically, but Clang does not; +// reading options.name there yields nullptr. They must match the corresponding +// `options` fields; vmsdk::module::OnLoad checks this once initialization is +// complete. +// +// ValkeyModule_Init has to come first because it is what establishes +// ValkeyModule_Alloc/Free, and RunDeferredStaticInitializers must allocate +// through them. There is no fallback allocator: until ValkeyModule_Init runs, +// ValkeyModule_Alloc is null and any allocation faults on the spot. See +// vmsdk/src/deferred_init.cc. +#define VALKEY_MODULE(options, module_name, module_version) \ namespace { \ extern "C" { \ int ValkeyModule_OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, \ int argc) { \ + if (ValkeyModule_Init(ctx, module_name, module_version, \ + VALKEYMODULE_APIVER_1) == VALKEYMODULE_ERR) { \ + return VALKEYMODULE_ERR; \ + } \ + vmsdk::RunDeferredStaticInitializers(); \ + /* Dynamically-initialized globals are usable from here on. */ \ if (!vmsdk::verifyLoadedOnlyOnce()) { \ VMSDK_LOG(NOTICE, ctx) << "Module cannot be loaded more than once"; \ return VALKEYMODULE_ERR; \ } \ vmsdk::TrackCurrentAsMainThread(); \ - if (auto status = vmsdk::module::OnLoad(ctx, argv, argc, options); \ + if (auto status = vmsdk::module::OnLoad(ctx, argv, argc, options, \ + module_name, module_version); \ status != VALKEYMODULE_OK) { \ return status; \ } \ @@ -72,7 +97,8 @@ struct CommandOptions { }; struct Options { - std::string name; + // Points at the same constexpr string passed to VALKEY_MODULE. + const char *name{nullptr}; std::list acl_categories; vmsdk::ValkeyVersion version; vmsdk::ValkeyVersion minimum_valkey_server_version; @@ -88,7 +114,8 @@ struct Options { }; int OnLoad(ValkeyModuleCtx *ctx, ValkeyModuleString **argv, int argc, - const Options &options); + const Options &options, const char *module_name, + vmsdk::ValkeyVersion module_version); int OnLoadDone(absl::Status status, ValkeyModuleCtx *ctx, const Options &options); absl::Status RegisterInfo(ValkeyModuleCtx *ctx, ValkeyModuleInfoFunc info); diff --git a/vmsdk/testing/CMakeLists.txt b/vmsdk/testing/CMakeLists.txt index 60bfc4799..301b9cfef 100644 --- a/vmsdk/testing/CMakeLists.txt +++ b/vmsdk/testing/CMakeLists.txt @@ -54,14 +54,6 @@ target_include_directories(status_macros_test PUBLIC ${CMAKE_CURRENT_LIST_DIR}) target_link_libraries(status_macros_test PUBLIC vmsdklib) finalize_test_flags(status_macros_test) -set(SRCS_MEMORY_ALLOCATION_TEST - ${CMAKE_CURRENT_LIST_DIR}/memory_allocation_test.cc) -add_executable(memory_allocation_test ${SRCS_MEMORY_ALLOCATION_TEST}) -target_include_directories(memory_allocation_test - PUBLIC ${CMAKE_CURRENT_LIST_DIR}) -target_link_libraries(memory_allocation_test PUBLIC vmsdklib) -finalize_test_flags(memory_allocation_test) - set(SRCS_UTILS_TEST ${CMAKE_CURRENT_LIST_DIR}/utils_test.cc) add_executable(utils_test ${SRCS_UTILS_TEST}) target_include_directories(utils_test PUBLIC ${CMAKE_CURRENT_LIST_DIR}) diff --git a/vmsdk/testing/memory_allocation_test.cc b/vmsdk/testing/memory_allocation_test.cc deleted file mode 100644 index 17856420e..000000000 --- a/vmsdk/testing/memory_allocation_test.cc +++ /dev/null @@ -1,696 +0,0 @@ -/* - * Copyright (c) 2025, valkey-search contributors - * All rights reserved. - * SPDX-License-Identifier: BSD 3-Clause - * - */ - -#include -#include -#include - -#include "gmock/gmock.h" -#include "gtest/gtest.h" -#include "vmsdk/src/memory_allocation_overrides.h" -#include "vmsdk/src/memory_tracker.h" -#include "vmsdk/src/testing_infra/module.h" -#include "vmsdk/src/testing_infra/utils.h" - -class MockSystemAlloc { - public: - // Prefixed with _ to avoid name collision with system functions. - MOCK_METHOD(void*, _malloc, (size_t size), (noexcept)); - MOCK_METHOD(void, _free, (void* ptr), (noexcept)); - MOCK_METHOD(void*, _calloc, (size_t nmemb, size_t size), (noexcept)); - MOCK_METHOD(void*, _realloc, (void* ptr, size_t size), (noexcept)); - MOCK_METHOD(void*, _aligned_alloc, (size_t alignment, size_t size), - (noexcept)); - MOCK_METHOD(size_t, _malloc_usable_size, (void* ptr), (noexcept)); - MOCK_METHOD(void*, _memalign, (size_t alignment, size_t size), (noexcept)); - MOCK_METHOD(int, _posix_memalign, (void** r, size_t alignment, size_t size), - (noexcept)); - MOCK_METHOD(void*, _pvalloc, (size_t size), (noexcept)); - MOCK_METHOD(void*, _valloc, (size_t size), (noexcept)); - MOCK_METHOD(void, _cfree, (void* ptr), (noexcept)); -}; - -MockSystemAlloc* kMockSystemAlloc; - -namespace vmsdk { - -namespace { -#ifndef TESTING_TMP_DISABLED -class MemoryAllocationTest : public ValkeyTest { - protected: - void SetUp() override { - ValkeyTest::SetUp(); - kMockSystemAlloc = new MockSystemAlloc(); - SetRealAllocators( - [](size_t size) { return kMockSystemAlloc->_malloc(size); }, - [](void* ptr) { kMockSystemAlloc->_free(ptr); }, - [](size_t nmemb, size_t size) { - return kMockSystemAlloc->_calloc(nmemb, size); - }, - [](void* ptr, size_t size) { - return kMockSystemAlloc->_realloc(ptr, size); - }, - [](size_t alignment, size_t size) { - return kMockSystemAlloc->_aligned_alloc(alignment, size); - }, - [](void** r, size_t alignment, size_t size) { - return kMockSystemAlloc->_posix_memalign(r, alignment, size); - }, - [](size_t size) { return kMockSystemAlloc->_valloc(size); }); - vmsdk::ResetValkeyAlloc(); - } - void TearDown() override { - SetRealAllocators(malloc, free, calloc, realloc, aligned_alloc, - posix_memalign, valloc); - ValkeyTest::TearDown(); - delete kMockSystemAlloc; - vmsdk::ResetValkeyAlloc(); - } -}; - -TEST_F(MemoryAllocationTest, SystemAllocIsDefault) { - size_t size = 10; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _malloc(size)) - .WillOnce(testing::Return(test_ptr)); - EXPECT_CALL(*kMockValkeyModule, Alloc(size)).Times(0); - void* ptr = __wrap_malloc(size); - EXPECT_EQ(ptr, test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - - EXPECT_CALL(*kMockSystemAlloc, _free(test_ptr)).Times(1); - __wrap_free(test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, SystemCallocIsDefault) { - size_t size = 10; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _calloc(size, sizeof(int))) - .WillOnce(testing::Return(test_ptr)); - EXPECT_CALL(*kMockValkeyModule, Calloc(size, sizeof(int))).Times(0); - void* ptr = __wrap_calloc(size, sizeof(int)); - EXPECT_EQ(ptr, test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - - EXPECT_CALL(*kMockSystemAlloc, _free(test_ptr)).Times(1); - __wrap_free(test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, SystemAlignedAllocIsDefault) { - size_t size = 10; - size_t align = 1024; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _aligned_alloc(align, size)) - .WillOnce(testing::Return(test_ptr)); - EXPECT_CALL(*kMockValkeyModule, Alloc(align)).Times(0); - void* ptr = __wrap_aligned_alloc(align, size); - EXPECT_EQ(ptr, test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - - EXPECT_CALL(*kMockSystemAlloc, _free(test_ptr)).Times(1); - __wrap_free(test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, MallocUsableSize) { - vmsdk::UseValkeyAlloc(); - size_t valkey_size = 20; - void* valkey_test_ptr = reinterpret_cast(0xBADF00D1); - EXPECT_CALL(*kMockValkeyModule, Alloc(valkey_size)) - .WillOnce(testing::Return(valkey_test_ptr)); - EXPECT_CALL(*kMockValkeyModule, MallocUsableSize(valkey_test_ptr)) - .Times(3) - .WillRepeatedly(testing::Return(valkey_size)); - - void* valkey_ptr = __wrap_malloc(valkey_size); - EXPECT_EQ(valkey_ptr, valkey_test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), valkey_size); - EXPECT_EQ(__wrap_malloc_usable_size(valkey_ptr), valkey_size); - - EXPECT_CALL(*kMockValkeyModule, Free(valkey_test_ptr)).Times(1); - __wrap_free(valkey_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, SwitchToValkeyAlloc) { - vmsdk::UseValkeyAlloc(); - - size_t size = 10; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _malloc(size)).Times(0); - EXPECT_CALL(*kMockValkeyModule, Alloc(size)) - .WillOnce(testing::Return(test_ptr)); - EXPECT_CALL(*kMockValkeyModule, MallocUsableSize(test_ptr)) - .Times(2) - .WillRepeatedly(testing::Return(size)); - - void* ptr = __wrap_malloc(10); - EXPECT_EQ(ptr, test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), size); - - EXPECT_CALL(*kMockValkeyModule, Free(test_ptr)).Times(1); - __wrap_free(test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, SwitchToValkeyCalloc) { - size_t size = 10; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _calloc(size, sizeof(int))).Times(0); - EXPECT_CALL(*kMockValkeyModule, Calloc(size, sizeof(int))) - .WillOnce(testing::Return(test_ptr)); - EXPECT_CALL(*kMockValkeyModule, MallocUsableSize(test_ptr)) - .Times(2) - .WillRepeatedly(testing::Return(size * sizeof(int))); - - vmsdk::UseValkeyAlloc(); - void* ptr = __wrap_calloc(size, sizeof(int)); - EXPECT_EQ(ptr, test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), size * sizeof(int)); - - EXPECT_CALL(*kMockValkeyModule, Free(test_ptr)).Times(1); - __wrap_free(test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, SwitchToValkeyAlignedAlloc) { - size_t size = 10; - size_t align = 1024; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _aligned_alloc(align, size)).Times(0); - EXPECT_CALL(*kMockValkeyModule, Alloc(align)) - .WillOnce(testing::Return(test_ptr)); - EXPECT_CALL(*kMockValkeyModule, MallocUsableSize(test_ptr)) - .Times(2) - .WillRepeatedly(testing::Return(align)); - - vmsdk::UseValkeyAlloc(); - void* ptr = __wrap_aligned_alloc(align, size); - EXPECT_EQ(ptr, test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), align); - - EXPECT_CALL(*kMockValkeyModule, Free(test_ptr)).Times(1); - __wrap_free(test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, FreeSystemAllocAfterSwitching) { - size_t size = 10; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _malloc(size)) - .WillOnce(testing::Return(test_ptr)); - void* ptr = __wrap_malloc(10); - EXPECT_EQ(ptr, test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - - vmsdk::UseValkeyAlloc(); - EXPECT_CALL(*kMockSystemAlloc, _free(test_ptr)).Times(1); - __wrap_free(test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, SystemFreeNullptr) { - EXPECT_CALL(*kMockSystemAlloc, _malloc_usable_size(testing::_)).Times(0); - EXPECT_CALL(*kMockSystemAlloc, _free(testing::_)).Times(0); - __wrap_free(nullptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, ValkeyFreeNullptr) { - vmsdk::UseValkeyAlloc(); - EXPECT_CALL(*kMockValkeyModule, MallocUsableSize(testing::_)).Times(0); - EXPECT_CALL(*kMockValkeyModule, Free(testing::_)).Times(0); - __wrap_free(nullptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, SystemAllocReturnsNullptr) { - size_t size = 10; - EXPECT_CALL(*kMockSystemAlloc, _malloc(size)) - .WillOnce(testing::Return(nullptr)); - EXPECT_CALL(*kMockSystemAlloc, _malloc_usable_size(testing::_)).Times(0); - void* ptr = __wrap_malloc(size); - EXPECT_EQ(ptr, nullptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, ValkeyAllocReturnsNullptr) { - size_t size = 10; - vmsdk::UseValkeyAlloc(); - EXPECT_CALL(*kMockValkeyModule, Alloc(size)) - .WillOnce(testing::Return(nullptr)); - EXPECT_CALL(*kMockValkeyModule, MallocUsableSize(testing::_)).Times(0); - void* ptr = __wrap_malloc(size); - EXPECT_EQ(ptr, nullptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} - -TEST_F(MemoryAllocationTest, SystemReallocBasic) { - size_t initial_size = 10; - size_t realloc_size = 20; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _malloc(initial_size)) - .WillOnce(testing::Return(test_ptr)); - - void* test_ptr_2 = reinterpret_cast(0xBADF00D1); - EXPECT_CALL(*kMockSystemAlloc, _realloc(test_ptr, realloc_size)) - .WillOnce(testing::Return(test_ptr_2)); - - void* ptr = __wrap_malloc(initial_size); - EXPECT_EQ(ptr, test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - void* ptr_2 = __wrap_realloc(ptr, realloc_size); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(ptr_2, test_ptr_2); - - EXPECT_CALL(*kMockSystemAlloc, _free(ptr_2)).Times(1); - __wrap_free(ptr_2); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} -TEST_F(MemoryAllocationTest, SystemReallocNullptr) { - size_t realloc_size = 20; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _realloc(nullptr, realloc_size)) - .WillOnce(testing::Return(test_ptr)); - - void* ptr = __wrap_realloc(nullptr, realloc_size); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(ptr, test_ptr); - - EXPECT_CALL(*kMockSystemAlloc, _free(test_ptr)).Times(1); - __wrap_free(ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} -TEST_F(MemoryAllocationTest, SystemReallocAfterSwitch) { - size_t initial_size = 10; - size_t realloc_size = 20; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _malloc(initial_size)) - .WillOnce(testing::Return(test_ptr)); - - void* test_ptr_2 = reinterpret_cast(0xBADF00D1); - EXPECT_CALL(*kMockSystemAlloc, _realloc(test_ptr, realloc_size)) - .WillOnce(testing::Return(test_ptr_2)); - - void* ptr = __wrap_malloc(initial_size); - EXPECT_EQ(ptr, test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - - vmsdk::UseValkeyAlloc(); - - void* ptr_2 = __wrap_realloc(ptr, realloc_size); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(ptr_2, test_ptr_2); - - EXPECT_CALL(*kMockSystemAlloc, _free(ptr_2)).Times(1); - __wrap_free(ptr_2); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} -TEST_F(MemoryAllocationTest, ValkeyReallocBasic) { - size_t initial_size = 10; - size_t realloc_size = 20; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockValkeyModule, Alloc(initial_size)) - .WillOnce(testing::Return(test_ptr)); - EXPECT_CALL(*kMockValkeyModule, MallocUsableSize(test_ptr)) - .Times(2) - .WillRepeatedly(testing::Return(initial_size)); - - void* test_ptr_2 = reinterpret_cast(0xBADF00D1); - EXPECT_CALL(*kMockValkeyModule, Realloc(test_ptr, realloc_size)) - .WillOnce(testing::Return(test_ptr_2)); - EXPECT_CALL(*kMockValkeyModule, MallocUsableSize(test_ptr_2)) - .Times(2) - .WillRepeatedly(testing::Return(realloc_size)); - - vmsdk::UseValkeyAlloc(); - - void* ptr = __wrap_malloc(initial_size); - EXPECT_EQ(ptr, test_ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), initial_size); - - void* ptr_2 = __wrap_realloc(ptr, realloc_size); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), realloc_size); - EXPECT_EQ(ptr_2, test_ptr_2); - - EXPECT_CALL(*kMockValkeyModule, Free(test_ptr_2)).Times(1); - __wrap_free(ptr_2); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} -TEST_F(MemoryAllocationTest, ValkeyReallocNullptr) { - size_t realloc_size = 20; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockValkeyModule, MallocUsableSize(test_ptr)) - .Times(2) - .WillRepeatedly(testing::Return(realloc_size)); - EXPECT_CALL(*kMockValkeyModule, Realloc(nullptr, realloc_size)) - .WillOnce(testing::Return(test_ptr)); - - vmsdk::UseValkeyAlloc(); - void* ptr = __wrap_realloc(nullptr, realloc_size); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), realloc_size); - EXPECT_EQ(ptr, test_ptr); - - EXPECT_CALL(*kMockValkeyModule, Free(ptr)).Times(1); - __wrap_free(ptr); - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); -} -TEST_F(MemoryAllocationTest, SystemFreeUntracksPointer) { - size_t size = 10; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _malloc(testing::_)) - .WillOnce(testing::Return(test_ptr)); - __wrap_malloc(size); - EXPECT_CALL(*kMockSystemAlloc, _free(testing::_)).Times(1); - __wrap_free(test_ptr); - - vmsdk::UseValkeyAlloc(); - - EXPECT_CALL(*kMockValkeyModule, Alloc(testing::_)) - .WillOnce(testing::Return(test_ptr)); - __wrap_malloc(size); - EXPECT_CALL(*kMockValkeyModule, Free(testing::_)).Times(1); - __wrap_free(test_ptr); -} -TEST_F(MemoryAllocationTest, SystemFreeDefaultsDuringBootstrap) { - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _free(testing::_)).Times(1); - __wrap_free(test_ptr); -} -TEST_F(MemoryAllocationTest, PosixMemalignOverride) { - size_t size = 10; - size_t align = 1024; - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _aligned_alloc(align, size)) - .WillOnce(testing::Return(test_ptr)); - void* out_ptr; - __wrap_posix_memalign(&out_ptr, align, size); - EXPECT_EQ(out_ptr, test_ptr); - __wrap_free(test_ptr); -} -TEST_F(MemoryAllocationTest, VallocOverride) { - size_t size = 10; - size_t page_size = sysconf(_SC_PAGESIZE); - void* test_ptr = reinterpret_cast(0xBAADF00D); - EXPECT_CALL(*kMockSystemAlloc, _aligned_alloc(page_size, size)) - .WillOnce(testing::Return(test_ptr)); - EXPECT_EQ(__wrap_valloc(size), test_ptr); - __wrap_free(test_ptr); -} - -TEST_F(MemoryAllocationTest, IsolatedMemoryScopeAllocationIsolation) { - vmsdk::UseValkeyAlloc(); - - MemoryPool outer_pool{0}; - MemoryPool inner_pool{0}; - - void* outer_ptr = nullptr; - void* inner_ptr = nullptr; - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - - { - IsolatedMemoryScope outer_scope{outer_pool}; - - EXPECT_CALL(*kMockRedisModule, Alloc(112)) - .WillOnce(testing::Return(reinterpret_cast(0x1000))); - EXPECT_CALL(*kMockRedisModule, - MallocUsableSize(reinterpret_cast(0x1000))) - .WillRepeatedly(testing::Return(128)); - outer_ptr = __wrap_malloc(100); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 128); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 128); - EXPECT_EQ(outer_pool.GetUsage(), 0); - - { - IsolatedMemoryScope inner_scope{inner_pool}; - - EXPECT_CALL(*kMockRedisModule, Alloc(80)) - .WillOnce(testing::Return(reinterpret_cast(0x2000))); - EXPECT_CALL(*kMockRedisModule, - MallocUsableSize(reinterpret_cast(0x2000))) - .WillRepeatedly(testing::Return(96)); - inner_ptr = __wrap_malloc(75); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 224); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 224); - EXPECT_EQ(outer_pool.GetUsage(), 0); - EXPECT_EQ(inner_pool.GetUsage(), 0); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 224); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 128); - EXPECT_EQ(outer_pool.GetUsage(), 0); - EXPECT_EQ(inner_pool.GetUsage(), 96); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 224); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - EXPECT_EQ(outer_pool.GetUsage(), 128); - EXPECT_EQ(inner_pool.GetUsage(), 96); - - __wrap_free(outer_ptr); - __wrap_free(inner_ptr); -} - -TEST_F(MemoryAllocationTest, IsolatedMemoryScopeFreeIsolation) { - vmsdk::UseValkeyAlloc(); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - - MemoryPool outer_pool{0}; - MemoryPool inner_pool{0}; - void* outer_ptr = nullptr; - void* inner_ptr = nullptr; - - // Allocate outer pool. - { - IsolatedMemoryScope scope{outer_pool}; - - EXPECT_CALL(*kMockRedisModule, Alloc(112)) - .WillOnce(testing::Return(reinterpret_cast(0x1000))); - EXPECT_CALL(*kMockRedisModule, - MallocUsableSize(reinterpret_cast(0x1000))) - .WillRepeatedly(testing::Return(128)); - outer_ptr = __wrap_malloc(100); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 128); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 128); - EXPECT_EQ(outer_pool.GetUsage(), 0); - EXPECT_EQ(inner_pool.GetUsage(), 0); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 128); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - EXPECT_EQ(outer_pool.GetUsage(), 128); - EXPECT_EQ(inner_pool.GetUsage(), 0); - - // Allocate inner pool. - { - IsolatedMemoryScope scope{inner_pool}; - - EXPECT_CALL(*kMockRedisModule, Alloc(80)) - .WillOnce(testing::Return(reinterpret_cast(0x2000))); - EXPECT_CALL(*kMockRedisModule, - MallocUsableSize(reinterpret_cast(0x2000))) - .WillRepeatedly(testing::Return(96)); - inner_ptr = __wrap_malloc(75); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 224); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 96); - EXPECT_EQ(outer_pool.GetUsage(), 128); - EXPECT_EQ(inner_pool.GetUsage(), 0); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 224); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - EXPECT_EQ(outer_pool.GetUsage(), 128); - EXPECT_EQ(inner_pool.GetUsage(), 96); - - { - IsolatedMemoryScope outer_scope{outer_pool}; - - { - IsolatedMemoryScope inner_scope{inner_pool}; - - EXPECT_CALL(*kMockRedisModule, Free(reinterpret_cast(0x2000))) - .Times(1); - __wrap_free(inner_ptr); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 128); - EXPECT_EQ(vmsdk::GetMemoryDelta(), -96); - EXPECT_EQ(outer_pool.GetUsage(), 128); - EXPECT_EQ(inner_pool.GetUsage(), 0); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 128); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - EXPECT_EQ(outer_pool.GetUsage(), 128); - EXPECT_EQ(inner_pool.GetUsage(), 0); - - EXPECT_CALL(*kMockRedisModule, Free(reinterpret_cast(0x1000))) - .Times(1); - __wrap_free(outer_ptr); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(vmsdk::GetMemoryDelta(), -128); - EXPECT_EQ(outer_pool.GetUsage(), 128); - EXPECT_EQ(inner_pool.GetUsage(), 0); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - EXPECT_EQ(outer_pool.GetUsage(), 0); - EXPECT_EQ(inner_pool.GetUsage(), 0); -} - -TEST_F(MemoryAllocationTest, NestedMemoryScopeAllocation) { - vmsdk::UseValkeyAlloc(); - - MemoryPool outer_pool{0}; - MemoryPool inner_pool{0}; - - void* outer_ptr = nullptr; - void* inner_ptr = nullptr; - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - - { - NestedMemoryScope outer_scope{outer_pool}; - - EXPECT_CALL(*kMockRedisModule, Alloc(112)) - .WillOnce(testing::Return(reinterpret_cast(0x1000))); - EXPECT_CALL(*kMockRedisModule, - MallocUsableSize(reinterpret_cast(0x1000))) - .WillRepeatedly(testing::Return(128)); - outer_ptr = __wrap_malloc(100); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 128); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 128); - EXPECT_EQ(outer_pool.GetUsage(), 0); - - { - NestedMemoryScope inner_scope{inner_pool}; - - EXPECT_CALL(*kMockRedisModule, Alloc(80)) - .WillOnce(testing::Return(reinterpret_cast(0x2000))); - EXPECT_CALL(*kMockRedisModule, - MallocUsableSize(reinterpret_cast(0x2000))) - .WillRepeatedly(testing::Return(96)); - inner_ptr = __wrap_malloc(75); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 224); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 224); - EXPECT_EQ(outer_pool.GetUsage(), 0); - EXPECT_EQ(inner_pool.GetUsage(), 0); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 224); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 224); - EXPECT_EQ(outer_pool.GetUsage(), 0); - EXPECT_EQ(inner_pool.GetUsage(), 96); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 224); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 224); - EXPECT_EQ(outer_pool.GetUsage(), 224); - EXPECT_EQ(inner_pool.GetUsage(), 96); - - __wrap_free(outer_ptr); - __wrap_free(inner_ptr); -} - -TEST_F(MemoryAllocationTest, NestedMemoryScopeFree) { - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - - MemoryPool outer_pool{0}; - MemoryPool inner_pool{0}; - void* outer_ptr = nullptr; - void* inner_ptr = nullptr; - - // Allocate outer pool - { - NestedMemoryScope scope{outer_pool}; - - EXPECT_CALL(*kMockRedisModule, Alloc(112)) - .WillOnce(testing::Return(reinterpret_cast(0x1000))); - EXPECT_CALL(*kMockRedisModule, - MallocUsableSize(reinterpret_cast(0x1000))) - .WillRepeatedly(testing::Return(128)); - outer_ptr = __wrap_malloc(100); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 128); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 128); - EXPECT_EQ(outer_pool.GetUsage(), 128); - EXPECT_EQ(inner_pool.GetUsage(), 0); - - // Allocate inner pool. - { - NestedMemoryScope scope{inner_pool}; - - EXPECT_CALL(*kMockRedisModule, Alloc(80)) - .WillOnce(testing::Return(reinterpret_cast(0x2000))); - EXPECT_CALL(*kMockRedisModule, - MallocUsableSize(reinterpret_cast(0x2000))) - .WillRepeatedly(testing::Return(96)); - inner_ptr = __wrap_malloc(75); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 224); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 224); - EXPECT_EQ(outer_pool.GetUsage(), 224); - EXPECT_EQ(inner_pool.GetUsage(), 96); - - { - NestedMemoryScope outer_scope{outer_pool}; - - { - NestedMemoryScope inner_scope{inner_pool}; - - EXPECT_CALL(*kMockRedisModule, Free(reinterpret_cast(0x2000))) - .Times(1); - __wrap_free(inner_ptr); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 128); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 128); - EXPECT_EQ(outer_pool.GetUsage(), 224); - EXPECT_EQ(inner_pool.GetUsage(), 96); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 128); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 128); - EXPECT_EQ(outer_pool.GetUsage(), 128); - EXPECT_EQ(inner_pool.GetUsage(), 0); - - EXPECT_CALL(*kMockRedisModule, Free(reinterpret_cast(0x1000))) - .Times(1); - __wrap_free(outer_ptr); - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - EXPECT_EQ(outer_pool.GetUsage(), 128); - EXPECT_EQ(inner_pool.GetUsage(), 0); - } - - EXPECT_EQ(vmsdk::GetUsedMemoryCnt(), 0); - EXPECT_EQ(vmsdk::GetMemoryDelta(), 0); - EXPECT_EQ(outer_pool.GetUsage(), 0); - EXPECT_EQ(inner_pool.GetUsage(), 0); -} - -#endif // TESTING_TMP_DISABLED -} // namespace - -} // namespace vmsdk diff --git a/vmsdk/versionscript.lds b/vmsdk/versionscript.lds index 4fbc0a01a..bcb3e9a4b 100644 --- a/vmsdk/versionscript.lds +++ b/vmsdk/versionscript.lds @@ -2,10 +2,36 @@ global: *; local: - __wrap_*; - __real_*; + /* + * The module defines the C allocator entry points itself; see + * vmsdk/src/memory_allocation_c_api.cc. Listing them here keeps them out of + * the dynamic symbol table, which hides them from the rest of the process + * and -- because a local symbol cannot be preempted -- makes every reference + * from inside the module bind to them at link time. + * + * That binding is the whole mechanism. A dlopened library's symbol lookups + * search the global scope first, and libc.so.6 defines malloc, so without + * this these definitions would simply be ignored -- even by the module's own + * operator new. + */ + malloc; + free; + calloc; + realloc; + aligned_alloc; + posix_memalign; + valloc; + malloc_usable_size; + strdup; + realpath; + getcwd; + /* + * operator new/delete come from the statically linked libstdc++. Keep them + * unexported so the module cannot interpose them for anything else in the + * process. + */ _Znwm*; _Znam*; _ZdlPv*; _ZdaPv*; -}; \ No newline at end of file +};