diff --git a/Builds/VisualStudio/stellar-core.vcxproj b/Builds/VisualStudio/stellar-core.vcxproj index 318f13f7f0..a47f8c94ed 100644 --- a/Builds/VisualStudio/stellar-core.vcxproj +++ b/Builds/VisualStudio/stellar-core.vcxproj @@ -908,6 +908,7 @@ exit /b 0 + @@ -1280,6 +1281,7 @@ exit /b 0 + diff --git a/Builds/VisualStudio/stellar-core.vcxproj.filters b/Builds/VisualStudio/stellar-core.vcxproj.filters index d1cf2654c5..0a2df18d3f 100644 --- a/Builds/VisualStudio/stellar-core.vcxproj.filters +++ b/Builds/VisualStudio/stellar-core.vcxproj.filters @@ -489,6 +489,9 @@ util + + util + util @@ -1737,6 +1740,9 @@ util + + util + util diff --git a/INSTALL.md b/INSTALL.md index 7a5ded2f7f..cd74c5efcb 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -268,3 +268,15 @@ The GUI depends on the `capstone`, `freetype` and `glfw` libraries and their hea # On MacOS $ brew install capstone freetype2 glfw + +On Windows, follow the [build and install instructions](https://github.com/wolfpld/tracy) from the main Tracy site. + +At a high level you need to install the required prerequisites to build clients, run in a shell: + + vcpkg.exe integrate install + vcpkg.exe install --triplet x64-windows-static capstone freetype glfw3 + +Then build one of the servers. Solutions for servers compatible with the version of stellar-core can be found under: + + * lib/tracy/profiler/build/win32 (GUI) + * lib/tracy/capture/build/win32 diff --git a/common.mk b/common.mk index 41ca4d835b..abae4f498a 100644 --- a/common.mk +++ b/common.mk @@ -31,6 +31,10 @@ if USE_TRACY AM_CPPFLAGS += -DUSE_TRACY $(tracy_CFLAGS) endif # USE_TRACY +if USE_TRACY_MEMORY_TRACKING +AM_CPPFLAGS += -DUSE_TRACY_MEMORY_TRACKING +endif # USE_TRACY_MEMORY_TRACKING + if BUILD_TESTS AM_CPPFLAGS += -DBUILD_TESTS=1 endif # BUILD_TESTS diff --git a/configure.ac b/configure.ac index ae77cd1760..cd692864b9 100644 --- a/configure.ac +++ b/configure.ac @@ -470,8 +470,13 @@ AM_CONDITIONAL(USE_POSTGRES, [test -n "$have_postgres"]) # tcmalloc_minimal - enabled by default on Linux for improved memory allocation performance # Disabled when using sanitizers (asan, memcheck, tsan) as they replace malloc # Uses vendored gperftools from lib/gperftools +AC_ARG_ENABLE(tcmalloc, + AS_HELP_STRING([--disable-tcmalloc], + [Disable tcmalloc support])) unset have_tcmalloc -if test "x$enable_asan" = "xyes" -o "x$enable_memcheck" = "xyes" -o "x$enable_threadsanitizer" = "xyes"; then +if test x"$enable_tcmalloc" = xno; then + AC_MSG_NOTICE([tcmalloc disabled by user]) +elif test "x$enable_asan" = "xyes" -o "x$enable_memcheck" = "xyes" -o "x$enable_threadsanitizer" = "xyes"; then AC_MSG_NOTICE([tcmalloc disabled due to sanitizer being enabled]) else case "${host_os}" in @@ -529,7 +534,12 @@ AC_ARG_ENABLE(tracy-memory-tracking, AS_HELP_STRING([--enable-tracy-memory-tracking], [Enable 'tracy' profiler/tracer memory tracking code (slow)])) AM_CONDITIONAL(USE_TRACY_MEMORY_TRACKING, [test x$enable_tracy_memory_tracking = xyes]) - +if test x"$enable_tracy_memory_tracking" = xyes -a x"$have_tcmalloc" = x1; then + AC_MSG_ERROR([--enable-tracy-memory-tracking requires --disable-tcmalloc]) +fi +if test x"$enable_tracy_memory_tracking" = xyes -a x"$enable_tracy" != xyes; then + AC_MSG_ERROR([--enable-tracy-memory-tracking requires --enable-tracy]) +fi if test x"$enable_tracy" = xyes -a x"$enable_asan" = xyes; then AC_MSG_ERROR([--enable-asan is not compatible with --enable-tracy]) fi diff --git a/docs/metrics.md b/docs/metrics.md index c538fbb413..5b96a8030d 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -284,7 +284,8 @@ soroban.config.bucket-list-target-size-byte | counter | soroban config settin soroban.module-cache.num-entries | counter | current number of entries in module cache soroban.module-cache.compilation-time | timer | times each contract compilation when adding to module cache soroban.module-cache.rebuild-time | timer | times each rebuild of module cache (including all compilations) -soroban.module-cache.rebuild-bytes | counter | bytes of WASM bytecode compiled in last rebuild of module cache +soroban.module-cache.rebuild-wasm-bytes | counter | bytes of WASM bytecode compiled in last rebuild of module cache +soroban.module-cache.rebuild-heap-bytes | counter | bytes of heap memory allocated in last rebuild of module cache soroban.in-memory-state.contract-code-size | counter | size in bytes of non-evicted ContractCode entries according to memory cost model soroban.in-memory-state.contract-data-size | counter | size in bytes of ContractData entries in memory soroban.in-memory-state.contract-code-entries | counter | number of ContractCode entries in memory diff --git a/performance-eval/performance-eval.md b/performance-eval/performance-eval.md index b3073f2f17..b72a5a0ec5 100644 --- a/performance-eval/performance-eval.md +++ b/performance-eval/performance-eval.md @@ -294,28 +294,84 @@ solution is, as root to run Stellar-core has built-in support for Tracy traces. -To install the visualizer, follow the [build and install instructions](https://github.com/wolfpld/tracy) from the main Tracy site. +To install the visualizer, follow the directions in [INSTALL.md](../INSTALL.md). -At a high level you need to +### General Visual Studio profiler -install the required pre-requesites to build clients, run in a shell: +The main page for the profiler built into Visual Studio Community Edition is located there: https://docs.microsoft.com/en-us/visualstudio/profiling/index - vcpkg.exe integrate install - vcpkg.exe install --triplet x64-windows-static capstone freetype glfw3 +## All platforms -Then build one of the servers. +Intel V-Tune (free, unlimited license 90 days renewal) https://software.intel.com/en-us/system-studio/choose-download -Solutions for servers compatible with the version of stellar-core can be found under: +# Memory-use profiling - * lib/tracy/profiler/build/win32 (GUI) - * lib/tracy/capture/build/win32 +Tools for memory use profiling are less well-developed than CPU profiling, but there are some options available. -Note: when connecting, use `localhost` instead of `127.0.0.1` as Tracy binds by default to IPV6 addresses. +## Tracy -### General Visual Studio profiler +Tracy has some built-in support for memory profiling, but turning it on will +slow down core significantly and will use memory in the tracy client _very +quickly_, easily overwhelming your workstation if you're not careful. So you can +usually only turn it on for a brief period of time. -The main page for the profiler built into Visual Studio Community Edition is located there: https://docs.microsoft.com/en-us/visualstudio/profiling/index +It is most useful for examining a small part of the code for a short period of +time, where you already have a fairly good idea of there being memory allocation +issues that you want to see a precise accounting of. Allocations get linked to +zones (as a list in each zone detail view) and are available for inspection in +the "memory" window, along with a total map of memory and a list of all +allocations. -## All platforms +Stellar-core has support for this mode separate from normal tracy tracing, +because it is so performance intensive and memory hungry. You need to configure +with --enable-tracy-memory-tracking and --disable-tcmalloc. + +## Heaptrack + +A better option for a high level "profile" of memory is the "heaptrack" tool, +which is available on Linux. + + $ sudo apt install heaptrack heaptrack-gui + +To use it you will also need to configure with --disable-tcmalloc, because it +works by intercepting malloc/free calls underlying the default operator +new/delete, and tcmalloc's operator new and delete will bypass that +interception. + +Heaptrack should also be run only for a moderate amount of time, otherwise the +recording will be huge. But it at least writes its recording to disk, and the +recording is much more compact than tracy's in-memory structure, so it can run +much longer than tracy in memory-recording mode without issue. + +Heaptrack can run a program as a subprocess or attach remotely. The remote +attach mode allows you to avoid starting it until the program is close to the +period you want to measure, so is recommended. You will need to enable ptrace +permissions. + +Heaptrack's default recording mode is very slow as it symbolicates all the +stacks while it runs. A better way is to record a _raw_ profile and then +symbolicate the data after the fact. + +Combining these facts, the best execution we've found is like the following: + + # in one terminal... + $ stellar-core ... + + # in another terminal... + $ echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope + $ heaptrack --raw $(pidof stellar-core) + + # switch back core and stop it with Ctrl-C when done + + # heaptrack will exit and write a file like + # heaptrack.stellar-core.12345.raw.zst along with, hopefully, instructions + # to run something like this to post-process the raw file into a more + # compact and symbolicated form. This will run a long time: + + $ zstd -dc < ".../heaptrack.stellar-core.12345.raw.zst" \ + | /usr/lib/heaptrack/libexec/heaptrack_interpret \ + | zstd -c > ".../heaptrack.stellar-core.12345.zst" + + # finally load the profile into the visualization tool + $ heaptrack_gui ".../heaptrack.stellar-core.12345.zst" -Intel V-Tune (free, unlimited license 90 days renewal) https://software.intel.com/en-us/system-studio/choose-download diff --git a/src/ledger/LedgerManagerImpl.cpp b/src/ledger/LedgerManagerImpl.cpp index 74c0939ad2..ecd935ab48 100644 --- a/src/ledger/LedgerManagerImpl.cpp +++ b/src/ledger/LedgerManagerImpl.cpp @@ -1028,8 +1028,10 @@ LedgerManagerImpl::ApplyState::finishPendingCompilation() releaseAssert(mPhase == Phase::SETTING_UP_STATE); releaseAssert(mCompiler); auto newCache = mCompiler->wait(); - getMetrics().mSorobanMetrics.mModuleCacheRebuildBytes.set_count( + getMetrics().mSorobanMetrics.mModuleCacheRebuildWasmBytes.set_count( (int64)mCompiler->getBytesCompiled()); + getMetrics().mSorobanMetrics.mModuleCacheRebuildHeapBytes.set_count( + mCompiler->getBytesAllocatedDuringCompilation()); getMetrics().mSorobanMetrics.mModuleCacheNumEntries.set_count( (int64)mCompiler->getContractsCompiled()); getMetrics().mSorobanMetrics.mModuleCacheRebuildTime.Update( @@ -1175,7 +1177,7 @@ LedgerManagerImpl::ApplyState::maybeRebuildModuleCache( // contract-set in the live BL as an event that warrants a rebuild. int64_t lastCompiledWasmBytesCount = - getMetrics().mSorobanMetrics.mModuleCacheRebuildBytes.count(); + getMetrics().mSorobanMetrics.mModuleCacheRebuildWasmBytes.count(); uint64_t lastCompiledWasmBytes = lastCompiledWasmBytesCount < 0 ? 0 diff --git a/src/ledger/SharedModuleCacheCompiler.cpp b/src/ledger/SharedModuleCacheCompiler.cpp index dca987bd43..ac01b46eca 100644 --- a/src/ledger/SharedModuleCacheCompiler.cpp +++ b/src/ledger/SharedModuleCacheCompiler.cpp @@ -7,6 +7,7 @@ #include "crypto/SHA.h" #include "rust/RustBridge.h" #include "util/Logging.h" +#include "util/TcmallocConfig.h" #include "xdr/Stellar-ledger-entries.h" #include #include @@ -139,6 +140,8 @@ SharedModuleCacheCompiler::start() { mStarted = std::chrono::steady_clock::now(); + mHeapSizeAtStart = getMallocBytesInUse(); + LOG_INFO(DEFAULT_LOG, "Launching 1 loading and {} compiling background threads", mNumThreads - 1); @@ -204,13 +207,20 @@ SharedModuleCacheCompiler::wait() auto end = std::chrono::steady_clock::now(); LOG_INFO( DEFAULT_LOG, - "Compiled {} contracts ({} bytes of Wasm) in {}ms real time, {}ms " + "Compiled {} contracts ({} of Wasm) in {}ms real time, {}ms " "CPU time", - mContractsCompiled, mBytesCompiled, + mContractsCompiled, formatSize(mBytesCompiled), std::chrono::duration_cast(end - mStarted) .count(), std::chrono::duration_cast(mTotalCompileTime) .count()); + mHeapSizeAtEnd = getMallocBytesInUse(); + int64_t heapDiff = static_cast(mHeapSizeAtEnd) - + static_cast(mHeapSizeAtStart); + LOG_INFO(DEFAULT_LOG, + "Heap changed from {} to {} during compilation ({} difference)", + formatSize(mHeapSizeAtStart), formatSize(mHeapSizeAtEnd), + formatSize(heapDiff)); return mModuleCache->shallow_clone(); } @@ -221,6 +231,14 @@ SharedModuleCacheCompiler::getBytesCompiled() return mBytesCompiled * mLedgerVersions.size(); } +int64_t +SharedModuleCacheCompiler::getBytesAllocatedDuringCompilation() +{ + std::unique_lock lock(mMutex); + return static_cast(mHeapSizeAtEnd) - + static_cast(mHeapSizeAtStart); +} + std::chrono::nanoseconds SharedModuleCacheCompiler::getCompileTime() { diff --git a/src/ledger/SharedModuleCacheCompiler.h b/src/ledger/SharedModuleCacheCompiler.h index 3bae557acf..ed039db58a 100644 --- a/src/ledger/SharedModuleCacheCompiler.h +++ b/src/ledger/SharedModuleCacheCompiler.h @@ -44,6 +44,8 @@ class SharedModuleCacheCompiler : NonMovableOrCopyable std::condition_variable mHaveContracts; std::chrono::steady_clock::time_point mStarted; + size_t mHeapSizeAtStart{0}; + size_t mHeapSizeAtEnd{0}; std::chrono::nanoseconds mTotalCompileTime{0}; void setFinishedLoading(size_t nContracts); @@ -61,6 +63,7 @@ class SharedModuleCacheCompiler : NonMovableOrCopyable void start(); ::rust::Box wait(); size_t getBytesCompiled(); + int64_t getBytesAllocatedDuringCompilation(); std::chrono::nanoseconds getCompileTime(); size_t getContractsCompiled(); }; diff --git a/src/ledger/SorobanMetrics.cpp b/src/ledger/SorobanMetrics.cpp index 5db27cd2ec..c7411e9e26 100644 --- a/src/ledger/SorobanMetrics.cpp +++ b/src/ledger/SorobanMetrics.cpp @@ -148,8 +148,10 @@ SorobanMetrics::SorobanMetrics(MetricsRegistry& metrics) metrics.NewTimer({"soroban", "module-cache", "compilation-time"})) , mModuleCacheRebuildTime( metrics.NewTimer({"soroban", "module-cache", "rebuild-time"})) - , mModuleCacheRebuildBytes( - metrics.NewCounter({"soroban", "module-cache", "rebuild-bytes"})) + , mModuleCacheRebuildWasmBytes( + metrics.NewCounter({"soroban", "module-cache", "rebuild-wasm-bytes"})) + , mModuleCacheRebuildHeapBytes( + metrics.NewCounter({"soroban", "module-cache", "rebuild-heap-bytes"})) , mContractCodeStateSize(metrics.NewCounter( {"soroban", "in-memory-state", "contract-code-size"})) , mContractDataStateSize(metrics.NewCounter( diff --git a/src/ledger/SorobanMetrics.h b/src/ledger/SorobanMetrics.h index 8c0499564c..edcd2569cd 100644 --- a/src/ledger/SorobanMetrics.h +++ b/src/ledger/SorobanMetrics.h @@ -193,7 +193,8 @@ class SorobanMetrics medida::Counter& mModuleCacheNumEntries; medida::Timer& mModuleCompilationTime; medida::Timer& mModuleCacheRebuildTime; - medida::Counter& mModuleCacheRebuildBytes; + medida::Counter& mModuleCacheRebuildWasmBytes; + medida::Counter& mModuleCacheRebuildHeapBytes; // In-memory state metrics medida::Counter& mContractCodeStateSize; diff --git a/src/main/main.cpp b/src/main/main.cpp index 8f0d63afbe..43b10c080e 100644 --- a/src/main/main.cpp +++ b/src/main/main.cpp @@ -345,6 +345,10 @@ checkStellarCoreMajorVersionProtocolIdentity() #ifdef USE_TRACY_MEMORY_TRACKING +#ifndef USE_TRACY +#error "USE_TRACY_MEMORY_TRACKING requires USE_TRACY" +#endif + #ifdef __has_feature #if __has_feature(address_sanitizer) #define ASAN_ENABLED diff --git a/src/transactions/test/InvokeHostFunctionTests.cpp b/src/transactions/test/InvokeHostFunctionTests.cpp index 18cfe946f2..2feed38490 100644 --- a/src/transactions/test/InvokeHostFunctionTests.cpp +++ b/src/transactions/test/InvokeHostFunctionTests.cpp @@ -7615,7 +7615,7 @@ TEST_CASE("module cache rebuild on incremental wasm uploads", REQUIRE(wasmsAreCached(*app, initialHashes)); auto& metrics = app->getLedgerManager().getSorobanMetrics(); - auto rebuildBytesAtStartup = metrics.mModuleCacheRebuildBytes.count(); + auto rebuildBytesAtStartup = metrics.mModuleCacheRebuildWasmBytes.count(); REQUIRE(rebuildBytesAtStartup > 0); auto uploader = app->getRoot(); @@ -7644,7 +7644,8 @@ TEST_CASE("module cache rebuild on incremental wasm uploads", // ledger close at apply start. closeLedger(*app); - if (metrics.mModuleCacheRebuildBytes.count() != rebuildBytesAtStartup) + if (metrics.mModuleCacheRebuildWasmBytes.count() != + rebuildBytesAtStartup) { rebuilt = true; uploadedRawAtTrigger = uploadedRawBytes; diff --git a/src/util/TcmallocConfig.cpp b/src/util/TcmallocConfig.cpp index 57575fa3df..3cc285aa2b 100644 --- a/src/util/TcmallocConfig.cpp +++ b/src/util/TcmallocConfig.cpp @@ -2,11 +2,27 @@ // under the Apache License, Version 2.0. See the COPYING file at the root // of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 +#include "util/TcmallocConfig.h" #include "config.h" #ifdef USE_TCMALLOC #include +namespace stellar +{ +size_t +getMallocBytesInUse() +{ + size_t out; + MallocExtension* ext = MallocExtension::instance(); + if (ext && ext->GetNumericProperty("generic.current_allocated_bytes", &out)) + { + return out; + } + return 0; +} +} + namespace { // Configure tcmalloc parameters at startup using a constructor attribute. @@ -44,4 +60,13 @@ initTcmallocConfig() } } } // namespace +#else +namespace stellar +{ +size_t +getMallocBytesInUse() +{ + return 0; +} +} #endif // USE_TCMALLOC diff --git a/src/util/TcmallocConfig.h b/src/util/TcmallocConfig.h new file mode 100644 index 0000000000..2fa51529b7 --- /dev/null +++ b/src/util/TcmallocConfig.h @@ -0,0 +1,12 @@ +// Copyright 2026 Stellar Development Foundation and contributors. Licensed +// under the Apache License, Version 2.0. See the COPYING file at the root +// of this distribution or at http://www.apache.org/licenses/LICENSE-2.0 + +#pragma once + +#include + +namespace stellar +{ +size_t getMallocBytesInUse(); +} \ No newline at end of file