Various memory-measuring improvements

Author: graydon

Builds/VisualStudio/stellar-core.vcxproj

@@ -908,6 +908,7 @@ exit /b 0
     <ClCompile Include="..\..\src\util\test\Uint128Tests.cpp" />
     <ClCompile Include="..\..\src\util\test\XDRStreamTests.cpp" />
     <ClCompile Include="..\..\src\util\TarjanSCCCalculator.cpp" />
+    <ClCompile Include="..\..\src\util\TcmallocConfig.cpp" />
     <ClCompile Include="..\..\src\util\Thread.cpp" />
     <ClCompile Include="..\..\src\util\TmpDir.cpp" />
     <ClCompile Include="..\..\src\util\Timer.cpp" />
@@ -1280,6 +1281,7 @@ exit /b 0
     <ClInclude Include="..\..\src\util\SociNoWarnings.h" />
     <ClInclude Include="..\..\src\util\StatusManager.h" />
     <ClInclude Include="..\..\src\util\TarjanSCCCalculator.h" />
+    <ClInclude Include="..\..\src\util\TcmallocConfig.h" />
     <ClInclude Include="..\..\src\util\Thread.h" />
     <ClInclude Include="..\..\src\util\TmpDir.h" />
     <ClInclude Include="..\..\src\util\Timer.h" />

Builds/VisualStudio/stellar-core.vcxproj.filters

@@ -489,6 +489,9 @@
     <ClCompile Include="..\..\src\util\TarjanSCCCalculator.cpp">
       <Filter>util</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\util\TcmallocConfig.cpp">
+      <Filter>util</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\util\Thread.cpp">
       <Filter>util</Filter>
     </ClCompile>
@@ -1737,6 +1740,9 @@
     <ClInclude Include="..\..\src\util\TarjanSCCCalculator.h">
       <Filter>util</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\src\util\TcmallocConfig.h">
+      <Filter>util</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\src\util\Thread.h">
       <Filter>util</Filter>
     </ClInclude>

INSTALL.md

@@ -268,3 +268,15 @@ The GUI depends on the `capstone`, `freetype` and `glfw` libraries and their hea
 
     # On MacOS
     $ brew install capstone freetype2 glfw
+
+On Windows, follow the [build and install instructions](https://github.com/wolfpld/tracy) from the main Tracy site.
+
+At a high level you need to install the required prerequisites to build clients, run in a shell:
+
+    vcpkg.exe integrate install
+    vcpkg.exe install --triplet x64-windows-static capstone freetype glfw3
+
+Then build one of the servers. Solutions for servers compatible with the version of stellar-core can be found under:
+
+    * lib/tracy/profiler/build/win32 (GUI)
+    * lib/tracy/capture/build/win32

common.mk

@@ -31,6 +31,10 @@ if USE_TRACY
 AM_CPPFLAGS += -DUSE_TRACY $(tracy_CFLAGS)
 endif # USE_TRACY
 
+if USE_TRACY_MEMORY_TRACKING
+AM_CPPFLAGS += -DUSE_TRACY_MEMORY_TRACKING
+endif # USE_TRACY_MEMORY_TRACKING
+
 if BUILD_TESTS
 AM_CPPFLAGS += -DBUILD_TESTS=1
 endif # BUILD_TESTS

configure.ac

@@ -534,7 +534,12 @@ AC_ARG_ENABLE(tracy-memory-tracking,
     AS_HELP_STRING([--enable-tracy-memory-tracking],
         [Enable 'tracy' profiler/tracer memory tracking code (slow)]))
 AM_CONDITIONAL(USE_TRACY_MEMORY_TRACKING, [test x$enable_tracy_memory_tracking = xyes])
-
+if test x"$enable_tracy_memory_tracking" = xyes -a x"$have_tcmalloc" = x1; then
+  AC_MSG_ERROR([--enable-tracy-memory-tracking requires --disable-tcmalloc])
+fi
+if test x"$enable_tracy_memory_tracking" = xyes -a x"$enable_tracy" != xyes; then
+  AC_MSG_ERROR([--enable-tracy-memory-tracking requires --enable-tracy])
+fi
 if test x"$enable_tracy" = xyes -a x"$enable_asan" = xyes; then
        AC_MSG_ERROR([--enable-asan is not compatible with --enable-tracy])
 fi

docs/metrics.md

@@ -284,7 +284,8 @@ soroban.config.bucket-list-target-size-byte  | counter   | soroban config settin
 soroban.module-cache.num-entries             | counter   | current number of entries in module cache
 soroban.module-cache.compilation-time        | timer     | times each contract compilation when adding to module cache
 soroban.module-cache.rebuild-time            | timer     | times each rebuild of module cache (including all compilations)
-soroban.module-cache.rebuild-bytes           | counter   | bytes of WASM bytecode compiled in last rebuild of module cache
+soroban.module-cache.rebuild-wasm-bytes      | counter   | bytes of WASM bytecode compiled in last rebuild of module cache
+soroban.module-cache.rebuild-heap-bytes      | counter   | bytes of heap memory allocated in last rebuild of module cache
 soroban.in-memory-state.contract-code-size   | counter   | size in bytes of non-evicted ContractCode entries according to memory cost model
 soroban.in-memory-state.contract-data-size   | counter   | size in bytes of ContractData entries in memory
 soroban.in-memory-state.contract-code-entries   | counter   | number of ContractCode entries in memory

performance-eval/performance-eval.md

@@ -294,28 +294,84 @@ solution is, as root to run
 
 Stellar-core has built-in support for Tracy traces.
 
-To install the visualizer, follow the [build and install instructions](https://github.com/wolfpld/tracy) from the main Tracy site.
+To install the visualizer, follow the directions in [INSTALL.md](../INSTALL.md).
 
-At a high level you need to
+### General Visual Studio profiler
 
-install the required pre-requesites to build clients, run in a shell:
+The main page for the profiler built into Visual Studio Community Edition is located there:  https://docs.microsoft.com/en-us/visualstudio/profiling/index
 
-    vcpkg.exe integrate install
-    vcpkg.exe install --triplet x64-windows-static capstone freetype glfw3
+## All platforms
 
-Then build one of the servers.
+Intel V-Tune (free, unlimited license 90 days renewal) https://software.intel.com/en-us/system-studio/choose-download
 
-Solutions for servers compatible with the version of stellar-core can be found under:
+# Memory-use profiling
 
-    * lib/tracy/profiler/build/win32 (GUI)
-    * lib/tracy/capture/build/win32
+Tools for memory use profiling are less well-developed than CPU profiling, but there are some options available.
 
-Note: when connecting, use `localhost` instead of `127.0.0.1` as Tracy binds by default to IPV6 addresses.
+## Tracy
 
-### General Visual Studio profiler
+Tracy has some built-in support for memory profiling, but turning it on will
+slow down core significantly and will use memory in the tracy client _very
+quickly_, easily overwhelming your workstation if you're not careful. So you can
+usually only turn it on for a brief period of time.
 
-The main page for the profiler built into Visual Studio Community Edition is located there:  https://docs.microsoft.com/en-us/visualstudio/profiling/index
+It is most useful for examining a small part of the code for a short period of
+time, where you already have a fairly good idea of there being memory allocation
+issues that you want to see a precise accounting of. Allocations get linked to
+zones (as a list in each zone detail view) and are available for inspection in
+the "memory" window, along with a total map of memory and a list of all
+allocations.
 
-## All platforms
+Stellar-core has support for this mode separate from normal tracy tracing,
+because it is so performance intensive and memory hungry. You need to configure
+with --enable-tracy-memory-tracking and --disable-tcmalloc.
+
+## Heaptrack
+
+A better option for a high level "profile" of memory is the "heaptrack" tool,
+which is available on Linux.
+
+    $ sudo apt install heaptrack heaptrack-gui
+
+To use it you will also need to configure with --disable-tcmalloc, because it
+works by intercepting malloc/free calls underlying the default operator
+new/delete, and tcmalloc's operator new and delete will bypass that
+interception.
+
+Heaptrack should also be run only for a moderate amount of time, otherwise the
+recording will be huge. But it at least writes its recording to disk, and the
+recording is much more compact than tracy's in-memory structure, so it can run
+much longer than tracy in memory-recording mode without issue.
+
+Heaptrack can run a program as a subprocess or attach remotely. The remote
+attach mode allows you to avoid starting it until the program is close to the
+period you want to measure, so is recommended. You will need to enable ptrace
+permissions.
+
+Heaptrack's default recording mode is very slow as it symbolicates all the
+stacks while it runs. A better way is to record a _raw_ profile and then
+symbolicate the data after the fact.
+
+Combining these facts, the best execution we've found is like the following:
+
+    # in one terminal...
+    $ stellar-core ...
+
+    # in another terminal...
+    $ echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope
+    $ heaptrack --raw $(pidof stellar-core)
+
+    # switch back core and stop it with Ctrl-C when done
+
+    # heaptrack will exit and write a file like
+    # heaptrack.stellar-core.12345.raw.zst along with, hopefully, instructions
+    # to run something like this to post-process the raw file into a more
+    # compact and symbolicated form. This will run a long time:
+
+    $ zstd -dc < ".../heaptrack.stellar-core.12345.raw.zst" \
+        | /usr/lib/heaptrack/libexec/heaptrack_interpret \
+        | zstd -c > ".../heaptrack.stellar-core.12345.zst"
+
+    # finally load the profile into the visualization tool
+    $ heaptrack_gui ".../heaptrack.stellar-core.12345.zst"
 
-Intel V-Tune (free, unlimited license 90 days renewal) https://software.intel.com/en-us/system-studio/choose-download

src/ledger/LedgerManagerImpl.cpp

@@ -1028,8 +1028,10 @@ LedgerManagerImpl::ApplyState::finishPendingCompilation()
     releaseAssert(mPhase == Phase::SETTING_UP_STATE);
     releaseAssert(mCompiler);
     auto newCache = mCompiler->wait();
-    getMetrics().mSorobanMetrics.mModuleCacheRebuildBytes.set_count(
+    getMetrics().mSorobanMetrics.mModuleCacheRebuildWasmBytes.set_count(
         (int64)mCompiler->getBytesCompiled());
+    getMetrics().mSorobanMetrics.mModuleCacheRebuildHeapBytes.set_count(
+        mCompiler->getBytesAllocatedDuringCompilation());
     getMetrics().mSorobanMetrics.mModuleCacheNumEntries.set_count(
         (int64)mCompiler->getContractsCompiled());
     getMetrics().mSorobanMetrics.mModuleCacheRebuildTime.Update(
@@ -1175,7 +1177,7 @@ LedgerManagerImpl::ApplyState::maybeRebuildModuleCache(
     // contract-set in the live BL as an event that warrants a rebuild.
 
     int64_t lastCompiledWasmBytesCount =
-        getMetrics().mSorobanMetrics.mModuleCacheRebuildBytes.count();
+        getMetrics().mSorobanMetrics.mModuleCacheRebuildWasmBytes.count();
     uint64_t lastCompiledWasmBytes =
         lastCompiledWasmBytesCount < 0
             ? 0

src/ledger/SharedModuleCacheCompiler.cpp

@@ -135,14 +135,12 @@ SharedModuleCacheCompiler::popAndCompileWasm(size_t thread,
     return true;
 }
 
-static size_t heap_at_compile_start = 0;
-
 void
 SharedModuleCacheCompiler::start()
 {
     mStarted = std::chrono::steady_clock::now();
 
-    heap_at_compile_start = getMallocBytesInUse();
+    mHeapSizeAtStart = getMallocBytesInUse();
 
     LOG_INFO(DEFAULT_LOG,
              "Launching 1 loading and {} compiling background threads",
@@ -209,17 +207,20 @@ SharedModuleCacheCompiler::wait()
     auto end = std::chrono::steady_clock::now();
     LOG_INFO(
         DEFAULT_LOG,
-        "Compiled {} contracts ({} bytes of Wasm) in {}ms real time, {}ms "
+        "Compiled {} contracts ({} of Wasm) in {}ms real time, {}ms "
         "CPU time",
-        mContractsCompiled, mBytesCompiled,
+        mContractsCompiled, formatSize(mBytesCompiled),
         std::chrono::duration_cast<std::chrono::milliseconds>(end - mStarted)
             .count(),
         std::chrono::duration_cast<std::chrono::milliseconds>(mTotalCompileTime)
             .count());
-    auto heap_at_compile_end = getMallocBytesInUse();
-    LOG_INFO(DEFAULT_LOG, "Heap grew from {} to {} bytes during ompilation, difference is {} bytes",
-        heap_at_compile_start, heap_at_compile_end, heap_at_compile_end - heap_at_compile_start);
-
+    mHeapSizeAtEnd = getMallocBytesInUse();
+    int64_t heapDiff = static_cast<int64_t>(mHeapSizeAtEnd) -
+                       static_cast<int64_t>(mHeapSizeAtStart);
+    LOG_INFO(DEFAULT_LOG,
+             "Heap changed from {} to {} during compilation ({} difference)",
+             formatSize(mHeapSizeAtStart), formatSize(mHeapSizeAtEnd),
+             formatSize(heapDiff));
     return mModuleCache->shallow_clone();
 }
 
@@ -230,6 +231,14 @@ SharedModuleCacheCompiler::getBytesCompiled()
     return mBytesCompiled * mLedgerVersions.size();
 }
 
+int64_t
+SharedModuleCacheCompiler::getBytesAllocatedDuringCompilation()
+{
+    std::unique_lock lock(mMutex);
+    return static_cast<int64_t>(mHeapSizeAtEnd) -
+           static_cast<int64_t>(mHeapSizeAtStart);
+}
+
 std::chrono::nanoseconds
 SharedModuleCacheCompiler::getCompileTime()
 {

src/ledger/SharedModuleCacheCompiler.h

@@ -44,6 +44,8 @@ class SharedModuleCacheCompiler : NonMovableOrCopyable
     std::condition_variable mHaveContracts;
 
     std::chrono::steady_clock::time_point mStarted;
+    size_t mHeapSizeAtStart{0};
+    size_t mHeapSizeAtEnd{0};
     std::chrono::nanoseconds mTotalCompileTime{0};
 
     void setFinishedLoading(size_t nContracts);
@@ -61,6 +63,7 @@ class SharedModuleCacheCompiler : NonMovableOrCopyable
     void start();
     ::rust::Box<stellar::rust_bridge::SorobanModuleCache> wait();
     size_t getBytesCompiled();
+    int64_t getBytesAllocatedDuringCompilation();
     std::chrono::nanoseconds getCompileTime();
     size_t getContractsCompiled();
 };