Print max CArena usage

Src/Base/AMReX_Arena.H

@@ -188,7 +188,7 @@ public:
     static std::size_t align (std::size_t sz);
 
     static void Initialize (bool minimal);
-    static void PrintUsage ();
+    static void PrintUsage (bool print_max_usage=false);
     static void PrintUsageToFiles (std::string const& filename, std::string const& message);
     static void Finalize ();
 
@@ -210,6 +210,8 @@ public:
      */
     [[nodiscard]] const ArenaInfo& arenaInfo () const { return arena_info; }
 
+    virtual void ResetMaxUsageCounter () {}
+
 protected:
 
     ArenaInfo arena_info;

Src/Base/AMReX_Arena.cpp

@@ -347,6 +347,7 @@ Arena::Initialize (bool minimal)
         BL_PROFILE("The_Arena::Initialize()");
         void *p = the_arena->alloc(static_cast<std::size_t>(the_arena_init_size));
         the_arena->free(p);
+        the_arena->ResetMaxUsageCounter();
 #endif
 #else
         the_arena = The_BArena();
@@ -406,25 +407,29 @@ Arena::Initialize (bool minimal)
         BL_PROFILE("The_Device_Arena::Initialize()");
         void *p = the_device_arena->alloc(the_device_arena_init_size);
         the_device_arena->free(p);
+        the_device_arena->ResetMaxUsageCounter();
     }
 
     if (the_managed_arena_init_size > 0 && the_managed_arena != the_arena) {
         BL_PROFILE("The_Managed_Arena::Initialize()");
         void *p = the_managed_arena->alloc(the_managed_arena_init_size);
         the_managed_arena->free(p);
+        the_managed_arena->ResetMaxUsageCounter();
     }
 
     if (the_pinned_arena_init_size > 0) {
         BL_PROFILE("The_Pinned_Arena::Initialize()");
         void *p = the_pinned_arena->alloc(the_pinned_arena_init_size);
         the_pinned_arena->free(p);
+        the_pinned_arena->ResetMaxUsageCounter();
     }
 
     if (the_comms_arena_init_size > 0 && the_comms_arena != the_arena
         && the_comms_arena != the_device_arena && the_comms_arena != the_pinned_arena) {
         BL_PROFILE("The_Comms_Arena::Initialize()");
         void *p = the_comms_arena->alloc(the_comms_arena_init_size);
         the_comms_arena->free(p);
+        the_comms_arena->ResetMaxUsageCounter();
     }
 
     the_cpu_arena = The_BArena();
@@ -436,7 +441,7 @@ Arena::Initialize (bool minimal)
 }
 
 void
-Arena::PrintUsage ()
+Arena::PrintUsage (bool print_max_usage)
 {
 #ifdef AMREX_USE_GPU
     const int IOProc = ParallelDescriptor::IOProcessorNumber();
@@ -468,32 +473,32 @@ Arena::PrintUsage ()
     if (The_Arena()) {
         auto* p = dynamic_cast<CArena*>(The_Arena());
         if (p) {
-            p->PrintUsage("The         Arena");
+            p->PrintUsage("The         Arena", print_max_usage);
         }
     }
     if (The_Device_Arena() && The_Device_Arena() != The_Arena()) {
         auto* p = dynamic_cast<CArena*>(The_Device_Arena());
         if (p) {
-            p->PrintUsage("The  Device Arena");
+            p->PrintUsage("The  Device Arena", print_max_usage);
         }
     }
     if (The_Managed_Arena() && The_Managed_Arena() != The_Arena()) {
         auto* p = dynamic_cast<CArena*>(The_Managed_Arena());
         if (p) {
-            p->PrintUsage("The Managed Arena");
+            p->PrintUsage("The Managed Arena", print_max_usage);
         }
     }
     if (The_Pinned_Arena()) {
         auto* p = dynamic_cast<CArena*>(The_Pinned_Arena());
         if (p) {
-            p->PrintUsage("The  Pinned Arena");
+            p->PrintUsage("The  Pinned Arena", print_max_usage);
         }
     }
     if (The_Comms_Arena() && The_Comms_Arena() != The_Device_Arena()
          && The_Comms_Arena() != The_Pinned_Arena()) {
         auto* p = dynamic_cast<CArena*>(The_Comms_Arena());
         if (p) {
-            p->PrintUsage("The   Comms Arena");
+            p->PrintUsage("The   Comms Arena", print_max_usage);
         }
     }
 }
@@ -560,7 +565,7 @@ Arena::Finalize ()
 #else
     if (amrex::Verbose() > 1) {
 #endif
-        PrintUsage();
+        PrintUsage(true);
     }
 
     initialized = false;

Src/Base/AMReX_CArena.H

@@ -82,10 +82,14 @@ public:
     //! Return the amount of memory in this pointer.  Return 0 for unknown pointer.
     std::size_t sizeOf (void* p) const noexcept;
 
-    void PrintUsage (std::string const& name) const;
+    void PrintUsage (std::string const& name, bool print_max_usage) const;
 
     void PrintUsage (std::ostream& os, std::string const& name, std::string const& space) const;
 
+    void ResetMaxUsageCounter () final {
+        m_max_actually_used = 0;
+    }
+
     //! The default memory hunk size to grab from the heap.
     constexpr static std::size_t DefaultHunkSize = 1024*1024*8;
 
@@ -181,8 +185,12 @@ protected:
     std::size_t m_hunk;
     //! The amount of heap space currently allocated.
     std::size_t m_used{0};
+    //! The max amount of heap space currently allocated.
+    std::size_t m_max_used{0};
     //! The amount of memory given out via alloc().
     std::size_t m_actually_used{0};
+    //! The max amount of memory given out via alloc().
+    std::size_t m_max_actually_used{0};
 
 
     std::mutex carena_mutex;

Src/Base/AMReX_CArena.cpp

@@ -71,6 +71,7 @@ CArena::alloc_protected (std::size_t nbytes)
         vp = allocate_system(N);
 
         m_used += N;
+        m_max_used = std::max(m_used, m_max_used);
 
         m_alloc.emplace_back(vp,N);
 
@@ -116,6 +117,7 @@ CArena::alloc_protected (std::size_t nbytes)
     }
 
     m_actually_used += nbytes;
+    m_max_actually_used = std::max(m_actually_used, m_max_actually_used);
 
     BL_ASSERT(vp != nullptr);
 
@@ -166,6 +168,7 @@ CArena::alloc_in_place (void* pt, std::size_t szmin, std::size_t szmax)
                 }
 #endif
                 m_actually_used += new_size - busy_it->size();
+                m_max_actually_used = std::max(m_actually_used, m_max_actually_used);
                 const_cast<Node&>(*busy_it).size(new_size);
                 return std::make_pair(pt, new_size);
             } else if (total_size >= szmin) {
@@ -179,6 +182,7 @@ CArena::alloc_in_place (void* pt, std::size_t szmin, std::size_t szmax)
                 }
 #endif
                 m_actually_used += total_size - busy_it->size();
+                m_max_actually_used = std::max(m_actually_used, m_max_actually_used);
                 const_cast<Node&>(*busy_it).size(total_size);
                 return std::make_pair(pt, total_size);
             }
@@ -453,25 +457,29 @@ CArena::sizeOf (void* p) const noexcept
 }
 
 void
-CArena::PrintUsage (std::string const& name) const
+CArena::PrintUsage (std::string const& name, bool print_max_usage) const
 {
-    Long min_megabytes = static_cast<Long>(heap_space_used() / (1024*1024));
+    Long min_megabytes = static_cast<Long>(
+        (print_max_usage ? m_max_used : heap_space_used()) / (1024*1024));
     Long max_megabytes = min_megabytes;
-    Long actual_min_megabytes = static_cast<Long>(heap_space_actually_used() / (1024*1024));
+    Long actual_min_megabytes = static_cast<Long>(
+        (print_max_usage ? m_max_actually_used : heap_space_actually_used()) / (1024*1024));
     Long actual_max_megabytes = actual_min_megabytes;
     const int IOProc = ParallelDescriptor::IOProcessorNumber();
     ParallelReduce::Min<Long>({min_megabytes, actual_min_megabytes},
                               IOProc, ParallelDescriptor::Communicator());
     ParallelReduce::Max<Long>({max_megabytes, actual_max_megabytes},
                               IOProc, ParallelDescriptor::Communicator());
+
+    const auto name_space = "[" + name + "] " + (print_max_usage ? "max " : "") + "space ";
 #ifdef AMREX_USE_MPI
-    amrex::Print() << "[" << name << "] space (MB) allocated spread across MPI: ["
+    amrex::Print() << name_space << "(MB) allocated spread across MPI: ["
                    << min_megabytes << " ... " << max_megabytes << "]\n"
-                   << "[" << name << "] space (MB) used      spread across MPI: ["
+                   << name_space << "(MB) used      spread across MPI: ["
                    << actual_min_megabytes << " ... " << actual_max_megabytes << "]\n";
 #else
-    amrex::Print() << "[" << name << "] space allocated (MB): " << min_megabytes << "\n";
-    amrex::Print() << "[" << name << "] space used      (MB): " << actual_min_megabytes << "\n";
+    amrex::Print() << name_space << "allocated (MB): " << min_megabytes << "\n";
+    amrex::Print() << name_space << "used      (MB): " << actual_min_megabytes << "\n";
 #endif
 }