Skip to content

Print max CArena usage #4468

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion Src/Base/AMReX_Arena.H
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ public:
static std::size_t align (std::size_t sz);

static void Initialize (bool minimal);
static void PrintUsage ();
static void PrintUsage (bool print_max_usage=false);
static void PrintUsageToFiles (std::string const& filename, std::string const& message);
static void Finalize ();

Expand All @@ -210,6 +210,8 @@ public:
*/
[[nodiscard]] const ArenaInfo& arenaInfo () const { return arena_info; }

virtual void ResetMaxUsageCounter () {}

protected:

ArenaInfo arena_info;
Expand Down
19 changes: 12 additions & 7 deletions Src/Base/AMReX_Arena.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,7 @@ Arena::Initialize (bool minimal)
BL_PROFILE("The_Arena::Initialize()");
void *p = the_arena->alloc(static_cast<std::size_t>(the_arena_init_size));
the_arena->free(p);
the_arena->ResetMaxUsageCounter();
#endif
#else
the_arena = The_BArena();
Expand Down Expand Up @@ -406,25 +407,29 @@ Arena::Initialize (bool minimal)
BL_PROFILE("The_Device_Arena::Initialize()");
void *p = the_device_arena->alloc(the_device_arena_init_size);
the_device_arena->free(p);
the_device_arena->ResetMaxUsageCounter();
}

if (the_managed_arena_init_size > 0 && the_managed_arena != the_arena) {
BL_PROFILE("The_Managed_Arena::Initialize()");
void *p = the_managed_arena->alloc(the_managed_arena_init_size);
the_managed_arena->free(p);
the_managed_arena->ResetMaxUsageCounter();
}

if (the_pinned_arena_init_size > 0) {
BL_PROFILE("The_Pinned_Arena::Initialize()");
void *p = the_pinned_arena->alloc(the_pinned_arena_init_size);
the_pinned_arena->free(p);
the_pinned_arena->ResetMaxUsageCounter();
}

if (the_comms_arena_init_size > 0 && the_comms_arena != the_arena
&& the_comms_arena != the_device_arena && the_comms_arena != the_pinned_arena) {
BL_PROFILE("The_Comms_Arena::Initialize()");
void *p = the_comms_arena->alloc(the_comms_arena_init_size);
the_comms_arena->free(p);
the_comms_arena->ResetMaxUsageCounter();
}

the_cpu_arena = The_BArena();
Expand All @@ -436,7 +441,7 @@ Arena::Initialize (bool minimal)
}

void
Arena::PrintUsage ()
Arena::PrintUsage (bool print_max_usage)
{
#ifdef AMREX_USE_GPU
const int IOProc = ParallelDescriptor::IOProcessorNumber();
Expand Down Expand Up @@ -468,32 +473,32 @@ Arena::PrintUsage ()
if (The_Arena()) {
auto* p = dynamic_cast<CArena*>(The_Arena());
if (p) {
p->PrintUsage("The Arena");
p->PrintUsage("The Arena", print_max_usage);
}
}
if (The_Device_Arena() && The_Device_Arena() != The_Arena()) {
auto* p = dynamic_cast<CArena*>(The_Device_Arena());
if (p) {
p->PrintUsage("The Device Arena");
p->PrintUsage("The Device Arena", print_max_usage);
}
}
if (The_Managed_Arena() && The_Managed_Arena() != The_Arena()) {
auto* p = dynamic_cast<CArena*>(The_Managed_Arena());
if (p) {
p->PrintUsage("The Managed Arena");
p->PrintUsage("The Managed Arena", print_max_usage);
}
}
if (The_Pinned_Arena()) {
auto* p = dynamic_cast<CArena*>(The_Pinned_Arena());
if (p) {
p->PrintUsage("The Pinned Arena");
p->PrintUsage("The Pinned Arena", print_max_usage);
}
}
if (The_Comms_Arena() && The_Comms_Arena() != The_Device_Arena()
&& The_Comms_Arena() != The_Pinned_Arena()) {
auto* p = dynamic_cast<CArena*>(The_Comms_Arena());
if (p) {
p->PrintUsage("The Comms Arena");
p->PrintUsage("The Comms Arena", print_max_usage);
}
}
}
Expand Down Expand Up @@ -560,7 +565,7 @@ Arena::Finalize ()
#else
if (amrex::Verbose() > 1) {
#endif
PrintUsage();
PrintUsage(true);
}

initialized = false;
Expand Down
10 changes: 9 additions & 1 deletion Src/Base/AMReX_CArena.H
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,14 @@ public:
//! Return the amount of memory in this pointer. Return 0 for unknown pointer.
std::size_t sizeOf (void* p) const noexcept;

void PrintUsage (std::string const& name) const;
void PrintUsage (std::string const& name, bool print_max_usage) const;

void PrintUsage (std::ostream& os, std::string const& name, std::string const& space) const;

void ResetMaxUsageCounter () final {
m_max_actually_used = 0;
}

//! The default memory hunk size to grab from the heap.
constexpr static std::size_t DefaultHunkSize = 1024*1024*8;

Expand Down Expand Up @@ -181,8 +185,12 @@ protected:
std::size_t m_hunk;
//! The amount of heap space currently allocated.
std::size_t m_used{0};
//! The max amount of heap space currently allocated.
std::size_t m_max_used{0};
//! The amount of memory given out via alloc().
std::size_t m_actually_used{0};
//! The max amount of memory given out via alloc().
std::size_t m_max_actually_used{0};


std::mutex carena_mutex;
Expand Down
22 changes: 15 additions & 7 deletions Src/Base/AMReX_CArena.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ CArena::alloc_protected (std::size_t nbytes)
vp = allocate_system(N);

m_used += N;
m_max_used = std::max(m_used, m_max_used);

m_alloc.emplace_back(vp,N);

Expand Down Expand Up @@ -116,6 +117,7 @@ CArena::alloc_protected (std::size_t nbytes)
}

m_actually_used += nbytes;
m_max_actually_used = std::max(m_actually_used, m_max_actually_used);

BL_ASSERT(vp != nullptr);

Expand Down Expand Up @@ -166,6 +168,7 @@ CArena::alloc_in_place (void* pt, std::size_t szmin, std::size_t szmax)
}
#endif
m_actually_used += new_size - busy_it->size();
m_max_actually_used = std::max(m_actually_used, m_max_actually_used);
const_cast<Node&>(*busy_it).size(new_size);
return std::make_pair(pt, new_size);
} else if (total_size >= szmin) {
Expand All @@ -179,6 +182,7 @@ CArena::alloc_in_place (void* pt, std::size_t szmin, std::size_t szmax)
}
#endif
m_actually_used += total_size - busy_it->size();
m_max_actually_used = std::max(m_actually_used, m_max_actually_used);
const_cast<Node&>(*busy_it).size(total_size);
return std::make_pair(pt, total_size);
}
Expand Down Expand Up @@ -453,25 +457,29 @@ CArena::sizeOf (void* p) const noexcept
}

void
CArena::PrintUsage (std::string const& name) const
CArena::PrintUsage (std::string const& name, bool print_max_usage) const
{
Long min_megabytes = static_cast<Long>(heap_space_used() / (1024*1024));
Long min_megabytes = static_cast<Long>(
(print_max_usage ? m_max_used : heap_space_used()) / (1024*1024));
Long max_megabytes = min_megabytes;
Long actual_min_megabytes = static_cast<Long>(heap_space_actually_used() / (1024*1024));
Long actual_min_megabytes = static_cast<Long>(
(print_max_usage ? m_max_actually_used : heap_space_actually_used()) / (1024*1024));
Long actual_max_megabytes = actual_min_megabytes;
const int IOProc = ParallelDescriptor::IOProcessorNumber();
ParallelReduce::Min<Long>({min_megabytes, actual_min_megabytes},
IOProc, ParallelDescriptor::Communicator());
ParallelReduce::Max<Long>({max_megabytes, actual_max_megabytes},
IOProc, ParallelDescriptor::Communicator());

const auto name_space = "[" + name + "] " + (print_max_usage ? "max " : "") + "space ";
#ifdef AMREX_USE_MPI
amrex::Print() << "[" << name << "] space (MB) allocated spread across MPI: ["
amrex::Print() << name_space << "(MB) allocated spread across MPI: ["
<< min_megabytes << " ... " << max_megabytes << "]\n"
<< "[" << name << "] space (MB) used spread across MPI: ["
<< name_space << "(MB) used spread across MPI: ["
<< actual_min_megabytes << " ... " << actual_max_megabytes << "]\n";
#else
amrex::Print() << "[" << name << "] space allocated (MB): " << min_megabytes << "\n";
amrex::Print() << "[" << name << "] space used (MB): " << actual_min_megabytes << "\n";
amrex::Print() << name_space << "allocated (MB): " << min_megabytes << "\n";
amrex::Print() << name_space << "used (MB): " << actual_min_megabytes << "\n";
#endif
}

Expand Down
Loading