Energy profiling tools: NVML-based measurement tool

CMakeLists.txt

@@ -103,6 +103,23 @@ else()
   set(KOKKOSTOOLS_HAS_VTUNE OFF)
 endif()
 
+# Check for NVML (required for energy profiler)
+set(MIN_CUDA_VERSION 12.6)
+find_package(CUDAToolkit ${MIN_CUDA_VERSION} QUIET)
+if (CUDAToolkit_FOUND)
+    find_package(CUDA::nvml QUIET)
+    if(TARGET CUDA::nvml)
+        message(STATUS "Found CUDA NVML (version ${CUDAToolkit_VERSION}), energy profiler will be built")
+        set(KOKKOSTOOLS_HAS_NVML ON)
+    else()
+        message(STATUS "CUDA::nvml target not found, energy profiler will be skipped")
+        set(KOKKOSTOOLS_HAS_NVML OFF)
+    endif()
+else()
+    message(STATUS "CUDAToolkit ${MIN_CUDA_VERSION} or higher not found, energy profiler will be skipped")
+    set(KOKKOSTOOLS_HAS_NVML OFF)
+endif()
+
 # make Kokkos profiling interface available for native profilers
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/profiling/all)
 
@@ -150,6 +167,11 @@ if(NOT WIN32)
   add_subdirectory(profiling/chrome-tracing)
   add_subdirectory(profiling/space-time-stack)
   add_subdirectory(profiling/perfetto-connector)
+  if(KOKKOSTOOLS_HAS_NVML)
+    add_subdirectory(profiling/energy-profiler)
+  else()
+    message(STATUS "Skipping energy-profiler (NVML not available)")
+  endif()
 endif()
 
 # External lib connectors

profiling/energy-profiler/CMakeLists.txt

@@ -0,0 +1,11 @@
+kp_add_library(kp_energy_profiler
+    kp_energy_profiler.cpp
+    timing_utils.cpp
+    timing_export.cpp
+    nvml_provider.cpp
+    power_sampler.cpp
+    daemon.cpp
+)
+
+target_link_libraries(kp_energy_profiler PRIVATE CUDA::nvml)
+target_compile_definitions(kp_energy_profiler PRIVATE KOKKOS_ENERGY_PROFILER_HAS_NVML)

profiling/energy-profiler/daemon.cpp

@@ -0,0 +1,45 @@
+//@HEADER
+// ************************************************************************
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
+// See https://kokkos.org/LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//@HEADER
+
+#include "daemon.hpp"
+#include <stdexcept>
+#include <thread>
+
+void Daemon::start() {
+  if (!running_) {
+    running_ = true;
+    thread_  = std::thread(&Daemon::run, this);
+  } else {
+    throw std::runtime_error("Daemon already started");
+  }
+}
+
+void Daemon::run() {
+  while (running_) {
+    auto next_run = std::chrono::high_resolution_clock::now() + interval_;
+    func_();
+    std::this_thread::sleep_until(next_run);
+  }
+}
+
+void Daemon::stop() {
+  if (running_) {
+    running_ = false;
+    thread_.join();
+  } else {
+    throw std::runtime_error("Daemon not started");
+  }
+}

profiling/energy-profiler/daemon.hpp

@@ -0,0 +1,39 @@
+//@HEADER
+// ************************************************************************
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
+// See https://kokkos.org/LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//@HEADER
+
+#pragma once
+
+#include <functional>
+#include <thread>
+#include <chrono>
+
+class Daemon {
+ public:
+  Daemon(std::function<void()> func, int interval_ms)
+      : interval_(interval_ms), func_(func){};
+
+  void start();
+  void run();
+  void stop();
+  bool is_running() const { return running_; }
+  std::thread& get_thread() { return thread_; }
+
+ private:
+  std::chrono::milliseconds interval_;
+  bool running_{false};
+  std::function<void()> func_;
+  std::thread thread_;
+};

profiling/energy-profiler/energy_profiler_constants.hpp

@@ -0,0 +1,38 @@
+//@HEADER
+// ************************************************************************
+//
+//                        Kokkos v. 4.0
+//       Copyright (2022) National Technology & Engineering
+//               Solutions of Sandia, LLC (NTESS).
+//
+// Under the terms of Contract DE-NA0003525 with NTESS,
+// the U.S. Government retains certain rights in this software.
+//
+// Part of Kokkos, under the Apache License v2.0 with LLVM Exceptions.
+// See https://kokkos.org/LICENSE for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//@HEADER
+
+#pragma once
+
+#include <cstddef>
+
+namespace KokkosTools {
+namespace EnergyProfiler {
+
+// Sampling interval in milliseconds
+constexpr int SAMPLING_INTERVAL_MS = 20;
+
+// Buffer size for hostname
+const size_t HOSTNAME_BUFFER_SIZE = 256;
+
+// Table formatting constants for timing export
+const int COLUMN_WIDTH_CATEGORY = 10;
+const int COLUMN_WIDTH_NAME     = 32;
+const int COLUMN_WIDTH_TYPE     = 14;
+const int COLUMN_WIDTH_TIME     = 17;
+const int COLUMN_WIDTH_DURATION = 13;
+
+}  // namespace EnergyProfiler
+}  // namespace KokkosTools