From 0faa2e55c17de9a1796d1d9aedb4d298dd206261 Mon Sep 17 00:00:00 2001 From: David Boehme Date: Thu, 29 Jan 2026 15:18:11 -0800 Subject: [PATCH 1/3] Remove roctracer from ConfigManager configs --- .../ROCmActivityProfileController.cpp | 19 +++++++-------- .../ROCmActivityReportController.cpp | 24 +++++++++---------- src/caliper/controllers/controllers.cpp | 8 +++---- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/src/caliper/controllers/ROCmActivityProfileController.cpp b/src/caliper/controllers/ROCmActivityProfileController.cpp index 2452a17a..729ff24c 100644 --- a/src/caliper/controllers/ROCmActivityProfileController.cpp +++ b/src/caliper/controllers/ROCmActivityProfileController.cpp @@ -42,16 +42,16 @@ class RocmActivityProfileController : public cali::ChannelController } std::string q_local = - " let act_count=first(sum#count,count) if rocm.activity" - ",dmin=scale(min#rocm.activity.duration,1e-9)" + " let " + " dmin=scale(min#rocm.activity.duration,1e-9)" ",davg=scale(avg#rocm.activity.duration,1e-9)" ",dmax=scale(max#rocm.activity.duration,1e-9)" - " select *,scale(sum#time.duration.ns,1e-9) as time" + " select *,scale(sum#rocm.host.duration,1e-9) as time" ",scale(sum#rocm.activity.duration,1e-9) as \"time (gpu)\"" ",min(dmin) as \"min time/inst\"" ",avg(davg) as \"avg time/inst\"" ",max(dmax) as \"max time/inst\"" - ",sum(act_count) as count" + ",sum(sum#rocm.activity.count) as instances" " group by path,rocm.kernel.name,rocm.activity,mpi.rank " " format "; @@ -127,14 +127,13 @@ const char* controller_spec = R"json( "name" : "rocm-activity-profile", "description" : "Record AMD ROCm activities and a write profile", "categories" : [ "adiak", "metric", "output", "region", "event" ], - "services" : [ "aggregate", "roctracer", "event", "timer" ], + "services" : [ "aggregate", "rocprofiler", "event" ], "config" : { - "CALI_CHANNEL_FLUSH_ON_EXIT" : "false", - "CALI_EVENT_ENABLE_SNAPSHOT_INFO" : "false", - "CALI_ROCTRACER_TRACE_ACTIVITIES" : "true", - "CALI_ROCTRACER_RECORD_KERNEL_NAMES": "true", - "CALI_ROCTRACER_SNAPSHOT_DURATION" : "false" + "CALI_CHANNEL_FLUSH_ON_EXIT": "false", + "CALI_EVENT_ENABLE_SNAPSHOT_INFO": "false", + "CALI_ROCPROFILER_ENABLE_ACTIVITY_TRACING": "true", + "CALI_ROCPROFILER_ENABLE_SNAPSHOT_TIMESTAMPS": "true" }, "defaults" : { "node.order": "true", "memcpy": "true" }, "options": diff --git a/src/caliper/controllers/ROCmActivityReportController.cpp b/src/caliper/controllers/ROCmActivityReportController.cpp index 5dc19ddf..ccabf12c 100644 --- a/src/caliper/controllers/ROCmActivityReportController.cpp +++ b/src/caliper/controllers/ROCmActivityReportController.cpp @@ -35,17 +35,17 @@ class RocmActivityReportController : public cali::ChannelController { // Config for local aggregation std::string local_select = - " inclusive_scale(sum#time.duration.ns,1e-9) as \"Host Time\"" + " inclusive_scale(sum#rocm.host.duration,1e-9) as \"Host Time\"" ",inclusive_scale(sum#rocm.activity.duration,1e-9) as \"GPU Time\"" - ",inclusive_ratio(sum#rocm.activity.duration,sum#time.duration.ns,100.0) as \"GPU %\""; + ",inclusive_ratio(sum#rocm.activity.duration,sum#rocm.host.duration,100.0) as \"GPU %\""; // Config for second aggregation step in MPI mode (cross-process aggregation) std::string cross_select = - " avg(iscale#sum#time.duration.ns) as \"Avg Host Time\"" - ",max(iscale#sum#time.duration.ns) as \"Max Host Time\"" + " avg(iscale#sum#rocm.host.duration) as \"Avg Host Time\"" + ",max(iscale#sum#rocm.host.duration) as \"Max Host Time\"" ",avg(iscale#sum#rocm.activity.duration) as \"Avg GPU Time\"" ",max(iscale#sum#rocm.activity.duration) as \"Max GPU Time\"" - ",ratio(iscale#sum#rocm.activity.duration,iscale#sum#time.duration.ns,100.0) as \"GPU %\""; + ",ratio(iscale#sum#rocm.activity.duration,iscale#sum#rocm.host.duration,100.0) as \"GPU %\""; std::string groupby = "path"; @@ -119,15 +119,16 @@ const char* controller_spec = R"json( { "name" : "rocm-activity-report", "description" : "Record and print AMD ROCm activities (kernel executions, memcopies, etc.)", - "categories" : [ "output", "region", "treeformatter", "event" ], - "services" : [ "aggregate", "roctracer", "event", "timer" ], + "categories" : [ "output", "region", "treeformatter", "event", "metric" ], + "services" : [ "aggregate", "rocprofiler", "event" ], "config" : { - "CALI_CHANNEL_FLUSH_ON_EXIT" : "false", - "CALI_EVENT_ENABLE_SNAPSHOT_INFO" : "false", - "CALI_ROCTRACER_TRACE_ACTIVITIES" : "true" + "CALI_CHANNEL_FLUSH_ON_EXIT" : "false", + "CALI_EVENT_ENABLE_SNAPSHOT_INFO" : "false", + "CALI_ROCPROFILER_ENABLE_ACTIVITY_TRACING" : "true", + "CALI_ROCPROFILER_ENABLE_SNAPSHOT_TIMESTAMPS" : "true" }, - "defaults" : { "order_as_visited": "true", "output.append": "true" }, + "defaults": { "order_as_visited": "true", "output.append": "true" }, "options": [ { @@ -137,7 +138,6 @@ const char* controller_spec = R"json( },{ "name": "show_kernels", "type": "bool", - "config": { "CALI_ROCTRACER_RECORD_KERNEL_NAMES": "true" }, "description": "Show kernel names" },{ "name": "output.append", diff --git a/src/caliper/controllers/controllers.cpp b/src/caliper/controllers/controllers.cpp index f322c600..04563a10 100644 --- a/src/caliper/controllers/controllers.cpp +++ b/src/caliper/controllers/controllers.cpp @@ -701,16 +701,16 @@ const char* builtin_rocm_option_specs = R"json( "type": "bool", "description": "Profile HIP API functions", "category": "region", - "services": [ "roctracer" ], - "config": { "CALI_ROCTRACER_TRACE_ACTIVITIES": "false" } + "services": [ "rocprofiler" ], + "config": { "CALI_ROCPROFILER_ENABLE_API_CALLBACKS": "true" } }, { "name": "rocm.gputime", "description": "Report GPU time in AMD ROCm activities", "type": "bool", "category": "metric", - "services": [ "roctracer" ], - "config": { "CALI_ROCTRACER_TRACE_ACTIVITIES": "true", "CALI_ROCTRACER_RECORD_KERNEL_NAMES": "false" }, + "services": [ "rocprofiler" ], + "config": { "CALI_ROCPROFILER_ENABLE_ACTIVITY_TRACING": "true" }, "query": { "local": "let t.gpu.r=first(sum#rocm.activity.duration,rocm.activity.duration) select inclusive_scale(t.gpu.r,1e-9) as \"GPU time (I)\" unit sec,scale(t.gpu.r,1e-9) as \"GPU time (E)\" unit sec", From 972208c9cbe932993526eb83bfafa92f1176d22d Mon Sep 17 00:00:00 2001 From: David Boehme Date: Thu, 29 Jan 2026 15:32:43 -0800 Subject: [PATCH 2/3] Remove rocprofiler's roctracer compatibility mode --- src/services/rocprofiler/CMakeLists.txt | 1 - src/services/rocprofiler/RocProfiler.cpp | 55 ------------------------ 2 files changed, 56 deletions(-) diff --git a/src/services/rocprofiler/CMakeLists.txt b/src/services/rocprofiler/CMakeLists.txt index 55c20fbb..f815fcb4 100644 --- a/src/services/rocprofiler/CMakeLists.txt +++ b/src/services/rocprofiler/CMakeLists.txt @@ -10,5 +10,4 @@ target_link_libraries(caliper-rocprofiler PRIVATE add_service_objlib("caliper-rocprofiler") add_caliper_service("rocprofiler CALIPER_HAVE_ROCPROFILER") -add_caliper_service("roctracer CALIPER_HAVE_ROCPROFILER") add_caliper_service("roctx CALIPER_HAVE_ROCPROFILER") diff --git a/src/services/rocprofiler/RocProfiler.cpp b/src/services/rocprofiler/RocProfiler.cpp index 473b3fc9..76631eea 100644 --- a/src/services/rocprofiler/RocProfiler.cpp +++ b/src/services/rocprofiler/RocProfiler.cpp @@ -711,7 +711,6 @@ class RocProfilerService public: static const char* s_spec; - static const char* s_roctracer_spec; static int tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data) { @@ -828,27 +827,6 @@ class RocProfilerService << " Activity tracing is " << (s_instance->m_enable_activity_tracing ? "on" : "off") << std::endl; } - - static void register_rocprofiler_as_roctracer(Caliper* c, Channel* channel) - { - // a compatibility layer to convert roctracer options into rocprofiler options - - Log(1).stream() << channel->name() << ": rocprofiler: Using roctracer compatibility layer.\n"; - - auto config = services::init_config_from_spec(channel->config(), s_roctracer_spec); - - bool enable_activity_tracing = config.get("trace_activities").to_bool(); - bool enable_snapshot_timestamps = - config.get("snapshot_duration").to_bool() || config.get("snapshot_timestamps").to_bool(); - - channel->config().set("CALI_ROCPROFILER_ENABLE_ACTIVITY_TRACING", enable_activity_tracing ? "true" : "false"); - channel->config().set( - "CALI_ROCPROFILER_ENABLE_SNAPSHOT_TIMESTAMPS", - enable_snapshot_timestamps ? "true" : "false" - ); - - register_rocprofiler(c, channel); - } }; RocProfilerService* RocProfilerService::s_instance = nullptr; @@ -895,36 +873,6 @@ const char* RocProfilerService::s_spec = R"json( } )json"; -const char* RocProfilerService::s_roctracer_spec = R"json( -{ - "name": "roctracer", - "description": "roctracer compatibility layer for rocprofiler service (deprecated)", - "config": - [ - { "name": "trace_activities", - "type": "bool", - "description": "Enable ROCm GPU activity tracing", - "value": "true" - }, - { "name": "record_kernel_names", - "type": "bool", - "description": "Record kernel names when activity tracing is enabled", - "value": "false" - }, - { "name": "snapshot_duration", - "type": "bool", - "description": "Record duration of host-side activities using ROCm timestamps", - "value": "false" - }, - { "name": "snapshot_timestamps", - "type": "bool", - "description": "Record host-side timestamps with ROCm", - "value": "false" - } - ] -} -)json"; - } // namespace extern "C" @@ -954,7 +902,4 @@ namespace cali CaliperService rocprofiler_service { ::RocProfilerService::s_spec, ::RocProfilerService::register_rocprofiler }; -CaliperService roctracer_service { ::RocProfilerService::s_roctracer_spec, - ::RocProfilerService::register_rocprofiler_as_roctracer }; - } // namespace cali \ No newline at end of file From e5816ba00c45c0ad204e93ef69d4bca58aa32979 Mon Sep 17 00:00:00 2001 From: David Boehme Date: Thu, 29 Jan 2026 15:46:40 -0800 Subject: [PATCH 3/3] Remove pre-rocprofiler roctracer and roctx services --- CMakeLists.txt | 77 +--- cmake/FindRocTX.cmake | 28 -- cmake/FindRoctracer.cmake | 61 --- src/services/CMakeLists.txt | 6 - src/services/roctracer/CMakeLists.txt | 9 - src/services/roctracer/RocTracer.cpp | 592 -------------------------- src/services/roctx/CMakeLists.txt | 8 - src/services/roctx/RocTX.cpp | 106 ----- 8 files changed, 22 insertions(+), 865 deletions(-) delete mode 100644 cmake/FindRocTX.cmake delete mode 100644 cmake/FindRoctracer.cmake delete mode 100644 src/services/roctracer/CMakeLists.txt delete mode 100644 src/services/roctracer/RocTracer.cpp delete mode 100644 src/services/roctx/CMakeLists.txt delete mode 100644 src/services/roctx/RocTX.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 6fb0b393..b7a390c8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -60,30 +60,28 @@ add_caliper_option(WITH_FORTRAN "Install Fortran interface" FALSE) add_caliper_option(WITH_PYTHON_BINDINGS "Install Python bindings" FALSE) add_caliper_option(WITH_TOOLS "Build Caliper tools" TRUE) -add_caliper_option(WITH_NVTX "Enable NVidia nvtx bindings for NVprof and NSight (requires CUDA)" FALSE) -add_caliper_option(WITH_CUPTI "Enable CUPTI service (CUDA performance analysis)" FALSE) -add_caliper_option(WITH_PAPI "Enable PAPI hardware counter service (requires papi)" FALSE) -add_caliper_option(WITH_LIBPFM "Enable libpfm (perf_event) sampling" FALSE) -add_caliper_option(WITH_LIBDW "Enable libdw support (for symbollookup service)" FALSE) -add_caliper_option(WITH_LIBUNWIND "Enable libunwind support (for callpath service)" FALSE) -add_caliper_option(WITH_MPI "Enable MPI" FALSE) -# add_caliper_option(WITH_MPIT "Enable MPI-T" FALSE) -add_caliper_option(WITH_OMPT "Enable OMPT" FALSE) -add_caliper_option(WITH_SAMPLER "Enable Linux sampler (x86 and PPC Linux only)" FALSE) -add_caliper_option(WITH_GOTCHA "Enable GOTCHA wrapping" ${CALIPER_HAVE_LINUX}) -add_caliper_option(WITH_ROCTX "Enable AMD RocTX support" FALSE) -add_caliper_option(WITH_ROCTRACER "Enable AMD RocTracer support" FALSE) -add_caliper_option(WITH_ROCM "Enable AMD RocTracer and RocTX support" FALSE) +add_caliper_option(WITH_NVTX "Enable NVidia nvtx bindings for NVprof and NSight (requires CUDA)" FALSE) +add_caliper_option(WITH_CUPTI "Enable CUPTI service (CUDA performance analysis)" FALSE) +add_caliper_option(WITH_PAPI "Enable PAPI hardware counter service (requires papi)" FALSE) +add_caliper_option(WITH_LIBPFM "Enable libpfm (perf_event) sampling" FALSE) +add_caliper_option(WITH_LIBDW "Enable libdw support (for symbollookup service)" FALSE) +add_caliper_option(WITH_LIBUNWIND "Enable libunwind support (for callpath service)" FALSE) +add_caliper_option(WITH_MPI "Enable MPI" FALSE) +# add_caliper_option(WITH_MPIT "Enable MPI-T" FALSE) +add_caliper_option(WITH_OMPT "Enable OMPT" FALSE) +add_caliper_option(WITH_SAMPLER "Enable Linux sampler (x86 and PPC Linux only)" FALSE) +add_caliper_option(WITH_GOTCHA "Enable GOTCHA wrapping" ${CALIPER_HAVE_LINUX}) +add_caliper_option(WITH_ROCM "Enable AMD ROCm profiling support (identical to WITH_ROCPROFILER)" FALSE) add_caliper_option(WITH_ROCPROFILER "Enable AMD rocprofiler-sdk support" FALSE) -add_caliper_option(WITH_TAU "Enable TAU service (TAU Performance System)" FALSE) -add_caliper_option(WITH_VTUNE "Enable Intel(R) VTune(tm) annotation bindings" FALSE) -add_caliper_option(WITH_ADIAK "Enable adiak support" FALSE) -add_caliper_option(WITH_KOKKOS "Enable Kokkos profiling support" TRUE) -add_caliper_option(WITH_PCP "Enable performance co-pilot support" FALSE) -add_caliper_option(WITH_VARIORUM "Enable Variorum support" FALSE) -add_caliper_option(WITH_UMPIRE "Enable Umpire statistics support" FALSE) -add_caliper_option(WITH_CRAYPAT "Enable CrayPAT region forwarding support" FALSE) -add_caliper_option(WITH_LDMS "Enable LDMS forwarder" FALSE) +add_caliper_option(WITH_TAU "Enable TAU service (TAU Performance System)" FALSE) +add_caliper_option(WITH_VTUNE "Enable Intel(R) VTune(tm) annotation bindings" FALSE) +add_caliper_option(WITH_ADIAK "Enable adiak support" FALSE) +add_caliper_option(WITH_KOKKOS "Enable Kokkos profiling support" TRUE) +add_caliper_option(WITH_PCP "Enable performance co-pilot support" FALSE) +add_caliper_option(WITH_VARIORUM "Enable Variorum support" FALSE) +add_caliper_option(WITH_UMPIRE "Enable Umpire statistics support" FALSE) +add_caliper_option(WITH_CRAYPAT "Enable CrayPAT region forwarding support" FALSE) +add_caliper_option(WITH_LDMS "Enable LDMS forwarder" FALSE) set(WITH_ARCH "" CACHE STRING "Enable features specific to the provided archspec CPU architecture name") if (NOT WITH_ARCH STREQUAL "") @@ -338,36 +336,7 @@ if (WITH_MPI) endif() endif() -if (WITH_ROCPROFILER AND (WITH_ROCTX OR WITH_ROCTRACER)) - message(WARNING "ROCPROFILER and ROCTX/ROCTRACER support are both enabled.\n" - "This is not recommended, please select either ROCPROFILER or ROCTRACER/ROCTX") -endif() - -if (WITH_ROCTX OR WITH_ROCM) - include(FindRocTX) - if (ROCTX_FOUND) - set(CALIPER_HAVE_ROCTX TRUE) - set(CALIPER_RocTX_CMAKE_MSG "Yes, using ${ROCTX_LIBRARIES}") - list(APPEND CALIPER_EXTERNAL_LIBS ${ROCTX_LIBRARIES}) - else() - message(WARNING "RocTX support requested but ROCm was not found!\n" - "Set ROCM_PATH to installation path and re-run cmake.") - endif() -endif() - -if (WITH_ROCTRACER OR WITH_ROCM) - include(FindRoctracer) - if (ROCTRACER_FOUND) - set(CALIPER_HAVE_ROCTRACER TRUE) - set(CALIPER_RocTracer_CMAKE_MSG "Yes, using ${ROCTRACER_LIBRARIES}") - list(APPEND CALIPER_EXTERNAL_LIBS ${ROCTRACER_LIBRARIES}) - else() - message(WARNING "RocTracer support requested but ROCm was not found!\n" - "Set ROCM_PATH to installation path and re-run cmake.") - endif() -endif() - -if (WITH_ROCPROFILER) +if (WITH_ROCPROFILER OR WITH_ROCM) find_package(rocprofiler-sdk REQUIRED) find_package(rocprofiler-sdk-roctx REQUIRED) if (rocprofiler-sdk_FOUND) @@ -641,8 +610,6 @@ set(CALIPER_MODULES Nvtx CUpti Kokkos - RocTracer - RocTX rocprofiler TAU VTune diff --git a/cmake/FindRocTX.cmake b/cmake/FindRocTX.cmake deleted file mode 100644 index 1b46a1fa..00000000 --- a/cmake/FindRocTX.cmake +++ /dev/null @@ -1,28 +0,0 @@ -# Find RocTX libraries/headers - -find_path(ROCM_PATH - NAMES include/roctracer/roctx.h -) - -find_library(ROCTX_LIBRARIES - NAMES roctx64 - HINTS ${ROCM_PATH}/lib -) - -find_path(ROCTX_INCLUDE_DIRS - NAMES roctx.h - HINTS ${ROCM_PATH}/include/roctracer -) - -include(FindPackageHandleStandardArgs) - -find_package_handle_standard_args(ROCTX - DEFAULT_MSG - ROCTX_LIBRARIES - ROCTX_INCLUDE_DIRS -) - -mark_as_advanced( - ROCTX_INCLUDE_DIRS - ROCTX_LIBRARIES -) diff --git a/cmake/FindRoctracer.cmake b/cmake/FindRoctracer.cmake deleted file mode 100644 index 83c3751b..00000000 --- a/cmake/FindRoctracer.cmake +++ /dev/null @@ -1,61 +0,0 @@ -# Find ROCTracer libraries/headers - -find_path(ROCM_PATH - NAMES include/roctracer/roctracer.h -) - -find_library(ROCTRACER_LIBROCTRACER - NAMES roctracer64 - HINTS ${ROCM_PATH}/lib -) -find_library(ROCTRACER_LIBHSARUNTIME - NAMES hsa-runtime64 - HINTS ${ROCM_PATH}/lib -) -find_library(ROCTRACER_LIBHSAKMT - NAMES hsakmt - HINTS ${ROCM_PATH}/lib -) -find_library(ROCTRACER_LIBHSAKMT - NAMES hsakmt - HINTS ${ROCM_PATH}/lib -) -find_library(ROCTRACER_AMDHIP64 - NAMES amdhip64 - HINTS ${ROCM_PATH}/lib -) - -find_path(ROCTRACER_INCLUDE_DIR - NAMES roctracer.h - HINTS ${ROCM_PATH}/include/roctracer -) - -find_path(HIP_INCLUDE_DIR - NAMES hip/hip_runtime.h - HINTS ${ROCM_PATH}/include) - -set(ROCTRACER_INCLUDE_DIRS - ${ROCTRACER_INCLUDE_DIR} - ${HIP_INCLUDE_DIR} -) -set(ROCTRACER_LIBRARIES - ${ROCTRACER_LIBROCTRACER} - ${ROCTRACER_LIBHSARUNTIME} - ${ROCTRACER_LIBHSAKMT} - ${ROCTRACER_AMDHIP64} -) - -include(FindPackageHandleStandardArgs) - -find_package_handle_standard_args(ROCTRACER - DEFAULT_MSG - ROCTRACER_LIBROCTRACER - ROCTRACER_LIBHSARUNTIME - ROCTRACER_LIBHSAKMT - ROCTRACER_INCLUDE_DIRS -) - -mark_as_advanced( - ROCTRACER_INCLUDE_DIRS - ROCTRACER_LIBRARIES -) diff --git a/src/services/CMakeLists.txt b/src/services/CMakeLists.txt index ce29701f..0dfaf84d 100644 --- a/src/services/CMakeLists.txt +++ b/src/services/CMakeLists.txt @@ -95,12 +95,6 @@ endif() if (CALIPER_HAVE_PCP) add_subdirectory(pcp) endif() -if (CALIPER_HAVE_ROCTX) - add_subdirectory(roctx) -endif() -if (CALIPER_HAVE_ROCTRACER) - add_subdirectory(roctracer) -endif() if (CALIPER_HAVE_ROCPROFILER) add_subdirectory(rocprofiler) endif() diff --git a/src/services/roctracer/CMakeLists.txt b/src/services/roctracer/CMakeLists.txt deleted file mode 100644 index 5a6a3cd2..00000000 --- a/src/services/roctracer/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -set(CALIPER_ROCTRACER_SOURCES - RocTracer.cpp) - -add_library(caliper-roctracer OBJECT ${CALIPER_ROCTRACER_SOURCES}) -target_compile_definitions(caliper-roctracer PRIVATE __HIP_PLATFORM_AMD__) -target_include_directories(caliper-roctracer PRIVATE ${ROCTRACER_INCLUDE_DIRS}) - -add_service_objlib("caliper-roctracer") -add_caliper_service("roctracer CALIPER_HAVE_ROCTRACER") diff --git a/src/services/roctracer/RocTracer.cpp b/src/services/roctracer/RocTracer.cpp deleted file mode 100644 index 8a67c4aa..00000000 --- a/src/services/roctracer/RocTracer.cpp +++ /dev/null @@ -1,592 +0,0 @@ -// Copyright (c) 2015-2022, Lawrence Livermore National Security, LLC. -// See top-level LICENSE file for details. - -#include "../Services.h" - -#include "caliper/Caliper.h" -#include "caliper/SnapshotRecord.h" - -#include "caliper/common/Attribute.h" -#include "caliper/common/Log.h" - -#include "../../common/util/unitfmt.h" -#include "../../common/util/demangle.h" - -#include -#include -#include - -#include -#include - -using namespace cali; - -namespace -{ - -class RocTracerService -{ - Attribute m_api_attr; - - Attribute m_activity_start_attr; - Attribute m_activity_end_attr; - Attribute m_activity_duration_attr; - Attribute m_activity_name_attr; - Attribute m_activity_queue_id_attr; - Attribute m_activity_device_id_attr; - Attribute m_activity_bytes_attr; - Attribute m_kernel_name_attr; - - Attribute m_host_starttime_attr; - Attribute m_host_duration_attr; - Attribute m_host_timestamp_attr; - - Attribute m_flush_region_attr; - - unsigned m_num_records; - unsigned m_num_flushed; - unsigned m_num_flushes; - - unsigned m_num_correlations_stored; - unsigned m_num_correlations_found; - unsigned m_num_correlations_missed; - - std::mutex m_correlation_map_mutex; - std::map m_correlation_map; - - roctracer_pool_t* m_roctracer_pool; - - Channel m_channel; - - bool m_enable_tracing; - bool m_record_names; - bool m_record_host_duration; - bool m_record_host_timestamp; - - static RocTracerService* s_instance; - - void create_callback_attributes(Caliper* c) - { - Attribute subs_attr = c->get_attribute("subscription_event"); - Variant v_true(true); - - m_api_attr = c->create_attribute("rocm.api", CALI_TYPE_STRING, CALI_ATTR_NESTED, 1, &subs_attr, &v_true); - } - - void create_activity_attributes(Caliper* c) - { - m_activity_start_attr = - c->create_attribute("rocm.starttime", CALI_TYPE_UINT, CALI_ATTR_ASVALUE | CALI_ATTR_SKIP_EVENTS); - m_activity_end_attr = - c->create_attribute("rocm.endtime", CALI_TYPE_UINT, CALI_ATTR_ASVALUE | CALI_ATTR_SKIP_EVENTS); - - m_activity_duration_attr = c->create_attribute( - "rocm.activity.duration", - CALI_TYPE_UINT, - CALI_ATTR_ASVALUE | CALI_ATTR_SKIP_EVENTS | CALI_ATTR_AGGREGATABLE - ); - - m_activity_name_attr = c->create_attribute("rocm.activity", CALI_TYPE_STRING, CALI_ATTR_SKIP_EVENTS); - m_activity_queue_id_attr = c->create_attribute("rocm.activity.queue", CALI_TYPE_UINT, CALI_ATTR_SKIP_EVENTS); - m_activity_device_id_attr = c->create_attribute("rocm.activity.device", CALI_TYPE_UINT, CALI_ATTR_SKIP_EVENTS); - m_activity_bytes_attr = c->create_attribute("rocm.activity.bytes", CALI_TYPE_UINT, CALI_ATTR_SKIP_EVENTS); - m_kernel_name_attr = c->create_attribute("rocm.kernel.name", CALI_TYPE_STRING, CALI_ATTR_SKIP_EVENTS); - - m_flush_region_attr = c->create_attribute("roctracer.flush", CALI_TYPE_STRING, CALI_ATTR_DEFAULT); - } - - void create_host_attributes(Caliper* c) - { - m_host_starttime_attr = - c->create_attribute("rocm.host.starttime", CALI_TYPE_UINT, CALI_ATTR_SCOPE_PROCESS | CALI_ATTR_SKIP_EVENTS); - - if (!(m_record_host_duration || m_record_host_timestamp)) - return; - - int hide_offset = m_record_host_timestamp ? 0 : CALI_ATTR_HIDDEN; - - m_host_timestamp_attr = c->create_attribute( - "rocm.host.timestamp", - CALI_TYPE_UINT, - CALI_ATTR_SCOPE_THREAD | CALI_ATTR_ASVALUE | CALI_ATTR_SKIP_EVENTS | hide_offset - ); - - if (m_record_host_duration) { - m_host_duration_attr = c->create_attribute( - "rocm.host.duration", - CALI_TYPE_UINT, - CALI_ATTR_SCOPE_THREAD | CALI_ATTR_ASVALUE | CALI_ATTR_SKIP_EVENTS | CALI_ATTR_AGGREGATABLE - ); - } - } - - void subscribe_attributes(Caliper* c) - { - m_channel.events().subscribe_attribute(c, m_api_attr); - } - - void push_correlation(uint64_t id, cali::Node* node) - { - std::lock_guard g(m_correlation_map_mutex); - - m_correlation_map[id] = node; - } - - cali::Node* pop_correlation(uint64_t id) - { - cali::Node* ret = nullptr; - - std::lock_guard g(m_correlation_map_mutex); - - auto it = m_correlation_map.find(id); - - if (it != m_correlation_map.end()) { - ret = it->second; - m_correlation_map.erase(it); - } - - return ret; - } - - static void hip_api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg) - { - // skip unneeded callbacks - if (cid == HIP_API_ID___hipPushCallConfiguration || cid == HIP_API_ID___hipPopCallConfiguration) - return; - - auto instance = static_cast(arg); - auto data = static_cast(callback_data); - Caliper c; - - if (data->phase == ACTIVITY_API_PHASE_ENTER) { - c.begin(instance->m_api_attr, Variant(roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, cid, 0))); - - if (instance->m_enable_tracing) { - // When tracing, store a correlation id with the kernel name and the - // current region context. We only store correlation IDs for the subset - // of calls that produce activities. - std::string kernel; - cali::Node* node = nullptr; - - switch (cid) { - case HIP_API_ID_hipLaunchKernel: - case HIP_API_ID_hipExtLaunchKernel: - { - Entry e = c.get(instance->m_api_attr); - if (e.is_reference()) - node = e.node(); - if (instance->m_record_names) { - kernel = hipKernelNameRefByPtr( - data->args.hipLaunchKernel.function_address, - data->args.hipLaunchKernel.stream - ); - } - } - break; - case HIP_API_ID_hipModuleLaunchKernel: - case HIP_API_ID_hipExtModuleLaunchKernel: - case HIP_API_ID_hipHccModuleLaunchKernel: - { - Entry e = c.get(instance->m_api_attr); - if (e.is_reference()) - node = e.node(); - if (instance->m_record_names) { - kernel = hipKernelNameRef(data->args.hipExtModuleLaunchKernel.f); - } - } - break; - case HIP_API_ID_hipMemcpy: - case HIP_API_ID_hipMemcpy2D: - case HIP_API_ID_hipMemcpy2DAsync: - case HIP_API_ID_hipMemcpy2DFromArray: - case HIP_API_ID_hipMemcpy2DFromArrayAsync: - case HIP_API_ID_hipMemcpy2DToArray: - case HIP_API_ID_hipMemcpy2DToArrayAsync: - case HIP_API_ID_hipMemcpy3D: - case HIP_API_ID_hipMemcpy3DAsync: - case HIP_API_ID_hipMemcpyAsync: - case HIP_API_ID_hipMemcpyAtoH: - case HIP_API_ID_hipMemcpyDtoD: - case HIP_API_ID_hipMemcpyDtoDAsync: - case HIP_API_ID_hipMemcpyDtoH: - case HIP_API_ID_hipMemcpyDtoHAsync: - case HIP_API_ID_hipMemcpyFromArray: - case HIP_API_ID_hipMemcpyFromSymbol: - case HIP_API_ID_hipMemcpyFromSymbolAsync: - case HIP_API_ID_hipMemcpyHtoA: - case HIP_API_ID_hipMemcpyHtoD: - case HIP_API_ID_hipMemcpyHtoDAsync: - case HIP_API_ID_hipMemcpyParam2D: - case HIP_API_ID_hipMemcpyParam2DAsync: - case HIP_API_ID_hipMemcpyPeer: - case HIP_API_ID_hipMemcpyPeerAsync: - case HIP_API_ID_hipMemcpyToArray: - case HIP_API_ID_hipMemcpyToSymbol: - case HIP_API_ID_hipMemcpyToSymbolAsync: - case HIP_API_ID_hipMemcpyWithStream: - case HIP_API_ID_hipMemset: - case HIP_API_ID_hipMemset2D: - case HIP_API_ID_hipMemset2DAsync: - case HIP_API_ID_hipMemset3D: - case HIP_API_ID_hipMemset3DAsync: - case HIP_API_ID_hipMemsetAsync: - case HIP_API_ID_hipMemsetD16: - case HIP_API_ID_hipMemsetD32: - case HIP_API_ID_hipMemsetD32Async: - case HIP_API_ID_hipMemsetD8: - case HIP_API_ID_hipMemsetD8Async: - { - Entry e = c.get(instance->m_api_attr); - if (e.is_reference()) - node = e.node(); - } - default: - break; - } - - if (!kernel.empty()) { - kernel = util::demangle(kernel.c_str()); - node = c.make_tree_entry(instance->m_kernel_name_attr, Variant(kernel.c_str()), node); - } - - if (node) { - instance->push_correlation(data->correlation_id, node); - ++instance->m_num_correlations_stored; - } - } - } else { - c.end(instance->m_api_attr); - } - } - - unsigned flush_record(Caliper* c, const roctracer_record_t* record) - { - unsigned num_records = 0; - - if (record->domain == ACTIVITY_DOMAIN_HIP_OPS || record->domain == ACTIVITY_DOMAIN_HCC_OPS) { - Attribute attr[7] = { m_activity_name_attr, m_activity_start_attr, m_activity_end_attr, - m_activity_duration_attr, m_activity_device_id_attr, m_activity_queue_id_attr, - m_activity_bytes_attr }; - Variant data[7] = { Variant(roctracer_op_string(record->domain, record->op, record->kind)), - Variant(cali_make_variant_from_uint(record->begin_ns)), - Variant(cali_make_variant_from_uint(record->end_ns)), - Variant(cali_make_variant_from_uint(record->end_ns - record->begin_ns)), - Variant(cali_make_variant_from_uint(record->device_id)), - Variant(cali_make_variant_from_uint(record->queue_id)), - Variant() }; - - size_t num = 6; - - if (record->op == HIP_OP_ID_COPY) - data[num++] = Variant(cali_make_variant_from_uint(record->bytes)); - - cali::Node* parent = pop_correlation(record->correlation_id); - - if (parent) { - ++m_num_correlations_found; - } else { - ++m_num_correlations_missed; - } - - FixedSizeSnapshotRecord<8> snapshot; - c->make_record(num, attr, data, snapshot.builder(), parent); - m_channel.events().process_snapshot(c, SnapshotView(), snapshot.view()); - - ++num_records; - } - - return num_records; - } - - void flush_activity_records(Caliper* c, const char* begin, const char* end) - { - c->begin(m_flush_region_attr, Variant("ROCTRACER FLUSH")); - - unsigned num_flushed = 0; - unsigned num_records = 0; - - const roctracer_record_t* record = reinterpret_cast(begin); - const roctracer_record_t* end_record = reinterpret_cast(end); - - while (record < end_record) { - num_flushed += flush_record(c, record); - - ++num_records; - if (roctracer_next_record(record, &record) != 0) - break; - } - - if (Log::verbosity() >= 2) { - Log(2).stream() << m_channel.name() << ": roctracer: Flushed " << num_records << " records (" << num_flushed - << " flushed, " << num_records - num_flushed << " skipped).\n"; - } - - m_num_flushed += num_flushed; - m_num_records += num_records; - m_num_flushes++; - - c->end(m_flush_region_attr); - } - - void pre_flush_cb() { roctracer_flush_activity_expl(m_roctracer_pool); } - - void snapshot_cb(Caliper* c, const SnapshotView, SnapshotBuilder& snapshot) - { - uint64_t timestamp = 0; - roctracer_get_timestamp(×tamp); - - Variant v_now(cali_make_variant_from_uint(timestamp)); - Variant v_prev = c->exchange(m_host_timestamp_attr, v_now); - - if (m_record_host_duration) - snapshot.append( - Entry(m_host_duration_attr, Variant(cali_make_variant_from_uint(timestamp - v_prev.to_uint()))) - ); - } - -#if 0 - static void rt_alloc(char** ptr, size_t size, void* arg) { - auto instance = static_cast(arg); - - char* buffer = new char[size]; - instance->m_trace_buffers.push_back(buffer); - ++instance->m_num_buffers; - *ptr = buffer; - - if (Log::verbosity() >= 2) { - unitfmt_result bytes = unitfmt(size, unitfmt_bytes); - Log(2).stream() << "roctracer: Allocated " - << bytes.val << bytes.symbol - << " trace buffer" << std::endl; - } - } -#endif - - static void rt_activity_callback(const char* begin, const char* end, void* arg) - { - auto instance = static_cast(arg); - - Caliper c; - instance->flush_activity_records(&c, begin, end); - - if (Log::verbosity() >= 2) { - unitfmt_result bytes = unitfmt(end - begin, unitfmt_bytes); - Log(2).stream() << instance->m_channel.name() << ": roctracer: processed " << bytes.val << bytes.symbol - << " buffer" << std::endl; - } - } - - void init_tracing(Channel* channel) - { - roctracer_properties_t properties {}; - memset(&properties, 0, sizeof(roctracer_properties_t)); - - properties.buffer_size = 0x800000; - // properties.alloc_fun = rt_alloc; - // properties.alloc_arg = this; - properties.buffer_callback_fun = rt_activity_callback; - properties.buffer_callback_arg = this; - - if (roctracer_open_pool_expl(&properties, &m_roctracer_pool) != 0) { - Log(0).stream() << channel->name() - << ": roctracer: roctracer_open_pool_expl(): " << roctracer_error_string() << std::endl; - return; - } - - if (roctracer_default_pool_expl(m_roctracer_pool) != 0) { - Log(0).stream() << channel->name() - << ": roctracer: roctracer_default_pool_expl(): " << roctracer_error_string() << std::endl; - return; - } - - if (roctracer_enable_domain_activity_expl(ACTIVITY_DOMAIN_HIP_OPS, m_roctracer_pool) != 0) { - Log(0).stream() << channel->name() - << ": roctracer: roctracer_enable_activity_expl(): " << roctracer_error_string() - << std::endl; - return; - } - if (roctracer_enable_domain_activity_expl(ACTIVITY_DOMAIN_HCC_OPS, m_roctracer_pool) != 0) { - Log(0).stream() << channel->name() - << ": roctracer: roctracer_enable_activity_expl(): " << roctracer_error_string() - << std::endl; - return; - } - - channel->events().pre_flush_evt.connect([this](Caliper*, ChannelBody*, SnapshotView) { this->pre_flush_cb(); }); - - Log(1).stream() << channel->name() << ": roctracer: Tracing initialized" << std::endl; - } - - void init_callbacks(Channel* channel) - { - roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, NULL); - - if (roctracer_enable_domain_callback(ACTIVITY_DOMAIN_HIP_API, RocTracerService::hip_api_callback, this) != 0) { - Log(0).stream() << channel->name() << ": roctracer: enable callback (HIP): " << roctracer_error_string() - << std::endl; - return; - } - - Log(1).stream() << channel->name() << ": roctracer: Callbacks initialized" << std::endl; - } - - void finish_tracing(Channel* channel) - { - roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HCC_OPS); - roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS); - roctracer_close_pool_expl(m_roctracer_pool); - m_roctracer_pool = nullptr; - - Log(1).stream() << channel->name() << ": roctracer: Tracing stopped" << std::endl; - } - - void finish_callbacks(Channel* channel) - { - roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API); - - Log(1).stream() << channel->name() << ": roctracer: Callbacks stopped" << std::endl; - } - - void post_init_cb(Caliper* c, Channel* channel) - { - subscribe_attributes(c); - - uint64_t starttime = 0; - roctracer_get_timestamp(&starttime); - - c->set(m_host_starttime_attr, cali_make_variant_from_uint(starttime)); - - if (m_record_host_timestamp || m_record_host_duration) { - c->set(m_host_timestamp_attr, cali_make_variant_from_uint(starttime)); - - channel->events().snapshot.connect([](Caliper* c, SnapshotView info, SnapshotBuilder& rec) { - s_instance->snapshot_cb(c, info, rec); - }); - } - - init_callbacks(channel); // apparently must happen before init_tracing() - - if (m_enable_tracing) - init_tracing(channel); - } - - void pre_finish_cb(Caliper* c, Channel* channel) - { - finish_callbacks(channel); - - if (m_enable_tracing) - finish_tracing(channel); - } - - void finish_cb(Caliper* c, Channel* channel) - { - if (m_enable_tracing) { - Log(1).stream() << channel->name() << ": roctracer: " << m_num_flushes << " activity flushes, " - << m_num_records << " records processed, " << m_num_flushed << " records flushed." - << std::endl; - - if (Log::verbosity() >= 2) { - Log(2).stream() << channel->name() << ": roctracer: " << m_num_correlations_stored - << " correlations stored; " << m_num_correlations_found << " correlations found, " - << m_num_correlations_missed << " missed." << std::endl; - } - } - } - - RocTracerService(Caliper* c, Channel* channel) - : m_num_records { 0 }, - m_num_flushed { 0 }, - m_num_flushes { 0 }, - m_num_correlations_stored { 0 }, - m_num_correlations_found { 0 }, - m_num_correlations_missed { 0 }, - m_roctracer_pool { nullptr }, - m_channel { *channel } - { - auto config = services::init_config_from_spec(channel->config(), s_spec); - - m_enable_tracing = config.get("trace_activities").to_bool(); - m_record_names = config.get("record_kernel_names").to_bool(); - m_record_host_duration = config.get("snapshot_duration").to_bool(); - m_record_host_timestamp = config.get("snapshot_timestamps").to_bool(); - - create_callback_attributes(c); - create_activity_attributes(c); - create_host_attributes(c); - } - - ~RocTracerService() - { -#if 0 - for (char* buffer : m_trace_buffers) - delete[] buffer; -#endif - } - -public: - - static const char* s_spec; - - static void register_roctracer(Caliper* c, Channel* channel) - { - if (s_instance) { - Log(0).stream() << channel->name() << ": roctracer service is already active, disabling!" << std::endl; - } - - s_instance = new RocTracerService(c, channel); - - channel->events().post_init_evt.connect([](Caliper* c, Channel* channel) { - s_instance->post_init_cb(c, channel); - }); - channel->events().pre_finish_evt.connect([](Caliper* c, Channel* channel) { - s_instance->pre_finish_cb(c, channel); - }); - channel->events().finish_evt.connect([](Caliper* c, Channel* channel) { - s_instance->finish_cb(c, channel); - delete s_instance; - s_instance = nullptr; - }); - - Log(1).stream() << channel->name() << ": Registered roctracer service." - << " Activity tracing is " << (s_instance->m_enable_tracing ? "on" : "off") << std::endl; - } -}; - -const char* RocTracerService::s_spec = R"json( -{ "name": "roctracer", - "description": "Record ROCm API and GPU activities", - "config": [ - { "name": "trace_activities", - "type": "bool", - "description": "Enable ROCm GPU activity tracing", - "value": "true" - }, - { "name": "record_kernel_names", - "type": "bool", - "description": "Record kernel names when activity tracing is enabled", - "value": "false" - }, - { "name": "snapshot_duration", - "type": "bool", - "description": "Record duration of host-side activities using ROCm timestamps", - "value": "false" - }, - { "name": "snapshot_timestamps", - "type": "bool", - "description": "Record host-side timestamps with ROCm", - "value": "false" - } - ] -} -)json"; - -RocTracerService* RocTracerService::s_instance = nullptr; - -} // namespace - -namespace cali -{ - -CaliperService roctracer_service { ::RocTracerService::s_spec, ::RocTracerService::register_roctracer }; - -} diff --git a/src/services/roctx/CMakeLists.txt b/src/services/roctx/CMakeLists.txt deleted file mode 100644 index 9335f5b2..00000000 --- a/src/services/roctx/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -set(CALIPER_ROCTX_SOURCES - RocTX.cpp) - -add_library(caliper-roctx OBJECT ${CALIPER_ROCTX_SOURCES}) -target_include_directories(caliper-roctx PRIVATE ${ROCTX_INCLUDE_DIRS}) - -add_service_objlib("caliper-roctx") -add_caliper_service("roctx CALIPER_HAVE_ROCTX") diff --git a/src/services/roctx/RocTX.cpp b/src/services/roctx/RocTX.cpp deleted file mode 100644 index fa1863b0..00000000 --- a/src/services/roctx/RocTX.cpp +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright (c) 2015-2022, Lawrence Livermore National Security, LLC. -// See top-level LICENSE file for details. - -// Caliper RocTX bindings - -#include "../Services.h" - -#include "../../caliper/AnnotationBinding.h" - -#include "caliper/common/Attribute.h" -#include "caliper/common/Log.h" - -#include - -#include -#include -#include - -using namespace cali; - -namespace -{ - -class RocTXBinding : public AnnotationBinding -{ - std::map> m_range_map; - std::mutex m_range_map_mutex; - - unsigned m_num_stack_errors { 0 }; - unsigned m_num_range_errors { 0 }; - -public: - - void on_begin(Caliper*, const Attribute& attr, const Variant& value) - { - const char* msg = nullptr; - std::string str; // string obj must not be deleted until end of function - - if (attr.type() == CALI_TYPE_STRING) { - msg = static_cast(value.data()); - } else { - str = value.to_string(); - msg = str.c_str(); - } - - // Use range stack for nested attributes, range start/stop for others - if (attr.is_nested()) { - roctxRangePush(msg); - } else { - roctx_range_id_t roctx_id = roctxRangeStart(msg); - - std::lock_guard g(m_range_map_mutex); - - m_range_map[attr.id()].push_back(roctx_id); - } - } - - void on_end(Caliper*, const Attribute& attr, const Variant&) - { - if (attr.is_nested()) { - if (roctxRangePop() < 0) - ++m_num_stack_errors; - } else { - bool found = false; - roctx_range_id_t roctx_id = 0; - - { - std::lock_guard g(m_range_map_mutex); - - auto it = m_range_map.find(attr.id()); - - if (it != m_range_map.end() && it->second.size() > 0) { - roctx_id = it->second.back(); - found = true; - it->second.pop_back(); - } - } - - if (found) - roctxRangeStop(roctx_id); - else - ++m_num_range_errors; - } - } - - void finalize(Caliper*, Channel* channel) - { - if (m_num_range_errors > 0) - Log(0).stream() << channel->name() << "roctx: " << m_num_range_errors << " range start/stop errors!" - << std::endl; - if (m_num_stack_errors > 0) - Log(0).stream() << channel->name() << "roctx: " << m_num_stack_errors << " region stack errors!" - << std::endl; - } - - const char* service_tag() const { return "roctx"; } -}; - -} // namespace - -namespace cali -{ - -CaliperService roctx_service { "roctx", AnnotationBinding::make_binding }; - -}