Skip to content

Add support for capturing roctx markers #730

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
daboehme merged 3 commits into from
Mar 13, 2026
Merged

Add support for capturing roctx markers #730

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a1e7184
Add support for capturing roctx ranges in rocprofiler
daboehme Mar 13, 2026
a9d7cba
Add profile.roctx option. Turn off HIP API callbacks by default.
daboehme Mar 13, 2026
bf3d55a
rocprofiler: Simplify context cleanup
daboehme Mar 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/caliper/controllers/ROCmActivityProfileController.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,6 +145,7 @@ const char* controller_spec = R"json(
{
"CALI_CHANNEL_FLUSH_ON_EXIT": "false",
"CALI_EVENT_ENABLE_SNAPSHOT_INFO": "false",
"CALI_ROCPROFILER_ENABLE_API_CALLBACKS": "true",
"CALI_ROCPROFILER_ENABLE_ACTIVITY_TRACING": "true",
"CALI_ROCPROFILER_ENABLE_SNAPSHOT_TIMESTAMPS": "true"
},
Expand Down
9 changes: 5 additions & 4 deletions src/caliper/controllers/ROCmActivityReportController.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,10 +136,11 @@ const char* controller_spec = R"json(
"services" : [ "aggregate", "rocprofiler", "event" ],
"config" :
{
"CALI_CHANNEL_FLUSH_ON_EXIT" : "false",
"CALI_EVENT_ENABLE_SNAPSHOT_INFO" : "false",
"CALI_ROCPROFILER_ENABLE_ACTIVITY_TRACING" : "true",
"CALI_ROCPROFILER_ENABLE_SNAPSHOT_TIMESTAMPS" : "true"
"CALI_CHANNEL_FLUSH_ON_EXIT": "false",
"CALI_EVENT_ENABLE_SNAPSHOT_INFO": "false",
"CALI_ROCPROFILER_ENABLE_API_CALLBACKS": "true",
"CALI_ROCPROFILER_ENABLE_ACTIVITY_TRACING": "true",
"CALI_ROCPROFILER_ENABLE_SNAPSHOT_TIMESTAMPS": "true"
},
"defaults": { "order_as_visited": "true", "output.append": "true" },
"options":
Expand Down
14 changes: 14 additions & 0 deletions src/caliper/controllers/controllers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@ const char* event_trace_spec = R"json(
"description": "Trace CUDA API events",
"type": "bool",
"services": [ "cupti" ]
},{
"name": "trace.hip",
"description": "Trace CUDA API events",
"type": "bool",
"services": [ "rocprofiler" ],
"config": { "CALI_ROCPROFILER_ENABLE_API_CALLBACKS": "true" }
},{
"name": "trace.openmp",
"description": "Trace OpenMP events",
Expand Down Expand Up @@ -704,6 +710,14 @@ const char* builtin_rocm_option_specs = R"json(
"services": [ "rocprofiler" ],
"config": { "CALI_ROCPROFILER_ENABLE_API_CALLBACKS": "true" }
},
{
"name": "profile.roctx",
"type": "bool",
"description": "Profile roctx markers",
"category": "region",
"services": [ "rocprofiler" ],
"config": { "CALI_ROCPROFILER_ENABLE_MARKER_CALLBACKS": "true" }
},
{
"name": "rocm.gputime",
"description": "Report GPU time in AMD ROCm activities",
Expand Down
138 changes: 98 additions & 40 deletions src/services/rocprofiler/RocProfiler.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@ auto make_array(Arg arg, Args&&... args)
class RocProfilerService
{
Attribute m_api_attr;
Attribute m_marker_attr;

Attribute m_kernel_name_attr;

Expand All @@ -95,6 +96,7 @@ class RocProfilerService
Attribute m_flush_region_attr;

bool m_enable_api_callbacks = false;
bool m_enable_marker_callbacks = false;
bool m_enable_activity_tracing = false;
bool m_enable_snapshot_timestamps = false;
bool m_enable_allocation_tracing = false;
Expand Down Expand Up @@ -129,6 +131,7 @@ class RocProfilerService
static RocProfilerService* s_instance;

static rocprofiler_context_id_t s_hip_api_ctx;
static rocprofiler_context_id_t s_marker_ctx;
static rocprofiler_context_id_t s_activity_ctx;
static rocprofiler_context_id_t s_rocprofiler_ctx;
static rocprofiler_context_id_t s_alloc_tracing_ctx;
Expand All @@ -142,6 +145,7 @@ class RocProfilerService
Variant v_true(true);

m_api_attr = c->create_attribute("rocm.api", CALI_TYPE_STRING, CALI_ATTR_NESTED, 1, &subs_attr, &v_true);
m_marker_attr = c->create_attribute("rocm.marker", CALI_TYPE_STRING, CALI_ATTR_NESTED, 1, &subs_attr, &v_true);

m_activity_start_attr =
c->create_attribute("rocm.starttime", CALI_TYPE_UINT, CALI_ATTR_ASVALUE | CALI_ATTR_SKIP_EVENTS);
Expand Down Expand Up @@ -425,7 +429,25 @@ class RocProfilerService
c.end(s_instance->m_flush_region_attr);
}

static void tool_api_callback(
static void code_obj_callback(
rocprofiler_callback_tracing_record_t record,
rocprofiler_user_data_t* user_data,
void* /* callback_data */
)
{
if (!s_instance)
return;
if (record.kind != ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT)
return;

if (record.operation == ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER &&
record.phase == ROCPROFILER_CALLBACK_PHASE_LOAD) {
auto* data = static_cast<rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t*>(record.payload);
s_instance->update_kernel_info(data->kernel_id, util::demangle(data->kernel_name));
}
}

static void hip_api_callback(
rocprofiler_callback_tracing_record_t record,
rocprofiler_user_data_t* user_data,
void* /* callback_data */
Expand All @@ -434,28 +456,45 @@ class RocProfilerService
if (!s_instance)
return;

if (record.kind == ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT) {
if (record.operation == ROCPROFILER_CODE_OBJECT_DEVICE_KERNEL_SYMBOL_REGISTER
&& record.phase == ROCPROFILER_CALLBACK_PHASE_LOAD) {
auto* data =
static_cast<rocprofiler_callback_tracing_code_object_kernel_symbol_register_data_t*>(record.payload
);
s_instance->update_kernel_info(data->kernel_id, util::demangle(data->kernel_name));
if (record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER) {
const char* name = nullptr;
uint64_t len = 0;
ROCPROFILER_CALL(
rocprofiler_query_callback_tracing_kind_operation_name(record.kind, record.operation, &name, &len)
);
if (!name) {
name = "UNKNOWN";
len = 7;
}
} else {
if (record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER) {
const char* name = nullptr;
uint64_t len = 0;
ROCPROFILER_CALL(
rocprofiler_query_callback_tracing_kind_operation_name(record.kind, record.operation, &name, &len)
);
if (!name) {
name = "UNKNOWN";
len = 7;
Caliper::instance().begin(s_instance->m_api_attr, Variant(CALI_TYPE_STRING, name, len));
} else if (record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT) {
Caliper::instance().end(s_instance->m_api_attr);
}
}

static void marker_callback(
rocprofiler_callback_tracing_record_t record,
rocprofiler_user_data_t* user_data,
void* /* callback_data */
)
{
if (!s_instance)
return;

if (record.kind == ROCPROFILER_CALLBACK_TRACING_MARKER_CORE_API &&
record.phase == ROCPROFILER_CALLBACK_PHASE_ENTER) {
switch (record.operation) {
case ROCPROFILER_MARKER_CORE_API_ID_roctxRangePushA:
{
const auto *data = static_cast<const rocprofiler_callback_tracing_marker_api_data_t*>(record.payload);
Caliper::instance().begin(s_instance->m_marker_attr, Variant(data->args.roctxRangePushA.message));
}
Caliper::instance().begin(s_instance->m_api_attr, Variant(CALI_TYPE_STRING, name, len));
} else if (record.phase == ROCPROFILER_CALLBACK_PHASE_EXIT) {
Caliper::instance().end(s_instance->m_api_attr);
break;
case ROCPROFILER_MARKER_CORE_API_ID_roctxRangePop:
Caliper::instance().end(s_instance->m_marker_attr);
break;
default:
break;
}
}
}
Expand Down Expand Up @@ -532,19 +571,29 @@ class RocProfilerService
if (m_enable_api_callbacks) {
channel->events().subscribe_attribute(c, m_api_attr);
ROCPROFILER_CALL(rocprofiler_start_context(s_hip_api_ctx));
Log(2).stream() << channel->name() << ": rocprofiler: HIP API callbacks activated\n";
}

if (m_enable_marker_callbacks) {
channel->events().subscribe_attribute(c, m_marker_attr);
ROCPROFILER_CALL(rocprofiler_start_context(s_marker_ctx));
Log(2).stream() << channel->name() << ": rocprofiler: marker callbacks activated\n";
}

if (m_enable_activity_tracing) {
ROCPROFILER_CALL(rocprofiler_start_context(s_rocprofiler_ctx));
ROCPROFILER_CALL(rocprofiler_start_context(s_activity_ctx));
Log(2).stream() << channel->name() << ": rocprofiler: activity tracing activated\n";
}

if (m_enable_allocation_tracing) {
ROCPROFILER_CALL(rocprofiler_start_context(s_alloc_tracing_ctx));
Log(2).stream() << channel->name() << ": rocprofiler: allocation tracking activated\n";
}

if (m_enable_counters) {
ROCPROFILER_CALL(rocprofiler_start_context(s_counter_ctx));
Log(2).stream() << channel->name() << ": rocprofiler: counter tracing activated\n";
}

if (m_enable_activity_tracing || m_enable_counters) {
Expand Down Expand Up @@ -573,19 +622,14 @@ class RocProfilerService

void pre_finish_cb(Caliper* c, Channel* channel)
{
int status = 0;
ROCPROFILER_CALL(rocprofiler_context_is_active(s_hip_api_ctx, &status));
if (status)
ROCPROFILER_CALL(rocprofiler_stop_context(s_hip_api_ctx));
ROCPROFILER_CALL(rocprofiler_context_is_active(s_rocprofiler_ctx, &status));
if (status)
ROCPROFILER_CALL(rocprofiler_stop_context(s_rocprofiler_ctx));
ROCPROFILER_CALL(rocprofiler_context_is_active(s_activity_ctx, &status));
if (status)
ROCPROFILER_CALL(rocprofiler_stop_context(s_activity_ctx));
ROCPROFILER_CALL(rocprofiler_context_is_active(s_counter_ctx, &status));
if (status)
ROCPROFILER_CALL(rocprofiler_stop_context(s_counter_ctx));
auto contexts = make_array(s_hip_api_ctx, s_marker_ctx, s_rocprofiler_ctx, s_activity_ctx, s_alloc_tracing_ctx, s_counter_ctx);

for (auto &ctx : contexts) {
int status = 0;
ROCPROFILER_CALL(rocprofiler_context_is_active(ctx, &status));
if (status)
ROCPROFILER_CALL(rocprofiler_stop_context(ctx));
}

Log(1).stream() << channel->name() << ": rocprofiler: wrote " << m_num_activity_records
<< " activity records, " << m_num_counter_records << " counter records.\n";
Expand Down Expand Up @@ -677,6 +721,7 @@ class RocProfilerService
auto config = services::init_config_from_spec(channel->config(), s_spec);

m_enable_api_callbacks = config.get("enable_api_callbacks").to_bool();
m_enable_marker_callbacks = config.get("enable_marker_callbacks").to_bool();
m_enable_activity_tracing = config.get("enable_activity_tracing").to_bool();
m_enable_snapshot_timestamps = config.get("enable_snapshot_timestamps").to_bool();
m_enable_allocation_tracing = config.get("enable_allocation_tracing").to_bool();
Expand Down Expand Up @@ -720,6 +765,7 @@ class RocProfilerService
static int tool_init(rocprofiler_client_finalize_t fini_func, void* tool_data)
{
ROCPROFILER_CALL(rocprofiler_create_context(&s_hip_api_ctx));
ROCPROFILER_CALL(rocprofiler_create_context(&s_marker_ctx));
ROCPROFILER_CALL(rocprofiler_create_context(&s_activity_ctx));
ROCPROFILER_CALL(rocprofiler_create_context(&s_rocprofiler_ctx));
ROCPROFILER_CALL(rocprofiler_create_context(&s_alloc_tracing_ctx));
Expand All @@ -730,15 +776,23 @@ class RocProfilerService
ROCPROFILER_CALLBACK_TRACING_HIP_RUNTIME_API,
nullptr,
0,
tool_api_callback,
hip_api_callback,
nullptr
));
ROCPROFILER_CALL(rocprofiler_configure_callback_tracing_service(
s_marker_ctx,
ROCPROFILER_CALLBACK_TRACING_MARKER_CORE_API,
nullptr,
0,
marker_callback,
nullptr
));
ROCPROFILER_CALL(rocprofiler_configure_callback_tracing_service(
s_rocprofiler_ctx,
ROCPROFILER_CALLBACK_TRACING_CODE_OBJECT,
nullptr,
0,
tool_api_callback,
code_obj_callback,
nullptr
));
#ifdef CALI_ROCPROFILER_HAVE_ALLOC_TRACING
Expand Down Expand Up @@ -832,15 +886,14 @@ class RocProfilerService
s_instance = nullptr;
});

Log(1).stream() << channel->name() << ": Registered rocprofiler service."
<< " Activity tracing is " << (s_instance->m_enable_activity_tracing ? "on" : "off")
<< std::endl;
Log(1).stream() << channel->name() << ": Registered rocprofiler service\n";
}
};

RocProfilerService* RocProfilerService::s_instance = nullptr;

rocprofiler_context_id_t RocProfilerService::s_hip_api_ctx = {};
rocprofiler_context_id_t RocProfilerService::s_marker_ctx = {};
rocprofiler_context_id_t RocProfilerService::s_activity_ctx = {};
rocprofiler_context_id_t RocProfilerService::s_rocprofiler_ctx = {};
rocprofiler_context_id_t RocProfilerService::s_alloc_tracing_ctx = {};
Expand All @@ -857,7 +910,12 @@ const char* RocProfilerService::s_spec = R"json(
{ "name": "enable_api_callbacks",
"type": "bool",
"description": "Enable HIP API interception callbacks",
"value": "true"
"value": "false"
},
{ "name": "enable_marker_callbacks",
"type": "bool",
"description": "Enable roctx marker callbacks",
"value": "false"
},
{ "name": "enable_activity_tracing",
"type": "bool",
Expand Down
Loading