diff --git a/libkineto/src/CuptiActivityProfiler.cpp b/libkineto/src/CuptiActivityProfiler.cpp index 41abef150..7d03505a0 100644 --- a/libkineto/src/CuptiActivityProfiler.cpp +++ b/libkineto/src/CuptiActivityProfiler.cpp @@ -1298,6 +1298,10 @@ void CuptiActivityProfiler::finalizeTrace(const Config& config, ActivityLogger& for (auto& session : sessions_){ auto trace_buffer = session->getTraceBuffer(); if (trace_buffer) { + // Set child start time to profiling start time if not set + if (trace_buffer->span.startTime == 0) { + trace_buffer->span.startTime = captureWindowStartTime_; + } traceBuffers_->cpu.push_back(std::move(trace_buffer)); } } diff --git a/libkineto/src/output_json.cpp b/libkineto/src/output_json.cpp index 41b38499e..a3e30ebf8 100644 --- a/libkineto/src/output_json.cpp +++ b/libkineto/src/output_json.cpp @@ -233,8 +233,10 @@ void ChromeTraceLogger::handleTraceSpan(const TraceSpan& span) { } uint64_t start = transToRelativeTime(span.startTime); - uint64_t dur = span.endTime - span.startTime; - + + // If endTime is 0 and start time is non-zero, dur can overflow. Add + // a guard to prevent this. + uint64_t dur = (span.endTime == 0) ? 0 : span.endTime - span.startTime; // clang-format off traceOf_ << fmt::format(R"JSON( {{ diff --git a/libkineto/stress_test/kineto_stress_test.cpp b/libkineto/stress_test/kineto_stress_test.cpp index a258f1a65..ecf02adf2 100644 --- a/libkineto/stress_test/kineto_stress_test.cpp +++ b/libkineto/stress_test/kineto_stress_test.cpp @@ -29,6 +29,8 @@ #include "kineto/libkineto/stress_test/tensor_cache.cuh" #include "kineto/libkineto/stress_test/utils.h" #include "kineto/libkineto/fb/nccl_profiler/NcclProfiler.h" +#include +#include using namespace kineto_stress_test; @@ -89,6 +91,8 @@ void read_inputs_from_json(std::string sJsonFile, stress_test_args *test_args, void trace_collection_thread(uint32_t trace_length_us, uint32_t cupti_buffer_mb) { + c10::ApproximateClockToUnixTimeConverter clockConverter; + if (cupti_buffer_mb > 0) { // Configure CUPTI buffer sizes @@ -111,6 +115,8 @@ void trace_collection_thread(uint32_t trace_length_us, auto& profiler = libkineto::api().activityProfiler(); libkineto::api().initProfilerIfRegistered(); profiler.prepareTrace(types); + auto converter = clockConverter.makeConverter(); + libkineto::get_time_converter() = converter; // Collect the trace profiler.startTrace();