diff --git a/libkineto/src/CuptiActivityProfiler.cpp b/libkineto/src/CuptiActivityProfiler.cpp index e02c325c1..2735c602c 100644 --- a/libkineto/src/CuptiActivityProfiler.cpp +++ b/libkineto/src/CuptiActivityProfiler.cpp @@ -103,6 +103,22 @@ std::function& get_time_converter() { }; return _time_converter; } +#ifdef HAS_ROCTRACER +timestamp_t getTimeOffset() { + int64_t t0, t00; + timespec t1; + t0 = libkineto::getApproximateTime(); + clock_gettime(CLOCK_MONOTONIC, &t1); + t00 = libkineto::getApproximateTime(); + + // Confvert to ns (if necessary) + t0 = libkineto::get_time_converter()(t0); + t00 = libkineto::get_time_converter()(t00); + + // Our stored timestamps (from roctracer and generated) are in CLOCK_MONOTONIC domain (in ns). + return (t0 >> 1) + (t00 >> 1) - timespec_to_ns(t1); +} +#endif #ifdef HAS_CUPTI bool& use_cupti_tsc() { @@ -340,6 +356,8 @@ void CuptiActivityProfiler::processTraceInternal(ActivityLogger& logger) { #ifdef HAS_ROCTRACER if (!cpuOnly_) { VLOG(0) << "Retrieving GPU activity buffers"; + timestamp_t offset = getTimeOffset(); + cupti_.setTimeOffset(offset); const int count = cupti_.processActivities( std::bind(&CuptiActivityProfiler::handleRoctracerActivity, this, std::placeholders::_1, &logger), std::bind(&CuptiActivityProfiler::handleCorrelationActivity, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); diff --git a/libkineto/src/RoctracerActivityApi.cpp b/libkineto/src/RoctracerActivityApi.cpp index 51392a084..ec4928962 100644 --- a/libkineto/src/RoctracerActivityApi.cpp +++ b/libkineto/src/RoctracerActivityApi.cpp @@ -8,11 +8,12 @@ #include "RoctracerActivityApi.h" +#include "ApproximateClock.h" #include #include #include #include - +#include "Logger.h" #include "Demangle.h" #include "output_base.h" #include "ThreadUtil.h" @@ -65,20 +66,16 @@ inline bool RoctracerActivityApi::isLogged(libkineto::ActivityType atype) { return activityMaskSnapshot_ & (1 << static_cast(atype)); } +void RoctracerActivityApi::setTimeOffset(timestamp_t toffset) { + toffset_ = toffset; +} + int RoctracerActivityApi::processActivities( std::function handler, std::function correlationHandler) { // Find offset to map from monotonic clock to system clock. // This will break time-ordering of events but is status quo. - timespec t0, t1, t00; - clock_gettime(CLOCK_REALTIME, &t0); - clock_gettime(CLOCK_MONOTONIC, &t1); - clock_gettime(CLOCK_REALTIME, &t00); - - const timestamp_t toffset = (timespec_to_ns(t0) >> 1) + (timespec_to_ns(t00) >> 1) - timespec_to_ns(t1); - // Our stored timestamps (from roctracer and generated) are in CLOCK_MONOTONIC domain (in ns). - int count = 0; // Process all external correlations pairs @@ -125,8 +122,8 @@ int RoctracerActivityApi::processActivities( } if (!filtered) { // Convert the begin and end timestamps from monotonic clock to system clock. - item->begin = item->begin + toffset; - item->end = item->end + toffset; + item->begin = item->begin + toffset_; + item->end = item->end + toffset_; handler(item); ++count; } diff --git a/libkineto/src/RoctracerActivityApi.h b/libkineto/src/RoctracerActivityApi.h index a02c2acf0..e66c9a1e7 100644 --- a/libkineto/src/RoctracerActivityApi.h +++ b/libkineto/src/RoctracerActivityApi.h @@ -52,6 +52,7 @@ class RoctracerActivityApi { const std::set& selected_activities); void clearActivities(); void teardownContext() {} + void setTimeOffset(timestamp_t toffset); virtual int processActivities( std::function handler, @@ -63,6 +64,7 @@ class RoctracerActivityApi { private: bool registered_{false}; + timestamp_t toffset_{0}; // Enabled Activity Filters uint32_t activityMask_{0};