Align Roctracer to TSC Clock (#991)

Summary:
Pull Request resolved: #991

Right now we align Roctracer events to system clock blindly regardless of what we are using in torch.profiler. We should use a clock based on  what is defined instead. This wont fix overlapping kernel events since we do a static offset when aligning but it will help make sure that kernel events always happen after kernel launches

Reviewed By: aaronenyeshi, briancoutinho

Differential Revision: D62984793

fbshipit-source-id: 4495a83de98dc3fb752754898588b93f4850e7a4

libkineto/src/CuptiActivityProfiler.cpp

@@ -103,6 +103,22 @@ std::function<time_t(approx_time_t)>& get_time_converter() {
   };
   return _time_converter;
 }
+#ifdef HAS_ROCTRACER
+timestamp_t getTimeOffset() {
+  int64_t t0, t00;
+  timespec t1;
+  t0 = libkineto::getApproximateTime();
+  clock_gettime(CLOCK_MONOTONIC, &t1);
+  t00 = libkineto::getApproximateTime();
+
+  // Confvert to ns (if necessary)
+  t0 = libkineto::get_time_converter()(t0);
+  t00 = libkineto::get_time_converter()(t00);
+
+  // Our stored timestamps (from roctracer and generated) are in CLOCK_MONOTONIC domain (in ns).
+  return (t0 >> 1) + (t00 >> 1) - timespec_to_ns(t1);
+}
+#endif
 
 #ifdef HAS_CUPTI
 bool& use_cupti_tsc() {
@@ -340,6 +356,8 @@ void CuptiActivityProfiler::processTraceInternal(ActivityLogger& logger) {
 #ifdef HAS_ROCTRACER
   if (!cpuOnly_) {
     VLOG(0) << "Retrieving GPU activity buffers";
+    timestamp_t offset = getTimeOffset();
+    cupti_.setTimeOffset(offset);
     const int count = cupti_.processActivities(
         std::bind(&CuptiActivityProfiler::handleRoctracerActivity, this, std::placeholders::_1, &logger),
         std::bind(&CuptiActivityProfiler::handleCorrelationActivity, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3));

libkineto/src/RoctracerActivityApi.cpp

@@ -8,11 +8,12 @@
 
 #include "RoctracerActivityApi.h"
 
+#include "ApproximateClock.h"
 #include <cstring>
 #include <chrono>
 #include <functional>
 #include <time.h>
-
+#include "Logger.h"
 #include "Demangle.h"
 #include "output_base.h"
 #include "ThreadUtil.h"
@@ -65,20 +66,16 @@ inline bool RoctracerActivityApi::isLogged(libkineto::ActivityType atype) {
   return activityMaskSnapshot_ & (1 << static_cast<uint32_t>(atype));
 }
 
+void RoctracerActivityApi::setTimeOffset(timestamp_t toffset) {
+  toffset_ = toffset;
+}
+
 int RoctracerActivityApi::processActivities(
     std::function<void(const roctracerBase*)> handler,
     std::function<void(uint64_t, uint64_t, RoctracerLogger::CorrelationDomain)> correlationHandler) {
   // Find offset to map from monotonic clock to system clock.
   // This will break time-ordering of events but is status quo.
 
-  timespec t0, t1, t00;
-  clock_gettime(CLOCK_REALTIME, &t0);
-  clock_gettime(CLOCK_MONOTONIC, &t1);
-  clock_gettime(CLOCK_REALTIME, &t00);
-
-  const timestamp_t toffset = (timespec_to_ns(t0) >> 1) + (timespec_to_ns(t00) >> 1) - timespec_to_ns(t1);
-  // Our stored timestamps (from roctracer and generated) are in CLOCK_MONOTONIC domain (in ns).
-
   int count = 0;
 
   // Process all external correlations pairs
@@ -125,8 +122,8 @@ int RoctracerActivityApi::processActivities(
     }
     if (!filtered) {
       // Convert the begin and end timestamps from monotonic clock to system clock.
-      item->begin = item->begin + toffset;
-      item->end = item->end + toffset;
+      item->begin = item->begin + toffset_;
+      item->end = item->end + toffset_;
       handler(item);
       ++count;
     }

libkineto/src/RoctracerActivityApi.h

@@ -52,6 +52,7 @@ class RoctracerActivityApi {
     const std::set<ActivityType>& selected_activities);
   void clearActivities();
   void teardownContext() {}
+  void setTimeOffset(timestamp_t toffset);
 
   virtual int processActivities(
     std::function<void(const roctracerBase*)> handler,
@@ -63,6 +64,7 @@ class RoctracerActivityApi {
 
  private:
   bool registered_{false};
+  timestamp_t toffset_{0};
 
   // Enabled Activity Filters
   uint32_t activityMask_{0};