Add API for Dynamic Activity Toggling [1/n] (#972)

Summary:
Pull Request resolved: #972

During PT2 there are many GPU/CPU events that are unneccessary to profile in between a given step. To remedy this, we can add an API that takes in a list of activities and an arg to toggle said activies or not. For this diff we are just adding the Kineto side to turn on/off the "CUDA" events which includes AMD/Roctracer. A follow up will be added for the generic profiler side.  Subsequent diffs will be added for CPU toggling and e2e testing.

Differential Revision: D60542040

fbshipit-source-id: 2608ec912812c9004cb87371bd8fca8145a95621

Author: sraikund16

libkineto/include/ActivityProfilerInterface.h

@@ -55,6 +55,10 @@ class ActivityProfilerInterface {
       const std::set<ActivityType>& activityTypes,
       const std::string& configStr = "") {}
 
+  // Toggle GPU tracing as a trace is running to omit certain parts of a graph
+  virtual void toggleCollectionDynamic(
+    const bool enable) {}
+
   // Start recording, potentially reusing any buffers allocated since
   // prepareTrace was called.
   virtual void startTrace() {}

libkineto/src/ActivityProfilerController.cpp

@@ -315,6 +315,10 @@ void ActivityProfilerController::prepareTrace(const Config& config) {
   profiler_->configure(config, now);
 }
 
+void ActivityProfilerController::toggleCollectionDynamic(const bool enable) {
+  profiler_->toggleCollectionDynamic(enable);
+}
+
 void ActivityProfilerController::startTrace() {
   UST_LOGGER_MARK_COMPLETED(kWarmUpStage);
   profiler_->startTrace(std::chrono::system_clock::now());

libkineto/src/ActivityProfilerController.h

@@ -58,6 +58,7 @@ class ActivityProfilerController : public ConfigLoader::ConfigHandler {
 
   // These API are used for Synchronous Tracing.
   void prepareTrace(const Config& config);
+  void toggleCollectionDynamic(const bool enable);
   void startTrace();
   void step();
   std::unique_ptr<ActivityTraceInterface> stopTrace();

libkineto/src/ActivityProfilerProxy.cpp

@@ -70,6 +70,10 @@ void ActivityProfilerProxy::prepareTrace(
   controller_->prepareTrace(config);
 }
 
+void ActivityProfilerProxy::toggleCollectionDynamic(const bool enable){
+  controller_->toggleCollectionDynamic(enable);
+}
+
 void ActivityProfilerProxy::startTrace() {
   controller_->startTrace();
 }

libkineto/src/ActivityProfilerProxy.h

@@ -52,6 +52,9 @@ class ActivityProfilerProxy : public ActivityProfilerInterface {
       const std::set<ActivityType>& activityTypes,
       const std::string& configStr = "") override;
 
+  void toggleCollectionDynamic(
+    const bool enable) override;
+    
   void startTrace() override;
   void step() override;
   std::unique_ptr<ActivityTraceInterface> stopTrace() override;

libkineto/src/CuptiActivityProfiler.cpp

@@ -1073,6 +1073,23 @@ void CuptiActivityProfiler::configure(
   currentRunloopState_ = RunloopState::Warmup;
 }
 
+void CuptiActivityProfiler::toggleCollectionDynamic(const bool enable){
+#ifdef HAS_CUPTI
+  if (enable) {
+    cupti_.enableCuptiActivities(derivedConfig_->profileActivityTypes());
+  } else {
+    cupti_.disableCuptiActivities(derivedConfig_->profileActivityTypes());
+  }
+#endif
+#ifdef HAS_ROCTRACER
+  if (enable) {
+    cupti_.enableActivities(derivedConfig_->profileActivityTypes());
+  } else {
+    cupti_.disableActivities(derivedConfig_->profileActivityTypes());
+  }
+#endif
+}
+
 void CuptiActivityProfiler::startTraceInternal(
     const time_point<system_clock>& now) {
   captureWindowStartTime_ = libkineto::timeSinceEpoch(now);

libkineto/src/CuptiActivityProfiler.h

@@ -179,6 +179,9 @@ class CuptiActivityProfiler {
   void configure(
       const Config& config,
       const std::chrono::time_point<std::chrono::system_clock>& now);
+  
+  // Toggle GPU tracing during a profile instance
+  void toggleCollectionDynamic(const bool enable);
 
   // Registered with client API to pass CPU trace events over
   void transferCpuTrace(