Configure activity types to be traced

Summary:
This allows configuring which activity type will be traced. For example if we are only interested in memory transfers between CPU-GPU, we can specify that from the configuration file.

Valid activity names are "gpu_memcpy", "gpu_memset", "concurrent_kernel", "external_correlation", "cuda_runtime". The configuration file key is "ACTIVITY_TYPES".

Reviewed By: gdankel

Differential Revision: D24157062

fbshipit-source-id: 7f0922ef28884edbca6c9190cc9fb603bf2db7a6

Author: valentinandrei

libkineto/src/ActivityProfiler.cpp

@@ -449,7 +449,7 @@ void ActivityProfiler::configure(
     if (VLOG_IS_ON(1)) {
       timestamp = high_resolution_clock::now();
     }
-    cupti_.enableCuptiActivities();
+    cupti_.enableCuptiActivities(config_->selectedActivityTypes());
     if (VLOG_IS_ON(1)) {
       auto t2 = high_resolution_clock::now();
       addOverheadSample(
@@ -477,7 +477,7 @@ void ActivityProfiler::endTrace() {
     if (VLOG_IS_ON(1)) {
       timestamp = high_resolution_clock::now();
     }
-    cupti_.disableCuptiActivities();
+    cupti_.disableCuptiActivities(config_->selectedActivityTypes());
     if (VLOG_IS_ON(1)) {
       auto t2 = high_resolution_clock::now();
       addOverheadSample(

libkineto/src/Config.cpp

@@ -54,6 +54,7 @@ const string kMaxEventProfilersPerGpuKey = "MAX_EVENT_PROFILERS_PER_GPU";
 
 // Activity Profiler
 const string kActivitiesEnabledKey = "ACTIVITIES_ENABLED";
+const string kActivityTypesKey = "ACTIVITY_TYPES";
 const string kActivitiesLogFileKey = "ACTIVITIES_LOG_FILE";
 const string kActivitiesDurationKey = "ACTIVITIES_DURATION_SECS";
 const string kActivitiesDurationMsecsKey = "ACTIVITIES_DURATION_MSECS";
@@ -66,6 +67,13 @@ const string kActivitiesWarmupDurationSecsKey = "ACTIVITIES_WARMUP_PERIOD_SECS";
 const string kActivitiesMaxGpuBufferSizeKey =
     "ACTIVITIES_MAX_GPU_BUFFER_SIZE_MB";
 
+// Valid configuration file entries for activity types
+const string kActivityMemcpy = "gpu_memcpy";
+const string kActivityMemset = "gpu_memset";
+const string kActivityConcurrentKernel = "concurrent_kernel";
+const string kActivityExternalCorrelation = "external_correlation";
+const string kActivityRuntime = "cuda_runtime";
+
 const string kDefaultLogFileFmt = "/tmp/libkineto_activities_{}.json";
 
 // Common
@@ -181,6 +189,32 @@ static time_point<system_clock> handleRequestTimestamp(int64_t ms) {
   return t;
 }
 
+void Config::addActivityTypes(
+  const std::vector<std::string>& selected_activities) {
+  if (selected_activities.size() > 0) {
+    for (const auto& activity : selected_activities) {
+      if (activity == "") {
+        continue;
+      } else if (activity == kActivityMemcpy) {
+        selectedActivityTypes_.insert(ActivityType::MEMCPY);
+      } else if (activity == kActivityMemset) {
+        selectedActivityTypes_.insert(ActivityType::MEMSET);
+      } else if (activity == kActivityConcurrentKernel) {
+        selectedActivityTypes_.insert(ActivityType::CONCURRENT_KERNEL);
+      } else if (activity == kActivityExternalCorrelation) {
+        selectedActivityTypes_.insert(ActivityType::EXTERNAL_CORRELATION);
+      } else if (activity == kActivityRuntime) {
+        selectedActivityTypes_.insert(ActivityType::RUNTIME);
+      } else {
+        throw std::invalid_argument(fmt::format(
+          "Invalid activity type selected: {}",
+          activity
+        ));
+      }
+    }
+  }
+}
+
 bool Config::handleOption(const std::string& name, std::string& val) {
   // Event Profiler
   if (name == kEventsKey) {
@@ -189,6 +223,9 @@ bool Config::handleOption(const std::string& name, std::string& val) {
   } else if (name == kMetricsKey) {
     vector<string> metric_names = splitAndTrim(val, ',');
     metricNames_.insert(metric_names.begin(), metric_names.end());
+  } else if (name == kActivityTypesKey) {
+    vector<string> activity_types = splitAndTrim(toLower(val), ',');
+    addActivityTypes(activity_types);
   } else if (name == kSamplePeriodKey) {
     samplePeriod_ = milliseconds(toInt32(val));
   } else if (name == kMultiplexPeriodKey) {
@@ -308,6 +345,10 @@ void Config::validate() {
     LOG(WARNING) << "Setting samples per report to " << max_samples_per_report;
     samplesPerReport_ = max_samples_per_report;
   }
+
+  if (selectedActivityTypes_.size() == 0) {
+    selectDefaultActivityTypes();
+  }
 }
 
 void Config::setReportPeriod(milliseconds msecs) {
@@ -340,6 +381,32 @@ void Config::printActivityProfilerConfig(std::ostream& s) const {
     << activitiesOnDemandExternalGpuOpCountThreshold() << std::endl;
   s << "Max GPU buffer size: " << activitiesMaxGpuBufferSize() / 1024 / 1024
     << "MB" << std::endl;
+
+  s << "Enabled activities: ";
+  for (const auto& activity : selectedActivityTypes_) {
+    switch(activity){
+      case ActivityType::MEMCPY:
+        s << kActivityMemcpy << " ";
+        break;
+      case ActivityType::MEMSET:
+        s << kActivityMemset << " ";
+        break;
+      case ActivityType::CONCURRENT_KERNEL:
+        s << kActivityConcurrentKernel << " ";
+        break;
+      case ActivityType::EXTERNAL_CORRELATION:
+        s << kActivityExternalCorrelation << " ";
+        break;
+      case ActivityType::RUNTIME:
+        s << kActivityRuntime << " ";
+        break;
+      default:
+        s << "UNKNOWN_ACTIVITY_NAME" << " ";
+        break;
+    }
+  }
+  s << std::endl;
+
   AbstractConfig::printActivityProfilerConfig(s);
 }
 

libkineto/src/Config.h

@@ -8,6 +8,7 @@
 #pragma once
 
 #include "AbstractConfig.h"
+#include "CuptiActivityType.h"
 
 #include <assert.h>
 #include <chrono>
@@ -139,6 +140,11 @@ class Config : public AbstractConfig {
     return eventProfilerMaxInstancesPerGpu_;
   }
 
+  // The types of activities selected in the configuration file
+  const std::set<ActivityType>& selectedActivityTypes() const {
+    return selectedActivityTypes_;
+  }
+
   // Trace for this long
   std::chrono::milliseconds activitiesOnDemandDuration() const {
     return activitiesOnDemandDuration_;
@@ -257,6 +263,20 @@ class Config : public AbstractConfig {
 
   uint8_t createDeviceMask(const std::string& val);
 
+  // Adds valid activity types from the user defined string list in the
+  // configuration file
+  void addActivityTypes(const std::vector<std::string>& selected_activities);
+
+  // Sets the default activity types to be traced
+  void selectDefaultActivityTypes() {
+    // If the user has not specified an activity list, add all types
+    selectedActivityTypes_.insert(ActivityType::MEMCPY);
+    selectedActivityTypes_.insert(ActivityType::MEMSET);
+    selectedActivityTypes_.insert(ActivityType::CONCURRENT_KERNEL);
+    selectedActivityTypes_.insert(ActivityType::EXTERNAL_CORRELATION);
+    selectedActivityTypes_.insert(ActivityType::RUNTIME);
+  }
+
   int verboseLogLevel_;
   std::vector<std::string> verboseLogModules_;
 
@@ -284,6 +304,7 @@ class Config : public AbstractConfig {
 
   // Activity profiler
   bool activityProfilerEnabled_;
+  std::set<ActivityType> selectedActivityTypes_;
 
   // The activity profiler settings are all on-demand
   std::string activitiesLogFile_;

libkineto/src/CuptiActivityInterface.cpp

@@ -195,27 +195,55 @@ void CUPTIAPI CuptiActivityInterface::bufferCompleted(
   }
 }
 
-void CuptiActivityInterface::enableCuptiActivities() {
+void CuptiActivityInterface::enableCuptiActivities(
+    const std::set<ActivityType>& selected_activities) {
   static bool registered = false;
   if (!registered) {
     CUPTI_CALL(
         cuptiActivityRegisterCallbacks(bufferRequested, bufferCompleted));
   }
-  CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY));
-  CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMSET));
-  CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL));
-  CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION));
-  CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME));
+
+  for (const auto& activity : selected_activities) {
+    if (activity == ActivityType::MEMCPY) {
+      CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY));
+    }
+    if (activity == ActivityType::MEMSET) {
+      CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMSET));
+    }
+    if (activity == ActivityType::CONCURRENT_KERNEL) {
+      CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL));
+    }
+    if (activity == ActivityType::EXTERNAL_CORRELATION) {
+      CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION));
+    }
+    if (activity == ActivityType::RUNTIME) {
+      CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME));
+    }
+  }
+
   // Explicitly enabled, so reset this flag if set
   stopCollection = false;
 }
 
-void CuptiActivityInterface::disableCuptiActivities() {
-  CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMCPY));
-  CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMSET));
-  CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL));
-  CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION));
-  CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_RUNTIME));
+void CuptiActivityInterface::disableCuptiActivities(
+    const std::set<ActivityType>& selected_activities) {
+  for (const auto& activity : selected_activities) {
+    if (activity == ActivityType::MEMCPY) {
+      CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMCPY));
+    }
+    if (activity == ActivityType::MEMSET) {
+      CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMSET));
+    }
+    if (activity == ActivityType::CONCURRENT_KERNEL) {
+      CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL));
+    }
+    if (activity == ActivityType::EXTERNAL_CORRELATION) {
+      CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION));
+    }
+    if (activity == ActivityType::RUNTIME) {
+      CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_RUNTIME));
+    }
+  }
 }
 
 } // namespace KINETO_NAMESPACE

libkineto/src/CuptiActivityInterface.h

@@ -7,10 +7,13 @@
 
 #pragma once
 
+#include "CuptiActivityType.h"
+
 #include <atomic>
 #include <cupti.h>
 #include <functional>
 #include <queue>
+#include <set>
 
 namespace KINETO_NAMESPACE {
 
@@ -25,8 +28,10 @@ class CuptiActivityInterface {
   static void pushCorrelationID(int id);
   static void popCorrelationID();
 
-  void enableCuptiActivities();
-  void disableCuptiActivities();
+  void enableCuptiActivities(
+    const std::set<ActivityType>& selected_activities);
+  void disableCuptiActivities(
+    const std::set<ActivityType>& selected_activities);
   void clearActivities();
 
   const std::pair<int, int> processActivities(

libkineto/src/CuptiActivityType.h

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ * All rights reserved.
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#pragma once
+
+namespace KINETO_NAMESPACE {
+
+enum class ActivityType {
+    MEMCPY,
+    MEMSET,
+    CONCURRENT_KERNEL,
+    EXTERNAL_CORRELATION,
+    RUNTIME
+};
+
+} // namespace KINETO_NAMESPACE

libkineto/test/ConfigTest.cpp

@@ -69,6 +69,48 @@ TEST(ParseTest, Format) {
   EXPECT_EQ(cfg.metricNames(), std::set<std::string>({"4", "5", "s i x"}));
 }
 
+TEST(ParseTest, DefaultActivityTypes) {
+  Config cfg;
+  cfg.validate();
+  EXPECT_EQ(cfg.selectedActivityTypes(),
+    std::set<ActivityType>({ActivityType::MEMCPY,
+                            ActivityType::MEMSET,
+                            ActivityType::CONCURRENT_KERNEL,
+                            ActivityType::EXTERNAL_CORRELATION,
+                            ActivityType::RUNTIME}));
+}
+
+TEST(ParseTest, ActivityTypes) {
+  Config cfg;
+  EXPECT_FALSE(cfg.parse("ACTIVITY_TYPES"));
+  EXPECT_TRUE(cfg.parse("ACTIVITY_TYPES="));
+  EXPECT_FALSE(cfg.parse("=ACTIVITY_TYPES="));
+
+  EXPECT_TRUE(cfg.parse("ACTIVITY_TYPES=gpu_memcpy,gpu_MeMsEt,concurrent_kernel"));
+  EXPECT_EQ(cfg.selectedActivityTypes(),
+    std::set<ActivityType>({ActivityType::MEMCPY,
+                            ActivityType::MEMSET,
+                            ActivityType::CONCURRENT_KERNEL}));
+
+  EXPECT_TRUE(cfg.parse("ACTIVITY_TYPES = cuda_Runtime,"));
+  EXPECT_EQ(cfg.selectedActivityTypes(),
+    std::set<ActivityType>({ActivityType::MEMCPY,
+                            ActivityType::MEMSET,
+                            ActivityType::RUNTIME,
+                            ActivityType::CONCURRENT_KERNEL}));
+
+  // Should throw an exception because incorrect activity name
+  EXPECT_FALSE(cfg.parse("ACTIVITY_TYPES = memcopy,cuda_runtime"));
+
+  EXPECT_TRUE(cfg.parse("ACTIVITY_TYPES = external_correlation"));
+  EXPECT_EQ(cfg.selectedActivityTypes(),
+    std::set<ActivityType>({ActivityType::MEMCPY,
+                            ActivityType::MEMSET,
+                            ActivityType::CONCURRENT_KERNEL,
+                            ActivityType::EXTERNAL_CORRELATION,
+                            ActivityType::RUNTIME}));
+}
+
 TEST(ParseTest, SamplePeriod) {
   Config cfg;
   EXPECT_TRUE(cfg.parse("SAMPLE_PERIOD_MSECS=10"));