[XPU profiler] Introduce XPU profiler by following kineto plugin design

libkineto/CMakeLists.txt

@@ -133,13 +133,20 @@ IF (NOT ROCM_SOURCE_DIR AND NOT ROCTRACER_INCLUDE_DIR)
     set(LIBKINETO_NOROCTRACER ON CACHE BOOL "" FORCE)
 endif()
 
+IF (DEFINED LIBKINETO_NOXPUPTI AND NOT LIBKINETO_NOXPUPTI)
+    add_subdirectory(src/plugin/xpupti)
+endif()
+
 # Define file lists
-if (LIBKINETO_NOCUPTI AND LIBKINETO_NOROCTRACER)
+if (LIBKINETO_NOCUPTI AND LIBKINETO_NOROCTRACER AND LIBKINETO_NOXPUPTI)
     get_filelist("get_libkineto_cpu_only_srcs(with_api=False)" LIBKINETO_SRCS)
     message(INFO " CUPTI unavailable or disabled - not building GPU profilers")
 elseif(NOT LIBKINETO_NOROCTRACER)
     get_filelist("get_libkineto_roctracer_srcs(with_api=False)" LIBKINETO_SRCS)
     message(INFO " Building with roctracer")
+elseif(DEFINED LIBKINETO_NOXPUPTI AND NOT LIBKINETO_NOXPUPTI)
+    get_filelist("get_libkineto_xpupti_srcs(with_api=False)" LIBKINETO_SRCS)
+    message(INFO " Building with xpupti")
 else()
     get_filelist("get_libkineto_cupti_srcs(with_api=False)" LIBKINETO_SRCS)
 endif()
@@ -171,6 +178,9 @@ endif()
 if (NOT LIBKINETO_NOCUPTI)
   list(APPEND KINETO_COMPILE_OPTIONS "-DHAS_CUPTI")
 endif()
+if (DEFINED LIBKINETO_NOXPUPTI AND NOT LIBKINETO_NOXPUPTI)
+  list(APPEND KINETO_COMPILE_OPTIONS ${XPUPTI_BUILD_FLAG})
+endif()
 if (CUDA_nvperf_host_LIBRARY)
   list(APPEND KINETO_COMPILE_OPTIONS "-DUSE_CUPTI_RANGE_PROFILER")
 endif()
@@ -239,6 +249,10 @@ target_include_directories(kineto_base PUBLIC
       $<BUILD_INTERFACE:${ROCTRACER_INCLUDE_DIR}>
       $<BUILD_INTERFACE:${ROCM_INCLUDE_DIRS}>)
 
+if(DEFINED LIBKINETO_NOXPUPTI AND NOT LIBKINETO_NOXPUPTI)
+  target_include_directories(kineto_base PUBLIC ${XPUPTI_INCLUDE_DIR})
+endif()
+
 target_include_directories(kineto_api PUBLIC
       $<BUILD_INTERFACE:${FMT_INCLUDE_DIR}>
       $<BUILD_INTERFACE:${LIBKINETO_INCLUDE_DIR}>)
@@ -278,6 +292,9 @@ endif()
 if(CUDA_nvperf_host_LIBRARY)
   target_link_libraries(kineto "${CUDA_nvperf_host_LIBRARY}")
 endif()
+if(DEFINED LIBKINETO_NOXPUPTI AND NOT LIBKINETO_NOXPUPTI)
+  target_link_libraries(kineto "${XPU_xpupti_LIBRARY}")
+endif()
 target_link_libraries(kineto $<BUILD_INTERFACE:fmt::fmt-header-only>)
 add_dependencies(kineto fmt::fmt-header-only)
 

libkineto/libkineto_defs.bzl

@@ -32,6 +32,13 @@ def get_libkineto_roctracer_srcs(with_api = True):
         "src/RoctracerLogger.cpp",
     ] + (get_libkineto_cpu_only_srcs(with_api))
 
+def get_libkineto_xpupti_srcs(with_api = True):
+    return [
+        "src/plugin/xpupti/XpuptiActivityApi.cpp",
+        "src/plugin/xpupti/XpuptiActivityProfiler.cpp",
+        "src/plugin/xpupti/XpuptiActivityHandlers.cpp",
+    ] + (get_libkineto_cpu_only_srcs(with_api))
+
 def get_libkineto_cpu_only_srcs(with_api = True):
     return [
         "src/AbstractConfig.cpp",

libkineto/src/init.cpp

@@ -22,6 +22,10 @@
 #include "CuptiRangeProfiler.h"
 #include "EventProfilerController.h"
 #endif
+#ifdef HAS_XPUPTI
+#include "plugin/xpupti/XpuptiActivityApi.h"
+#include "plugin/xpupti/XpuptiActivityProfiler.h"
+#endif
 #include "libkineto.h"
 
 #include "Logger.h"
@@ -194,6 +198,20 @@ void libkineto_init(bool cpuOnly, bool logOnError) {
   libkineto::api().registerProfiler(
       std::make_unique<ActivityProfilerProxy>(cpuOnly, config_loader));
 
+#ifdef HAS_XPUPTI
+  // register xpu pti profiler
+  libkineto::api().registerProfilerFactory([]() -> std::unique_ptr<IActivityProfiler> {
+    auto returnCode = ptiViewGPULocalAvailable();
+    if (returnCode != PTI_SUCCESS) {
+      std::string errCode = std::to_string(returnCode);
+      std::string errMsg(
+          "Fail to enable Kineto Profiler on XPU due to error code: ");
+      throw std::runtime_error(errMsg + errCode);
+    }
+    return std::make_unique<XPUActivityProfiler>();
+  });
+#endif // HAS_XPUPTI
+
 #if __linux__
   // When CUDA/GPU is used the profiler initialization happens on the
   // creation of the first CUDA stream (see initProfilers()).

libkineto/src/plugin/xpupti/CMakeLists.txt

@@ -0,0 +1,53 @@
+# xpupti: XPU implementation for kineto profiler
+# outputs:
+#  SYCL_INCLUDE_DIR   --  SYCL include dir
+#  SYCL_LIBRARY       --  SYCL library file
+#  XPU_xpupti_LIBRARY --  XPUPTI dependencies lib
+#  XPUPTI_BUILD_FLAG  --  XPUPTI build flags
+
+if((NOT SYCL_INCLUDE_DIR) OR (NOT SYCL_LIBRARY_DIR))
+    include(FindSYCLToolkit.cmake)
+    if(NOT SYCLTOOLKIT_FOUND)
+        set(LIBKINETO_NOXPUPTI ON CACHE STRING "" FORCE PARENT_SCOPE)
+        message(WARNING "XPU PTI has not built because ${SYCL_NOT_FOUND_MESSAGE}")
+        return()
+    endif()
+endif()
+
+message(INFO " SYCL_INCLUDE_DIR = ${SYCL_INCLUDE_DIR}")
+message(INFO " SYCL_LIBRARY = ${SYCL_LIBRARY}")
+
+set(XPUPTI_INCLUDE_DIR ${SYCL_INCLUDE_DIR})
+
+# find xpupti sdk
+find_package(Pti REQUIRED)
+if(TARGET Pti::pti_view)
+    message(INFO " Found XPUPTI")
+
+    get_target_property(PTI_INCLUDE_DIR Pti::pti_view INTERFACE_INCLUDE_DIRECTORIES)
+    find_library(PTI_VIEW_LIBRARY NAMES pti_view PATHS "${PTI_INCLUDE_DIR}/../lib")
+    set(PTI_LIBRARY ${PTI_VIEW_LIBRARY} CACHE STRING "Imported PTI library.")
+    set(PTI_INCLUDE_DIR ${PTI_INCLUDE_DIR} CACHE STRING "PTI include directory.")
+
+    # find dependent lib
+    set(XPU_xpupti_LIBRARY ${SYCL_LIBRARY})
+    list(APPEND XPU_xpupti_LIBRARY ${PTI_LIBRARY})
+    set(XPU_xpupti_LIBRARY ${XPU_xpupti_LIBRARY} PARENT_SCOPE)
+
+    # find dependent include
+    list(APPEND XPUPTI_INCLUDE_DIR ${PTI_INCLUDE_DIR})
+    set(XPUPTI_INCLUDE_DIR ${XPUPTI_INCLUDE_DIR} PARENT_SCOPE)
+
+    set(XPUPTI_BUILD_FLAG "-DHAS_XPUPTI" PARENT_SCOPE)
+
+    message(INFO " XPU_xpupti_LIBRARY = ${XPU_xpupti_LIBRARY}")
+    message(INFO " XPUPTI_INCLUDE_DIR = ${XPUPTI_INCLUDE_DIR}")
+    message(INFO " XPUPTI_BUILD_FLAG = ${XPUPTI_BUILD_FLAG}")
+else()
+    set(LIBKINETO_NOXPUPTI ON CACHE STRING "" FORCE PARENT_SCOPE)
+    set(XPU_xpupti_LIBRARY PARENT_SCOPE)
+    set(XPUPTI_BUILD_FLAG PARENT_SCOPE)
+    set(XPUPTI_INCLUDE_DIR PARENT_SCOPE)
+    message(WARNING " Could not find XPUPTI for building kineto")
+    return()
+endif()

libkineto/src/plugin/xpupti/FindSYCLToolkit.cmake

@@ -0,0 +1,110 @@
+#[=======================================================================[.rst:
+SYCLConfig
+-------
+
+Library to verify SYCL compatability of CMAKE_CXX_COMPILER
+and passes relevant compiler flags.
+
+Result Variables
+^^^^^^^^^^^^^^^^
+
+This will define the following variables:
+
+``SYCLTOOLKIT_FOUND``
+  True if the system has the SYCL library.
+``SYCL_COMPILER``
+  SYCL compiler executable.
+``SYCL_INCLUDE_DIR``
+  Include directories needed to use SYCL.
+``SYCL_LIBRARY_DIR``
+  Libaray directories needed to use SYCL.
+
+#]=======================================================================]
+
+set(SYCLTOOLKIT_FOUND False)
+include(${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake)
+
+set(SYCL_ROOT "")
+if(DEFINED ENV{SYCL_ROOT})
+  set(SYCL_ROOT $ENV{SYCL_ROOT})
+elseif(DEFINED ENV{CMPLR_ROOT})
+  set(SYCL_ROOT $ENV{CMPLR_ROOT})
+endif()
+
+if(WIN32)
+  set(SYCL_EXECUTABLE_NAME icx)
+else()
+  set(SYCL_EXECUTABLE_NAME icpx)
+endif()
+
+if(NOT SYCL_ROOT)
+  execute_process(
+    COMMAND which ${SYCL_EXECUTABLE_NAME}
+    OUTPUT_VARIABLE SYCL_CMPLR_FULL_PATH
+    OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+  if(NOT EXISTS "${SYCL_CMPLR_FULL_PATH}")
+    message(WARNING "Cannot find ENV{CMPLR_ROOT} or icpx, please setup SYCL compiler Tool kit enviroment before building!!")
+    return()
+  endif()
+
+  get_filename_component(SYCL_BIN_DIR "${SYCL_CMPLR_FULL_PATH}" DIRECTORY)
+  set(SYCL_ROOT ${SYCL_BIN_DIR}/..)
+endif()
+
+find_program(
+  SYCL_COMPILER
+  NAMES ${SYCL_EXECUTABLE_NAME}
+  PATHS "${SYCL_ROOT}"
+  PATH_SUFFIXES bin bin64
+  NO_DEFAULT_PATH
+  )
+
+string(COMPARE EQUAL "${SYCL_COMPILER}" "" nocmplr)
+if(nocmplr)
+  set(SYCLTOOLKIT_FOUND False)
+  set(SYCL_REASON_FAILURE "SYCL: CMAKE_CXX_COMPILER not set!!")
+  set(SYCL_NOT_FOUND_MESSAGE "${SYCL_REASON_FAILURE}")
+endif()
+
+find_file(
+  SYCL_INCLUDE_DIR
+  NAMES include
+  HINTS ${SYCL_ROOT}
+  NO_DEFAULT_PATH
+  )
+
+find_file(
+  SYCL_INCLUDE_SYCL_DIR
+  NAMES sycl
+  HINTS ${SYCL_ROOT}/include
+  NO_DEFAULT_PATH
+  )
+
+list(APPEND SYCL_INCLUDE_DIR ${SYCL_INCLUDE_SYCL_DIR})
+
+find_file(
+  SYCL_LIBRARY_DIR
+  NAMES lib lib64
+  HINTS ${SYCL_ROOT}
+  NO_DEFAULT_PATH
+  )
+
+find_library(
+  SYCL_LIBRARY
+  NAMES sycl
+  HINTS ${SYCL_LIBRARY_DIR}
+  NO_DEFAULT_PATH
+  )
+
+if((NOT SYCL_INCLUDE_DIR) OR (NOT SYCL_LIBRARY_DIR) OR (NOT SYCL_LIBRARY))
+  set(SYCLTOOLKIT_FOUND False)
+  set(SYCL_REASON_FAILURE "SYCL sdk is incomplete!!")
+  set(SYCL_NOT_FOUND_MESSAGE "${SYCL_REASON_FAILURE}")
+  return()
+endif()
+
+message(DEBUG "The SYCL compiler is ${SYCL_COMPILER}")
+message(DEBUG "The SYCL Flags are ${SYCL_FLAGS}")
+
+set(SYCLTOOLKIT_FOUND True)