Skip to content

Commit 0cded49

Browse files
zejun-chenfacebook-github-bot
zejun-chen
authored and
facebook-github-bot
committed
Introduce XPU profiler by following kineto plugin design (#961)
Summary: As XPU became a PyTorch built-in device, the profiler support is indispensable part of functionality completeness. In this PR, the XPU profiler is introduced by following kineto plugin design under libkineto/src/plugin/xpupti. The XPU profiler plugin is built on the foundation of intel PTI toolkit (https://github.com/intel/pti-gpu), and underlying SYCL runtime. The LIBKINETO_NOXPUPTI option is added to enable or disable the XPU profiler plugin during kineto build stage. CC: aaronenyeshi briancoutinho davidberard98 sraikund16 Pull Request resolved: #961 Reviewed By: xuzhao9 Differential Revision: D60830913 Pulled By: aaronenyeshi fbshipit-source-id: a24444e1ab1ed074bfcf5a9012076fa7c193b178
1 parent 1bb9b76 commit 0cded49

12 files changed

+1248
-1
lines changed

‎libkineto/CMakeLists.txt

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,13 +133,20 @@ IF (NOT ROCM_SOURCE_DIR AND NOT ROCTRACER_INCLUDE_DIR)
133133
set(LIBKINETO_NOROCTRACER ON CACHE BOOL "" FORCE)
134134
endif()
135135

136+
IF (DEFINED LIBKINETO_NOXPUPTI AND NOT LIBKINETO_NOXPUPTI)
137+
add_subdirectory(src/plugin/xpupti)
138+
endif()
139+
136140
# Define file lists
137-
if (LIBKINETO_NOCUPTI AND LIBKINETO_NOROCTRACER)
141+
if (LIBKINETO_NOCUPTI AND LIBKINETO_NOROCTRACER AND LIBKINETO_NOXPUPTI)
138142
get_filelist("get_libkineto_cpu_only_srcs(with_api=False)" LIBKINETO_SRCS)
139143
message(INFO " CUPTI unavailable or disabled - not building GPU profilers")
140144
elseif(NOT LIBKINETO_NOROCTRACER)
141145
get_filelist("get_libkineto_roctracer_srcs(with_api=False)" LIBKINETO_SRCS)
142146
message(INFO " Building with roctracer")
147+
elseif(DEFINED LIBKINETO_NOXPUPTI AND NOT LIBKINETO_NOXPUPTI)
148+
get_filelist("get_libkineto_xpupti_srcs(with_api=False)" LIBKINETO_SRCS)
149+
message(INFO " Building with xpupti")
143150
else()
144151
get_filelist("get_libkineto_cupti_srcs(with_api=False)" LIBKINETO_SRCS)
145152
endif()
@@ -171,6 +178,9 @@ endif()
171178
if (NOT LIBKINETO_NOCUPTI)
172179
list(APPEND KINETO_COMPILE_OPTIONS "-DHAS_CUPTI")
173180
endif()
181+
if (DEFINED LIBKINETO_NOXPUPTI AND NOT LIBKINETO_NOXPUPTI)
182+
list(APPEND KINETO_COMPILE_OPTIONS ${XPUPTI_BUILD_FLAG})
183+
endif()
174184
if (CUDA_nvperf_host_LIBRARY)
175185
list(APPEND KINETO_COMPILE_OPTIONS "-DUSE_CUPTI_RANGE_PROFILER")
176186
endif()
@@ -239,6 +249,10 @@ target_include_directories(kineto_base PUBLIC
239249
$<BUILD_INTERFACE:${ROCTRACER_INCLUDE_DIR}>
240250
$<BUILD_INTERFACE:${ROCM_INCLUDE_DIRS}>)
241251

252+
if(DEFINED LIBKINETO_NOXPUPTI AND NOT LIBKINETO_NOXPUPTI)
253+
target_include_directories(kineto_base PUBLIC ${XPUPTI_INCLUDE_DIR})
254+
endif()
255+
242256
target_include_directories(kineto_api PUBLIC
243257
$<BUILD_INTERFACE:${FMT_INCLUDE_DIR}>
244258
$<BUILD_INTERFACE:${LIBKINETO_INCLUDE_DIR}>)
@@ -278,6 +292,9 @@ endif()
278292
if(CUDA_nvperf_host_LIBRARY)
279293
target_link_libraries(kineto "${CUDA_nvperf_host_LIBRARY}")
280294
endif()
295+
if(DEFINED LIBKINETO_NOXPUPTI AND NOT LIBKINETO_NOXPUPTI)
296+
target_link_libraries(kineto "${XPU_xpupti_LIBRARY}")
297+
endif()
281298
target_link_libraries(kineto $<BUILD_INTERFACE:fmt::fmt-header-only>)
282299
add_dependencies(kineto fmt::fmt-header-only)
283300

‎libkineto/libkineto_defs.bzl

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@ def get_libkineto_roctracer_srcs(with_api = True):
3232
"src/RoctracerLogger.cpp",
3333
] + (get_libkineto_cpu_only_srcs(with_api))
3434

35+
def get_libkineto_xpupti_srcs(with_api = True):
36+
return [
37+
"src/plugin/xpupti/XpuptiActivityApi.cpp",
38+
"src/plugin/xpupti/XpuptiActivityProfiler.cpp",
39+
"src/plugin/xpupti/XpuptiActivityHandlers.cpp",
40+
] + (get_libkineto_cpu_only_srcs(with_api))
41+
3542
def get_libkineto_cpu_only_srcs(with_api = True):
3643
return [
3744
"src/AbstractConfig.cpp",

‎libkineto/src/init.cpp

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,10 @@
2222
#include "CuptiRangeProfiler.h"
2323
#include "EventProfilerController.h"
2424
#endif
25+
#ifdef HAS_XPUPTI
26+
#include "plugin/xpupti/XpuptiActivityApi.h"
27+
#include "plugin/xpupti/XpuptiActivityProfiler.h"
28+
#endif
2529
#include "libkineto.h"
2630

2731
#include "Logger.h"
@@ -194,6 +198,20 @@ void libkineto_init(bool cpuOnly, bool logOnError) {
194198
libkineto::api().registerProfiler(
195199
std::make_unique<ActivityProfilerProxy>(cpuOnly, config_loader));
196200

201+
#ifdef HAS_XPUPTI
202+
// register xpu pti profiler
203+
libkineto::api().registerProfilerFactory([]() -> std::unique_ptr<IActivityProfiler> {
204+
auto returnCode = ptiViewGPULocalAvailable();
205+
if (returnCode != PTI_SUCCESS) {
206+
std::string errCode = std::to_string(returnCode);
207+
std::string errMsg(
208+
"Fail to enable Kineto Profiler on XPU due to error code: ");
209+
throw std::runtime_error(errMsg + errCode);
210+
}
211+
return std::make_unique<XPUActivityProfiler>();
212+
});
213+
#endif // HAS_XPUPTI
214+
197215
#if __linux__
198216
// When CUDA/GPU is used the profiler initialization happens on the
199217
// creation of the first CUDA stream (see initProfilers()).

‎libkineto/src/plugin/xpupti/CMakeLists.txt

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
# xpupti: XPU implementation for kineto profiler
2+
# outputs:
3+
# SYCL_INCLUDE_DIR -- SYCL include dir
4+
# SYCL_LIBRARY -- SYCL library file
5+
# XPU_xpupti_LIBRARY -- XPUPTI dependencies lib
6+
# XPUPTI_BUILD_FLAG -- XPUPTI build flags
7+
8+
if((NOT SYCL_INCLUDE_DIR) OR (NOT SYCL_LIBRARY_DIR))
9+
include(FindSYCLToolkit.cmake)
10+
if(NOT SYCLTOOLKIT_FOUND)
11+
set(LIBKINETO_NOXPUPTI ON CACHE STRING "" FORCE PARENT_SCOPE)
12+
message(WARNING "XPU PTI has not built because ${SYCL_NOT_FOUND_MESSAGE}")
13+
return()
14+
endif()
15+
endif()
16+
17+
message(INFO " SYCL_INCLUDE_DIR = ${SYCL_INCLUDE_DIR}")
18+
message(INFO " SYCL_LIBRARY = ${SYCL_LIBRARY}")
19+
20+
set(XPUPTI_INCLUDE_DIR ${SYCL_INCLUDE_DIR})
21+
22+
# find xpupti sdk
23+
find_package(Pti REQUIRED)
24+
if(TARGET Pti::pti_view)
25+
message(INFO " Found XPUPTI")
26+
27+
get_target_property(PTI_INCLUDE_DIR Pti::pti_view INTERFACE_INCLUDE_DIRECTORIES)
28+
find_library(PTI_VIEW_LIBRARY NAMES pti_view PATHS "${PTI_INCLUDE_DIR}/../lib")
29+
set(PTI_LIBRARY ${PTI_VIEW_LIBRARY} CACHE STRING "Imported PTI library.")
30+
set(PTI_INCLUDE_DIR ${PTI_INCLUDE_DIR} CACHE STRING "PTI include directory.")
31+
32+
# find dependent lib
33+
set(XPU_xpupti_LIBRARY ${SYCL_LIBRARY})
34+
list(APPEND XPU_xpupti_LIBRARY ${PTI_LIBRARY})
35+
set(XPU_xpupti_LIBRARY ${XPU_xpupti_LIBRARY} PARENT_SCOPE)
36+
37+
# find dependent include
38+
list(APPEND XPUPTI_INCLUDE_DIR ${PTI_INCLUDE_DIR})
39+
set(XPUPTI_INCLUDE_DIR ${XPUPTI_INCLUDE_DIR} PARENT_SCOPE)
40+
41+
set(XPUPTI_BUILD_FLAG "-DHAS_XPUPTI" PARENT_SCOPE)
42+
43+
message(INFO " XPU_xpupti_LIBRARY = ${XPU_xpupti_LIBRARY}")
44+
message(INFO " XPUPTI_INCLUDE_DIR = ${XPUPTI_INCLUDE_DIR}")
45+
message(INFO " XPUPTI_BUILD_FLAG = ${XPUPTI_BUILD_FLAG}")
46+
else()
47+
set(LIBKINETO_NOXPUPTI ON CACHE STRING "" FORCE PARENT_SCOPE)
48+
set(XPU_xpupti_LIBRARY PARENT_SCOPE)
49+
set(XPUPTI_BUILD_FLAG PARENT_SCOPE)
50+
set(XPUPTI_INCLUDE_DIR PARENT_SCOPE)
51+
message(WARNING " Could not find XPUPTI for building kineto")
52+
return()
53+
endif()

‎libkineto/src/plugin/xpupti/FindSYCLToolkit.cmake

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
#[=======================================================================[.rst:
2+
SYCLConfig
3+
-------
4+
5+
Library to verify SYCL compatability of CMAKE_CXX_COMPILER
6+
and passes relevant compiler flags.
7+
8+
Result Variables
9+
^^^^^^^^^^^^^^^^
10+
11+
This will define the following variables:
12+
13+
``SYCLTOOLKIT_FOUND``
14+
True if the system has the SYCL library.
15+
``SYCL_COMPILER``
16+
SYCL compiler executable.
17+
``SYCL_INCLUDE_DIR``
18+
Include directories needed to use SYCL.
19+
``SYCL_LIBRARY_DIR``
20+
Libaray directories needed to use SYCL.
21+
22+
#]=======================================================================]
23+
24+
set(SYCLTOOLKIT_FOUND False)
25+
include(${CMAKE_ROOT}/Modules/FindPackageHandleStandardArgs.cmake)
26+
27+
set(SYCL_ROOT "")
28+
if(DEFINED ENV{SYCL_ROOT})
29+
set(SYCL_ROOT $ENV{SYCL_ROOT})
30+
elseif(DEFINED ENV{CMPLR_ROOT})
31+
set(SYCL_ROOT $ENV{CMPLR_ROOT})
32+
endif()
33+
34+
if(WIN32)
35+
set(SYCL_EXECUTABLE_NAME icx)
36+
else()
37+
set(SYCL_EXECUTABLE_NAME icpx)
38+
endif()
39+
40+
if(NOT SYCL_ROOT)
41+
execute_process(
42+
COMMAND which ${SYCL_EXECUTABLE_NAME}
43+
OUTPUT_VARIABLE SYCL_CMPLR_FULL_PATH
44+
OUTPUT_STRIP_TRAILING_WHITESPACE)
45+
46+
if(NOT EXISTS "${SYCL_CMPLR_FULL_PATH}")
47+
message(WARNING "Cannot find ENV{CMPLR_ROOT} or icpx, please setup SYCL compiler Tool kit enviroment before building!!")
48+
return()
49+
endif()
50+
51+
get_filename_component(SYCL_BIN_DIR "${SYCL_CMPLR_FULL_PATH}" DIRECTORY)
52+
set(SYCL_ROOT ${SYCL_BIN_DIR}/..)
53+
endif()
54+
55+
find_program(
56+
SYCL_COMPILER
57+
NAMES ${SYCL_EXECUTABLE_NAME}
58+
PATHS "${SYCL_ROOT}"
59+
PATH_SUFFIXES bin bin64
60+
NO_DEFAULT_PATH
61+
)
62+
63+
string(COMPARE EQUAL "${SYCL_COMPILER}" "" nocmplr)
64+
if(nocmplr)
65+
set(SYCLTOOLKIT_FOUND False)
66+
set(SYCL_REASON_FAILURE "SYCL: CMAKE_CXX_COMPILER not set!!")
67+
set(SYCL_NOT_FOUND_MESSAGE "${SYCL_REASON_FAILURE}")
68+
endif()
69+
70+
find_file(
71+
SYCL_INCLUDE_DIR
72+
NAMES include
73+
HINTS ${SYCL_ROOT}
74+
NO_DEFAULT_PATH
75+
)
76+
77+
find_file(
78+
SYCL_INCLUDE_SYCL_DIR
79+
NAMES sycl
80+
HINTS ${SYCL_ROOT}/include
81+
NO_DEFAULT_PATH
82+
)
83+
84+
list(APPEND SYCL_INCLUDE_DIR ${SYCL_INCLUDE_SYCL_DIR})
85+
86+
find_file(
87+
SYCL_LIBRARY_DIR
88+
NAMES lib lib64
89+
HINTS ${SYCL_ROOT}
90+
NO_DEFAULT_PATH
91+
)
92+
93+
find_library(
94+
SYCL_LIBRARY
95+
NAMES sycl
96+
HINTS ${SYCL_LIBRARY_DIR}
97+
NO_DEFAULT_PATH
98+
)
99+
100+
if((NOT SYCL_INCLUDE_DIR) OR (NOT SYCL_LIBRARY_DIR) OR (NOT SYCL_LIBRARY))
101+
set(SYCLTOOLKIT_FOUND False)
102+
set(SYCL_REASON_FAILURE "SYCL sdk is incomplete!!")
103+
set(SYCL_NOT_FOUND_MESSAGE "${SYCL_REASON_FAILURE}")
104+
return()
105+
endif()
106+
107+
message(DEBUG "The SYCL compiler is ${SYCL_COMPILER}")
108+
message(DEBUG "The SYCL Flags are ${SYCL_FLAGS}")
109+
110+
set(SYCLTOOLKIT_FOUND True)

0 commit comments

Comments
 (0)