Skip to content

Commit c06c8ac

Browse files
alugoreychinmaydk99
authored andcommitted
[ROCm] - Reduce generated CK kernel files and build by default (pytorch#178310)
Change to turn CK build on by default. Various filters were added to reduce the number of kernels generated by default Pull Request resolved: pytorch#178310 Approved by: https://github.com/jeffdaily, https://github.com/pruthvistony Co-authored-by: Chinmay Kuchinad <ChinmayDattanand.Kuchinad@amd.com>
1 parent cb841d0 commit c06c8ac

5 files changed

Lines changed: 270 additions & 127 deletions

File tree

aten/src/ATen/CMakeLists.txt

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -202,9 +202,15 @@ file(GLOB native_flash_attn_api_cpp "native/transformers/cuda/flash_attn/flash_a
202202
file(GLOB flash_attention_hip_hip "native/transformers/hip/flash_attn/*.hip")
203203
# if USE_FLASH_ATTENTION is set, ensure CK instances get generated
204204
if(USE_FLASH_ATTENTION)
205-
if("$ENV{USE_CK_FLASH_ATTENTION}" STREQUAL "1")
206-
message(STATUS "USE_CK_FLASH_ATTENTION is being deprecated. Please use USE_ROCM_CK_SDPA instead")
207-
caffe2_update_option(USE_ROCM_CK_SDPA ON)
205+
# Now building CK by default
206+
if(USE_ROCM)
207+
if(DEFINED ENV{USE_ROCM_CK_SDPA})
208+
if(ENV{USE_ROCM_CK_SDPA} EQUAL 1)
209+
caffe2_update_option(USE_ROCM_CK_SDPA ON)
210+
endif()
211+
else()
212+
caffe2_update_option(USE_ROCM_CK_SDPA ON)
213+
endif()
208214
endif()
209215
# Now building CK by default
210216
if(USE_ROCM)

aten/src/ATen/native/transformers/hip/flash_attn/ck/CMakeLists.txt

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# generate a list of kernels, but not actually emit files at config stage
22
execute_process(
3-
COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py
4-
--api fwd --receipt 4 --list_blobs ${CMAKE_CURRENT_LIST_DIR}/fwd_blob_list.txt
3+
COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --optdim=32,64,128,256
4+
--api fwd --receipt 4 --filter "*_lse*ntrload*nsink*" --list_blobs ${CMAKE_CURRENT_LIST_DIR}/fwd_blob_list.txt
55
RESULT_VARIABLE ret
66
)
77

@@ -10,8 +10,8 @@ if(ret AND NOT ret EQUAL 0)
1010
endif()
1111

1212
execute_process(
13-
COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py
14-
--api fwd_splitkv --receipt 4 --list_blobs ${CMAKE_CURRENT_LIST_DIR}/fwd_splitkv_blob_list.txt
13+
COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --optdim=32,64,128,256
14+
--api fwd_splitkv --receipt 4 --filter "*psdv*_lse*_nsquant*" --list_blobs ${CMAKE_CURRENT_LIST_DIR}/fwd_splitkv_blob_list.txt
1515
RESULT_VARIABLE ret
1616
)
1717

@@ -20,8 +20,8 @@ if(ret AND NOT ret EQUAL 0)
2020
endif()
2121

2222
execute_process(
23-
COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py
24-
--api fwd_appendkv --receipt 4 --list_blobs ${CMAKE_CURRENT_LIST_DIR}/fwd_appendkv_blob_list.txt
23+
COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --optdim=32,64,128,256
24+
--api fwd_appendkv --receipt 4 --filter "*psskddv_*" --list_blobs ${CMAKE_CURRENT_LIST_DIR}/fwd_appendkv_blob_list.txt
2525
RESULT_VARIABLE ret
2626
)
2727

@@ -31,7 +31,7 @@ endif()
3131

3232
execute_process(
3333
COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py
34-
--api bwd --receipt 4 --list_blobs ${CMAKE_CURRENT_LIST_DIR}/bwd_blob_list.txt
34+
--api bwd --optdim=32,64,128,256 --receipt 4 --filter "*psdv*@*psd*@*_pd1dv1*_ntrload*" --list_blobs ${CMAKE_CURRENT_LIST_DIR}/bwd_blob_list.txt
3535
RESULT_VARIABLE ret
3636
)
3737

@@ -40,28 +40,28 @@ if(ret AND NOT ret EQUAL 0)
4040
endif()
4141

4242
# Generate the files for both fwd, fwd_splitkv, fwd_appendkv, and bwd
43-
execute_process(COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --api fwd --receipt 4 --output_dir ${CMAKE_CURRENT_LIST_DIR}
43+
execute_process(COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --api fwd --optdim=32,64,128,256 --receipt 4 --filter "*_lse*ntrload*nsink*" --output_dir ${CMAKE_CURRENT_LIST_DIR}
4444
)
4545

4646
if(ret AND NOT ret EQUAL 0)
4747
message( FATAL_ERROR "CK Tile FMHA FAILED to generate FWD kernels.")
4848
endif()
4949

50-
execute_process(COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --api fwd_splitkv --receipt 4 --output_dir ${CMAKE_CURRENT_LIST_DIR}
50+
execute_process(COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --api fwd_splitkv --optdim=32,64,128,256 --receipt 4 --filter "*psdv*_lse*_nsquant*" --output_dir ${CMAKE_CURRENT_LIST_DIR}
5151
)
5252

5353
if(ret AND NOT ret EQUAL 0)
5454
message( FATAL_ERROR "CK Tile FMHA FAILED to generate FWD_SPLITKV kernels.")
5555
endif()
5656

57-
execute_process(COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --api fwd_appendkv --receipt 4 --output_dir ${CMAKE_CURRENT_LIST_DIR}
57+
execute_process(COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --api fwd_appendkv --optdim=32,64,128,256 --receipt 4 --filter "*psskddv_*" --output_dir ${CMAKE_CURRENT_LIST_DIR}
5858
)
5959

6060
if(ret AND NOT ret EQUAL 0)
6161
message( FATAL_ERROR "CK Tile FMHA FAILED to generate FWD_APPENDKV kernels.")
6262
endif()
6363

64-
execute_process(COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --api bwd --receipt 4 --output_dir ${CMAKE_CURRENT_LIST_DIR}
64+
execute_process(COMMAND python3 ${CMAKE_SOURCE_DIR}/third_party/composable_kernel/example/ck_tile/01_fmha/generate.py --api bwd --optdim=32,64,128,256 --receipt 4 --filter "*psdv*@*psd*@*_pd1dv1*_ntrload*" --output_dir ${CMAKE_CURRENT_LIST_DIR}
6565
RESULT_VARIABLE ret
6666
)
6767

aten/src/ATen/native/transformers/hip/flash_attn/ck/fav_v3/CMakeLists.txt

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,38 @@
11
include(CMakePrintHelpers)
22

3-
# Generate AITER/CK Asm code
3+
# Create an isolated venv for codegen.py which requires pandas>=2.1.0 (for
4+
# the "future.no_silent_downcasting" option). This avoids polluting the CI /
5+
# host Python environment while still satisfying the dependency.
6+
set(CODEGEN_VENV_DIR "${CMAKE_CURRENT_BINARY_DIR}/_codegen_venv")
7+
8+
if(NOT EXISTS "${CODEGEN_VENV_DIR}/bin/python3")
9+
message(STATUS "Creating isolated codegen venv at ${CODEGEN_VENV_DIR}")
10+
execute_process(
11+
COMMAND python3 -m venv "${CODEGEN_VENV_DIR}"
12+
RESULT_VARIABLE ret
13+
)
14+
if(ret AND NOT ret EQUAL 0)
15+
message(FATAL_ERROR "Failed to create codegen venv")
16+
endif()
17+
18+
execute_process(
19+
COMMAND "${CODEGEN_VENV_DIR}/bin/pip" install --quiet "pandas>=2.1.0,<=2.3.3" numpy
20+
RESULT_VARIABLE ret
21+
)
22+
if(ret AND NOT ret EQUAL 0)
23+
message(FATAL_ERROR "Failed to install pandas/numpy into codegen venv")
24+
endif()
25+
endif()
26+
27+
# Generate AITER/CK Asm code (using the venv python so pandas is new enough)
428
execute_process(
529
COMMAND ${CMAKE_COMMAND} -E env "AITER_GPU_ARCHS=gfx942;gfx950"
6-
python3 ${CMAKE_SOURCE_DIR}/third_party/aiter/hsa/codegen.py -m fmha_v3_bwd --output_dir ${CMAKE_CURRENT_LIST_DIR}
30+
"${CODEGEN_VENV_DIR}/bin/python3" ${CMAKE_SOURCE_DIR}/third_party/aiter/hsa/codegen.py -m fmha_v3_bwd --output_dir ${CMAKE_CURRENT_LIST_DIR}
731
RESULT_VARIABLE ret
832
)
933

1034
if(ret AND NOT ret EQUAL 0)
11-
message( FATAL_ERROR "Failed to generate FAv3 CK Kernels")
35+
message(FATAL_ERROR "Failed to generate FAv3 CK Kernels")
1236
endif()
1337

1438
execute_process(COMMAND bash -c "cp ${CMAKE_SOURCE_DIR}/third_party/aiter/csrc/cpp_itfs/mha_bwd.cu ${CMAKE_CURRENT_LIST_DIR}/mha_bwd.hip")

aten/src/ATen/native/transformers/hip/flash_attn/ck/mha_fwd_ck.hip

Lines changed: 143 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,100 @@
66
#include <fmha_fwd.hpp>
77
#include <mask.hpp>
88

9+
#include <cstdio>
10+
11+
namespace {
12+
/* Debug traces. Comment in to aid in debug of 'Invalid argument to fmha_fwd' error
13+
const char* mask_enum_str(mask_enum m) {
14+
switch (m) {
15+
case mask_enum::no_mask: return "no_mask";
16+
case mask_enum::mask_top_left: return "mask_top_left";
17+
case mask_enum::mask_bottom_right:return "mask_bottom_right";
18+
case mask_enum::window_generic: return "window_generic";
19+
default: return "unknown";
20+
}
21+
}
22+
23+
const char* bias_enum_str(bias_enum b) {
24+
switch (b) {
25+
case bias_enum::no_bias: return "no_bias";
26+
case bias_enum::elementwise_bias: return "elementwise_bias";
27+
case bias_enum::alibi: return "alibi";
28+
default: return "unknown";
29+
}
30+
}
31+
32+
const char* qscale_enum_str(quant_scale_enum q) {
33+
switch (q) {
34+
case quant_scale_enum::no_scale: return "no_scale";
35+
case quant_scale_enum::pertensor: return "pertensor";
36+
case quant_scale_enum::blockscale: return "blockscale";
37+
case quant_scale_enum::kv_blockscale: return "kv_blockscale";
38+
default: return "unknown";
39+
}
40+
}
41+
42+
void dump_fmha_fwd_kernel_selection_params(const fmha_fwd_traits& t,
43+
const fmha_fwd_args& a) {
44+
std::printf("=== fmha_fwd kernel selection params ===\n");
45+
std::printf("--- traits ---\n");
46+
std::printf(" data_type : %s\n", t.data_type.c_str());
47+
std::printf(" hdim_q : %d\n", t.hdim_q);
48+
std::printf(" hdim_v : %d\n", t.hdim_v);
49+
std::printf(" is_group_mode : %s\n", t.is_group_mode ? "true" : "false");
50+
std::printf(" is_v_rowmajor : %s\n", t.is_v_rowmajor ? "true" : "false");
51+
std::printf(" has_logits_soft_cap: %s\n", t.has_logits_soft_cap ? "true" : "false");
52+
std::printf(" mask_type : %s (%d)\n", mask_enum_str(t.mask_type),
53+
static_cast<int>(t.mask_type));
54+
std::printf(" bias_type : %s (%d)\n", bias_enum_str(t.bias_type),
55+
static_cast<int>(t.bias_type));
56+
std::printf(" has_lse : %s\n", t.has_lse ? "true" : "false");
57+
std::printf(" has_dropout : %s\n", t.has_dropout ? "true" : "false");
58+
std::printf(" qscale_type : %s (%d)\n", qscale_enum_str(t.qscale_type),
59+
static_cast<int>(t.qscale_type));
60+
std::printf(" skip_min_seqlen_q : %s\n", t.skip_min_seqlen_q ? "true" : "false");
61+
std::printf(" has_sink : %s\n", t.has_sink ? "true" : "false");
62+
std::printf("--- args ---\n");
63+
std::printf(" hdim_q : %d\n", static_cast<int>(a.hdim_q));
64+
std::printf(" hdim_v : %d\n", static_cast<int>(a.hdim_v));
65+
std::printf(" seqlen_k : %d\n", static_cast<int>(a.seqlen_k));
66+
std::printf(" cu_seqlen_k_ptr : %p\n", a.cu_seqlen_k_ptr);
67+
std::printf(" batch : %d\n", static_cast<int>(a.batch));
68+
std::printf(" nhead_q : %d\n", static_cast<int>(a.nhead_q));
69+
std::printf(" max_seqlen_q : %d\n", static_cast<int>(a.max_seqlen_q));
70+
std::printf("========================================\n");
71+
std::fflush(stdout);
72+
}
73+
*/
74+
} // anonymous namespace
975

1076
namespace pytorch_flash {
1177

78+
// SFINAE trait to detect block-scale quantization fields in fmha_fwd_args.
79+
// Newer versions of composable_kernel add these fields; older versions don't.
80+
// This lets the PyTorch integration layer work with either version.
81+
template <typename T, typename = void>
82+
struct has_fmha_block_scale_fields : std::false_type {};
83+
84+
template <typename T>
85+
struct has_fmha_block_scale_fields<T,
86+
std::void_t<decltype(std::declval<T&>().block_scale_size_q)>> : std::true_type {};
87+
88+
template <typename Args>
89+
void set_fmha_fwd_block_scale_fields([[maybe_unused]] Args& args) {
90+
if constexpr (has_fmha_block_scale_fields<Args>::value) {
91+
args.block_scale_seqstart_q_ptr = nullptr;
92+
args.block_scale_seqstart_k_ptr = nullptr;
93+
args.nhead_stride_q_descale = 0;
94+
args.nhead_stride_k_descale = 0;
95+
args.nhead_stride_v_descale = 0;
96+
args.batch_stride_q_descale = 0;
97+
args.batch_stride_k_descale = 0;
98+
args.batch_stride_v_descale = 0;
99+
args.block_scale_size_q = 0;
100+
args.block_scale_size_kv = 0;
101+
}
102+
}
12103

13104
fmha_fwd_traits get_ck_fmha_fwd_traits(const mask_info &mask,
14105
std::string dtype,
@@ -96,61 +187,58 @@ fmha_fwd_args get_ck_fmha_fwd_args(bool has_lse,
96187
nhead_stride_bias = a_b.stride(1);
97188
batch_stride_bias = a_b.stride(0);
98189
}
99-
return fmha_fwd_args{q.data_ptr(),
100-
k.data_ptr(),
101-
v.data_ptr(),
102-
attn_bias_ptr, // bias
103-
nullptr, // q_descale_ptr
104-
nullptr, // k_descale_ptr
105-
nullptr, // v_descale_ptr
106-
has_dropout_randval ? dropout_randval.data_ptr() : nullptr,
107-
has_lse ? softmax_lse.data_ptr() : nullptr,
108-
out.data_ptr(),
109-
nullptr, // seqstart_q
110-
nullptr, // seqstart_k
111-
nullptr, // seqlen_q_ptr
112-
nullptr, // seqlen_k_ptr
113-
nullptr, // cu_seqlen_q_ptr
114-
nullptr, // cu_seqlen_k_ptr
115-
nullptr, // sink_ptr
116-
seqlen_q,
117-
seqlen_k,
118-
b,
119-
seqlen_q, // max_seqlen_q
120-
d, // hdim_q
121-
d, // hdim_v
122-
h, // nhead
123-
h_k, // nhead_k
124-
softmax_scale, // scale_s
125-
0.0f, // logits_soft_cap
126-
stride_q,
127-
stride_k,
128-
stride_v,
129-
stride_attn_bias,
130-
stride_randval,
131-
stride_o,
132-
nhead_stride_q,
133-
nhead_stride_k,
134-
nhead_stride_v,
135-
nhead_stride_bias, // nhead_stride_bias
136-
nhead_stride_randval,
137-
nhead_stride_lse,
138-
nhead_stride_o,
139-
batch_stride_q,
140-
batch_stride_k,
141-
batch_stride_v,
142-
batch_stride_bias, // batch_stride_bias
143-
batch_stride_randval,
144-
batch_stride_lse,
145-
batch_stride_o,
146-
mask.left,
147-
mask.right,
148-
0, // sink_size
149-
static_cast<ck_tile::index_t>(mask.type),
150-
-1, // min_seqlen_q
151-
p_dropout,
152-
has_dropout_randval,
153-
drop_seed_offset};
190+
fmha_fwd_args args{};
191+
args.q_ptr = q.data_ptr();
192+
args.k_ptr = k.data_ptr();
193+
args.v_ptr = v.data_ptr();
194+
args.bias_ptr = attn_bias_ptr;
195+
args.q_descale_ptr = nullptr;
196+
args.k_descale_ptr = nullptr;
197+
args.v_descale_ptr = nullptr;
198+
args.rand_val_ptr = has_dropout_randval ? dropout_randval.data_ptr() : nullptr;
199+
args.lse_ptr = has_lse ? softmax_lse.data_ptr() : nullptr;
200+
args.o_ptr = out.data_ptr();
201+
args.sink_ptr = nullptr;
202+
args.seqlen_q = seqlen_q;
203+
args.seqlen_k = seqlen_k;
204+
args.batch = b;
205+
args.max_seqlen_q = seqlen_q;
206+
args.hdim_q = d;
207+
args.hdim_v = d;
208+
args.nhead_q = h;
209+
args.nhead_k = h_k;
210+
args.scale_s = softmax_scale;
211+
args.logits_soft_cap = 0.0f;
212+
args.stride_q = stride_q;
213+
args.stride_k = stride_k;
214+
args.stride_v = stride_v;
215+
args.stride_bias = stride_attn_bias;
216+
args.stride_randval = stride_randval;
217+
args.stride_o = stride_o;
218+
args.nhead_stride_q = nhead_stride_q;
219+
args.nhead_stride_k = nhead_stride_k;
220+
args.nhead_stride_v = nhead_stride_v;
221+
args.nhead_stride_bias = nhead_stride_bias;
222+
args.nhead_stride_randval = nhead_stride_randval;
223+
args.nhead_stride_lse = nhead_stride_lse;
224+
args.nhead_stride_o = nhead_stride_o;
225+
args.batch_stride_q = batch_stride_q;
226+
args.batch_stride_k = batch_stride_k;
227+
args.batch_stride_v = batch_stride_v;
228+
args.batch_stride_bias = batch_stride_bias;
229+
args.batch_stride_randval = batch_stride_randval;
230+
args.batch_stride_lse = batch_stride_lse;
231+
args.batch_stride_o = batch_stride_o;
232+
args.window_size_left = mask.left;
233+
args.window_size_right = mask.right;
234+
args.sink_size = 0;
235+
args.mask_type = static_cast<ck_tile::index_t>(mask.type);
236+
args.min_seqlen_q = -1;
237+
args.p_drop = p_dropout;
238+
args.s_randval = has_dropout_randval;
239+
args.drop_seed_offset = drop_seed_offset;
240+
set_fmha_fwd_block_scale_fields(args);
241+
return args;
154242
}
155243

156244
std::tuple<at::Tensor, at::Tensor, at::Tensor, at::Tensor, at::Tensor, at::Tensor, at::Tensor, at::Tensor>

0 commit comments

Comments
 (0)