|
| 1 | +/* |
| 2 | + * Copyright (c) 2020-2023, NVIDIA CORPORATION. All rights reserved. |
| 3 | + * |
| 4 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | + * you may not use this file except in compliance with the License. |
| 6 | + * You may obtain a copy of the License at |
| 7 | + * |
| 8 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | + * |
| 10 | + * Unless required by applicable law or agreed to in writing, software |
| 11 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | + * See the License for the specific language governing permissions and |
| 14 | + * limitations under the License. |
| 15 | + */ |
| 16 | +#include <cuda_fp16.h> |
| 17 | + |
| 18 | +#include <cstddef> |
| 19 | +#include <cstdint> |
| 20 | +#include <functional> |
| 21 | +#include <type_traits> |
| 22 | +#include <vector> |
| 23 | + |
| 24 | +#include "flashinfer/gemm/cutlass_gemm_configs.h" |
| 25 | +#include "flashinfer/gemm/fp4_gemm_cutlass.h" |
| 26 | +#include "flashinfer/gemm/fp4_gemm_cutlass_template_sm103.h" |
| 27 | +#include "tvm_ffi_utils.h" |
| 28 | + |
| 29 | +using flashinfer::gemm::ClusterShape; |
| 30 | +using flashinfer::gemm::CutlassFp4GemmRunner; |
| 31 | +using flashinfer::gemm::CutlassFp4GemmRunnerInterface; |
| 32 | +using flashinfer::gemm::CutlassGemmConfig; |
| 33 | +using flashinfer::gemm::CutlassTileConfigSM100; |
| 34 | +using flashinfer::gemm::EpilogueScheduleType; |
| 35 | +using flashinfer::gemm::FP4GemmType; |
| 36 | +using flashinfer::gemm::MainloopScheduleType; |
| 37 | + |
| 38 | +namespace flashinfer { |
| 39 | +namespace gemm { |
| 40 | +template class CutlassFp4GemmRunner<__nv_bfloat16, FP4GemmType::W4A4_NVFP4_NVFP4>; |
| 41 | +template class CutlassFp4GemmRunner<half, FP4GemmType::W4A4_NVFP4_NVFP4>; |
| 42 | +} // namespace gemm |
| 43 | +} // namespace flashinfer |
| 44 | + |
| 45 | +namespace torch_ext { |
| 46 | + |
| 47 | +namespace { |
| 48 | + |
| 49 | +CutlassGemmConfig getFp4GemmConfig(int64_t m, int64_t n, int64_t k, int64_t tactic) { |
| 50 | + auto getCutlassFp4GemmConfigs = []() { |
| 51 | + CutlassFp4GemmRunner<__nv_bfloat16, FP4GemmType::W4A4_NVFP4_NVFP4> gemmRunner; |
| 52 | + return gemmRunner.getConfigs(); |
| 53 | + }; |
| 54 | + static std::vector<CutlassGemmConfig> globalConfigs = getCutlassFp4GemmConfigs(); |
| 55 | + TVM_FFI_ICHECK(tactic >= 0 && tactic < globalConfigs.size()) |
| 56 | + << "tactic must be between 0 and " << globalConfigs.size(); |
| 57 | + return globalConfigs[tactic]; |
| 58 | +} |
| 59 | + |
| 60 | +template <typename T> |
| 61 | +void runGemm(TensorView out, TensorView mat1, TensorView mat2, TensorView mat1Scale, |
| 62 | + TensorView mat2Scale, TensorView globalScale, int64_t m, int64_t n, int64_t k, |
| 63 | + int64_t batch_count, CutlassGemmConfig const& gemmConfig, |
| 64 | + TensorView workspace_buffer) { |
| 65 | + CutlassFp4GemmRunner<T, FP4GemmType::W4A4_NVFP4_NVFP4> gemmRunner; |
| 66 | + |
| 67 | + int64_t const required_workspace_size = gemmRunner.getWorkspaceSize(m, n, k, batch_count); |
| 68 | + int64_t const provided_workspace_size = |
| 69 | + workspace_buffer.numel() * get_element_size(workspace_buffer); |
| 70 | + |
| 71 | + auto runKernel = [&](void* workspace) { |
| 72 | + gemmRunner.gemm(out.data_ptr(), mat1.data_ptr(), mat2.data_ptr(), mat1Scale.data_ptr(), |
| 73 | + mat2Scale.data_ptr(), static_cast<float*>(globalScale.data_ptr()), m, n, k, |
| 74 | + batch_count, gemmConfig, reinterpret_cast<char*>(workspace), |
| 75 | + required_workspace_size, get_stream(mat1.device())); |
| 76 | + }; |
| 77 | + |
| 78 | + if (provided_workspace_size < required_workspace_size) { |
| 79 | + Tensor new_workspace = |
| 80 | + alloc_tensor({required_workspace_size}, DLDataType{kDLInt, 8, 1}, mat1.device()); |
| 81 | + runKernel(new_workspace.data_ptr()); |
| 82 | + } else { |
| 83 | + runKernel(workspace_buffer.data_ptr()); |
| 84 | + } |
| 85 | +} |
| 86 | + |
| 87 | +constexpr auto FLOAT4_E2M1X2 = dl_uint8; // uint8_t |
| 88 | +constexpr auto SF_DTYPE = dl_uint8; // uint8_t |
| 89 | + |
| 90 | +// mat1: [B, M, K / 2], FLOAT4_E2M1X2 or [B, M, K], FLOAT8_E4M3FN |
| 91 | +// mat2: [B, N, K / 2], FLOAT4_E2M1X2 |
| 92 | +// out: [B, M, N], fp16/bf16/fp32 |
| 93 | +// mat1Scale: ceil(M / 128) * 128 * ceil(K / sfVecSize / 4) * 4, SF_DTYPE (UE4M3 or UE8M0) |
| 94 | +// mat2Scale: ceil(N / 128) * 128 * ceil(K / sfVecSize / 4) * 4, SF_DTYPE (UE4M3 or UE8M0) |
| 95 | +// globalScale: [1], 1 / (((448 * 6) / mat1.abs().max()) * ((448 * 6) / mat2.abs().max())) |
| 96 | +// B = 1 for GEMM op as a special case |
| 97 | +void fp4_bmm_impl(TensorView mat1, TensorView mat2, TensorView mat1Scale, TensorView mat2Scale, |
| 98 | + TensorView globalScale, TensorView out, TensorView workspace_buffer, |
| 99 | + int64_t tactic) { |
| 100 | + CHECK_INPUT_AND_TYPE(mat1, FLOAT4_E2M1X2); |
| 101 | + CHECK_INPUT_AND_TYPE(mat2, FLOAT4_E2M1X2); |
| 102 | + |
| 103 | + int mat2_k_scale = 1; |
| 104 | + |
| 105 | + CHECK_INPUT_AND_TYPE(mat1Scale, SF_DTYPE); |
| 106 | + CHECK_INPUT_AND_TYPE(mat2Scale, SF_DTYPE); |
| 107 | + |
| 108 | + CHECK_INPUT_AND_TYPE(globalScale, dl_float32); |
| 109 | + |
| 110 | + int64_t m, n, k, b; |
| 111 | + if (mat1.ndim() == 2) { |
| 112 | + TVM_FFI_ICHECK_EQ(mat2.ndim(), 2) << "mat2 must be a matrix"; |
| 113 | + TVM_FFI_ICHECK_EQ(mat1.size(1), mat2.size(1) * mat2_k_scale) |
| 114 | + << "mat1 and mat2 shapes cannot be multiplied (" << mat1.size(0) << "x" << mat1.size(1) |
| 115 | + << " and " << mat2.size(0) << "x" << mat2.size(1) << ")"; |
| 116 | + m = mat1.size(0); |
| 117 | + n = mat2.size(0); |
| 118 | + k = mat2.size(1) * 2; |
| 119 | + b = 1; |
| 120 | + } else if (mat1.ndim() == 3) { |
| 121 | + TVM_FFI_ICHECK_EQ(mat2.ndim(), 3) << "mat2 must be a batch of matrices"; |
| 122 | + TVM_FFI_ICHECK_EQ(mat1.size(0), mat2.size(0)) << "mat1 and mat2 must have the same batch size (" |
| 123 | + << mat1.size(0) << " and " << mat2.size(0) << ")"; |
| 124 | + TVM_FFI_ICHECK_EQ(mat1.size(2), mat2.size(2) * mat2_k_scale) |
| 125 | + << "mat1 and mat2 shapes cannot be multiplied (" << mat1.size(1) << "x" << mat1.size(2) |
| 126 | + << " and " << mat2.size(1) << "x" << mat2.size(2) << ")"; |
| 127 | + m = mat1.size(1); |
| 128 | + n = mat2.size(1); |
| 129 | + k = mat2.size(2) * 2; |
| 130 | + b = mat1.size(0); |
| 131 | + } else { |
| 132 | + TVM_FFI_LOG_AND_THROW(NotImplementedError) << "mat1 must be a matrix or a batch of matrices"; |
| 133 | + } |
| 134 | + |
| 135 | + // No heuristic for now, we rely on the autotuner to select the best tactic. |
| 136 | + if (tactic == -1) { |
| 137 | + tactic = 0; |
| 138 | + } |
| 139 | + auto config = getFp4GemmConfig(m, n, k, tactic); |
| 140 | + |
| 141 | + constexpr int alignment = 32; |
| 142 | + TVM_FFI_ICHECK_EQ(k % alignment, 0) |
| 143 | + << "Expected k to be divisible by " << alignment << ", but got mat1 shape: (" << mat1.size(0) |
| 144 | + << "x" << mat1.size(1) << "), k: " << k << "."; |
| 145 | + TVM_FFI_ICHECK_EQ(n % alignment, 0) |
| 146 | + << "Expected n to be divisible by " << alignment << ", but got mat2 shape: (" << mat2.size(0) |
| 147 | + << "x" << mat2.size(1) << ")."; |
| 148 | + |
| 149 | + // Validate out dimensions |
| 150 | + std::vector<int64_t> out_shape = |
| 151 | + mat1.ndim() == 2 ? std::vector<int64_t>{m, n} : std::vector<int64_t>{b, m, n}; |
| 152 | + TVM_FFI_ICHECK_EQ(out.ndim(), out_shape.size()) |
| 153 | + << "out must have " << out_shape.size() << " dimensions, but got " << out.ndim(); |
| 154 | + for (int i = 0; i < out_shape.size(); ++i) { |
| 155 | + TVM_FFI_ICHECK_EQ(out.size(i), out_shape[i]) |
| 156 | + << "out shape mismatch at dimension " << i << ": expected " << out_shape[i] << ", got " |
| 157 | + << out.size(i); |
| 158 | + } |
| 159 | + |
| 160 | + switch (encode_dlpack_dtype(out.dtype())) { |
| 161 | + case float16_code: |
| 162 | + runGemm<half>(out, mat1, mat2, mat1Scale, mat2Scale, globalScale, m, n, k, b, config, |
| 163 | + workspace_buffer); |
| 164 | + break; |
| 165 | + case bfloat16_code: |
| 166 | + runGemm<__nv_bfloat16>(out, mat1, mat2, mat1Scale, mat2Scale, globalScale, m, n, k, b, config, |
| 167 | + workspace_buffer); |
| 168 | + break; |
| 169 | + default: |
| 170 | + TVM_FFI_ICHECK(false) << "out_dtype must be one of fp16/bf16."; |
| 171 | + } |
| 172 | +} |
| 173 | + |
| 174 | +} // namespace |
| 175 | + |
| 176 | +void fp4_gemm(TensorView mat1, TensorView mat2, TensorView mat1Scale, TensorView mat2Scale, |
| 177 | + TensorView globalScale, TensorView out, TensorView workspace_buffer, int64_t tactic) { |
| 178 | + fp4_bmm_impl(mat1, mat2, mat1Scale, mat2Scale, globalScale, out, workspace_buffer, tactic); |
| 179 | +} |
| 180 | + |
| 181 | +int64_t fp4_gemm_tactic_num() { |
| 182 | + auto getCutlassConfigs = []() { |
| 183 | + CutlassFp4GemmRunner<__nv_bfloat16, FP4GemmType::W4A4_NVFP4_NVFP4> gemmRunner; |
| 184 | + return gemmRunner.getConfigs(); |
| 185 | + }; |
| 186 | + static int64_t totalTactics = getCutlassConfigs().size(); |
| 187 | + return totalTactics; |
| 188 | +} |
| 189 | + |
| 190 | +} // namespace torch_ext |
| 191 | + |
| 192 | +TVM_FFI_DLL_EXPORT_TYPED_FUNC(fp4_gemm, torch_ext::fp4_gemm); |
| 193 | +TVM_FFI_DLL_EXPORT_TYPED_FUNC(fp4_gemm_tactic_num, torch_ext::fp4_gemm_tactic_num); |
0 commit comments