diff --git a/common/arg.cpp b/common/arg.cpp index ab23b77e0218..1ad44ff6acdf 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -765,6 +765,7 @@ static void common_params_print_completion(common_params_context & ctx_arg) { "llama-batched", "llama-batched-bench", "llama-bench", + "llama-benchmark-matmul", "llama-cli", "llama-completion", "llama-convert-llama2c-to-ggml", diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index a29dc707c3dc..68efcb159749 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -15,6 +15,7 @@ llama_add_compile_flags() if (EMSCRIPTEN) else() add_subdirectory(batched) + add_subdirectory(benchmark) add_subdirectory(debug) add_subdirectory(embedding) add_subdirectory(eval-callback) diff --git a/examples/benchmark/CMakeLists.txt b/examples/benchmark/CMakeLists.txt new file mode 100644 index 000000000000..2c7970b6647c --- /dev/null +++ b/examples/benchmark/CMakeLists.txt @@ -0,0 +1,5 @@ +set(TARGET llama-bench-matmult) +add_executable(${TARGET} benchmark-matmult.cpp) +install(TARGETS ${TARGET} RUNTIME) +target_link_libraries(${TARGET} PRIVATE llama-common llama ${CMAKE_THREAD_LIBS_INIT}) +target_compile_features(${TARGET} PRIVATE cxx_std_17) diff --git a/examples/benchmark/benchmark-matmult-q4.cpp b/examples/benchmark/benchmark-matmult-q4.cpp new file mode 100644 index 000000000000..fb8ac85f5fd2 --- /dev/null +++ b/examples/benchmark/benchmark-matmult-q4.cpp @@ -0,0 +1,345 @@ +#include "common.h" +#include "ggml.h" +#include "build-info.h" +#include "../../ggml/src/ggml-common.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) { + struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, nullptr); + + if (plan.work_size > 0) { + buf.resize(plan.work_size); + plan.work_data = buf.data(); + } + + ggml_graph_compute(graph, &plan); +} + +static float tensor_sum_elements(const ggml_tensor * tensor) { + double sum1; + if (tensor->type == GGML_TYPE_F32) { + for (int j = 0; j < tensor->ne[1]; j++) { + for (int k = 0; k < tensor->ne[0]; k++) { + sum1 = sum1 + ((float *) tensor->data)[j*tensor->ne[0] + k]; + } + printf("\n"); + } + } + return sum1; +} + +static void tensor_dump(const ggml_tensor * tensor, const char * name) { + printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi) - \n", name, + tensor->type, ggml_type_name(tensor->type), + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->nb[0], tensor->nb[1], tensor->nb[2]); + float sum = tensor_sum_elements(tensor); + printf("Sum of tensor %s is %6.2f\n", name, sum); + printf("tensor data\n"); + int rows = tensor->ne[1]; + int cols = tensor->ne[0]; + float *data = (float*)tensor->data; + for (int r = 0; r < rows; r++) { + for (int c = 0; c < cols; c++) { + printf("%.2f ", data[r*cols+c]); + } + printf("\n"); + } +} + +/*static void tensor_dump_dequant(const ggml_tensor * tensor, const char * name) { + int rows = tensor->ne[1]; + int cols = tensor->ne[0]; + + // allocate temp buffer + std::vector tmp(ggml_nelements(tensor)); + ggml_get_data_f32(tensor, tmp.data()); + + printf("%15s (dequantized): type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 "\n", + name, tensor->type, ggml_type_name(tensor->type), tensor->ne[0], tensor->ne[1]); + + for (int r = 0; r < rows; r++) { + for (int c = 0; c < cols; c++) { + printf("%.2f ", tmp[r*cols + c]); + } + printf("\n"); + } +}*/ + +#define TENSOR_DUMP(tensor) tensor_dump(tensor, #tensor) + +struct benchmark_params_struct { + int n_threads = 1; + int32_t n_iterations = 10; +}; + +static void print_usage(int /*argc*/, char ** argv, struct benchmark_params_struct params) { + fprintf(stderr, "usage: %s [options]\n", argv[0]); + fprintf(stderr, "\n"); + fprintf(stderr, "options:\n"); + fprintf(stderr, " -h, --help show this help message and exit\n"); + fprintf(stderr, " -t N, --threads N number of threads to use during computation (default: %d)\n", params.n_threads); + fprintf(stderr, " -i N, --iter N number of iterations to use during computation (default: %d)\n", params.n_iterations); + fprintf(stderr, "\n"); +} + +int main(int argc, char ** argv) { + struct benchmark_params_struct benchmark_params; + + bool invalid_param = false; + std::string arg; + for (int i = 1; i < argc; i++) { + arg = argv[i]; + + if (arg == "-t" || arg == "--threads") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.n_threads = std::stoi(argv[i]); + } else if (arg == "-i" || arg == "--iter") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.n_iterations = std::stoi(argv[i]); + } else if (arg == "-h" || arg == "--help") { + print_usage(argc, argv, benchmark_params); + exit(0); + } + } + if (invalid_param) { + fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); + print_usage(argc, argv, benchmark_params); + exit(1); + } + + llama_print_build_info(); + printf("Starting Test\n"); + + // create the ggml context + struct ggml_context * ctx; + //const int sizex = 4096; + //const int sizey = 11008; + +#undef VERBOSE_DEBUGGING +#ifndef VERBOSE_DEBUGGING + /* + const int sizey = 4096; + const int sizex = 11008; + const int sizez = 128; + */ + const int sizey = 16; + const int sizex = 3*32; + const int sizez = 16; +#else + /* Working - let's increase size */ + const int sizey = 1; + const int sizex = (8*32); + const int sizez = 1; + + /*const int sizey = 1; + const int sizex = 3*(8*32); + const int sizez = 1;*/ +#endif + + //printf("Memsize required = %i\n", sizex*sizex); + + // TODO: perform the bench for all types or for a user specified type + const ggml_type qtype1 = GGML_TYPE_Q8_0; + const ggml_type qtype2 = GGML_TYPE_Q4_0; + + size_t ctx_size = 0; + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizez); + ctx_size += ggml_row_size(qtype1, sizex*sizey); + ctx_size += ggml_row_size(qtype1, sizex*sizez); + ctx_size += ggml_row_size(qtype1, sizex*sizey); + ctx_size += ggml_row_size(qtype2, sizex*sizey); + ctx_size += ggml_row_size(qtype2, sizex*sizez); + ctx_size += ggml_row_size(qtype2, sizex*sizey); + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); // BLAS + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); // BLAS + ctx_size += 1024*1024*16; + + printf("Allocating Memory of size %zi bytes, %zi MB\n",ctx_size, (ctx_size/1024/1024)); + + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /* no_alloc =*/ 0 + }; + + ctx = ggml_init(params); + if (!ctx) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return 1; + } + + + printf("Creating new tensors\n"); + // printf("Creating new tensor m1\n"); + struct ggml_tensor * m11 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizey); + //ggml_set_f32(m11, -1.23f); + float *m11_data = (float *) m11->data; + float x = 1.0f; + for (int r = 0; r < m11->ne[1]; r++) { + for (int c = 0; c < m11->ne[0]; c++) { + m11_data[r*m11->ne[0] + c] = x++;//(float)(r*m11->ne[0] + c + 1); + } + } + + TENSOR_DUMP(m11); + // printf("Creating new tensor m1\n"); + struct ggml_tensor * m12 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizey); + ggml_set_f32(m12, 1.5f); + + // printf("Creating new tensor m2\n"); + struct ggml_tensor * m2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizez); + float *m2_data = (float *) m2->data; + //ggml_set_f32(m2, -12.23f); + float y = 2.0f; + for (int r = 0; r < m2->ne[1]; r++) { + for (int c = 0; c < m2->ne[0]; c++) { + m2_data[r*m2->ne[0] + c] = y++;//(float)(r*m11->ne[0] + c + 1); + } + } + + printf("\n------ Test 1 - Matrix Mult via F32 code\n"); + // printf("Creating new tensor m11xm2\n"); + struct ggml_tensor * m11xm2 = ggml_mul_mat(ctx, m11, m2); + + // printf("Creating compute graph\n"); + struct ggml_cgraph * gf = ggml_new_graph(ctx); + ggml_build_forward_expand(gf, m11xm2); + + printf("n_threads=%i\n", benchmark_params.n_threads); + + //TENSOR_DUMP(m11); + //TENSOR_DUMP(m2); + + std::vector work_buffer; + + ggml_graph_compute_helper(work_buffer, gf, benchmark_params.n_threads); + + //TENSOR_DUMP(ggml_graph_node(gf, 0)); + + printf("\n------ Test 2 - Matrix Mult via %s and %s code\n", ggml_type_name(qtype1), ggml_type_name(qtype2)); + + int32_t nelements = sizex*sizey; + int32_t nelements2 = sizex*sizez; + + // Set up a the benchmark matrices + printf("Creating new tensor of type Q8_0 & Running quantize\n"); + struct ggml_tensor * q1 = ggml_new_tensor_2d(ctx, qtype1, sizex, sizey); + ggml_quantize_chunk(qtype1, (const float *) m11->data, q1->data, 0, nelements/m11->ne[0], m11->ne[0], nullptr); + //tensor_dump_dequant(q1, "Q4_0 q1 tensor"); + + printf("Creating new tensor of type Q4_0 & Running quantize\n"); + struct ggml_tensor * q2 = ggml_new_tensor_2d(ctx, qtype2, sizex, sizez); + ggml_quantize_chunk(qtype2, (const float *) m2->data, q2->data, 0, nelements2/m2->ne[0], m2->ne[0], nullptr); + //TENSOR_QDUMP(q2); + + printf("Creating new tensor of type Q8_0 & Running quantize\n"); + struct ggml_tensor * q3 = ggml_new_tensor_2d(ctx, qtype1, sizex, sizez); + ggml_quantize_chunk(qtype1, (const float *) m2->data, q3->data, 0, nelements2/m2->ne[0], m2->ne[0], nullptr); + TENSOR_DUMP(q3); + + // Set up a the compute graph + // printf("Creating new tensor q31\n"); + struct ggml_tensor * qprod = ggml_mul_mat(ctx, q2, q1); + //struct ggml_tensor * qprod = ggml_mul_mat(ctx, q3, q1); + + // printf("Creating compute graph\n"); + struct ggml_cgraph * gf31 = ggml_new_graph(ctx); + ggml_build_forward_expand(gf31, qprod); + + // Set up a second graph computation to make sure we override the CPU cache lines + // printf("Creating new tensor q12 & Running quantize\n"); + struct ggml_tensor * q12 = ggml_new_tensor_2d(ctx, qtype1, sizex, sizey); + ggml_quantize_chunk(qtype1, (const float *) m12->data, q12->data, 0, nelements/m12->ne[0], m12->ne[0], nullptr); + + // printf("Creating new tensor q32\n"); + struct ggml_tensor * q32 = ggml_mul_mat(ctx, q12, m2); + + //printf("Creating compute graph\n"); + struct ggml_cgraph * gf32 = ggml_new_graph(ctx); + ggml_build_forward_expand(gf32, q32); + printf("n_threads=%i\n", benchmark_params.n_threads); + + const int dimx = sizex; + const int dimy = sizey; + const int dimz = sizez; + long long int flops_per_dot_product = dimy + dimy; + long long int flops_per_matrix = flops_per_dot_product * dimx * dimz; ; + printf("Matrix Multiplication of (%i,%i,%i) x (%i,%i,%i) - about %6.2f gFLOPS\n\n", sizex, sizey, 1, sizex, sizez, 1, 1.0f*flops_per_matrix / 1000 / 1000 / 1000); + + + // Let's use the F32 result from above as a reference for the quantized multiplication + float sum_of_F32_reference = tensor_sum_elements(ggml_graph_node(gf, 0)); + + printf("Iteration;NThreads; SizeX; SizeY; SizeZ; Required_FLOPS; Elapsed_u_Seconds; gigaFLOPS\n"); + printf("=====================================================================================\n"); + + double gflops_sum = 0; + for (int i=0;i allowed_delta) { + printf("\nABORT - ERROR in Matrix Multiplication result - expected %6.2f, got %6.2f (delta %6.2f > allowed_delta %6.2f)\n", + sum_of_F32_reference, + sum_of_Q4_result, + delta, + allowed_delta + ); + exit(0); + } + + // Running a different graph computation to make sure we override the CPU cache lines + ggml_graph_compute_helper(work_buffer, gf32, benchmark_params.n_threads); + } + printf("\n"); + printf("Average%78.2f\n",gflops_sum/((double)benchmark_params.n_iterations)); + printf("=====================================================================================\n"); +} diff --git a/examples/benchmark/benchmark-matmult-q8.cpp b/examples/benchmark/benchmark-matmult-q8.cpp new file mode 100644 index 000000000000..cb60bcd1db3c --- /dev/null +++ b/examples/benchmark/benchmark-matmult-q8.cpp @@ -0,0 +1,319 @@ +#include "common.h" +#include "ggml.h" +#include "build-info.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) { + struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, nullptr); + + if (plan.work_size > 0) { + buf.resize(plan.work_size); + plan.work_data = buf.data(); + } + + ggml_graph_compute(graph, &plan); +} + +static float tensor_sum_elements(const ggml_tensor * tensor) { + double sum1; + //printf("sum inside = %f\n", sum1); + if (tensor->type == GGML_TYPE_F32) { + for (int j = 0; j < tensor->ne[1]; j++) { + for (int k = 0; k < tensor->ne[0]; k++) { + //printf("sum inside = %f\n", sum1); + printf("%f \t ", ((float *) tensor->data)[j*tensor->ne[0] + k]); + sum1 = sum1 + ((float *) tensor->data)[j*tensor->ne[0] + k]; + //printf("sum inside = %f\n", sum1); + } + printf("\n"); + } + } + return sum1; +} + +static void tensor_dump(const ggml_tensor * tensor, const char * name) { + printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi) - ", name, + tensor->type, ggml_type_name(tensor->type), + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->nb[0], tensor->nb[1], tensor->nb[2]); + float sum = tensor_sum_elements(tensor); + printf("Sum of tensor %s is %6.2f\n", name, sum); +} + +#define TENSOR_DUMP(tensor) tensor_dump(tensor, #tensor) + +struct benchmark_params_struct { + int n_threads = 1; + int32_t n_iterations = 10; + int m = 512; + int n = 512; + int k = 64; +}; +static void print_usage(int /*argc*/, char ** argv, struct benchmark_params_struct params) { + fprintf(stderr, "usage: %s [options]\n", argv[0]); + fprintf(stderr, "\n"); + fprintf(stderr, "options:\n"); + fprintf(stderr, " -h, --help show this help message and exit\n"); + fprintf(stderr, " -t N, --threads N number of threads to use (default: %d)\n", params.n_threads); + fprintf(stderr, " -i N, --iter N number of iterations to use (default: %d)\n", params.n_iterations); + fprintf(stderr, " -m N, --m N matrix dimension m (default: %d)\n", params.m); + fprintf(stderr, " -n N, --n N matrix dimension n (default: %d)\n", params.n); + fprintf(stderr, " -k N, --k N matrix dimension k (default: %d)\n", params.k); + fprintf(stderr, "\n"); +} + + +int main(int argc, char ** argv) { + struct benchmark_params_struct benchmark_params; + + bool invalid_param = false; + std::string arg; + for (int i = 1; i < argc; i++) { + arg = argv[i]; + + if (arg == "-t" || arg == "--threads") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.n_threads = std::stoi(argv[i]); + } else if (arg == "-i" || arg == "--iter") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.n_iterations = std::stoi(argv[i]); + } else if (arg == "-m") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.m = std::stoi(argv[i]); + } else if (arg == "-n") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.n = std::stoi(argv[i]); + }else if (arg == "-k") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.k = std::stoi(argv[i]); + }else if (arg == "-h" || arg == "--help") { + print_usage(argc, argv, benchmark_params); + exit(0); + } + } + if (invalid_param) { + fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); + print_usage(argc, argv, benchmark_params); + exit(1); + } + + llama_print_build_info(); + printf("Starting Test\n"); + + // create the ggml context + struct ggml_context * ctx; + //const int sizex = 4096; + //const int sizey = 11008; + +#undef VERBOSE_DEBUGGING +#ifndef VERBOSE_DEBUGGING + /* + const int sizey = 4096; + const int sizex = 11008; + const int sizez = 128; + */ + const int sizey = benchmark_params.m; //8; // m + const int sizex = 32*benchmark_params.k; //32*2; // k + const int sizez = benchmark_params.n; //4; // n +#else + /* Working - let's increase size */ + const int sizey = 1; + const int sizex = (8*32); + const int sizez = 1; + + /*const int sizey = 1; + const int sizex = 3*(8*32); + const int sizez = 1;*/ +#endif + + //printf("Memsize required = %i\n", sizex*sizex); + + // TODO: perform the bench for all types or for a user specified type + const ggml_type qtype = GGML_TYPE_Q8_0; + + size_t ctx_size = 0; + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizez); + ctx_size += ggml_row_size(qtype, sizex*sizey); + ctx_size += ggml_row_size(qtype, sizex*sizez); + ctx_size += ggml_row_size(qtype, sizex*sizey); + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); // BLAS + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); // BLAS + ctx_size += 1024*1024*16; + + printf("Allocating Memory of size %zi bytes, %zi MB\n",ctx_size, (ctx_size/1024/1024)); + + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /* no_alloc =*/ 0 + }; + + ctx = ggml_init(params); + if (!ctx) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return 1; + } + + + printf("Creating new tensors\n"); + // printf("Creating new tensor m1\n"); + struct ggml_tensor * m11 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizey); + ggml_set_f32(m11, 1.0f); + + // printf("Creating new tensor m1\n"); + struct ggml_tensor * m12 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizey); + ggml_set_f32(m12, 33.0f); + + // printf("Creating new tensor m2\n"); + struct ggml_tensor * m2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizez); + ggml_set_f32(m2, 2.0f); + + printf("\n------ Test 1 - Matrix Mult via F32 code\n"); + // printf("Creating new tensor m11xm2\n"); + struct ggml_tensor * m11xm2 = ggml_mul_mat(ctx, m11, m2); + + // printf("Creating compute graph\n"); + struct ggml_cgraph * gf = ggml_new_graph(ctx); + ggml_build_forward_expand(gf, m11xm2); + + printf("n_threads=%i\n", benchmark_params.n_threads); + printf("m =%d n = %d k = %d\n", benchmark_params.m, benchmark_params.n, benchmark_params.k); + //printf("Printing inputs fp32 matrices\n"); + //TENSOR_DUMP(m11); + //TENSOR_DUMP(m2); + + std::vector work_buffer; + + ggml_graph_compute_helper(work_buffer, gf, benchmark_params.n_threads); + + //TENSOR_DUMP(ggml_graph_node(gf, 0)); + + printf("\n------ Test 2 - Matrix Mult via %s code\n", ggml_type_name(qtype)); + + int32_t nelements = sizex*sizey; + int32_t nelements2 = sizex*sizez; + + // Set up a the benchmark matrices + // printf("Creating new tensor q11 & Running quantize\n"); + struct ggml_tensor * q11 = ggml_new_tensor_2d(ctx, qtype, sizex, sizey); + ggml_quantize_chunk(qtype, (const float *) m11->data, q11->data, 0, nelements/m11->ne[0], m11->ne[0], nullptr); + //TENSOR_DUMP(q11); + + // printf("Creating new tensor q2 & Running quantize\n"); + struct ggml_tensor * q2 = ggml_new_tensor_2d(ctx, qtype, sizex, sizez); + ggml_quantize_chunk(qtype, (const float *) m2->data, q2->data, 0, nelements2/m2->ne[0], m2->ne[0], nullptr); + //TENSOR_DUMP(q2); + + // Set up a the compute graph + // printf("Creating new tensor q31\n"); + struct ggml_tensor * q31 = ggml_mul_mat(ctx, q11, q2); + + // printf("Creating compute graph\n"); + struct ggml_cgraph * gf31 = ggml_new_graph(ctx); + ggml_build_forward_expand(gf31, q31); + + // Set up a second graph computation to make sure we override the CPU cache lines + // printf("Creating new tensor q12 & Running quantize\n"); + struct ggml_tensor * q12 = ggml_new_tensor_2d(ctx, qtype, sizex, sizey); + ggml_quantize_chunk(qtype, (const float *) m12->data, q12->data, 0, nelements/m12->ne[0], m12->ne[0], nullptr); + + // printf("Creating new tensor q32\n"); + struct ggml_tensor * q32 = ggml_mul_mat(ctx, q12, m2); + + //printf("Creating compute graph\n"); + struct ggml_cgraph * gf32 = ggml_new_graph(ctx); + ggml_build_forward_expand(gf32, q32); + printf("n_threads=%i\n", benchmark_params.n_threads); + + const int dimx = sizex; + const int dimy = sizey; + const int dimz = sizez; + long long int flops_per_dot_product = dimy + dimy; + long long int flops_per_matrix = flops_per_dot_product * dimx * dimz; ; + printf("Matrix Multiplication of (%i,%i,%i) x (%i,%i,%i) - about %6.2f gFLOPS\n\n", sizex, sizey, 1, sizex, sizez, 1, 1.0f*flops_per_matrix / 1000 / 1000 / 1000); + + + // Let's use the F32 result from above as a reference for the quantized multiplication + float sum_of_F32_reference = tensor_sum_elements(ggml_graph_node(gf, 0)); + + printf("Iteration;NThreads; SizeX; SizeY; SizeZ; Required_FLOPS; Elapsed_u_Seconds; gigaFLOPS\n"); + printf("=====================================================================================\n"); + + double gflops_sum = 0; + for (int i=0;i allowed_delta) { + printf("\nABORT - ERROR in Matrix Multiplication result - expected %6.2f, got %6.2f (delta %6.2f > allowed_delta %6.2f)\n", + sum_of_F32_reference, + sum_of_Q8_result, + delta, + allowed_delta + ); + exit(0); + } + + // Running a different graph computation to make sure we override the CPU cache lines + ggml_graph_compute_helper(work_buffer, gf32, benchmark_params.n_threads); + } + printf("\n"); + printf("Average%78.2f\n",gflops_sum/((double)benchmark_params.n_iterations)); + printf("=====================================================================================\n"); +} diff --git a/examples/benchmark/benchmark-matmult.cpp b/examples/benchmark/benchmark-matmult.cpp new file mode 100644 index 000000000000..cb60bcd1db3c --- /dev/null +++ b/examples/benchmark/benchmark-matmult.cpp @@ -0,0 +1,319 @@ +#include "common.h" +#include "ggml.h" +#include "build-info.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(_MSC_VER) +#pragma warning(disable: 4244 4267) // possible loss of data +#endif + +static void ggml_graph_compute_helper(std::vector & buf, ggml_cgraph * graph, int n_threads) { + struct ggml_cplan plan = ggml_graph_plan(graph, n_threads, nullptr); + + if (plan.work_size > 0) { + buf.resize(plan.work_size); + plan.work_data = buf.data(); + } + + ggml_graph_compute(graph, &plan); +} + +static float tensor_sum_elements(const ggml_tensor * tensor) { + double sum1; + //printf("sum inside = %f\n", sum1); + if (tensor->type == GGML_TYPE_F32) { + for (int j = 0; j < tensor->ne[1]; j++) { + for (int k = 0; k < tensor->ne[0]; k++) { + //printf("sum inside = %f\n", sum1); + printf("%f \t ", ((float *) tensor->data)[j*tensor->ne[0] + k]); + sum1 = sum1 + ((float *) tensor->data)[j*tensor->ne[0] + k]; + //printf("sum inside = %f\n", sum1); + } + printf("\n"); + } + } + return sum1; +} + +static void tensor_dump(const ggml_tensor * tensor, const char * name) { + printf("%15s: type = %i (%5s) ne = %5" PRIi64 " x %5" PRIi64 " x %5" PRIi64 ", nb = (%5zi, %5zi, %5zi) - ", name, + tensor->type, ggml_type_name(tensor->type), + tensor->ne[0], tensor->ne[1], tensor->ne[2], tensor->nb[0], tensor->nb[1], tensor->nb[2]); + float sum = tensor_sum_elements(tensor); + printf("Sum of tensor %s is %6.2f\n", name, sum); +} + +#define TENSOR_DUMP(tensor) tensor_dump(tensor, #tensor) + +struct benchmark_params_struct { + int n_threads = 1; + int32_t n_iterations = 10; + int m = 512; + int n = 512; + int k = 64; +}; +static void print_usage(int /*argc*/, char ** argv, struct benchmark_params_struct params) { + fprintf(stderr, "usage: %s [options]\n", argv[0]); + fprintf(stderr, "\n"); + fprintf(stderr, "options:\n"); + fprintf(stderr, " -h, --help show this help message and exit\n"); + fprintf(stderr, " -t N, --threads N number of threads to use (default: %d)\n", params.n_threads); + fprintf(stderr, " -i N, --iter N number of iterations to use (default: %d)\n", params.n_iterations); + fprintf(stderr, " -m N, --m N matrix dimension m (default: %d)\n", params.m); + fprintf(stderr, " -n N, --n N matrix dimension n (default: %d)\n", params.n); + fprintf(stderr, " -k N, --k N matrix dimension k (default: %d)\n", params.k); + fprintf(stderr, "\n"); +} + + +int main(int argc, char ** argv) { + struct benchmark_params_struct benchmark_params; + + bool invalid_param = false; + std::string arg; + for (int i = 1; i < argc; i++) { + arg = argv[i]; + + if (arg == "-t" || arg == "--threads") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.n_threads = std::stoi(argv[i]); + } else if (arg == "-i" || arg == "--iter") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.n_iterations = std::stoi(argv[i]); + } else if (arg == "-m") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.m = std::stoi(argv[i]); + } else if (arg == "-n") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.n = std::stoi(argv[i]); + }else if (arg == "-k") { + if (++i >= argc) { + invalid_param = true; + break; + } + benchmark_params.k = std::stoi(argv[i]); + }else if (arg == "-h" || arg == "--help") { + print_usage(argc, argv, benchmark_params); + exit(0); + } + } + if (invalid_param) { + fprintf(stderr, "error: invalid parameter for argument: %s\n", arg.c_str()); + print_usage(argc, argv, benchmark_params); + exit(1); + } + + llama_print_build_info(); + printf("Starting Test\n"); + + // create the ggml context + struct ggml_context * ctx; + //const int sizex = 4096; + //const int sizey = 11008; + +#undef VERBOSE_DEBUGGING +#ifndef VERBOSE_DEBUGGING + /* + const int sizey = 4096; + const int sizex = 11008; + const int sizez = 128; + */ + const int sizey = benchmark_params.m; //8; // m + const int sizex = 32*benchmark_params.k; //32*2; // k + const int sizez = benchmark_params.n; //4; // n +#else + /* Working - let's increase size */ + const int sizey = 1; + const int sizex = (8*32); + const int sizez = 1; + + /*const int sizey = 1; + const int sizex = 3*(8*32); + const int sizez = 1;*/ +#endif + + //printf("Memsize required = %i\n", sizex*sizex); + + // TODO: perform the bench for all types or for a user specified type + const ggml_type qtype = GGML_TYPE_Q8_0; + + size_t ctx_size = 0; + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizez); + ctx_size += ggml_row_size(qtype, sizex*sizey); + ctx_size += ggml_row_size(qtype, sizex*sizez); + ctx_size += ggml_row_size(qtype, sizex*sizey); + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); // BLAS + ctx_size += ggml_row_size(GGML_TYPE_F32, sizex*sizey); // BLAS + ctx_size += 1024*1024*16; + + printf("Allocating Memory of size %zi bytes, %zi MB\n",ctx_size, (ctx_size/1024/1024)); + + struct ggml_init_params params = { + /*.mem_size =*/ ctx_size, + /*.mem_buffer =*/ NULL, + /* no_alloc =*/ 0 + }; + + ctx = ggml_init(params); + if (!ctx) { + fprintf(stderr, "%s: ggml_init() failed\n", __func__); + return 1; + } + + + printf("Creating new tensors\n"); + // printf("Creating new tensor m1\n"); + struct ggml_tensor * m11 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizey); + ggml_set_f32(m11, 1.0f); + + // printf("Creating new tensor m1\n"); + struct ggml_tensor * m12 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizey); + ggml_set_f32(m12, 33.0f); + + // printf("Creating new tensor m2\n"); + struct ggml_tensor * m2 = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, sizex, sizez); + ggml_set_f32(m2, 2.0f); + + printf("\n------ Test 1 - Matrix Mult via F32 code\n"); + // printf("Creating new tensor m11xm2\n"); + struct ggml_tensor * m11xm2 = ggml_mul_mat(ctx, m11, m2); + + // printf("Creating compute graph\n"); + struct ggml_cgraph * gf = ggml_new_graph(ctx); + ggml_build_forward_expand(gf, m11xm2); + + printf("n_threads=%i\n", benchmark_params.n_threads); + printf("m =%d n = %d k = %d\n", benchmark_params.m, benchmark_params.n, benchmark_params.k); + //printf("Printing inputs fp32 matrices\n"); + //TENSOR_DUMP(m11); + //TENSOR_DUMP(m2); + + std::vector work_buffer; + + ggml_graph_compute_helper(work_buffer, gf, benchmark_params.n_threads); + + //TENSOR_DUMP(ggml_graph_node(gf, 0)); + + printf("\n------ Test 2 - Matrix Mult via %s code\n", ggml_type_name(qtype)); + + int32_t nelements = sizex*sizey; + int32_t nelements2 = sizex*sizez; + + // Set up a the benchmark matrices + // printf("Creating new tensor q11 & Running quantize\n"); + struct ggml_tensor * q11 = ggml_new_tensor_2d(ctx, qtype, sizex, sizey); + ggml_quantize_chunk(qtype, (const float *) m11->data, q11->data, 0, nelements/m11->ne[0], m11->ne[0], nullptr); + //TENSOR_DUMP(q11); + + // printf("Creating new tensor q2 & Running quantize\n"); + struct ggml_tensor * q2 = ggml_new_tensor_2d(ctx, qtype, sizex, sizez); + ggml_quantize_chunk(qtype, (const float *) m2->data, q2->data, 0, nelements2/m2->ne[0], m2->ne[0], nullptr); + //TENSOR_DUMP(q2); + + // Set up a the compute graph + // printf("Creating new tensor q31\n"); + struct ggml_tensor * q31 = ggml_mul_mat(ctx, q11, q2); + + // printf("Creating compute graph\n"); + struct ggml_cgraph * gf31 = ggml_new_graph(ctx); + ggml_build_forward_expand(gf31, q31); + + // Set up a second graph computation to make sure we override the CPU cache lines + // printf("Creating new tensor q12 & Running quantize\n"); + struct ggml_tensor * q12 = ggml_new_tensor_2d(ctx, qtype, sizex, sizey); + ggml_quantize_chunk(qtype, (const float *) m12->data, q12->data, 0, nelements/m12->ne[0], m12->ne[0], nullptr); + + // printf("Creating new tensor q32\n"); + struct ggml_tensor * q32 = ggml_mul_mat(ctx, q12, m2); + + //printf("Creating compute graph\n"); + struct ggml_cgraph * gf32 = ggml_new_graph(ctx); + ggml_build_forward_expand(gf32, q32); + printf("n_threads=%i\n", benchmark_params.n_threads); + + const int dimx = sizex; + const int dimy = sizey; + const int dimz = sizez; + long long int flops_per_dot_product = dimy + dimy; + long long int flops_per_matrix = flops_per_dot_product * dimx * dimz; ; + printf("Matrix Multiplication of (%i,%i,%i) x (%i,%i,%i) - about %6.2f gFLOPS\n\n", sizex, sizey, 1, sizex, sizez, 1, 1.0f*flops_per_matrix / 1000 / 1000 / 1000); + + + // Let's use the F32 result from above as a reference for the quantized multiplication + float sum_of_F32_reference = tensor_sum_elements(ggml_graph_node(gf, 0)); + + printf("Iteration;NThreads; SizeX; SizeY; SizeZ; Required_FLOPS; Elapsed_u_Seconds; gigaFLOPS\n"); + printf("=====================================================================================\n"); + + double gflops_sum = 0; + for (int i=0;i allowed_delta) { + printf("\nABORT - ERROR in Matrix Multiplication result - expected %6.2f, got %6.2f (delta %6.2f > allowed_delta %6.2f)\n", + sum_of_F32_reference, + sum_of_Q8_result, + delta, + allowed_delta + ); + exit(0); + } + + // Running a different graph computation to make sure we override the CPU cache lines + ggml_graph_compute_helper(work_buffer, gf32, benchmark_params.n_threads); + } + printf("\n"); + printf("Average%78.2f\n",gflops_sum/((double)benchmark_params.n_iterations)); + printf("=====================================================================================\n"); +} diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index f3eccff7d726..a7ab7433e721 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -400,7 +400,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name) if (EXTRACTED_NUMBER GREATER_EQUAL 10) list(APPEND ARCH_FLAGS -mcpu=power10) elseif (EXTRACTED_NUMBER EQUAL 9) - list(APPEND ARCH_FLAGS -mcpu=power9) + list(APPEND ARCH_FLAGS -mcpu=future) elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "ppc64le") list(APPEND ARCH_FLAGS -mcpu=powerpc64le -mtune=native) else() diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index cd5c61a81879..33f4333f25fb 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -821,6 +821,7 @@ struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) { const size_t n1 = tensor->nb[1]; char * const data = tensor->data; + float v = value; switch (tensor->type) { case GGML_TYPE_I8: @@ -862,7 +863,7 @@ struct ggml_tensor * ggml_set_f32(struct ggml_tensor * tensor, float value) { { assert(tensor->nb[0] == sizeof(float)); for (int i = 0; i < n; i++) { - ggml_vec_set_f32(nc, (float *)(data + i*n1), value); + ggml_vec_set_f32(nc, (float *)(data + i*n1), v++); } } break; default: diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index e13828e3be6f..c03efabbf9f5 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -1797,6 +1797,8 @@ class tinyBLAS_Q0_AVX { //PPC Implementation #if defined(__MMA__) +#define MMA_PLUS 1 + #define SAVE_ACC(ACC, ii, jj) \ __builtin_mma_disassemble_acc(vec_C, ACC); \ for (int I = 0; I < 4; I++) { \ @@ -1821,14 +1823,31 @@ struct mma_instr { __builtin_mma_xvf16ger2pp(acc, a, b); } }; +#if defined(MMA_PLUS) +template +struct mma_plus_instr; -template +template<> +struct mma_plus_instr { + static inline void outer_product(__dmr1024 *acc, __vector_pair* a, vec_t b) { + __builtin_mma_dmxvbf16gerx2pp(acc, *a, b); + } +}; + +template<> +struct mma_plus_instr { + static inline void outer_product(__dmr1024 *acc, __vector_pair* a, vec_t b) { + __builtin_mma_dmxvf16gerx2pp(acc, *a, b); + } +}; +#endif +template class tinyBLAS_HP16_PPC { public: tinyBLAS_HP16_PPC(int64_t k, - const TA *A, int64_t lda, - const TB *B, int64_t ldb, - TC *C, int64_t ldc, + const T *A, int64_t lda, + const T *B, int64_t ldb, + float *C, int64_t ldc, int ith, int nth) : A(A), B(B), C(C), k(k), lda(lda), ldb(ldb), ldc(ldc), ith(ith), nth(nth) { } @@ -1838,6 +1857,30 @@ class tinyBLAS_HP16_PPC { } private: + #if defined(MMA_PLUS) + inline void save_dmr1024(__dmr1024* acc, int ii, int jj) { + vec_t vec_C[8]; + __builtin_mma_disassemble_dmr(vec_C, acc); + for (int I = 0; I < 8; I++) { + int target_row = ii + (7 - I); + for (int J = 0; J < 4; J++) { + float* dest = (float *)(C + target_row + ((jj + J) * ldc)); + *dest = *((float *)&vec_C[I] + J); + } + } +} + void dump_vec_bf16(const char * name, vec_t vec) { + ggml_bf16_t* ptr = (ggml_bf16_t*)&vec; + + printf("%s:\t", name); + for (int i = 0; i < 8; i++) { + float val = ggml_compute_bf16_to_fp32(ptr[i]); + printf("%-12.4f", val); + } + printf("\n"); +} + +#endif void vector_permute_store(vec_t *c, int numVec, unsigned char *vecOffset) { vec_t t[8], s[8]; vec_t swiz1 = {0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23}; @@ -1880,120 +1923,145 @@ class tinyBLAS_HP16_PPC { s[5] = vec_perm(t[4], t[6], swiz4); s[6] = vec_perm(t[5], t[7], swiz3); s[7] = vec_perm(t[5], t[7], swiz4); - for (int i = 0; i < 8; ++i) - vec_xst(s[i], 0, (vec_t*)(vecOffset + i * 16)); + vec_xst(s[0], 0, (vec_t*)(vecOffset + 0 * 16)); + vec_xst(s[4], 0, (vec_t*)(vecOffset + 1 * 16)); + vec_xst(s[1], 0, (vec_t*)(vecOffset + 2 * 16)); + vec_xst(s[5], 0, (vec_t*)(vecOffset + 3 * 16)); + vec_xst(s[2], 0, (vec_t*)(vecOffset + 4 * 16)); + vec_xst(s[6], 0, (vec_t*)(vecOffset + 5 * 16)); + vec_xst(s[3], 0, (vec_t*)(vecOffset + 6 * 16)); + vec_xst(s[7], 0, (vec_t*)(vecOffset + 7 * 16)); + /*for (int i = 0; i < 8; ++i) + vec_xst(s[i], 0, (vec_t*)(vecOffset + i * 16));*/ } } - - void packNormal(const TA* a, int64_t lda, int rows, int cols, unsigned char* vec) { - int64_t i, j; - TA *aoffset = NULL; - unsigned char *vecOffset = NULL; - TA * aoffsets[8]; - vector unsigned char c_arr[8]; - aoffset = const_cast(a); - vecOffset = vec; - j = (rows >> 3); - if (j > 0) { - do { - if (cols == 4) { - aoffsets[0] = aoffset; - for (int it = 1; it < 4; ++it) - aoffsets[it] = aoffsets[it-1] + lda; - aoffset += 4 * lda; - for (int i = 0; i < 4; ++i) - c_arr[i] = vec_xl(0, (vector unsigned char*)aoffsets[i]); - vector_permute_store(c_arr, 4, vecOffset); - for (int i = 0; i<4; i++) - aoffsets[i] = aoffsets[i]+lda; - vecOffset +=64; - } - i = (cols >> 3); - if (i > 0) { - aoffsets[0] = aoffset; - for (int it = 1; it < 8; ++it) { - aoffsets[it] = aoffsets[it-1] + lda; - } - aoffset += 8 * lda; - do { - for (int it = 0; it < 8; ++it) - c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]); - vector_permute_store(c_arr, 8, vecOffset); - for (int it = 0; it < 8; ++it) - aoffsets[it] = aoffsets[it] + 8*lda; - vecOffset += 128; - i--; - } while(i > 0); - } - j--; - } while(j > 0); - } - if (rows & 4) { +void packNormal(const T* a, int64_t lda, int rows, int cols, unsigned char* vec) { + int64_t i, j; + T *aoffset = NULL; + unsigned char *vecOffset = NULL; + T * aoffsets[8]; + vector unsigned char c_arr[8]; + aoffset = const_cast(a); + vecOffset = vec; + j = (rows >> 3); + if (j > 0) { + do { aoffsets[0] = aoffset; - for (int it = 1; it < 4; ++it) - aoffsets[it] = aoffsets[it-1] + lda; - aoffset += 4 * lda; if (cols == 4) { - for (int it = 0; it < 4; ++it) - c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]); - vector_permute_store(c_arr, 2, vecOffset); - for (int it = 0; it< 4; it++) - aoffsets[it] = aoffsets[it] + lda; - vecOffset += 32; + for (int it = 1; it < 4; ++it) + aoffsets[it] = aoffsets[it-1] + lda; + aoffset += 4 * lda; + for (int i = 0; i < 4; ++i) + c_arr[i] = vec_xl(0, (vector unsigned char*)aoffsets[i]); + vector_permute_store(c_arr, 4, vecOffset); + for (int i = 0; i<4; i++) + aoffsets[i] = aoffsets[i]+lda; + vecOffset +=64; } i = (cols >> 3); if (i > 0) { + for (int it = 1; it < 8; ++it) { + aoffsets[it] = aoffsets[it-1] + lda; + } + aoffset += 8 * lda; do { - for (int it = 0; it < 4; ++it) + for (int it = 0; it < 8; ++it) c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]); - vector_permute_store(c_arr, 4, vecOffset); - for (int it = 0; it< 4; it++) + vector_permute_store(c_arr, 8, vecOffset); + for (int it = 0; it < 8; ++it) aoffsets[it] = aoffsets[it] + 8*lda; - vecOffset += 64; + vecOffset += 128; i--; } while(i > 0); } + j--; + } while(j > 0); + } + if (rows & 4) { + aoffsets[0] = aoffset; + for (int it = 1; it < 4; ++it) + aoffsets[it] = aoffsets[it-1] + lda; + aoffset += 4 * lda; + if (cols == 4) { + for (int it = 0; it < 4; ++it) + c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]); + vector_permute_store(c_arr, 2, vecOffset); + for (int it = 0; it< 4; it++) + aoffsets[it] = aoffsets[it] + lda; + vecOffset += 32; } - if (rows & 3) { - aoffsets[0] = aoffset; - for (int it = 1; it < 4; ++it) - aoffsets[it] = aoffsets[it-1] + lda; - if (cols == 4) { + i = (cols >> 3); + if (i > 0) { + do { + for (int it = 0; it < 4; ++it) + c_arr[it] = vec_xl(0, (vector unsigned char*)aoffsets[it]); + vector_permute_store(c_arr, 4, vecOffset); + for (int it = 0; it< 4; it++) + aoffsets[it] = aoffsets[it] + 8*lda; + vecOffset += 64; + i--; + } while(i > 0); + } + } + if (rows & 3) { + aoffsets[0] = aoffset; + for (int it = 1; it < 4; ++it) + aoffsets[it] = aoffsets[it-1] + lda; + if (cols == 4) { + switch(rows) { + case 3: c_arr[2] = vec_xl(0, (vector unsigned char*)aoffsets[2]); + case 2: c_arr[1] = vec_xl(0, (vector unsigned char*)aoffsets[1]); + case 1: c_arr[0] = vec_xl(0, (vector unsigned char*)aoffsets[0]); + break; + } + vector_permute_store(c_arr, 2, vecOffset); + for (int it = 0; it< 4; it++) + aoffsets[it] = aoffsets[it] + lda; + vecOffset += 32; + } + i = (cols >> 3); + if (i > 0) { + do { switch(rows) { case 3: c_arr[2] = vec_xl(0, (vector unsigned char*)aoffsets[2]); case 2: c_arr[1] = vec_xl(0, (vector unsigned char*)aoffsets[1]); case 1: c_arr[0] = vec_xl(0, (vector unsigned char*)aoffsets[0]); break; } - vector_permute_store(c_arr, 2, vecOffset); - for (int it = 0; it< 4; it++) - aoffsets[it] = aoffsets[it] + lda; - vecOffset += 32; - } - i = (cols >> 3); - if (i > 0) { - do { - switch(rows) { - case 3: c_arr[2] = vec_xl(0, (vector unsigned char*)aoffsets[2]); - case 2: c_arr[1] = vec_xl(0, (vector unsigned char*)aoffsets[1]); - case 1: c_arr[0] = vec_xl(0, (vector unsigned char*)aoffsets[0]); - break; - } - vector_permute_store(c_arr, 4, vecOffset); - for (int it = 0; it <4; it++) - aoffsets[it] = aoffsets[it] + 8* lda; - vecOffset += 64; - i--; - } while(i > 0); - } + vector_permute_store(c_arr, 4, vecOffset); + for (int it = 0; it <4; it++) + aoffsets[it] = aoffsets[it] + 8* lda; + vecOffset += 64; + i--; + } while(i > 0); } } +} + void mnpack(int64_t m0, int64_t m, int64_t n0, int64_t n) { int64_t mc, nc, mp, np; - int m_rem = MIN(m - m0, 8); - int n_rem = MIN(n - n0, 8); - + int m_rem = MIN(m - m0, 16); + int n_rem = MIN(n - n0, 16); + #if defined(MMA_PLUS) + if (m_rem >= 16 && n_rem >= 16) { + mc = 16; + nc = 16; + gemm<16,16>(m0, m, n0, n); + } else if (m_rem >= 16 && n_rem >= 8) { + mc = 16; + nc = 8; + gemm<16,8>(m0, m, n0, n); + } else if (m_rem >=16 && n_rem >=4){ + mc = 16; + nc = 4; + gemm<16,4>(m0, m, n0, n); + } else if (m_rem >=8 && n_rem >= 16){ + mc = 8; + nc = 16; + gemm<8,16>(m0, m, n0, n); + } else + #endif if (m_rem >= 8 && n_rem >= 8) { mc = 8; nc = 8; @@ -2143,14 +2211,171 @@ class tinyBLAS_HP16_PPC { packNormal((A+(ii*lda)+l), lda, 4, 8, (uint8_t*)vec_A); packNormal((B+(jj*ldb)+l), ldb, 8, 8, (uint8_t*)vec_B); for (int x = 0; x < 4; x++) { - mma_instr::outer_product(&acc_0, vec_A[x], vec_B[x]); - mma_instr::outer_product(&acc_1, vec_A[x], vec_B[x+4]); + mma_instr::outer_product(&acc_0, vec_A[x], vec_B[2*x]); + mma_instr::outer_product(&acc_1, vec_A[x], vec_B[2*x+1]); } } SAVE_ACC(&acc_0, ii, jj); SAVE_ACC(&acc_1, ii, jj+4); } +#if defined(MMA_PLUS) + void KERNEL_8x4(int64_t ii, int64_t jj) { + vec_t vec_A[8], vec_B[4]; + __dmr1024 acc_0; + __vector_pair vec_A0; + + __builtin_mma_dmsetdmrz(&acc_0); + + for (int l = 0; l < k; l += 8) { + packNormal((A + (ii * lda) + l), lda, 8, 8, (uint8_t*)vec_A); + packNormal((B + (jj * ldb) + l), ldb, 8, 4, (uint8_t*)vec_B); + + for (int x = 0; x < 4; x++) { + vec_A0 = __builtin_vsx_lxvp(x * 32, (__vector_pair*)vec_A); + mma_plus_instr::outer_product(&acc_0, &vec_A0, vec_B[x]); + } + } + save_dmr1024(&acc_0, ii, jj); + } + + void KERNEL_8x8(int64_t ii, int64_t jj) { + vec_t vec_A[8]; + vec_t vec_B[8]; + __dmr1024 acc_0, acc_1; + __vector_pair vec_A0; + + __builtin_mma_dmsetdmrz(&acc_0); + __builtin_mma_dmsetdmrz(&acc_1); + + for (int l = 0; l < k; l += 8) { + packNormal((A + (ii * lda) + l), lda, 8, 8, (uint8_t*)vec_A); + packNormal((B + (jj * ldb) + l), ldb, 8, 8, (uint8_t*)vec_B); + + for (int x = 0; x < 4; x++) { + vec_A0 = __builtin_vsx_lxvp(x * 32, (__vector_pair*)vec_A); + mma_plus_instr::outer_product(&acc_0, &vec_A0, vec_B[2*x]); + mma_plus_instr::outer_product(&acc_1, &vec_A0, vec_B[2*x + 1]); + } + } + save_dmr1024(&acc_0, ii, jj); + save_dmr1024(&acc_1, ii, jj+4); + } + + void KERNEL_16x16(int64_t ii, int64_t jj) { + vec_t vec_A[16], vec_B[16]; + __vector_pair vec_A0, vec_A1; + __dmr1024 acc[8]; + + for(int i=0; i<8; i++) __builtin_mma_dmsetdmrz(&acc[i]); + + for (int l = 0; l < k; l += 8) { + packNormal(A + (ii * lda) + l, lda, 16, 8, (uint8_t*)vec_A); + packNormal(B + (jj * ldb) + l, ldb, 8, 16, (uint8_t*)vec_B); + for (int x = 0; x < 4; x++) { + vec_A0 = __builtin_vsx_lxvp(x * 32, (__vector_pair*)vec_A); + vec_A1 = __builtin_vsx_lxvp(128 + (x * 32), (__vector_pair*)vec_A); + + mma_plus_instr::outer_product(&acc[0], &vec_A0, vec_B[2*x]); + mma_plus_instr::outer_product(&acc[1], &vec_A0, vec_B[2*x+1]); + mma_plus_instr::outer_product(&acc[2], &vec_A0, vec_B[2*x+8]); + mma_plus_instr::outer_product(&acc[3], &vec_A0, vec_B[2*x+9]); + + mma_plus_instr::outer_product(&acc[4], &vec_A1, vec_B[2*x]); + mma_plus_instr::outer_product(&acc[5], &vec_A1, vec_B[2*x+1]); + mma_plus_instr::outer_product(&acc[6], &vec_A1, vec_B[2*x+8]); + mma_plus_instr::outer_product(&acc[7], &vec_A1, vec_B[2*x+9]); + } + } + save_dmr1024(&acc[0], ii, jj); + save_dmr1024(&acc[1], ii, jj + 4); + save_dmr1024(&acc[2], ii, jj + 8); + save_dmr1024(&acc[3], ii, jj + 12); + + save_dmr1024(&acc[4], ii + 8, jj); + save_dmr1024(&acc[5], ii + 8, jj + 4); + save_dmr1024(&acc[6], ii + 8, jj + 8); + save_dmr1024(&acc[7], ii + 8, jj + 12); + } + + void KERNEL_16x4(int64_t ii, int64_t jj) { + vec_t vec_A[16], vec_B[4], vec_C[8]; + __dmr1024 acc_0, acc_1; + __vector_pair vec_A0, vec_A1; + + __builtin_mma_dmsetdmrz(&acc_0); + __builtin_mma_dmsetdmrz(&acc_1); + for (int l = 0; l < k; l+=8) { + packNormal(A+(ii*lda)+l, lda, 16, 8, (uint8_t*)vec_A); + packNormal(B+(jj*ldb)+l, ldb, 8, 4, (uint8_t*)vec_B); + for (int x = 0; x < 4; x++) { + vec_A0 = __builtin_vsx_lxvp(x * 32, (__vector_pair*)vec_A); + vec_A1 = __builtin_vsx_lxvp(128 + (x * 32), (__vector_pair*)vec_A); + + mma_plus_instr::outer_product(&acc_0, &vec_A0, vec_B[x]); + mma_plus_instr::outer_product(&acc_1, &vec_A1, vec_B[x]); + } + } + + save_dmr1024(&acc_0, ii, jj); + save_dmr1024(&acc_1, ii+8, jj); + } + + void KERNEL_16x8(int64_t ii, int64_t jj) { + vec_t vec_A[16], vec_B[8], vec_C[8]; + __dmr1024 acc_0, acc_1, acc_2, acc_3; + __vector_pair vec_A0, vec_A1; + __builtin_mma_dmsetdmrz(&acc_0); + __builtin_mma_dmsetdmrz(&acc_1); + __builtin_mma_dmsetdmrz(&acc_2); + __builtin_mma_dmsetdmrz(&acc_3); + for (int l = 0; l < k; l+=8) { + packNormal(A+(ii*lda)+l, lda, 16, 8, (uint8_t*)vec_A); + packNormal(B+(jj*ldb)+l, ldb, 8, 8, (uint8_t*)vec_B); + for (int x = 0; x < 4; x++) { + vec_A0 = __builtin_vsx_lxvp(x * 32, (__vector_pair*)vec_A); + vec_A1 = __builtin_vsx_lxvp(128 + (x * 32), (__vector_pair*)vec_A); + + mma_plus_instr::outer_product(&acc_0, &vec_A0, vec_B[2*x]); + mma_plus_instr::outer_product(&acc_1, &vec_A1, vec_B[2*x]); + mma_plus_instr::outer_product(&acc_2, &vec_A0, vec_B[2*x + 1]); + mma_plus_instr::outer_product(&acc_3, &vec_A1, vec_B[2*x + 1]); + } + } + + save_dmr1024(&acc_0, ii, jj); + save_dmr1024(&acc_1, ii + 8, jj); + save_dmr1024(&acc_2, ii, jj + 4); + save_dmr1024(&acc_3, ii + 8, jj + 4); + } + + void KERNEL_8x16(int64_t ii, int64_t jj) { + vec_t vec_A[8], vec_B[16], vec_C[8]; + __dmr1024 acc_0, acc_1, acc_2, acc_3; + __vector_pair vec_A0; + __builtin_mma_dmsetdmrz(&acc_0); + __builtin_mma_dmsetdmrz(&acc_1); + __builtin_mma_dmsetdmrz(&acc_2); + __builtin_mma_dmsetdmrz(&acc_3); + for (int l = 0; l < k; l+=8) { + packNormal(A+(ii*lda)+l, lda, 8, 8, (uint8_t*)vec_A); + packNormal(B+(jj*ldb)+l, ldb, 8, 16, (uint8_t*)vec_B); + for (int x = 0; x < 4; x++) { + vec_A0 = __builtin_vsx_lxvp(x * 32, (__vector_pair*)vec_A); + + mma_plus_instr::outer_product(&acc_0, &vec_A0, vec_B[2*x]); + mma_plus_instr::outer_product(&acc_1, &vec_A0, vec_B[2*x + 1]); + mma_plus_instr::outer_product(&acc_2, &vec_A0, vec_B[2*x + 8]); + mma_plus_instr::outer_product(&acc_3, &vec_A0, vec_B[2*x + 9]); + } + } + + save_dmr1024(&acc_0, ii, jj); + save_dmr1024(&acc_1, ii, jj + 4); + save_dmr1024(&acc_2, ii, jj + 8); + save_dmr1024(&acc_3, ii, jj + 12); + } + #else void KERNEL_8x4(int64_t ii, int64_t jj) { vec_t vec_A[8], vec_B[4] , vec_C[4]; acc_t acc_0, acc_1; @@ -2160,8 +2385,8 @@ class tinyBLAS_HP16_PPC { packNormal((A+(ii*lda)+l), lda, 8, 8, (uint8_t*)vec_A); packNormal((B+(jj*ldb)+l), ldb, 8, 4, (uint8_t*)vec_B); for (int x = 0; x < 4; x++) { - mma_instr::outer_product(&acc_0, vec_A[x], vec_B[x]); - mma_instr::outer_product(&acc_1, vec_A[x+4], vec_B[x]); + mma_instr::outer_product(&acc_0, vec_A[2*x], vec_B[x]); + mma_instr::outer_product(&acc_1, vec_A[2*x+1], vec_B[x]); } } SAVE_ACC(&acc_0, ii, jj); @@ -2180,10 +2405,10 @@ class tinyBLAS_HP16_PPC { packNormal(A+(ii*lda)+l, lda, 8, 8, (uint8_t*)vec_A); packNormal(B+(jj*ldb)+l, ldb, 8, 8, (uint8_t*)vec_B); for (int x = 0; x < 4; x++) { - mma_instr::outer_product(&acc_0, vec_A[x], vec_B[x]); - mma_instr::outer_product(&acc_1, vec_A[x], vec_B[x+4]); - mma_instr::outer_product(&acc_2, vec_A[x+4], vec_B[x]); - mma_instr::outer_product(&acc_3, vec_A[x+4], vec_B[x+4]); + mma_instr::outer_product(&acc_0, vec_A[2*x], vec_B[2*x]); + mma_instr::outer_product(&acc_1, vec_A[2*x], vec_B[2*x+1]); + mma_instr::outer_product(&acc_2, vec_A[2*x+1], vec_B[2*x]); + mma_instr::outer_product(&acc_3, vec_A[2*x+1], vec_B[2*x+1]); } } @@ -2192,7 +2417,7 @@ class tinyBLAS_HP16_PPC { SAVE_ACC(&acc_2, ii+4, jj); SAVE_ACC(&acc_3, ii+4, jj+4); } - +#endif template void gemm_small(int64_t m0, int64_t m, int64_t n0, int64_t n) { int64_t ytiles = (m - m0) / RM; @@ -2214,13 +2439,13 @@ class tinyBLAS_HP16_PPC { packNormal(A+(ii*lda)+l, lda, RM, 4, (uint8_t*)vec_A); packNormal(B+(jj*ldb)+l, ldb, RN, 4, (uint8_t*)vec_B); for (int x = 0; x<2; x++) { - mma_instr::outer_product(&acc_0, vec_A[x], vec_B[x]); + mma_instr::outer_product(&acc_0, vec_A[x], vec_B[x]); } } __builtin_mma_disassemble_acc(vec_C, &acc_0); for (int I = 0; I < RM; I++) { for (int J = 0; J < RN; J++) { - *((TC*)(C+ii+((jj+J)*ldc)+I)) = *((TC*)&vec_C[I]+J); + *((float*)(C+ii+((jj+J)*ldc)+I)) = *((float*)&vec_C[I]+J); } } } @@ -2249,20 +2474,20 @@ class tinyBLAS_HP16_PPC { packNormal(A+(ii*lda)+l, lda, RM, 8, (uint8_t*)vec_A); packNormal(B+(jj*ldb)+l, ldb, RN, 8, (uint8_t*)vec_B); for (int x = 0; x<4; x++) { - mma_instr::outer_product(&acc_0, vec_A[x], vec_B[x]); - mma_instr::outer_product(&acc_1, vec_A[x], vec_B[x+4]); + mma_instr::outer_product(&acc_0, vec_A[x], vec_B[2*x]); + mma_instr::outer_product(&acc_1, vec_A[x], vec_B[2*x+1]); } } __builtin_mma_disassemble_acc(vec_C, &acc_0); for (int I = 0; I < RM; I++) { for (int J = 0; J < 4; J++) { - *((TC*)(C+ii+((jj+J)*ldc)+I)) = *((TC*)&vec_C[I]+J); + *((float*)(C+ii+((jj+J)*ldc)+I)) = *((float*)&vec_C[I]+J); } } __builtin_mma_disassemble_acc(vec_C, &acc_1); for (int I = 0; I < RM; I++) { for (int J = 0; J < 4; J++) { - *((TC*)(C+ii+((jj+4+J)*ldc)+I)) = *((TC*)&vec_C[I]+J); + *((float*)(C+ii+((jj+4+J)*ldc)+I)) = *((float*)&vec_C[I]+J); } } } @@ -2270,6 +2495,17 @@ class tinyBLAS_HP16_PPC { template inline void kernel(int64_t ii, int64_t jj) { + #if defined(MMA_PLUS) + if constexpr(RM == 16 && RN == 16) { + KERNEL_16x16(ii,jj); + } else if constexpr(RM == 16 && RN == 8) { + KERNEL_16x8(ii,jj); + } else if constexpr(RM == 16 && RN == 4) { + KERNEL_16x4(ii,jj); + } else if constexpr(RM == 8 && RN == 16) { + KERNEL_8x16(ii, jj); + } else + #endif if constexpr(RM == 4 && RN == 8) { KERNEL_4x8(ii,jj); } else if constexpr(RM == 8 && RN == 8) { @@ -2298,9 +2534,9 @@ class tinyBLAS_HP16_PPC { } } - const TA *const A; - const TB *const B; - TC *C; + const T *const A; + const T *const B; + float *C; const int64_t k; const int64_t lda; const int64_t ldb; @@ -2399,7 +2635,43 @@ class tinyBLAS_Q0_PPC { fin_res[s_idx + i] = vec_madd(res[i], vs[s_idx + i], fin_res[s_idx + i]); } } - + #if defined(MMA_PLUS) + template + inline void compute_8x4(__dmr1024 * ACC, int c_idx, int s_idx, ArrayType & comparray, vector float * vs, vector float * fin_res) { + vector signed int vec_C[8]; + vector float CA; + vector float res; + __builtin_mma_disassemble_dmr(vec_C, ACC); + for (int i = 0; i < 8; i++) { + int r_idx = 7-i; + CA = vec_splats((float)(((double)comparray[c_idx + r_idx]) * -128.0)); + vector float res = vec_add(vec_ctf(vec_C[i], 0), CA); + fin_res[s_idx + r_idx] = vec_madd(res, vs[s_idx + r_idx], fin_res[s_idx + r_idx]); + } + } + inline void save_dmr1024(__dmr1024* acc, int ii, int jj) { + vec_t vec_C[8]; + __builtin_mma_disassemble_dmr(vec_C, acc); + for (int I = 0; I < 8; I++) { + int target_row = ii + (7 - I); + for (int J = 0; J < 4; J++) { + float* dest = (float *)(C + target_row + ((jj + J) * ldc)); + *dest = *((float *)&vec_C[I] + J); + } + } +} +inline void add_save_dmr1024(__dmr1024* acc, int ii, int jj) { + vec_t vec_C[8]; + __builtin_mma_disassemble_dmr(vec_C, acc); + for (int I = 0; I < 8; I++) { + int target_row = ii + (7 - I); + for (int J = 0; J < 4; J++) { + float* dest = (float *)(C + target_row + ((jj + J) * ldc)); + *dest += *((float *)&vec_C[I] + J); + } + } +} +#endif inline void process_q4_elements(vector signed char (&c)[2], int * ca) { const vector signed char lowMask = vec_splats((signed char)0xF); const vector unsigned char v4 = vec_splats((unsigned char)0x4); @@ -2607,6 +2879,7 @@ class tinyBLAS_Q0_PPC { aoffset = const_cast(a); vecOffset = vec; j = (rows >> 3); + int idx = 0; if (j > 0) { do { aoffset1 = aoffset; @@ -2630,14 +2903,14 @@ class tinyBLAS_Q0_PPC { c7[1] = vec_xl(0, (const vector signed char *)aoffset7->qs); c8[1] = vec_xl(0, (const vector signed char *)aoffset8->qs); - process_q4_elements(c1, & comparray[0]); - process_q4_elements(c2, & comparray[1]); - process_q4_elements(c3, & comparray[2]); - process_q4_elements(c4, & comparray[3]); - process_q4_elements(c5, & comparray[4]); - process_q4_elements(c6, & comparray[5]); - process_q4_elements(c7, & comparray[6]); - process_q4_elements(c8, & comparray[7]); + process_q4_elements(c1, & comparray[idx + 0]); + process_q4_elements(c2, & comparray[idx + 1]); + process_q4_elements(c3, & comparray[idx + 2]); + process_q4_elements(c4, & comparray[idx + 3]); + process_q4_elements(c5, & comparray[idx + 4]); + process_q4_elements(c6, & comparray[idx + 5]); + process_q4_elements(c7, & comparray[idx + 6]); + process_q4_elements(c8, & comparray[idx + 7]); vector_permute_store(c1[0], c2[0], c3[0], c4[0], vecOffset, false); vector_permute_store(c1[1], c2[1], c3[1], c4[1], vecOffset + 64, false); vector_permute_store(c5[0], c6[0], c7[0], c8[0], vecOffset + 128, false); @@ -2655,6 +2928,7 @@ class tinyBLAS_Q0_PPC { } while (i > 0); } j--; + idx += 8; } while (j > 0); } @@ -2818,7 +3092,21 @@ class tinyBLAS_Q0_PPC { int n_rem = MIN(n - n0, 16); int mc = 0, nc = 0; - + #if defined(MMA_PLUS) + if (m_rem >= 16 && n_rem >=16) { + mc = 16; + nc = 16; + gemm<16, 16>(m0, m, n0, n); + } else if (m_rem >= 16 && n_rem >= 8) { + mc = 16; + nc = 8; + gemm<16, 8>(m0, m, n0, n); + } else if (m_rem > = 8 && n_rem >= 16) { + mc = 8; + nc = 16; + gemm<8, 16>(m0, m, n0, n); + } else + #endif if (m_rem >= 8 && n_rem >= 8) { mc = 8; nc = 8; @@ -2894,7 +3182,333 @@ class tinyBLAS_Q0_PPC { save_res(ii, jj, 0, fin_res); save_res(ii, jj + 4, 4, fin_res); } +#if defined(MMA_PLUS) + void KERNEL_8x4(int64_t ii, int64_t jj) { + vec_t vec_A[16], vec_B[8] = {0}; + __vector_pair vec_A0; + __dmr1024 acc_0; + std::array comparray {}; + vector float fin_res[8] = {0}; + vector float vs[8] = {0}; + bool isAblock_q4 = std::is_same_v; + for (int l = 0; l < k; l++) { + __builtin_mma_dmsetdmrz(& acc_0); + if (std::is_same_v) { + packNormalInt4<8>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray); + } else { + packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 8, 8, (int8_t *)vec_A, false); + } + packNormal((B + (jj * ldb) + l), ldb, 4, 8, (uint8_t *)vec_B, true); + for (int x = 0; x < 8; x++) { + __builtin_vsx_build_pair(&vec_A0, vec_A[x], vec_A[x+8]); + __builtin_mma_dmxvi8gerx4pp(& acc_0, vec_A0, vec_B[x]); + } + for (int I = 0; I < 8; I++) { + for (int J = 0; J < 4; J++) { + *((float *)&vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d)); + } + } + if (!isAblock_q4) { + auto aoffset = A + (ii * lda) + l; + for (int i = 0; i < 8; i++) { + comparray[i] = 0; + int ca = 0; + auto *at = aoffset->qs; + for (int j = 0; j < 32; j++) + ca += (int)*at++; + comparray[i] = ca; + aoffset += lda; + } + } + compute_8x4(& acc_0, 0, 0, comparray, vs, fin_res); + } + save_res(ii, jj, 0, fin_res, 8, 4); + } +void KERNEL_8x8(int64_t ii, int64_t jj) { + vec_t vec_A[16], vec_B[16] = {0}; + __dmr1024 acc_0, acc_1; + __vector_pair vec_A0; + std::array comparray {}; + vector float fin_res[16] = {0}; + vector float vs[16] = {0}; + bool isAblock_q4 = std::is_same_v; + for (int l = 0; l < k; l++) { + __builtin_mma_dmsetdmrz(& acc_0); + __builtin_mma_dmsetdmrz(& acc_1); + if (std::is_same_v) { + packNormalInt4<8>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray); + } else { + packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 8, 8, (int8_t *)vec_A, false); + } + Base::template packNormal((B + (jj * ldb) + l), ldb, 8, 8, (uint8_t *)vec_B, true); + for(int x = 0; x < 8; x++) { + __builtin_vsx_build_pair(&vec_A0, vec_A[x], vec_A[x+8]); + __builtin_mma_dmxvi8gerx4pp(& acc_0, vec_A0, vec_B[x]); + __builtin_mma_dmxvi8gerx4pp(& acc_1, vec_A0, vec_B[x + 8]); + } + for (int I = 0; I < 8 ; I++) { + for (int J = 0; J < 4; J++) { + *((float *)& vs[I] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J) * ldb) + l)->d)); + *((float *)& vs[I + 8] + J) = (unhalf((A + ((ii + I) * lda) + l)->d) * unhalf((B + ((jj + J + 4) * ldb) + l)->d)); + } + } + if (!isAblock_q4) { + auto aoffset = A + (ii * lda) + l; + for (int i = 0; i < 8; i++) { + comparray[i] = 0; + int ca = 0; + auto *at = aoffset->qs; + for (int j = 0; j < 32; j++) + ca += (int)*at++; + comparray[i] = ca; + aoffset += lda; + } + } + compute_8x4(& acc_0, 0, 0, comparray, vs, fin_res); + compute_8x4(& acc_1, 0, 8, comparray, vs, fin_res); + } + save_res(ii, jj, 0, fin_res, 8, 4); + save_res(ii, jj + 4, 8, fin_res, 8, 4); + } +void KERNEL_8x16(int64_t ii, int64_t jj) { + vec_t vec_A[16], vec_B[32] = {0}; + __dmr1024 acc_0, acc_1, acc_2, acc_3; + __vector_pair vec_A0; + std::array comparray {}; + vector float fin_res[32] = {0}; + vector float vs[32] = {0}; + bool isAblock_q4 = std::is_same_v; + for (int l = 0; l < k; l++) { + __builtin_mma_dmsetdmrz(& acc_0); + __builtin_mma_dmsetdmrz(& acc_1); + __builtin_mma_dmsetdmrz(& acc_2); + __builtin_mma_dmsetdmrz(& acc_3); + if (std::is_same_v) { + packNormalInt4<8>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray); + } else { + packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 8, 8, (int8_t *)vec_A, false); + } + packNormal((B + (jj * ldb) + l), ldb, 16, 8, (uint8_t *)vec_B, true); + for(int x = 0; x < 8; x++) { + __builtin_vsx_build_pair(&vec_A0, vec_A[x], vec_A[x+8]); + __builtin_mma_dmxvi8gerx4pp(& acc_0, vec_A0, vec_B[x]); + __builtin_mma_dmxvi8gerx4pp(& acc_1, vec_A0, vec_B[x+8]); + __builtin_mma_dmxvi8gerx4pp(& acc_2, vec_A0, vec_B[x+16]); + __builtin_mma_dmxvi8gerx4pp(& acc_3, vec_A0, vec_B[x+24]); + } + for (int I = 0; I < 8; I++) { + float sA = unhalf((A + ((ii + I) * lda) + l)->d); + for (int J = 0; J < 4; J++) { + *((float *)&vs[I] + J) = sA * unhalf((B + ((jj + J) * ldb) + l)->d); + *((float *)&vs[I + 8] + J) = sA * unhalf((B + ((jj + J + 4) * ldb) + l)->d); + *((float *)&vs[I + 16] + J) = sA * unhalf((B + ((jj + J + 8) * ldb) + l)->d); + *((float *)&vs[I + 24] + J) = sA * unhalf((B + ((jj + J + 12) * ldb) + l)->d); + } + } + if (!isAblock_q4) { + for (int i = 0; i < 8; i++) { + int ca = 0; + auto *at = (A + ((ii + i) * lda) + l)->qs; + for (int j = 0; j < 32; j++) ca += (int)at[j]; + comparray[i] = ca; + } + } + compute_8x4(& acc_0, 0, 0, comparray, vs, fin_res); + compute_8x4(& acc_1, 0, 8, comparray, vs, fin_res); + compute_8x4(& acc_2, 0, 16, comparray, vs, fin_res); + compute_8x4(& acc_3, 0, 24, comparray, vs, fin_res); + } + save_res(ii, jj, 0, fin_res, 8, 4); + save_res(ii, jj + 4, 8, fin_res, 8, 4); + save_res(ii, jj + 8, 16, fin_res, 8, 4); + save_res(ii, jj + 12, 24, fin_res, 8, 4); + } +void KERNEL_16x8(int64_t ii, int64_t jj) { + vec_t vec_A[32], vec_B[16] = {0}; + __dmr1024 acc_0, acc_1, acc_2, acc_3; + __vector_pair vec_A0, vec_A1; + std::array comparray {}; + vector float fin_res[32] = {0}; + vector float vs[32] = {0}; + bool isAblock_q4 = std::is_same_v; + for (int l = 0; l < k; l++) { + __builtin_mma_dmsetdmrz(& acc_0); + __builtin_mma_dmsetdmrz(& acc_1); + __builtin_mma_dmsetdmrz(& acc_2); + __builtin_mma_dmsetdmrz(& acc_3); + if (std::is_same_v) { + packNormalInt4<16>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray); + } else { + packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 16, 8, (int8_t *)vec_A, false); + } + packNormal((B + (jj * ldb) + l), ldb, 8, 8, (uint8_t *)vec_B, true); + for(int x = 0; x < 8; x++) { + __builtin_vsx_build_pair(&vec_A0, vec_A[x], vec_A[x+8]); + __builtin_vsx_build_pair(&vec_A1, vec_A[x+16], vec_A[x+24]); + __builtin_mma_dmxvi8gerx4pp(& acc_0, vec_A0, vec_B[x]); + __builtin_mma_dmxvi8gerx4pp(& acc_1, vec_A1, vec_B[x]); + __builtin_mma_dmxvi8gerx4pp(& acc_2, vec_A0, vec_B[x+8]); + __builtin_mma_dmxvi8gerx4pp(& acc_3, vec_A1, vec_B[x+8]); + } + for (int I = 0; I < 16; I++) { + float sA = unhalf((A + ((ii + I) * lda) + l)->d); + for (int J = 0; J < 4; J++) { + *((float *)&vs[I] + J) = sA * unhalf((B + ((jj + J) * ldb) + l)->d); + *((float *)&vs[I + 16] + J) = sA * unhalf((B + ((jj + J + 4) * ldb) + l)->d); + } + } + if (!isAblock_q4) { + for (int i = 0; i < 16; i++) { + int ca = 0; + auto *at = (A + ((ii + i) * lda) + l)->qs; + for (int j = 0; j < 32; j++) ca += (int)at[j]; + comparray[i] = ca; + } + } + compute_8x4(& acc_0, 0, 0, comparray, vs, fin_res); + compute_8x4(& acc_1, 8, 8, comparray, vs, fin_res); + compute_8x4(& acc_2, 0, 16, comparray, vs, fin_res); + compute_8x4(& acc_3, 8, 24, comparray, vs, fin_res); + } + save_res(ii, jj, 0, fin_res, 8, 4); + save_res(ii + 8, jj, 8, fin_res, 8, 4); + save_res(ii, jj + 4, 16, fin_res, 8, 4); + save_res(ii + 8, jj + 4, 24, fin_res, 8, 4); + } + void KERNEL_16x16(int64_t ii, int64_t jj) { + vec_t vec_A[32], vec_B[32] = {0}; + __dmr1024 acc_0, acc_1, acc_2, acc_3; + __dmr1024 acc_4, acc_5, acc_6, acc_7; + __vector_pair vec_A0, vec_A1; + std::array comparray {}; + vector float fin_res[64] = {0}; + vector float vs[64] = {0}; + bool isAblock_q4 = std::is_same_v; + for (int l = 0; l < k; l++) { + __builtin_mma_dmsetdmrz(& acc_0); + __builtin_mma_dmsetdmrz(& acc_1); + __builtin_mma_dmsetdmrz(& acc_2); + __builtin_mma_dmsetdmrz(& acc_3); + __builtin_mma_dmsetdmrz(& acc_4); + __builtin_mma_dmsetdmrz(& acc_5); + __builtin_mma_dmsetdmrz(& acc_6); + __builtin_mma_dmsetdmrz(& acc_7); + if (std::is_same_v) { + packNormalInt4<16>((A + (ii * lda) + l), lda, 8, 4, (int8_t *)vec_A, comparray); + } else { + packNormal((const block_q8_0 *)(A + (ii * lda) + l), lda, 16, 8, (int8_t *)vec_A, false); + } + packNormal((B + (jj * ldb) + l), ldb, 16, 8, (uint8_t *)vec_B, true); + for(int x = 0; x < 8; x++) { + __builtin_vsx_build_pair(&vec_A0, vec_A[x], vec_A[x+8]); + __builtin_vsx_build_pair(&vec_A1, vec_A[x+16], vec_A[x+24]); + + __builtin_mma_dmxvi8gerx4pp(& acc_0, vec_A0, vec_B[x]); + __builtin_mma_dmxvi8gerx4pp(& acc_1, vec_A1, vec_B[x]); + __builtin_mma_dmxvi8gerx4pp(& acc_2, vec_A0, vec_B[x+8]); + __builtin_mma_dmxvi8gerx4pp(& acc_3, vec_A1, vec_B[x+8]); + __builtin_mma_dmxvi8gerx4pp(& acc_4, vec_A0, vec_B[x+16]); + __builtin_mma_dmxvi8gerx4pp(& acc_5, vec_A1, vec_B[x+16]); + __builtin_mma_dmxvi8gerx4pp(& acc_6, vec_A0, vec_B[x+24]); + __builtin_mma_dmxvi8gerx4pp(& acc_7, vec_A1, vec_B[x+24]); + } + for (int I = 0; I < 16; I++) { + float sA = unhalf((A + ((ii + I) * lda) + l)->d); + for (int J = 0; J < 4; J++) { + *((float *)&vs[I] + J) = sA * unhalf((B + ((jj + J) * ldb) + l)->d); + *((float *)&vs[I + 16] + J) = sA * unhalf((B + ((jj + J + 4) * ldb) + l)->d); + *((float *)&vs[I + 32] + J) = sA * unhalf((B + ((jj + J + 8) * ldb) + l)->d); + *((float *)&vs[I + 48] + J) = sA * unhalf((B + ((jj + J + 12) * ldb) + l)->d); + } + } + if (!isAblock_q4) { + for (int i = 0; i < 16; i++) { + int ca = 0; + auto *at = (A + ((ii + i) * lda) + l)->qs; + for (int j = 0; j < 32; j++) ca += (int)at[j]; + comparray[i] = ca; + } + } + compute_8x4(& acc_0, 0, 0, comparray, vs, fin_res); + compute_8x4(& acc_1, 8, 8, comparray, vs, fin_res); + compute_8x4(& acc_2, 0, 16, comparray, vs, fin_res); + compute_8x4(& acc_3, 8, 24, comparray, vs, fin_res); + compute_8x4(& acc_4, 0, 32, comparray, vs, fin_res); + compute_8x4(& acc_5, 8, 40, comparray, vs, fin_res); + compute_8x4(& acc_6, 0, 48, comparray, vs, fin_res); + compute_8x4(& acc_7, 8, 56, comparray, vs, fin_res); + } + save_res(ii, jj, 0, fin_res, 8, 4); + save_res(ii + 8, jj, 8, fin_res, 8, 4); + save_res(ii, jj + 4, 16, fin_res, 8, 4); + save_res(ii + 8, jj + 4, 24, fin_res, 8, 4); + save_res(ii, jj + 8, 32, fin_res, 8, 4); + save_res(ii + 8, jj + 8, 40, fin_res, 8, 4); + save_res(ii, jj + 12, 48, fin_res, 8, 4); + save_res(ii + 8, jj + 12, 56, fin_res, 8, 4); + } + + void KERNEL_Q0(int64_t ii, int64_t jj, int64_t mc, int64_t nc, int64_t kc, int64_t l, vec_t * vec_A, vec_t * vec_B) { + printf("In kernel Q0 matmul tiled\n"); + __dmr1024 acc[8]; + for (int i = 0; i < mc ; i += 16) { + for (int j = 0; j < nc; j += 16) { + int A0_base = (i / 16) * (2 * 32 * kc); + int B0_base = (j / 16) * (2 * 32 * kc); + for (int x = 0; x < 8; x++) { + __builtin_mma_dmsetdmrz(&acc[x]); + } + for (int64_t kk = 0; kk < kc; kk++) { + int A0_block_idx = A0_base + kk * 32; + int B0_block_idx = B0_base + kk * 32; + int A1_block_idx = A0_block_idx + 32 * kc; + int B1_block_idx = B0_block_idx + 32 * kc; + vec_t * A0_block = & vec_A[A0_block_idx]; + vec_t * B0_block = & vec_B[B0_block_idx]; + vec_t * A1_block = & vec_A[A1_block_idx]; + vec_t * B1_block = & vec_B[B1_block_idx]; + __vector_pair vec_A0, vec_A1; + for (int it = 0; it < 4; it++) { + for (int x = 0; x < 4; x++) { + __builtin_vsx_build_pair(&vec_A0, A0_block[8 * it + x], A0_block[8* it + x + 4]); + __builtin_vsx_build_pair(&vec_A1, A1_block[8 * it + x], A1_block[8* it + x + 4]); + + __builtin_mma_dmxvf16gerx2pp(& acc[0], vec_A0, B0_block[8 * it + x]); + __builtin_mma_dmxvf16gerx2pp(& acc[1], vec_A0, B0_block[8 * it + x + 4]); + __builtin_mma_dmxvf16gerx2pp(& acc[4], vec_A1, B0_block[8 * it + x]); + __builtin_mma_dmxvf16gerx2pp(& acc[5], vec_A1, B0_block[8 * it+ x + 4]); + + __builtin_mma_dmxvf16gerx2pp(& acc[2], vec_A0, B1_block[8 * it + x ]); + __builtin_mma_dmxvf16gerx2pp(& acc[3], vec_A0, B1_block[8 * it + x + 4]); + __builtin_mma_dmxvf16gerx2pp(& acc[6], vec_A1, B1_block[8 * it + x ]); + __builtin_mma_dmxvf16gerx2pp(& acc[7], vec_A1, B1_block[8 * it+ x + 4]); + } + } + } + if (l == 0) { + save_dmr1024(& acc[0], ii + i, jj + j); + save_dmr1024(& acc[1], ii + i, jj + j + 4); + save_dmr1024(& acc[4], ii + i + 8, jj + j); + save_dmr1024(& acc[5], ii + i + 8, jj + j + 4); + save_dmr1024(& acc[2], ii + i, jj + j + 8); + save_dmr1024(& acc[3], ii + i, jj + j + 12); + save_dmr1024(& acc[6], ii + i + 8, jj + j + 8); + save_dmr1024(& acc[7], ii + i + 8, jj + j + 12); + } else { + add_save_dmr1024(& acc[0], ii + i, jj + j); + add_save_dmr1024(& acc[1], ii + i, jj + j + 4); + add_save_dmr1024(& acc[4], ii + i + 8, jj + j); + add_save_dmr1024(& acc[5], ii + i + 8, jj + j + 4); + add_save_dmr1024(& acc[2], ii + i, jj + j + 8); + add_save_dmr1024(& acc[3], ii + i, jj + j + 12); + add_save_dmr1024(& acc[6], ii + i + 8, jj + j + 8); + add_save_dmr1024(& acc[7], ii + i + 8, jj + j + 12); + } + } + } + } +#else void KERNEL_8x4(int64_t ii, int64_t jj) { vec_t vec_A[16], vec_B[8] = {0}; acc_t acc_0, acc_1; @@ -3045,7 +3659,7 @@ class tinyBLAS_Q0_PPC { } } } - +#endif void matmul_tiled(int64_t m, int64_t n, int64_t mc, int64_t nc, int64_t kc) { vec_t A_pack[mc * kc * 4]; vec_t B_pack[nc * kc * 4]; @@ -3144,6 +3758,15 @@ class tinyBLAS_Q0_PPC { template inline void kernel(int64_t ii, int64_t jj) { + #if defined(MMA_PLUS) + if constexpr(RM == 16 && RN == 16) { + KERNEL_16x16(ii, jj); + } else if constexpr(RM == 16 && RN == 8) { + KERNEL_16x8(ii, jj); + } else if constexpr(RM == 8 && RN == 16) { + KERNEL_8x16(ii, jj); + } else + #endif if constexpr(RM == 4 && RN == 8) { KERNEL_4x8(ii,jj); } else if constexpr(RM == 8 && RN == 4) { @@ -3808,7 +4431,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 } if (Btype == GGML_TYPE_BF16) { - tinyBLAS_HP16_PPC tb{ k, + tinyBLAS_HP16_PPC tb{ k, (const ggml_bf16_t *)A, lda, (const ggml_bf16_t *)B, ldb, (float *)C, ldc, @@ -3912,7 +4535,7 @@ bool llamafile_sgemm(const struct ggml_compute_params * params, int64_t m, int64 } if (Btype == GGML_TYPE_F16) { - tinyBLAS_HP16_PPC tb{ k, + tinyBLAS_HP16_PPC tb{ k, (const ggml_fp16_t *)A, lda, (const ggml_fp16_t *)B, ldb, (float *)C, ldc, diff --git a/ggml/src/ggml-cpu/vec.h b/ggml/src/ggml-cpu/vec.h index bcd68da9aa9e..6a2359b9495a 100644 --- a/ggml/src/ggml-cpu/vec.h +++ b/ggml/src/ggml-cpu/vec.h @@ -86,7 +86,7 @@ inline static void ggml_vec_sub_f16 (const int n, ggml_fp16_t * z, const ggml_fp z[i] = GGML_CPU_FP32_TO_FP16(GGML_CPU_FP16_TO_FP32(x[i]) - GGML_CPU_FP16_TO_FP32(y[i])); } } -inline static void ggml_vec_set_f32 (const int n, float * x, const float v) { for (int i = 0; i < n; ++i) x[i] = v; } +inline static void ggml_vec_set_f32 (const int n, float * x, float v) { for (int i = 0; i < n; ++i) x[i] = v++; } inline static void ggml_vec_cpy_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = x[i]; } inline static void ggml_vec_neg_f32 (const int n, float * y, const float * x) { for (int i = 0; i < n; ++i) y[i] = -x[i]; } inline static void ggml_vec_neg_f16 (const int n, ggml_fp16_t * y, const ggml_fp16_t * x) {