diff --git a/lib/Conversion/StructuredToMemref/StructuredToMemref.cpp b/lib/Conversion/StructuredToMemref/StructuredToMemref.cpp index 4fce354cb..102402984 100644 --- a/lib/Conversion/StructuredToMemref/StructuredToMemref.cpp +++ b/lib/Conversion/StructuredToMemref/StructuredToMemref.cpp @@ -409,6 +409,30 @@ struct MakeTensorPtrConverter // clampedOff - targetOffset // d1 = -------------------- // strideRows + // + //////////////////////////////////////////////////////////////////////////// + // + // cols + // + // wrappedAroundOff + // --------------*--------------------- + // | | + // | targetOffset | + // | *------------| | + // | | | | + // | | | | + // rows| rowSize | | | + // | | | | + // | | | | + // | *------------| | + // | nextOff | + // | | + // | clampedOff | + // --------------*--------------------- + // + // For the case that clampedOff is not overflown + // d1 = min(d1, rowSize) + // auto resultType = getResultMemrefType( op, /* offset */ ShapedType::kDynamic, @@ -443,6 +467,7 @@ struct MakeTensorPtrConverter rewriter.create(loc, modRow, wrappedAroundOff); Value d1 = rewriter.create(loc, clampedOff, targetOffset); d1 = rewriter.create(loc, d1, strideRow); + d1 = rewriter.create(loc, d1, rowSize); SmallVector sizes1{d1, colSize}; memref::ReinterpretCastOp cast1 = @@ -685,11 +710,10 @@ struct LoadConverter : public OpConversionPattern { ConversionPatternRewriter &rewriter) const { OpFoldResult subviewRowFull = dims[0]; OpFoldResult subviewColFull = dims[1]; - OpFoldResult col1 = + OpFoldResult subviewCol1 = rewriter.create(loc, block1, 1).getResult(); - OpFoldResult subviewCol1 = minOFRs(col1, subviewColFull, loc, rewriter); OpFoldResult subviewCol2 = - subOFRs(subviewColFull, subviewCol1, loc, rewriter); + rewriter.create(loc, block2, 1).getResult(); SmallVector offsets(dims.size(), rewriter.getIndexAttr(0)); SmallVector strides(dims.size(), rewriter.getIndexAttr(1)); @@ -707,11 +731,10 @@ struct LoadConverter : public OpConversionPattern { ConversionPatternRewriter &rewriter) const { OpFoldResult subviewRowFull = dims[0]; OpFoldResult subviewColFull = dims[1]; - OpFoldResult row1 = + OpFoldResult subviewRow1 = rewriter.create(loc, block1, 0).getResult(); - OpFoldResult subviewRow1 = minOFRs(row1, subviewRowFull, loc, rewriter); OpFoldResult subviewRow2 = - subOFRs(subviewRowFull, subviewRow1, loc, rewriter); + rewriter.create(loc, block2, 0).getResult(); SmallVector offsets(dims.size(), rewriter.getIndexAttr(0)); SmallVector strides(dims.size(), rewriter.getIndexAttr(1)); diff --git a/python/examples/test_mm.py b/python/examples/test_mm.py new file mode 100644 index 000000000..5bbdcd5d6 --- /dev/null +++ b/python/examples/test_mm.py @@ -0,0 +1,161 @@ + +import torch +import triton +import pytest +import triton.language as tl +import benchmark + +@triton.jit +def prev_multiple_of(a, b): + # the largest x 1 and a.stride(1) > 1: + a = a.contiguous() + if b.stride(0) > 1 and b.stride(1) > 1: + b = b.contiguous() + # checks constraints + assert a.shape[1] == b.shape[0], "incompatible dimensions" + M, K = a.shape + _, N = b.shape + # allocates output + c_dtype = get_higher_dtype(a.dtype, b.dtype) + c = torch.empty((M, N), device=device, dtype=c_dtype) + # launch kernel + grid = lambda META: ( + triton.cdiv(M, META["BLOCK_M"]) * triton.cdiv(N, META["BLOCK_N"]), + ) + + mm_kernel[grid]( + a, + b, + c, + M, + N, + K, + a.stride(0), + a.stride(1), + b.stride(0), + b.stride(1), + c.stride(0), + c.stride(1), + 32, + 32, + 32, + GROUP_M=8, + ) + + return c + +@pytest.mark.interpreter +@pytest.mark.parametrize("M, N, K", [(1, 1, 32), (15, 160, 1024), (495, 5333, 71)]) +@pytest.mark.parametrize("dtype", [torch.float32]) +def test_accuracy_mm(M, N, K, dtype): + device = 'cpu' + a = torch.randn((M, K), dtype=dtype, device=device) + b = torch.randn((K, N), dtype=dtype, device=device) + + ref_out = torch.mm(a, b) + res_out = mm(a, b) + + torch.testing.assert_close(res_out, ref_out, atol=1e-2, rtol=0) + + +if __name__ == "__main__": + benchmark.select_cpu_backend() + M, N, K = (495, 5333, 71) + test_accuracy_mm(M, N, K, torch.float32) diff --git a/test/Conversion/StructuredToMemref/wraparound_side_by_side.mlir b/test/Conversion/StructuredToMemref/wraparound_side_by_side.mlir index 9a77b3bf9..5c7f4f4eb 100644 --- a/test/Conversion/StructuredToMemref/wraparound_side_by_side.mlir +++ b/test/Conversion/StructuredToMemref/wraparound_side_by_side.mlir @@ -96,20 +96,18 @@ module { // CHECK-DAG: [[VAR_reinterpret_cast_1_:%.+]] = memref.reinterpret_cast [[PARAM_0_]] to offset: {{.}}[[VAR_15_]]{{.}}, sizes: {{.}}[[CST_4_]], [[VAR_19_]]{{.}}, strides: {{.}}[[VAR_0_]], [[VAR_3_]]{{.}} : memref<*xf32> to memref> // CHECK-DAG: [[RES_:%.+]] = memref.alloc() : memref<4x4xf32> // CHECK: linalg.fill ins([[CST_minus_9_dot_900000_]] : f32) outs([[RES_]] : memref<4x4xf32>) -// CHECK: [[VAR_20_:%.+]] = arith.minsi [[VAR_18_]], [[CST_4_]] : index -// CHECK-DAG: [[VAR_21_:%.+]] = arith.subi [[CST_4_]], [[VAR_20_]] : index -// CHECK-DAG: [[VAR_subview_:%.+]] = memref.subview [[VAR_reinterpret_cast_0_]][0, 0] [2, [[VAR_20_]]{{.}} [1, 1] : memref> to memref<2x?xf32, strided<[?, ?], offset: ?>> +// CHECK-DAG: [[VAR_subview_:%.+]] = memref.subview [[VAR_reinterpret_cast_0_]][0, 0] [2, [[VAR_18_]]{{.}} [1, 1] : memref> to memref<2x?xf32, strided<[?, ?], offset: ?>> // CHECK-NOT: separator of consecutive DAGs -// CHECK-DAG: [[VAR_subview_2_:%.+]] = memref.subview [[VAR_reinterpret_cast_1_]][0, 0] [2, [[VAR_21_]]{{.}} [1, 1] : memref> to memref<2x?xf32, strided<[?, ?], offset: ?>> -// CHECK-DAG: [[VAR_subview_3_:%.+]] = memref.subview [[RES_]][0, 0] [2, [[VAR_20_]]{{.}} [1, 1] : memref<4x4xf32> to memref<2x?xf32, strided<[4, 1]>> -// CHECK-DAG: [[VAR_subview_4_:%.+]] = memref.subview [[RES_]][0, [[VAR_20_]]{{.}} [2, [[VAR_21_]]{{.}} [1, 1] : memref<4x4xf32> to memref<2x?xf32, strided<[4, 1], offset: ?>> +// CHECK-DAG: [[VAR_subview_2_:%.+]] = memref.subview [[VAR_reinterpret_cast_1_]][0, 0] [2, [[VAR_19_]]{{.}} [1, 1] : memref> to memref<2x?xf32, strided<[?, ?], offset: ?>> +// CHECK-DAG: [[VAR_subview_3_:%.+]] = memref.subview [[RES_]][0, 0] [2, [[VAR_18_]]{{.}} [1, 1] : memref<4x4xf32> to memref<2x?xf32, strided<[4, 1]>> +// CHECK-DAG: [[VAR_subview_4_:%.+]] = memref.subview [[RES_]][0, [[VAR_18_]]{{.}} [2, [[VAR_19_]]{{.}} [1, 1] : memref<4x4xf32> to memref<2x?xf32, strided<[4, 1], offset: ?>> // CHECK: memref.copy [[VAR_subview_]], [[VAR_subview_3_]] : memref<2x?xf32, strided<[?, ?], offset: ?>> to memref<2x?xf32, strided<[4, 1]>> // CHECK: memref.copy [[VAR_subview_2_]], [[VAR_subview_4_]] : memref<2x?xf32, strided<[?, ?], offset: ?>> to memref<2x?xf32, strided<[4, 1], offset: ?>> -// CHECK: [[VAR_22_:%.+]] = bufferization.to_tensor [[RES_]] restrict writable : memref<4x4xf32> -// CHECK: bufferization.materialize_in_destination [[VAR_22_]] in writable [[VAR_reinterpret_cast_]] : (tensor<4x4xf32>, memref<4x4xf32, strided<[?, ?], offset: ?>>) -> () -// CHECK-DAG: [[VAR_23_:%.+]] = arith.addi [[VAR_arg15_]], [[VAR_9_]] : index -// CHECK-DAG: [[VAR_24_:%.+]] = arith.addi [[VAR_arg16_]], [[VAR_11_]] : index -// CHECK: scf.yield [[VAR_23_]], [[VAR_24_]] : index, index +// CHECK: [[VAR_20_:%.+]] = bufferization.to_tensor [[RES_]] restrict writable : memref<4x4xf32> +// CHECK: bufferization.materialize_in_destination [[VAR_20_]] in writable [[VAR_reinterpret_cast_]] : (tensor<4x4xf32>, memref<4x4xf32, strided<[?, ?], offset: ?>>) -> () +// CHECK-DAG: [[VAR_21_:%.+]] = arith.addi [[VAR_arg15_]], [[VAR_9_]] : index +// CHECK-DAG: [[VAR_22_:%.+]] = arith.addi [[VAR_arg16_]], [[VAR_11_]] : index +// CHECK: scf.yield [[VAR_21_]], [[VAR_22_]] : index, index // CHECK: } // CHECK: return // CHECK: } diff --git a/test/Conversion/StructuredToMemref/wraparound_stacked.mlir b/test/Conversion/StructuredToMemref/wraparound_stacked.mlir index 5c27f121b..4e5ce945b 100644 --- a/test/Conversion/StructuredToMemref/wraparound_stacked.mlir +++ b/test/Conversion/StructuredToMemref/wraparound_stacked.mlir @@ -83,26 +83,25 @@ module { // CHECK: [[VAR_13_:%.+]] = arith.addi [[VAR_3_]], [[VAR_12_]] : index // CHECK: [[VAR_14_:%.+]] = arith.subi [[VAR_13_]], [[VAR_11_]] : index // CHECK: [[VAR_15_:%.+]] = arith.divsi [[VAR_14_]], [[VAR_1_]] : index -// CHECK-DAG: [[VAR_reinterpret_cast_0_:%.+]] = memref.reinterpret_cast [[PARAM_0_]] to offset: {{.}}[[VAR_11_]]{{.}}, sizes: {{.}}[[VAR_15_]], [[CST_4_]]{{.}}, strides: {{.}}[[VAR_1_]], [[VAR_4_]]{{.}} : memref<*xf32> to memref> -// CHECK-DAG: [[VAR_16_:%.+]] = arith.subi [[CST_4_]], [[VAR_15_]] : index +// CHECK: [[VAR_16_:%.+]] = arith.minsi [[VAR_15_]], [[CST_4_]] : index +// CHECK-DAG: [[VAR_reinterpret_cast_0_:%.+]] = memref.reinterpret_cast [[PARAM_0_]] to offset: {{.}}[[VAR_11_]]{{.}}, sizes: {{.}}[[VAR_16_]], [[CST_4_]]{{.}}, strides: {{.}}[[VAR_1_]], [[VAR_4_]]{{.}} : memref<*xf32> to memref> +// CHECK-DAG: [[VAR_17_:%.+]] = arith.subi [[CST_4_]], [[VAR_16_]] : index // CHECK-NOT: separator of consecutive DAGs -// CHECK-DAG: [[VAR_reinterpret_cast_1_:%.+]] = memref.reinterpret_cast [[PARAM_0_]] to offset: {{.}}[[VAR_12_]]{{.}}, sizes: {{.}}[[VAR_16_]], [[CST_4_]]{{.}}, strides: {{.}}[[VAR_1_]], [[VAR_4_]]{{.}} : memref<*xf32> to memref> +// CHECK-DAG: [[VAR_reinterpret_cast_1_:%.+]] = memref.reinterpret_cast [[PARAM_0_]] to offset: {{.}}[[VAR_12_]]{{.}}, sizes: {{.}}[[VAR_17_]], [[CST_4_]]{{.}}, strides: {{.}}[[VAR_1_]], [[VAR_4_]]{{.}} : memref<*xf32> to memref> // CHECK-DAG: [[RES_:%.+]] = memref.alloc() : memref<4x4xf32> // CHECK: linalg.fill ins([[CST_minus_9_dot_900000_]] : f32) outs([[RES_]] : memref<4x4xf32>) -// CHECK: [[VAR_17_:%.+]] = arith.minsi [[VAR_15_]], [[CST_4_]] : index -// CHECK-DAG: [[VAR_18_:%.+]] = arith.subi [[CST_4_]], [[VAR_17_]] : index -// CHECK-DAG: [[VAR_subview_:%.+]] = memref.subview [[VAR_reinterpret_cast_0_]][0, 0] {{.}}[[VAR_17_]], 3] [1, 1] : memref> to memref> +// CHECK-DAG: [[VAR_subview_:%.+]] = memref.subview [[VAR_reinterpret_cast_0_]][0, 0] {{.}}[[VAR_16_]], 3] [1, 1] : memref> to memref> // CHECK-NOT: separator of consecutive DAGs -// CHECK-DAG: [[VAR_subview_2_:%.+]] = memref.subview [[VAR_reinterpret_cast_1_]][0, 0] {{.}}[[VAR_18_]], 3] [1, 1] : memref> to memref> -// CHECK-DAG: [[VAR_subview_3_:%.+]] = memref.subview [[RES_]][0, 0] {{.}}[[VAR_17_]], 3] [1, 1] : memref<4x4xf32> to memref> -// CHECK-DAG: [[VAR_subview_4_:%.+]] = memref.subview [[RES_]]{{.}}[[VAR_17_]], 0] {{.}}[[VAR_18_]], 3] [1, 1] : memref<4x4xf32> to memref> +// CHECK-DAG: [[VAR_subview_2_:%.+]] = memref.subview [[VAR_reinterpret_cast_1_]][0, 0] {{.}}[[VAR_17_]], 3] [1, 1] : memref> to memref> +// CHECK-DAG: [[VAR_subview_3_:%.+]] = memref.subview [[RES_]][0, 0] {{.}}[[VAR_16_]], 3] [1, 1] : memref<4x4xf32> to memref> +// CHECK-DAG: [[VAR_subview_4_:%.+]] = memref.subview [[RES_]]{{.}}[[VAR_16_]], 0] {{.}}[[VAR_17_]], 3] [1, 1] : memref<4x4xf32> to memref> // CHECK: memref.copy [[VAR_subview_]], [[VAR_subview_3_]] : memref> to memref> // CHECK: memref.copy [[VAR_subview_2_]], [[VAR_subview_4_]] : memref> to memref> -// CHECK: [[VAR_19_:%.+]] = bufferization.to_tensor [[RES_]] restrict writable : memref<4x4xf32> -// CHECK: bufferization.materialize_in_destination [[VAR_19_]] in writable [[VAR_reinterpret_cast_]] : (tensor<4x4xf32>, memref<4x4xf32, strided<[?, ?], offset: ?>>) -> () -// CHECK-DAG: [[VAR_20_:%.+]] = arith.addi [[VAR_arg15_]], [[VAR_9_]] : index -// CHECK-DAG: [[VAR_21_:%.+]] = arith.addi [[VAR_arg16_]], [[VAR_9_]] : index -// CHECK: scf.yield [[VAR_20_]], [[VAR_21_]] : index, index +// CHECK: [[VAR_18_:%.+]] = bufferization.to_tensor [[RES_]] restrict writable : memref<4x4xf32> +// CHECK: bufferization.materialize_in_destination [[VAR_18_]] in writable [[VAR_reinterpret_cast_]] : (tensor<4x4xf32>, memref<4x4xf32, strided<[?, ?], offset: ?>>) -> () +// CHECK-DAG: [[VAR_19_:%.+]] = arith.addi [[VAR_arg15_]], [[VAR_9_]] : index +// CHECK-DAG: [[VAR_20_:%.+]] = arith.addi [[VAR_arg16_]], [[VAR_9_]] : index +// CHECK: scf.yield [[VAR_19_]], [[VAR_20_]] : index, index // CHECK: } // CHECK: return // CHECK: } diff --git a/test/Conversion/StructuredToMemref/wraparound_stacked_load.mlir b/test/Conversion/StructuredToMemref/wraparound_stacked_load.mlir new file mode 100644 index 000000000..e41209164 --- /dev/null +++ b/test/Conversion/StructuredToMemref/wraparound_stacked_load.mlir @@ -0,0 +1,48 @@ +// RUN: triton-shared-opt --split-input-file --structured-to-memref %s | FileCheck %s + +module { + tt.func public @wrap_stacked_load(%arg0: !tt.ptr, %M: index, %N: index) -> tensor<4x4xf32> { + %0 = arith.muli %M, %N : index + %1 = tts.make_tptr %arg0 to sizes: [4, 4], strides: [%N, 1], offsets: [0, 0], shape: [%0, 0], order: [] : to tensor<4x4x!tt.ptr> + %2 = "tts.load"(%1) <{operandSegmentSizes = array, static_mask_dims = array}> : (tensor<4x4x!tt.ptr>) -> tensor<4x4xf32> + tt.return %2 : tensor<4x4xf32> + } +} + +// CHECK-LABEL: tt.func public @wrap_stacked_load( +// CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !tt.ptr, +// CHECK-SAME: %[[VAL_1:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: index, +// CHECK-SAME: %[[VAL_2:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: index) -> tensor<4x4xf32> { +// CHECK: %[[VAL_3:.*]] = builtin.unrealized_conversion_cast %[[VAL_0]] : !tt.ptr to memref<*xf32> +// CHECK: %[[VAL_4:.*]] = arith.muli %[[VAL_1]], %[[VAL_2]] : index +// CHECK: %[[VAL_5:.*]] = arith.constant 0 : index +// CHECK: %[[VAL_6:.*]] = arith.constant 4 : index +// CHECK: %[[VAL_7:.*]] = arith.constant 4 : index +// CHECK: %[[VAL_8:.*]] = arith.constant 1 : index +// CHECK: %[[VAL_9:.*]] = arith.remsi %[[VAL_5]], %[[VAL_2]] : index +// CHECK: %[[VAL_10:.*]] = arith.addi %[[VAL_4]], %[[VAL_9]] : index +// CHECK: %[[VAL_11:.*]] = arith.subi %[[VAL_10]], %[[VAL_5]] : index +// CHECK: %[[VAL_12:.*]] = arith.divsi %[[VAL_11]], %[[VAL_2]] : index +// CHECK: %[[VAL_13:.*]] = arith.minsi %[[VAL_12]], %[[VAL_6]] : index +// CHECK: %[[VAL_14:.*]] = memref.reinterpret_cast %[[VAL_3]] to offset: {{\[}}%[[VAL_5]]], sizes: {{\[}}%[[VAL_13]], %[[VAL_7]]], strides: {{\[}}%[[VAL_2]], %[[VAL_8]]] : memref<*xf32> to memref> +// CHECK: %[[VAL_15:.*]] = arith.subi %[[VAL_6]], %[[VAL_13]] : index +// CHECK: %[[VAL_16:.*]] = memref.reinterpret_cast %[[VAL_3]] to offset: {{\[}}%[[VAL_9]]], sizes: {{\[}}%[[VAL_15]], %[[VAL_7]]], strides: {{\[}}%[[VAL_2]], %[[VAL_8]]] : memref<*xf32> to memref> +// CHECK: %[[VAL_17:.*]] = builtin.unrealized_conversion_cast %[[VAL_14]], %[[VAL_16]] : memref>, memref> to tensor<4x4x!tt.ptr> {wrap_stacked} +// CHECK: %[[VAL_18:.*]] = memref.alloc() : memref<4x4xf32> +// CHECK: %[[VAL_19:.*]] = arith.constant 0 : index +// CHECK: %[[VAL_20:.*]] = arith.constant 1 : index +// CHECK: %[[VAL_21:.*]] = arith.constant 0 : index +// CHECK: %[[VAL_22:.*]] = memref.dim %[[VAL_14]], %[[VAL_21]] : memref> +// CHECK: %[[VAL_23:.*]] = arith.constant 1 : index +// CHECK: %[[VAL_24:.*]] = memref.dim %[[VAL_14]], %[[VAL_23]] : memref> +// CHECK: %[[VAL_25:.*]] = arith.constant 0 : index +// CHECK: %[[VAL_26:.*]] = memref.dim %[[VAL_16]], %[[VAL_25]] : memref> +// CHECK: %[[VAL_27:.*]] = arith.constant 1 : index +// CHECK: %[[VAL_28:.*]] = memref.dim %[[VAL_16]], %[[VAL_27]] : memref> +// CHECK: %[[VAL_29:.*]] = memref.subview %[[VAL_18]]{{\[}}%[[VAL_19]], %[[VAL_19]]] {{\[}}%[[VAL_22]], %[[VAL_24]]] {{\[}}%[[VAL_20]], %[[VAL_20]]] : memref<4x4xf32> to memref> +// CHECK: %[[VAL_30:.*]] = memref.subview %[[VAL_18]]{{\[}}%[[VAL_22]], %[[VAL_19]]] {{\[}}%[[VAL_26]], %[[VAL_28]]] {{\[}}%[[VAL_20]], %[[VAL_20]]] : memref<4x4xf32> to memref> +// CHECK: memref.copy %[[VAL_14]], %[[VAL_29]] : memref> to memref> +// CHECK: memref.copy %[[VAL_16]], %[[VAL_30]] : memref> to memref> +// CHECK: %[[VAL_31:.*]] = bufferization.to_tensor %[[VAL_18]] restrict writable : memref<4x4xf32> to tensor<4x4xf32> +// CHECK: tt.return %[[VAL_31]] : tensor<4x4xf32> +// CHECK: }