Add nightly benchmarking on Triton pytorch and triton-main versions

[xuzhao9]

Add nightly benchmarking CI on Triton's pytorch and triton-main versions.

Test plan:

$ python benchmarks/nightly/run.py

Output:

{
    "env": {
        "cuda_version": "12.4",
        "device": "NVIDIA H100",
        "pytorch_commit": "aa019efbc3137e971c1a81b1f61a18ac9998d930",
        "triton_commit": "unknown",
        "tritonbench_commit": "b146d3e0def7379caae463fb330b856bf4252914"
    },
    "metrics": {
        "tritonbench_launch_latency_fwd[x_0-nop_python_function]_latency": 0.0029120000544935465,
        "tritonbench_launch_latency_fwd[x_0-nop_python_function]_walltime": 2.437883566664313e-05,
        "tritonbench_launch_latency_fwd[x_0-nop_triton_kernel]_latency": 0.004544000141322613,
        "tritonbench_launch_latency_fwd[x_0-nop_triton_kernel]_walltime": 0.009655900224497466,
        "tritonbench_launch_latency_fwd[x_0-nop_triton_compiled_kernel_run]_latency": 0.004544000141322613,
        "tritonbench_launch_latency_fwd[x_0-nop_triton_compiled_kernel_run]_walltime": 0.0027005181459927607,
        "tritonbench_launch_latency_fwd[x_0-nop_inductor_kernel]_latency": 0.004767999984323978,
        "tritonbench_launch_latency_fwd[x_0-nop_inductor_kernel]_walltime": 0.014456475353731476,
        "tritonbench_launch_latency_fwd[x_19-nop_python_function]_latency": 0.0029120000544935465,
        "tritonbench_launch_latency_fwd[x_19-nop_python_function]_walltime": 2.5523167311428935e-05,
        "tritonbench_launch_latency_fwd[x_19-nop_triton_kernel]_latency": 0.004544000141322613,
        "tritonbench_launch_latency_fwd[x_19-nop_triton_kernel]_walltime": 0.018005142061199148,
        "tritonbench_launch_latency_fwd[x_19-nop_triton_compiled_kernel_run]_latency": 0.004544000141322613,
        "tritonbench_launch_latency_fwd[x_19-nop_triton_compiled_kernel_run]_walltime": 0.004151842380697663,
        "tritonbench_launch_latency_fwd[x_19-nop_inductor_kernel]_latency": 0.005024000070989132,
        "tritonbench_launch_latency_fwd[x_19-nop_inductor_kernel]_walltime": 0.02036418371681101,
        "tritonbench_launch_latency_fwd[nop_python_function]-latency-avg": 0.0029120000544935465,
        "tritonbench_launch_latency_fwd[nop_python_function]-walltime-avg": 2.4951001489036032e-05,
        "tritonbench_launch_latency_fwd[nop_triton_kernel]-latency-avg": 0.004544000141322613,
        "tritonbench_launch_latency_fwd[nop_triton_kernel]-walltime-avg": 0.013830521142848308,
        "tritonbench_launch_latency_fwd[nop_triton_compiled_kernel_run]-latency-avg": 0.004544000141322613,
        "tritonbench_launch_latency_fwd[nop_triton_compiled_kernel_run]-walltime-avg": 0.003426180263345212,
        "tritonbench_launch_latency_fwd[nop_inductor_kernel]-latency-avg": 0.004896000027656555,
        "tritonbench_launch_latency_fwd[nop_inductor_kernel]-walltime-avg": 0.017410329535271243,
        "tritonbench_softmax_fwd[x_[4096, 256]-naive_softmax]_latency": 0.03209599852561951,
        "tritonbench_softmax_fwd[x_[4096, 256]-naive_softmax]_gbps": 261.3599322452638,
        "tritonbench_softmax_fwd[x_[4096, 256]-triton_softmax]_latency": 0.009920000098645687,
        "tritonbench_softmax_fwd[x_[4096, 256]-triton_softmax]_gbps": 845.6257980426071,
        "tritonbench_softmax_fwd[x_[4096, 384]-naive_softmax]_latency": 0.03683200106024742,
        "tritonbench_softmax_fwd[x_[4096, 384]-naive_softmax]_gbps": 341.6298772205638,
        "tritonbench_softmax_fwd[x_[4096, 384]-triton_softmax]_latency": 0.011776000261306763,
        "tritonbench_softmax_fwd[x_[4096, 384]-triton_softmax]_gbps": 1068.521715420181,
        "tritonbench_softmax_fwd[x_[4096, 512]-naive_softmax]_latency": 0.04131200164556503,
        "tritonbench_softmax_fwd[x_[4096, 512]-naive_softmax]_gbps": 406.1099760776439,
        "tritonbench_softmax_fwd[x_[4096, 512]-triton_softmax]_latency": 0.013311999849975109,
        "tritonbench_softmax_fwd[x_[4096, 512]-triton_softmax]_gbps": 1260.3077065112325,
        "tritonbench_softmax_fwd[x_[4096, 640]-naive_softmax]_latency": 0.04608000069856644,
        "tritonbench_softmax_fwd[x_[4096, 640]-naive_softmax]_gbps": 455.1111042116896,
        "tritonbench_softmax_fwd[x_[4096, 640]-triton_softmax]_latency": 0.01500799972563982,
        "tritonbench_softmax_fwd[x_[4096, 640]-triton_softmax]_gbps": 1397.3561023040293,
        "tritonbench_softmax_fwd[x_[4096, 768]-naive_softmax]_latency": 0.051231998950242996,
        "tritonbench_softmax_fwd[x_[4096, 768]-naive_softmax]_gbps": 491.21300194515715,
        "tritonbench_softmax_fwd[x_[4096, 768]-triton_softmax]_latency": 0.017184000462293625,
        "tritonbench_softmax_fwd[x_[4096, 768]-triton_softmax]_gbps": 1464.4915807131565,
        "tritonbench_softmax_fwd[x_[4096, 896]-naive_softmax]_latency": 0.05628800019621849,
        "tritonbench_softmax_fwd[x_[4096, 896]-naive_softmax]_gbps": 521.6054558280871,
        "tritonbench_softmax_fwd[x_[4096, 896]-triton_softmax]_latency": 0.018848000094294548,
        "tritonbench_softmax_fwd[x_[4096, 896]-triton_softmax]_gbps": 1557.7317409334887,
        "tritonbench_softmax_fwd[naive_softmax]-latency-avg": 0.043973333512743316,
        "tritonbench_softmax_fwd[naive_softmax]-gbps-avg": 412.83822458806753,
        "tritonbench_softmax_fwd[triton_softmax]-latency-avg": 0.014341333415359259,
        "tritonbench_softmax_fwd[triton_softmax]-gbps-avg": 1265.672440654116,
        "tritonbench_gemm_fwd[x_(256, 256, 256)-aten_matmul]_tflops": 4.315127532440617,
        "tritonbench_gemm_fwd[x_(256, 256, 256)-aten_matmul]_latency": 0.0077760000713169575,
        "tritonbench_gemm_fwd[x_(256, 256, 256)-triton_tutorial_matmul]_tflops": 4.681142792935275,
        "tritonbench_gemm_fwd[x_(256, 256, 256)-triton_tutorial_matmul]_latency": 0.0071680000983178616,
        "tritonbench_gemm_fwd[x_(384, 384, 384)-aten_matmul]_tflops": 12.078307577336817,
        "tritonbench_gemm_fwd[x_(384, 384, 384)-aten_matmul]_latency": 0.009375999681651592,
        "tritonbench_gemm_fwd[x_(384, 384, 384)-triton_tutorial_matmul]_tflops": 14.099379026156042,
        "tritonbench_gemm_fwd[x_(384, 384, 384)-triton_tutorial_matmul]_latency": 0.008031999692320824,
        "tritonbench_gemm_fwd[x_(512, 512, 512)-aten_matmul]_tflops": 29.433713587834777,
        "tritonbench_gemm_fwd[x_(512, 512, 512)-aten_matmul]_latency": 0.009119999594986439,
        "tritonbench_gemm_fwd[x_(512, 512, 512)-triton_tutorial_matmul]_tflops": 29.852698997369032,
        "tritonbench_gemm_fwd[x_(512, 512, 512)-triton_tutorial_matmul]_latency": 0.008991999551653862,
        "tritonbench_gemm_fwd[x_(640, 640, 640)-aten_matmul]_tflops": 43.22955092493062,
        "tritonbench_gemm_fwd[x_(640, 640, 640)-aten_matmul]_latency": 0.012128000147640705,
        "tritonbench_gemm_fwd[x_(640, 640, 640)-triton_tutorial_matmul]_tflops": 52.17834321176448,
        "tritonbench_gemm_fwd[x_(640, 640, 640)-triton_tutorial_matmul]_latency": 0.010048000141978264,
        "tritonbench_gemm_fwd[aten_matmul]-tflops-avg": 22.26417490563571,
        "tritonbench_gemm_fwd[aten_matmul]-latency-avg": 0.009599999873898923,
        "tritonbench_gemm_fwd[triton_tutorial_matmul]-tflops-avg": 25.202891007056206,
        "tritonbench_gemm_fwd[triton_tutorial_matmul]-latency-avg": 0.008559999871067703
    }
}

[facebook-github-bot]

@xuzhao9 has imported this pull request. If you are a Meta employee, you can view this diff on Phabricator.

[facebook-github-bot]

@xuzhao9 merged this pull request in 70f3ea2.