feat: added p90 step time logic for GEMM and HBM tail latencies

Author: simrankaurb

Ironwood/src/benchmark_hbm.py

@@ -89,14 +89,24 @@ def single_device_hbm_copy_calculate_metrics(
     )
     print(
         f"Tensor size: {tensor_size_bytes / 1024**2} MB, "
-        f"time taken (median): {time_statistics.statistics["p50"]:.4f} ms, "
-        f"bandwidth (median): {statistics.statistics["p50"]:.3f} GB/s"
+        f"time taken (p10): {time_statistics.statistics['p10']:.4f} ms, "
+        f"time taken (median): {time_statistics.statistics['p50']:.4f} ms, "
+        f"time taken (p90): {time_statistics.statistics['p90']:.4f} ms, "
+        f"bandwidth (p10): {statistics.statistics['p10']:.3f} GB/s, "
+        f"bandwidth (median): {statistics.statistics['p50']:.3f} GB/s, "
+        f"bandwidth (p90): {statistics.statistics['p90']:.3f} GB/s"
     )
     print()
     # Gather the metrics to report.
     metadata.update(
         {
             "tensor_size_gbytes": tensor_size_gbytes,
+            "time_taken(p10,ms)": time_statistics.statistics["p10"],
+            "time_taken(median,ms)": time_statistics.statistics["p50"],
+            "time_taken(p90,ms)": time_statistics.statistics["p90"],
+            "bandwidth(p10,GB/s)": statistics.statistics["p10"],
+            "bandwidth(median,GB/s)": statistics.statistics["p50"],
+            "bandwidth(p90,GB/s)": statistics.statistics["p90"],
         }
     )
     metrics.update(time_statistics.serialize_statistics())

Ironwood/src/benchmark_utils.py

@@ -858,6 +858,7 @@ def _calculate_statistics(self) -> Dict[str, float]:
         if not self.metrics_list:
             return {}  # Return an empty dict if metrics_list is empty
         return {
+            "p10": np.percentile(self.metrics_list, 10),
             "p50": np.percentile(self.metrics_list, 50),
             "p90": np.percentile(self.metrics_list, 90),
             "p95": np.percentile(self.metrics_list, 95),
@@ -1288,28 +1289,63 @@ def unified_flops_metrics(
     dtype_prefix = f"[{dtype}] " if dtype is not None else ""
     print(
         f"{dtype_prefix}"
-        f"Total floating-point ops: {total_flops}, Step Time (median): "
-        f"{average_time_ms_statistics.statistics["p50"]:.2f}, "
+        f"Total floating-point ops: {total_flops}, "
+        f"Step Time (p10): "
+        f"{average_time_ms_statistics.statistics['p10']:.2f}, "
+        f"Step Time (median): "
+        f"{average_time_ms_statistics.statistics['p50']:.2f}, "
+        f"Step Time (p90): "
+        f"{average_time_ms_statistics.statistics['p90']:.2f}, "
+        f"Throughput (p10): "
+        f"{tflops_per_sec_statistics.statistics['p10']:.2f}"
+        f" TFLOP / second / device, "
         f"Throughput (median): "
-        f"{tflops_per_sec_statistics.statistics["p50"]:.2f}"
+        f"{tflops_per_sec_statistics.statistics['p50']:.2f}"
+        f" TFLOP / second / device, "
+        f"Throughput (p90): "
+        f"{tflops_per_sec_statistics.statistics['p90']:.2f}"
         f" TFLOP / second / device, "
+        f"TotalThroughput (p10): "
+        f"{tflops_per_sec_all_devices_statistics.statistics['p10']:.2f} "
+        f"TFLOP / second, "
         f"TotalThroughput (median): "
-        f"{tflops_per_sec_all_devices_statistics.statistics["p50"]:.2f} "
+        f"{tflops_per_sec_all_devices_statistics.statistics['p50']:.2f} "
+        f"TFLOP / second, "
+        f"TotalThroughput (p90): "
+        f"{tflops_per_sec_all_devices_statistics.statistics['p90']:.2f} "
         f"TFLOP / second, "
-        f"MFU: {mfu_statistics.statistics["p50"]:.2%}"
+        f"MFU (p10): {mfu_statistics.statistics['p10']:.2%}, "
+        f"MFU (median): {mfu_statistics.statistics['p50']:.2%}, "
+        f"MFU (p90): {mfu_statistics.statistics['p90']:.2%}"
     )
 
     # Gather the metrics to report.
     metadata.update(
         {
+            "StepTime(p10,ms)": average_time_ms_statistics.statistics["p10"],
             "StepTime(median,ms)": average_time_ms_statistics.statistics["p50"],
+            "StepTime(p90,ms)": average_time_ms_statistics.statistics["p90"],
+            "Throughput(p10,TFLOP/s/device)": (
+                tflops_per_sec_statistics.statistics["p10"]
+            ),
             "Throughput(median,TFLOP/s/device)": (
                 tflops_per_sec_statistics.statistics["p50"]
             ),
+            "Throughput(p90,TFLOP/s/device)": (
+                tflops_per_sec_statistics.statistics["p90"]
+            ),
+            "TotalThroughput(p10,TFLOP/s)": (
+                tflops_per_sec_all_devices_statistics.statistics["p10"]
+            ),
             "TotalThroughput(median,TFLOP/s)": (
                 tflops_per_sec_all_devices_statistics.statistics["p50"]
             ),
-            "MFU": mfu_statistics.statistics["p50"],
+            "TotalThroughput(p90,TFLOP/s)": (
+                tflops_per_sec_all_devices_statistics.statistics["p90"]
+            ),
+            "MFU(p10)": mfu_statistics.statistics["p10"],
+            "MFU(median)": mfu_statistics.statistics["p50"],
+            "MFU(p90)": mfu_statistics.statistics["p90"],
             "total_flops": total_flops,
         }
     )
@@ -1373,25 +1409,41 @@ def unified_bytes_metrics(
         type_prefix = f"[d={dtype}] "
     print(
         f"{type_prefix}"
-        f"Total bytes: {total_bytes}, Step Time (median): "
-        f"{average_time_ms_statistics.statistics["p50"]:.2f}, "
-        f"Throughput (median):"
-        f"{gigabytes_per_sec_statistics.statistics["p50"]:.2f} "
-        f"GBytes / second / device, "
-        f"TotalThroughput (median): "
-        f"{gigabytes_per_sec_all_devices_statistics.statistics["p50"]:.2f} "
-        f"GBytes / second"
+        f"Total bytes: {total_bytes}, "
+        f"Step Time (p10): {average_time_ms_statistics.statistics['p10']:.2f}, "
+        f"Step Time (median): {average_time_ms_statistics.statistics['p50']:.2f}, "
+        f"Step Time (p90): {average_time_ms_statistics.statistics['p90']:.2f}, "
+        f"Throughput (p10): {gigabytes_per_sec_statistics.statistics['p10']:.2f} GBytes / second / device, "
+        f"Throughput (median): {gigabytes_per_sec_statistics.statistics['p50']:.2f} GBytes / second / device, "
+        f"Throughput (p90): {gigabytes_per_sec_statistics.statistics['p90']:.2f} GBytes / second / device, "
+        f"TotalThroughput (p10): {gigabytes_per_sec_all_devices_statistics.statistics['p10']:.2f} GBytes / second, "
+        f"TotalThroughput (median): {gigabytes_per_sec_all_devices_statistics.statistics['p50']:.2f} GBytes / second, "
+        f"TotalThroughput (p90): {gigabytes_per_sec_all_devices_statistics.statistics['p90']:.2f} GBytes / second"
     )
     print()
     metadata.update(
         {
+            "StepTime(p10,ms)": average_time_ms_statistics.statistics["p10"],
             "StepTime(median,ms)": average_time_ms_statistics.statistics["p50"],
+            "StepTime(p90,ms)": average_time_ms_statistics.statistics["p90"],
+            "Throughput(p10,GBytes/s/device)": (
+                gigabytes_per_sec_statistics.statistics["p10"]
+            ),
             "Throughput(median,GBytes/s/device)": (
                 gigabytes_per_sec_statistics.statistics["p50"]
             ),
+            "Throughput(p90,GBytes/s/device)": (
+                gigabytes_per_sec_statistics.statistics["p90"]
+            ),
+            "TotalThroughput(p10,GBytes/s)": (
+                gigabytes_per_sec_all_devices_statistics.statistics["p10"]
+            ),
             "TotalThroughput(median,GBytes/s)": (
                 gigabytes_per_sec_all_devices_statistics.statistics["p50"]
             ),
+            "TotalThroughput(p90,GBytes/s)": (
+                gigabytes_per_sec_all_devices_statistics.statistics["p90"]
+            ),
             "total_bytes": total_bytes,
         }
     )