Skip to content

Commit 9551f0a

Browse files
committed
further
Signed-off-by: dafnapension <[email protected]>
1 parent d217aa3 commit 9551f0a

File tree

1 file changed

+12
-33
lines changed

1 file changed

+12
-33
lines changed

performance/compare_benchmark_performance_results.py

+12-33
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
import argparse
22
import json
3-
import os
43
import sys
54

65
# Argument parser to get file paths from the command line
@@ -23,24 +22,11 @@
2322
print(f'dataset_query = "{main_perf["dataset_query"]}"')
2423
print(f"used_eager_mode in main = {main_perf['used_eager_mode']}")
2524
print(f"used_eager_mode in PR = {pr_perf['used_eager_mode']}")
26-
print(f"use Mocked inference = {os.environ['UNITXT_MOCK_INFERENCE_MODE']}")
2725

2826
ratio1 = (
29-
(pr_perf["generate_benchmark_dataset_time"] - pr_perf["load_time_no_initial_ms"])
30-
/ (
31-
main_perf["generate_benchmark_dataset_time"]
32-
- main_perf["load_time_no_initial_ms"]
33-
)
34-
if (
35-
main_perf["generate_benchmark_dataset_time"]
36-
- main_perf["load_time_no_initial_ms"]
37-
)
38-
> 0
39-
else 1
40-
)
41-
ratio2 = (
42-
pr_perf["evaluation_time"] / main_perf["evaluation_time"]
43-
if main_perf["evaluation_time"] > 0
27+
(pr_perf["generate_benchmark_dataset_time"] - pr_perf["load_time"])
28+
/ (main_perf["generate_benchmark_dataset_time"] - main_perf["load_time"])
29+
if (main_perf["generate_benchmark_dataset_time"] - main_perf["load_time"]) > 0
4430
else 1
4531
)
4632
# Markdown table formatting
@@ -49,29 +35,22 @@
4935
line2 = "--------------------|-------------|-------------|---------------\n"
5036
line3 = f" Total time | {main_perf['total_time']:>11} | {pr_perf['total_time']:>11} | {pr_perf['total_time'] / main_perf['total_time']:.2f}\n"
5137
ratio_line4 = (
52-
pr_perf["load_time_no_initial_ms"] / main_perf["load_time_no_initial_ms"]
53-
if main_perf["load_time_no_initial_ms"] > 0
54-
else 1
38+
pr_perf["load_time"] / main_perf["load_time"] if main_perf["load_time"] > 0 else 1
5539
)
56-
line4 = f" Load time | {main_perf['load_time_no_initial_ms']:>11} | {pr_perf['load_time_no_initial_ms']:>11} | {ratio_line4:.2f}\n"
40+
line4 = f" Load time | {main_perf['load_time']:>11} | {pr_perf['load_time']:>11} | {ratio_line4:.2f}\n"
5741
line5 = f" DS Gen. inc. Load | {main_perf['generate_benchmark_dataset_time']:>11} | {pr_perf['generate_benchmark_dataset_time']:>11} | {pr_perf['generate_benchmark_dataset_time'] / main_perf['generate_benchmark_dataset_time']:.2f}\n"
58-
line6 = f" DS Gen. exc. Load | {round(main_perf['generate_benchmark_dataset_time'] - main_perf['load_time_no_initial_ms'], 3):>11} | {round(pr_perf['generate_benchmark_dataset_time'] - pr_perf['load_time_no_initial_ms'], 3):>11} | {ratio1:.2f}\n"
59-
line7 = f" Inference time | {main_perf['inference_time']:>11} | {pr_perf['inference_time']:>11} | {pr_perf['inference_time'] / main_perf['inference_time']:.2f}\n"
60-
line8 = f" Evaluate time | {main_perf['evaluation_time']:>11} | {pr_perf['evaluation_time']:>11} | {ratio2:.2f}\n"
61-
line9 = f" Benchmark Instant. | {main_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time'] / main_perf['instantiate_benchmark_time']:.2f}\n"
62-
line10 = f" Model Instantiation| {main_perf['instantiate_model_time']:>11} | {pr_perf['instantiate_model_time']:>11} | {pr_perf['instantiate_model_time'] / main_perf['instantiate_model_time']:.2f}\n"
42+
line6 = f" DS Gen. exc. Load | {round(main_perf['generate_benchmark_dataset_time'] - main_perf['load_time'], 3):>11} | {round(pr_perf['generate_benchmark_dataset_time'] - pr_perf['load_time'], 3):>11} | {ratio1:.2f}\n"
43+
line7 = f" Benchmark Instant. | {main_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time'] / main_perf['instantiate_benchmark_time']:.2f}\n"
6344

6445
print("### Performance Comparison Results, time expressed in seconds:\n")
65-
print(line1 + line2 + line3 + line4 + line5 + line6 + line7 + line8 + line9 + line10)
46+
print(line1 + line2 + line3 + line4 + line5 + line6 + line7)
6647
print("\n\n")
6748
# Performance degradation check (5% threshold)
68-
if ratio1 > 1.05 or ratio2 > 1.05:
69-
print(
70-
"\n**Warning**: Performance degradation in Dataset Generation and/or Evaluation exceeds 5%!"
71-
)
49+
if ratio1 > 1.05:
50+
print("\n**Warning**: Performance degradation in Dataset Generation exceeds 5%!")
7251
print(
73-
"Explore branch performance via 'python performance/bluebench_profiler.py --output_file=``path to json file``',"
74-
"followed by 'snakeviz ``the performance.prof file specified in the output json file``'."
52+
"Explore branch performance via 'python performance/bluebench_profiler.py --output_file=<path to json file>',"
53+
"followed by 'snakeviz <the performance.prof file specified in the output json file>'."
7554
)
7655
sys.exit(1)
7756

0 commit comments

Comments
 (0)