diff --git a/performance/compare_benchmark_performance_results.py b/performance/compare_benchmark_performance_results.py index 5765727d4..f1de94511 100644 --- a/performance/compare_benchmark_performance_results.py +++ b/performance/compare_benchmark_performance_results.py @@ -1,6 +1,5 @@ import argparse import json -import os import sys # Argument parser to get file paths from the command line @@ -23,24 +22,11 @@ print(f'dataset_query = "{main_perf["dataset_query"]}"') print(f"used_eager_mode in main = {main_perf['used_eager_mode']}") print(f"used_eager_mode in PR = {pr_perf['used_eager_mode']}") -print(f"use Mocked inference = {os.environ['UNITXT_MOCK_INFERENCE_MODE']}") ratio1 = ( - (pr_perf["generate_benchmark_dataset_time"] - pr_perf["load_time_no_initial_ms"]) - / ( - main_perf["generate_benchmark_dataset_time"] - - main_perf["load_time_no_initial_ms"] - ) - if ( - main_perf["generate_benchmark_dataset_time"] - - main_perf["load_time_no_initial_ms"] - ) - > 0 - else 1 -) -ratio2 = ( - pr_perf["evaluation_time"] / main_perf["evaluation_time"] - if main_perf["evaluation_time"] > 0 + (pr_perf["generate_benchmark_dataset_time"] - pr_perf["load_time"]) + / (main_perf["generate_benchmark_dataset_time"] - main_perf["load_time"]) + if (main_perf["generate_benchmark_dataset_time"] - main_perf["load_time"]) > 0 else 1 ) # Markdown table formatting @@ -49,29 +35,22 @@ line2 = "--------------------|-------------|-------------|---------------\n" line3 = f" Total time | {main_perf['total_time']:>11} | {pr_perf['total_time']:>11} | {pr_perf['total_time'] / main_perf['total_time']:.2f}\n" ratio_line4 = ( - pr_perf["load_time_no_initial_ms"] / main_perf["load_time_no_initial_ms"] - if main_perf["load_time_no_initial_ms"] > 0 - else 1 + pr_perf["load_time"] / main_perf["load_time"] if main_perf["load_time"] > 0 else 1 ) -line4 = f" Load time | {main_perf['load_time_no_initial_ms']:>11} | {pr_perf['load_time_no_initial_ms']:>11} | {ratio_line4:.2f}\n" +line4 = f" Load time | {main_perf['load_time']:>11} | {pr_perf['load_time']:>11} | {ratio_line4:.2f}\n" line5 = f" DS Gen. inc. Load | {main_perf['generate_benchmark_dataset_time']:>11} | {pr_perf['generate_benchmark_dataset_time']:>11} | {pr_perf['generate_benchmark_dataset_time'] / main_perf['generate_benchmark_dataset_time']:.2f}\n" -line6 = f" DS Gen. exc. Load | {round(main_perf['generate_benchmark_dataset_time'] - main_perf['load_time_no_initial_ms'], 3):>11} | {round(pr_perf['generate_benchmark_dataset_time'] - pr_perf['load_time_no_initial_ms'], 3):>11} | {ratio1:.2f}\n" -line7 = f" Inference time | {main_perf['inference_time']:>11} | {pr_perf['inference_time']:>11} | {pr_perf['inference_time'] / main_perf['inference_time']:.2f}\n" -line8 = f" Evaluate time | {main_perf['evaluation_time']:>11} | {pr_perf['evaluation_time']:>11} | {ratio2:.2f}\n" -line9 = f" Benchmark Instant. | {main_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time'] / main_perf['instantiate_benchmark_time']:.2f}\n" -line10 = f" Model Instantiation| {main_perf['instantiate_model_time']:>11} | {pr_perf['instantiate_model_time']:>11} | {pr_perf['instantiate_model_time'] / main_perf['instantiate_model_time']:.2f}\n" +line6 = f" DS Gen. exc. Load | {round(main_perf['generate_benchmark_dataset_time'] - main_perf['load_time'], 3):>11} | {round(pr_perf['generate_benchmark_dataset_time'] - pr_perf['load_time'], 3):>11} | {ratio1:.2f}\n" +line7 = f" Benchmark Instant. | {main_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time'] / main_perf['instantiate_benchmark_time']:.2f}\n" print("### Performance Comparison Results, time expressed in seconds:\n") -print(line1 + line2 + line3 + line4 + line5 + line6 + line7 + line8 + line9 + line10) +print(line1 + line2 + line3 + line4 + line5 + line6 + line7) print("\n\n") # Performance degradation check (5% threshold) -if ratio1 > 1.05 or ratio2 > 1.05: - print( - "\n**Warning**: Performance degradation in Dataset Generation and/or Evaluation exceeds 5%!" - ) +if ratio1 > 1.05: + print("\n**Warning**: Performance degradation in Dataset Generation exceeds 5%!") print( - "Explore branch performance via 'python performance/bluebench_profiler.py --output_file=``path to json file``'," - "followed by 'snakeviz ``the performance.prof file specified in the output json file``'." + "Explore branch performance via 'python performance/bluebench_profiler.py --output_file='," + "followed by 'snakeviz '." ) sys.exit(1)