|
1 | 1 | import argparse
|
2 | 2 | import json
|
3 |
| -import os |
4 | 3 | import sys
|
5 | 4 |
|
6 | 5 | # Argument parser to get file paths from the command line
|
|
23 | 22 | print(f'dataset_query = "{main_perf["dataset_query"]}"')
|
24 | 23 | print(f"used_eager_mode in main = {main_perf['used_eager_mode']}")
|
25 | 24 | print(f"used_eager_mode in PR = {pr_perf['used_eager_mode']}")
|
26 |
| -print(f"use Mocked inference = {os.environ['UNITXT_MOCK_INFERENCE_MODE']}") |
27 | 25 |
|
28 | 26 | ratio1 = (
|
29 |
| - (pr_perf["generate_benchmark_dataset_time"] - pr_perf["load_time_no_initial_ms"]) |
30 |
| - / ( |
31 |
| - main_perf["generate_benchmark_dataset_time"] |
32 |
| - - main_perf["load_time_no_initial_ms"] |
33 |
| - ) |
34 |
| - if ( |
35 |
| - main_perf["generate_benchmark_dataset_time"] |
36 |
| - - main_perf["load_time_no_initial_ms"] |
37 |
| - ) |
38 |
| - > 0 |
39 |
| - else 1 |
40 |
| -) |
41 |
| -ratio2 = ( |
42 |
| - pr_perf["evaluation_time"] / main_perf["evaluation_time"] |
43 |
| - if main_perf["evaluation_time"] > 0 |
| 27 | + (pr_perf["generate_benchmark_dataset_time"] - pr_perf["load_time"]) |
| 28 | + / (main_perf["generate_benchmark_dataset_time"] - main_perf["load_time"]) |
| 29 | + if (main_perf["generate_benchmark_dataset_time"] - main_perf["load_time"]) > 0 |
44 | 30 | else 1
|
45 | 31 | )
|
46 | 32 | # Markdown table formatting
|
|
49 | 35 | line2 = "--------------------|-------------|-------------|---------------\n"
|
50 | 36 | line3 = f" Total time | {main_perf['total_time']:>11} | {pr_perf['total_time']:>11} | {pr_perf['total_time'] / main_perf['total_time']:.2f}\n"
|
51 | 37 | ratio_line4 = (
|
52 |
| - pr_perf["load_time_no_initial_ms"] / main_perf["load_time_no_initial_ms"] |
53 |
| - if main_perf["load_time_no_initial_ms"] > 0 |
54 |
| - else 1 |
| 38 | + pr_perf["load_time"] / main_perf["load_time"] if main_perf["load_time"] > 0 else 1 |
55 | 39 | )
|
56 |
| -line4 = f" Load time | {main_perf['load_time_no_initial_ms']:>11} | {pr_perf['load_time_no_initial_ms']:>11} | {ratio_line4:.2f}\n" |
| 40 | +line4 = f" Load time | {main_perf['load_time']:>11} | {pr_perf['load_time']:>11} | {ratio_line4:.2f}\n" |
57 | 41 | line5 = f" DS Gen. inc. Load | {main_perf['generate_benchmark_dataset_time']:>11} | {pr_perf['generate_benchmark_dataset_time']:>11} | {pr_perf['generate_benchmark_dataset_time'] / main_perf['generate_benchmark_dataset_time']:.2f}\n"
|
58 |
| -line6 = f" DS Gen. exc. Load | {round(main_perf['generate_benchmark_dataset_time'] - main_perf['load_time_no_initial_ms'], 3):>11} | {round(pr_perf['generate_benchmark_dataset_time'] - pr_perf['load_time_no_initial_ms'], 3):>11} | {ratio1:.2f}\n" |
59 |
| -line7 = f" Inference time | {main_perf['inference_time']:>11} | {pr_perf['inference_time']:>11} | {pr_perf['inference_time'] / main_perf['inference_time']:.2f}\n" |
60 |
| -line8 = f" Evaluate time | {main_perf['evaluation_time']:>11} | {pr_perf['evaluation_time']:>11} | {ratio2:.2f}\n" |
61 |
| -line9 = f" Benchmark Instant. | {main_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time'] / main_perf['instantiate_benchmark_time']:.2f}\n" |
62 |
| -line10 = f" Model Instantiation| {main_perf['instantiate_model_time']:>11} | {pr_perf['instantiate_model_time']:>11} | {pr_perf['instantiate_model_time'] / main_perf['instantiate_model_time']:.2f}\n" |
| 42 | +line6 = f" DS Gen. exc. Load | {round(main_perf['generate_benchmark_dataset_time'] - main_perf['load_time'], 3):>11} | {round(pr_perf['generate_benchmark_dataset_time'] - pr_perf['load_time'], 3):>11} | {ratio1:.2f}\n" |
| 43 | +line7 = f" Benchmark Instant. | {main_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time']:>11} | {pr_perf['instantiate_benchmark_time'] / main_perf['instantiate_benchmark_time']:.2f}\n" |
63 | 44 |
|
64 | 45 | print("### Performance Comparison Results, time expressed in seconds:\n")
|
65 |
| -print(line1 + line2 + line3 + line4 + line5 + line6 + line7 + line8 + line9 + line10) |
| 46 | +print(line1 + line2 + line3 + line4 + line5 + line6 + line7) |
66 | 47 | print("\n\n")
|
67 | 48 | # Performance degradation check (5% threshold)
|
68 |
| -if ratio1 > 1.05 or ratio2 > 1.05: |
69 |
| - print( |
70 |
| - "\n**Warning**: Performance degradation in Dataset Generation and/or Evaluation exceeds 5%!" |
71 |
| - ) |
| 49 | +if ratio1 > 1.05: |
| 50 | + print("\n**Warning**: Performance degradation in Dataset Generation exceeds 5%!") |
72 | 51 | print(
|
73 |
| - "Explore branch performance via 'python performance/bluebench_profiler.py --output_file=``path to json file``'," |
74 |
| - "followed by 'snakeviz ``the performance.prof file specified in the output json file``'." |
| 52 | + "Explore branch performance via 'python performance/bluebench_profiler.py --output_file=<path to json file>'," |
| 53 | + "followed by 'snakeviz <the performance.prof file specified in the output json file>'." |
75 | 54 | )
|
76 | 55 | sys.exit(1)
|
77 | 56 |
|
|
0 commit comments