Skip to content

Also compare batch measurements in nvbench_compare.py #263

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Also compare batch measurements in nvbench_compare.py #263

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
58d1f8c
Also compare batch measurements in nvbench_compare.py
bernhardmgruber Aug 14, 2025
d140027
Fix
bernhardmgruber Aug 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 51 additions & 0 deletions scripts/nvbench_compare.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ def compare_benches(ref_benches, cmp_benches, threshold, plot):
headers.append("Status")
colalign.append("center")

added_batch_headers = False

for device_id in device_ids:
rows = []
plot_data = {"cmp": {}, "ref": {}, "cmp_noise": {}, "ref_noise": {}}
Expand Down Expand Up @@ -179,6 +181,12 @@ def lookup_summary(summaries, tag):
ref_noise_summary = lookup_summary(
ref_summaries, "nv/cold/time/gpu/stdev/relative"
)
cmp_batch_summary = lookup_summary(
cmp_summaries, "nv/batch/time/gpu/mean"
)
ref_batch_summary = lookup_summary(
ref_summaries, "nv/batch/time/gpu/mean"
)

# TODO: Use other timings, too. Maybe multiple rows, with a
# "Timing" column + values "CPU/GPU/Batch"?
Expand All @@ -192,6 +200,20 @@ def lookup_summary(summaries, tag):
):
continue

has_batch_data = cmp_batch_summary and ref_batch_summary
if has_batch_data and not added_batch_headers:
headers.append("B Ref Time")
colalign.append("right")
headers.append("B Cmp Time")
colalign.append("right")
headers.append("B Diff")
colalign.append("right")
headers.append("B %Diff")
colalign.append("right")
headers.append("B Status")
colalign.append("center")
added_batch_headers = True

def extract_value(summary):
summary_data = summary["data"]
value_data = next(
Expand All @@ -204,6 +226,9 @@ def extract_value(summary):
ref_time = extract_value(ref_time_summary)
cmp_noise = extract_value(cmp_noise_summary)
ref_noise = extract_value(ref_noise_summary)
if has_batch_data:
cmp_batch_time = extract_value(cmp_batch_summary)
ref_batch_time = extract_value(ref_batch_summary)

# Convert string encoding to expected numerics:
cmp_time = float(cmp_time)
Expand All @@ -212,6 +237,12 @@ def extract_value(summary):
diff = cmp_time - ref_time
frac_diff = diff / ref_time

if has_batch_data:
cmp_batch_time = float(cmp_batch_time)
ref_batch_time = float(ref_batch_time)
diff_batch = cmp_batch_time - ref_batch_time
frac_diff_batch = diff_batch / ref_batch_time

if ref_noise and cmp_noise:
ref_noise = float(ref_noise)
cmp_noise = float(cmp_noise)
Expand Down Expand Up @@ -269,6 +300,19 @@ def extract_value(summary):
failure_count += 1
status = Fore.RED + "SLOW" + Fore.RESET

if has_batch_data:
if (
abs(frac_diff_batch) <= 0.01
): # TODO(bgruber): what value to use here?
Copy link
Collaborator

@alliepiper alliepiper Aug 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have no idea, let's get some input internally on that.

Copy link
Collaborator

@alliepiper alliepiper Aug 20, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe just pick a sensible default and let the user override with command-line opts?

Copy link
Collaborator

@oleksandr-pavlyk oleksandr-pavlyk Oct 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should use min_noise estimated from abs(frac_diff_batch) <= min_noise if available, like we do for cold measurements, and use yellow tint if min_noise is not available.

Perhaps in case when has_batch_data is True, min_noise should always be available.

pass_count += 1
batch_status = Fore.BLUE + "SAME" + Fore.RESET
elif diff_batch < 0:
failure_count += 1
batch_status = Fore.GREEN + "FAST" + Fore.RESET
else:
failure_count += 1
batch_status = Fore.RED + "SLOW" + Fore.RESET

if abs(frac_diff) >= threshold:
row.append(format_duration(ref_time))
row.append(format_percentage(ref_noise))
Expand All @@ -278,6 +322,13 @@ def extract_value(summary):
row.append(format_percentage(frac_diff))
row.append(status)

if has_batch_data:
row.append(format_duration(ref_batch_time))
row.append(format_duration(cmp_batch_time))
row.append(format_duration(diff_batch))
row.append(format_percentage(frac_diff_batch))
row.append(batch_status)

rows.append(row)

if len(rows) == 0:
Expand Down
Loading