From 708939ff173ecbf50ff51c75418520e4943555af Mon Sep 17 00:00:00 2001 From: Mohaned Qunaibit Date: Wed, 12 Oct 2022 12:12:07 +0300 Subject: [PATCH 1/2] Add machine comparison list to published results --- asv/commands/publish.py | 11 ++ asv/config.py | 2 + asv/plugins/comparisonlist.py | 210 +++++++++++++++++++++ asv/www/asv.js | 2 + asv/www/comparisonlist.css | 68 +++++++ asv/www/comparisonlist.js | 337 ++++++++++++++++++++++++++++++++++ asv/www/index.html | 9 + 7 files changed, 639 insertions(+) create mode 100644 asv/plugins/comparisonlist.py create mode 100644 asv/www/comparisonlist.css create mode 100644 asv/www/comparisonlist.js diff --git a/asv/commands/publish.py b/asv/commands/publish.py index e5eb0bce5..c4242bd94 100644 --- a/asv/commands/publish.py +++ b/asv/commands/publish.py @@ -58,6 +58,14 @@ def setup_arguments(cls, subparsers): '--html-dir', '-o', default=None, help=( "Optional output directory. Default is 'html_dir' " "from asv config")) + parser.add_argument( + '--baseline-machine', default=None, help=( + "Optional baseline comparisons between machines. Provide " + "machine name")) + parser.add_argument( + '--generate-markdown', action='store_true', dest='generate_markdown', + help=("Optional output a generated markdown file comparisons " + "between machines in the 'html_dir'.")) parser.set_defaults(func=cls.run_from_args) @@ -67,6 +75,9 @@ def setup_arguments(cls, subparsers): def run_from_conf_args(cls, conf, args): if args.html_dir is not None: conf.html_dir = args.html_dir + if args.baseline_machine is not None: + conf.baseline_machine = args.baseline_machine + conf.generate_markdown = args.generate_markdown return cls.run(conf=conf, range_spec=args.range, pull=not args.no_pull) @staticmethod diff --git a/asv/config.py b/asv/config.py index 473007c20..5137b5597 100644 --- a/asv/config.py +++ b/asv/config.py @@ -42,6 +42,8 @@ def __init__(self): self.build_command = None self.install_command = None self.uninstall_command = None + self.baseline_machine = None + self.generate_markdown = False @classmethod def load(cls, path=None): diff --git a/asv/plugins/comparisonlist.py b/asv/plugins/comparisonlist.py new file mode 100644 index 000000000..575785f74 --- /dev/null +++ b/asv/plugins/comparisonlist.py @@ -0,0 +1,210 @@ +# Licensed under a 3-clause BSD style license - see LICENSE.rst +import os +import itertools + +from ..console import log +from ..publishing import OutputPublisher +from .. import util + + +def benchmark_param_iter(benchmark): + """ + Iterate over all combinations of parameterized benchmark parameters. + + Yields + ------ + idx : int + Combination flat index. `None` if benchmark not parameterized. + params : tuple + Tuple of parameter values. + + """ + if not benchmark['params']: + yield None, () + else: + for item in enumerate(itertools.product(*benchmark['params'])): + yield item + +time_units = [ + ['`ps`', 'picoseconds', 0.000000000001], + ['`ns`', 'nanoseconds', 0.000000001], + ['`μs`', 'microseconds', 0.000001], + ['`ms`', 'milliseconds', 0.001], + ['`s`', 'seconds', 1], + ['`m`', 'minutes', 60], + ['`h`', 'hours', 60 * 60], + ['`d`', 'days', 60 * 60 * 24], + ['`w`', 'weeks', 60 * 60 * 24 * 7], + ['`y`', 'years', 60 * 60 * 24 * 7 * 52], + ['`C`', 'centuries', 60 * 60 * 24 * 7 * 52 * 100] +] + +def pretty_time_unit(x, unit): + if unit == 'seconds': + for i in range(len(time_units) - 1): + if abs(x) < time_units[i+1][2]: + return '%.3f' % (x / time_units[i][2]) + time_units[i][0] + return 'inf' + else: + return '%.3f' % x + unit + +class ComparisonList(OutputPublisher): + name = "comparisonlist" + button_label = "List view" + description = "Display as a list" + order = 1 + + @classmethod + def publish(cls, conf, repo, benchmarks, graphs, revisions): + machines = list(graphs.get_params()["machine"]) + num_machines = len(machines) + baseline_machine_idx = -1 + if conf.baseline_machine: + baseline_machine_idx = machines.index(conf.baseline_machine) + + result = { + "machines" : machines, + "benchmarks" : None, + } + benchmarks_result = [] + warned = False + + # Investigate all benchmarks + for benchmark_name, benchmark in sorted(benchmarks.items()): + log.dot() + + benchmark_graphs = graphs.get_graph_group(benchmark_name) + + # For parameterized benchmarks, consider each combination separately + for idx, benchmark_param in benchmark_param_iter(benchmark): + pretty_name = benchmark_name + + if benchmark.get('pretty_name'): + pretty_name = benchmark['pretty_name'] + + if idx is not None: + pretty_name = '{0}({1})'.format(pretty_name, + ", ".join(benchmark_param)) + + # Each environment parameter combination is reported + # separately on the comparisonlist page + benchmark_graphs = graphs.get_graph_group(benchmark_name) + benchmark_data = None + best_val = None + worst_val = None + for graph in benchmark_graphs: + machine_idx = machines.index(graph.params["machine"]) + if not benchmark_data: + benchmark_data = { + "name" : benchmark_name, + "pretty_name" : pretty_name, + "idx" : idx, + "best" : -1, + "worst" : -1, + "last_rev" : [None] * num_machines, + "last_value" : [None] * num_machines, + "last_err" : [None] * num_machines, + "cmp_percent" : [0.] * num_machines, + } + + # Produce interesting information, based on + # stepwise fit on the benchmark data (reduces noise) + steps = graph.get_steps() + if idx is not None and steps: + steps = graph.get_steps()[idx] + + last_value = None + last_err = None + last_rev = None + + if not steps: + # No data + pass + else: + last_piece = steps[-1] + last_value = last_piece[2] + if best_val is None or last_value < best_val: + benchmark_data["best"] = machine_idx + best_val = last_value + if worst_val is None or last_value > worst_val: + benchmark_data["worst"] = machine_idx + worst_val = last_value + last_err = last_piece[4] + last_rev = last_piece[1] - 1 + if not warned and benchmark_data["last_value"][machine_idx]: + warned = True + log.warning("There are two machines that has the same name '%s'" % machines[machine_idx]) + benchmark_data["last_value"][machine_idx] = last_value + benchmark_data["last_err"][machine_idx] = last_err + benchmark_data["last_rev"][machine_idx] = last_rev + if benchmark_data and best_val: + benchmarks_result.append(benchmark_data) + + if baseline_machine_idx != -1: + num_benchmarks = len(benchmarks_result) + cmp_list = [0.] * num_machines + for bench_idx in range(num_benchmarks): + values = benchmarks_result[bench_idx]["last_value"] + b = values[baseline_machine_idx] + if b: + for machine_idx in range(num_machines): + v = values[machine_idx] + if v: + p = (v - b) / b * 100 + cmp_list[machine_idx] += p + benchmarks_result[bench_idx]["cmp_percent"][machine_idx] = p + + benchmarks_average_cmp = [None] * num_machines + for machine_idx in range(num_machines): + benchmarks_average_cmp[machine_idx] = cmp_list[machine_idx]/num_benchmarks + result["average"] = benchmarks_average_cmp + result["baseline"] = baseline_machine_idx + + + def machine_idx_sort(row): + idx = row['best'] + if idx == -1: + return 9999 + if baseline_machine_idx != -1: + if idx == baseline_machine_idx: + v = max(row["cmp_percent"])/100 + return idx - v + else: + v = row["cmp_percent"][idx]/100 + return idx + v + + return idx + result["benchmarks"] = sorted(benchmarks_result, key=machine_idx_sort) + # Write results to file + util.write_json(os.path.join(conf.html_dir, "comparison.json"), result, compact=True) + + if conf.generate_markdown: + # Generate a markdown page + with open(os.path.join(conf.html_dir, "comparison.md"), "w") as fp: + machines = result["machines"] + num_machines = len(machines) + fp.write('# Benchmark Machine\n') + fp.write('* CPU: %s\n' % list(graphs.get_params()["cpu"])[0]) + fp.write('* CPU Cores: %s\n' % list(graphs.get_params()["num_cpu"])[0]) + fp.write('* OS: %s\n' % list(graphs.get_params()["os"])[0]) + fp.write('* RAM: %dGB\n' % (int(list(graphs.get_params()["ram"])[0])//1000000)) + fp.write('\n\n') + fp.write('# Results\n') + fp.write('| No. |' + '|'.join(machines + ["Benchmarks"]) + '|\n') + fp.write('| :-- |' + '|'.join([":--"] * (num_machines + 1)) + '|\n') + if baseline_machine_idx != -1: + avg = ['%.2f%%' % v for v in result["average"]] + fp.write('| - |' + '|'.join(avg + ["Average"]) + '|\n') + count = 1 + for benchmark in result["benchmarks"]: + if None in benchmark["last_value"]: + continue + unit = benchmarks[benchmark["name"]]["unit"] + row = '| %d ' % count + count += 1 + for machine_idx in range(num_machines): + row += '|' + pretty_time_unit(benchmark["last_value"][machine_idx], unit) + if baseline_machine_idx != -1 and baseline_machine_idx != machine_idx: + row += ' `%.2f%%`' % benchmark["cmp_percent"][machine_idx] + row += '|' + benchmark["pretty_name"] + '|\n' + fp.write(row) diff --git a/asv/www/asv.js b/asv/www/asv.js index c3c54ac5c..7c224601f 100644 --- a/asv/www/asv.js +++ b/asv/www/asv.js @@ -382,6 +382,7 @@ $(document).ready(function() { $("#graph-display").hide(); $("#summarygrid-display").hide(); $("#summarylist-display").hide(); + $("#comparisonlist-display").hide(); $('#regressions-display').hide(); $('.tooltip').remove(); loaded_pages[name](params); @@ -461,6 +462,7 @@ $(document).ready(function() { $('#regressions-display').hide(); $('#summarygrid-display').hide(); $('#summarylist-display').hide(); + $('#comparisonlist-display').hide(); hashchange(); }).fail(function () { diff --git a/asv/www/comparisonlist.css b/asv/www/comparisonlist.css new file mode 100644 index 000000000..899d647e3 --- /dev/null +++ b/asv/www/comparisonlist.css @@ -0,0 +1,68 @@ +#comparisonlist-body { + padding-left: 2em; + padding-right: 2em; + padding-top: 1em; + padding-bottom: 2em; +} + +#comparisonlist-body table thead th { + background: white; + cursor: pointer; + white-space: nowrap; + position: sticky; + top: 0; /* Don't forget this, required for the stickiness */ +} + +#comparisonlist-body table thead th.desc:after { + content: ' \2191'; +} + +#comparisonlist-body table thead th.asc:after { + content: ' \2193'; +} + +#comparisonlist-body table.ignored { + padding-top: 1em; + color: #ccc; + background-color: #eee; +} + +#comparisonlist-body table.ignored a { + color: #82abda; +} + +#comparisonlist-body table tbody td.positive-change { + background-color: #fdd; +} + +#comparisonlist-body table tbody td.negative-change { + background-color: #dfd; +} + +#comparisonlist-body table tbody td.stats { + white-space: nowrap; + font-weight: bold; +} + +#comparisonlist-body table tbody td.value { + white-space: nowrap; +} + +#comparisonlist-body table tbody td.value-best { + white-space: nowrap; + color: green; +} + +#comparisonlist-body table tbody td.value-worst { + white-space: nowrap; + color: red; +} + +#comparisonlist-body table tbody td.change a { + color: black; + white-space: nowrap; +} + +#comparisonlist-body table tbody td.change-date { + white-space: nowrap; +} diff --git a/asv/www/comparisonlist.js b/asv/www/comparisonlist.js new file mode 100644 index 000000000..64cae5aa4 --- /dev/null +++ b/asv/www/comparisonlist.js @@ -0,0 +1,337 @@ +'use strict'; + +$(document).ready(function() { + /* The state of the parameters in the sidebar. Dictionary mapping + strings to values determining the "enabled" configurations. */ + var state = null; + /* Cache of constructed tables, {data_path: table_dom_id} */ + var table_cache = {}; + var table_cache_counter = 0; + + function setup_display(state_selection) { + var new_state = setup_state(state_selection); + var same_state = (state !== null); + + /* Avoid needless UI updates, e.g., on table sort */ + + if (same_state) { + $.each(state, function (key, value) { + if (value != new_state[key]) { + same_state = false; + } + }); + } + + if (!same_state) { + state = new_state; + + $("#comparisonlist-body table").hide(); + $("#comparisonlist-body .message").remove(); + + if (table_cache['comparisonlist'] !== undefined) { + $(table_cache['comparisonlist']).show(); + } + else { + $("#comparisonlist-body").append($("

Loading...

")); + $.ajax({ + url: "comparison.json", + dataType: "json", + cache: true + }).done(function (data) { + var table = construct_machines_comparison_benchmark_table(data); + var table_name = 'comparisonlist-table-' + table_cache_counter; + ++table_cache_counter; + + table.attr('id', table_name); + table_cache['comparisonlist'] = '#' + table_name; + $("#comparisonlist-body .message").remove(); + $("#comparisonlist-body").append(table); + table.show(); + }); + } + } + } + + function update_state_url(key, value) { + var info = $.asv.parse_hash_string(window.location.hash); + var new_state = get_valid_state(state, key, value); + + $.each($.asv.master_json.params, function(param, values) { + if (values.length > 1) { + info.params[param] = [new_state[param]]; + } + else if (info.params[param]) { + delete info.params[param]; + } + }); + + window.location.hash = $.asv.format_hash_string(info); + } + + function obj_copy(obj) { + var newobj = {}; + $.each(obj, function(key, val) { + newobj[key] = val; + }); + return newobj; + } + + function obj_diff(obj1, obj2) { + var count = 0; + $.each(obj1, function(key, val) { + if (obj2[key] != val) { + ++count + } + }); + return count; + } + + function get_valid_state(tmp_state, wanted_key, wanted_value) { + /* + Get an available state with wanted_key having wanted_value, + preferably as a minor modification of tmp_state. + */ + var best_params = null; + var best_diff = 1e99; + var best_hit = false; + + tmp_state = obj_copy(tmp_state); + if (wanted_key !== undefined) { + tmp_state[wanted_key] = wanted_value; + } + + $.each($.asv.master_json.graph_param_list, function(idx, params) { + var diff = obj_diff(tmp_state, params); + var hit = (wanted_key === undefined || params[wanted_key] == wanted_value); + + if ((!best_hit && hit) || (hit == best_hit && diff < best_diff)) { + best_params = params; + best_diff = diff; + best_hit = hit; + } + }); + + if (best_params === null) { + best_params = $.asv.master_json.graph_param_list[0]; + } + + return obj_copy(best_params); + } + + function setup_state(state_selection) { + var index = $.asv.master_json; + var state = {}; + + state.machine = index.params.machine; + + $.each(index.params, function(param, values) { + state[param] = values[0]; + }); + + if (state_selection !== null) { + /* Select a specific generic parameter state */ + $.each(index.params, function(param, values) { + if (state_selection[param]) { + state[param] = state_selection[param][0]; + } + }); + } + + return get_valid_state(state); + } + + function construct_machines_comparison_benchmark_table(data) { + var index = $.asv.master_json; + + var table = $(''); + + /* Form a new table */ + var machines = data.machines; + var benchmarks = data.benchmarks; + var average = data.average; + var baseline_machine = data.baseline; + + var second_row = $(''); + var total = $('' + var percent = $('' + + '' + machines_head + + ''); + table.append(table_head); + + var table_body = $(''); + if (average !== undefined) { + table_body.append(second_row); + } else { + baseline_machine = -1; + } + + + $.each(benchmarks, function(row_idx, row) { + var tr = $(''); + var name_td = $('
'); + total.text('Total number of benchmarks: ' + benchmarks.length); + second_row.append(total); + var machines_head = ''; + $.each(machines, function(machine_idx, machine) { + machines_head += '' + machine + ''); + if (average !== undefined) { + percent.append(average[machine_idx].toFixed(2) + '%'); + second_row.append(percent); + } + + }); + var table_head = $('
Benchmark
'); + var name = $(''); + var benchmark_url_args = {}; + var benchmark_full_url; + var benchmark_base_url; + + /* Format benchmark url */ + benchmark_url_args.location = [row.name]; + benchmark_url_args.params = {}; + $.each($.asv.master_json.params, function (key, values) { + if (values.length > 1) { + benchmark_url_args.params[key] = [state[key]]; + } + }); + benchmark_base_url = $.asv.format_hash_string(benchmark_url_args); + if (row.idx !== null) { + var benchmark = $.asv.master_json.benchmarks[row.name]; + $.each($.asv.param_selection_from_flat_idx(benchmark.params, row.idx).slice(1), + function(i, param_values) { + benchmark_url_args.params['p-'+benchmark.param_names[i]] + = [benchmark.params[i][param_values[0]]]; + }); + } + benchmark_full_url = $.asv.format_hash_string(benchmark_url_args); + + /* Benchmark name column */ + var bm_link; + if (row.idx === null) { + bm_link = $('').attr('href', benchmark_base_url).text(row.pretty_name); + name_td.append(bm_link); + } + else { + var basename = row.pretty_name; + var args = null; + var m = row.pretty_name.match(/(.*)\(.*$/); + if (m) { + basename = m[1]; + args = row.pretty_name.slice(basename.length); + } + bm_link = $('').attr('href', benchmark_base_url).text(basename); + name_td.append(bm_link); + if (args) { + var bm_idx_link; + var graph_url; + bm_idx_link = $('').attr('href', benchmark_full_url).text(' ' + args); + name_td.append(bm_idx_link); + graph_url = $.asv.graph_to_path(row.name, state); + $.asv.ui.hover_graph(bm_idx_link, graph_url, row.name, row.idx, null); + } + } + $.asv.ui.hover_summary_graph(bm_link, row.name); + tr.append(name_td); + + /* Values column */ + $.each(machines, function(machine_idx, machine) { + var last_value = row.last_value[machine_idx]; + var last_err = row.last_err[machine_idx]; + var value_class = "value"; + if (row.best === machine_idx) { + value_class = "value-best"; + } else if (row.worst === machine_idx) { + value_class = "value-worst"; + } + var value_td = $(''); + if (last_value !== null) { + var value, err, err_str, sort_value; + var unit = $.asv.master_json.benchmarks[row.name].unit; + value = $.asv.pretty_unit(last_value, unit); + if (unit == "seconds") { + sort_value = last_value * 1e100; + } + else { + sort_value = last_value; + } + var baseline_percent = ''; + if (baseline_machine !== -1 && baseline_machine !== machine_idx) { + baseline_percent = ' (' + row.cmp_percent[machine_idx].toFixed(2) + '%)'; + } + var value_span = $('').text(value + baseline_percent); + + err = 100*last_err/last_value; + if (err == err) { + err_str = " \u00b1 " + err.toFixed(0.1) + '%'; + } + else { + err_str = ""; + } + value_span.attr('data-toggle', 'tooltip'); + value_span.attr('title', value + err_str); + value_td.append(value_span); + value_td.attr('data-sort-value', sort_value); + } + else { + value_td.attr('data-sort-value', -1e99); + } + tr.append(value_td); + }); + table_body.append(tr); + }); + + table_body.find('[data-toggle="tooltip"]').tooltip(); + + /* Finalize */ + table.append(table_body); + // setup_sort(table); + + return table; + } + + function setup_sort(table) { + var info = $.asv.parse_hash_string(window.location.hash); + + table.stupidtable(); + + table.on('aftertablesort', function (event, data) { + var info = $.asv.parse_hash_string(window.location.hash); + info.params['sort'] = [data.column]; + info.params['dir'] = [data.direction]; + window.location.hash = $.asv.format_hash_string(info); + + /* Update appearance */ + table.find('thead th').removeClass('asc'); + table.find('thead th').removeClass('desc'); + var th_to_sort = table.find("thead th").eq(parseInt(data.column)); + if (th_to_sort) { + th_to_sort.addClass(data.direction); + } + }); + + if (info.params.sort && info.params.dir) { + var th_to_sort = table.find("thead th").eq(parseInt(info.params.sort[0])); + th_to_sort.stupidsort(info.params.dir[0]); + } + else { + var th_to_sort = table.find("thead th").eq(0); + th_to_sort.stupidsort("asc"); + } + } + + /* + * Entry point + */ + $.asv.register_page('comparisonlist', function(params) { + var state_selection = null; + + if (Object.keys(params).length > 0) { + state_selection = params; + } + + setup_display(state_selection); + + $('#comparisonlist-display').show(); + $("#title").text("List of benchmarks"); + }); +}); diff --git a/asv/www/index.html b/asv/www/index.html index 5e2f65100..8703284ae 100644 --- a/asv/www/index.html +++ b/asv/www/index.html @@ -33,6 +33,9 @@ + @@ -42,6 +45,7 @@ + @@ -57,6 +61,7 @@
  • Benchmark grid
    • +