From 32f7def611cbdbdefcba074a2e48788584408866 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 14 Oct 2017 22:52:52 +0100 Subject: [PATCH 1/7] disable cache when building docker images --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 8e128c7..34473e3 100755 --- a/build.sh +++ b/build.sh @@ -1,3 +1,3 @@ #!/bin/bash -docker build -t magic/benchmark $(dirname $0) +docker build --no-cache -t magic/benchmark "$(dirname $0)" From 4e5290b84d6775f9d9b6d599d6c6d3d59bfef67f Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 14 Oct 2017 22:53:53 +0100 Subject: [PATCH 2/7] Update uvloop to avoid exception --- servers/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/servers/requirements.txt b/servers/requirements.txt index 539b598..c3fbd28 100644 --- a/servers/requirements.txt +++ b/servers/requirements.txt @@ -4,4 +4,4 @@ gevent==1.1.1 tornado==4.3 Twisted==16.1.1 httptools==0.0.9 -uvloop==0.4.28 +uvloop==0.8.1 From a5ababe7439df0d4555719719de5a6617ddb11f6 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sun, 15 Oct 2017 14:17:16 +0100 Subject: [PATCH 3/7] fudge to make sure container has been stopped --- run_benchmarks | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/run_benchmarks b/run_benchmarks index e38641b..e4c25d9 100755 --- a/run_benchmarks +++ b/run_benchmarks @@ -293,11 +293,15 @@ def server_container_exists(): def kill_server(): if server_is_running(): print('Shutting down server...') - subprocess.check_output(['docker', 'stop', 'magicbench']) - - if server_container_exists(): - print('Removing server container...') - subprocess.check_output(['docker', 'rm', 'magicbench']) + subprocess.check_output(['docker', 'stop', '-t10', 'magicbench']) + i = 0 + while server_container_exists(): + if i == 100: + subprocess.check_output(['docker', 'rm', '-f', 'magicbench']) + elif i == 110: + raise IOError("container still exists a minute after being sigkilled") + time.sleep(0.1) + i += 1 def format_report(data, target_file): From c4314cf657525f5090147a30318d2a3bd11708bb Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sun, 15 Oct 2017 14:17:52 +0100 Subject: [PATCH 4/7] ensure a type of benchmark is selected so html report doesn't blow up --- run_benchmarks | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/run_benchmarks b/run_benchmarks index e4c25d9..f38db6e 100755 --- a/run_benchmarks +++ b/run_benchmarks @@ -445,6 +445,9 @@ def main(): parser.add_argument('--benchmarks', type=str, help='comma-separated list of benchmarks to run ' + '(regular expressions are supported)') + parser.add_argument('type', type=str, + choices={b['name'].split('-', 1)[0] for b in benchmarks}, + help='type of benchmark to run') parser.add_argument('--concurrency-levels', type=int, default=[10], nargs='+', help='a list of concurrency levels to use') @@ -461,11 +464,14 @@ def main(): if not os.path.exists(_socket): os.mkdir(_socket) + benchmarks_to_run = [b for b in benchmarks + if b['name'].startswith(args.type)] if args.benchmarks: - benchmarks_to_run = [re.compile(b) for b in args.benchmarks.split(',')] - else: - benchmarks_to_run = [re.compile(re.escape(b['name'])) - for b in benchmarks] + patterns = [re.compile(p) for p in args.benchmarks.split(',')] + benchmarks_to_run = [ + b + for b in benchmarks_to_run + if any(p.match(b['name']) for p in patterns)] benchmarks_data = [] @@ -489,10 +495,7 @@ def main(): warmup = ['--msize=1024', '--duration=10', '--concurrency={}'.format(warmup_concurrency)] - for benchmark in benchmarks: - if not any(b.match(benchmark['name']) for b in benchmarks_to_run): - continue - + for benchmark in benchmarks_to_run: print(benchmark['title']) print('=' * len(benchmark['title'])) print() From 96b375a1bd20805eb754bf06000c32ec8cfc0aff Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sun, 15 Oct 2017 13:57:40 +0100 Subject: [PATCH 5/7] consistently use *iB --- run_benchmarks | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_benchmarks b/run_benchmarks index f38db6e..93673a8 100755 --- a/run_benchmarks +++ b/run_benchmarks @@ -480,7 +480,7 @@ def main(): for concurrency in sorted(args.concurrency_levels): for msgsize in sorted(args.payload_size_levels): variations.append({ - 'title': '{}kb messages, concurrency {}'.format( + 'title': '{}KiB messages, concurrency {}'.format( round(msgsize / 1024, 1), concurrency ), 'concurrency': concurrency, From 2ed20fc01ffcb0b067ec851674a08db825955c95 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sun, 15 Oct 2017 14:12:36 +0100 Subject: [PATCH 6/7] update and elaborate in README.rst --- README.rst | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 8b1d150..85bce70 100755 --- a/README.rst +++ b/README.rst @@ -12,9 +12,10 @@ Installation Install the following: -- Docker -- Python 3 -- Numpy +- `Docker` (tested with *17.09.0-ce*) +- Python 3.5+ (tested with *3.5.2*) +- `numpy` (tested with *1.13.3*) +- `wrk` (tested with *4.0.0*) Build the docker image containing the servers being tested by running ``./build.sh``. @@ -22,3 +23,9 @@ Build the docker image containing the servers being tested by running The benchmarks can then be ran with ``./run_benchmarks``. Use ``./run_benchmarks --help`` for various options, including selective benchmark running. + +To run the http benchmarks and save results to ``./results.html``: + +.. code:: + + ./run_benchmarks http --duration=60 --save-html=results.html From a94f928f2943f9752aad60425f33cbf9a21e34e9 Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Sat, 21 Oct 2017 12:57:08 +0100 Subject: [PATCH 7/7] move from using --benchmarks to positional args to filter benchmarks --- README.rst | 2 +- run_benchmarks | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 85bce70..26c1f82 100755 --- a/README.rst +++ b/README.rst @@ -28,4 +28,4 @@ To run the http benchmarks and save results to ``./results.html``: .. code:: - ./run_benchmarks http --duration=60 --save-html=results.html + ./run_benchmarks --duration=60 --save-html=results.html http diff --git a/run_benchmarks b/run_benchmarks index 93673a8..1ee1a5b 100755 --- a/run_benchmarks +++ b/run_benchmarks @@ -442,12 +442,11 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument('--duration', '-D', default=30, type=int, help='duration of each benchmark in seconds') - parser.add_argument('--benchmarks', type=str, - help='comma-separated list of benchmarks to run ' + - '(regular expressions are supported)') parser.add_argument('type', type=str, choices={b['name'].split('-', 1)[0] for b in benchmarks}, help='type of benchmark to run') + parser.add_argument('benchmark_patterns', type=str, nargs='*', + help='run benchmarks that match any of these regexes') parser.add_argument('--concurrency-levels', type=int, default=[10], nargs='+', help='a list of concurrency levels to use') @@ -464,14 +463,15 @@ def main(): if not os.path.exists(_socket): os.mkdir(_socket) - benchmarks_to_run = [b for b in benchmarks - if b['name'].startswith(args.type)] - if args.benchmarks: - patterns = [re.compile(p) for p in args.benchmarks.split(',')] - benchmarks_to_run = [ - b - for b in benchmarks_to_run - if any(p.match(b['name']) for p in patterns)] + if args.benchmark_patterns: + patterns = [re.compile(p) for p in args.benchmark_patterns] + else: + patterns = [re.compile('')] + benchmarks_to_run = [] + for benchmark in benchmarks: + type_, name = benchmark['name'].split('-', 1) + if type_ == args.type and any(p.match(name) for p in patterns): + benchmarks_to_run.append(benchmark) benchmarks_data = []