|
4 | 4 | This measures how fast skim can ingest items and display matched results. |
5 | 5 |
|
6 | 6 | Usage: bench.py [BINARY_PATH ...] [-n|--num-items NUM] [-q|--query QUERY] |
7 | | - [-r|--runs RUNS] [-f|--file FILE] [-g|--generate-file FILE] |
8 | | - [-j|--json] [-- EXTRA_ARGS...] |
| 7 | + [-r|--runs RUNS] [-w|--warmup N] [-f|--file FILE] |
| 8 | + [-g|--generate-file FILE] [-j|--json] [-- EXTRA_ARGS...] |
9 | 9 |
|
10 | 10 | Arguments: |
11 | 11 | BINARY_PATH ... One or more paths to binaries (default: ./target/release/sk) |
|
14 | 14 | -n, --num-items NUM Number of items to generate (default: 1000000) |
15 | 15 | -q, --query QUERY Query string to search (default: "test") |
16 | 16 | -r, --runs RUNS Number of benchmark runs per binary (default: 1) |
| 17 | + -w, --warmup N Number of warmup runs per binary (default: 1) |
17 | 18 | -f, --file FILE Use existing file as input instead of generating |
18 | 19 | -g, --generate-file FILE Generate test data to file and exit |
19 | 20 | -j, --json Output results as JSON |
|
44 | 45 | DEFAULT_NUM_ITEMS = 1_000_000 |
45 | 46 | DEFAULT_QUERY = "test" |
46 | 47 | DEFAULT_RUNS = 1 |
| 48 | +DEFAULT_WARMUP = 1 |
47 | 49 |
|
48 | 50 | # Stability / timeout tuning (mirrors bench.sh values) |
49 | 51 | REQUIRED_STABLE_S = 5.0 # seconds the matched count must be unchanged |
@@ -129,6 +131,7 @@ def parse_args(argv): |
129 | 131 | parser.add_argument("-n", "--num-items", type=int, default=DEFAULT_NUM_ITEMS) |
130 | 132 | parser.add_argument("-q", "--query", default=DEFAULT_QUERY) |
131 | 133 | parser.add_argument("-r", "--runs", type=int, default=DEFAULT_RUNS) |
| 134 | + parser.add_argument("-w", "--warmup", type=int, default=DEFAULT_WARMUP) |
132 | 135 | parser.add_argument("-f", "--file", default="") |
133 | 136 | parser.add_argument("-g", "--generate-file", default="") |
134 | 137 | parser.add_argument("-j", "--json", action="store_true") |
@@ -423,12 +426,13 @@ def _max(values): |
423 | 426 |
|
424 | 427 |
|
425 | 428 | def aggregate(results: list) -> dict: |
426 | | - times = [r["elapsed_s"] for r in results] |
427 | | - rates = [r["rate"] for r in results] |
428 | | - matched = [r["matched"] for r in results] |
429 | | - mems = [r["peak_mem_kb"] for r in results] |
430 | | - cpus = [r["peak_cpu"] for r in results] |
431 | | - completed = sum(1 for r in results if r["completed"]) |
| 429 | + completed_results = [r for r in results if r["completed"]] |
| 430 | + times = [r["elapsed_s"] for r in completed_results] |
| 431 | + rates = [r["rate"] for r in completed_results] |
| 432 | + matched = [r["matched"] for r in completed_results] |
| 433 | + mems = [r["peak_mem_kb"] for r in completed_results] |
| 434 | + cpus = [r["peak_cpu"] for r in completed_results] |
| 435 | + completed = len(completed_results) |
432 | 436 |
|
433 | 437 | return { |
434 | 438 | "completed": completed, |
@@ -611,6 +615,7 @@ def main(): |
611 | 615 | num_items = opts.num_items |
612 | 616 | query = opts.query |
613 | 617 | runs = opts.runs |
| 618 | + warmup = opts.warmup |
614 | 619 | input_file = opts.file |
615 | 620 | generate_file = opts.generate_file |
616 | 621 | as_json = opts.json |
@@ -645,14 +650,33 @@ def main(): |
645 | 650 | print(f"=== Skim Ingestion + Matching Benchmark ===", file=sys.stderr) |
646 | 651 | print( |
647 | 652 | f"Binaries: {binary_list} | Items: {num_items} | " |
648 | | - f"Query: '{query}' | Runs: {runs} (per binary)", |
| 653 | + f"Query: '{query}' | Warmup: {warmup} | Runs: {runs} (per binary)", |
649 | 654 | file=sys.stderr, |
650 | 655 | ) |
651 | 656 | if input_file: |
652 | 657 | print(f"Input file: {input_file}", file=sys.stderr) |
653 | 658 | if extra_args: |
654 | 659 | print(f"Extra args: {' '.join(extra_args)}", file=sys.stderr) |
655 | 660 |
|
| 661 | + # ---- warmup (results discarded) ------------------------------------- |
| 662 | + if warmup > 0: |
| 663 | + print(f"\n=== Warmup ({warmup} run(s) per binary) ===", file=sys.stderr) |
| 664 | + for bi, binary in enumerate(binaries): |
| 665 | + for wu in range(1, warmup + 1): |
| 666 | + print( |
| 667 | + f" Warmup {wu}/{warmup} — {binary} ...", |
| 668 | + file=sys.stderr, |
| 669 | + ) |
| 670 | + run_once( |
| 671 | + binary_path=binary, |
| 672 | + query=query, |
| 673 | + tmp_file=tmp_file, |
| 674 | + num_items=num_items, |
| 675 | + extra_args=extra_args, |
| 676 | + run_index=wu, |
| 677 | + session_suffix=f"warmup_b{bi}", |
| 678 | + ) |
| 679 | + |
656 | 680 | # ---- run benchmark in round-robin ----------------------------------- |
657 | 681 | # all_results[i] = list of per-run dicts for binaries[i] |
658 | 682 | all_results = [[] for _ in binaries] |
|
0 commit comments