Skip to content

Commit d7fec0a

Browse files
dionhaefnerdionhaefner
committed
more useful benchmarks
1 parent 10f902a commit d7fec0a

5 files changed

Lines changed: 137 additions & 47 deletions

File tree

‎benchmarks/bench_array_encoding.py‎

Lines changed: 29 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,12 @@
1515
from functools import partial
1616

1717
from pydantic import BaseModel
18-
from utils import BenchmarkSuite, create_test_array, run_benchmark
18+
from utils import (
19+
DEFAULT_MIN_DURATION_S,
20+
BenchmarkSuite,
21+
create_test_array,
22+
run_benchmark,
23+
)
1924

2025
from tesseract_core.runtime.schema_types import Array, Float64
2126

@@ -54,17 +59,18 @@ def _roundtrip(model: BaseModel, encoding: str, **context: str) -> ArrayModel:
5459

5560

5661
def benchmark_encoding(
57-
iterations: int = 50,
62+
min_iterations: int | None = None,
5863
array_sizes: list[int] | None = None,
5964
profile: bool = False,
65+
min_duration_s: float = DEFAULT_MIN_DURATION_S,
6066
) -> BenchmarkSuite:
6167
"""Run encoding benchmarks for all methods and sizes."""
6268
if array_sizes is None:
6369
array_sizes = DEFAULT_ARRAY_SIZES
6470

6571
suite = BenchmarkSuite(
6672
name="array_encoding",
67-
metadata={"iterations": iterations, "array_sizes": array_sizes},
73+
metadata={"min_iterations": min_iterations, "array_sizes": array_sizes},
6874
)
6975

7076
for i, size in enumerate(array_sizes):
@@ -89,26 +95,28 @@ def benchmark_encoding(
8995
result = run_benchmark(
9096
name=f"encode_{encoding}_{size:,}",
9197
func=partial(_encode, model, encoding, **ctx),
92-
iterations=iterations,
98+
min_iterations=min_iterations,
9399
profile=profile,
100+
min_duration_s=min_duration_s,
94101
)
95102
suite.add_result(result)
96103

97104
return suite
98105

99106

100107
def benchmark_decoding(
101-
iterations: int = 50,
108+
min_iterations: int | None = None,
102109
array_sizes: list[int] | None = None,
103110
profile: bool = False,
111+
min_duration_s: float = DEFAULT_MIN_DURATION_S,
104112
) -> BenchmarkSuite:
105113
"""Run decoding benchmarks for all methods and sizes."""
106114
if array_sizes is None:
107115
array_sizes = DEFAULT_ARRAY_SIZES
108116

109117
suite = BenchmarkSuite(
110118
name="array_decoding",
111-
metadata={"iterations": iterations, "array_sizes": array_sizes},
119+
metadata={"min_iterations": min_iterations, "array_sizes": array_sizes},
112120
)
113121

114122
for i, size in enumerate(array_sizes):
@@ -131,26 +139,28 @@ def benchmark_decoding(
131139
result = run_benchmark(
132140
name=f"decode_{encoding}_{size:,}",
133141
func=partial(_decode, encoded, **ctx),
134-
iterations=iterations,
142+
min_iterations=min_iterations,
135143
profile=profile,
144+
min_duration_s=min_duration_s,
136145
)
137146
suite.add_result(result)
138147

139148
return suite
140149

141150

142151
def benchmark_roundtrip(
143-
iterations: int = 50,
152+
min_iterations: int | None = None,
144153
array_sizes: list[int] | None = None,
145154
profile: bool = False,
155+
min_duration_s: float = DEFAULT_MIN_DURATION_S,
146156
) -> BenchmarkSuite:
147157
"""Run roundtrip (encode + decode) benchmarks."""
148158
if array_sizes is None:
149159
array_sizes = DEFAULT_ARRAY_SIZES
150160

151161
suite = BenchmarkSuite(
152162
name="array_roundtrip",
153-
metadata={"iterations": iterations, "array_sizes": array_sizes},
163+
metadata={"min_iterations": min_iterations, "array_sizes": array_sizes},
154164
)
155165

156166
for i, size in enumerate(array_sizes):
@@ -172,25 +182,30 @@ def benchmark_roundtrip(
172182
result = run_benchmark(
173183
name=f"roundtrip_{encoding}_{size:,}",
174184
func=partial(_roundtrip, model, encoding, **ctx),
175-
iterations=iterations,
185+
min_iterations=min_iterations,
176186
profile=profile,
187+
min_duration_s=min_duration_s,
177188
)
178189
suite.add_result(result)
179190

180191
return suite
181192

182193

183194
def run_all(
184-
iterations: int = 50,
195+
min_iterations: int | None = None,
185196
array_sizes: list[int] | None = None,
186197
profile: bool = False,
198+
min_duration_s: float = DEFAULT_MIN_DURATION_S,
187199
) -> list[BenchmarkSuite]:
188200
"""Run all array encoding benchmarks."""
189201
results = []
190202
for benchmark_func in [benchmark_encoding, benchmark_decoding, benchmark_roundtrip]:
191203
print(f"Running {benchmark_func.__name__}...")
192204
suite = benchmark_func(
193-
iterations=iterations, array_sizes=array_sizes, profile=profile
205+
min_iterations=min_iterations,
206+
array_sizes=array_sizes,
207+
profile=profile,
208+
min_duration_s=min_duration_s,
194209
)
195210
results.append(suite)
196211
return results
@@ -199,10 +214,10 @@ def run_all(
199214
if __name__ == "__main__":
200215
import sys
201216

202-
iterations = int(sys.argv[1]) if len(sys.argv) > 1 else 50
217+
min_iterations: int | None = int(sys.argv[1]) if len(sys.argv) > 1 else None
203218

204219
print("Running array encoding benchmarks...")
205-
suites = run_all(iterations)
220+
suites = run_all(min_iterations=min_iterations)
206221

207222
for suite in suites:
208223
print(f"\n=== {suite.name} ===")

‎benchmarks/bench_tesseract.py‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,12 @@
2424
import tempfile
2525
from pathlib import Path
2626

27-
from utils import BenchmarkSuite, create_test_array, run_benchmark
27+
from utils import (
28+
DEFAULT_MIN_DURATION_S,
29+
BenchmarkSuite,
30+
create_test_array,
31+
run_benchmark,
32+
)
2833

2934
# Path to the no-op tesseract for benchmarking
3035
NOOP_TESSERACT_PATH = Path(__file__).parent / "tesseract_noop" / "tesseract_api.py"
@@ -74,9 +79,10 @@ def _build_noop_tesseract() -> str | None:
7479

7580

7681
def benchmark_from_tesseract_api(
77-
iterations: int = 50,
82+
min_iterations: int | None = None,
7883
array_sizes: list[int] | None = None,
7984
profile: bool = False,
85+
min_duration_s: float = DEFAULT_MIN_DURATION_S,
8086
) -> BenchmarkSuite:
8187
"""Benchmark non-containerized Tesseract via from_tesseract_api().
8288
@@ -90,7 +96,7 @@ def benchmark_from_tesseract_api(
9096

9197
suite = BenchmarkSuite(
9298
name="from_tesseract_api",
93-
metadata={"iterations": iterations, "array_sizes": array_sizes},
99+
metadata={"min_iterations": min_iterations, "array_sizes": array_sizes},
94100
)
95101

96102
with tempfile.TemporaryDirectory() as tmpdir:
@@ -114,18 +120,20 @@ def call_apply(t=tesseract, inp=inputs):
114120
result = run_benchmark(
115121
name=f"apply_{size:,}",
116122
func=call_apply,
117-
iterations=iterations,
123+
min_iterations=min_iterations,
118124
profile=profile,
125+
min_duration_s=min_duration_s,
119126
)
120127
suite.add_result(result)
121128

122129
return suite
123130

124131

125132
def benchmark_containerized_http(
126-
iterations: int = 20,
133+
min_iterations: int | None = None,
127134
array_sizes: list[int] | None = None,
128135
profile: bool = False,
136+
min_duration_s: float = DEFAULT_MIN_DURATION_S,
129137
image_name: str | None = None,
130138
) -> BenchmarkSuite | None:
131139
"""Benchmark containerized Tesseract via HTTP (Tesseract.from_image).
@@ -151,7 +159,7 @@ def benchmark_containerized_http(
151159

152160
suite = BenchmarkSuite(
153161
name="containerized_http",
154-
metadata={"iterations": iterations, "array_sizes": array_sizes},
162+
metadata={"min_iterations": min_iterations, "array_sizes": array_sizes},
155163
)
156164

157165
# Build the benchmark tesseract image if not provided
@@ -191,9 +199,10 @@ def call_apply(t=tesseract, inp=inputs):
191199
result = run_benchmark(
192200
name=f"apply_{size:,}",
193201
func=call_apply,
194-
iterations=iterations,
202+
min_iterations=min_iterations,
195203
warmup=2,
196204
profile=profile,
205+
min_duration_s=min_duration_s,
197206
)
198207
suite.add_result(result)
199208

@@ -205,9 +214,10 @@ def call_apply(t=tesseract, inp=inputs):
205214

206215

207216
def benchmark_containerized_cli(
208-
iterations: int = 10,
217+
min_iterations: int | None = None,
209218
array_sizes: list[int] | None = None,
210219
profile: bool = False,
220+
min_duration_s: float = DEFAULT_MIN_DURATION_S,
211221
image_name: str | None = None,
212222
) -> BenchmarkSuite | None:
213223
"""Benchmark containerized Tesseract via CLI (`tesseract run`).
@@ -234,7 +244,7 @@ def benchmark_containerized_cli(
234244

235245
suite = BenchmarkSuite(
236246
name="containerized_cli",
237-
metadata={"iterations": iterations, "array_sizes": array_sizes},
247+
metadata={"min_iterations": min_iterations, "array_sizes": array_sizes},
238248
)
239249

240250
if image_name is None:
@@ -314,34 +324,41 @@ def run_cli(
314324
result = run_benchmark(
315325
name=f"apply_{size:,}",
316326
func=run_cli,
317-
iterations=iterations,
327+
min_iterations=min_iterations,
318328
warmup=1,
319329
profile=profile,
330+
min_duration_s=min_duration_s,
320331
)
321332
suite.add_result(result)
322333

323334
return suite
324335

325336

326337
def run_all(
327-
iterations: int = 50,
338+
min_iterations: int | None = None,
328339
include_containerized: bool = False,
329340
array_sizes: list[int] | None = None,
341+
min_duration_s: float = DEFAULT_MIN_DURATION_S,
330342
) -> list[BenchmarkSuite]:
331343
"""Run all Tesseract benchmarks.
332344
333345
Args:
334-
iterations: Number of iterations per benchmark
346+
min_iterations: Minimum iterations per benchmark (None = auto-calibrate only)
335347
include_containerized: Whether to include Docker-based benchmarks
336348
array_sizes: Array sizes to benchmark (defaults to DEFAULT_ARRAY_SIZES)
349+
min_duration_s: Minimum duration per case when auto-calibrating
337350
338351
Returns:
339352
List of BenchmarkSuites
340353
"""
341354
results = []
342355

343356
print("Running benchmark_from_tesseract_api...")
344-
suite = benchmark_from_tesseract_api(iterations=iterations, array_sizes=array_sizes)
357+
suite = benchmark_from_tesseract_api(
358+
min_iterations=min_iterations,
359+
array_sizes=array_sizes,
360+
min_duration_s=min_duration_s,
361+
)
345362
if suite is not None:
346363
results.append(suite)
347364

@@ -355,9 +372,10 @@ def run_all(
355372
]:
356373
print(f"Running {benchmark_func.__name__}...")
357374
suite = benchmark_func(
358-
iterations=iterations,
375+
min_iterations=min_iterations,
359376
array_sizes=array_sizes,
360377
image_name=image_name,
378+
min_duration_s=min_duration_s,
361379
)
362380
if suite is not None:
363381
results.append(suite)
@@ -366,13 +384,15 @@ def run_all(
366384

367385

368386
if __name__ == "__main__":
369-
iterations = int(sys.argv[1]) if len(sys.argv) > 1 else 50
387+
min_iterations: int | None = int(sys.argv[1]) if len(sys.argv) > 1 else None
370388
exclude_docker = "--no-docker" in sys.argv
371389

372390
print(
373-
f"Running Tesseract benchmarks (iterations={iterations}, docker={not exclude_docker})..."
391+
f"Running Tesseract benchmarks (min_iterations={min_iterations}, docker={not exclude_docker})..."
392+
)
393+
suites = run_all(
394+
min_iterations=min_iterations, include_containerized=not exclude_docker
374395
)
375-
suites = run_all(iterations, include_containerized=not exclude_docker)
376396

377397
for suite in suites:
378398
print(f"\n=== {suite.name} ===")

‎benchmarks/compare_benchmarks.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
from pathlib import Path
1515

1616
# Percentage change threshold for flagging a benchmark as notable.
17-
NOTABLE_THRESHOLD_PCT = 5
17+
NOTABLE_THRESHOLD_PCT = 10
1818

1919

2020
def _get_median_ms(result: dict) -> float:

0 commit comments

Comments
 (0)