Skip to content

Commit 830a8d6

Browse files
authored
Merge branch 'main' into cov_workspaces
2 parents 63bd880 + 55054d6 commit 830a8d6

11 files changed

Lines changed: 564 additions & 57 deletions

File tree

.github/workflows/benchmark_harness.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,11 @@ jobs:
6161
- uses: dart-lang/setup-dart@e51d8e571e22473a2ddebf0ef8a2123f0ab2c02c
6262
with:
6363
sdk: ${{ matrix.sdk }}
64+
# Node 22 has wasmGC enabled, which allows the wasm tests to run!
65+
- name: Setup Node.js 22
66+
uses: actions/setup-node@v3
67+
with:
68+
node-version: 22
6469
- id: install
6570
name: Install dependencies
6671
run: dart pub get

pkgs/benchmark_harness/CHANGELOG.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
## 2.4.0-wip
2+
3+
- Added a `bench` command.
4+
15
## 2.3.1
26

37
- Move to `dart-lang/tools` monorepo.

pkgs/benchmark_harness/README.md

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,3 +114,56 @@ Template(RunTime): 0.1568472448997197 us.
114114
This is the average amount of time it takes to run `run()` 10 times for
115115
`BenchmarkBase` and once for `AsyncBenchmarkBase`.
116116
> µs is an abbreviation for microseconds.
117+
118+
## `bench` command
119+
120+
A convenience command available in `package:benchmark_harness`.
121+
122+
If a package depends on `benchmark_harness`, invoke the command by running
123+
124+
```shell
125+
dart run benchmark_harness:bench
126+
```
127+
128+
If not, you can use this command by activating it.
129+
130+
```shell
131+
dart pub global activate benchmark_harness
132+
dart pub global run benchmark_harness:bench
133+
```
134+
135+
Output from `dart run benchmark_harness:bench --help`
136+
137+
```
138+
Runs a dart script in a number of runtimes.
139+
140+
Meant to make it easy to run a benchmark executable across runtimes to validate
141+
performance impacts.
142+
143+
-f, --flavor
144+
[aot] Compile and run as a native binary.
145+
[jit] Run as-is without compilation, using the just-in-time (JIT) runtime.
146+
[js] Compile to JavaScript and run on node.
147+
[wasm] Compile to WebAssembly and run on node.
148+
149+
--target The target script to compile and run.
150+
(defaults to "benchmark/benchmark.dart")
151+
-h, --help Print usage information and quit.
152+
-v, --verbose Print the full stack trace if an exception is thrown.
153+
```
154+
155+
Example usage:
156+
157+
```shell
158+
dart run benchmark_harness:bench --flavor aot --target example/template.dart
159+
160+
AOT - COMPILE
161+
/dart_installation/dart-sdk/bin/dart compile exe example/template.dart -o /temp_dir/bench_1747680526905_GtfAeM/out.exe
162+
163+
Generated: /temp_dir/bench_1747680526905_GtfAeM/out.exe
164+
165+
AOT - RUN
166+
/temp_dir/bench_1747680526905_GtfAeM/out.exe
167+
168+
Template(RunTime): 0.005620051244379949 us.
169+
```
Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:io';
6+
7+
import 'package:benchmark_harness/src/bench_command/bench_options.dart';
8+
import 'package:benchmark_harness/src/bench_command/compile_and_run.dart';
9+
10+
Future<void> main(List<String> args) async {
11+
BenchOptions? options;
12+
13+
try {
14+
options = BenchOptions.fromArgs(args);
15+
if (options.help) {
16+
print('''
17+
\nRuns a dart script in a number of runtimes.
18+
19+
Meant to make it easy to run a benchmark executable across runtimes to validate
20+
performance impacts.
21+
''');
22+
print(BenchOptions.usage);
23+
return;
24+
}
25+
26+
await compileAndRun(options);
27+
} on FormatException catch (e) {
28+
print(e.message);
29+
print(BenchOptions.usage);
30+
exitCode = 64; // command line usage error
31+
} on BenchException catch (e, stack) {
32+
print(e.message);
33+
if (options?.verbose ?? true) {
34+
print(e);
35+
print(stack);
36+
}
37+
exitCode = e.exitCode;
38+
}
39+
}
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'package:args/args.dart';
6+
7+
enum RuntimeFlavor {
8+
aot(help: 'Compile and run as a native binary.'),
9+
jit(
10+
help: 'Run as-is without compilation, '
11+
'using the just-in-time (JIT) runtime.',
12+
),
13+
js(help: 'Compile to JavaScript and run on node.'),
14+
wasm(help: 'Compile to WebAssembly and run on node.');
15+
16+
const RuntimeFlavor({required this.help});
17+
18+
final String help;
19+
}
20+
21+
class BenchOptions {
22+
BenchOptions({
23+
required this.flavor,
24+
required this.target,
25+
this.help = false,
26+
this.verbose = false,
27+
}) {
28+
if (!help && flavor.isEmpty) {
29+
// This is the wrong exception to use, except that it's caught in the
30+
// program, so it makes implementation easy.
31+
throw const FormatException('At least one `flavor` must be provided', 64);
32+
}
33+
}
34+
35+
factory BenchOptions.fromArgs(List<String> args) {
36+
final result = _parserForBenchOptions.parse(args);
37+
38+
if (result.rest.isNotEmpty) {
39+
throw FormatException('All arguments must be provided via `--` options. '
40+
'Not sure what to do with "${result.rest.join()}".');
41+
}
42+
43+
return BenchOptions(
44+
flavor:
45+
result.multiOption('flavor').map(RuntimeFlavor.values.byName).toSet(),
46+
target: result.option('target')!,
47+
help: result.flag('help'),
48+
verbose: result.flag('verbose'),
49+
);
50+
}
51+
52+
final String target;
53+
54+
final Set<RuntimeFlavor> flavor;
55+
56+
final bool help;
57+
58+
final bool verbose;
59+
60+
static String get usage => _parserForBenchOptions.usage;
61+
62+
static final _parserForBenchOptions = ArgParser()
63+
..addMultiOption('flavor',
64+
abbr: 'f',
65+
allowed: RuntimeFlavor.values.map((e) => e.name),
66+
allowedHelp: {
67+
for (final flavor in RuntimeFlavor.values) flavor.name: flavor.help
68+
})
69+
..addOption('target',
70+
defaultsTo: 'benchmark/benchmark.dart',
71+
help: 'The target script to compile and run.')
72+
..addFlag('help',
73+
defaultsTo: false,
74+
negatable: false,
75+
help: 'Print usage information and quit.',
76+
abbr: 'h')
77+
..addFlag('verbose',
78+
defaultsTo: false,
79+
negatable: false,
80+
help: 'Print the full stack trace if an exception is thrown.',
81+
abbr: 'v');
82+
}
Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
2+
// for details. All rights reserved. Use of this source code is governed by a
3+
// BSD-style license that can be found in the LICENSE file.
4+
5+
import 'dart:io';
6+
7+
import 'bench_options.dart';
8+
9+
// TODO(kevmoo): allow the user to specify custom flags – for compile and/or run
10+
11+
Future<void> compileAndRun(BenchOptions options) async {
12+
if (!FileSystemEntity.isFileSync(options.target)) {
13+
throw BenchException(
14+
'The target Dart program `${options.target}` does not exist',
15+
2, // standard bash code for file doesn't exist
16+
);
17+
}
18+
19+
for (var mode in options.flavor) {
20+
await _Runner(flavor: mode, target: options.target).run();
21+
}
22+
}
23+
24+
class BenchException implements Exception {
25+
const BenchException(this.message, this.exitCode) : assert(exitCode > 0);
26+
final String message;
27+
final int exitCode;
28+
29+
@override
30+
String toString() => 'BenchException: $message ($exitCode)';
31+
}
32+
33+
/// Base name for output files.
34+
const _outputFileRoot = 'out';
35+
36+
/// Denote the "stage" of the compile/run step for logging.
37+
enum _Stage { compile, run }
38+
39+
/// Base class for runtime-specific runners.
40+
abstract class _Runner {
41+
_Runner._({required this.target, required this.flavor})
42+
: assert(FileSystemEntity.isFileSync(target), '$target is not a file');
43+
44+
factory _Runner({required RuntimeFlavor flavor, required String target}) {
45+
return (switch (flavor) {
46+
RuntimeFlavor.jit => _JITRunner.new,
47+
RuntimeFlavor.aot => _AOTRunner.new,
48+
RuntimeFlavor.js => _JSRunner.new,
49+
RuntimeFlavor.wasm => _WasmRunner.new,
50+
})(target: target);
51+
}
52+
53+
final String target;
54+
final RuntimeFlavor flavor;
55+
late Directory _tempDirectory;
56+
57+
/// Executes the compile and run cycle.
58+
///
59+
/// Takes care of creating and deleting the corresponding temp directory.
60+
Future<void> run() async {
61+
_tempDirectory = Directory.systemTemp
62+
.createTempSync('bench_${DateTime.now().millisecondsSinceEpoch}_');
63+
try {
64+
await _runImpl();
65+
} finally {
66+
_tempDirectory.deleteSync(recursive: true);
67+
}
68+
}
69+
70+
/// Overridden in implementations to handle the compile and run cycle.
71+
Future<void> _runImpl();
72+
73+
/// Executes the specific [executable] with the provided [args].
74+
///
75+
/// Also prints out a nice message before execution denoting the [flavor] and
76+
/// the [stage].
77+
Future<void> _runProc(
78+
_Stage stage, String executable, List<String> args) async {
79+
print('''
80+
\n${flavor.name.toUpperCase()} - ${stage.name.toUpperCase()}
81+
$executable ${args.join(' ')}
82+
''');
83+
84+
final proc = await Process.start(executable, args,
85+
mode: ProcessStartMode.inheritStdio);
86+
87+
final exitCode = await proc.exitCode;
88+
89+
if (exitCode != 0) {
90+
throw ProcessException(executable, args, 'Process errored', exitCode);
91+
}
92+
}
93+
94+
String _outputFile(String ext) =>
95+
_tempDirectory.uri.resolve('$_outputFileRoot.$ext').toFilePath();
96+
}
97+
98+
class _JITRunner extends _Runner {
99+
_JITRunner({required super.target}) : super._(flavor: RuntimeFlavor.jit);
100+
101+
@override
102+
Future<void> _runImpl() async {
103+
await _runProc(_Stage.run, Platform.executable, [target]);
104+
}
105+
}
106+
107+
class _AOTRunner extends _Runner {
108+
_AOTRunner({required super.target}) : super._(flavor: RuntimeFlavor.aot);
109+
110+
@override
111+
Future<void> _runImpl() async {
112+
final outFile = _outputFile('exe');
113+
await _runProc(_Stage.compile, Platform.executable, [
114+
'compile',
115+
'exe',
116+
target,
117+
'-o',
118+
outFile,
119+
]);
120+
121+
await _runProc(_Stage.run, outFile, []);
122+
}
123+
}
124+
125+
class _JSRunner extends _Runner {
126+
_JSRunner({required super.target}) : super._(flavor: RuntimeFlavor.js);
127+
128+
@override
129+
Future<void> _runImpl() async {
130+
final outFile = _outputFile('js');
131+
await _runProc(_Stage.compile, Platform.executable, [
132+
'compile',
133+
'js',
134+
target,
135+
'-O4', // default for Flutter
136+
'-o',
137+
outFile,
138+
]);
139+
140+
await _runProc(_Stage.run, 'node', [outFile]);
141+
}
142+
}
143+
144+
class _WasmRunner extends _Runner {
145+
_WasmRunner({required super.target}) : super._(flavor: RuntimeFlavor.wasm);
146+
147+
@override
148+
Future<void> _runImpl() async {
149+
final outFile = _outputFile('wasm');
150+
await _runProc(_Stage.compile, Platform.executable, [
151+
'compile',
152+
'wasm',
153+
target,
154+
'-O2', // default for Flutter
155+
'-o',
156+
outFile,
157+
]);
158+
159+
final jsFile =
160+
File.fromUri(_tempDirectory.uri.resolve('$_outputFileRoot.js'));
161+
jsFile.writeAsStringSync(_wasmInvokeScript);
162+
163+
await _runProc(_Stage.run, 'node', [jsFile.path]);
164+
}
165+
166+
static const _wasmInvokeScript = '''
167+
import { readFile } from 'node:fs/promises'; // For async file reading
168+
import { fileURLToPath } from 'url';
169+
import { dirname, join } from 'path';
170+
171+
// Get the current directory name
172+
const __filename = fileURLToPath(import.meta.url);
173+
const __dirname = dirname(__filename);
174+
175+
const wasmFilePath = join(__dirname, '$_outputFileRoot.wasm');
176+
const wasmBytes = await readFile(wasmFilePath);
177+
178+
const mjsFilePath = join(__dirname, '$_outputFileRoot.mjs');
179+
const dartModule = await import(mjsFilePath);
180+
const {compile} = dartModule;
181+
182+
const compiledApp = await compile(wasmBytes);
183+
const instantiatedApp = await compiledApp.instantiate({});
184+
await instantiatedApp.invokeMain();
185+
''';
186+
}

0 commit comments

Comments
 (0)