Skip to content

Commit 5148d42

Browse files
DonachDonach
committed
feat: add isolated Wurst benchmark mode
1 parent f383a55 commit 5148d42

18 files changed

Lines changed: 1645 additions & 3 deletions

File tree

‎de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/CompilationProcess.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package de.peeeq.wurstio;
22

33
import org.wurstscript.projectconfig.WurstProjectConfigData;
4+
import de.peeeq.wurstio.benchmark.BenchmarkOptions;
5+
import de.peeeq.wurstio.benchmark.BenchmarkResult;
6+
import de.peeeq.wurstio.benchmark.BenchmarkWorkerOutput;
7+
import de.peeeq.wurstio.benchmark.RunBenchmarks;
48
import de.peeeq.wurstio.languageserver.requests.RunTests;
59
import de.peeeq.wurstio.mpq.MpqEditor;
610
import de.peeeq.wurstio.utils.FileUtils;
@@ -19,6 +23,8 @@
1923
import java.io.File;
2024
import java.io.IOException;
2125
import java.io.PrintStream;
26+
import java.nio.file.Path;
27+
import java.nio.file.Paths;
2228
import java.util.Optional;
2329
import java.util.function.Supplier;
2430

@@ -80,6 +86,11 @@ public CompilationProcess(WurstGui gui, RunArgs runArgs) {
8086
return null;
8187
}
8288

89+
if (runArgs.isRunBenchmarks()) {
90+
timeTaker.measure("Run benchmark worker", () -> runBenchmarks(compiler));
91+
return null;
92+
}
93+
8394
if (runArgs.isRunTests()) {
8495
timeTaker.measure("Run tests",
8596
() -> runTests(compiler.getImTranslator(), compiler, runArgs.getTestTimeout(), runArgs.getTestFilter()));
@@ -116,6 +127,32 @@ public CompilationProcess(WurstGui gui, RunArgs runArgs) {
116127
return mapScript;
117128
}
118129

130+
private void runBenchmarks(WurstCompilerJassImpl compiler) {
131+
try {
132+
RunBenchmarks runner = new RunBenchmarks();
133+
Path output = Paths.get(runArgs.getBenchmarkOutput());
134+
if (runArgs.isBenchmarkList()) {
135+
BenchmarkWorkerOutput.writeDiscovery(
136+
output,
137+
runner.discover(compiler.getImProg(), Optional.ofNullable(runArgs.getBenchmarkFilter())));
138+
} else {
139+
BenchmarkResult result = runner.run(
140+
compiler.getImTranslator(),
141+
compiler.getImProg(),
142+
runArgs.getBenchmarkName(),
143+
new BenchmarkOptions(runArgs.getBenchmarkWarmup(), runArgs.getBenchmarkIterations(), 1_000_000L));
144+
BenchmarkWorkerOutput.writeResult(output, result);
145+
}
146+
} catch (Throwable e) {
147+
String message = e.getMessage() == null ? e.getClass().getSimpleName() : e.getMessage();
148+
gui.sendError(new CompileError(
149+
null,
150+
"Benchmark worker failed: " + message,
151+
CompileError.ErrorType.ERROR,
152+
e));
153+
}
154+
}
155+
119156
private boolean runPjass(File outputMapscript) {
120157
File commonJ = new File(outputMapscript.getParent(), "common.j");
121158
File blizzJ = new File(outputMapscript.getParent(), "blizzard.j");

‎de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/Main.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -191,7 +191,10 @@ public static void main(String[] args) {
191191
compiledScript = compilationProcess.doCompilation(null, true);
192192
}
193193

194-
if (compiledScript != null) {
194+
if (runArgs.isRunBenchmarks()) {
195+
// Benchmark workers write their JSON result during compilation;
196+
// a null script is the successful worker result, not a failure.
197+
} else if (compiledScript != null) {
195198
File scriptFile = new File("compiled.j.txt");
196199
Files.write(compiledScript.toString().getBytes(Charsets.UTF_8), scriptFile);
197200
}

‎de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkClock.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package de.peeeq.wurstio.benchmark;
2+
3+
@FunctionalInterface
4+
public interface BenchmarkClock {
5+
long nanoTime();
6+
}

‎de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkOptions.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package de.peeeq.wurstio.benchmark;
2+
3+
public record BenchmarkOptions(int warmupIterations, int measurementIterations, long minimumSampleNanos) {
4+
public BenchmarkOptions {
5+
if (warmupIterations < 0) {
6+
throw new IllegalArgumentException("warmupIterations must be non-negative");
7+
}
8+
if (measurementIterations <= 0) {
9+
throw new IllegalArgumentException("measurementIterations must be positive");
10+
}
11+
if (minimumSampleNanos < 0) {
12+
throw new IllegalArgumentException("minimumSampleNanos must be non-negative");
13+
}
14+
}
15+
}

‎de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkResult.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package de.peeeq.wurstio.benchmark;
2+
3+
import java.util.List;
4+
import java.util.Objects;
5+
6+
public record BenchmarkResult(
7+
String qualifiedName,
8+
int checksum,
9+
int batchSize,
10+
List<Long> samplesNanos,
11+
BenchmarkStatistics statistics
12+
) {
13+
public BenchmarkResult {
14+
Objects.requireNonNull(qualifiedName, "qualifiedName");
15+
if (batchSize <= 0) {
16+
throw new IllegalArgumentException("batchSize must be positive");
17+
}
18+
samplesNanos = List.copyOf(Objects.requireNonNull(samplesNanos, "samplesNanos"));
19+
statistics = Objects.requireNonNull(statistics, "statistics");
20+
BenchmarkStatistics expected = BenchmarkStatistics.fromSamples(samplesNanos);
21+
if (!expected.equals(statistics)) {
22+
throw new IllegalArgumentException("statistics do not match benchmark samples");
23+
}
24+
}
25+
}

‎de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkStatistics.java‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package de.peeeq.wurstio.benchmark;
2+
3+
import java.util.ArrayList;
4+
import java.util.List;
5+
import java.util.Objects;
6+
7+
public record BenchmarkStatistics(
8+
double mean,
9+
double standardDeviation,
10+
long min,
11+
long max,
12+
long median,
13+
long p90,
14+
long p95
15+
) {
16+
public BenchmarkStatistics {
17+
if (!Double.isFinite(mean) || mean < 0.0) {
18+
throw new IllegalArgumentException("mean must be finite and non-negative");
19+
}
20+
if (!Double.isFinite(standardDeviation) || standardDeviation < 0.0) {
21+
throw new IllegalArgumentException("standard deviation must be finite and non-negative");
22+
}
23+
if (min < 0 || max < 0 || median < 0 || p90 < 0 || p95 < 0) {
24+
throw new IllegalArgumentException("statistics values must be non-negative");
25+
}
26+
if (min > median || median > p90 || p90 > p95 || p95 > max) {
27+
throw new IllegalArgumentException("percentiles must be ordered between min and max");
28+
}
29+
if (mean < min || mean > max) {
30+
throw new IllegalArgumentException("mean must be between min and max");
31+
}
32+
}
33+
34+
public static BenchmarkStatistics fromSamples(List<Long> samples) {
35+
Objects.requireNonNull(samples, "samples");
36+
if (samples.isEmpty()) {
37+
throw new IllegalArgumentException("at least one benchmark sample is required");
38+
}
39+
for (Long sample : samples) {
40+
if (sample == null || sample < 0) {
41+
throw new IllegalArgumentException("benchmark samples must be non-negative");
42+
}
43+
}
44+
45+
List<Long> sorted = new ArrayList<>(samples);
46+
sorted.sort(Long::compareTo);
47+
double mean = samples.stream()
48+
.mapToDouble(Long::doubleValue)
49+
.average()
50+
.orElseThrow();
51+
double variance = samples.stream()
52+
.mapToDouble(Long::doubleValue)
53+
.map(value -> {
54+
double delta = value - mean;
55+
return delta * delta;
56+
})
57+
.average()
58+
.orElseThrow();
59+
60+
return new BenchmarkStatistics(
61+
mean,
62+
Math.sqrt(variance),
63+
sorted.get(0),
64+
sorted.get(sorted.size() - 1),
65+
nearestRank(sorted, 0.50),
66+
nearestRank(sorted, 0.90),
67+
nearestRank(sorted, 0.95));
68+
}
69+
70+
public static BenchmarkStatistics of(List<Long> samples) {
71+
return fromSamples(samples);
72+
}
73+
74+
private static long nearestRank(List<Long> sorted, double percentile) {
75+
int rank = (int) Math.ceil(percentile * sorted.size());
76+
return sorted.get(Math.max(0, rank - 1));
77+
}
78+
}

‎de.peeeq.wurstscript/src/main/java/de/peeeq/wurstio/benchmark/BenchmarkWorkerOutput.java‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
package de.peeeq.wurstio.benchmark;
2+
3+
import com.google.gson.Gson;
4+
import com.google.gson.GsonBuilder;
5+
import com.google.gson.JsonArray;
6+
import com.google.gson.JsonObject;
7+
8+
import java.io.IOException;
9+
import java.nio.charset.StandardCharsets;
10+
import java.nio.file.AtomicMoveNotSupportedException;
11+
import java.nio.file.Files;
12+
import java.nio.file.Path;
13+
import java.nio.file.StandardCopyOption;
14+
import java.nio.file.StandardOpenOption;
15+
import java.util.List;
16+
import java.util.Objects;
17+
18+
/** Writes the machine-readable result of one isolated benchmark compiler worker. */
19+
public final class BenchmarkWorkerOutput {
20+
public static final String SCHEMA = "wurst-benchmark-worker-v1";
21+
22+
@FunctionalInterface
23+
public interface TemporaryFileWriter {
24+
void write(Path temporary, String json) throws IOException;
25+
}
26+
27+
private static final Gson GSON = new GsonBuilder()
28+
.disableHtmlEscaping()
29+
.create();
30+
31+
private BenchmarkWorkerOutput() {
32+
}
33+
34+
public static void writeDiscovery(Path output, List<String> benchmarkNames) throws IOException {
35+
Objects.requireNonNull(benchmarkNames, "benchmarkNames");
36+
JsonObject json = envelope("discovery");
37+
JsonArray benchmarks = new JsonArray();
38+
for (String benchmarkName : benchmarkNames) {
39+
benchmarks.add(Objects.requireNonNull(benchmarkName, "benchmarkName"));
40+
}
41+
json.add("benchmarks", benchmarks);
42+
writeAtomically(output, GSON.toJson(json));
43+
}
44+
45+
public static void writeResult(Path output, BenchmarkResult result) throws IOException {
46+
Objects.requireNonNull(result, "result");
47+
JsonObject json = envelope("execution");
48+
json.addProperty("qualifiedName", result.qualifiedName());
49+
json.addProperty("checksum", result.checksum());
50+
json.addProperty("batchSize", result.batchSize());
51+
json.add("samplesNanos", GSON.toJsonTree(result.samplesNanos()));
52+
json.add("statistics", GSON.toJsonTree(result.statistics()));
53+
writeAtomically(output, GSON.toJson(json));
54+
}
55+
56+
/**
57+
* Write a complete JSON document to a sibling temporary file, then rename it
58+
* over the destination. Serialization happens before touching the destination.
59+
*/
60+
public static void writeAtomically(Path output, String json) throws IOException {
61+
writeAtomically(output, json, BenchmarkWorkerOutput::writeTemporaryFile);
62+
}
63+
64+
public static void writeAtomically(
65+
Path output,
66+
String json,
67+
TemporaryFileWriter temporaryFileWriter
68+
) throws IOException {
69+
Objects.requireNonNull(output, "output");
70+
Objects.requireNonNull(json, "json");
71+
Objects.requireNonNull(temporaryFileWriter, "temporaryFileWriter");
72+
Path absoluteOutput = output.toAbsolutePath();
73+
Path parent = absoluteOutput.getParent();
74+
if (parent == null) {
75+
throw new IOException("benchmark output has no parent directory: " + output);
76+
}
77+
Files.createDirectories(parent);
78+
Path temporary = Files.createTempFile(parent, "wurst-benchmark-", ".tmp");
79+
try {
80+
temporaryFileWriter.write(temporary, json);
81+
try {
82+
Files.move(
83+
temporary,
84+
absoluteOutput,
85+
StandardCopyOption.ATOMIC_MOVE,
86+
StandardCopyOption.REPLACE_EXISTING);
87+
} catch (AtomicMoveNotSupportedException e) {
88+
Files.move(temporary, absoluteOutput, StandardCopyOption.REPLACE_EXISTING);
89+
}
90+
} finally {
91+
Files.deleteIfExists(temporary);
92+
}
93+
}
94+
95+
private static void writeTemporaryFile(Path temporary, String json) throws IOException {
96+
Files.writeString(
97+
temporary,
98+
json,
99+
StandardCharsets.UTF_8,
100+
StandardOpenOption.WRITE,
101+
StandardOpenOption.TRUNCATE_EXISTING);
102+
}
103+
104+
private static JsonObject envelope(String mode) {
105+
JsonObject json = new JsonObject();
106+
json.addProperty("schema", SCHEMA);
107+
json.addProperty("mode", mode);
108+
return json;
109+
}
110+
}

0 commit comments

Comments
 (0)