Tune JMH configurations for stable measurements

core/src/jmh/java/com/ibm/wala/benchmarks/callgraph/CallGraphBenchmark.java

@@ -28,14 +28,14 @@
  * <p>The benchmark is stable to roughly one percent across forks, but with only a single fork and a
  * handful of iterations JMH reports a wildly inflated error: its printed margin is {@code t(n-1,
  * 99.9%) x} the standard error, and with {@code n = 3} that t-multiplier is about 32. Each fork is
- * therefore treated as one independent sample, and four forks (with two measured invocations each)
+ * therefore treated as one independent sample, and six forks (with six measured invocations each)
  * give a printed error that is small enough to track relative changes in runtime.
  */
 @BenchmarkMode(Mode.SingleShotTime)
-@Fork(4)
-@Measurement(iterations = 2)
+@Fork(6)
+@Measurement(iterations = 6)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
-@Warmup(iterations = 1)
+@Warmup(iterations = 3)
 public class CallGraphBenchmark {
 
   /**

core/src/jmh/java/com/ibm/wala/benchmarks/dataflow/DataflowBenchmark.java

@@ -42,17 +42,14 @@
  *
  * <p>These are sub-millisecond analyses, so a single invocation is noisy (allocation and GC jitter
  * are on the same scale as the work itself). The {@code @Warmup} of 60 iterations covers the JIT
- * ramp, which extends well past 30 iterations, and the {@code @Measurement} of 20 iterations across
- * 4 forks averages the noise down to a per-run error of roughly {@code 5-9%}. Repeated runs agree
- * with each other within that noise: e.g. the context-insensitive mean reproduced at {@code 0.255}
- * and {@code 0.250} ms and the context-sensitive mean at {@code 0.236} and {@code 0.224} ms across
- * two consecutive runs.
+ * ramp, which extends well past 30 iterations, and the {@code @Measurement} of 40 iterations across
+ * 6 forks averages the noise down.
  *
  * <p>Run with {@code ./gradlew :core:jmh}.
  */
 @BenchmarkMode(Mode.SingleShotTime)
-@Fork(4)
-@Measurement(iterations = 20)
+@Fork(6)
+@Measurement(iterations = 40)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 @State(Scope.Thread)
 @Warmup(iterations = 60)

core/src/jmh/java/com/ibm/wala/benchmarks/slicer/SlicerBenchmark.java

@@ -42,14 +42,14 @@
  * roughly 40ms after five warmup invocations to a stable 28.6ms only after roughly sixty. The
  * {@code @Warmup(iterations = 60)} setting absorbs that entire ramp, so the
  * {@code @Measurement(iterations = 10)} measured invocations in each fork are steady and
- * independent. Together with {@code @Fork(4)} — four independent JVMs, so that no single JVM's JIT
+ * independent. Together with {@code @Fork(5)} — five independent JVMs, so that no single JVM's JIT
  * decisions dominate the reported error — this benchmark is reproducible to within about 1.5%,
  * tight enough to track relative changes in runtime.
  *
  * <p>Run with {@code ./gradlew :core:jmh}.
  */
 @BenchmarkMode(Mode.SingleShotTime)
-@Fork(4)
+@Fork(5)
 @Measurement(iterations = 10)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 @State(Scope.Thread)

core/src/jmh/java/com/ibm/wala/core/util/strings/AtomBenchmark.java

@@ -49,18 +49,17 @@
  *
  * <p>Run with {@code ./gradlew :core:jmh}. Results are written to {@code core/build/results/jmh/}.
  * These settings trade total run time for precision, so that they can detect relative regressions:
- * each method runs in three forks, with three one-second warmup iterations and three one-second
+ * each method runs in four forks, with five one-second warmup iterations and five one-second
  * measurement iterations. A single fork is measurably noisier (the per-fork mean is stable, but
- * between-fork variance dominates for several methods), so three forks cut the reported error from
- * up to ~10% down to ~1-2%. A few methods with idiosyncratic noise get their own overrides; see
- * their Javadoc. Pass {@code -f}, {@code -wi}, {@code -i}, and similar JMH command-line flags for
- * one-off adjustments.
+ * between-fork variance dominates for several methods), so four forks reduce the reported error. A
+ * few methods with idiosyncratic noise get their own overrides; see their Javadoc. Pass {@code -f},
+ * {@code -wi}, {@code -i}, and similar JMH command-line flags for one-off adjustments.
  */
 @BenchmarkMode(Mode.Throughput)
-@Fork(3)
-@Measurement(iterations = 3, time = 1)
+@Fork(4)
+@Measurement(iterations = 5, time = 1)
 @OutputTimeUnit(TimeUnit.MICROSECONDS)
-@Warmup(iterations = 3, time = 1)
+@Warmup(iterations = 5, time = 1)
 public class AtomBenchmark {
 
   /** Number of distinct, never-before-interned byte arrays in each miss-benchmark iteration. */
@@ -184,20 +183,15 @@ public void regenerate() {
     }
   }
 
-  /**
-   * Looks up an already-interned {@link Atom} from its bytes.
-   *
-   * <p>One more fork than the class default, because this benchmark's mean has drifted more between
-   * forks than the other microbenchmarks in repeated runs.
-   */
+  /** Looks up an already-interned {@link Atom} from its bytes. */
   @Benchmark
-  @Fork(4)
   public Atom findOrCreateFromByteArray(AtomFixture fixture) {
     return Atom.findOrCreate(fixture.nextByteArray());
   }
 
   /** Looks up an already-interned {@link Atom} from a freshly copied byte array. */
   @Benchmark
+  @Measurement(iterations = 6, time = 1)
   public Atom findOrCreateFromFreshByteArray(AtomFixture fixture) {
     final byte[] original = fixture.nextByteArray();
     return Atom.findOrCreate(Arrays.copyOf(original, original.length));
@@ -217,13 +211,15 @@ public Atom findOrCreateAsciiAtom(AtomFixture fixture) {
 
   /** Looks up an already-interned {@link Atom} from a whole byte-array slice. */
   @Benchmark
+  @Measurement(iterations = 6, time = 1)
   public Atom findOrCreateFromByteArraySlice(AtomFixture fixture) {
     final byte[] bytes = fixture.nextByteArray();
     return Atom.findOrCreate(bytes, 0, bytes.length);
   }
 
   /** Creates the trailing half of an already-interned {@link Atom}. */
   @Benchmark
+  @Measurement(iterations = 6, time = 1)
   public Atom right(AtomFixture fixture) {
     final Atom atom = fixture.nextAtom();
     return atom.right(atom.length() / 2);
@@ -236,14 +232,8 @@ public boolean startsWith(AtomFixture fixture) {
     return fixture.atoms[index].startsWith(fixture.prefixes[index]);
   }
 
-  /**
-   * Concatenates two already-interned {@link Atom}s to an already-interned result.
-   *
-   * <p>One more fork than the class default, because this benchmark's mean has drifted more between
-   * forks than the other microbenchmarks in repeated runs.
-   */
+  /** Concatenates two already-interned {@link Atom}s to an already-interned result. */
   @Benchmark
-  @Fork(4)
   public Atom concatAtoms(AtomFixture fixture) {
     final int index = fixture.nextIndex();
     return Atom.concat(
@@ -271,13 +261,15 @@ public byte[] getValArray(AtomFixture fixture) {
 
   /** Reads one byte from an {@link Atom}. */
   @Benchmark
+  @Measurement(iterations = 6, time = 1)
   public byte getVal(AtomFixture fixture) {
     final Atom atom = fixture.nextAtom();
     return atom.getVal(atom.length() / 2);
   }
 
   /** Searches an {@link Atom} for a byte that is usually absent. */
   @Benchmark
+  @Measurement(iterations = 6, time = 1)
   public int rIndex(AtomFixture fixture) {
     return fixture.nextAtom().rIndex((byte) 'a');
   }
@@ -299,9 +291,9 @@ public int atomHashCode(AtomFixture fixture) {
   /** Interns a fresh pool of genuinely new {@link Atom}s. */
   @Benchmark
   @BenchmarkMode(Mode.SingleShotTime)
-  @Measurement(iterations = 10)
+  @Measurement(iterations = 20)
   @OperationsPerInvocation(MISS_POOL_SIZE)
-  @Warmup(iterations = 10)
+  @Warmup(iterations = 20)
   public void findOrCreateMiss(FreshBytes freshBytes, Blackhole blackhole) {
     for (final byte[] bytes : freshBytes.pool) {
       blackhole.consume(Atom.findOrCreate(bytes));