benchmark code

Author: dkuku

bench/.gitignore

@@ -0,0 +1 @@
+out/

bench/ab.exs

@@ -0,0 +1,81 @@
+# Measures two generated parser modules against each other in one VM.
+#
+#     BENCH_EBIN=<dir> mix run bench/ab.exs
+#
+# Expects BenchOld and BenchNew, as produced by bench/gen_module.exs. With only
+# BenchNew on the path it reports that side's absolute numbers instead.
+# bench/compare.sh drives the whole thing; run this directly only if you have
+# already built the modules yourself.
+#
+# The beam directory arrives by env var rather than `elixir -pa`, because Mix
+# prunes the code path down to the project's own load paths.
+#
+# Env: BENCH_EBIN, BENCH_ROUNDS (default 21), BENCH_BUDGET_US per round (20000).
+Code.require_file("support.exs", __DIR__)
+
+if ebin = System.get_env("BENCH_EBIN"), do: Code.prepend_path(ebin)
+
+alias Bench.{Measure, Table}
+
+rounds = System.get_env("BENCH_ROUNDS", "21") |> String.to_integer()
+budget = System.get_env("BENCH_BUDGET_US", "20000") |> String.to_integer()
+
+old? = Code.ensure_loaded?(BenchOld)
+if not Code.ensure_loaded?(BenchNew), do: raise("BenchNew is not on the code path")
+
+call = fn module, parser, input -> fn -> apply(module, parser, [input]) end end
+
+rows =
+  for {label, parser, input} <- Bench.Cases.inputs() do
+    new = call.(BenchNew, parser, input)
+
+    # One iteration count for both sides, so the reduction counts stay directly
+    # comparable
+    iters = Measure.calibrate(new, budget)
+
+    if old? do
+      old = call.(BenchOld, parser, input)
+
+      if apply(BenchOld, parser, [input]) != apply(BenchNew, parser, [input]) do
+        raise "#{label}: the two revisions disagree on the result"
+      end
+
+      t_old = Measure.time(old, iters, rounds)
+      t_new = Measure.time(new, iters, rounds)
+      # The control: the same code timed twice. Its distance from 1.000x is how
+      # much of the speedup column is noise.
+      t_control = Measure.time(old, iters, rounds)
+
+      r_old = Measure.reductions(old)
+      r_new = Measure.reductions(new)
+
+      [
+        label,
+        Table.ns(t_old),
+        Table.ns(t_new),
+        Table.ratio(t_old / t_new),
+        Table.ratio(t_old / t_control),
+        Table.int(r_old),
+        Table.signed(r_new - r_old)
+      ]
+    else
+      [label, Table.ns(Measure.time(new, iters, rounds)), Table.int(Measure.reductions(new))]
+    end
+  end
+
+IO.puts("")
+
+if old? do
+  Table.print(["case", "old(ns)", "new(ns)", "speedup", "control", "red.", "red.diff"], rows)
+
+  IO.puts("""
+
+  speedup   old/new wall clock, min-of-#{rounds}. Only trust it if it clears `control`.
+  control   the old side timed twice. This run's noise floor.
+  red.diff  reductions for one call, new minus old. Exact: 0 means both revisions
+            perform exactly the same work, whatever the clock says.
+  """)
+else
+  Table.print(["case", "ns/call", "red./call"], rows)
+  IO.puts("\nBenchOld not on the path, so this is the working tree alone.\n")
+end

bench/compare.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Compares the working tree against a git ref.
+#
+#     bench/compare.sh            # vs HEAD
+#     bench/compare.sh master     # vs any ref
+#
+# Both revisions are measured in a single VM, from parser source dumped by
+# bench/gen_module.exs, so neither side pays for a recompile and neither warms
+# the machine up for the other. The dumps are left in bench/out/ to be diffed.
+#
+# The ref is checked out into a throwaway worktree, and this directory's scripts
+# are copied in, so the ref may predate them and both sides are driven by the
+# same case definitions.
+set -euo pipefail
+
+REF="${1:-HEAD}"
+ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+OUT="$ROOT/bench/out"
+TMP="$(mktemp -d)"
+WT="$TMP/worktree"
+
+cleanup() {
+  [ -d "$WT" ] && git -C "$ROOT" worktree remove --force "$WT" >/dev/null 2>&1 || true
+  git -C "$ROOT" worktree prune >/dev/null 2>&1 || true
+  rm -rf "$TMP"
+}
+trap cleanup EXIT
+
+cd "$ROOT"
+mkdir -p "$OUT"
+
+echo "==> working tree"
+mix run bench/gen_module.exs New "$OUT/new.ex"
+
+echo "==> $REF ($(git rev-parse --short "$REF"))"
+git worktree add --quiet --detach "$WT" "$REF"
+mkdir -p "$WT/bench"
+cp bench/*.exs "$WT/bench/"
+(cd "$WT" && mix run bench/gen_module.exs Old "$OUT/old.ex")
+
+echo
+echo "==> generated code"
+if diff -q <(sed 's/BenchOld/Bench/' "$OUT/old.ex") \
+           <(sed 's/BenchNew/Bench/' "$OUT/new.ex") >/dev/null; then
+  echo "    identical -- any difference in the table below is noise"
+else
+  echo "    differs: diff bench/out/old.ex bench/out/new.ex"
+  echo "    ($(diff <(sed 's/BenchOld/Bench/' "$OUT/old.ex") \
+                     <(sed 's/BenchNew/Bench/' "$OUT/new.ex") | grep -c '^[<>]') changed lines;"
+  echo "     renumbered functions and reordered clauses are expected and harmless)"
+fi
+
+mkdir -p "$TMP/ebin"
+elixirc --ignore-module-conflict -o "$TMP/ebin" "$OUT/old.ex" "$OUT/new.ex"
+BENCH_EBIN="$TMP/ebin" mix run bench/ab.exs

bench/gen_module.exs

@@ -0,0 +1,40 @@
+# Dumps the parsers in Bench.Cases as plain, formatted source.
+#
+#     mix run bench/gen_module.exs <ModuleSuffix> <output.ex>
+#
+# Two purposes:
+#
+#   * The code two revisions generate can be compiled into a single VM and
+#     measured side by side, without either paying for a recompile.
+#   * The dumps are diffable. An empty diff settles a performance question
+#     outright: identical code cannot run at different speeds. A non-empty one
+#     still needs reading, since function numbering and clause order shift
+#     around harmlessly.
+Code.require_file("support.exs", __DIR__)
+
+[suffix, out] = System.argv()
+
+body =
+  for {name, combinator} <- Bench.Cases.parsers() do
+    {defs, inline} = NimbleParsec.Compiler.compile(name, Enum.reverse(combinator), inline: true)
+
+    clauses =
+      for {fun, args, guards, body} <- defs do
+        head = Macro.to_string({fun, [], args})
+        head = if guards == true, do: head, else: "#{head} when #{Macro.to_string(guards)}"
+        "defp #{head} do\n#{Macro.to_string(body)}\nend"
+      end
+
+    # Calls the generated entry point directly, minus the `defparsec` wrapper's
+    # argument handling
+    """
+    @compile {:inline, #{inspect(inline)}}
+    def #{name}(binary), do: #{name}__0(binary, [], [], %{}, {1, 0}, 0)
+
+    #{Enum.join(clauses, "\n\n")}
+    """
+  end
+
+source = "defmodule Bench#{suffix} do\n#{Enum.join(body, "\n")}\nend\n"
+File.write!(out, [Code.format_string!(source), "\n"])
+IO.puts("wrote #{out} (Bench#{suffix})")

bench/string_bench.exs

@@ -0,0 +1,47 @@
+# Absolute numbers for the working tree, through the public API.
+#
+#     mix run bench/string_bench.exs
+#
+# This goes through the parsers `defparsec` defines, wrapper and all, so the
+# numbers are what a caller actually pays. For comparing two revisions use
+# bench/compare.sh, which measures both in one VM and can prove when the
+# generated code is identical.
+#
+# Env: BENCH_ROUNDS (default 21), BENCH_BUDGET_US per round (default 20000).
+Code.require_file("support.exs", __DIR__)
+
+alias Bench.{Measure, Table}
+
+defmodule Bench.Parsers do
+  import NimbleParsec
+
+  # `defparsec` evaluates its combinator in the module body, so the shared
+  # definitions can be fed to it straight from Bench.Cases
+  for {name, combinator} <- Bench.Cases.parsers() do
+    defparsec(name, combinator)
+  end
+end
+
+rounds = System.get_env("BENCH_ROUNDS", "21") |> String.to_integer()
+budget = System.get_env("BENCH_BUDGET_US", "20000") |> String.to_integer()
+
+rows =
+  for {label, parser, input} <- Bench.Cases.inputs() do
+    fun = fn -> apply(Bench.Parsers, parser, [input]) end
+    iters = Measure.calibrate(fun, budget)
+
+    [
+      label,
+      Table.ns(Measure.time(fun, iters, rounds)),
+      Table.ns(Measure.reductions(fun, 2_000))
+    ]
+  end
+
+IO.puts("")
+Table.print(~w(case ns/call red./call), rows)
+
+IO.puts("""
+
+ns/call    min-of-#{rounds} rounds, each in a fresh process.
+red./call  reductions, including a constant loop overhead. Deterministic.
+""")