Add persistent Benchee benchmarks

.formatter.exs

@@ -1,5 +1,5 @@
 # Used by "mix format"
 [
-  inputs: ["{mix,.formatter}.exs", "{lib,test}/**/*.{ex,exs}"],
+  inputs: ["{mix,.formatter}.exs", "{lib,test,bench}/**/*.{ex,exs}"],
   import_deps: [:stream_data]
 ]

.github/workflows/benchmark.yml

@@ -0,0 +1,133 @@
+name: benchmark
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+  schedule:
+    - cron: "17 10 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: write
+  pull-requests: write
+
+concurrency:
+  group: benchmark-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  row-binary:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+
+      - id: beam
+        uses: erlef/setup-beam@v1
+        with:
+          elixir-version: "1.19"
+          otp-version: "28"
+
+      - uses: actions/cache@v6
+        with:
+          path: |
+            deps
+            _build
+          key: benchmark-${{ runner.os }}-${{ steps.beam.outputs.elixir-version }}-${{ steps.beam.outputs.otp-version }}-${{ hashFiles('mix.lock') }}
+          restore-keys: benchmark-${{ runner.os }}-${{ steps.beam.outputs.elixir-version }}-
+
+      - run: mix deps.get --only dev
+      - run: mix compile --warnings-as-errors
+      - run: mix benchmark
+
+      - name: Locate benchmark result
+        id: benchmark-result
+        run: |
+          result_path=$(cat bench/output/current-result-path)
+          machine=$(jq --raw-output .machine "$result_path")
+          version=$(jq --raw-output .benchmark_version "$result_path")
+          echo "path=$result_path" >> "$GITHUB_OUTPUT"
+          echo "machine=$machine" >> "$GITHUB_OUTPUT"
+          echo "version=$version" >> "$GITHUB_OUTPUT"
+
+      - name: Install ClickHouse Local
+        env:
+          DO_NOT_TRACK: "1"
+        run: |
+          curl --fail --silent --show-error https://clickhouse.com/cli | sh
+          "$HOME/.local/bin/clickhousectl" local use latest
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+      - name: Load benchmark history
+        id: benchmark-history
+        run: |
+          result_path="${{ steps.benchmark-result.outputs.path }}"
+          machine="${{ steps.benchmark-result.outputs.machine }}"
+
+          if git fetch origin benchmark-results; then
+            git worktree add bench-history FETCH_HEAD
+          else
+            git worktree add --detach bench-history
+            git -C bench-history switch --orphan benchmark-results
+          fi
+
+          mkdir -p bench/compare
+          cp "$result_path" bench/compare/current.json
+
+          if test -f "bench-history/latest/$machine/benchmark.json"; then
+            cp "bench-history/latest/$machine/benchmark.json" bench/compare/baseline.json
+          else
+            cp bench/compare/current.json bench/compare/baseline.json
+            echo "note=No prior baseline exists; this run establishes it." >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Compare with latest master benchmark
+        run: |
+          version="${{ steps.benchmark-result.outputs.version }}"
+          machine="${{ steps.benchmark-result.outputs.machine }}"
+          baseline_note="${{ steps.benchmark-history.outputs.note }}"
+          if test -z "$baseline_note"; then
+            baseline_note="Compared with the latest master result from the same CI machine. Changes below 5% are reported as noise."
+          fi
+          clickhouse local --queries-file bench/compare.sql > bench/compare/comment.md
+          {
+            echo "## RowBinary benchmark"
+            echo
+            echo "$baseline_note"
+            echo
+            cat bench/compare/comment.md
+            echo
+            echo "Commit: \`$version\` · Machine: \`$machine\` · Profiler: TProf · Full JSON results are attached to this workflow run."
+          } > bench/compare/pr-comment.md
+          cat bench/compare/pr-comment.md
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: row-binary-benchmark-${{ steps.benchmark-result.outputs.version }}
+          path: ${{ steps.benchmark-result.outputs.path }}
+
+      - name: Comment on pull request
+        if: github.event_name == 'pull_request'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: gh pr comment "${{ github.event.pull_request.number }}" --edit-last --create-if-none --body-file bench/compare/pr-comment.md
+
+      - name: Store master result
+        if: github.event_name == 'push' || github.event_name == 'schedule'
+        run: |
+          result_path="${{ steps.benchmark-result.outputs.path }}"
+          machine="${{ steps.benchmark-result.outputs.machine }}"
+          version="${{ steps.benchmark-result.outputs.version }}"
+          destination="data/machine=$machine/version=$version"
+          mkdir -p "bench-history/$destination" "bench-history/latest/$machine"
+          cp "$result_path" "bench-history/$destination/benchmark.json"
+          cp "bench-history/$destination/benchmark.json" "bench-history/latest/$machine/benchmark.json"
+          git -C bench-history add data latest
+          git -C bench-history config user.name github-actions[bot]
+          git -C bench-history config user.email 41898282+github-actions[bot]@users.noreply.github.com
+          git -C bench-history commit -m "Benchmark $version"
+          git -C bench-history push origin HEAD:benchmark-results

AGENTS.md

@@ -0,0 +1,83 @@
+# Benchmark history
+
+Performance results live on the orphan `benchmark-results` branch. They are stored as self-describing Benchee JSON under:
+
+```text
+data/machine=<machine>/version=<git-sha>/benchmark.json
+latest/<machine>/benchmark.json
+```
+
+Machine partitions are derived from the execution environment, OS, CPU model, online core count, total memory, and architecture. CI partitions start with `github-actions-`, while local partitions start with `local-`. Never compare results across machine partitions. A benchmark version is the commit SHA whose code was measured.
+
+The queries below use `clickhouse local`. If the ClickHouse binary is not installed, install it with `curl https://clickhouse.com/cli | sh` followed by `~/.local/bin/clickhousectl local use latest`.
+
+Run benchmarks locally with `mix benchmark`. Use `mix benchmark --profile` to include Benchee's built-in TProf pass; CI profiles automatically. JSON results are written below `bench/output/`, partitioned by the detected machine specifications and current Git SHA. Each file includes schema/run identity; timestamp and CI provenance; OS, architecture, CPU, cores, memory, Elixir, Erlang, and JIT details; benchmark configuration and units; and raw samples plus runtime, memory, and reduction statistics for every scenario.
+
+An abridged result from the demo RowBinary benchmark looks like this. Actual files contain all samples and statistics, plus similarly shaped `memory_usage` and `reductions` collections for each scenario.
+
+```json
+{
+  "schema_version": 1,
+  "benchmark_version": "e19815a...",
+  "machine": "local-macOS-Apple-M2-8-cores-8-GB-aarch64-apple-darwin",
+  "generated_at": "2026-08-03T12:43:41Z",
+  "ci": false,
+  "units": {
+    "configuration_time": "nanosecond",
+    "run_time_samples": "nanosecond",
+    "run_time_ips": "iterations_per_second",
+    "memory_usage_samples": "byte",
+    "reductions_samples": "count"
+  },
+  "system": {
+    "os": "macOS",
+    "architecture": "aarch64-apple-darwin",
+    "cpu_speed": "Apple M2",
+    "num_cores": 8,
+    "available_memory": "8 GB",
+    "elixir": "1.20.2",
+    "erlang": "29.0.3",
+    "jit_enabled?": true
+  },
+  "configuration": {
+    "time": 20000000.0,
+    "warmup": 10000000.0,
+    "parallel": 1,
+    "percentiles": [50, 99],
+    "max_sample_size": 20000
+  },
+  "scenarios": [
+    {
+      "job_name": "RowBinary.encode_rows/2",
+      "input_name": "1000 rows",
+      "run_time": {
+        "samples": [357000, 339500, 551708],
+        "statistics": {
+          "ips": 2485.05,
+          "average": 402405.88,
+          "median": 388562.5,
+          "std_dev_ratio": 0.1445,
+          "percentiles": {"25": 349646.0, "50": 388562.5, "75": 454552.5, "99": 555458.0},
+          "sample_size": 50
+        }
+      }
+    }
+  ]
+}
+```
+
+To query history without modifying the working tree:
+
+```sh
+git fetch origin benchmark-results
+git worktree add .context/benchmark-results origin/benchmark-results
+clickhouse local -q "SELECT _file, scenario.job_name, scenario.input_name, round(scenario.run_time.statistics.ips, 2) AS ips FROM file('.context/benchmark-results/data/machine=github-actions-*/version=*/benchmark.json', JSONEachRow) ARRAY JOIN scenarios AS scenario ORDER BY _file DESC, scenario.job_name, scenario.input_name FORMAT PrettyCompact"
+```
+
+For the same comparison used by CI, copy two result files to `bench/compare/baseline.json` and `bench/compare/current.json`, then run:
+
+```sh
+clickhouse local --queries-file bench/compare.sql
+```
+
+The SQL treats an absolute throughput change below 5% as noise (`no material change`). Inspect raw samples in `scenario.run_time.samples` before attributing small changes to code. Remove the `.context/benchmark-results` worktree with `git worktree remove .context/benchmark-results` when finished.

bench/benchmark.exs

@@ -0,0 +1,61 @@
+alias Ch.RowBinary
+
+Code.require_file("json_formatter.exs", __DIR__)
+
+types = ["UInt64", "String", "Array(UInt8)", "DateTime64(3, 'UTC')", "DateTime"]
+titles = ["Golang SQL database driver", "Phoenix app event", "billing webhook payload"]
+bytes = Enum.to_list(1..16)
+base_datetime = DateTime.from_naive!(~N[2026-01-01 00:00:00.000], "Etc/UTC")
+base_naive = ~N[2026-01-01 00:00:00]
+
+make_rows = fn count ->
+  Enum.map(1..count, fn i ->
+    [
+      i,
+      Enum.at(titles, rem(i, length(titles))),
+      Enum.take(bytes, rem(i, length(bytes)) + 1),
+      DateTime.add(base_datetime, i * 17, :millisecond),
+      NaiveDateTime.add(base_naive, i, :second)
+    ]
+  end)
+end
+
+inputs =
+  for count <- [1_000, 10_000], into: %{} do
+    rows = make_rows.(count)
+
+    {"#{count} rows",
+     %{rows: rows, encoded: rows |> RowBinary.encode_rows(types) |> IO.iodata_to_binary()}}
+  end
+
+version =
+  case System.cmd("git", ["rev-parse", "HEAD"], stderr_to_stdout: true) do
+    {sha, 0} -> String.trim(sha)
+    _ -> "working-tree"
+  end
+
+github_actions? = System.get_env("GITHUB_ACTIONS") == "true"
+profile_after = if github_actions? or "--profile" in System.argv(), do: :tprof, else: false
+
+Benchee.run(
+  %{
+    "RowBinary.encode_rows/2" => fn %{rows: rows} -> RowBinary.encode_rows(rows, types) end,
+    "RowBinary.decode_rows/2" => fn %{encoded: encoded} ->
+      RowBinary.decode_rows(encoded, types)
+    end
+  },
+  inputs: inputs,
+  time: 3,
+  warmup: 1,
+  max_sample_size: 20_000,
+  measure_function_call_overhead: true,
+  pre_check: true,
+  profile_after: profile_after,
+  formatters: [
+    {Benchee.Formatters.Console, extended_statistics: true},
+    {Ch.Bench.JSONFormatter,
+     output_root: "bench/output",
+     benchmark_version: version,
+     machine_prefix: if(github_actions?, do: "github-actions", else: "local")}
+  ]
+)

bench/compare.sql

@@ -0,0 +1,44 @@
+SET join_use_nulls = 1;
+
+WITH
+baseline AS (
+  SELECT
+    scenario.job_name AS job_name,
+    scenario.input_name AS input_name,
+    scenario.run_time.statistics.ips AS baseline_ips,
+    scenario.run_time.statistics.median AS baseline_median_ns
+  FROM file('bench/compare/baseline.json', JSONEachRow)
+  ARRAY JOIN scenarios AS scenario
+),
+current AS (
+  SELECT
+    scenario.job_name AS job_name,
+    scenario.input_name AS input_name,
+    scenario.run_time.statistics.ips AS current_ips,
+    scenario.run_time.statistics.median AS current_median_ns
+  FROM file('bench/compare/current.json', JSONEachRow)
+  ARRAY JOIN scenarios AS scenario
+)
+SELECT
+  coalesce(current.job_name, baseline.job_name) AS benchmark,
+  coalesce(current.input_name, baseline.input_name) AS input,
+  round(baseline_ips, 2) AS baseline_ips,
+  round(current_ips, 2) AS current_ips,
+  multiIf(
+    baseline_ips IS NULL, 'new',
+    current_ips IS NULL, 'removed',
+    abs((current_ips / baseline_ips - 1) * 100) < 5, 'no material change',
+    current_ips > baseline_ips, 'faster',
+    'slower'
+  ) AS result,
+  if(
+    baseline_ips IS NULL OR current_ips IS NULL,
+    NULL,
+    round((current_ips / baseline_ips - 1) * 100, 2)
+  ) AS change_percent,
+  round(baseline_median_ns / 1000000, 3) AS baseline_median_ms,
+  round(current_median_ns / 1000000, 3) AS current_median_ms
+FROM baseline
+FULL OUTER JOIN current USING (job_name, input_name)
+ORDER BY benchmark, input
+FORMAT Markdown;