perf: delay JavaScript optimizations until emission

bench/delayed-js-optimizations/README.md

@@ -0,0 +1,45 @@
+# Delayed JavaScript optimization benchmark
+
+This benchmark compares the eager (previous) and delayed JavaScript optimizer
+schedules through real Dune builds. It covers cold and leaf-edit builds of both
+the complete `@melange` output and the CMJ critical path, serial and parallel
+execution, one and two JavaScript module systems, and four dependency shapes:
+
+- `chain`: a long critical dependency path
+- `wide`: independent modules feeding one entry point
+- `diamond`: a branching dependency DAG
+- `optimizer`: a chain with substantially more JavaScript optimizer work
+
+`--optimizer-width` controls the number of exported helper functions in every
+module of the optimizer topology (default: 64). Raise it to stress large JS IR
+without increasing the Dune action count.
+
+Run the full matrix from the repository's Nix development shell:
+
+```sh
+nix develop
+bench/delayed-js-optimizations/run.sh \
+  --output /tmp/melange-delay-js-results
+```
+
+For a quick smoke measurement:
+
+```sh
+bench/delayed-js-optimizations/run.sh \
+  --output /tmp/melange-delay-js-smoke \
+  --runs 2 --warmup 0 --modules 20 \
+  --topologies chain --module-systems single \
+  --jobs 1 --scenarios cold
+```
+
+The runner produces Hyperfine JSON, Chrome-compatible Dune action traces for
+both schedules, `summary.json`, and a compact `summary.md`. The wall-clock
+results measure tracing-disabled production behavior. A separate diagnostic
+build enables `--mel-action-trace`, allowing the summary to report compiler
+phase counts and durations without contaminating the timing samples.
+
+Treat results as meaningful only when repeated runs are stable and compare the
+same checkout, machine, power mode, Dune cache setting, and job count. The dual
+module-system cases are especially important: delayed optimization shortens
+the CMJ action, but repeats JavaScript optimization once per emitted module
+system.

bench/delayed-js-optimizations/generate.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""Generate a deterministic Melange/Dune benchmark project."""
+
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+import shutil
+
+
+def module_name(index: int) -> str:
+    return f"Unit_{index:04d}"
+
+
+def filename(index: int) -> str:
+    return f"unit_{index:04d}.ml"
+
+
+def sum_expression(terms: list[str]) -> str:
+    while len(terms) > 1:
+        terms = [
+            "(" + " + ".join(terms[index : index + 16]) + ")"
+            for index in range(0, len(terms), 16)
+        ]
+    return terms[0]
+
+
+def source_for(index: int, topology: str, optimizer_width: int) -> str:
+    if index == 0:
+        dependency = "1"
+    elif topology in {"chain", "optimizer"}:
+        dependency = f"{module_name(index - 1)}.value + 1"
+    elif topology == "wide":
+        dependency = str(index + 1)
+    elif topology == "diamond":
+        left = module_name(index - 1)
+        right = module_name(index // 2)
+        dependency = f"{left}.value + {right}.value + 1"
+    else:
+        raise ValueError(f"unknown topology: {topology}")
+
+    if topology != "optimizer":
+        return f"let value = {dependency}\n"
+
+    helpers = "\n".join(
+        f"let helper_{helper} x = x + {helper + 1}"
+        for helper in range(optimizer_width)
+    )
+    pipeline = " |> ".join(
+        f"helper_{helper}" for helper in range(optimizer_width)
+    )
+    return (
+        f"{helpers}\n"
+        "let dead_branch x = if x = 0 then x + 1000 else x\n"
+        f"let value = {dependency} |> {pipeline} |> dead_branch\n"
+    )
+
+
+def main_source(modules: int, topology: str) -> str:
+    if topology == "wide":
+        terms = [f"{module_name(index)}.value" for index in range(modules)]
+        expression = sum_expression(terms)
+    else:
+        expression = f"{module_name(modules - 1)}.value"
+    return f"let result = {expression}\n"
+
+
+def generate(args: argparse.Namespace) -> None:
+    root = args.root.resolve()
+    if root in {Path(root.anchor), Path.home(), Path.cwd()}:
+        raise SystemExit(f"refusing unsafe project root: {root}")
+    marker = root / ".melange-delay-js-benchmark"
+    if root.exists():
+        if not marker.is_file():
+            raise SystemExit(f"refusing to replace unmarked directory: {root}")
+        shutil.rmtree(root)
+    root.mkdir(parents=True)
+    marker.write_text("generated benchmark project\n", encoding="utf-8")
+
+    flag = {
+        "eager": "--mel-eager-js-optimizations",
+        "delayed": "--mel-delay-js-optimizations",
+    }[args.mode]
+    trace_flag = " --mel-action-trace" if args.action_trace else ""
+    module_systems = {
+        "single": "(module_systems (commonjs js))",
+        "dual": "(module_systems (commonjs cjs) (esm mjs))",
+    }[args.module_systems]
+
+    (root / "dune-project").write_text(
+        "(lang dune 3.24)\n(using melange 1.0)\n", encoding="utf-8"
+    )
+    (root / "dune").write_text(
+        "(melange.emit\n"
+        " (target output)\n"
+        " (emit_stdlib false)\n"
+        f" {module_systems}\n"
+        f" (compile_flags :standard {flag}{trace_flag} --mel-cross-module-opt))\n",
+        encoding="utf-8",
+    )
+
+    for index in range(args.modules):
+        source = source_for(index, args.topology, args.optimizer_width)
+        (root / filename(index)).write_text(source, encoding="utf-8")
+    (root / "main.ml").write_text(
+        main_source(args.modules, args.topology), encoding="utf-8"
+    )
+
+    leaf = root / filename(0)
+    baseline = leaf.read_text(encoding="utf-8")
+    (root / f"{filename(0)}.baseline").write_text(baseline, encoding="utf-8")
+    (root / f"{filename(0)}.changed").write_text(
+        baseline + "let benchmark_edit = 1\n", encoding="utf-8"
+    )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--root", type=Path, required=True)
+    parser.add_argument("--modules", type=int, default=200)
+    parser.add_argument(
+        "--topology", choices=("chain", "wide", "diamond", "optimizer"), required=True
+    )
+    parser.add_argument("--mode", choices=("eager", "delayed"), required=True)
+    parser.add_argument("--module-systems", choices=("single", "dual"), required=True)
+    parser.add_argument("--optimizer-width", type=int, default=64)
+    parser.add_argument("--action-trace", action="store_true")
+    args = parser.parse_args()
+    if args.modules < 2:
+        parser.error("--modules must be at least 2")
+    if args.optimizer_width < 1:
+        parser.error("--optimizer-width must be at least 1")
+    return args
+
+
+if __name__ == "__main__":
+    generate(parse_args())

bench/delayed-js-optimizations/run.sh

@@ -0,0 +1,147 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+script_dir=$(cd "$(dirname "$0")" && pwd)
+repo_root=$(cd "$script_dir/../.." && pwd)
+
+runs=5
+warmup=1
+modules=200
+optimizer_width=64
+output=""
+topologies=chain,wide,diamond,optimizer
+module_systems=single,dual
+jobs=1,auto
+scenarios=cold,incremental,cmj-cold,cmj-incremental
+
+usage() {
+  echo "usage: $0 --output DIR [--runs N] [--warmup N] [--modules N]"
+  echo "          [--optimizer-width N]"
+  echo "          [--topologies LIST] [--module-systems LIST] [--jobs LIST]"
+  echo "          [--scenarios LIST]"
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --output) output=$2; shift 2 ;;
+    --runs) runs=$2; shift 2 ;;
+    --warmup) warmup=$2; shift 2 ;;
+    --modules) modules=$2; shift 2 ;;
+    --optimizer-width) optimizer_width=$2; shift 2 ;;
+    --topologies) topologies=$2; shift 2 ;;
+    --module-systems) module_systems=$2; shift 2 ;;
+    --jobs) jobs=$2; shift 2 ;;
+    --scenarios) scenarios=$2; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$output" ]]; then
+  echo "--output is required so benchmark artifacts have an explicit destination" >&2
+  exit 2
+fi
+
+case "$output" in
+  /|"$repo_root")
+    echo "refusing unsafe output directory: $output" >&2
+    exit 2
+    ;;
+  *"'"*)
+    echo "output directory cannot contain a single quote: $output" >&2
+    exit 2
+    ;;
+esac
+
+if [[ "$(uname -s)" == Darwin ]]; then
+  auto_jobs=$(sysctl -n hw.logicalcpu)
+else
+  auto_jobs=$(getconf _NPROCESSORS_ONLN)
+fi
+
+output=$(mkdir -p "$output" && cd "$output" && pwd)
+
+(cd "$repo_root" && \
+  dune build --display=quiet --sandbox=none --only-packages melange @install)
+export PATH="$repo_root/_build/install/default/bin:$PATH"
+export OCAMLPATH="$repo_root/_build/install/default/lib${OCAMLPATH:+:$OCAMLPATH}"
+
+IFS=, read -r -a topology_values <<< "$topologies"
+IFS=, read -r -a system_values <<< "$module_systems"
+IFS=, read -r -a job_values <<< "$jobs"
+IFS=, read -r -a scenario_values <<< "$scenarios"
+
+for topology in "${topology_values[@]}"; do
+  for systems in "${system_values[@]}"; do
+    for job_value in "${job_values[@]}"; do
+      if [[ "$job_value" == auto ]]; then
+        job_count=$auto_jobs
+      else
+        job_count=$job_value
+      fi
+      for scenario in "${scenario_values[@]}"; do
+        case_dir="$output/cases/$topology/$systems/j$job_count/$scenario"
+        mkdir -p "$case_dir"
+
+        for mode in eager delayed; do
+          python3 "$script_dir/generate.py" \
+            --root "$case_dir/$mode" \
+            --modules "$modules" \
+            --topology "$topology" \
+            --mode "$mode" \
+            --optimizer-width "$optimizer_width" \
+            --module-systems "$systems"
+        done
+
+        if [[ "$scenario" == cmj-* ]]; then
+          build_target=.output.mobjs/melange/melange__Main.cmj
+        else
+          build_target=@melange
+        fi
+
+        if [[ "$scenario" == cold || "$scenario" == cmj-cold ]]; then
+          prepare="dune clean --root '$case_dir/{mode}'"
+        elif [[ "$scenario" == incremental || "$scenario" == cmj-incremental ]]; then
+          prepare="cp '$case_dir/{mode}/unit_0000.ml.baseline' '$case_dir/{mode}/unit_0000.ml'; dune clean --root '$case_dir/{mode}'; dune build --root '$case_dir/{mode}' --cache=disabled -j '$job_count' '$build_target' >/dev/null; cp '$case_dir/{mode}/unit_0000.ml.changed' '$case_dir/{mode}/unit_0000.ml'"
+        else
+          echo "unknown scenario: $scenario" >&2
+          exit 2
+        fi
+        command="dune build --root '$case_dir/{mode}' --cache=disabled -j '$job_count' '$build_target'"
+
+        hyperfine \
+          --runs "$runs" \
+          --warmup "$warmup" \
+          --parameter-list mode eager,delayed \
+          --prepare "$prepare" \
+          --export-json "$case_dir/hyperfine.json" \
+          "$command"
+
+        for mode in eager delayed; do
+          trace_root="$case_dir/trace-$mode-project"
+          python3 "$script_dir/generate.py" \
+            --root "$trace_root" \
+            --modules "$modules" \
+            --topology "$topology" \
+            --mode "$mode" \
+            --optimizer-width "$optimizer_width" \
+            --module-systems "$systems" \
+            --action-trace
+          if [[ "$scenario" == incremental || "$scenario" == cmj-incremental ]]; then
+            dune build --display=quiet --root "$trace_root" --cache=disabled \
+              -j "$job_count" "$build_target"
+            cp "$trace_root/unit_0000.ml.changed" "$trace_root/unit_0000.ml"
+          fi
+          dune build --display=quiet --root "$trace_root" --cache=disabled \
+            -j "$job_count" --trace-file="$case_dir/trace-$mode.csexp" \
+            "$build_target"
+          dune trace cat --trace-file="$case_dir/trace-$mode.csexp" \
+            --chrome-trace > "$case_dir/trace-$mode.json"
+        done
+      done
+    done
+  done
+done
+
+python3 "$script_dir/summarize.py" "$output"
+echo "Raw measurements, traces, and summaries: $output"