compile_lmbench.sh

#!/usr/bin/env bash
set -euo pipefail

# lmbench WASM build helper for lind-wasm-apps
#
# High level:
#   1) Reuse the merged `libc.a` augmentation unless its inputs changed.
#   2) Rebuild lmbench only when the toolchain or build inputs changed.
#   3) Stage outputs under build/lmbench/bin (canonical location).
#   4) Treat optimization and precompilation as optional, heavier follow-up
#      work instead of always doing them on the default path.
#   5) Mirror canonical outputs to build/bin/lmbench/wasm32-wasi (legacy path).

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"

if [[ -z "${LIND_WASM_ROOT:-}" ]]; then
  LIND_WASM_ROOT="$(cd "$REPO_ROOT/.." && pwd)"
fi

APPS_BUILD="$REPO_ROOT/build"
APPS_OVERLAY="$APPS_BUILD/sysroot_overlay"
MERGED_SYSROOT="$APPS_BUILD/sysroot_merged"
APPS_LMBENCH_CANON_ROOT="$APPS_BUILD/lmbench/bin"
APPS_LMBENCH_LEGACY_ROOT="$APPS_BUILD/bin/lmbench"
TOOL_ENV="$APPS_BUILD/.toolchain.env"
MAX_WASM_MEMORY="${MAX_WASM_MEMORY:-67108864}"
ENABLE_WASI_THREADS="${ENABLE_WASI_THREADS:-1}"
WASM_OPT="${WASM_OPT:-$LIND_WASM_ROOT/tools/binaryen/bin/wasm-opt}"
LIND_BOOT="${LIND_BOOT:-$LIND_WASM_ROOT/build/lind-boot}"
JOBS="${JOBS:-$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN || echo 4)}"
##################
# These flags expose the speed/cleanliness tradeoff directly at the script
# layer. The default is a faster developer-oriented path, but the caller can
# still request a scratch rebuild or the full artifact set without changing
# lmbench's own makefiles.
##################
ARTIFACT_MODE="${ARTIFACT_MODE:-fast}"
FORCE_CLEAN="${FORCE_CLEAN:-0}"
FORCE_CONFIGURE="${FORCE_CONFIGURE:-0}"
CACHE_REVISION="2"
STATE_DIR="$APPS_BUILD/.lmbench_state"
LIBC_SIG_FILE="$STATE_DIR/libc.sig"
BUILD_SIG_FILE="$STATE_DIR/build.sig"
COMB_DIR="$APPS_BUILD/.lmbench_libc_objs"
##################
# The state directory stores signatures for the two expensive phases in this
# script: rebuilding the combined libc archive and deciding whether a clean
# lmbench rebuild is actually necessary. Without that, repeated invocations
# pay the same cost even when the inputs are unchanged.
##################
mkdir -p "$STATE_DIR"

if [[ -r "$TOOL_ENV" ]]; then
  # shellcheck disable=SC1090
  . "$TOOL_ENV"
else
  echo "[lmbench] ERROR: missing toolchain env '$TOOL_ENV' (run 'make preflight' first)" >&2
  exit 1
fi

: "${CLANG:?missing CLANG in $TOOL_ENV}"
: "${AR:?missing AR in $TOOL_ENV}"
: "${RANLIB:?missing RANLIB in $TOOL_ENV}"

##################
# These helpers exist for two different reasons: hashing makes the cache
# decisions deterministic, and the bounded-background helpers let the optional
# post-processing loops run concurrently without spawning an unbounded number
# of `wasm-opt` or `lind-boot` jobs.
##################
supports_cflag() {
  local flag="$1"
  printf 'int main(void){return 0;}\n' | \
    "$CLANG" --target=wasm32-unknown-wasi --sysroot="$MERGED_SYSROOT" \
    $flag -x c -c -o /dev/null - >/dev/null 2>&1
}

hash_file() {
  local path="$1"
  if [[ -e "$path" ]]; then
    sha256sum "$path" | awk '{print $1}'
  else
    echo "missing"
  fi
}

list_hashes() {
  local item
  for item in "$@"; do
    if [[ -e "$item" ]]; then
      printf '%s %s\n' "$(sha256sum "$item" | awk '{print $1}')" "$item"
    else
      printf 'missing %s\n' "$item"
    fi
  done
}

run_limited() {
  local limit="$1"
  shift
  while (( $(jobs -rp | wc -l) >= limit )); do
    wait -n || true
  done
  "$@" &
}

wait_for_background_jobs() {
  local rc=0
  while [[ -n "$(jobs -rp)" ]]; do
    if ! wait -n; then
      rc=1
    fi
  done
  return "$rc"
}

run_wasm_opt_replace() {
  local src="$1"
  local raw="$2"
  local out="$3"
  local tmp="${out}.tmp"
  cp "$src" "$raw"
  if "$WASM_OPT" --epoch-injection --asyncify --debuginfo -O2 "$raw" -o "$tmp"; then
    mv "$tmp" "$out"
  else
    rm -f "$tmp"
    return 1
  fi
}

mirror_legacy_outputs() {
  local legacy_out="$APPS_LMBENCH_LEGACY_ROOT/wasm32-wasi"
  rm -rf "$legacy_out"
  mkdir -p "$legacy_out"
  cp -a "$OUT_DIR/." "$legacy_out/"
  echo "[lmbench] mirrored artifacts to legacy path: $legacy_out"
}


BASE_LIBC="$MERGED_SYSROOT/lib/wasm32-wasi/libc.a"
TIRPC_MERGE_DIR="$APPS_OVERLAY/usr/lib/wasm32-wasi/merge_tmp"

if [[ ! -f "$BASE_LIBC" ]]; then
  echo "[lmbench] ERROR: merged sysroot libc.a not found at: $BASE_LIBC" >&2
  echo "[lmbench] Hint: run 'make merge-sysroot' before 'make lmbench'." >&2
  exit 1
fi

if [[ ! -d "$TIRPC_MERGE_DIR" ]]; then
  echo "[lmbench] ERROR: expected libtirpc .o dir '$TIRPC_MERGE_DIR' not found" >&2
  echo "[lmbench] Hint: did 'make libtirpc' succeed?" >&2
  exit 1
fi

shopt -s nullglob
tirpc_objs=("$TIRPC_MERGE_DIR"/*.o)
shopt -u nullglob

if (( ${#tirpc_objs[@]} == 0 )); then
  echo "[lmbench] ERROR: no libtirpc .o files under $TIRPC_MERGE_DIR" >&2
  exit 1
fi

##################
# lmbench is special in this repo because it needs a libc archive that includes
# the libtirpc objects. Rebuilding that archive every time is unnecessary, so
# we fingerprint the base libc plus the libtirpc merge inputs and only recreate
# the combined archive when one of those inputs actually changed.
##################
libc_signature() {
  {
    echo "rev=$CACHE_REVISION"
    echo "base=$(hash_file "$BASE_LIBC")"
    echo "ar=$AR"
    echo "ranlib=$RANLIB"
    list_hashes "${tirpc_objs[@]}"
  } | sha256sum | awk '{print $1}'
}

LM_BENCH_BIN_DIR="$REPO_ROOT/lmbench/bin/wasm32-wasi"
mkdir -p "$LM_BENCH_BIN_DIR"

REAL_CC="$CLANG --target=wasm32-unknown-wasi --sysroot=$MERGED_SYSROOT"
CFLAGS="-DNO_PORTMAPPER -O2 -g -I$MERGED_SYSROOT/include -I$MERGED_SYSROOT/include/wasm32-wasi -I$MERGED_SYSROOT/include/tirpc"
LDFLAGS_WASM=(
  "-Wl,--import-memory,--export-memory,--max-memory=${MAX_WASM_MEMORY},--export=__stack_pointer,--export=__stack_low,--export=__tls_base"
  "-L$MERGED_SYSROOT/lib/wasm32-wasi"
  "-L$MERGED_SYSROOT/usr/lib/wasm32-wasi"
)

if [[ "$ENABLE_WASI_THREADS" == "1" ]]; then
  thread_flag="-mthread-model=posix"
  if ! supports_cflag "$thread_flag"; then
    thread_flag=""
  fi
  if supports_cflag "-pthread" && supports_cflag "-matomics" && supports_cflag "-mbulk-memory"; then
    CFLAGS+=" -pthread -matomics -mbulk-memory"
    if [[ -n "$thread_flag" ]]; then
      CFLAGS+=" $thread_flag"
    else
      echo "[lmbench] WARNING: clang does not support -mthread-model=posix; skipping thread model flag."
    fi
    LDFLAGS_WASM+=("-Wl,--shared-memory")
  else
    echo "[lmbench] WARNING: clang does not support wasi-threads flags; disabling shared memory."
    ENABLE_WASI_THREADS="0"
  fi
fi

LDFLAGS="${LDFLAGS_WASM[*]}"
LDLIBS="-ltirpc -lm"

##################
# The compile step has its own signature because a clean rebuild only makes
# sense when the toolchain, memory/threading configuration, or lmbench make
# inputs changed. This keeps the script from doing `make clean` on every run.
##################
build_signature() {
  {
    echo "rev=$CACHE_REVISION"
    echo "makefile=$(hash_file "$REPO_ROOT/lmbench/src/Makefile")"
    echo "clang=$CLANG"
    echo "merged=$MERGED_SYSROOT"
    echo "memory=$MAX_WASM_MEMORY"
    echo "threads=$ENABLE_WASI_THREADS"
    echo "cflags=$CFLAGS"
    echo "ldflags=$LDFLAGS"
    echo "ldlibs=$LDLIBS"
  } | sha256sum | awk '{print $1}'
}

mkdir -p "$APPS_BUILD/lib"
COMBINED_LIBC="$APPS_BUILD/lib/libc.a"
current_libc_sig="$(libc_signature)"

##################
# This block is intentionally separate from the main build so it is clear that
# "rebuild combined libc" and "rebuild lmbench" are different invalidation
# problems. If the libc inputs are unchanged, we can safely reuse the existing
# archive and just make sure the merged sysroot copy matches it.
##################
if [[ "$FORCE_CLEAN" == "1" || "$FORCE_CONFIGURE" == "1" || ! -f "$LIBC_SIG_FILE" || "$(cat "$LIBC_SIG_FILE" 2>/dev/null || true)" != "$current_libc_sig" || ! -f "$COMBINED_LIBC" ]]; then
  rm -rf "$COMB_DIR"
  mkdir -p "$COMB_DIR"

  echo "[lmbench] extracting base libc objects..."
  (
    cd "$COMB_DIR"
    "$AR" x "$BASE_LIBC"
  )

  echo "[lmbench] adding libtirpc objects from $TIRPC_MERGE_DIR..."
  cp "${tirpc_objs[@]}" "$COMB_DIR/"

  echo "[lmbench] creating combined libc.a -> $COMBINED_LIBC"
  (
    cd "$COMB_DIR"
    "$AR" rcs "$COMBINED_LIBC" ./*.o
    "$RANLIB" "$COMBINED_LIBC" || true
  )

  printf '%s\n' "$current_libc_sig" >"$LIBC_SIG_FILE"
else
  echo "[lmbench] reusing combined libc.a"
fi

if ! cmp -s "$COMBINED_LIBC" "$BASE_LIBC"; then
  cp "$COMBINED_LIBC" "$BASE_LIBC"
fi

##################
# This is the second cache decision: even if the combined libc is current, the
# lmbench binaries themselves may or may not need a clean rebuild. We only ask
# for `make clean` when the build signature changed or the expected output tree
# is missing.
##################
current_build_sig="$(build_signature)"
need_clean=0
if [[ "$FORCE_CLEAN" == "1" || "$FORCE_CONFIGURE" == "1" ]]; then
  need_clean=1
fi
if [[ ! -f "$BUILD_SIG_FILE" ]] || [[ "$(cat "$BUILD_SIG_FILE" 2>/dev/null || true)" != "$current_build_sig" ]]; then
  need_clean=1
fi
if [[ ! -d "$LM_BENCH_BIN_DIR" ]]; then
  need_clean=1
fi

echo "[lmbench] building suite with REAL_CC='$REAL_CC'"
(
  cd "$REPO_ROOT/lmbench/src"

  if (( need_clean )); then
    echo "[lmbench] cleaning previous outputs..."
    make clean || true
  else
    echo "[lmbench] reusing previous build outputs..."
  fi

  make -j"$JOBS" \
    OS="wasm32-wasi" \
    O="../bin/wasm32-wasi" \
    CC="$REAL_CC" \
    CFLAGS="$CFLAGS" \
    CPPFLAGS="-I$MERGED_SYSROOT/include/tirpc" \
    LDFLAGS="$LDFLAGS" \
    LDLIBS="$LDLIBS" \
    all
)
printf '%s\n' "$current_build_sig" >"$BUILD_SIG_FILE"

mkdir -p "$APPS_LMBENCH_CANON_ROOT"
OUT_DIR="$APPS_LMBENCH_CANON_ROOT"

echo "[lmbench] staging binaries from $LM_BENCH_BIN_DIR -> $OUT_DIR"

if [[ ! -d "$LM_BENCH_BIN_DIR" ]]; then
  echo "[lmbench] ERROR: expected lmbench output dir '$LM_BENCH_BIN_DIR' not found" >&2
  exit 1
fi

##################
# The original script removed the entire staging directory unconditionally.
# Here we only clear staged files inside it, which is enough to keep the
# outputs accurate without paying for more directory churn than necessary.
##################
mkdir -p "$OUT_DIR"
find "$OUT_DIR" -maxdepth 1 -type f \( -name '*.wasm' -o -name '*.raw.wasm' -o -name '*.opt.wasm' -o -name '*.cwasm' -o -name '*.opt.wasm.cwasm' -o -name '*' \) -delete

shopt -s nullglob
bin_files=("$LM_BENCH_BIN_DIR"/*)
shopt -u nullglob

have_files=0
for f in "${bin_files[@]}"; do
  case "$f" in
    *.o|*.a|*.cwasm|*.opt.wasm|*.opt.wasm.cwasm) continue ;;
  esac
  cp "$f" "$OUT_DIR/"
  have_files=1
done

if (( have_files == 0 )); then
  echo "[lmbench] ERROR: no non-.o binaries found in $LM_BENCH_BIN_DIR" >&2
  exit 1
fi

echo "[lmbench] staged binaries under $OUT_DIR"

##################
# Lind expects the runtime module to include the epoch-injection transform, so
# the staged raw lmbench binaries are best understood as intermediate files,
# not the final runnable artifacts. We therefore preserve the raw link output
# under a debug-friendly `.raw.wasm` name and rewrite the traditional lmbench
# program name to be the runnable optimized module.
##################
if [[ ! -x "$WASM_OPT" ]]; then
  echo "[lmbench] ERROR: wasm-opt not found at '$WASM_OPT'; cannot produce runnable Lind artifacts."
  exit 1
fi

echo "[lmbench] post-processing staged binaries under $OUT_DIR ..."

shopt -s nullglob
stage_bins=("$OUT_DIR"/*)
shopt -u nullglob

for f in "${stage_bins[@]}"; do
  case "$f" in
    *.o|*.a|*.cwasm|*.opt.wasm|*.opt.wasm.cwasm|*.raw.wasm) continue ;;
  esac
  base="$(basename -- "$f")"
  RAW_OUT="$OUT_DIR/${base}.raw.wasm"
  run_limited "$JOBS" run_wasm_opt_replace "$f" "$RAW_OUT" "$f"
done

wait_for_background_jobs

##################
# Full mode still preserves the historical `.opt.wasm` names expected by some
# tooling, but those are now aliases of the runnable default outputs rather
# than the only runnable versions. Fast mode stops after creating the optimized
# default program names.
##################
if [[ "$ARTIFACT_MODE" != "full" ]]; then
  echo "[lmbench] artifact mode is fast; keeping default lmbench program names runnable and skipping cwasm generation."
  mirror_legacy_outputs
  exit 0
fi

shopt -s nullglob
optimized_bins=("$OUT_DIR"/*)
shopt -u nullglob
for f in "${optimized_bins[@]}"; do
  case "$f" in
    *.o|*.a|*.cwasm|*.opt.wasm|*.opt.wasm.cwasm|*.raw.wasm) continue ;;
  esac
  cp "$f" "$OUT_DIR/$(basename -- "$f").opt.wasm"
done

##################
# In full mode we still keep the richer artifact set, but we do not need to
# run the post-processing strictly one file at a time. The bounded worker loop
# lets us use `JOBS` without oversubscribing wildly.
##################
if [[ -x "$LIND_BOOT" ]]; then
  echo "[lmbench] generating cwasm via lind-boot --precompile..."
  shopt -s nullglob
  opt_files=("$OUT_DIR"/*.opt.wasm)
  shopt -u nullglob
  for w in "${opt_files[@]}"; do
    run_limited "$JOBS" "$LIND_BOOT" --precompile "$w"
  done
  wait_for_background_jobs

  for w in "${opt_files[@]}"; do
    OPT_CWASM="${w%.wasm}.cwasm"
    CLEAN_CWASM="${OPT_CWASM/.opt/}"
    if [[ "$OPT_CWASM" != "$CLEAN_CWASM" && -f "$OPT_CWASM" ]]; then
      mv "$OPT_CWASM" "$CLEAN_CWASM"
    fi
  done
else
  echo "[lmbench] NOTE: lind-boot not found at '$LIND_BOOT'; skipping cwasm generation."
fi

mirror_legacy_outputs
echo "[lmbench] post-processing complete."