Make per-request RNGs deterministic by default (reproducible QPS)

benchpress/config/jobs.yml

@@ -502,6 +502,7 @@
     - '--rpc-fanout-scale={rpc_fanout_scale}'
     - '--server-zstd={server_zstd}'
     - '--sla-p95-ms={sla_p95_ms}'
+    - '--depth={depth}'
     - '{extra_args}'
   vars:
     # Hardcode num_instances=1: this job exists specifically for single-instance
@@ -512,7 +513,12 @@
     - 'io_mean=200'
     - 'workload=dlrm'
     - 'dlrm_model=models/dlrm_small.pt'
-    - 'dlrm_batch_size=64'
+    # RMSE-winning recipe (2026-08, supersedes t43 c7): minimizes all-metrics
+    # RMSE vs prod multifeed_aggregator (uArch + instruction mix + hot-function)
+    # on BGM & Grace, and preserves gen-over-gen ranking across all 6 LSSTs.
+    # Pairs with the integer-domain feature-extractor helpers + memory-streaming
+    # stride-sweep (N=16, code default) earlier in this stack.
+    - 'dlrm_batch_size=32'
     - 'dlrm_threads=1'
     - 'dlrm_inferences=1'
     - 'client_side_features=0'
@@ -521,19 +527,25 @@
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
-    - 'feature_complexity=5'
+    - 'feature_complexity=8'
     - 'num_stories=400'
-    - 'extractors_per_story=280'
+    - 'extractors_per_story=240'
     - 'story_processors_per_story=2'
-    - 'stories_per_processor_pass=100'
+    - 'stories_per_processor_pass=150'
     - 'silesia_dir=silesia'
     - 'stories_per_request=10'
     - 'mock_tls=1'
-    - 'mock_zstd_frac=0.75'
+    - 'mock_zstd_frac=0.9'
     - 'mock_keepalive_interval_ms=200'
-    - 'rpc_fanout_scale=0.05'
-    - 'server_zstd=0'
+    - 'rpc_fanout_scale=0.10'
+    - 'server_zstd=1'
     - 'sla_p95_ms=700'
+    # Driver pipeline depth (max outstanding requests per driver connection).
+    # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA
+    # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often
+    # needed on high-perf ARM cores. With adaptive depth on (the default), this is
+    # the starting floor the peak search raises from; see README "Driver depth".
+    - 'depth=1'
     - 'extra_args='
   hooks:
     - hook: cpu-mpstat
@@ -592,18 +604,20 @@
     - '--rpc-fanout-scale={rpc_fanout_scale}'
     - '--server-zstd={server_zstd}'
     - '--sla-p95-ms={sla_p95_ms}'
+    - '--depth={depth}'
     - '{extra_args}'
   vars:
     - 'num_instances=-1'
     - 'io_dist=fixed'
     - 'io_mean=200'
     - 'workload=dlrm'
     - 'dlrm_model=models/dlrm_small.pt'
-    # t43 c7: bs=64 is the balanced choice across both gen-over-gen
-    # correlation (BGM/CPL = 4.25× → 2nd best of 12 cells) and prod
-    # hot-function gap (CPL 4th, BGM 5th). bs=32 wins peak QPS alone but
-    # breaks correlation (BGM stretches to 4.75× CPL).
-    - 'dlrm_batch_size=64'
+    # RMSE-winning recipe (2026-08, supersedes t43 c7): minimizes all-metrics
+    # RMSE vs prod (uArch + instruction mix + hot-function) on BGM & Grace.
+    # bs=32 (with the integer-domain extractor helpers + stride-sweep earlier in
+    # this stack) raises SIMD toward prod and, at eps=240, preserves gen-over-gen
+    # ranking across all 6 LSSTs (validated 2026-08-11).
+    - 'dlrm_batch_size=32'
     - 'dlrm_threads=1'
     - 'dlrm_inferences=1'
     - 'client_side_features=0'
@@ -612,29 +626,34 @@
     - 'client_feature_seed=42'
     - 'client_num_dense=13'
     - 'client_num_sparse=26'
-    - 'feature_complexity=5'
-    # 400 x 280 = 112K extractor calls/req. Combined with the scaled-up
-    # story-processor pipeline (below) this matches prod's FE:story CPU
-    # ratio of ~1.3:1 (multifeed: FE 29%, story 22% on BGM Strobelight).
+    - 'feature_complexity=8'
+    # 400 x 240 = 96K extractor calls/req. eps=240 lands Ranking-FeatureExtraction
+    # near prod on BGM (~22% vs 22.8%); the integer-domain helpers made each call
+    # cheaper, so eps rose from 280 to hold the FE share after batch32/zstd/rpc grew.
     - 'num_stories=400'
-    - 'extractors_per_story=280'
-    # Story-processor pipeline scaled 8x from t32 first attempt (was
-    # 1 pass/story * 25 stories/pass = 25 cmpns/story → 1.1% Ranking-Story
-    # on BGM, vs prod 9.3%). Bumped to 2 passes * 100 stories = 200
-    # comparisons/story, target ~9% Ranking-Story share.
+    - 'extractors_per_story=240'
+    # Story-processor pipeline: 2 passes * 150 stories = 300 comparisons/story,
+    # target ~9-10% Ranking-Story share (≈ prod 9.3% on BGM).
     - 'story_processors_per_story=2'
-    - 'stories_per_processor_pass=100'
+    - 'stories_per_processor_pass=150'
     - 'silesia_dir=silesia'
     - 'stories_per_request=10'
-    # t43 c7: TLS+keepalive on mock_services, 75% of channels compress,
-    # rpc_fanout_scale=0.05 (~188 RPCs/session). Server-side ZSTD off
-    # (mock-side ZSTD provides the prod-realistic Compression share).
+    # TLS+keepalive on mock_services; mock_zstd_frac=0.9 + server_zstd=1 land
+    # Compression ≈ prod (BGM ~10% vs 10.7%); rpc_fanout_scale=0.10 lands
+    # RPC-AsyncIO ≈ prod (~15%). DRIVER_TLS stays off (default): it overshoots
+    # Encryption without moving throughput.
     - 'mock_tls=1'
-    - 'mock_zstd_frac=0.75'
+    - 'mock_zstd_frac=0.9'
     - 'mock_keepalive_interval_ms=200'
-    - 'rpc_fanout_scale=0.05'
-    - 'server_zstd=0'
+    - 'rpc_fanout_scale=0.10'
+    - 'server_zstd=1'
     - 'sla_p95_ms=700'
+    # Driver pipeline depth (max outstanding requests per driver connection).
+    # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA
+    # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often
+    # needed on high-perf ARM cores. With adaptive depth on (the default), this is
+    # the starting floor the peak search raises from; see README "Driver depth".
+    - 'depth=1'
     - 'extra_args='
   hooks:
     - hook: cpu-mpstat
@@ -695,6 +714,7 @@
     - '--rpc-fanout-scale={rpc_fanout_scale}'
     - '--server-zstd={server_zstd}'
     - '--sla-p95-ms={sla_p95_ms}'
+    - '--depth={depth}'
     - '{extra_args}'
   vars:
     - 'num_instances=-1'
@@ -735,6 +755,12 @@
     - 'rpc_fanout_scale=0.05'
     - 'server_zstd=0'
     - 'sla_p95_ms=700'
+    # Driver pipeline depth (max outstanding requests per driver connection).
+    # Default 1. Raise (e.g. 2) if the final phase saturates neither CPU nor SLA
+    # latency (final p95 well below sla_p95_ms while CPU util < ~90%) — often
+    # needed on high-perf ARM cores. With adaptive depth on (the default), this is
+    # the starting floor the peak search raises from; see README "Driver depth".
+    - 'depth=1'
     - 'extra_args='
   hooks:
     - hook: copymove

packages/feedsim/README.md

@@ -55,7 +55,7 @@ taskset -c 0-15 ./benchpress_cli.py install feedsim_dlrm
 Unlike FeedSim v1 which spawns a new FeedSim instance per 100 CPU cores,
 `feedsim_dlrm` is pinned to **one FeedSim instance per host** because the
 redesigned threading model in FeedSim v2 has overcome the scalability issue
-on ultra-high-core-count CPUs and ARM CPUs. 
+on ultra-high-core-count CPUs and ARM CPUs.
 
 The runner searches for the QPS that keeps 95th-percentile end-to-end
 latency at or below **700 ms**. When it converges it runs a final 5-minute
@@ -64,7 +64,7 @@ counters) during that final window. We expect the total wall-clock runtime
 to be around 30 minutes.
 
 Please make sure to turn CPU turbo-boost on before starting, or FeedSim may
-fail to converge and report a low QPS. 
+fail to converge and report a low QPS.
 
 ### Result report
 
@@ -211,7 +211,7 @@ feedsim server, driver and mock_services instances.
 
 2. Use the `feedsim_autoscale_dlrm` job. This autoscale job will spawn `ceil(nproc / 100)`
 FeedSim instances, each pinned to its own CPU range via `taskset`, plus one driver
-and one `mock_services` process per instance (also `taskset`-isolated). For example: 
+and one `mock_services` process per instance (also `taskset`-isolated). For example:
 ```
 ./benchpress_cli.py run feedsim_autoscale_dlrm
 ```
@@ -220,6 +220,41 @@ In multi-instance mode, the overall QPS is the sum across all instances. and the
 average latency will be the average of p95 latency values observed across all
 instances.
 
+### Driver depth (fixing CPU/latency under-utilization)
+
+The `depth` parameter sets the driver's pipeline depth — the maximum number of
+outstanding (in-flight) requests per driver connection. The driver's total
+offered concurrency is `driver_threads × connections × depth`, so with the
+default `depth=1` the driver can cap the achievable load below what the server
+can actually handle.
+
+**Increase `depth` beyond 1 when the final benchmarking phase saturates neither
+CPU nor latency** — i.e. the final achieved p95 latency is well below the SLA
+limit (`sla_p95_ms`, default 700 ms) *and* the CPU utilization during the final
+5-minute benchmarking phase is less than ~90%. In that situation the reported QPS
+is limited by driver concurrency rather than by the server, so it understates the
+hardware's true capacity. Raising `depth` (start with `2`) lets the driver offer
+more concurrent load until the server becomes the bottleneck — either CPU-bound
+(~100% utilization) or latency-bound (p95 ≈ SLA). **This is likely necessary on
+high-performance ARM cores** (e.g. NVIDIA Grace), which can otherwise sit at
+80–90% CPU with p95 far below the SLA at `depth=1`.
+
+```
+# Force driver depth 2
+./benchpress_cli.py run feedsim_dlrm -i '{"depth": 2}'
+```
+
+There is also an **adaptive depth** mechanism (on by default) that raises the
+depth automatically during the peak-finding stage until the server saturates
+(system CPU ≥ 95% or p95 ≥ SLA). It catches *severe* under-utilization early, but
+because it evaluates saturation on the high-load peak/search probes rather than
+on the final SLA-converged operating point, it **may not catch all
+under-utilization cases**. If you still observe under-utilization in the final
+result (low CPU + p95 well under SLA), increase `depth` manually as above. When
+adaptive depth is on, a manually-set `depth` acts as the starting floor the
+adaptive search raises from; to pin an exact fixed depth, also set the
+`FEEDSIM_ADAPTIVE_DEPTH_MAX=0` environment variable to disable adaptive search.
+
 ### Other parameters
 
 This section lists additional parameters in `feedsim_dlrm` benchmark. These parameters
@@ -233,6 +268,7 @@ Job-level parameters (can be passed via `-i` flag in Benchpress CLI):
 |---|---|---|
 | `num_instances` | Number of FeedSim instances to run in parallel. Defaults to 1 in `feedsim_dlrm`; set to -1 to autoscale for `feedsim_autoscale_dlrm`. | `1` |
 | `sla_p95_ms` | SLA target in ms. The runner searches for the highest QPS keeping p95 ≤ this. | `700` |
+| `depth` | Driver pipeline depth (max outstanding requests per connection; total in-flight = `driver_threads × connections × depth`). Raise (e.g. `2`) when the final phase saturates neither CPU nor latency — often needed on high-perf ARM. See [Driver depth](#driver-depth-fixing-cpulatency-under-utilization). | `1` |
 | `io_dist` | I/O latency distribution: `fixed`, `exponential`, or `lognormal`. | `fixed` |
 | `io_mean` | Mean I/O latency in ms. | `200` |
 | `workload` | Ranking workload: `pagerank` or `dlrm`. `dlrm` is v2. | `dlrm` |

packages/feedsim/install_feedsim.sh

@@ -10,7 +10,20 @@ FEEDSIM_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd -P)
 BENCHPRESS_ROOT="$(readlink -f "$FEEDSIM_ROOT/../..")"
 FEEDSIM_ROOT_SRC="${BENCHPRESS_ROOT}/benchmarks/feedsim"
 FEEDSIM_THIRD_PARTY_SRC="${FEEDSIM_ROOT_SRC}/third_party"
-LIBTORCH_VERSION="2.8.0"
+LIBTORCH_VERSION="${LIBTORCH_VERSION:-2.13.0}"
+# When 1, fetch LibTorch by extracting it from the prebuilt torch CPU wheel
+# (download.pytorch.org/whl/cpu) instead of the libtorch-shared-with-deps zip.
+# Required for LibTorch >=2.9 (2.13.0 and later publish a wheel but no
+# standalone zip); harmless for older versions. Default 1 pairs with the
+# LIBTORCH_VERSION=2.13.0 default so the out-of-box install works without
+# additional env overrides.
+LIBTORCH_FROM_WHEEL="${LIBTORCH_FROM_WHEEL:-1}"
+# Dependency versions are env-overridable so experiments can bump them without
+# forking this script; defaults reproduce the v2 baseline exactly.
+JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}"
+LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}"
+# Export so the aarch64 sub-installer (dispatched below) inherits the pins.
+export LIBTORCH_VERSION LIBTORCH_FROM_WHEEL FEEDSIM_JEMALLOC_VERSION FEEDSIM_LIBEVENT_VERSION
 DLRM_MODEL_URL="https://github.com/facebookresearch/DCPerf-datasets/releases/download/feedsim-dlrm/dlrm_small.tar.gz"
 echo "BENCHPRESS_ROOT is ${BENCHPRESS_ROOT}"
 
@@ -45,7 +58,7 @@ dnf install -y bc ninja-build flex bison git texinfo binutils-devel \
     libsodium-devel libunwind-devel bzip2-devel double-conversion-devel \
     libzstd-devel lz4-devel xz-devel snappy-devel libtool bzip2 openssl-devel \
     zlib-devel libdwarf libdwarf-devel libaio-devel libatomic patch jq \
-    xxhash xxhash-devel unzip rsync liburing-devel
+    xxhash xxhash-devel unzip rsync liburing-devel python3-pip
 
 # Creates feedsim directory under benchmarks/
 mkdir -p "${BENCHPRESS_ROOT}/benchmarks/feedsim"
@@ -178,30 +191,30 @@ else
 fi
 
 # Installing JEMalloc
-if ! [ -d "jemalloc-5.3.0" ]; then
-    wget "https://github.com/jemalloc/jemalloc/releases/download/5.3.0/jemalloc-5.3.0.tar.bz2"
-    bunzip2 "jemalloc-5.3.0.tar.bz2"
-    tar -xvf "jemalloc-5.3.0.tar"
-    cd "jemalloc-5.3.0"
+if ! [ -d "jemalloc-${JEMALLOC_VERSION}" ]; then
+    wget "https://github.com/jemalloc/jemalloc/releases/download/${JEMALLOC_VERSION}/jemalloc-${JEMALLOC_VERSION}.tar.bz2"
+    bunzip2 "jemalloc-${JEMALLOC_VERSION}.tar.bz2"
+    tar -xvf "jemalloc-${JEMALLOC_VERSION}.tar"
+    cd "jemalloc-${JEMALLOC_VERSION}"
     ./configure --enable-prof --enable-prof-libunwind
     make -j"$(nproc)"
     make install
     cd ../
 else
-    msg "[SKIPPED] jemalloc-5.3.0"
+    msg "[SKIPPED] jemalloc-${JEMALLOC_VERSION}"
 fi
 
 # Installing libevent
-if ! [ -d "libevent-2.1.12-stable" ]; then
-    wget "https://github.com/libevent/libevent/releases/download/release-2.1.12-stable/libevent-2.1.12-stable.tar.gz"
-    tar -xzf "libevent-2.1.12-stable.tar.gz"
-    cd "libevent-2.1.12-stable"
+if ! [ -d "libevent-${LIBEVENT_VERSION}" ]; then
+    wget "https://github.com/libevent/libevent/releases/download/release-${LIBEVENT_VERSION}/libevent-${LIBEVENT_VERSION}.tar.gz"
+    tar -xzf "libevent-${LIBEVENT_VERSION}.tar.gz"
+    cd "libevent-${LIBEVENT_VERSION}"
     ./configure
     make -j"$(nproc)"
     make install
     cd ../
 else
-    msg "[SKIPPED] libevent-2.1.12-stable"
+    msg "[SKIPPED] libevent-${LIBEVENT_VERSION}"
 fi
 
 msg "Installing third-party dependencies ... DONE"
@@ -218,12 +231,33 @@ else
 fi
 
 if ! [ -d "libtorch" ]; then
-    msg "Downloading LibTorch ${LIBTORCH_VERSION}..."
-    wget "${LIBTORCH_URL}" -O libtorch.zip
-    msg "Extracting LibTorch..."
-    unzip -q libtorch.zip
-    rm libtorch.zip
-    msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch"
+    if [ "${LIBTORCH_FROM_WHEEL}" = "1" ]; then
+        # Extract LibTorch from the prebuilt torch CPU wheel. The wheel's
+        # torch/ dir has the same lib/ include/ share/cmake/Torch/ layout as
+        # the standalone libtorch zip, so we just rename it to libtorch/.
+        msg "Downloading LibTorch ${LIBTORCH_VERSION} from torch CPU wheel..."
+        # pip on the box (3.9, or an internal stale mirror) can't see the cp310
+        # 2.13 wheels, so resolve the wheel href straight from the PEP-503 index
+        # and wget it. The C++ libtorch inside (torch/lib, torch/share/cmake) is
+        # Python-version independent, so the cp310 wheel is fine for our C++ link.
+        WHEEL_HREF="$(curl -s "https://download.pytorch.org/whl/cpu/torch/" \
+            | grep -oE "https://[^\"]*torch-${LIBTORCH_VERSION}[^\"]*cp310-cp310-manylinux_2_28_x86_64\.whl" \
+            | head -1)"
+        [ -n "${WHEEL_HREF}" ] || die "Could not find torch ${LIBTORCH_VERSION} x86_64 wheel in index"
+        msg "Wheel: ${WHEEL_HREF}"
+        wget "${WHEEL_HREF}" -O torch.whl
+        unzip -q torch.whl -d ./_torch_whl_x
+        mv ./_torch_whl_x/torch libtorch
+        rm -rf ./_torch_whl_x torch.whl
+        msg "LibTorch ${LIBTORCH_VERSION} extracted from wheel to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch"
+    else
+        msg "Downloading LibTorch ${LIBTORCH_VERSION}..."
+        wget "${LIBTORCH_URL}" -O libtorch.zip
+        msg "Extracting LibTorch..."
+        unzip -q libtorch.zip
+        rm libtorch.zip
+        msg "LibTorch installed to ${FEEDSIM_THIRD_PARTY_SRC}/libtorch"
+    fi
 else
     msg "[SKIPPED] LibTorch already installed"
 fi

packages/feedsim/install_feedsim_aarch64.sh

@@ -182,11 +182,15 @@ else
     msg "[SKIPPED] glog-${DEP_GFLAGS_VERSION}"
 fi
 
-DEP_JEMALLOC_VERSION="5.3.0"
+DEP_JEMALLOC_VERSION="${FEEDSIM_JEMALLOC_VERSION:-5.3.0}"
 # Installing JEMalloc
 if ! [ -d "jemalloc-${DEP_JEMALLOC_VERSION}" ]; then
     wget "https://github.com/jemalloc/jemalloc/releases/download/${DEP_JEMALLOC_VERSION}/jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" -O "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2"
-    verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa"
+    if [ "${DEP_JEMALLOC_VERSION}" = "5.3.0" ]; then
+        verify_checksum "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2" "2db82d1e7119df3e71b7640219b6dfe84789bc0537983c3b7ac4f7189aecfeaa"
+    else
+        msg "[WARN] no pinned checksum for jemalloc ${DEP_JEMALLOC_VERSION}; skipping verify (official github release over https)"
+    fi
     bunzip2 "jemalloc-${DEP_JEMALLOC_VERSION}.tar.bz2"
     tar -xvf "jemalloc-${DEP_JEMALLOC_VERSION}.tar"
     cd "jemalloc-${DEP_JEMALLOC_VERSION}"
@@ -198,11 +202,15 @@ else
     msg "[SKIPPED] jemalloc-${DEP_JEMALLOC_VERSION}"
 fi
 
-DEP_LIBEVENT_VERSION="2.1.12-stable"
+DEP_LIBEVENT_VERSION="${FEEDSIM_LIBEVENT_VERSION:-2.1.12-stable}"
 # Installing libevent
 if ! [ -d "libevent-${DEP_LIBEVENT_VERSION}" ]; then
     wget "https://github.com/libevent/libevent/releases/download/release-${DEP_LIBEVENT_VERSION}/libevent-${DEP_LIBEVENT_VERSION}.tar.gz" -O "libevent-${DEP_LIBEVENT_VERSION}.tar.gz"
-    verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb"
+    if [ "${DEP_LIBEVENT_VERSION}" = "2.1.12-stable" ]; then
+        verify_checksum "libevent-${DEP_LIBEVENT_VERSION}.tar.gz" "92e6de1be9ec176428fd2367677e61ceffc2ee1cb119035037a27d346b0403bb"
+    else
+        msg "[WARN] no pinned checksum for libevent ${DEP_LIBEVENT_VERSION}; skipping verify (official github release over https)"
+    fi
     tar -xzf "libevent-${DEP_LIBEVENT_VERSION}.tar.gz"
     cd "libevent-${DEP_LIBEVENT_VERSION}"
     ./configure
@@ -242,9 +250,14 @@ if ! [ -d "libtorch" ]; then
     export PATH="${CONDA_DIR}/bin:${PATH}"
 
     # Install CPU-only PyTorch via pip — this is the only reliable way to get
-    # CPU-only libtorch on aarch64
-    msg "Installing PyTorch CPU-only via pip..."
-    pip install torch --index-url https://download.pytorch.org/whl/cpu
+    # CPU-only libtorch on aarch64. LIBTORCH_VERSION (env) pins the version;
+    # unset reproduces the v2 baseline (latest).
+    msg "Installing PyTorch CPU-only via pip (version='${LIBTORCH_VERSION:-latest}')..."
+    if [ -n "${LIBTORCH_VERSION:-}" ]; then
+        pip install "torch==${LIBTORCH_VERSION}+cpu" --index-url https://download.pytorch.org/whl/cpu
+    else
+        pip install torch --index-url https://download.pytorch.org/whl/cpu
+    fi
 
     # Also install libstdcxx-ng to ensure compatible C++ runtime
     eval "$("${CONDA_DIR}/bin/conda" shell.bash hook)"