fix: reduce peak memory for multi-player CFR generation (#244)

Three changes to prevent OOM when generating 3-player CFR games:

- Configure jemalloc with 1-second dirty/muzzy page decay (down from
  default 10s). The ~17GB CFR tree freed between games was overlapping
  with the next allocation, spiking RSS to ~34GB.

- Drop HoldemSimulation immediately after run() in generate, compare,
  and comparison runner. This frees the CFR tree before stats processing
  or potentially-blocking channel sends.

- Use 47MB stack for spawned background threads to match the main
  binary's linker-configured stack, preventing stack overflow during
  deep CFR recursion with 3+ players.

Also fixes agent builder ordering in comparison runner to call
cfr_context() before game_state(), avoiding wasted eager CFR allocation.

Author: elliottneilclark

src/arena/comparison/runner.rs

@@ -259,11 +259,13 @@ impl ArenaComparison {
                 .map(|(idx, &agent_idx)| {
                     let mut builder = ConfigAgentBuilder::new(agent_configs[agent_idx].clone())
                         .expect("Failed to create agent builder")
-                        .player_idx(idx)
-                        .game_state(game_state.clone());
+                        .player_idx(idx);
+                    // Inject shared CFR context BEFORE game_state to avoid
+                    // wasted eager allocation in game_state()
                     if let Some((ref cfr_states, ref ts)) = cfr_context {
                         builder = builder.cfr_context(cfr_states.clone(), ts.clone());
                     }
+                    builder = builder.game_state(game_state.clone());
                     if let Some(ref pool) = self.config.thread_pool {
                         builder = builder.thread_pool(pool.clone());
                     }
@@ -311,6 +313,10 @@ impl ArenaComparison {
                 "Completed permutation"
             );
 
+            // Drop the simulation immediately to free the CFR tree
+            // before we read stats or invoke the callback.
+            drop(sim);
+
             // Extract statistics from the historian via the shared storage
             let stats = stats_storage.try_read().map_err(|e| {
                 ComparisonError::SimulationError(format!("Failed to read stats: {}", e))

src/bin/rsp/arena/compare.rs

@@ -174,10 +174,17 @@ fn run_comparison_with_tui(comparison: ArenaComparison) -> Result<(), CompareErr
 
     let (tx, rx) = std::sync::mpsc::sync_channel::<SimMessage<GameResult>>(1024);
 
+    // Must use a large stack to match the main binary's linker-configured stack
+    // (47 MB via -Wl,-zstack-size), since CFR traversal with 3+ players recurses
+    // deeply and overflows the default 8 MB thread stack.
     let bg_hand_store = hand_store.clone();
-    std::thread::spawn(move || {
-        run_comparison_background(comparison, tx, bg_hand_store, ohh_path);
-    });
+    const STACK_SIZE: usize = 47 * 1024 * 1024;
+    std::thread::Builder::new()
+        .stack_size(STACK_SIZE)
+        .spawn(move || {
+            run_comparison_background(comparison, tx, bg_hand_store, ohh_path);
+        })
+        .expect("failed to spawn comparison thread");
 
     let handler = EventHandler::new(rx, Duration::from_millis(33));
     let mut tui_app = App::new(Some(total_games));

src/bin/rsp/arena/generate.rs

@@ -340,6 +340,7 @@ fn run_generation(args: &GenerateArgs, configs: &[AgentConfig]) -> Result<(), Ge
         };
 
         sim.run(&mut ctx.rng);
+        drop(sim);
         ctx.record_success();
 
         if ctx.games_completed.is_multiple_of(report_interval) {
@@ -436,6 +437,12 @@ fn run_generation_inner(
 
         sim.run(&mut ctx.rng);
 
+        // Drop the simulation immediately to free the CFR tree (~19GB for 3
+        // players) before we snapshot stats or block on the channel send.
+        // Without this, the tree stays alive through the potentially-blocking
+        // tx.send(), keeping peak RSS ~2x higher than necessary.
+        drop(sim);
+
         // Record the byte offset so the TUI can fetch this hand on demand
         hand_store.push_offset(pre_offset);
 
@@ -447,7 +454,6 @@ fn run_generation_inner(
         let seat_stats: Vec<SeatStats> = (0..setup.num_players)
             .map(|i| SeatStats::from_storage(&stats_snap, i))
             .collect();
-        // stats_snap (with its 40+ Vecs) is dropped here, on the generation thread
         drop(stats_snap);
 
         let game_result = GameResult {
@@ -483,13 +489,20 @@ fn run_generation_with_tui(
 
     let hand_store = HandStore::new(args.output.clone());
 
-    // Spawn simulation in background thread
+    // Spawn simulation in background thread.
+    // Must use a large stack to match the main binary's linker-configured stack
+    // (47 MB via -Wl,-zstack-size), since CFR traversal with 3+ players recurses
+    // deeply and overflows the default 8 MB thread stack.
     let bg_args = args.clone();
     let bg_configs = configs.clone();
     let bg_hand_store = hand_store.clone();
-    std::thread::spawn(move || {
-        run_generation_background(bg_args, bg_configs, tx, bg_hand_store);
-    });
+    const STACK_SIZE: usize = 47 * 1024 * 1024;
+    std::thread::Builder::new()
+        .stack_size(STACK_SIZE)
+        .spawn(move || {
+            run_generation_background(bg_args, bg_configs, tx, bg_hand_store);
+        })
+        .expect("failed to spawn generation thread");
 
     let handler = EventHandler::new(rx, std::time::Duration::from_millis(33));
     let mut tui_app = App::new(games_target);

src/bin/rsp/main.rs

@@ -2,6 +2,19 @@
 #[global_allocator]
 static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
+/// Configure jemalloc to purge freed pages after 1 second instead of the
+/// default 10 seconds.
+///
+/// CFR solvers allocate ~17GB per game. With the default 10-second dirty page
+/// decay, the old tree's pages can overlap with the new allocation, spiking
+/// peak RSS to ~34GB. A 1-second decay is fast enough to reclaim pages between
+/// games (3-player CFR games take several seconds) while avoiding the syscall
+/// overhead of immediate purging (decay_ms:0).
+#[cfg(not(target_env = "msvc"))]
+#[allow(non_upper_case_globals)]
+#[unsafe(no_mangle)]
+pub static malloc_conf: &[u8; 40] = b"dirty_decay_ms:1000,muzzy_decay_ms:1000\0";
+
 mod arena;
 mod common;
 mod holdem;