batch invariant configs should have max off policy = 0 (#3910)

as title
without this max off policy is 3 inherited from base config so we hsould
set this to 0 in the batch invariant configs to see 0 diff between
trainer and generator per step
<img width="1498" height="458" alt="Screenshot 2026-07-13 at 2 36 12 PM"
src="https://github.com/user-attachments/assets/9b563480-fc66-45df-8d83-7a658279311d"
/>

Author: liangel-02

torchtitan/experiments/rl/examples/alphabet_sort/config_registry.py

@@ -209,6 +209,10 @@ def rl_grpo_qwen3_0_6b_flex_batch_invariant() -> Controller.Config:
     config.async_loop.batcher = dataclasses.replace(
         config.async_loop.batcher, per_sample_pad_multiple=block_size
     )
+    # Batch invariance requires strict on-policy: the generator must run the
+    # latest weights before generating so trainer/generator logprobs stay
+    # bitwise-identical (bit_wise/logprob_diff == 0) every step, not just step 1.
+    config.async_loop.max_offpolicy_steps = 0
     config.trainer = dataclasses.replace(
         config.trainer,
         debug=_BATCH_INVARIANT_DEBUG,
@@ -357,6 +361,9 @@ def rl_grpo_gpt_oss_debug_varlen_batch_invariant() -> Controller.Config:
         hf_assets_path="tests/assets/tokenizer",
         async_loop=AsyncLoopConfig(
             num_training_steps=3,
+            # Batch invariance: strict on-policy so trainer/generator logprobs
+            # stay bitwise-identical every step.
+            max_offpolicy_steps=0,
             num_groups_per_train_step=5,
             group_size=group_size,
             validation=ValidationConfig(num_samples=20),
@@ -663,6 +670,9 @@ def rl_grpo_qwen3_moe_debug_varlen_batch_invariant() -> Controller.Config:
         hf_assets_path="tests/assets/tokenizer",
         async_loop=AsyncLoopConfig(
             num_training_steps=10,
+            # Batch invariance: strict on-policy so trainer/generator logprobs
+            # stay bitwise-identical every step.
+            max_offpolicy_steps=0,
             num_groups_per_train_step=8,
             group_size=group_size,
             validation=ValidationConfig(num_samples=20),
@@ -837,6 +847,9 @@ def rl_grpo_qwen3_0_6b_varlen_batch_invariant() -> Controller.Config:
         num_generators=3,
         async_loop=AsyncLoopConfig(
             num_training_steps=10,
+            # Batch invariance: strict on-policy so trainer/generator logprobs
+            # stay bitwise-identical every step.
+            max_offpolicy_steps=0,
             num_groups_per_train_step=8,
             group_size=group_size,
             validation=ValidationConfig(num_samples=20),