fix multi-gpu mfsdp tests (#1401)

turn off clip grad norm until we can fix BIO-3

Signed-off-by: Peter St. John <pstjohn@nvidia.com>

Author: pstjohn

bionemo-recipes/recipes/esm2_native_te/train_mfsdp.py

@@ -153,7 +153,8 @@ def main(args: DictConfig) -> float | None:
             loss.backward()
 
             # Compute and clip gradient norms.
-            total_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0).item()
+            # This is causing training to hang in 25.12 torch base image for multi-process mFSDP.
+            # total_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0).item()
 
             # Step optimizer.
             optimizer.step()
@@ -164,7 +165,7 @@ def main(args: DictConfig) -> float | None:
                 step=step,
                 batch=batch,
                 outputs=outputs,
-                grad_norm=total_norm,
+                grad_norm=0.0,  # total_norm,
                 lr=optimizer.param_groups[0]["lr"],
             )