Skip to content

Commit 887ce52

Browse files
committed
fix varlen ci test hang
1 parent 51c197c commit 887ce52

1 file changed

Lines changed: 7 additions & 1 deletion

File tree

torchtitan/experiments/rl/tests/test_bitwise_parity.py

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -679,13 +679,19 @@ def setUpClass(cls):
679679

680680
@classmethod
681681
def tearDownClass(cls):
682+
if dist.is_initialized():
683+
dist.barrier()
682684
if hasattr(cls, "engine"):
683-
cls.engine.engine_core.shutdown()
685+
renderer = getattr(cls.engine, "renderer", None)
686+
if renderer is not None:
687+
renderer.shutdown()
684688
del cls.engine
685689
if hasattr(cls, "model"):
686690
del cls.model
687691
gc.collect()
688692
torch.cuda.empty_cache()
693+
if dist.is_initialized():
694+
dist.destroy_process_group()
689695

690696
def _assert_logprobs_equal(self, name, a, b, label_a="A", label_b="B"):
691697
"""Assert two logprob sequences are bitwise identical."""

0 commit comments

Comments
 (0)