Skip to content

Commit 6cc46a5

Browse files
committed
fix lazy stack dataloader batching
1 parent a1a061e commit 6cc46a5

2 files changed

Lines changed: 13 additions & 0 deletions

File tree

tensordict/_lazy.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2696,6 +2696,8 @@ def recompose(converted_idx, stack_dim=new_stack_dim):
26962696
result.lock_()
26972697
return result
26982698

2699+
__getitems__ = __getitem__
2700+
26992701
def __eq__(self, other):
27002702
return self._dispatch_comparison(other, "__eq__", "__eq__", default=False)
27012703

test/tensordict/test_lazy.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,17 @@ def test_to_mds(self, tmpdir):
275275
test_td = LazyStackedTensorDict(*batches)
276276
assert_allclose_td(td, test_td)
277277

278+
def test_dataloader_ragged_lazy_stack(self):
279+
dataset = lazy_stack([TensorDict(x=torch.zeros(i)) for i in range(10)])
280+
batches = list(
281+
torch.utils.data.DataLoader( # noqa: TOR401
282+
dataset, batch_size=4, collate_fn=lambda batch: batch
283+
)
284+
)
285+
assert [len(batch) for batch in batches] == [4, 4, 2]
286+
assert all(isinstance(batch, LazyStackedTensorDict) for batch in batches)
287+
assert [len(td["x"]) for batch in batches for td in batch] == list(range(10))
288+
278289
def test_all_keys(self):
279290
td = TensorDict({"a": torch.zeros(1)}, [])
280291
td2 = TensorDict({"a": torch.zeros(2)}, [])

0 commit comments

Comments
 (0)