Skip to content

Commit f8a5396

Browse files
Fix OOM: lazy reader registered with zero chunk_bytes bypassing cache budget
register_loader had chunk_bytes default to 0. The lazy reader path omitted it, so the 7.4 GB sample chunk bypassed the budget check and was loaded unchecked, spiking memory and triggering the OOM killer. Make chunk_bytes a required argument so this class of bug is caught at call time.
1 parent 2b85302 commit f8a5396

2 files changed

Lines changed: 54 additions & 2 deletions

File tree

tests/test_vcz.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1229,6 +1229,56 @@ def test_memory_pressure_deadlock(self):
12291229
cache.shutdown()
12301230

12311231

1232+
class TestRegisterLoaderChunkBytes:
1233+
"""chunk_bytes is required — omitting it must be a TypeError."""
1234+
1235+
def test_register_loader_requires_chunk_bytes(self):
1236+
cache = vcz.ScheduledCache(
1237+
max_bytes=100,
1238+
refcounts={("s", 0): 1},
1239+
chunk_order=[("s", 0)],
1240+
)
1241+
with pytest.raises(TypeError):
1242+
cache.register_loader("s", lambda idx: _arr(10))
1243+
1244+
def test_chunk_bytes_zero_still_accepted(self):
1245+
"""Explicitly passing 0 is allowed (small chunks)."""
1246+
cache = vcz.ScheduledCache(
1247+
max_bytes=100,
1248+
refcounts={("s", 0): 1},
1249+
chunk_order=[("s", 0)],
1250+
)
1251+
cache.register_loader("s", lambda idx: _arr(10), chunk_bytes=0)
1252+
cache.start()
1253+
result = cache.get(("s", 0))
1254+
assert result is not None
1255+
cache.shutdown()
1256+
1257+
def test_large_chunk_bytes_blocks_worker(self):
1258+
"""A chunk whose estimated size exceeds the budget is not loaded
1259+
until space is freed."""
1260+
cache = vcz.ScheduledCache(
1261+
max_bytes=20,
1262+
refcounts={("s", 0): 1, ("big", 0): 1},
1263+
chunk_order=[("s", 0), ("big", 0)],
1264+
)
1265+
cache.register_loader("s", lambda idx: _arr(10), chunk_bytes=10)
1266+
cache.register_loader("big", lambda idx: _arr(10), chunk_bytes=30)
1267+
cache.start()
1268+
# Small chunk fits (10 <= 20)
1269+
cache.get(("s", 0))
1270+
assert ("s", 0) in cache._chunks
1271+
# Big chunk can't fit (10 + 30 > 20), so it shouldn't be loaded yet
1272+
time.sleep(0.1)
1273+
assert ("big", 0) not in cache._chunks
1274+
# Free the small chunk
1275+
cache.record_read(("s", 0))
1276+
# Now big chunk still can't fit (0 + 30 > 20), stays blocked
1277+
time.sleep(0.1)
1278+
assert ("big", 0) not in cache._chunks
1279+
cache.shutdown()
1280+
1281+
12321282
# ---------------------------------------------------------------------------
12331283
# AlleleMapper
12341284
# ---------------------------------------------------------------------------

tsinfer/vcz.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1273,7 +1273,7 @@ def register_loader(
12731273
self,
12741274
source_name: str,
12751275
load_fn,
1276-
chunk_bytes: int = 0,
1276+
chunk_bytes: int,
12771277
) -> None:
12781278
"""Register a loader ``load_fn(chunk_idx) -> ndarray`` for a source."""
12791279
self._loaders[source_name] = load_fn
@@ -2153,7 +2153,9 @@ def _get_reader(self, source_name: str) -> VCZHaplotypeReader:
21532153
_allele_mapper=self._allele_mapper,
21542154
)
21552155
self._readers[source_name] = reader
2156-
self._cache.register_loader(source_name, reader._do_load_chunk)
2156+
self._cache.register_loader(
2157+
source_name, reader._do_load_chunk, reader.chunk_bytes
2158+
)
21572159
return reader
21582160

21592161
def log_cache_state(self) -> None:

0 commit comments

Comments
 (0)