Fix loading manifests from stdin (#810)

Piping a manifest into `lhotse split - <out>` (or any other command using
load_manifest_lazy_or_eager) failed with a JSONDecodeError. Root cause:
load_manifest_lazy_or_eager routes "-" to load_manifest_lazy, which
consumes one line of stdin to detect the manifest class and then builds
a LazyManifestIterator that re-opens the input on every iteration. stdin
is a one-shot stream that cannot be re-opened, so subsequent reads see
either truncated or empty data.

Fix:
- load_manifest_lazy_or_eager now eagerly slurps stdin into a list and
  dispatches to from_dicts (mirroring load_manifest's logic), so the
  manifest is fully materialized in a single pass.
- The `lhotse split` and `lhotse split-lazy` CLI commands derived output
  filenames from the input path's stem/suffix, which produced bogus names
  like `out/-.0` (no extension) when reading from stdin. They now fall
  back to a `manifest.<idx>.jsonl.gz` naming scheme when the input is "-".

Tests:
- test/test_load_manifest_stdin.py exercises the stdin path with
  monkeypatched sys.stdin, covering full-load, split(), double-iteration,
  type detection, empty input, explicit manifest_cls, and garbage input.

Author: deekshaNVIDIA

lhotse/bin/modes/manipulation.py

@@ -168,14 +168,22 @@ def split(
 
     output_dir = Path(output_dir)
     manifest = Path(manifest)
-    suffix = "".join(manifest.suffixes)
+    is_stdin = str(manifest) == "-"
+    if is_stdin:
+        # The "-" placeholder for stdin has no useful stem or suffix to derive
+        # output filenames from, so fall back to sensible defaults.
+        stem = "manifest"
+        suffix = ".jsonl.gz"
+    else:
+        stem = manifest.stem
+        suffix = "".join(manifest.suffixes)
     any_set = load_manifest_lazy_or_eager(manifest)
     parts = any_set.split(num_splits=num_splits, shuffle=shuffle)
     output_dir.mkdir(parents=True, exist_ok=True)
     num_digits = len(str(num_splits))
     for idx, part in enumerate(parts, start=start_idx):
         idx = f"{idx}".zfill(num_digits) if pad else str(idx)
-        part.to_file((output_dir / manifest.stem).with_suffix(f".{idx}{suffix}"))
+        part.to_file((output_dir / stem).with_suffix(f".{idx}{suffix}"))
 
 
 @cli.command()
@@ -206,11 +214,12 @@ def split_lazy(
 
     output_dir = Path(output_dir)
     manifest = Path(manifest)
+    prefix = "manifest" if str(manifest) == "-" else manifest.stem
     any_set = load_manifest_lazy_or_eager(manifest)
     any_set.split_lazy(
         output_dir=output_dir,
         chunk_size=chunk_size,
-        prefix=manifest.stem,
+        prefix=prefix,
         start_idx=start_idx,
     )
 

lhotse/serialization.py

@@ -491,13 +491,52 @@ def load_manifest_lazy_or_eager(
     """
     Generic utility for reading an arbitrary manifest.
     If possible, opens the manifest lazily, otherwise reads everything into memory.
+
+    .. note::
+        When ``path`` is ``"-"`` (stdin), the manifest is always loaded eagerly,
+        because stdin is a one-shot stream and the lazy reader needs to re-open
+        the input to iterate over it (see GitHub issue #810).
     """
-    if extension_contains(".jsonl", path) or str(path) == "-":
+    if str(path) == "-":
+        # stdin cannot be re-opened or seeked, so lazy iteration (which
+        # reopens the input on every pass) does not work for it. Eagerly
+        # consume stdin into a list and build the manifest from it.
+        return _load_manifest_from_stdin(manifest_cls=manifest_cls)
+    if extension_contains(".jsonl", path):
         return load_manifest_lazy(path)
     else:
         return load_manifest(path, manifest_cls=manifest_cls)
 
 
+def _load_manifest_from_stdin(manifest_cls=None) -> Optional[Manifest]:
+    """
+    Read a JSONL manifest from standard input and build it eagerly.
+
+    Stdin is a one-shot stream that cannot be re-opened, which is incompatible
+    with Lhotse's lazy iterators. We therefore slurp every line once and then
+    dispatch to the same ``from_dicts`` logic used by :func:`load_manifest`.
+    """
+    from lhotse import CutSet, FeatureSet, RecordingSet, SupervisionSet
+
+    raw_data = list(load_jsonl("-"))
+    if not raw_data:
+        return None  # empty manifest
+
+    if manifest_cls is not None:
+        candidates = [manifest_cls]
+    else:
+        candidates = [RecordingSet, SupervisionSet, FeatureSet, CutSet]
+    for manifest_type in candidates:
+        try:
+            data_set = manifest_type.from_dicts(raw_data)
+            if len(data_set) == 0:
+                raise RuntimeError()
+            return data_set
+        except Exception:
+            pass
+    raise ValueError("Unknown type of manifest read from stdin.")
+
+
 def resolve_manifest_set_class(item):
     """Returns the right *Set class for a manifest, e.g. Recording -> RecordingSet."""
     from lhotse import (

test/test_load_manifest_stdin.py

@@ -0,0 +1,91 @@
+"""Tests for loading manifests from stdin (``"-"``).
+
+See https://github.com/lhotse-speech/lhotse/issues/810.
+"""
+
+import io
+import json
+import sys
+
+import pytest
+
+from lhotse import RecordingSet
+from lhotse.serialization import load_manifest_lazy_or_eager
+from lhotse.testing.dummies import dummy_recording, dummy_supervision
+
+
+def _redirect_stdin(monkeypatch, payload: str):
+    monkeypatch.setattr(sys, "stdin", io.StringIO(payload))
+
+
+def _to_jsonl(items) -> str:
+    return "\n".join(json.dumps(item.to_dict()) for item in items) + "\n"
+
+
+def test_load_manifest_from_stdin_returns_full_manifest(monkeypatch):
+    recordings = [dummy_recording(i) for i in range(8)]
+    _redirect_stdin(monkeypatch, _to_jsonl(recordings))
+
+    manifest = load_manifest_lazy_or_eager("-")
+
+    assert isinstance(manifest, RecordingSet)
+    assert len(manifest) == 8
+    assert [r.id for r in manifest] == [r.id for r in recordings]
+
+
+def test_load_manifest_from_stdin_supports_split(monkeypatch):
+    """Regression test for #810: ``load + split`` used to fail because the lazy
+    loader tried to re-read stdin to materialize the iterator.
+    """
+    recordings = [dummy_recording(i) for i in range(8)]
+    _redirect_stdin(monkeypatch, _to_jsonl(recordings))
+
+    manifest = load_manifest_lazy_or_eager("-")
+    parts = manifest.split(num_splits=4)
+
+    assert len(parts) == 4
+    assert sum(len(p) for p in parts) == len(recordings)
+
+
+def test_load_manifest_from_stdin_can_be_iterated_twice(monkeypatch):
+    recordings = [dummy_recording(i) for i in range(3)]
+    _redirect_stdin(monkeypatch, _to_jsonl(recordings))
+
+    manifest = load_manifest_lazy_or_eager("-")
+    first_pass = [r.id for r in manifest]
+    second_pass = [r.id for r in manifest]
+
+    assert first_pass == second_pass == [r.id for r in recordings]
+
+
+def test_load_manifest_from_stdin_detects_supervision_set(monkeypatch):
+    supervisions = [dummy_supervision(i) for i in range(2)]
+    _redirect_stdin(monkeypatch, _to_jsonl(supervisions))
+
+    manifest = load_manifest_lazy_or_eager("-")
+
+    from lhotse import SupervisionSet
+
+    assert isinstance(manifest, SupervisionSet)
+    assert len(manifest) == 2
+
+
+def test_load_manifest_from_stdin_returns_none_for_empty_input(monkeypatch):
+    _redirect_stdin(monkeypatch, "")
+    assert load_manifest_lazy_or_eager("-") is None
+
+
+def test_load_manifest_from_stdin_with_explicit_manifest_cls(monkeypatch):
+    recordings = [dummy_recording(i) for i in range(2)]
+    _redirect_stdin(monkeypatch, _to_jsonl(recordings))
+
+    manifest = load_manifest_lazy_or_eager("-", manifest_cls=RecordingSet)
+
+    assert isinstance(manifest, RecordingSet)
+    assert len(manifest) == 2
+
+
+def test_load_manifest_from_stdin_garbage_raises(monkeypatch):
+    _redirect_stdin(monkeypatch, "this is not json\n")
+    with pytest.raises(Exception):
+        load_manifest_lazy_or_eager("-")