Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions src/huggingface_hub/utils/tqdm.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@
import io
import logging
import os
import threading
import warnings
from collections.abc import Iterator
from contextlib import contextmanager, nullcontext
Expand Down Expand Up @@ -295,6 +296,20 @@ def _inner_read(size: int | None = -1) -> bytes:
pbar.close()


class _SafeTqdm(old_tqdm):
"""tqdm subclass that uses a thread lock instead of a multiprocessing lock.

Used as a fallback when the standard tqdm cannot initialize its multiprocessing
lock (e.g. when stderr.fileno() returns -1 in Textual, Jupyter, pytest, etc.).
"""

_lock = threading.RLock()

@classmethod
def get_lock(cls):
return cls._lock


def _create_progress_bar(*, cls: type[old_tqdm], log_level: int, name: str | None = None, **kwargs) -> old_tqdm:
"""Create a progress bar.

Expand All @@ -309,12 +324,26 @@ def _create_progress_bar(*, cls: type[old_tqdm], log_level: int, name: str | Non
"""
# issubclass() crashes on non-class callables (e.g. functools.partial), guard with isinstance.
if not (isinstance(cls, type) and issubclass(cls, tqdm)):
return cls(**kwargs) # type: ignore[return-value]
try:
return cls(**kwargs)
except (OSError, ValueError):
# Caller opted into custom progress; a visible fallback bar would
# corrupt their output (e.g. TUIs), so disable it.
return _SafeTqdm(disable=True, **kwargs)

# HF subclass: keep the historical log-level / TTY behavior. Group-based
# disabling is already handled in `tqdm.__init__`.
disable = is_tqdm_disabled(log_level)
return cls(disable=disable, name=name, **kwargs) # type: ignore[return-value]
try:
return cls(disable=disable, name=name, **kwargs)
except (OSError, ValueError):
warnings.warn(
"Progress bar could not be initialized in this environment. "
"Download will continue without progress reporting. "
"To suppress this warning, call `disable_progress_bars()` or set HF_HUB_DISABLE_PROGRESS_BARS=1.",
stacklevel=2,
)
return _SafeTqdm(disable=disable, **kwargs)


def _get_progress_bar_context(
Expand Down
35 changes: 35 additions & 0 deletions tests/test_file_download.py
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,41 @@ def test_hf_hub_download_custom_cache_permission(self):
finally:
os.umask(previous_umask)

# tqdm's failed construction leaves an object without `disable` set; its destructor raises AttributeError on
# garbage collection. Python silently ignores exceptions in destructors, so this is only visible in pytest.
@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
def test_hf_hub_download_survives_bad_fileno(self):
"""hf_hub_download should not crash when stderr.fileno() returns -1.

Environments like Textual, Jupyter, and pytest replace stderr with objects
whose fileno() returns -1 or raises. tqdm's multiprocessing lock init
crashes on these, but _create_progress_bar should catch that gracefully.
"""

class FakeStderr:
def write(self, s):
pass

def flush(self):
pass

def isatty(self):
return True

def fileno(self):
return -1

with SoftTemporaryDirectory() as tmpdir:
with warnings.catch_warnings():
warnings.simplefilter("ignore", UserWarning)
with patch("sys.stderr", FakeStderr()):
filepath = hf_hub_download(
DUMMY_MODEL_ID,
filename=constants.CONFIG_NAME,
cache_dir=tmpdir,
)
self.assertTrue(os.path.exists(filepath))

def test_download_from_a_renamed_repo_with_hf_hub_download(self):
"""Checks `hf_hub_download` works also on a renamed repo.

Expand Down
31 changes: 31 additions & 0 deletions tests/test_utils_tqdm.py
Original file line number Diff line number Diff line change
Expand Up @@ -297,3 +297,34 @@ def test_custom_tqdm_class_no_name_kwarg(self):
)
with bar as pbar:
pbar.update(10)

# Failed tqdm construction leaves an object without `disable` set; its
# destructor raises AttributeError on GC. Python silently ignores it, but
# pytest surfaces it as an unraisable warning — same pattern as
# test_hf_hub_download_survives_bad_fileno.
@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
def test_custom_tqdm_class_crash_fallback_is_disabled(self):
"""When a custom tqdm_class raises in __init__, the _SafeTqdm fallback
must be constructed with disable=True so it doesn't render a surprise
bar into the caller's output stream.

Regression guard for the TUI case where a custom class crashes on
multiprocessing lock init (stderr.fileno()==-1) and the visible
fallback bar was corrupting the app with ANSI escapes.
"""

class ExplodingTqdm(vanilla_tqdm):
def __init__(self, *args, **kwargs):
raise ValueError("simulated multiprocessing lock failure")

bar = _get_progress_bar_context(
desc="test",
log_level=logging.INFO,
total=10,
tqdm_class=ExplodingTqdm,
name="huggingface_hub.test",
)
with bar as pbar:
assert pbar.disable is True
pbar.update(5)
pbar.update(5)