Skip to content

Commit 853829b

Browse files
authored
[data] limit low-resource warning log (#52973)
#52502 removed the cache in `update_reservation`. now the low-resource warning will spam the console when cluster resources are low. Add a `log_once` check to fix this. --------- Signed-off-by: Hao Chen <[email protected]>
1 parent c5eb554 commit 853829b

File tree

1 file changed

+5
-2
lines changed

1 file changed

+5
-2
lines changed

python/ray/data/_internal/execution/resource_manager.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from ray.data._internal.execution.operators.zip_operator import ZipOperator
1919
from ray.data._internal.execution.util import memory_string
2020
from ray.data.context import DataContext
21+
from ray.util.debug import log_once
2122

2223
if TYPE_CHECKING:
2324
from ray.data._internal.execution.streaming_executor_state import OpState, Topology
@@ -484,11 +485,13 @@ def _update_reservation(self):
484485
reserved_for_tasks = ExecutionResources(
485486
0, 0, min_resource_usage.object_store_memory
486487
)
487-
if index == 0:
488+
# Add `id(self)` to the log_once key so that it will be logged once
489+
# per execution.
490+
if index == 0 and log_once(f"low_resource_warning_{id(self)}"):
488491
# Log a warning if even the first operator cannot reserve
489492
# the minimum resources.
490493
logger.warning(
491-
f"Cluster resource are not engough to run any task from {op}."
494+
f"Cluster resources are not engough to run any task from {op}."
492495
" The job may hang forever unless the cluster scales up."
493496
)
494497

0 commit comments

Comments
 (0)