[client] Bound read timeout on streaming API requests to detect dead connections

`sky status`/`sky logs`/`sky jobs logs` stream their results from the API
server's `/api/stream` (and `/logs`, `/provision_logs`, `/jobs/logs`) endpoints.
The client made these streaming reads with no read timeout
(`timeout=(connect, None)`), so when a streaming connection silently dies -- a
proxy/CDN drops the long-lived connection, or the peer goes away mid-stream --
the client blocks in recv() forever. We observed `sky status` wedged ~20 min in
a single call (kernel stack stuck in `tcp_recvmsg`), while a fresh `sky status`
returned instantly, confirming a dead connection rather than a slow server.

The server already sends a heartbeat every 30s on these streams
(`stream_utils.py:_HEARTBEAT_INTERVAL`, #5750) to keep them busy through
idle-timeout proxies. This adds the missing client-side counterpart: a read
timeout of 4x the heartbeat (120s). A healthy stream is reset by each 30s
heartbeat so the timeout never fires; a dead stream raises `ReadTimeout`, which
these readers already retry (`retry_transient_errors`) on a fresh connection.

Scope: the retry-wrapped sync console-log streams -- `tail_logs`,
`tail_provision_logs`, `stream_and_get` (sky/client/sdk.py) and `jobs.tail_logs`
(sky/jobs/client/sdk.py). Left for follow-up (each needs its own retry wrapper
first, else a timeout would surface instead of retry): `serve.tail_logs`, the
async client streams, and `jobs.download_logs_streaming`. Heartbeat-less
endpoints (`/api/get` long-poll, `format=plain` log streams) keep an unbounded
read, since they may legitimately block with no intermediate output.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: zpoint

sky/client/common.py

@@ -63,6 +63,18 @@
 # Connection timeout when sending requests to the API server.
 API_SERVER_REQUEST_CONNECTION_TIMEOUT_SECONDS = 5
 
+# Read timeout (seconds allowed between bytes from the server) for streaming
+# requests to the API server's heartbeat-emitting endpoints (e.g. /api/stream
+# log following). The server sends a heartbeat every 30s on these streams
+# (sky/server/stream_utils.py:_HEARTBEAT_INTERVAL, added for #5750) to keep the
+# connection busy through idle-timeout proxies/CDNs. This client-side read
+# timeout is the counterpart: set well above the 30s heartbeat so a healthy
+# stream never trips it, but a dead/stalled connection (no heartbeat received
+# for this long) raises ReadTimeout and is retried on a fresh connection instead
+# of blocking forever. NOT applied to heartbeat-less long-poll endpoints (e.g.
+# /api/get), which may legitimately block on a slow request with no output.
+API_SERVER_REQUEST_STREAM_READ_TIMEOUT_SECONDS = 120  # 4x the 30s heartbeat
+
 
 def download_logs_from_api_server(
         paths_on_api_server: Iterable[str],

sky/client/sdk.py

@@ -1165,7 +1165,7 @@ def tail_logs(
         json=json.loads(body.model_dump_json()),
         stream=True,
         timeout=(client_common.API_SERVER_REQUEST_CONNECTION_TIMEOUT_SECONDS,
-                 None))
+                 client_common.API_SERVER_REQUEST_STREAM_READ_TIMEOUT_SECONDS))
     request_id: server_common.RequestId[int] = server_common.get_request_id(
         response)
     if preload_content:
@@ -1225,7 +1225,7 @@ def tail_provision_logs(cluster_name: str,
         params=params,
         stream=True,
         timeout=(client_common.API_SERVER_REQUEST_CONNECTION_TIMEOUT_SECONDS,
-                 None))
+                 client_common.API_SERVER_REQUEST_STREAM_READ_TIMEOUT_SECONDS))
     # Check for HTTP errors before streaming the response
     if response.status_code != 200:
         with ux_utils.print_exception_no_traceback():
@@ -2442,7 +2442,7 @@ def stream_and_get(
         params=params,
         retry=False,
         timeout=(client_common.API_SERVER_REQUEST_CONNECTION_TIMEOUT_SECONDS,
-                 None),
+                 client_common.API_SERVER_REQUEST_STREAM_READ_TIMEOUT_SECONDS),
         stream=True)
     if response.status_code in [404, 400]:
         detail = response.json().get('detail')

sky/jobs/client/sdk.py

@@ -543,7 +543,8 @@ def tail_logs(
         '/jobs/logs',
         json=json.loads(body.model_dump_json()),
         stream=True,
-        timeout=(5, None))
+        timeout=(client_common.API_SERVER_REQUEST_CONNECTION_TIMEOUT_SECONDS,
+                 client_common.API_SERVER_REQUEST_STREAM_READ_TIMEOUT_SECONDS))
     request_id: server_common.RequestId[int] = server_common.get_request_id(
         response)
     if preload_content: