Skip to content

Commit 4cddd8b

Browse files
committed
fix: per-write deadline on listener + per-read deadline on source; ReadHeaderTimeout
Recurring symptom: HTTPS to the server stops responding after some time even though the process is alive — TLS handshake completes (verified with openssl s_client) but the HTTP layer hangs and curl returns 'i/o timeout' for every request, stream and metadata POST alike. CLOSE_WAIT socket count climbs into the dozens. Root cause: handler-side I/O has no deadline. - serveStreamData's out.Write / w.Write to a slow or half-disconnected listener blocks indefinitely (TCP send buffer full, never drains). The handler goroutine sits there, the listener slot stays registered, the conn ends up in CLOSE_WAIT, and over time enough of these accumulate that the runtime can no longer service new HTTPS requests in a timely way. - handleSource's bufrw.Read on a hijacked conn has the same problem in the other direction: a source that drops without a FIN (network blip, encoder crash) wedges the handler. The fix is the standard Go-network-server defence: - Listener: http.NewResponseController(w).SetWriteDeadline before every Write. 30 s — typical Write to a healthy listener finishes in <1 ms; this only fires when the receiver is gone. - Source: conn.SetReadDeadline(60 s) refreshed after every successful read. A continuously-streaming source never sees it; a silent connection dies cleanly. Also: switched both http.Server's ReadTimeout (10 s, applies to the WHOLE request including streaming bodies, wrong for a server that ingests source audio over PUT/SOURCE) to ReadHeaderTimeout. Same slowloris protection without penalising long-running uploads.
1 parent 2bcb8e7 commit 4cddd8b

2 files changed

Lines changed: 45 additions & 7 deletions

File tree

server/handlers_stream.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,15 @@ func (s *Server) handleSource(w http.ResponseWriter, r *http.Request) {
152152
return
153153
}
154154
defer conn.Close()
155+
// Per-read deadline on the source. Without it, a source that
156+
// silently drops (network blip, encoder crash without FIN)
157+
// holds the handler goroutine in bufrw.Read forever; the stream
158+
// stays mounted, transcoders keep their input subscription, and
159+
// the next reconnect from the same encoder leaves zombies. The
160+
// deadline is refreshed after every successful read so a
161+
// continuously-streaming source is never penalised — only
162+
// genuinely silent connections die.
163+
const sourceReadTimeout = 60 * time.Second
155164

156165
bufrw.WriteString("HTTP/1.0 200 OK\r\nServer: Icecast 2.4.4\r\nConnection: Keep-Alive\r\n\r\n")
157166
bufrw.Flush()
@@ -178,6 +187,7 @@ func (s *Server) handleSource(w http.ResponseWriter, r *http.Request) {
178187

179188
buf := make([]byte, 8192)
180189
for {
190+
_ = conn.SetReadDeadline(time.Now().Add(sourceReadTimeout))
181191
n, err := bufrw.Read(buf)
182192
if n > 0 {
183193
stream.Broadcast(buf[:n], s.Relay)
@@ -504,6 +514,22 @@ func (s *Server) serveStreamData(w http.ResponseWriter, r *http.Request, stream
504514
}
505515
}
506516

517+
// Per-write deadline. Without it, a Write to a slow / dead /
518+
// half-disconnected listener blocks forever — the handler
519+
// goroutine sits in TCP-write-blocked state, the listener slot
520+
// stays registered on the stream, and the conn ends up in
521+
// CLOSE_WAIT. With many of those accumulating, the runtime ran
522+
// out of usable handler slots and NEW HTTPS requests started
523+
// timing out wholesale (recurring "i/o timeout" symptom).
524+
//
525+
// 30 s is conservative — typical TCP send to a healthy listener
526+
// returns in <1 ms; 30 s only fires when the kernel send buffer
527+
// is full and not draining (i.e. the receiver is gone). On
528+
// deadline expiry Write returns an error → we exit the handler →
529+
// Unsubscribe runs → the conn cleans up.
530+
rc := http.NewResponseController(w)
531+
const writeTimeout = 30 * time.Second
532+
507533
// 64 KiB read chunk: we flush after draining the buffer on each signal,
508534
// so a larger chunk means fewer syscalls / TCP writes per second without
509535
// adding latency. 4 KiB was causing tiny-segment writes whenever the
@@ -573,6 +599,7 @@ func (s *Server) serveStreamData(w http.ResponseWriter, r *http.Request, stream
573599
}
574600
offset = next
575601

602+
_ = rc.SetWriteDeadline(time.Now().Add(writeTimeout))
576603
if _, err := out.Write(buf[:n]); err != nil {
577604
return false
578605
}
@@ -592,6 +619,7 @@ func (s *Server) serveStreamData(w http.ResponseWriter, r *http.Request, stream
592619
res[0] = byte(l)
593620
copy(res[1:], meta)
594621

622+
_ = rc.SetWriteDeadline(time.Now().Add(writeTimeout))
595623
if _, err := w.Write(res); err != nil {
596624
return false
597625
}

server/listener.go

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -280,11 +280,18 @@ func (s *Server) startHTTPS(handler http.Handler, addr string) error {
280280
}
281281

282282
httpsSrv := &http.Server{
283-
Addr: httpsAddr,
284-
Handler: handler,
285-
ReadTimeout: 10 * time.Second,
286-
WriteTimeout: 0,
287-
IdleTimeout: 120 * time.Second,
283+
Addr: httpsAddr,
284+
Handler: handler,
285+
// ReadHeaderTimeout instead of ReadTimeout — the latter
286+
// bounds the WHOLE request including streaming bodies,
287+
// which is wrong for source ingest. Headers must be in
288+
// within 10 s; bodies use their own per-read deadline
289+
// (handleSource refreshes a 60 s deadline on every Read).
290+
// Listener writes get a per-write deadline via
291+
// http.NewResponseController inside the handler.
292+
ReadHeaderTimeout: 10 * time.Second,
293+
WriteTimeout: 0,
294+
IdleTimeout: 120 * time.Second,
288295
}
289296
if s.certManager != nil {
290297
httpsSrv.TLSConfig = s.certManager.TLSConfig()
@@ -312,8 +319,11 @@ func (s *Server) startHTTPS(handler http.Handler, addr string) error {
312319
}
313320
http.Redirect(w, r, target, http.StatusMovedPermanently)
314321
}),
315-
ReadTimeout: 10 * time.Second,
316-
IdleTimeout: 120 * time.Second,
322+
// Same rationale as the HTTPS server above: ReadHeaderTimeout
323+
// caps slowloris attacks without breaking PUT/SOURCE streams
324+
// that use Hijack and do their own per-read deadlines.
325+
ReadHeaderTimeout: 10 * time.Second,
326+
IdleTimeout: 120 * time.Second,
317327
}
318328
s.httpServers = append(s.httpServers, httpSrv)
319329

0 commit comments

Comments
 (0)