feat(httputil): adds ResponseChain.maxBodySize field

Signed-off-by: Dwi Siswanto <git@dw1.io>

Author: dwisiswant0

http/normalization.go

@@ -76,12 +76,12 @@ func readNNormalizeRespBody(rc *ResponseChain, body *bytes.Buffer) (err error) {
 	if err != nil {
 		wrapped = origBody
 	}
-	limitReader := io.LimitReader(wrapped, int64(DefaultMaxBodySize))
+	limitReader := io.LimitReader(wrapped, rc.maxBodySize)
 
-	// Read body using ReadFrom for efficiency, but cap growth at DefaultMaxBodySize.
+	// Read body using ReadFrom for efficiency, but cap growth at maxBodySize.
 	// We use a custom limitedBuffer wrapper to prevent bytes.Buffer from
 	// over-allocating (it normally grows to 2x when size is unknown).
-	limitedBuf := &limitedBuffer{buf: body, maxCap: DefaultMaxBodySize}
+	limitedBuf := &limitedBuffer{buf: body, maxCap: int(rc.maxBodySize)}
 	_, err = limitedBuf.ReadFrom(limitReader)
 	if err != nil {
 		if strings.Contains(err.Error(), "gzip: invalid header") {

http/respChain.go

@@ -154,7 +154,16 @@ func getBuffer() *bytes.Buffer {
 	return buff
 }
 
-// putBuffer returns a buffer to the pool
+// putBuffer returns a buffer to the pool for reuse.
+//
+// Buffers larger than [DefaultMaxBodySize] are discarded.
+// Buffers larger than or equal to largeBufferThreshold are subject to
+// maxLargeBuffers limiting.
+//
+// TODO(dwisiswant0): Current threshold is global. Consider making it
+// configurable per instance (via [ResponseChain.maxBodySize]) if needed.
+// The current implementation is to prevents memory bloat in typical use-cases.
+// And the pool is shared, so per-instance thresholds might cause confusion.
 func putBuffer(buf *bytes.Buffer) {
 	cap := buf.Cap()
 	if cap > DefaultMaxBodySize {
@@ -205,10 +214,11 @@ func resetBuffer() {
 // on every call to previous it returns the previous response
 // if it was redirected.
 type ResponseChain struct {
-	headers  *bytes.Buffer
-	body     *bytes.Buffer
-	resp     *http.Response
-	reloaded bool // if response was reloaded to its previous redirect
+	headers     *bytes.Buffer
+	body        *bytes.Buffer
+	resp        *http.Response
+	reloaded    bool // if response was reloaded to its previous redirect
+	maxBodySize int64
 }
 
 // NewResponseChain creates a new response chain for a http request
@@ -225,9 +235,10 @@ func NewResponseChain(resp *http.Response, maxBody int64) *ResponseChain {
 	}
 
 	return &ResponseChain{
-		headers: getBuffer(),
-		body:    getBuffer(),
-		resp:    resp,
+		headers:     getBuffer(),
+		body:        getBuffer(),
+		resp:        resp,
+		maxBodySize: maxBody,
 	}
 }