security(url-reader): enforce size limit via streaming body cap (SEC-022)

web_url_read's size limit was advisory only: the HEAD Content-Length preflight
is non-fatal, so a server using chunked encoding, a failing/absent HEAD, or a
GET body larger than its reported Content-Length could make the unbounded
`await response.text()` buffer the whole body into memory (DoS). Read the
response body via a bounded stream that counts decompressed bytes and cancels
the reader once URL_READ_MAX_CONTENT_LENGTH_BYTES is exceeded, returning the
existing content-too-large message before any conversion or cache write. The
same bounded read caps the !response.ok error-body snippet. HEAD preflight is
kept as a cheap early-out; the streaming cap is authoritative.

Co-Authored-By: Claude <noreply@anthropic.com>
Co-Authored-By: Codex <noreply@openai.com>

Author: 3 people

CONFIGURATION.md

@@ -57,7 +57,7 @@ Self-hosting SearXNG with JSON output enabled remains the recommended setup. The
 | Variable | Required | Default | Description |
 |---|---|---|---|
 | `URL_READ_MAX_CHARS` | No | — | Default maximum characters returned by `web_url_read` when the caller omits `maxLength`. Explicit `maxLength` always wins. Invalid values are ignored. |
-| `URL_READ_MAX_CONTENT_LENGTH_BYTES` | No | `5242880` | Maximum `Content-Length` allowed by the `web_url_read` HEAD preflight before downloading a page. Invalid values fall back to the default. HEAD failures are non-fatal and the GET proceeds. |
+| `URL_READ_MAX_CONTENT_LENGTH_BYTES` | No | `5242880` | Maximum decompressed response-body bytes `web_url_read` will read while streaming a page. A HEAD `Content-Length` preflight may reject oversized pages before GET, but the streaming cap is authoritative. Invalid values fall back to the default. |
 | `CACHE_TTL_MS` | No | `86400000` | URL cache TTL in milliseconds. Invalid or non-positive values fall back to the default (24 hours). |
 | `CACHE_MAX_ENTRIES` | No | `500` | Maximum number of cached URLs. When the cache exceeds this size, the least frequently used entry is evicted, with oldest entry used as the tie-breaker. Invalid or non-positive values fall back to the default. |
 
@@ -129,6 +129,8 @@ For direct URL-reader requests without a proxy, DNS answers are validated before
 
 When a URL-reader proxy is configured (`URL_READER_HTTP_PROXY`, `URL_READER_HTTPS_PROXY`, `HTTP_PROXY`, or `HTTPS_PROXY`), the proxy performs DNS resolution. Client-side DNS-answer validation cannot inspect proxied resolutions, so proxied deployments should rely on proxy, firewall, and egress controls.
 
+`URL_READ_MAX_CONTENT_LENGTH_BYTES` is enforced while streaming the response body, including chunked responses and responses whose GET body is larger than the HEAD `Content-Length` value. The limit is measured after transparent response decompression.
+
 Set `MCP_HTTP_ALLOW_PRIVATE_URLS=true` only when internal URL reads are intentional for your deployment. This also allows hostnames that DNS-resolve to private/internal addresses.
 
 

SECURITY.md

@@ -94,6 +94,10 @@ The server auto-detects system CA bundles on Linux and macOS for outbound HTTPS
 
 The `web_url_read` tool manually follows redirects (up to 5 hops). Each intermediate URL is validated against the private-IP blocklist before the request is made. On the direct no-proxy path, each redirect hop also goes through DNS-answer validation before connecting.
 
+### URL Reader Size Limits
+
+`web_url_read` enforces `URL_READ_MAX_CONTENT_LENGTH_BYTES` while streaming the response body. The HEAD `Content-Length` check remains as a cheap early rejection path, but the streaming cap is authoritative and also applies when the server omits `Content-Length`, uses chunked transfer encoding, or sends more data than it reported. The cap is measured after undici's transparent Content-Encoding decompression, which bounds the in-memory content size used for HTML-to-Markdown conversion.
+
 ## Deployment Recommendations
 
 ### Minimal / Local

__tests__/unit/url-reader.test.ts

@@ -392,6 +392,194 @@ async function runTests() {
     }
   }, results);
 
+  await testFunction('Streaming GET body without Content-Length is capped', async () => {
+    const mockServer = createMockServer();
+    urlCache.clear();
+    envManager.set('URL_READ_MAX_CONTENT_LENGTH_BYTES', '64');
+
+    const seenMethods: string[] = [];
+    const { url, close } = await startHttpServer((req, res) => {
+      seenMethods.push(req.method || 'UNKNOWN');
+      if (req.method === 'HEAD') {
+        res.writeHead(200);
+        res.end();
+        return;
+      }
+
+      res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });
+      res.write('<html><body><h1>Chunked</h1>');
+      res.write('x'.repeat(128));
+      res.end('</body></html>');
+    });
+
+    try {
+      const result = await fetchAndConvertToMarkdown(mockServer as any, url);
+      assert.ok(result.includes('Content too large'), `Expected content-too-large message, got: ${result}`);
+      assert.deepEqual(seenMethods, ['HEAD', 'GET']);
+    } finally {
+      await close();
+      envManager.restore();
+      urlCache.clear();
+    }
+  }, results);
+
+  await testFunction('Streaming GET body with understated HEAD Content-Length is capped', async () => {
+    const mockServer = createMockServer();
+    urlCache.clear();
+    envManager.set('URL_READ_MAX_CONTENT_LENGTH_BYTES', '64');
+
+    const seenMethods: string[] = [];
+    const { url, close } = await startHttpServer((req, res) => {
+      seenMethods.push(req.method || 'UNKNOWN');
+      if (req.method === 'HEAD') {
+        res.writeHead(200, { 'content-length': '10' });
+        res.end();
+        return;
+      }
+
+      res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });
+      res.write('<html><body><h1>Understated</h1>');
+      res.write('x'.repeat(128));
+      res.end('</body></html>');
+    });
+
+    try {
+      const result = await fetchAndConvertToMarkdown(mockServer as any, url);
+      assert.ok(result.includes('Content too large'), `Expected content-too-large message, got: ${result}`);
+      assert.deepEqual(seenMethods, ['HEAD', 'GET']);
+    } finally {
+      await close();
+      envManager.restore();
+      urlCache.clear();
+    }
+  }, results);
+
+  await testFunction('Streaming GET body just under limit is returned in full', async () => {
+    const mockServer = createMockServer();
+    urlCache.clear();
+    envManager.set('URL_READ_MAX_CONTENT_LENGTH_BYTES', '128');
+
+    const html = '<html><body><h1>Within Limit</h1><p>Complete body.</p></body></html>';
+    const { url, close } = await startHttpServer((req, res) => {
+      if (req.method === 'HEAD') {
+        res.writeHead(200);
+        res.end();
+        return;
+      }
+
+      res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });
+      res.end(html);
+    });
+
+    try {
+      assert.ok(Buffer.byteLength(html, 'utf8') < 128, 'Test body must stay below configured cap');
+      const result = await fetchAndConvertToMarkdown(mockServer as any, url);
+      assert.ok(result.includes('Within Limit'), `Expected converted content, got: ${result}`);
+      assert.ok(result.includes('Complete body'), `Expected complete converted body, got: ${result}`);
+    } finally {
+      await close();
+      envManager.restore();
+      urlCache.clear();
+    }
+  }, results);
+
+  await testFunction('Over-limit streaming GET result is not cached', async () => {
+    const mockServer = createMockServer();
+    urlCache.clear();
+    envManager.set('URL_READ_MAX_CONTENT_LENGTH_BYTES', '64');
+
+    let headCount = 0;
+    let getCount = 0;
+    const { url, close } = await startHttpServer((req, res) => {
+      if (req.method === 'HEAD') {
+        headCount++;
+        res.writeHead(200);
+        res.end();
+        return;
+      }
+
+      getCount++;
+      res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });
+      res.write('<html><body><h1>Not cached</h1>');
+      res.write('x'.repeat(128));
+      res.end('</body></html>');
+    });
+
+    try {
+      const first = await fetchAndConvertToMarkdown(mockServer as any, url);
+      const second = await fetchAndConvertToMarkdown(mockServer as any, url);
+
+      assert.ok(first.includes('Content too large'), `Expected first read to be capped, got: ${first}`);
+      assert.ok(second.includes('Content too large'), `Expected second read to be capped, got: ${second}`);
+      assert.equal(headCount, 2, 'Second over-limit read should repeat HEAD instead of using cache');
+      assert.equal(getCount, 2, 'Second over-limit read should re-fetch instead of using cache');
+    } finally {
+      await close();
+      envManager.restore();
+      urlCache.clear();
+    }
+  }, results);
+
+  await testFunction('Oversized HTTP error response body is capped', async () => {
+    const mockServer = createMockServer();
+    urlCache.clear();
+    envManager.set('URL_READ_MAX_CONTENT_LENGTH_BYTES', '64');
+
+    let chunksWritten = 0;
+    let responseClosed = false;
+    let resolveResponseClosed: () => void = () => {};
+    const responseClosedPromise = new Promise<void>((resolve) => {
+      resolveResponseClosed = resolve;
+    });
+    const totalChunks = 100;
+    const { url, close } = await startHttpServer((req, res) => {
+      if (req.method === 'HEAD') {
+        res.writeHead(500);
+        res.end();
+        return;
+      }
+
+      res.writeHead(500, { 'content-type': 'text/plain; charset=utf-8' });
+      res.on('close', () => {
+        responseClosed = true;
+        resolveResponseClosed();
+      });
+
+      const writeNext = () => {
+        if (res.destroyed || chunksWritten >= totalChunks) {
+          res.end();
+          return;
+        }
+
+        chunksWritten++;
+        res.write('x'.repeat(32));
+        setImmediate(writeNext);
+      };
+
+      writeNext();
+    });
+
+    try {
+      await fetchAndConvertToMarkdown(mockServer as any, url);
+      assert.fail('Expected server error');
+    } catch (error: any) {
+      assert.ok(
+        error.message.includes('Website Error (500)') || error.name === 'MCPSearXNGError',
+        `Expected server error, got: ${error.message}`,
+      );
+      await Promise.race([
+        responseClosedPromise,
+        new Promise<void>((resolve) => setTimeout(resolve, 50)),
+      ]);
+      assert.ok(responseClosed, 'Expected response stream to be closed');
+      assert.ok(chunksWritten < totalChunks, `Expected capped error-body read, wrote all ${chunksWritten} chunks`);
+    } finally {
+      await close();
+      envManager.restore();
+      urlCache.clear();
+    }
+  }, results);
+
   await testFunction('Invalid URL_READ_MAX_CONTENT_LENGTH_BYTES falls back to default cap', async () => {
     const mockServer = createMockServer();
     urlCache.clear();

src/url-reader.ts

@@ -26,6 +26,10 @@ interface PaginationOptions {
   readHeadings?: boolean;
 }
 
+type BoundedBodyReadResult =
+  | { exceeded: false; text: string; bytesRead: number }
+  | { exceeded: true; bytesRead: number };
+
 const REDIRECT_STATUS_CODES = new Set([301, 302, 303, 307, 308]);
 const MAX_REDIRECTS = 5;
 export const DEFAULT_MAX_CONTENT_LENGTH_BYTES = 5 * 1024 * 1024;
@@ -224,6 +228,53 @@ function createContentTooLargeMessage(contentLength: number, maxBytes: number):
   );
 }
 
+function concatenateChunks(chunks: Uint8Array[], totalBytes: number): Uint8Array {
+  const result = new Uint8Array(totalBytes);
+  let offset = 0;
+
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+
+  return result;
+}
+
+async function readResponseBodyWithLimit(response: Response, maxBytes: number): Promise<BoundedBodyReadResult> {
+  if (response.body === null) {
+    return { exceeded: false, text: "", bytesRead: 0 };
+  }
+
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let bytesRead = 0;
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) {
+        break;
+      }
+      if (!value) {
+        continue;
+      }
+
+      bytesRead += value.byteLength;
+      if (bytesRead > maxBytes) {
+        await reader.cancel();
+        return { exceeded: true, bytesRead };
+      }
+
+      chunks.push(value);
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const bodyBytes = concatenateChunks(chunks, bytesRead);
+  return { exceeded: false, text: new TextDecoder("utf-8").decode(bodyBytes), bytesRead };
+}
+
 export async function fetchAndConvertToMarkdown(
   mcpServer: McpServer,
   url: string,
@@ -345,7 +396,10 @@ export async function fetchAndConvertToMarkdown(
     if (!response.ok) {
       let responseBody: string;
       try {
-        responseBody = await response.text();
+        const bodyRead = await readResponseBodyWithLimit(response, maxContentLengthBytes);
+        responseBody = bodyRead.exceeded
+          ? createContentTooLargeMessage(bodyRead.bytesRead, maxContentLengthBytes)
+          : bodyRead.text;
       } catch {
         responseBody = '[Could not read response body]';
       }
@@ -357,7 +411,11 @@ export async function fetchAndConvertToMarkdown(
     // Retrieve HTML content
     let htmlContent: string;
     try {
-      htmlContent = await response.text();
+      const bodyRead = await readResponseBodyWithLimit(response, maxContentLengthBytes);
+      if (bodyRead.exceeded) {
+        return createContentTooLargeMessage(bodyRead.bytesRead, maxContentLengthBytes);
+      }
+      htmlContent = bodyRead.text;
     } catch (error: any) {
       throw createContentError(
         `Failed to read website content: ${error.message || 'Unknown error reading content'}`,