fix(sse): scale the direct response-start window by request payload size

.env.example

@@ -1923,6 +1923,10 @@ CURSOR_USER_AGENT="Cursor/3.4"
 #                                     # Used when the direct upstream URL is private/LAN
 #                                     # so native Ollama cold-start TTFB can exceed 30s.
 #                                     # Default: 300000 = 5 min. See docs/guides/LOCAL_OLLAMA_DOCKER.md
+# OMNIROUTE_DIRECT_HEADERS_VOLUME_ADAPTIVE=1  # Raise the direct response-start window by
+#                                     # payload tier (>512KB ×2, >2MB ×3, capped at 180s,
+#                                     # never lowering a higher floor). Set 0/false/off
+#                                     # to pin the flat window. Default: 1.
 # OMNIROUTE_DIRECT_RESPONSE_RETRY_TIMEOUT_MS=600000  # Ceiling (ms) for the fresh-socket
 #                                     # RETRY attempt above (#13703). Only applies when
 #                                     # the caller already attached its own deadline

changelog.d/fixes/18404-direct-headers-volume-tier.md

@@ -0,0 +1 @@
+- **fix(sse):** the direct response-start window now scales with request payload size (>512 KB ×2, >2 MB ×3, capped at 180 s; `OMNIROUTE_DIRECT_HEADERS_VOLUME_ADAPTIVE=0` pins the flat window) ([#18404](https://github.com/cheaper-inference/OmniRoute/pull/18404)) — thanks @jjxyxsjr

docs/reference/ENVIRONMENT.md

@@ -862,6 +862,7 @@ REQUEST_TIMEOUT_MS (global override)
 | `OMNIROUTE_FETCH_START_TIMEOUT_CAP_MS`          | `110000`                                                          | Maximum wait for response headers on streaming requests (ms). `0` disables this cap; an explicit provider `fetchStartTimeoutCapMs` takes precedence.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           |
 | `OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS`           | `30000` (30s)                                                     | Maximum response-start wait (ms) for each direct no-proxy attempt. A timeout retries once on a fresh socket; set `0` to disable the bound and retain the previous behavior.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
 | `OMNIROUTE_LOCAL_DIRECT_HEADERS_TIMEOUT_MS`     | `300000` (5 min)                                                  | Response-start floor (ms) for direct no-proxy attempts whose target host is private/LAN/loopback (for example `localhost`, `host.docker.internal`, `192.168.x.x`), so a native Ollama cold start is not cut off by the 30s default above. Acts as a floor: the effective wait is the larger of this value and `OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS`; `0` or an invalid value removes the local floor. Remote hosts are unaffected.                                                                                                                                                                                                                                                             |
+| `OMNIROUTE_DIRECT_HEADERS_VOLUME_ADAPTIVE`      | `1`                                                               | Raise the direct response-start window by request payload tier: bodies over 512 KB get ×2, over 2 MB ×3, capped at 180 s and never lowering an already-higher floor (the operator override or the local/LAN floor above). Large contexts prefill proportionally to input size with no reasoning selector for the reasoning ceiling to key on, so the flat window cut healthy slow-prefill upstreams at exactly the floor. Set `0`/`false`/`off` to pin the flat behavior.                                                                                                                                                                                                                      |
 | `OMNIROUTE_DIRECT_RESPONSE_RETRY_TIMEOUT_MS`    | `600000` (10 min)                                                 | Ceiling (ms) for the fresh-socket RETRY attempt after a pooled-attempt response-start timeout above (#13703). Applies only when the caller already attached its own deadline signal (the resolved connection/model/provider/`FETCH_TIMEOUT_MS` cascade); that signal is the real bound and fires first in the intended path, so this is a generous backstop rather than a flat cap — without it, the retry reused the identical short `OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS` window as the pooled attempt and 504'd healthy slow-TTFB reasoning models. Never allowed below the flat floor above; when the caller supplies no deadline signal at all, the retry keeps the flat floor unchanged. |
 | `FETCH_BODY_TIMEOUT_MS`                         | = `FETCH_TIMEOUT_MS`                                              | Time to receive the full response body.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        |
 | `FETCH_CONNECT_TIMEOUT_MS`                      | `30000`                                                           | TCP connection establishment timeout.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |

open-sse/utils/directResponseStartTimeout.ts

@@ -52,6 +52,46 @@ function resolveLocalDirectHeadersFloorMs(
   return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : 0;
 }
 
+// Volume-aware probe-window tier. Prefill scales with input size: a large
+// context legitimately sits silent well past the 30s default before the first
+// byte, with no reasoning selector in the body for the ceiling above to key
+// on — so the flat floor cut healthy slow-prefill upstreams at exactly the
+// floor. Tiers: ≤512KB unchanged (a small body overrunning the probe still
+// smells like a dead connection), >512KB ×2, >2MB ×3, capped at 180s.
+// OMNIROUTE_DIRECT_HEADERS_VOLUME_ADAPTIVE=0/false/off pins the flat behavior.
+const LARGE_BODY_TIER_BYTES = 512 * 1024;
+const HUGE_BODY_TIER_BYTES = 2 * 1024 * 1024;
+const VOLUME_AWARE_WINDOW_CAP_MS = 180_000;
+
+function isVolumeAdaptiveDirectHeadersEnabled(
+  env: Record<string, string | undefined> = process.env
+): boolean {
+  const raw = env.OMNIROUTE_DIRECT_HEADERS_VOLUME_ADAPTIVE;
+  if (raw == null || raw.trim() === "") return true;
+  return !["0", "false", "off"].includes(raw.trim().toLowerCase());
+}
+
+function bodyByteLength(body: string | null | undefined): number {
+  if (!body) return 0;
+  return new TextEncoder().encode(body).length;
+}
+
+export function resolveVolumeAwareDirectHeadersTimeoutMs(
+  baseMs: number,
+  bodyBytes: number,
+  env: Record<string, string | undefined> = process.env
+): number {
+  if (!(baseMs > 0) || !(bodyBytes > 0)) return baseMs;
+  if (!isVolumeAdaptiveDirectHeadersEnabled(env)) return baseMs;
+  let factor = 1;
+  if (bodyBytes > HUGE_BODY_TIER_BYTES) factor = 3;
+  else if (bodyBytes > LARGE_BODY_TIER_BYTES) factor = 2;
+  if (factor === 1) return baseMs;
+  // Raise-only: the ceiling must never pull the window below an
+  // already-higher floor (operator override or the local/LAN floor).
+  return Math.max(baseMs, Math.min(baseMs * factor, VOLUME_AWARE_WINDOW_CAP_MS));
+}
+
 function resolveFlatDirectHeadersFloorMs(
   env: Record<string, string | undefined>,
   body?: string | null,
@@ -72,7 +112,9 @@ function resolveFlatDirectHeadersFloorMs(
   if (isLocalDirectTarget(targetUrl)) {
     base = Math.max(base, resolveLocalDirectHeadersFloorMs(env));
   }
-  return base;
+  // Volume tier applies last with raise-only semantics, so it composes with
+  // the effort/local raises above without order dependence.
+  return resolveVolumeAwareDirectHeadersTimeoutMs(base, bodyByteLength(body), env);
 }
 
 // #13703: `attempt` 0 is the pooled dispatcher (flat floor, #10214's

tests/unit/direct-response-start-timeout-volume-tier.test.ts

@@ -0,0 +1,69 @@
+// Regression guard for large-context requests cut by the flat direct
+// response-start window.
+//
+// Root cause: resolveFlatDirectHeadersFloorMs resolves the response-start
+// budget with awareness of reasoning effort and local/LAN targets, but NOT of
+// request payload size. Prefill scales with input size: a multi-hundred-KB
+// context legitimately sits silent well past the 30s default before the first
+// byte, with no reasoning selector in the body for the existing ceiling to
+// key on. The fetch-layer guard then cut healthy slow-prefill upstreams at
+// exactly the floor and surfaced 504s the stream-readiness layer would have
+// permitted.
+//
+// The fix: raise the flat floor by payload tier — ≤512KB unchanged (a small
+// body overrunning the probe still smells like a dead connection), >512KB ×2,
+// >2MB ×3, capped at 180s and never lowering an already-higher floor
+// (operator override or local/LAN). OMNIROUTE_DIRECT_HEADERS_VOLUME_ADAPTIVE=0
+// pins the old flat behavior.
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import {
+  directHeadersTimeoutResolver,
+  resolveDirectHeadersTimeoutMs,
+} from "../../open-sse/utils/directResponseStartTimeout.ts";
+
+const TIER1_BYTES = 512 * 1024;
+const TIER2_BYTES = 2 * 1024 * 1024;
+
+function bodyOf(approxBytes: number): string {
+  return "x".repeat(approxBytes);
+}
+
+const NO_OVERRIDES: Record<string, string | undefined> = {};
+
+test("1. small bodies keep the flat 30s floor (dead-connection detection intact)", () => {
+  assert.equal(resolveDirectHeadersTimeoutMs(NO_OVERRIDES, bodyOf(64 * 1024)), 30_000);
+  assert.equal(resolveDirectHeadersTimeoutMs(NO_OVERRIDES, bodyOf(TIER1_BYTES)), 30_000);
+});
+
+test("2. >512KB doubles the floor", () => {
+  assert.equal(resolveDirectHeadersTimeoutMs(NO_OVERRIDES, bodyOf(TIER1_BYTES + 1)), 60_000);
+});
+
+test("3. >2MB triples the floor", () => {
+  assert.equal(resolveDirectHeadersTimeoutMs(NO_OVERRIDES, bodyOf(TIER2_BYTES + 1)), 90_000);
+});
+
+test("4. tier raise is capped at the 180s volume ceiling", () => {
+  const env = { OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS: "90000" };
+  // 90s floor × 3 = 270s, but the ceiling clamps to 180s (which is still a raise).
+  assert.equal(resolveDirectHeadersTimeoutMs(env, bodyOf(TIER2_BYTES + 1)), 180_000);
+});
+
+test("5. the tier never lowers an already-higher floor (raise-only semantics)", () => {
+  const env = { OMNIROUTE_DIRECT_HEADERS_TIMEOUT_MS: "240000" };
+  // 240s override × 3 would clamp to 180s — the override must win.
+  assert.equal(resolveDirectHeadersTimeoutMs(env, bodyOf(TIER2_BYTES + 1)), 240_000);
+});
+
+test("6. OMNIROUTE_DIRECT_HEADERS_VOLUME_ADAPTIVE=0 pins the flat behavior", () => {
+  const env = { OMNIROUTE_DIRECT_HEADERS_VOLUME_ADAPTIVE: "0" };
+  assert.equal(resolveDirectHeadersTimeoutMs(env, bodyOf(TIER2_BYTES + 1)), 30_000);
+  const envOff = { OMNIROUTE_DIRECT_HEADERS_VOLUME_ADAPTIVE: "off" };
+  assert.equal(resolveDirectHeadersTimeoutMs(envOff, bodyOf(TIER2_BYTES + 1)), 30_000);
+});
+
+test("7. resolver binds the body once per request (attempt 0 window follows the tier)", () => {
+  const resolve = directHeadersTimeoutResolver({ body: bodyOf(TIER1_BYTES + 1) });
+  assert.equal(resolve(0), 60_000);
+});