CHORE: added documentation for streaming utils and made timeouts configurable (#180)

Author: michelle-hadfield-nava

frontend/src/app/[locale]/generate-referrals/page.tsx

@@ -49,6 +49,46 @@ export default function Page() {
   const [showResultsView, setShowResultsView] = useState(false);
 
   // ========== Resources Streaming (Custom Hook) ==========
+  /**
+   * STREAMING STATE LIFECYCLE:
+   *
+   * The streaming process follows these state transitions:
+   *
+   * 1. INITIAL STATE (before search):
+   *    - loading: false
+   *    - isStreamingResources: false
+   *    - hasReceivedFirstResource: false
+   *    - streamingResources: null
+   *    - retainedResources: undefined
+   *
+   * 2. LOADING STATE (search initiated):
+   *    - loading: true (triggers loading UI)
+   *    - isStreamingResources: true
+   *    - showResultsView: true
+   *    - Starts 12-second timeout to show "No resources found" if nothing arrives
+   *
+   * 3. STREAMING STATE (first chunk arrives):
+   *    - hasReceivedFirstResource: true (clears "No resources found" timeout)
+   *    - streamingResources: PartialResource[] (updates with each chunk)
+   *    - ResourcesList displays streamingResources with loading skeleton for incomplete data
+   *
+   * 4. STREAM COMPLETE STATE:
+   *    - isStreamingResources: false
+   *    - streamingResources: null (cleared)
+   *    - retainedResources: Resource[] (final validated resources)
+   *    - ResourcesList switches to displaying retainedResources
+   *
+   * 5. ERROR STATE (at any point):
+   *    - isStreamingResources: false
+   *    - loading: false
+   *    - streamingResources: null
+   *    - errorMessage: string (displayed in ResourcesList)
+   *
+   * State management notes:
+   * - streamingResources are partial/incomplete, retainedResources are final/complete
+   * - retainedResources is user-editable (can remove items), streamingResources is read-only
+   * - The hook manages streaming lifecycle, this component manages retained results
+   */
   const {
     loading,
     isStreamingResources,

frontend/src/app/api/fcc-lookup/route.ts

@@ -1,4 +1,5 @@
 import { NextResponse } from "next/server";
+import { LOCATION_FETCH_TIMEOUT } from "@/config/timeouts";
 
 interface FCCCounty {
   FIPS: string;
@@ -45,8 +46,7 @@ export async function GET(request: Request) {
     const fccUrl = `https://geo.fcc.gov/api/census/block/find?${params.toString()}`;
 
     const fccResponse = await fetch(fccUrl, {
-      // Add a timeout
-      signal: AbortSignal.timeout(5000),
+      signal: AbortSignal.timeout(LOCATION_FETCH_TIMEOUT), //5000
     });
     if (!fccResponse.ok) {
       return NextResponse.json(

frontend/src/hooks/useResourceRemoval.ts

@@ -1,5 +1,6 @@
 import { useState, useCallback } from "react";
 import { Resource } from "@/types/resources";
+import { UNDO_NOTIFICATION_TIMEOUT } from "@/config/timeouts";
 
 export interface UseResourceRemovalReturn {
   recentlyRemoved: Resource | null;
@@ -34,7 +35,7 @@ export function useResourceRemoval(): UseResourceRemovalReturn {
 
       setRecentlyRemoved(resourceToRemove);
 
-      // Auto-clear the undo notification after 7.5 seconds
+      // Auto-clear the undo notification after timeout period
       setTimeout(() => {
         setRecentlyRemoved((current) => {
           // If the resource is still marked as recently removed, clear it
@@ -44,7 +45,7 @@ export function useResourceRemoval(): UseResourceRemovalReturn {
           return current;
         });
         setRemovedResourceIndex(null);
-      }, 7500);
+      }, UNDO_NOTIFICATION_TIMEOUT); //7500
     },
     [],
   );

frontend/src/hooks/useResourcesStreaming.ts

@@ -6,6 +6,7 @@ import {
   buildRequestWithResolvedZipCodes,
   RequestParams,
 } from "@/util/resolveZipCodes";
+import { NO_RESOURCES_TIMEOUT } from "@/config/timeouts";
 
 export interface UseResourcesStreamingReturn {
   loading: boolean;
@@ -62,14 +63,14 @@ export function useResourcesStreaming(): UseResourcesStreamingReturn {
       setStreamingResources(null);
       setErrorMessage(undefined);
 
-      // 12-second timeout to show "No resources found" if nothing arrives
+      // Show "No resources found" if nothing arrives within timeout period - 12 seconds
       const timeoutId = setTimeout(() => {
         if (!hasReceivedFirstResourceRef.current) {
           setErrorMessage("No resources found.");
           setIsStreamingResources(false);
           setLoading(false);
         }
-      }, 12000);
+      }, NO_RESOURCES_TIMEOUT);
 
       try {
         const request = await buildRequestWithResolvedZipCodes(params);

frontend/src/util/createStreamingFetcher.ts

@@ -3,10 +3,12 @@
  * Handles abort controllers, timeouts, and SSE parsing.
  */
 
+import { STREAMING_TIMEOUT } from "@/config/timeouts";
+
 export interface StreamingConfig<T, TPartial> {
   /** The API endpoint URL */
   url: string;
-  /** Timeout in milliseconds (default: 600000 = 10 minutes) */
+  /** Timeout in milliseconds (default: STREAMING_TIMEOUT = 10 minutes) */
   timeout?: number;
   /** Request body to send */
   requestBody: Record<string, unknown>;
@@ -39,7 +41,7 @@ export async function createStreamingFetcher<T, TPartial>(
 ): Promise<StreamingResult<T>> {
   const {
     url,
-    timeout = 600_000,
+    timeout = STREAMING_TIMEOUT,
     requestBody,
     onChunk,
     onComplete,

frontend/src/util/emailResult.ts

@@ -4,6 +4,7 @@ import {
   EmailResultResponse,
   EmailFullResultResponse,
 } from "@/types/api";
+import { EMAIL_TIMEOUT } from "@/config/timeouts";
 
 export async function emailResult(
   resultId: string,
@@ -33,7 +34,7 @@ export async function emailResult(
       };
 
   const ac = new AbortController();
-  const timer = setTimeout(() => ac.abort(), 300_000);
+  const timer = setTimeout(() => ac.abort(), EMAIL_TIMEOUT); //300_000
 
   try {
     const response = await fetch(url, {

frontend/src/util/fetchActionPlan.ts

@@ -3,6 +3,7 @@ import { getApiDomain } from "./apiDomain";
 import { fixJsonControlCharacters, extractField } from "./parseStreamingUtils";
 import { createStreamingFetcher } from "./createStreamingFetcher";
 import { GenerateActionPlanResponse } from "@/types/api";
+import { ACTION_PLAN_TIMEOUT, STREAMING_TIMEOUT } from "@/config/timeouts";
 
 export interface ActionPlan {
   title: string;
@@ -45,7 +46,7 @@ export async function fetchActionPlan(
   };
 
   const ac = new AbortController();
-  const timer = setTimeout(() => ac.abort(), 120_000);
+  const timer = setTimeout(() => ac.abort(), ACTION_PLAN_TIMEOUT); //120_000
 
   try {
     const requestBody: {
@@ -135,7 +136,7 @@ export async function fetchActionPlanStreaming(
 
   const result = await createStreamingFetcher<ActionPlan, PartialActionPlan>({
     url,
-    timeout: 600_000, // 10 minutes
+    timeout: STREAMING_TIMEOUT, // 10 minutes
     requestBody: {
       model: "generate_action_plan", // Pipeline name as model
       messages: [{ role: "user", content: userQuery }],

frontend/src/util/fetchResourcesStreaming.ts

@@ -7,6 +7,7 @@ import {
 import { fixJsonControlCharacters } from "./parseStreamingUtils";
 import { ResourcesSchema } from "@/types/resources";
 import { createStreamingFetcher } from "./createStreamingFetcher";
+import { STREAMING_TIMEOUT } from "@/config/timeouts";
 
 /**
  * Fetches resources with streaming support using Server-Sent Events (SSE).
@@ -58,7 +59,7 @@ export async function fetchResourcesStreaming(
 
   const result = await createStreamingFetcher<Resource[], PartialResource[]>({
     url,
-    timeout: 600_000, // 10 minutes
+    timeout: STREAMING_TIMEOUT, // 10 minutes
     requestBody,
     onChunk,
     onComplete,

frontend/src/util/parseStreamingResources.ts

@@ -27,7 +27,27 @@ export function parseStreamingResources(jsonStr: string): PartialResource[] {
     const arrayStartPos =
       resourcesKeyMatch.index! + resourcesKeyMatch[0].length;
 
-    // Manually find the matching closing bracket by tracking depth
+    /**
+     * BRACKET DEPTH TRACKING ALGORITHM
+     *
+     * This section finds the matching closing bracket for the "resources" array
+     * by maintaining a depth counter. This is necessary because:
+     * 1. The array may contain nested arrays (e.g., addresses, phones)
+     * 2. We need to handle incomplete streaming JSON (no closing bracket yet)
+     * 3. We must ignore brackets inside string values
+     *
+     * Algorithm:
+     * - bracketDepth starts at 1 (we're already inside the resources array)
+     * - For each '[', increment depth (entering nested array)
+     * - For each ']', decrement depth (exiting nested array)
+     * - When depth reaches 0, we've found the matching closing bracket
+     * - If we reach end of string with depth > 0, the array is incomplete (streaming)
+     *
+     * String state tracking (inString flag):
+     * - Prevents counting brackets that appear inside string values
+     * - Example: "address": "123 Main St [Apt 4]" should not affect depth
+     * - Toggles on/off when encountering unescaped quotes (not preceded by \)
+     */
     let bracketDepth = 1; // We're already inside the resources array
     let inString = false;
     let prevChar = "";
@@ -61,8 +81,29 @@ export function parseStreamingResources(jsonStr: string): PartialResource[] {
     // the closing bracket or the end of the string for incomplete streams)
     const arrayContent = jsonStr.substring(arrayStartPos, arrayEndPos);
 
-    // Find all resource object boundaries by tracking brace depth
-    // We want to identify each `{...}` at the top level of the array
+    /**
+     * RESOURCE OBJECT BOUNDARY DETECTION
+     *
+     * This section identifies individual resource objects within the array by
+     * tracking brace depth. Similar to bracket tracking above, but for objects.
+     *
+     * We want to identify each top-level `{...}` in the array:
+     * Example: [{"name": "A"}, {"name": "B", "addresses": ["X", "Y"]}, {"name"
+     *           ^-- resource 1 --^ ^-- resource 2 (has nested array) ----^ ^-- incomplete
+     *
+     * Algorithm:
+     * - braceDepth = 0 means we're between objects
+     * - When braceDepth goes from 0→1, we mark the start of a new resource
+     * - When braceDepth goes from 1→0, we mark the end of that resource
+     * - Nested objects (e.g., in a field value) don't trigger new resources
+     *   because we only care about 0→1 transitions
+     *
+     * Handling incomplete resources:
+     * - If we reach end of array content with braceDepth > 0, the last resource
+     *   is incomplete (still streaming from server)
+     * - We include it anyway so users see partial data during streaming
+     * - parsePartialResource handles incomplete JSON gracefully
+     */
     const resourceRanges: Array<{ start: number; end: number }> = [];
     let braceDepth = 0;
     inString = false; // Reset the string tracking flag

frontend/src/util/parseStreamingUtils.ts

@@ -58,9 +58,38 @@ export function fixJsonControlCharacters(jsonString: string): string {
 
 /**
  * Extracts a field value from partial JSON, handling incomplete strings
- * Pattern matches escaped characters: (?:[^"\\]|\\.)*
- * - [^"\\] matches any char except quote or backslash
- * - \\. matches backslash followed by any char (handles \", \n, etc)
+ *
+ * Uses regex to find string field values in streaming JSON, even when the closing
+ * quote hasn't arrived yet. Handles escaped characters properly.
+ *
+ * **Pattern explanation:** `(?:[^"\\]|\\.)*`
+ * - `[^"\\]` matches any char except quote or backslash
+ * - `\\.` matches backslash followed by any char (handles \", \n, etc)
+ * - `(?:...)` non-capturing group
+ * - `*` matches zero or more times
+ *
+ * **Examples:**
+ * ```typescript
+ * // Complete field
+ * extractField('{"name": "John Doe"}', 'name')
+ * // → "John Doe"
+ *
+ * // Incomplete field (no closing quote yet)
+ * extractField('{"name": "John Do', 'name')
+ * // → "John Do"
+ *
+ * // Escaped characters
+ * extractField('{"description": "Line 1\\nLine 2"}', 'description')
+ * // → "Line 1\nLine 2" (unescaped)
+ *
+ * // Escaped quotes in value
+ * extractField('{"text": "He said \\"Hello\\""}', 'text')
+ * // → "He said \"Hello\""
+ *
+ * // Field not found
+ * extractField('{"name": "John"}', 'age')
+ * // → undefined
+ * ```
  */
 export function extractField(
   jsonStr: string,
@@ -87,7 +116,40 @@ export function extractField(
 
 /**
  * Extracts array items from partial JSON
- * Handles both complete and incomplete array strings
+ *
+ * Handles both complete and incomplete array strings, finding all complete items
+ * and optionally the incomplete last item (string without closing quote).
+ *
+ * Uses two regex patterns:
+ * 1. Global pattern to find all complete items: `/"((?:[^"\\]|\\.)*)"/g`
+ * 2. Anchored pattern to find incomplete last item: `/"((?:[^"\\]|\\.)*?)$/`
+ *
+ * **Examples:**
+ * ```typescript
+ * // Complete array
+ * extractArrayField('{"phones": ["555-1234", "555-5678"]}', 'phones')
+ * // → ["555-1234", "555-5678"]
+ *
+ * // Incomplete array (no closing bracket)
+ * extractArrayField('{"phones": ["555-1234", "555-5678"', 'phones')
+ * // → ["555-1234", "555-5678"]
+ *
+ * // Incomplete last item (no closing quote)
+ * extractArrayField('{"phones": ["555-1234", "555-56', 'phones')
+ * // → ["555-1234", "555-56"]
+ *
+ * // Escaped characters in items
+ * extractArrayField('{"lines": ["Line 1\\nLine 2", "Line 3"]}', 'lines')
+ * // → ["Line 1\nLine 2", "Line 3"]
+ *
+ * // Empty array
+ * extractArrayField('{"phones": []}', 'phones')
+ * // → []
+ *
+ * // Array not found
+ * extractArrayField('{"name": "John"}', 'phones')
+ * // → []
+ * ```
  */
 export function extractArrayField(
   jsonStr: string,