feat: Enhance MetaSearchAgent with optional search functions and improve error handling in search handlers

vtempest · vtempest · commit 4aae01cfd47c · 2026-07-06T20:58:20.000-07:00
diff --git a/apps/qwksearch-web/components/Settings/Sections/Models/TestModelsButton.tsx b/apps/qwksearch-web/components/Settings/Sections/Models/TestModelsButton.tsx
@@ -131,29 +131,29 @@ export default function TestModelsButton({
                       Total Tested
                     </p>
                     <p className="text-2xl font-semibold text-black dark:text-white">
-                      {results.totalModels}
+                      {results.totalModels ?? 0}
                     </p>
                   </div>
                   <div className="p-4 rounded-lg border border-green-200 dark:border-green-800 bg-green-50 dark:bg-green-950/30">
                     <p className="text-xs text-green-700 dark:text-green-400 mb-1">
                       Available
                     </p>
                     <p className="text-2xl font-semibold text-green-700 dark:text-green-400">
-                      {results.availableModels.length}
+                      {results.availableModels?.length ?? 0}
                     </p>
                   </div>
                   <div className="p-4 rounded-lg border border-red-200 dark:border-red-800 bg-red-50 dark:bg-red-950/30">
                     <p className="text-xs text-red-700 dark:text-red-400 mb-1">
                       Unavailable
                     </p>
                     <p className="text-2xl font-semibold text-red-700 dark:text-red-400">
-                      {results.unavailableModels.length}
+                      {results.unavailableModels?.length ?? 0}
                     </p>
                   </div>
                 </div>
 
                 {/* Available Models */}
-                {results.availableModels.length > 0 && (
+                {results.availableModels && results.availableModels.length > 0 && (
                   <div>
                     <h3 className="text-sm font-medium mb-2 text-black dark:text-white flex items-center gap-2">
                       <Check size={16} className="text-green-600 dark:text-green-400" />
@@ -192,7 +192,7 @@ export default function TestModelsButton({
                 )}
 
                 {/* Unavailable Models */}
-                {results.unavailableModels.length > 0 && (
+                {results.unavailableModels && results.unavailableModels.length > 0 && (
                   <div>
                     <h3 className="text-sm font-medium mb-2 text-black dark:text-white flex items-center gap-2">
                       <X size={16} className="text-red-600 dark:text-red-400" />
diff --git a/packages/agent-toolkit/src/tools/search/meta-search-types.ts b/packages/agent-toolkit/src/tools/search/meta-search-types.ts
@@ -44,6 +44,16 @@ export interface Config {
   queryGeneratorFewShots: FewShotExample[];
   responsePrompt: string;
   activeEngines: string[];
+  /** Optional: Function to fetch documents from URLs */
+  getDocumentsFromLinks?: (options: { links: string[] }) => Promise<any[]>;
+  /** Optional: Function to search via SearXNG */
+  searchSearxng?: (query: string, options: any) => Promise<{ results: any[]; suggestions: string[] }>;
+  /** Optional: Function to search via Tavily */
+  searchTavily?: (query: string, options: any) => Promise<{ results: any[]; suggestions: string[] }>;
+  /** Optional: Function to check if Tavily is configured */
+  isTavilyConfigured?: () => boolean;
+  /** Optional: Function to scrape a URL */
+  scrapeURL?: (url: string, options: any) => Promise<string>;
 }
 
 export type BasicChainInput = {
diff --git a/packages/agent-toolkit/src/tools/search/metaSearchAgent.ts b/packages/agent-toolkit/src/tools/search/metaSearchAgent.ts
@@ -5,11 +5,7 @@
  */
 import { generateText, streamText, type LanguageModel } from "ai";
 import { LineOutputParser, LineListOutputParser } from "../../utils/outputParser";
-import { getDocumentsFromLinks } from "extract-webpage/utils/documents";
 import type { Document } from "./document";
-import { searchSearxng } from "extract-webpage/search/public-searxng";
-import { searchTavily, isTavilyConfigured } from "extract-webpage/search/tavily";
-import { scrapeURL } from "extract-webpage/search/url-to-html";
 
 /** Strip HTML tags and decode entities — works in Cloudflare edge runtime */
 function htmlToText(html: string): string {
@@ -26,7 +22,6 @@ function htmlToText(html: string): string {
     .replace(/\s+/g, ' ')
     .trim();
 }
-import { getSourceScrapeTimeout } from "../../config/serverRegistry";
 import { formatChatHistoryAsString } from "../../utils";
 import EventEmitter from "events";
 import type {
@@ -127,10 +122,10 @@ class MetaSearchAgent implements MetaSearchAgentType {
       question = "latest information";
     }
 
-    if (links.length > 0) {
+    if (links.length > 0 && this.config.getDocumentsFromLinks) {
       if (question.length === 0) question = "summarize";
 
-      const linkDocs = await getDocumentsFromLinks({ links });
+      const linkDocs = await this.config.getDocumentsFromLinks({ links });
       const docs = await groupAndSummarizeDocs(llm, linkDocs, question);
 
       return { query: question, docs };
@@ -174,53 +169,60 @@ class MetaSearchAgent implements MetaSearchAgentType {
 
     let res: { results: any[]; suggestions: string[] };
 
-    const runSearxng = () => searchSearxng(question, {
-      language: "en",
-      engines: this.config.activeEngines,
-      categories: [category],
-    });
-
-    if (
-      isTavilyConfigured() &&
-      this.config.activeEngines.length === 0 &&
-      category === "general"
-    ) {
-      try {
-        res = await searchTavily(question, { searchDepth: "basic", maxResults: 10 });
-      } catch (error) {
-        console.error("Tavily search failed, falling back to SearXNG:", error);
-        res = await runSearxng();
-      }
+    // If no search functions provided, return empty results
+    if (!this.config.searchSearxng && !this.config.searchTavily) {
+      res = { results: [], suggestions: [] };
     } else {
-      if (isTavilyConfigured()) {
+      const runSearxng = () => this.config.searchSearxng!(question, {
+        language: "en",
+        engines: this.config.activeEngines,
+        categories: [category],
+      });
+
+      const isTavilyConfigured = this.config.isTavilyConfigured?.() ?? false;
+
+      if (
+        isTavilyConfigured &&
+        this.config.activeEngines.length === 0 &&
+        category === "general"
+      ) {
         try {
-          res = await Promise.race([
-            runSearxng(),
-            new Promise<{ results: any[]; suggestions: string[] }>((_, reject) =>
-              setTimeout(() => reject(new Error("Timeout")), 10000)
-            )
-          ]);
-        } catch (err: any) {
-          if (err.message === "Timeout") {
-            console.warn("[MetaSearchAgent] SearXNG search did not respond in 10 seconds, falling back to Tavily.");
-            try {
-              res = await searchTavily(question, { searchDepth: "basic", maxResults: 10 });
-            } catch (tavilyErr) {
-              console.error("[MetaSearchAgent] Tavily fallback also failed, awaiting SearXNG directly:", tavilyErr);
-              res = await runSearxng();
-            }
-          } else {
-            console.error("[MetaSearchAgent] SearXNG search failed, falling back to Tavily:", err);
-            try {
-              res = await searchTavily(question, { searchDepth: "basic", maxResults: 10 });
-            } catch (tavilyErr) {
-              console.error("[MetaSearchAgent] Tavily fallback also failed:", tavilyErr);
-              throw err;
+          res = await this.config.searchTavily!(question, { searchDepth: "basic", maxResults: 10 });
+        } catch (error) {
+          console.error("Tavily search failed, falling back to SearXNG:", error);
+          res = this.config.searchSearxng ? await runSearxng() : { results: [], suggestions: [] };
+        }
+      } else {
+        if (isTavilyConfigured && this.config.searchTavily) {
+          try {
+            res = await Promise.race([
+              runSearxng(),
+              new Promise<{ results: any[]; suggestions: string[] }>((_, reject) =>
+                setTimeout(() => reject(new Error("Timeout")), 10000)
+              )
+            ]);
+          } catch (err: any) {
+            if (err.message === "Timeout") {
+              console.warn("[MetaSearchAgent] SearXNG search did not respond in 10 seconds, falling back to Tavily.");
+              try {
+                res = await this.config.searchTavily(question, { searchDepth: "basic", maxResults: 10 });
+              } catch (tavilyErr) {
+                console.error("[MetaSearchAgent] Tavily fallback also failed, awaiting SearXNG directly:", tavilyErr);
+                res = this.config.searchSearxng ? await runSearxng() : { results: [], suggestions: [] };
+              }
+            } else {
+              console.error("[MetaSearchAgent] SearXNG search failed, falling back to Tavily:", err);
+              try {
+                res = await this.config.searchTavily(question, { searchDepth: "basic", maxResults: 10 });
+              } catch (tavilyErr) {
+                console.error("[MetaSearchAgent] Tavily fallback also failed:", tavilyErr);
+                throw err;
+              }
             }
           }
+        } else {
+          res = this.config.searchSearxng ? await runSearxng() : { results: [], suggestions: [] };
         }
-      } else {
-        res = await runSearxng();
       }
     }
 
@@ -253,7 +255,8 @@ class MetaSearchAgent implements MetaSearchAgentType {
       perSourceTimeout = Math.max(2, Math.floor(thinkingTimeLimit / scrapeCount));
     } else if (sourceExtractionEnabled) {
       scrapeCount = 3;
-      perSourceTimeout = Math.max(1, getSourceScrapeTimeout());
+      // Default to 5 seconds if no config function available
+      perSourceTimeout = 5;
     } else {
       scrapeCount = 0;
       perSourceTimeout = 0;
@@ -263,12 +266,12 @@ class MetaSearchAgent implements MetaSearchAgentType {
       const docsToScrape = documents.slice(0, scrapeCount);
       emitSearching("running", `Extracting top ${docsToScrape.length} sources`, "extract");
 
-      const extractionTasks = docsToScrape.map(async (doc, idx) => {
+      const extractionTasks = this.config.scrapeURL ? docsToScrape.map(async (doc, idx) => {
         const url = doc.metadata?.url;
-        if (!url) return;
+        if (!url || !this.config.scrapeURL) return;
         try {
           const result = await waitWithTimeout(
-            scrapeURL(url, { timeout: perSourceTimeout }),
+            this.config.scrapeURL(url, { timeout: perSourceTimeout }),
             perSourceTimeout * 1000 + 1500,
           );
           if (typeof result === "string" && result.length > 100) {
@@ -284,7 +287,7 @@ class MetaSearchAgent implements MetaSearchAgentType {
         } catch {
           // Keep original snippet on scraping failure or timeout
         }
-      });
+      }) : [];
 
       await Promise.allSettled(extractionTasks);
       emitSearching("done", `Extracting top ${docsToScrape.length} sources`, "extract");
diff --git a/packages/agent-toolkit/src/tools/search/search-handlers.ts b/packages/agent-toolkit/src/tools/search/search-handlers.ts
@@ -1,11 +1,20 @@
 /**
  * @module research/search/index
  * @description Research library module.
+ *
+ * Note: To use these search handlers, you need to provide search functions
+ * (searchSearxng, searchTavily, etc.) from extract-webpage package.
+ * See extract-webpage/src/search/index.ts for an example.
  */
 import MetaSearchAgent from "./metaSearchAgent";
 import prompts from "../../language-generation/prompts/search-prompts";
+import type { Config } from "./meta-search-types";
 
-export const searchHandlers: Record<string, MetaSearchAgent> = {
+/**
+ * Creates search handler instances with provided search functions.
+ * Pass in search functions from extract-webpage to enable web search.
+ */
+export const createSearchHandlers = (searchFunctions?: Partial<Config>) => ({
   webSearch: new MetaSearchAgent({
     activeEngines: [],
     queryGeneratorPrompt: prompts.webSearchRetrieverPrompt,
@@ -14,6 +23,7 @@ export const searchHandlers: Record<string, MetaSearchAgent> = {
     rerank: true,
     rerankThreshold: 0.3,
     searchWeb: true,
+    ...searchFunctions,
   }),
   academicSearch: new MetaSearchAgent({
     activeEngines: ["arxiv", "google scholar", "pubmed"],
@@ -23,6 +33,7 @@ export const searchHandlers: Record<string, MetaSearchAgent> = {
     rerank: true,
     rerankThreshold: 0,
     searchWeb: true,
+    ...searchFunctions,
   }),
   writingAssistant: new MetaSearchAgent({
     activeEngines: [],
@@ -32,6 +43,7 @@ export const searchHandlers: Record<string, MetaSearchAgent> = {
     rerank: true,
     rerankThreshold: 0,
     searchWeb: true,
+    ...searchFunctions,
   }),
   wolframAlphaSearch: new MetaSearchAgent({
     activeEngines: ["wolframalpha"],
@@ -41,6 +53,7 @@ export const searchHandlers: Record<string, MetaSearchAgent> = {
     rerank: false,
     rerankThreshold: 0,
     searchWeb: true,
+    ...searchFunctions,
   }),
   youtubeSearch: new MetaSearchAgent({
     activeEngines: ["youtube"],
@@ -50,6 +63,7 @@ export const searchHandlers: Record<string, MetaSearchAgent> = {
     rerank: true,
     rerankThreshold: 0.3,
     searchWeb: true,
+    ...searchFunctions,
   }),
   redditSearch: new MetaSearchAgent({
     activeEngines: ["reddit"],
@@ -59,5 +73,13 @@ export const searchHandlers: Record<string, MetaSearchAgent> = {
     rerank: true,
     rerankThreshold: 0.3,
     searchWeb: true,
+    ...searchFunctions,
   }),
-};
+});
+
+/**
+ * Default search handlers without search functions.
+ * These will not perform actual web searches unless search functions are added to the Config.
+ * @deprecated Use createSearchHandlers() with search functions instead
+ */
+export const searchHandlers = createSearchHandlers();
