Merge pull request #56 from TBosak/suggestion-engine

Updated suggestion engine heuristics + added loading indicator, deduplication in strict mode, etc.

Author: TBosak

frontend/src/components/forms/WebScrapingForm.tsx

@@ -1,3 +1,4 @@
+import { useState } from "react";
 import {
   UseFormRegister,
   Control,
@@ -30,6 +31,7 @@ import {
 import { Info, Sparkles, Link, Wand2 } from "lucide-react";
 import { SelectorField } from "./SelectorField";
 import { SelectorPlayground } from "./SelectorPlayground";
+import { LoadingSpinner } from "@/components/ui/loading-spinner";
 
 interface WebScrapingFormProps {
   register: UseFormRegister<FeedFormData>;
@@ -47,13 +49,15 @@ export const WebScrapingForm = ({
   feedUrl,
 }: WebScrapingFormProps) => {
   const dateFormat = watch("dateFormat");
+  const [isLoadingSelectors, setIsLoadingSelectors] = useState(false);
 
   const handleAutoFillSelectors = async () => {
     if (!feedUrl) {
       alert("Please enter a target URL first.");
       return;
     }
 
+    setIsLoadingSelectors(true);
     try {
       const response = await fetch("/utils/suggest-selectors", {
         method: "POST",
@@ -89,11 +93,19 @@ export const WebScrapingForm = ({
     } catch (error) {
       console.error("Error:", error);
       alert("An error occurred while auto-filling selectors.");
+    } finally {
+      setIsLoadingSelectors(false);
     }
   };
 
   return (
     <>
+      {isLoadingSelectors && (
+        <LoadingSpinner
+          fullscreen
+          message="Analyzing page structure and suggesting selectors..."
+        />
+      )}
       <SelectorPlayground
         feedUrl={feedUrl}
         setValue={setValue}
@@ -116,9 +128,10 @@ export const WebScrapingForm = ({
             variant="outline"
             onClick={handleAutoFillSelectors}
             className="w-full"
+            disabled={isLoadingSelectors}
           >
             <Sparkles className="mr-2 h-4 w-4" />
-            Suggest Selectors
+            {isLoadingSelectors ? "Analyzing..." : "Suggest Selectors"}
           </Button>
         </div>
 

index.ts

@@ -1200,40 +1200,34 @@ async function determineIsRelativeAndBaseUrl(
   userBaseUrl: string | undefined,
   feedUrl: string | undefined,
 ): Promise<{ isRelative: boolean; baseUrl: string | undefined }> {
+  // If user explicitly set both isRelative and baseUrl, use those
   if (typeof userIsRelative === "boolean" && userBaseUrl) {
     return { isRelative: userIsRelative, baseUrl: userBaseUrl };
   }
+
+  // If user explicitly set isRelative
   if (typeof userIsRelative === "boolean") {
     if (userIsRelative && !userBaseUrl && feedUrl) {
       return { isRelative: true, baseUrl: feedUrl };
     }
     return { isRelative: userIsRelative, baseUrl: userBaseUrl };
   }
+
+  // If user provided baseUrl but not isRelative, detect from URL format
   if (userBaseUrl) {
-    if (isLikelyAbsoluteUrl(url)) {
-      return { isRelative: false, baseUrl: userBaseUrl };
-    } else {
-      return { isRelative: true, baseUrl: userBaseUrl };
-    }
+    const isAbs = isLikelyAbsoluteUrl(url);
+    return { isRelative: !isAbs, baseUrl: userBaseUrl };
   }
-  if (isLikelyAbsoluteUrl(url)) {
-    try {
-      const resp = await axios.head(url, {
-        maxRedirects: 2,
-        validateStatus: () => true,
-      });
-      if (resp.status >= 200 && resp.status < 600) {
-        return { isRelative: false, baseUrl: undefined };
-      }
-    } catch {
-      if (feedUrl) return { isRelative: true, baseUrl: feedUrl };
-      return { isRelative: true, baseUrl: undefined };
-    }
-    if (feedUrl) return { isRelative: true, baseUrl: feedUrl };
-    return { isRelative: true, baseUrl: undefined };
+
+  // Auto-detect based on URL format
+  const isAbsolute = isLikelyAbsoluteUrl(url);
+
+  if (isAbsolute) {
+    return { isRelative: false, baseUrl: undefined };
+  } else {
+    // Relative URL - use feedUrl as base if available
+    return { isRelative: true, baseUrl: feedUrl };
   }
-  if (feedUrl) return { isRelative: true, baseUrl: feedUrl };
-  return { isRelative: true, baseUrl: undefined };
 }
 
 function extractSampleUrlFromHtml(
@@ -1242,12 +1236,27 @@ function extractSampleUrlFromHtml(
   attribute?: string,
 ): string {
   const $ = cheerio.load(html);
-  const el = $(selector).first();
-  if (!el.length) return "";
-  if (attribute) {
-    return el.attr(attribute) || "";
+  const elements = $(selector).slice(0, 5); // Check first 5 elements
+
+  if (elements.length === 0) return "";
+
+  // Try to find a non-empty URL from the sample
+  for (let i = 0; i < elements.length; i++) {
+    const el = elements.eq(i);
+    let url = "";
+    if (attribute) {
+      url = el.attr(attribute) || "";
+    } else {
+      url = el.attr("href") || el.attr("src") || "";
+    }
+
+    // Return first non-empty URL found
+    if (url && url.trim()) {
+      return url.trim();
+    }
   }
-  return el.attr("href") || el.attr("src") || "";
+
+  return "";
 }
 
 async function buildCSSTarget(
@@ -1287,6 +1296,7 @@ async function buildCSSTarget(
     );
     isRelative = result.isRelative;
     baseUrl = result.baseUrl;
+    console.log(`[Preview ${prefix}] Sample URL: "${urlSample}" → isRelative: ${isRelative}, baseUrl: ${baseUrl}`);
   }
 
   // Extract drill chain data directly if it was pre-processed into an array of objects
@@ -1564,9 +1574,12 @@ async function generatePreview(feedConfig: any) {
           },
           maxContentLength: 2 * 1024 * 1024,
           maxBodyLength: 2 * 1024 * 1024,
+          timeout: 30000, // 30 second timeout
         });
+        console.log("[Preview] Page fetched, building RSS...");
         const html = response.data;
         rssXml = await buildRSS(html, feedConfig);
+        console.log("[Preview] RSS build complete");
       }
     } else if (feedConfig.feedType === "api") {
       // feedConfig.config contains API call details (baseUrl, route, method, params, apiSpecificHeaders, apiSpecificBody)

package.json

@@ -2,7 +2,7 @@
   "name": "mkfd",
   "module": "index.ts",
   "type": "module",
-  "version": "2.0.6",
+  "version": "2.0.7",
   "scripts": {
     "build": "cd frontend && bun run build",
     "start": "bun run index.ts",