docs: improve doc parsing

.changeset/fifty-words-lick.md

@@ -0,0 +1,5 @@
+---
+"superfill.ai": patch
+---
+
+add observability in the document parser

.changeset/huge-suns-prove.md

@@ -0,0 +1,5 @@
+---
+"superfill.ai": minor
+---
+
+Updated doc parser prompt to stop redundant memory creation and added logic for deduplication of memory

.changeset/tender-months-wish.md

@@ -0,0 +1,5 @@
+---
+"superfill.ai": major
+---
+
+Updated the deduplication logic to give the preview of the new and existing duplicate memory in the UI

src/components/features/document/document-import-dialog.tsx

@@ -1,5 +1,6 @@
 import { FileTextIcon, UploadIcon } from "lucide-react";
-import { useRef, useState } from "react";
+import { useCallback, useRef, useState } from "react";
+import { toast } from "sonner";
 import {
   ImportDialogFooter,
   ImportDialogShell,
@@ -10,13 +11,15 @@ import {
 } from "@/components/features/import/import-dialog-shared";
 import { Button } from "@/components/ui/button";
 import { useImportDialog } from "@/hooks/use-import-dialog";
+import { useMemories } from "@/hooks/use-memories";
 import {
   convertToImportItems,
   type DocumentImportItem,
   type DocumentParserStatus,
   parseDocument,
 } from "@/lib/document/document-parser";
 import { createLogger } from "@/lib/logger";
+import { findDuplicates } from "@/lib/storage/memories";
 
 const logger = createLogger("component:document-import-dialog");
 
@@ -43,6 +46,7 @@ function getDescription(status: DocumentParserStatus): string {
     case "success":
       return "Select the information you want to import.";
     case "reading":
+      return "Reading your document...";
     case "parsing":
       return "AI is extracting your information...";
     case "error":
@@ -63,6 +67,11 @@ export function DocumentImportDialog({
 }: DocumentImportDialogProps) {
   const fileInputRef = useRef<HTMLInputElement>(null);
   const [fileName, setFileName] = useState<string | null>(null);
+  const lastImportKeyRef = useRef<string | null>(null);
+  const lastImportTimeRef = useRef<number>(0);
+  const abortControllerRef = useRef<AbortController | null>(null);
+
+  const { entries: existingMemories } = useMemories();
 
   const {
     status,
@@ -91,65 +100,111 @@ export function DocumentImportDialog({
 
   const progress = PROGRESS_BY_STATUS[status];
 
-  const handleFileSelect = async (
-    event: React.ChangeEvent<HTMLInputElement>,
-  ) => {
-    const file = event.target.files?.[0];
-    if (!file) return;
-
-    const isPdf =
-      file.type === "application/pdf" ||
-      file.name.toLowerCase().endsWith(".pdf");
-    const isTxt =
-      file.type === "text/plain" || file.name.toLowerCase().endsWith(".txt");
-
-    if (!isPdf && !isTxt) {
-      setError("Please select a PDF or text file");
-      setStatus("error");
-      event.target.value = "";
-      return;
-    }
-
-    setFileName(file.name);
-    setStatus("parsing");
-    setError(null);
-    setImportItems([]);
-
-    const currentRequestId = ++requestIdRef.current;
-
-    try {
-      const result = await parseDocument(file);
+  const handleFileSelect = useCallback(
+    async (event: React.ChangeEvent<HTMLInputElement>) => {
+      const file = event.target.files?.[0];
+      if (!file) return;
 
-      if (requestIdRef.current !== currentRequestId) return;
+      const isPdf =
+        file.type === "application/pdf" ||
+        file.name.toLowerCase().endsWith(".pdf");
+      const isTxt =
+        file.type === "text/plain" || file.name.toLowerCase().endsWith(".txt");
 
-      if (!result.success || !result.items) {
+      if (!isPdf && !isTxt) {
+        setError("Please select a PDF or text file");
         setStatus("error");
-        setError(result.error || "Failed to extract data from document");
+        event.target.value = "";
         return;
       }
 
-      const items = convertToImportItems(result.items);
-      setImportItems(items);
-      setStatus("success");
+      const importKey = `${file.name}:${file.size}`;
+      const now = Date.now();
+
+      if (
+        importKey === lastImportKeyRef.current &&
+        now - lastImportTimeRef.current < 5_000
+      ) {
+        logger.debug("Skipping duplicate import for:", file.name);
+        event.target.value = "";
+        return;
+      }
+
+      lastImportKeyRef.current = importKey;
+      lastImportTimeRef.current = now;
+      abortControllerRef.current?.abort();
+      const controller = new AbortController();
+      abortControllerRef.current = controller;
 
-      logger.debug("Successfully extracted document data:", items.length);
-    } catch (err) {
-      if (requestIdRef.current !== currentRequestId) return;
+      setFileName(file.name);
+      setStatus("reading");
+      setError(null);
+      setImportItems([]);
 
-      logger.error("Import error:", err);
-      setStatus("error");
-      setError(
-        err instanceof Error ? err.message : "An unexpected error occurred",
-      );
-    }
+      const currentRequestId = ++requestIdRef.current;
+      const requestId = String(currentRequestId);
 
-    if (fileInputRef.current) {
-      fileInputRef.current.value = "";
-    }
-  };
+      try {
+        const result = await parseDocument(file, {
+          requestId,
+          signal: controller.signal,
+          onStageChange: (stage) => {
+            if (requestIdRef.current !== currentRequestId) return;
+            setStatus(stage);
+          },
+        });
+
+        if (requestIdRef.current !== currentRequestId) return;
+
+        if (!result.success || !result.items) {
+          if (result.error === "cancelled") return;
+
+          const errorMsg =
+            result.error || "Failed to extract data from document";
+          setStatus("error");
+          setError(errorMsg);
+          toast.error(errorMsg);
+          return;
+        }
+
+        const items = convertToImportItems(result.items);
+        const duplicatesMap = await findDuplicates(items, existingMemories);
+        const enrichedItems = items.map((item, i) => {
+          const duplicate = duplicatesMap.get(i);
+          return duplicate ? { ...item, existingDuplicate: duplicate } : item;
+        });
+
+        setImportItems(enrichedItems);
+        setStatus("success");
+
+        logger.debug(
+          `[req:${requestId}] Successfully extracted document data:`,
+          items.length,
+          "items",
+        );
+      } catch (err) {
+        if (requestIdRef.current !== currentRequestId) return;
+        if (err instanceof Error && err.name === "AbortError") return;
+
+        const errMsg =
+          err instanceof Error ? err.message : "An unexpected error occurred";
+        logger.error(`[req:${requestId}] Import error:`, err);
+        setStatus("error");
+        setError(errMsg);
+        toast.error(errMsg);
+      }
+
+      if (fileInputRef.current) {
+        fileInputRef.current.value = "";
+      }
+    },
+    [requestIdRef, setStatus, setError, setImportItems, existingMemories],
+  );
 
   const handleCloseWrapper = (open: boolean) => {
     if (!open) {
+      abortControllerRef.current?.abort();
+      abortControllerRef.current = null;
       setFileName(null);
     }
     handleClose(open);

src/components/features/import/import-dialog-shared.tsx

@@ -24,6 +24,7 @@ import {
 } from "@/components/ui/dialog";
 import { Progress } from "@/components/ui/progress";
 import { ScrollArea } from "@/components/ui/scroll-area";
+import { cn } from "@/lib/cn";
 import type { BaseImportItem } from "@/types/import";
 import type { AllowedCategory } from "@/types/memory";
 
@@ -79,6 +80,7 @@ export function ImportItemsList<T extends BaseImportItem>({
   );
 
   const allSelected = items.every((item) => item.selected);
+  const duplicateCount = items.filter((item) => item.existingDuplicate).length;
 
   return (
     <div className="space-y-4">
@@ -88,6 +90,15 @@ export function ImportItemsList<T extends BaseImportItem>({
           <span className="text-sm font-medium">
             Found {items.length} items
           </span>
+          {duplicateCount > 0 && (
+            <Badge
+              variant="outline"
+              size="sm"
+              className="border-amber-400 text-amber-600 bg-amber-50 dark:bg-amber-950/30"
+            >
+              {duplicateCount} duplicate{duplicateCount > 1 ? "s" : ""}
+            </Badge>
+          )}
           {headerExtra}
         </div>
         <Button
@@ -122,7 +133,11 @@ export function ImportItemsList<T extends BaseImportItem>({
                 {categoryItems?.map((item) => (
                   <div
                     key={item.id}
-                    className="flex items-start gap-3 p-2 rounded-lg hover:bg-muted/50 transition-colors w-full"
+                    className={cn(
+                      "flex items-start gap-3 p-2 rounded-lg hover:bg-muted/50 transition-colors w-full",
+                      item.existingDuplicate &&
+                        "border border-amber-200 dark:border-amber-800/50 bg-amber-50/50 dark:bg-amber-950/10",
+                    )}
                   >
                     <Checkbox
                       id={`${itemIdPrefix}-${item.id}`}
@@ -134,12 +149,86 @@ export function ImportItemsList<T extends BaseImportItem>({
                       htmlFor={`${itemIdPrefix}-${item.id}`}
                       className="flex-1 min-w-0 cursor-pointer"
                     >
-                      <p className="text-sm font-medium truncate">
-                        {item.label}
-                      </p>
-                      <p className="text-xs text-muted-foreground line-clamp-2">
-                        {item.answer}
-                      </p>
+                      <div className="flex items-center gap-1.5 flex-wrap">
+                        <p className="text-sm font-medium truncate">
+                          {item.label}
+                        </p>
+                        {item.existingDuplicate && (
+                          <Badge
+                            variant="outline"
+                            size="sm"
+                            className="shrink-0 border-amber-400 text-amber-600 bg-amber-50 dark:bg-amber-950/30"
+                          >
+                            duplicate
+                          </Badge>
+                        )}
+                      </div>
+                      {item.question &&
+                        item.question.toLowerCase() !==
+                          item.label.toLowerCase() && (
+                          <p className="text-xs text-muted-foreground/70 italic truncate">
+                            {item.question}
+                          </p>
+                        )}
+                      <div
+                        className={cn(
+                          "mt-0.5",
+                          item.existingDuplicate && "space-y-1.5",
+                        )}
+                      >
+                        {item.existingDuplicate && (
+                          <p className="text-[11px] font-medium text-muted-foreground/60 uppercase tracking-wide">
+                            New
+                          </p>
+                        )}
+                        <p className="text-xs text-muted-foreground line-clamp-2">
+                          {item.answer}
+                        </p>
+                        {item.existingDuplicate && (
+                          <>
+                            <p className="text-[11px] font-medium text-muted-foreground/60 uppercase tracking-wide mt-1">
+                              Currently saved
+                            </p>
+                            <p className="text-xs text-muted-foreground/70 line-clamp-2 italic">
+                              {item.existingDuplicate.answer}
+                            </p>
+                            <div className="flex gap-1.5 mt-2">
+                              <button
+                                type="button"
+                                onClick={(e) => {
+                                  e.preventDefault();
+                                  e.stopPropagation();
+                                  if (!item.selected) onToggleItem(item.id);
+                                }}
+                                className={cn(
+                                  "text-[11px] px-2 py-0.5 rounded border font-medium transition-colors",
+                                  item.selected
+                                    ? "bg-primary text-primary-foreground border-primary"
+                                    : "bg-background text-muted-foreground border-border hover:border-primary hover:text-primary",
+                                )}
+                              >
+                                Use new
+                              </button>
+                              <button
+                                type="button"
+                                onClick={(e) => {
+                                  e.preventDefault();
+                                  e.stopPropagation();
+                                  if (item.selected) onToggleItem(item.id);
+                                }}
+                                className={cn(
+                                  "text-[11px] px-2 py-0.5 rounded border font-medium transition-colors",
+                                  !item.selected
+                                    ? "bg-primary text-primary-foreground border-primary"
+                                    : "bg-background text-muted-foreground border-border hover:border-primary hover:text-primary",
+                                )}
+                              >
+                                Keep existing
+                              </button>
+                            </div>
+                          </>
+                        )}
+                      </div>
                     </label>
                   </div>
                 ))}

src/hooks/use-import-dialog.ts

@@ -108,13 +108,18 @@ export function useImportDialog<
         description: successDescription,
       });
 
-      logger.debug("Successfully imported memories:", entries.length);
+      logger.debug(
+        `Import success — saved: ${entries.length}, total selected: ${selectedItems.length}`,
+      );
 
       resetState();
       onOpenChange(false);
       onSuccess?.();
     } catch (err) {
-      logger.error("Failed to save memories:", err);
+      logger.error(
+        `Import error — attempted: ${importItems.filter((i) => i.selected).length} items —`,
+        err,
+      );
       toast.error(
         err instanceof Error ? err.message : "Failed to save memories",
       );