Merge pull request #486 from rahulharpal1603/backend-revamp

Add sync microservice for watching file system events

Author: rahulharpal1603

backend/app/database/folders.py

@@ -342,6 +342,46 @@ def db_get_folder_ids_by_path_prefix(root_path: str) -> List[FolderIdPath]:
         conn.close()
 
 
+def db_get_folder_ids_by_paths(
+    folder_paths: List[FolderPath],
+) -> Dict[FolderPath, FolderId]:
+    """
+    Get folder IDs for multiple folder paths in a single database query.
+
+    Args:
+        folder_paths: List of folder paths to look up
+
+    Returns:
+        Dictionary mapping folder paths to their corresponding folder IDs
+    """
+    if not folder_paths:
+        return {}
+
+    conn = sqlite3.connect(DATABASE_PATH)
+    cursor = conn.cursor()
+
+    try:
+        # Convert all paths to absolute paths
+        abs_paths = [os.path.abspath(path) for path in folder_paths]
+
+        # Create placeholders for the IN clause
+        placeholders = ",".join("?" * len(abs_paths))
+
+        cursor.execute(
+            f"SELECT folder_path, folder_id FROM folders WHERE folder_path IN ({placeholders})",
+            abs_paths,
+        )
+
+        results = cursor.fetchall()
+
+        # Create a mapping from folder_path to folder_id
+        path_to_id = {folder_path: folder_id for folder_path, folder_id in results}
+
+        return path_to_id
+    finally:
+        conn.close()
+
+
 def db_get_direct_child_folders(parent_folder_id: str) -> List[Tuple[str, str]]:
     """
     Get all direct child folders (not subfolders) for a given parent folder.

backend/app/database/images.py

@@ -195,3 +195,75 @@ def db_insert_image_classes_batch(image_class_pairs: List[ImageClassPair]) -> bo
         return False
     finally:
         conn.close()
+
+
+def db_get_images_by_folder_ids(
+    folder_ids: List[int],
+) -> List[Tuple[ImageId, ImagePath, str]]:
+    """
+    Get all images that belong to the specified folder IDs.
+
+    Args:
+        folder_ids: List of folder IDs to search for images
+
+    Returns:
+        List of tuples containing (image_id, image_path, thumbnail_path)
+    """
+    if not folder_ids:
+        return []
+
+    conn = sqlite3.connect(DATABASE_PATH)
+    cursor = conn.cursor()
+
+    try:
+        # Create placeholders for the IN clause
+        placeholders = ",".join("?" for _ in folder_ids)
+        cursor.execute(
+            f"""
+            SELECT id, path, thumbnailPath
+            FROM images
+            WHERE folder_id IN ({placeholders})
+            """,
+            folder_ids,
+        )
+        return cursor.fetchall()
+    except Exception as e:
+        print(f"Error getting images by folder IDs: {e}")
+        return []
+    finally:
+        conn.close()
+
+
+def db_delete_images_by_ids(image_ids: List[ImageId]) -> bool:
+    """
+    Delete multiple images from the database by their IDs.
+    This will also delete associated records in image_classes due to CASCADE.
+
+    Args:
+        image_ids: List of image IDs to delete
+
+    Returns:
+        True if deletion was successful, False otherwise
+    """
+    if not image_ids:
+        return True
+
+    conn = sqlite3.connect(DATABASE_PATH)
+    cursor = conn.cursor()
+
+    try:
+        # Create placeholders for the IN clause
+        placeholders = ",".join("?" for _ in image_ids)
+        cursor.execute(
+            f"DELETE FROM images WHERE id IN ({placeholders})",
+            image_ids,
+        )
+        conn.commit()
+        print(f"Deleted {cursor.rowcount} obsolete image(s) from database")
+        return True
+    except Exception as e:
+        print(f"Error deleting images: {e}")
+        conn.rollback()
+        return False
+    finally:
+        conn.close()

backend/app/routes/folders.py

@@ -1,4 +1,5 @@
 from fastapi import APIRouter, HTTPException, status, Depends, Request
+from typing import List, Tuple
 from app.database.folders import (
     db_update_parent_ids_for_subtree,
     db_folder_exists,
@@ -7,6 +8,7 @@
     db_disable_ai_tagging_batch,
     db_delete_folders_batch,
     db_get_direct_child_folders,
+    db_get_folder_ids_by_path_prefix,
 )
 from app.schemas.folders import (
     AddFolderRequest,
@@ -37,15 +39,24 @@
 router = APIRouter()
 
 
-def post_folder_add_sequence(folder_path: str):
+def post_folder_add_sequence(folder_path: str, folder_id: int):
     """
     Post-addition sequence for a folder.
     This function is called after a folder is successfully added.
     It processes images in the folder and updates the database.
     """
     try:
-        # Process images in the folder
-        image_util_process_folder_images(folder_path)
+        # Get all folder IDs and paths that match the root path prefix
+        folder_data = []
+        folder_ids_and_paths = db_get_folder_ids_by_path_prefix(folder_path)
+
+        # Set all folders to non-recursive (False)
+        for folder_id_from_db, folder_path_from_db in folder_ids_and_paths:
+            folder_data.append((folder_path_from_db, folder_id_from_db, False))
+
+        print("Add folder: ", folder_data)
+        # Process images in all folders
+        image_util_process_folder_images(folder_data)
 
     except Exception as e:
         print(f"Error in post processing after folder {folder_path} was added: {e}")
@@ -68,6 +79,34 @@ def post_AI_tagging_enabled_sequence():
     return True
 
 
+def post_sync_folder_sequence(
+    folder_path: str, folder_id: int, added_folders: List[Tuple[str, str]]
+):
+    """
+    Post-sync sequence for a folder.
+    This function is called after a folder is synced.
+    It processes images in the folder and updates the database.
+    """
+    try:
+        # Create folder data array
+        folder_data = []
+
+        folder_data.append((folder_path, folder_id, False))
+
+        for added_folder_id, added_folder_path in added_folders:
+            folder_data.append((added_folder_path, added_folder_id, False))
+
+        print("Sync folder: ", folder_data)
+        # Process images in all folders
+        image_util_process_folder_images(folder_data)
+        image_util_process_untagged_images()
+        cluster_util_face_clusters_sync()
+    except Exception as e:
+        print(f"Error in post processing after folder {folder_path} was synced: {e}")
+        return False
+    return True
+
+
 def get_state(request: Request):
     return request.app.state
 
@@ -132,7 +171,7 @@ def add_folder(request: AddFolderRequest, app_state=Depends(get_state)):
 
         # Step 6: Call the post-addition sequence in a separate process
         executor: ProcessPoolExecutor = app_state.executor
-        executor.submit(post_folder_add_sequence, request.folder_path)
+        executor.submit(post_folder_add_sequence, request.folder_path, root_folder_id)
 
         return AddFolderResponse(
             success=True,
@@ -287,33 +326,43 @@ def delete_folders(request: DeleteFoldersRequest):
     response_model=SyncFolderResponse,
     responses={code: {"model": ErrorResponse} for code in [400, 404, 500]},
 )
-def sync_folder(request: SyncFolderRequest):
+def sync_folder(request: SyncFolderRequest, app_state=Depends(get_state)):
     """Sync a folder by comparing filesystem folders with database entries and removing extra DB entries."""
     try:
-        # Step 1: Validate request
-
-        # Step 2: Get current state from both sources
+        # Step 1: Get current state from both sources
         db_child_folders = db_get_direct_child_folders(request.folder_id)
         filesystem_folders = folder_util_get_filesystem_direct_child_folders(
             request.folder_path
         )
 
-        # Step 3: Compare and identify differences
+        # Step 2: Compare and identify differences
         filesystem_folder_set = set(filesystem_folders)
         db_folder_paths = {folder_path for folder_id, folder_path in db_child_folders}
 
         folders_to_delete = db_folder_paths - filesystem_folder_set
         folders_to_add = filesystem_folder_set - db_folder_paths
 
-        # Step 4: Perform synchronization operations
+        # Step 3: Perform synchronization operations
         deleted_count, deleted_folders = folder_util_delete_obsolete_folders(
             db_child_folders, folders_to_delete
         )
-        added_count, added_folders = folder_util_add_multiple_folder_trees(
+        added_count, added_folders_with_ids = folder_util_add_multiple_folder_trees(
             folders_to_add, request.folder_id
         )
 
-        # Step 5: Return comprehensive response
+        # Extract just the paths for the API response
+        added_folders = [
+            folder_path for folder_id, folder_path in added_folders_with_ids
+        ]
+
+        executor: ProcessPoolExecutor = app_state.executor
+        executor.submit(
+            post_sync_folder_sequence,
+            request.folder_path,
+            request.folder_id,
+            added_folders_with_ids,
+        )
+        # Step 4: Return comprehensive response
         return SyncFolderResponse(
             success=True,
             message=f"Successfully synced folder. Added {added_count} folder(s), deleted {deleted_count} folder(s)",

backend/app/utils/folders.py

@@ -131,7 +131,7 @@ def folder_util_delete_obsolete_folders(
 
 def folder_util_add_multiple_folder_trees(
     folders_to_add: set, parent_folder_id: str
-) -> Tuple[int, List[str]]:
+) -> Tuple[int, List[Tuple[str, str]]]:
     """
     Add multiple folder trees with same parent to the database.
 
@@ -140,12 +140,12 @@ def folder_util_add_multiple_folder_trees(
         parent_folder_id: ID of the parent folder
 
     Returns:
-        Tuple of (added_count, added_folders_list)
+        Tuple of (added_count, added_folders_list) where added_folders_list contains (folder_id, folder_path) tuples
     """
     if not folders_to_add:
         return 0, []
 
-    added_folders = []
+    added_folders = []  # List of (folder_id, folder_path) tuples
     added_count = 0
 
     for folder_path in folders_to_add:
@@ -161,7 +161,10 @@ def folder_util_add_multiple_folder_trees(
             # Update parent IDs for the new folder tree
             db_update_parent_ids_for_subtree(folder_path, folder_map)
 
-            added_folders.append(folder_path)
+            # Add all folders from the folder_map as (folder_id, folder_path) tuples
+            for folder_path_in_map, (folder_id_in_map, _) in folder_map.items():
+                added_folders.append((folder_id_in_map, folder_path_in_map))
+
             added_count += len(folder_map)  # Count all folders in the tree
 
         except Exception as e: