feat(perf): optimize callback insertion with parallel batching

- feat(backend): implement Promise.all batch parallel DB insertion in callback-route for 10-20x speedup
- refactor(phase5): cleanup obsolete pgvector/hybrid search code (schema, services, api)
- refactor(phase5): consolidate DriveConfig to DriveFolder model
- refactor(frontend): remove obsolete 'Balance' slider and alpha parameter
- docs: update roadmap, architecture, and api docs for Phase 5 completion

Author: namtroi

apps/ai-worker/src/embedder.py

@@ -17,6 +17,7 @@
 
 from .callback import send_callback
 from .config import settings
+from .hybrid_embedder import HybridEmbedder
 from .logging_config import configure_logging, get_logger
 from .metrics import MetricsCollector
 from .models import (
@@ -71,7 +72,6 @@ async def readiness_check():
 @app.post("/embed", response_model=EmbedResponse)
 async def embed_texts(request: EmbedRequest):
     """Generate dense-only embeddings for a list of texts (backward compatibility)."""
-    from .hybrid_embedder import HybridEmbedder
 
     if not request.texts:
         return EmbedResponse(embeddings=[])
@@ -92,7 +92,6 @@ async def embed_query(request: dict):
 
     Returns both dense (384d) and sparse (BM25) vectors for Qdrant hybrid search.
     """
-    from .hybrid_embedder import HybridEmbedder
     from .models import HybridEmbedResponse, SparseVectorModel
 
     text = request.get("text", "")

apps/ai-worker/src/main.py

@@ -96,13 +96,15 @@ class TestEmbeddingDimensions:
     """Regression tests for embedding dimensions."""
 
     def test_embedding_dimensions_unchanged(self):
-        """Embeddings should still be 384 dimensions."""
-        from src.embedder import Embedder
+        """HybridEmbedder should return 384 dense dimensions."""
+        from src.hybrid_embedder import HybridEmbedder
 
-        embedder = Embedder()
-        embedding = embedder.embed("Test text")
+        embedder = HybridEmbedder()
+        result = embedder.embed(["Test text"])
 
-        assert len(embedding) == 384
+        # HybridEmbedder returns list of HybridVector
+        assert len(result) == 1
+        assert len(result[0].dense) == 384
 
 
 class TestChunkStructure:

apps/ai-worker/tests/fixtures/sample.pdf

@@ -188,9 +188,10 @@ async def test_embed_success(self):
         from fastapi.testclient import TestClient
         from src.main import app
 
-        with patch("src.embedder.Embedder") as MockEmbedder:
+        # HybridEmbedder is now imported at top of main.py, so mock works
+        with patch("src.main.HybridEmbedder") as MockEmbedder:
             mock_instance = MockEmbedder.return_value
-            mock_instance.embed.return_value = [[0.1] * 384, [0.2] * 384]
+            mock_instance.embed_dense_only.return_value = [[0.1] * 384, [0.2] * 384]
 
             with TestClient(app) as client:
                 response = client.post(

apps/ai-worker/tests/regression/test_existing_formats.py

@@ -64,7 +64,7 @@ model Document {
   // Phase 2E: Drive Sync
   sourceType          SourceType    @default(MANUAL) @map("source_type")
   driveFileId         String?       @unique @map("drive_file_id")
-  driveConfigId       String?       @map("drive_config_id")
+  driveFolderId       String?       @map("drive_folder_id")
   driveWebViewLink    String?       @map("drive_web_view_link")
   driveModifiedTime   DateTime?     @map("drive_modified_time")
   lastSyncedAt        DateTime?     @map("last_synced_at")
@@ -77,15 +77,15 @@ model Document {
   formatCategory  String?       @map("format_category")
 
   chunks              Chunk[]
-  driveConfig         DriveConfig?  @relation(fields: [driveConfigId], references: [id], onDelete: SetNull)
+  driveFolder         DriveFolder?  @relation(fields: [driveFolderId], references: [id], onDelete: SetNull)
   processingMetrics   ProcessingMetrics?
   
   // Phase 5: Processing Profile
   processingProfileId  String?  @map("processing_profile_id")
   processingProfile    ProcessingProfile? @relation(fields: [processingProfileId], references: [id], onDelete: SetNull)
 
   @@index([sourceType])
-  @@index([driveConfigId])
+  @@index([driveFolderId])
   @@index([isActive])
   @@index([connectionState])
   @@index([formatCategory])
@@ -114,9 +114,6 @@ model Chunk {
   breadcrumbs    String[]  @default([])
   tokenCount     Int?      @map("token_count")
 
-  // Hybrid Search: BM25 full-text search vector
-  searchVector   Unsupported("tsvector")?  @map("search_vector")
-
   // Phase 5D: Qdrant Sync (Outbox Pattern)
   syncStatus     ChunkSyncStatus  @default(PENDING) @map("sync_status")
   denseVector    Float[]          @default([]) @map("dense_vector")
@@ -128,7 +125,6 @@ model Chunk {
   @@unique([documentId, chunkIndex])
   @@index([qualityScore])
   @@index([chunkType])
-  @@index([searchVector], type: Gin)
   @@index([syncStatus])
   @@map("chunks")
 }
@@ -140,7 +136,7 @@ enum ChunkSyncStatus {
   FAILED
 }
 
-model DriveConfig {
+model DriveFolder {
   id            String      @id @default(uuid())
   folderId      String      @unique @map("folder_id")
   folderName    String      @map("folder_name")
@@ -154,11 +150,6 @@ model DriveConfig {
   createdAt     DateTime    @default(now()) @map("created_at")
   updatedAt     DateTime    @updatedAt @map("updated_at")
 
-  // Phase 5A: AES-256-GCM Encrypted OAuth Tokens
-  encryptedRefreshToken  String?  @map("encrypted_refresh_token")
-  tokenIv                String?  @map("token_iv")
-  tokenAuthTag           String?  @map("token_auth_tag")
-
   documents     Document[]
 
   // Phase 5: Processing Profile
@@ -167,7 +158,7 @@ model DriveConfig {
 
   @@index([enabled])
   @@index([processingProfileId])
-  @@map("drive_configs")
+  @@map("drive_folders")
 }
 
 // Phase 5F: OAuth Credentials (encrypted refresh tokens)
@@ -264,8 +255,8 @@ model ProcessingProfile {
   embeddingDimension      Int     @default(384) @map("embedding_dimension")
   embeddingMaxTokens      Int     @default(512) @map("embedding_max_tokens")
   
-  documents     Document[]
-  driveConfigs  DriveConfig[]
+  documents      Document[]
+  driveFolders   DriveFolder[]
   
   @@map("processing_profiles")
 }

apps/ai-worker/tests/test_embedder.py

@@ -26,8 +26,6 @@ import { disconnectPrisma } from './services/database.js';
 import { overviewRoute, processingRoute, qualityRoute, documentsRoute } from './routes/analytics/index.js';
 import { chunksRoute } from './routes/chunks/index.js';
 import { profileRoutes } from './routes/profiles/index.js';
-// Hybrid Search Infrastructure
-import { initializeHybridSearch } from './services/hybrid-search-init.js';
 // Default Profile
 import { ensureDefaultProfile } from './services/default-profile.js';
 // Phase 5F: OAuth
@@ -83,10 +81,6 @@ export async function createApp(): Promise<FastifyInstance> {
     initializeCronJobs().catch(err => {
       console.error('Failed to initialize cron jobs:', err);
     });
-    // Initialize hybrid search (tsvector trigger)
-    initializeHybridSearch().catch(err => {
-      console.error('Failed to initialize hybrid search:', err);
-    });
   }
 
   // Register protected routes (Auth applied here)

apps/ai-worker/tests/test_main.py

@@ -13,13 +13,13 @@ import cron, { ScheduledTask } from 'node-cron';
 const scheduledTasks: Map<string, ScheduledTask> = new Map();
 
 /**
- * Initialize cron jobs for all enabled DriveConfigs
+ * Initialize cron jobs for all enabled DriveFolders
  */
 export async function initializeCronJobs(): Promise<void> {
     const prisma = getPrismaClient();
 
     // Get all enabled configs
-    const configs = await prisma.driveConfig.findMany({
+    const configs = await prisma.driveFolder.findMany({
         where: { enabled: true },
         select: { id: true, syncCron: true },
     });