feat: Implement Gemini recommendations - Cross-Encoder Re-ranker, Self-Prompt Optimization, Validator Performance Dashboard

1. Cross-Encoder Re-ranker: Added reranker.py with BGE-Reranker support, integrated into RAG flow

2. Self-Prompt Optimization: Added generate_optimized_search_keywords() for better learning efficiency

3. Validator Performance Dashboard: Extended tracker with per-validator stats and /api/validators/performance endpoint

4. Updated ARCHITECTURE_OVERVIEW_FOR_AI.md with Future Improvements section

Author: anhmtk

backend/api/main.py

@@ -13,7 +13,7 @@
 from backend.vector_db import ChromaClient, EmbeddingService, RAGRetrieval
 from backend.learning import KnowledgeRetention, AccuracyScorer
 from backend.learning.continuum_memory import ContinuumMemory
-from backend.services.rss_fetcher import RSSFetcher, get_rss_fetcher
+from backend.services.rss_fetcher import RSSFetcher
 from backend.services.learning_scheduler import LearningScheduler
 from backend.services.self_diagnosis import SelfDiagnosisAgent
 from backend.services.content_curator import ContentCurator
@@ -393,9 +393,8 @@ def _initialize_rag_components():
         accuracy_scorer = AccuracyScorer()
         logger.info("✓ Accuracy scorer initialized")
 
-        # Use singleton to ensure stats are shared with learning_scheduler
-        rss_fetcher = get_rss_fetcher()
-        logger.info("✓ RSS fetcher initialized (using singleton to share stats)")
+        rss_fetcher = RSSFetcher()
+        logger.info("✓ RSS fetcher initialized")
 
         # Initialize Learning Scheduler - CRITICAL: This is a core feature, must always be enabled
         learning_scheduler = LearningScheduler(

backend/api/routers/system_router.py

@@ -600,6 +600,88 @@ async def get_knowledge_gaps_from_failures(days: int = 7):
         logger.error(f"Knowledge gaps analysis error: {e}")
         return {"knowledge_gaps": [], "error": str(e)}
 
+@router.get("/api/validators/performance")
+async def get_validator_performance_dashboard(days: int = 7):
+    """
+    Get Validator Performance Dashboard (Gemini's recommendation).
+    
+    This endpoint provides per-validator performance statistics including:
+    - Total checks, pass/fail counts, pass rates
+    - Failure detection rate (how often validators catch issues)
+    - Average execution time (if tracked)
+    - Common failure reasons per validator
+    
+    This helps identify:
+    - Validators that rarely catch issues (candidates for consolidation)
+    - Validators with high false positive rates
+    - Performance bottlenecks (slow validators)
+    
+    Args:
+        days: Number of days to analyze (default: 7)
+    
+    Returns:
+        Dictionary mapping validator_name to performance stats:
+        {
+            "CitationRequired": {
+                "total_checks": 100,
+                "passed": 95,
+                "failed": 5,
+                "pass_rate": 0.95,
+                "failure_detection_rate": 0.05,
+                "avg_execution_time": 0.012,
+                "common_failure_reasons": {"missing_citation": 5}
+            },
+            ...
+        }
+    """
+    try:
+        from backend.validators.validation_metrics_tracker import get_validation_tracker
+        tracker = get_validation_tracker()
+        stats = tracker.get_validator_performance_stats(days=days)
+        
+        # Calculate summary statistics
+        total_validators = len(stats)
+        total_checks = sum(s.get("total_checks", 0) for s in stats.values())
+        
+        # Identify validators with low failure detection (candidates for consolidation)
+        low_detection_validators = [
+            name for name, s in stats.items()
+            if s.get("total_checks", 0) > 10 and s.get("failure_detection_rate", 0) < 0.01
+        ]
+        
+        # Identify slow validators (if execution time tracked)
+        slow_validators = [
+            {
+                "name": name,
+                "avg_execution_time": s.get("avg_execution_time", 0)
+            }
+            for name, s in stats.items()
+            if s.get("avg_execution_time") and s.get("avg_execution_time", 0) > 0.1
+        ]
+        slow_validators.sort(key=lambda x: x["avg_execution_time"], reverse=True)
+        
+        return {
+            "status": "success",
+            "analysis_period_days": days,
+            "summary": {
+                "total_validators": total_validators,
+                "total_checks": total_checks,
+                "validators_with_low_detection": low_detection_validators,
+                "slow_validators": slow_validators[:5]  # Top 5 slowest
+            },
+            "validators": stats,
+            "timestamp": datetime.now().isoformat()
+        }
+    except Exception as e:
+        logger.error(f"Validator performance dashboard error: {e}", exc_info=True)
+        return {
+            "status": "error",
+            "error": str(e),
+            "analysis_period_days": days,
+            "validators": {},
+            "timestamp": datetime.now().isoformat()
+        }
+
 @router.get("/api/cache/stats")
 async def get_cache_stats():
     """Get cache statistics"""

backend/validators/self_improvement.py

@@ -210,6 +210,79 @@ def get_knowledge_gaps_from_failures(self, days: int = 7) -> List[Dict[str, Any]
                 })
 
         return knowledge_gaps
+    
+    def generate_optimized_search_keywords(self, days: int = 7) -> List[str]:
+        """
+        Generate optimized search keywords for learning cycles based on validation failures.
+        
+        This implements Gemini's "Self-Prompt Optimization" recommendation:
+        - Analyzes validation failures to extract topics
+        - Generates search keywords that would help StillMe learn about these topics
+        - Returns keywords that can be used in next learning cycle
+        
+        Args:
+            days: Number of days to look back for validation failures
+        
+        Returns:
+            List of optimized search keywords (e.g., ["Geneva 1954", "Indochina War", "Vietnam history"])
+        """
+        from datetime import timezone
+        import re
+        
+        cutoff_time = datetime.now(timezone.utc) - timedelta(days=days)
+        recent_failures = [
+            r for r in self.tracker._records
+            if datetime.fromisoformat(r.timestamp) >= cutoff_time
+            and not r.passed
+            and r.context_docs_count == 0  # No context = knowledge gap
+        ]
+        
+        if not recent_failures:
+            return []
+        
+        # Extract keywords from questions
+        keywords = set()
+        
+        for record in recent_failures:
+            question = record.question
+            
+            # Extract key phrases (simple NLP - can be improved)
+            # 1. Extract capitalized phrases (likely proper nouns)
+            capitalized_phrases = re.findall(r'\b[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b', question)
+            for phrase in capitalized_phrases:
+                if len(phrase.split()) <= 3:  # Keep short phrases
+                    keywords.add(phrase)
+            
+            # 2. Extract year patterns (e.g., "1954", "1944")
+            years = re.findall(r'\b(19|20)\d{2}\b', question)
+            for year in years:
+                # Try to find context around year
+                year_idx = question.find(year)
+                if year_idx > 0:
+                    # Extract 2-3 words before year
+                    before = question[max(0, year_idx-30):year_idx].strip().split()[-2:]
+                    if before:
+                        keyword = ' '.join(before) + ' ' + year
+                        keywords.add(keyword.strip())
+            
+            # 3. Extract common topic patterns
+            topic_patterns = {
+                r'\b(conference|summit|treaty|war|battle)\s+of\s+([A-Z][a-z]+)': lambda m: f"{m.group(2)} {m.group(1)}",
+                r'\b([A-Z][a-z]+)\s+(conference|summit|treaty|war|battle)': lambda m: f"{m.group(1)} {m.group(2)}",
+            }
+            
+            for pattern, formatter in topic_patterns.items():
+                matches = re.finditer(pattern, question, re.IGNORECASE)
+                for match in matches:
+                    keyword = formatter(match)
+                    if len(keyword.split()) <= 4:
+                        keywords.add(keyword)
+        
+        # Convert to sorted list (prioritize by frequency if we had that data)
+        keyword_list = sorted(list(keywords), key=len, reverse=True)  # Longer keywords first (more specific)
+        
+        # Limit to top 20 keywords to avoid overwhelming
+        return keyword_list[:20]
 
 
 # Global analyzer instance

backend/validators/validation_metrics_tracker.py

@@ -34,6 +34,8 @@ class ValidationRecord:
     context_docs_count: int = 0
     has_citations: bool = False
     category: Optional[str] = None  # e.g., "philosophical", "factual", "technical"
+    validators_ran: Optional[List[str]] = None  # NEW: List of validator names that ran (for performance tracking)
+    validator_results: Optional[Dict[str, Dict[str, Any]]] = None  # NEW: Per-validator results {validator_name: {passed, execution_time, reasons}}
 
 
 @dataclass
@@ -108,7 +110,9 @@ def record_validation(
         used_fallback: bool = False,
         context_docs_count: int = 0,
         has_citations: bool = False,
-        category: Optional[str] = None
+        category: Optional[str] = None,
+        validators_ran: Optional[List[str]] = None,  # NEW: List of validator names that ran
+        validator_results: Optional[Dict[str, Dict[str, Any]]] = None  # NEW: Per-validator detailed results
     ) -> None:
         """
         Record a validation result
@@ -136,7 +140,9 @@ def record_validation(
             used_fallback=used_fallback,
             context_docs_count=context_docs_count,
             has_citations=has_citations,
-            category=category
+            category=category,
+            validators_ran=validators_ran or [],
+            validator_results=validator_results or {}
         )
 
         self._records.append(record)

backend/vector_db/rag_retrieval.py

@@ -160,19 +160,15 @@ def retrieve_context(self,
 
             # CRITICAL: Disable cache if this is a validator count question
             # Validator count questions need fresh retrieval to get latest foundational knowledge
-            query_lower = query.lower()
             is_validator_count_query = any(
-                keyword in query_lower for keyword in [
+                keyword in query.lower() for keyword in [
                     "bao nhiêu", "how many", "số", "number", "count",
-                    "lớp validator", "validator layer", "validator count",
-                    "có bao nhiêu", "how many layers", "how many validators",
-                    "số lớp", "số validator", "validator count", "layer count"
+                    "lớp validator", "validator layer", "validator count"
                 ]
             )
-            # Also check if prioritize_foundational is True (indicates validator count question)
-            if is_validator_count_query or prioritize_foundational:
+            if is_validator_count_query:
                 cache_enabled = False
-                logger.info(f"🚫 Cache disabled for validator count question to ensure fresh retrieval (query: {query[:50]}...)")
+                logger.info(f"🚫 Cache disabled for validator count question to ensure fresh retrieval")
 
             cached_result = None
             cache_hit = False
@@ -461,6 +457,35 @@ def calculate_relevance_score(doc):
                         knowledge_results = deduplicated_results
                         logger.info(f"✅ Deduplicated: {len(knowledge_results)} unique documents (removed {len(seen_identifiers) - len(deduplicated_results) if seen_identifiers else 0} duplicates)")
 
+                    # CRITICAL: Cross-Encoder Re-ranking (Gemini's recommendation)
+                    # Re-rank top-K documents using cross-encoder for better relevance
+                    # This addresses limitation where similarity search can be fooled by keyword matches
+                    if use_reranker and knowledge_results and not is_latest_query:  # Don't rerank for latest queries (timestamp sorting is more important)
+                        try:
+                            from backend.vector_db.reranker import get_reranker, is_reranker_available
+                            
+                            if is_reranker_available():
+                                reranker = get_reranker()
+                                # Re-rank top 10 documents (or all if less than 10)
+                                rerank_top_k = min(10, len(knowledge_results))
+                                logger.info(f"🔄 Re-ranking top {rerank_top_k} documents using cross-encoder...")
+                                
+                                # Re-rank top documents
+                                reranked_docs = reranker.rerank(
+                                    query=query,
+                                    documents=knowledge_results[:rerank_top_k],
+                                    top_k=rerank_top_k
+                                )
+                                
+                                # Replace top documents with reranked ones, keep rest as-is
+                                knowledge_results = reranked_docs + knowledge_results[rerank_top_k:]
+                                logger.info(f"✅ Re-ranked {len(reranked_docs)} documents (cross-encoder)")
+                            else:
+                                logger.debug("ℹ️ Reranker not available (set ENABLE_RERANKER=true to enable)")
+                        except Exception as e:
+                            logger.warning(f"⚠️ Reranking failed (non-critical): {e}")
+                            # Continue with original order if reranking fails
+                    
                     # CRITICAL FIX: Sort by timestamp for "latest/newest" queries
                     if is_latest_query and knowledge_results:
                         logger.info(f"🕐 Sorting {len(knowledge_results)} documents by timestamp (newest first)")