397 cross project data leak (#427)

* Refactored filtering logic for retrieving chunks for a given course and conversation

* adding fixes to retrieve chunks for a particular conversation id and course prevent cross project data leak

* adding the missed must condition

Author: CodingWithJS

ai_ta_backend/database/vector.py

@@ -379,7 +379,14 @@ def process_clinicaltrials_results(results):
   def _create_search_filter(self, course_name: str, doc_groups: List[str], admin_disabled_doc_groups: List[str],
                             public_doc_groups: List[dict]) -> models.Filter:
     """
-    Create search conditions for the vector search.
+    Create search conditions for regular searches (no conversation filtering).
+    Excludes chunks with any conversation_id.
+    
+    Args:
+        course_name: The course/project name to filter by
+        doc_groups: List of document groups to include
+        admin_disabled_doc_groups: List of document groups to exclude
+        public_doc_groups: List of public document groups that can be accessed
     """
 
     must_conditions = []
@@ -390,6 +397,12 @@ def _create_search_filter(self, course_name: str, doc_groups: List[str], admin_d
     if admin_disabled_doc_groups:
       must_not_conditions.append(FieldCondition(key='doc_groups', match=MatchAny(any=admin_disabled_doc_groups)))
 
+    # For regular searches, only include chunks that have NO conversation_id field
+    # This ensures we only get regular course chunks and prevents cross-conversation leaks
+    must_conditions.append(models.IsEmptyCondition(
+        is_empty={"key": "conversation_id"}  # Only include chunks where conversation_id field is empty/missing
+    ))
+    
     # Handle public_doc_groups
     if public_doc_groups:
       for public_doc_group in public_doc_groups:
@@ -411,12 +424,31 @@ def _create_search_filter(self, course_name: str, doc_groups: List[str], admin_d
     # Add the own_course_condition to should_conditions
     should_conditions.append(own_course_condition)
 
-    # Construct the final filter
-    vector_search_filter = models.Filter(should=should_conditions, must_not=must_not_conditions)
+    # Construct the final filter (apply must to enforce no conversation_id)
+    vector_search_filter = models.Filter(must=must_conditions, should=should_conditions, must_not=must_not_conditions)
 
     print(f"Vector search filter: {vector_search_filter}")
     return vector_search_filter
 
+  def _create_conversation_search_filter(self, conversation_id: str) -> models.Filter:
+    """
+    Create search conditions for conversation-specific chunks.
+    Only includes chunks with the specified conversation_id.
+    
+    Args:
+        conversation_id: The specific conversation ID to filter by
+    """
+
+    must_conditions = []
+
+    # Conversation ID filter - this is sufficient since conversation_id is unique
+    must_conditions.append(FieldCondition(
+        key='conversation_id', 
+        match=MatchValue(value=conversation_id)
+    ))
+    
+    return models.Filter(must=must_conditions)
+
   def delete_data(self, collection_name: str, key: str, value: str):
     """
     Delete data from the vector database.
@@ -460,14 +492,35 @@ def _create_conversation_filter(self, conversation_id: str) -> models.Filter:
         ]
     )
 
-  def _combine_filters(self, search_filter: models.Filter, conversation_filter: models.Filter) -> models.Filter:
+  def _combine_filters(self, search_filter: models.Filter, conversation_filter: models.Filter = None) -> models.Filter:
     """
-    Combine search filter with conversation filter using OR logic.
-    This allows searching both regular course documents AND conversation-specific documents.
+    Combine search filter with conversation filter using AND logic.
+    
+    Args:
+        search_filter: The main search filter (course_name, doc_groups, etc.)
+        conversation_filter: The conversation-specific filter (optional)
+    
+    Returns:
+        Combined filter using AND logic for security
     """
-    return models.Filter(
-        should=[search_filter, conversation_filter]
-    )
+    combined_conditions = []
+    
+    # Add conditions from search filter
+    if search_filter.must:
+        combined_conditions.extend(search_filter.must)
+    
+    # Add conditions from conversation filter if provided
+    if conversation_filter and conversation_filter.must:
+        combined_conditions.extend(conversation_filter.must)
+    
+    # Combine must_not conditions
+    combined_must_not = []
+    if search_filter.must_not:
+        combined_must_not.extend(search_filter.must_not)
+    if conversation_filter and conversation_filter.must_not:
+        combined_must_not.extend(conversation_filter.must_not)
+    
+    return models.Filter(must=combined_conditions, must_not=combined_must_not)
 
   def vector_search_with_filter(self, search_query, course_name, doc_groups: List[str], 
                                  user_query_embedding, top_n, disabled_doc_groups: List[str], 

ai_ta_backend/service/retrieval_service.py

@@ -33,7 +33,7 @@
 # from ai_ta_backend.service.nomic_service import NomicService
 from ai_ta_backend.service.posthog_service import PosthogService
 from ai_ta_backend.service.sentry_service import SentryService
-
+from qdrant_client.http import models
 
 # Qwen query instruction for Illinois Chat retrieval.
 # Docs are embedded without instruction during ingest; only queries get this prefix.
@@ -543,10 +543,19 @@ def vector_search(self,
     else:
       # Handle conversation filtering for normal courses
       if conversation_id:
-          conversation_filter = self._create_conversation_filter(conversation_id)
-          combined_filter = self._combine_filters(
-              self._create_search_filter(course_name, doc_groups, disabled_doc_groups, public_doc_groups),
-              conversation_filter
+          # For chat conversations: get BOTH regular course documents AND conversation-specific documents
+          
+          # Get regular course documents (course_name + no conversation_id)
+          regular_filter = self.vdb._create_search_filter(
+              course_name, doc_groups, disabled_doc_groups, public_doc_groups
+          )
+          
+          # Get conversation-specific documents (this conversation_id)
+          chat_filter = self.vdb._create_conversation_search_filter(conversation_id)
+          
+          # Combine both filters with OR logic to get both types of documents
+          combined_filter = models.Filter(
+              should=[regular_filter, chat_filter]
           )
 
           search_results = self.vdb.vector_search_with_filter(
@@ -821,41 +830,7 @@ def _create_conversation_filter(self, conversation_id: str):
         ]
     )
 
-  def _combine_filters(self, filter1, filter2):
-    """Combine two Qdrant filters with AND logic."""
-    from qdrant_client.http import models
-    combined_conditions = []
-    
-    # Add conditions from first filter
-    if filter1.must:
-        combined_conditions.extend(filter1.must)
-    
-    # Add conditions from second filter  
-    if filter2.must:
-        combined_conditions.extend(filter2.must)
-    
-    return models.Filter(must=combined_conditions)
-
-  def _create_search_filter(self, course_name, doc_groups, disabled_doc_groups, public_doc_groups):
-    """
-    Create a Qdrant filter for course, doc groups, and public/disabled doc groups.
-    """
-    from qdrant_client.http import models
-    
-    must_conditions = []
-    if course_name:
-        must_conditions.append(models.FieldCondition(
-            key="course_name",
-            match=models.MatchValue(value=course_name)
-        ))
-    if doc_groups and 'All Documents' not in doc_groups:
-        must_conditions.append(models.FieldCondition(
-            key="doc_groups",
-            match=models.MatchAny(any=doc_groups)  # Fixed: use 'any' parameter instead of 'value'
-        ))
-    # Optionally, you can add filters for disabled/public doc groups if needed
-    # (depends on your schema and use case)
-    return models.Filter(must=must_conditions)
+# Removed duplicate methods - now using consolidated methods from VectorDatabase
 
   # Add all these methods at the end of the RetrievalService class
 
@@ -1116,6 +1091,7 @@ def _store_conversation_content(self, text_content: str, conversation_id: str,
                 documents.append(doc)
 
             except Exception as e:
+                print("Error in _store_conversation_content: ", e)
                 pass
                 continue