fix(agentic_rag): honor similarity_threshold in deduplicate_chunks

examples/python_agent_nodes/agentic_rag/skills.py

@@ -177,20 +177,35 @@ def find_quote_in_chunk(claim: str, chunk_text: str, min_length: int = 20) -> st
     return best_sentence
 
 
+def _token_set(text: str) -> set:
+    """Return the set of whitespace-delimited lower-case tokens."""
+    return set(text.lower().split())
+
+
+def _jaccard_similarity(a: set, b: set) -> float:
+    """Compute the Jaccard similarity between two token sets."""
+    if not a or not b:
+        return 0.0
+    return len(a & b) / len(a | b)
+
+
 def deduplicate_chunks(
     chunks: List[Dict], similarity_threshold: float = 0.9
 ) -> List[Dict]:
-    """Remove duplicate chunks based on text similarity"""
+    """Remove duplicate chunks based on text similarity."""
     unique_chunks = []
-    seen_texts = set()
 
     for chunk in chunks:
-        text_normalized = " ".join(chunk["text"].lower().split())
+        chunk_tokens = _token_set(chunk["text"])
+
+        if any(
+            _jaccard_similarity(chunk_tokens, _token_set(unique["text"]))
+            >= similarity_threshold
+            for unique in unique_chunks
+        ):
+            continue
 
-        # Simple deduplication based on normalized text
-        if text_normalized not in seen_texts:
-            unique_chunks.append(chunk)
-            seen_texts.add(text_normalized)
+        unique_chunks.append(chunk)
 
     return unique_chunks
 

examples/python_agent_nodes/agentic_rag/test_skills.py

@@ -0,0 +1,47 @@
+"""Regression tests for agentic_rag skills."""
+
+from skills import deduplicate_chunks
+
+
+def test_deduplicate_chunks_exact_duplicates():
+    chunks = [
+        {"text": "The quick brown fox jumps over the lazy dog."},
+        {"text": "The quick brown fox jumps over the lazy dog."},
+        {"text": "A completely different sentence for testing."},
+    ]
+    result = deduplicate_chunks(chunks, similarity_threshold=0.9)
+    assert len(result) == 2
+    assert result[0]["text"] == chunks[0]["text"]
+    assert result[1]["text"] == chunks[2]["text"]
+
+
+def test_deduplicate_chunks_near_duplicates():
+    base = " ".join(str(i) for i in range(1, 20))
+    near = " ".join(str(i) for i in range(1, 19)) + " twenty"
+    chunks = [
+        {"text": base},
+        {"text": near},
+        {"text": "something else entirely that does not match"},
+    ]
+    result = deduplicate_chunks(chunks, similarity_threshold=0.9)
+    assert len(result) == 2
+    assert result[0]["text"] == base
+    assert result[1]["text"] == chunks[2]["text"]
+
+
+def test_deduplicate_chunks_respects_threshold():
+    base = "alpha beta gamma delta epsilon"
+    similar = "alpha beta gamma delta zeta"
+    chunks = [
+        {"text": base},
+        {"text": similar},
+    ]
+    high = deduplicate_chunks(chunks, similarity_threshold=0.9)
+    assert len(high) == 2
+
+    low = deduplicate_chunks(chunks, similarity_threshold=0.5)
+    assert len(low) == 1
+
+
+def test_deduplicate_chunks_empty_input():
+    assert deduplicate_chunks([]) == []