Adjust search index

Author: heussd

python/ai_search.py

@@ -7,6 +7,10 @@
 
 
 def add(searchDocs: List[dict]):
+    if len(searchDocs) == 0:
+        print("No documents to add.")
+        return
+
     data = {"value": [searchDoc for searchDoc in searchDocs]}
 
     res = requests.post(
@@ -27,8 +31,8 @@ def search(query: str):
     data = json.dumps(
         {
             "search": query,
-            "select": "title, content, url, author, language, date, excerpt, baseUrl",
-            "facets": ["date", "language", "author", "baseUrl"],
+            "top": 10,
+            "select": "title, content, url, author, language, date, baseUrl",
             "vectorQueries": [
                 {
                     "kind": "vector",

python/compose.yml

@@ -1,7 +1,10 @@
 services:
   service:
-    build: .
+    build:
+      context: .
+      target: search-mcp-server
     env_file:
       - ../.env
     ports:
       - "8000:8000"
+

python/index.py

@@ -5,6 +5,7 @@
 
 import config
 from ai_search import add
+import base64
 
 
 def prepare(searchDoc: Union[SearchDoc, List[SearchDoc]]):
@@ -13,24 +14,29 @@ def prepare(searchDoc: Union[SearchDoc, List[SearchDoc]]):
     else:
         documents = searchDoc
 
-    embeddings_list = list(config.model.embed([doc.content for doc in documents]))
+    embeddings_list = list(config.model.embed([f"{doc.title} {doc.excerpt} {doc.content}" for doc in documents]))
 
-    jsonDocs = [
-        {
+    jsonDocs = []
+
+    for idx, searchDoc in enumerate(documents):
+        if not searchDoc.url:
+            continue
+
+        jsonDocs.append(
+            {
             "@search.action": "mergeOrUpload",
-            "id": searchDoc.id,
+            "id": base64.b64encode(searchDoc.url.encode()).decode(),
             "title": searchDoc.title,
             "excerpt": searchDoc.excerpt,
             "author": searchDoc.author,
             "language": searchDoc.language,
-            "url": searchDoc.url or "",
+            "url": searchDoc.url,
             "baseUrl": "/".join(searchDoc.url.split("/")[:3]) if searchDoc.url else "",
             "content": searchDoc.content,
-            "vector": embeddings_list[documents.index(searchDoc)],
+            "vector": embeddings_list[idx],
             "date": searchDoc.date or str(datetime.now().astimezone().isoformat()),
-        }
-        for searchDoc in documents
-    ]
+            }
+        )
 
     return jsonDocs
 
@@ -43,13 +49,15 @@ def prepare(searchDoc: Union[SearchDoc, List[SearchDoc]]):
             author="John Doe",
             language="en",
             content="Gday this is an important Australian message.",
+            url="https://example.com/doc1",
         ),
         SearchDoc(
             title="Example Document",
             excerpt="This is an example excerpt.",
             author="John Doe",
             language="de",
             content="Guten Tag das ist eine wichtige deutsche Nachricht.",
+            url="https://example.com/doc2",
         ),
     ]
 

python/model.py

@@ -3,12 +3,9 @@
 from typing import List, Union
 from numpyencoder import NumpyEncoder
 from pydantic import BaseModel
-from uuid import uuid4
-from pydantic import Field
 
 
 class SearchDoc(BaseModel):
-    id: str = Field(default_factory=lambda: str(uuid4()))
     title: str = ""
     excerpt: str = ""
     author: str = ""

python/search.py

@@ -4,5 +4,5 @@
 
 
 if __name__ == "__main__":
-    query = "space"
+    query = "deutsch"
     print(json.dumps(search(query), indent=2))