update pathway vector store, fix tests (opea-project#940)

Signed-off-by: Berke <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: berkecanrizai

comps/vectorstores/pathway/requirements.txt

@@ -1,4 +1,7 @@
+langchain
+langchain-community
+langchain_huggingface
 langchain_openai
-pathway[xpack-llm] >= 0.14.1
+pathway[xpack-llm]
 sentence-transformers
-unstructured[all-docs] >= 0.10.28,<0.15
+unstructured[all-docs] >= 0.16

comps/vectorstores/pathway/vectorstore_pathway.py

@@ -6,7 +6,8 @@
 
 import pathway as pw
 from langchain import text_splitter
-from langchain_community.embeddings import HuggingFaceBgeEmbeddings, HuggingFaceHubEmbeddings
+from langchain_community.embeddings import HuggingFaceBgeEmbeddings
+from langchain_huggingface import HuggingFaceEndpointEmbeddings
 from pathway.xpacks.llm.parsers import ParseUnstructured
 from pathway.xpacks.llm.vector_store import VectorStoreServer
 
@@ -42,7 +43,7 @@
     if tei_embedding_endpoint:
         # create embeddings using TEI endpoint service
         logging.info(f"Initializing the embedder from tei_embedding_endpoint: {tei_embedding_endpoint}")
-        embeddings = HuggingFaceHubEmbeddings(model=tei_embedding_endpoint)
+        embeddings = HuggingFaceEndpointEmbeddings(model=tei_embedding_endpoint)
     else:
         # create embeddings using local embedding model
         embeddings = HuggingFaceBgeEmbeddings(model_name=EMBED_MODEL)

tests/vectorstores/test_vectorstores_pathway.sh

@@ -47,7 +47,7 @@ function start_service() {
 
     docker run -d --name="test-comps-vectorstore-pathway-ms" -e PATHWAY_HOST=${PATHWAY_HOST} -e PATHWAY_PORT=${PATHWAY_PORT} -e TEI_EMBEDDING_ENDPOINT=${TEI_EMBEDDING_ENDPOINT} -e http_proxy=$http_proxy -e https_proxy=$https_proxy -v $WORKPATH/comps/vectorstores/pathway/README.md:/app/data/README.md -p ${PATHWAY_PORT}:${PATHWAY_PORT} --network="host" opea/vectorstore-pathway:comps
 
-    sleep 45s
+    sleep 70s
 
     export PATHWAY_HOST=$ip_address  # needed in order to reach to vector store
 
@@ -60,8 +60,15 @@ function validate_microservice() {
     result=$(http_proxy=''
     curl http://${PATHWAY_HOST}:$PATHWAY_PORT/v1/retrieve \
         -X POST \
-        -d "{\"query\":\"test\",\"k\":3}" \
+        -d "{\"query\":\"pathway\",\"k\":5}" \
         -H 'Content-Type: application/json')
+
+    docs=$(http_proxy=''
+    curl http://${PATHWAY_HOST}:$PATHWAY_PORT/v1/inputs \
+        -X POST \
+        -H 'Content-Type: application/json')
+
+    echo "Indexed documents: $docs"
     if [[ $result == *"Pathway"* ]]; then
         echo "Result correct."
     else