[CAI-718] Fix doc id when adding missing folders to vector index (#1963)

* Fix getting refenrece folder dirnames that belongs to static documents

* Fix getting refenrece folder dirnames that belongs to static documents

* Update schedule time to 22:00 for the workflow

* Add changeset

Author: mdciri

.changeset/floppy-ears-stare.md

@@ -0,0 +1,5 @@
+---
+"chatbot-index": patch
+---
+
+Fix to consider only docIDs that belong to static docs when adding missing folders md files to vector index

.github/workflows/chatbot_refresh_vector_index.yaml

@@ -25,7 +25,7 @@ on:
           - add missing static
 
   schedule:
-    - cron: '0 1 * * *'  # Run daily at 01:00 AM UTC (03:00 AM CEST) to refresh the index in prod
+    - cron: '0 22 * * *'  # Run daily at 10:00 PM UTC (11:00 PM CEST) to refresh the index in prod
 
 permissions:
   id-token: write

apps/chatbot-index/src/modules/add_missing_static_docs.py

@@ -3,6 +3,7 @@
     get_one_metadata_from_s3,
     get_folders_list,
     StaticMetadata,
+    DOCS_PARENT_FOLDER,
 )
 from src.modules.vector_index import DiscoveryVectorIndex
 from src.modules.settings import SETTINGS
@@ -24,7 +25,9 @@
     index = VECTOR_INDEX.get_index()
     ref_doc_info = index.storage_context.docstore.get_all_ref_doc_info()
     ref_doc_ids = list(ref_doc_info.keys())
-    ref_folders = [doc_id.split("/")[2] for doc_id in ref_doc_ids]
+    ref_folders = [
+        doc_id.split("/")[2] for doc_id in ref_doc_ids if DOCS_PARENT_FOLDER in doc_id
+    ]
     ref_folders = list(set(ref_folders))
 
     static_docs_to_add = []
@@ -57,6 +60,9 @@
             folders_to_remove.append(ref_folder)
 
     if index:
-        VECTOR_INDEX.refresh_index_static_docs(index, static_docs_to_add, [])
-        VECTOR_INDEX.remove_docs_in_folder(index, folders_to_remove)
+        if static_docs_to_add:
+            VECTOR_INDEX.refresh_index_static_docs(index, static_docs_to_add, [])
+        if folders_to_remove:
+            for folder_to_remove in folders_to_remove:
+                VECTOR_INDEX.remove_docs_in_folder(index, folder_to_remove)
         LOGGER.info("Static docs refresh process completed.")