ChatQnA fixes (#553)

Signed-off-by: Yeoh, Hoong Tee <hoong.tee.yeoh@intel.com>
Co-authored-by: 14pankaj <pankaj.kumar.singh@intel.com>
Co-authored-by: Pankaj Kumar Singh <97222471+14pankaj@users.noreply.github.com>
Co-authored-by: Madhuri Kumari <madhuri.rai07@gmail.com>

Author: 3 people

sample-applications/chat-question-and-answer-core/.gitignore

@@ -0,0 +1,12 @@
+*.pyc
+__pycache__/
+.*_cache/
+**/charts
+.vscode
+.venv
+coverage
+.coverage
+.coverage-report/
+*.lock
+!poetry.lock
+.vscode

sample-applications/chat-question-and-answer-core/app/.env

@@ -1,4 +1,4 @@
-from .config import Settings
+from .config import config
 from .utils import login_to_huggingface, download_huggingface_model, convert_model
 from .document import load_file_document
 from .logger import logger
@@ -14,73 +14,55 @@
 import os
 import pandas as pd
 
-config = Settings()
 vectorstore = None
 
 # The RUN_TEST flag is used to bypass the model download and conversion steps during pytest unit testing.
-# By default, the flag is set to 'false', enabling the model download and conversion process in a normal run.
-# To skip these steps, set the flag to 'true'.
-# Check environment flag
-RUN_TEST = os.getenv('RUN_TEST', False)
-
-if not RUN_TEST:
+# If RUN_TEST is set to "True", the model download and conversion steps are skipped.
+# This flag is set in the conftest.py file before running the tests.
+if os.getenv("RUN_TEST", "").lower() != "true":
     # login huggingface
     login_to_huggingface(config.HF_ACCESS_TOKEN)
 
     # Download convert the model to openvino optimized
-    download_huggingface_model(config.EMBEDDING_MODEL_ID, config.CACHE_DIR)
-    download_huggingface_model(config.RERANKER_MODEL_ID, config.CACHE_DIR)
-    download_huggingface_model(config.LLM_MODEL_ID, config.CACHE_DIR)
+    download_huggingface_model(config.EMBEDDING_MODEL_ID, config._CACHE_DIR)
+    download_huggingface_model(config.RERANKER_MODEL_ID, config._CACHE_DIR)
+    download_huggingface_model(config.LLM_MODEL_ID, config._CACHE_DIR)
 
     # Convert to openvino IR
-    convert_model(config.EMBEDDING_MODEL_ID, config.CACHE_DIR, "embedding")
-    convert_model(config.RERANKER_MODEL_ID, config.CACHE_DIR, "reranker")
-    convert_model(config.LLM_MODEL_ID, config.CACHE_DIR, "llm")
-
-    # Define RAG prompt
-    template = """
-    Use the following pieces of context from retrieved
-    dataset to answer the question. Do not make up an answer if there is no
-    context provided to help answer it.
+    convert_model(config.EMBEDDING_MODEL_ID, config._CACHE_DIR, "embedding")
+    convert_model(config.RERANKER_MODEL_ID, config._CACHE_DIR, "reranker")
+    convert_model(config.LLM_MODEL_ID, config._CACHE_DIR, "llm")
 
-    Context:
-    ---------
-    {context}
 
-    ---------
-    Question: {question}
-    ---------
-
-    Answer:
-    """
+    template = config.PROMPT_TEMPLATE
 
     prompt = ChatPromptTemplate.from_template(template)
 
     # Initialize Embedding Model
     embedding = OpenVINOBgeEmbeddings(
-        model_name_or_path=f"{config.CACHE_DIR}/{config.EMBEDDING_MODEL_ID}",
+        model_name_or_path=f"{config._CACHE_DIR}/{config.EMBEDDING_MODEL_ID}",
         model_kwargs={"device": config.EMBEDDING_DEVICE, "compile": False},
     )
     embedding.ov_model.compile()
 
     # Initialize Reranker Model
     reranker = OpenVINOReranker(
-        model_name_or_path=f"{config.CACHE_DIR}/{config.RERANKER_MODEL_ID}",
+        model_name_or_path=f"{config._CACHE_DIR}/{config.RERANKER_MODEL_ID}",
         model_kwargs={"device": config.RERANKER_DEVICE},
         top_n=2,
     )
 
     # Initialize LLM
     llm = HuggingFacePipeline.from_model_id(
-        model_id=f"{config.CACHE_DIR}/{config.LLM_MODEL_ID}",
+        model_id=f"{config._CACHE_DIR}/{config.LLM_MODEL_ID}",
         task="text-generation",
         backend="openvino",
         model_kwargs={
             "device": config.LLM_DEVICE,
             "ov_config": {
                 "PERFORMANCE_HINT": "LATENCY",
                 "NUM_STREAMS": "1",
-                "CACHE_DIR": f"{config.CACHE_DIR}/{config.LLM_MODEL_ID}/model_cache",
+                "CACHE_DIR": f"{config._CACHE_DIR}/{config.LLM_MODEL_ID}/model_cache",
             },
             "trust_remote_code": True,
         },
@@ -287,4 +269,4 @@ def delete_embedding_from_vectordb(document: str = "", delete_all: bool = False)
 
     vectorstore.delete(chunk_list)
 
-    return True
+    return True

sample-applications/chat-question-and-answer-core/app/chain.py

@@ -1,50 +1,114 @@
+from pydantic import PrivateAttr
 from pydantic_settings import BaseSettings
-from os.path import dirname, abspath, join
-
+from os.path import dirname, abspath
+from .prompt import get_prompt_template
+import os
+import yaml
 
 class Settings(BaseSettings):
     """
-    Settings for the Chatqna-Core application.
+    Settings class for configuring the Chatqna-Core application.
+    This class manages application-wide configuration, including model settings, device preferences,
+    supported file formats, and paths for caching and configuration files. It loads additional
+    configuration from a YAML file if provided, and updates its attributes accordingly.
 
     Attributes:
-        APP_DISPLAY_NAME (str): The display name of the application.
-        BASE_DIR (str): The base directory of the application.
-        SUPPORTED_FORMATS (set): A set of supported file formats.
+        APP_DISPLAY_NAME (str): Display name of the application.
+        BASE_DIR (str): Base directory of the application.
+        SUPPORTED_FORMATS (set): Supported document file formats.
         DEBUG (bool): Flag to enable or disable debug mode.
-        TMP_FILE_PATH (str): The temporary file path for documents.
-        HF_ACCESS_TOKEN (str): The Hugging Face access token.
-        EMBEDDING_MODEL_ID (str): The ID of the embedding model.
-        RERANKER_MODEL_ID (str): The ID of the reranker model.
-        LLM_MODEL_ID (str): The ID of the large language model.
-        EMBEDDING_DEVICE (str): The device used for embedding.
-        RERANKER_DEVICE (str): The device used for reranker.
-        LLM_DEVICE (str): The device used for LLM inferencing.
-        CACHE_DIR (str): The directory used for caching.
-        HF_DATASETS_CACHE (str): The cache directory for Hugging Face datasets.
-        MAX_TOKENS (int): The maximum number of output tokens.
+        HF_ACCESS_TOKEN (str): Hugging Face access token for model downloads.
+        EMBEDDING_MODEL_ID (str): Model ID for embeddings.
+        RERANKER_MODEL_ID (str): Model ID for reranker.
+        LLM_MODEL_ID (str): Model ID for large language model.
+        PROMPT_TEMPLATE (str): Prompt template for the LLM.
+        EMBEDDING_DEVICE (str): Device to run embedding model on.
+        RERANKER_DEVICE (str): Device to run reranker model on.
+        LLM_DEVICE (str): Device to run LLM on.
+        MAX_TOKENS (int): Maximum number of tokens for LLM responses.
         ENABLE_RERANK (bool): Flag to enable or disable reranking.
+        _CACHE_DIR (str): Directory for model cache (private).
+        _HF_DATASETS_CACHE (str): Directory for Hugging Face datasets cache (private).
+        _TMP_FILE_PATH (str): Temporary file path for documents (private).
+        _DEFAULT_MODEL_CONFIG (str): Path to default model configuration YAML (private).
+        _MODEL_CONFIG_PATH (str): Path to user-provided model configuration YAML (private).
 
-    Config:
-        env_file (str): The path to the environment file.
+    Methods:
+        __init__(**kwargs): Initializes the Settings object, loads configuration from YAML file,
+            and updates attributes accordingly.
     """
 
     APP_DISPLAY_NAME: str = "Chatqna-Core"
     BASE_DIR: str = dirname(dirname(abspath(__file__)))
     SUPPORTED_FORMATS: set = {".pdf", ".txt", ".docx"}
     DEBUG: bool = False
 
-    HF_ACCESS_TOKEN: str = ...
-    EMBEDDING_MODEL_ID: str = ...
-    RERANKER_MODEL_ID: str = ...
-    LLM_MODEL_ID: str = ...
-    EMBEDDING_DEVICE: str = ...
-    RERANKER_DEVICE: str = ...
-    LLM_DEVICE: str = ...
-    CACHE_DIR: str = ...
-    HF_DATASETS_CACHE: str = ...
-    MAX_TOKENS: int = ...
-    ENABLE_RERANK: bool = ...
-    TMP_FILE_PATH: str = ...
-
-    class Config:
-        env_file = join(dirname(abspath(__file__)), ".env")
+    HF_ACCESS_TOKEN: str = ""
+    EMBEDDING_MODEL_ID: str = ""
+    RERANKER_MODEL_ID: str = ""
+    LLM_MODEL_ID: str = ""
+    PROMPT_TEMPLATE: str = ""
+    EMBEDDING_DEVICE: str = "CPU"
+    RERANKER_DEVICE: str = "CPU"
+    LLM_DEVICE: str = "CPU"
+    MAX_TOKENS: int = 1024
+    ENABLE_RERANK: bool = True
+
+    # These fields will not be affected by environment variables
+    _CACHE_DIR: str = PrivateAttr("/tmp/model_cache")
+    _HF_DATASETS_CACHE: str = PrivateAttr("/tmp/model_cache")
+    _TMP_FILE_PATH: str = PrivateAttr("/tmp/chatqna/documents")
+    _DEFAULT_MODEL_CONFIG: str = PrivateAttr("/tmp/model_config/default_model.yaml")
+    _MODEL_CONFIG_PATH: str = PrivateAttr("/tmp/model_config/config.yaml")
+
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        # The RUN_TEST flag is used to bypass the model config loading during pytest unit testing.
+        # If RUN_TEST is set to "True", the model config loading is skipped.
+        # This flag is set in the conftest.py file before running the tests.
+        if os.getenv("RUN_TEST", "").lower() == "true":
+            print("INFO - Skipping model config loading in test mode.")
+            return
+
+        config_file = self._MODEL_CONFIG_PATH if os.path.isfile(self._MODEL_CONFIG_PATH) else self._DEFAULT_MODEL_CONFIG
+
+        if config_file == self._MODEL_CONFIG_PATH:
+            print(f"INFO - Model configuration yaml from user found in {config_file}. Loading configuration from {config_file}")
+
+        else:
+            print("WARNING - User did not provide model configuration yaml file via MODEL_CONFIG_PATH.")
+            print(f"INFO - Proceeding with default settings from {config_file}")
+
+        with open(config_file, 'r') as f:
+            config = yaml.safe_load(f)
+
+        for section in ("model_settings", "device_settings"):
+            for key, value in config.get(section, {}).items():
+                if hasattr(self, key):
+                    setattr(self, key, value)
+
+        self._validate_model_ids()
+
+        self._check_and_validate_prompt_template()
+
+    def _validate_model_ids(self):
+        for model_name in ["EMBEDDING_MODEL_ID", "RERANKER_MODEL_ID", "LLM_MODEL_ID"]:
+            model_id = getattr(self, model_name)
+            if not model_id:
+                raise ValueError(f"{model_name} must not be an empty string.")
+
+    def _check_and_validate_prompt_template(self):
+        if not self.PROMPT_TEMPLATE:
+            print("INFO - PROMPT_TEMPLATE is not set. Getting default prompt_template.")
+            self.PROMPT_TEMPLATE = get_prompt_template(self.LLM_MODEL_ID)
+
+        # Validate PROMPT_TEMPLATE
+        required_placeholders = ["{context}", "{question}"]
+        for placeholder in required_placeholders:
+            if placeholder not in self.PROMPT_TEMPLATE:
+                raise ValueError(f"PROMPT_TEMPLATE must include the placeholder {placeholder}.")
+
+
+config = Settings()

sample-applications/chat-question-and-answer-core/app/config.py

@@ -1,5 +1,5 @@
 import os
-from .config import Settings
+from .config import config
 from .logger import logger
 from pathlib import Path
 from fastapi import UploadFile
@@ -9,8 +9,6 @@
     TextLoader
 )
 
-config = Settings()
-
 
 def validate_document(file_object: UploadFile):
     """
@@ -45,7 +43,7 @@ async def save_document(file_object: UploadFile):
                If the file is saved successfully, the error will be None. If an error occurs, the path will be None.
     """
 
-    tmp_path = Path(config.TMP_FILE_PATH) / file_object.filename
+    tmp_path = Path(config._TMP_FILE_PATH) / file_object.filename
     if not tmp_path.parent.exists():
         tmp_path.parent.mkdir(parents=True, exist_ok=True)
 
@@ -85,4 +83,4 @@ def load_file_document(file_path):
 
     docs = loader.load()
 
-    return docs
+    return docs

sample-applications/chat-question-and-answer-core/app/document.py

@@ -1,9 +1,8 @@
-from .config import Settings
+from .config import config
 from typing import Optional
 import logging
 import sys
 
-config = Settings()
 
 def initialize_logger(name: Optional[str] = None) -> logging.Logger:
     """