Merge pull request #1 from hteeyeoh/config_yaml

chatqna-core config yaml

Author: hteeyeoh

sample-applications/chat-question-and-answer-core/app/.env

@@ -1,4 +1,4 @@
-from .config import Settings
+from .config import config
 from .utils import login_to_huggingface, download_huggingface_model, convert_model
 from .document import load_file_document
 from .logger import logger
@@ -14,9 +14,10 @@
 import os
 import pandas as pd
 
-config = Settings()
 vectorstore = None
 
+logger.info(f"Chatqna-Core application started with config: {config}")
+
 # The RUN_TEST flag is used to bypass the model download and conversion steps during pytest unit testing.
 # By default, the flag is set to 'false', enabling the model download and conversion process in a normal run.
 # To skip these steps, set the flag to 'true'.
@@ -38,21 +39,22 @@
     convert_model(config.LLM_MODEL_ID, config.CACHE_DIR, "llm")
 
     # Define RAG prompt
-    template = """
-    Use the following pieces of context from retrieved
-    dataset to answer the question. Do not make up an answer if there is no
-    context provided to help answer it.
-
-    Context:
-    ---------
-    {context}
-
-    ---------
-    Question: {question}
-    ---------
-
-    Answer:
-    """
+    # template = """
+    # Use the following pieces of context from retrieved
+    # dataset to answer the question. Do not make up an answer if there is no
+    # context provided to help answer it.
+
+    # Context:
+    # ---------
+    # {context}
+
+    # ---------
+    # Question: {question}
+    # ---------
+
+    # Answer:
+    # """
+    template = config.PROMPT_TEMPLATE
 
     prompt = ChatPromptTemplate.from_template(template)
 

sample-applications/chat-question-and-answer-core/app/chain.py

@@ -1,50 +1,87 @@
 from pydantic_settings import BaseSettings
-from os.path import dirname, abspath, join
-
+from os.path import dirname, abspath
+from .prompt import get_prompt_template
+import os
+import yaml
 
 class Settings(BaseSettings):
     """
     Settings for the Chatqna-Core application.
+    This class manages configuration settings for the application, supporting loading from a YAML file,
+    environment variables, or default values. It includes model identifiers, device settings, cache paths,
+    and other application-specific options.
 
     Attributes:
         APP_DISPLAY_NAME (str): The display name of the application.
         BASE_DIR (str): The base directory of the application.
-        SUPPORTED_FORMATS (set): A set of supported file formats.
+        SUPPORTED_FORMATS (set): Supported file formats for input documents.
         DEBUG (bool): Flag to enable or disable debug mode.
-        TMP_FILE_PATH (str): The temporary file path for documents.
-        HF_ACCESS_TOKEN (str): The Hugging Face access token.
-        EMBEDDING_MODEL_ID (str): The ID of the embedding model.
-        RERANKER_MODEL_ID (str): The ID of the reranker model.
-        LLM_MODEL_ID (str): The ID of the large language model.
-        EMBEDDING_DEVICE (str): The device used for embedding.
-        RERANKER_DEVICE (str): The device used for reranker.
-        LLM_DEVICE (str): The device used for LLM inferencing.
-        CACHE_DIR (str): The directory used for caching.
-        HF_DATASETS_CACHE (str): The cache directory for Hugging Face datasets.
-        MAX_TOKENS (int): The maximum number of output tokens.
+        HF_ACCESS_TOKEN (str): Hugging Face access token for model downloads.
+        EMBEDDING_MODEL_ID (str): Identifier for the embedding model.
+        RERANKER_MODEL_ID (str): Identifier for the reranker model.
+        LLM_MODEL_ID (str): Identifier for the large language model.
+        PROMPT_TEMPLATE (str): Template for prompts used by the LLM.
+        EMBEDDING_DEVICE (str): Device to run the embedding model on (e.g., "CPU", "GPU").
+        RERANKER_DEVICE (str): Device to run the reranker model on.
+        LLM_DEVICE (str): Device to run the LLM on.
+        CACHE_DIR (str): Directory for caching models.
+        HF_DATASETS_CACHE (str): Directory for caching Hugging Face datasets.
+        MAX_TOKENS (int): Maximum number of tokens for LLM input/output.
         ENABLE_RERANK (bool): Flag to enable or disable reranking.
+        TMP_FILE_PATH (str): Temporary file path for storing documents.
+        MODEL_CONFIG_PATH (str): Path to the YAML configuration file.
 
-    Config:
-        env_file (str): The path to the environment file.
+    Methods:
+        __init__(**kwargs):
+            Initializes the Settings instance, loading configuration from a YAML file if it exists,
+            and overriding attributes with values from the file. If PROMPT_TEMPLATE is not set,
+            it is determined based on the LLM_MODEL_ID.
     """
 
     APP_DISPLAY_NAME: str = "Chatqna-Core"
     BASE_DIR: str = dirname(dirname(abspath(__file__)))
     SUPPORTED_FORMATS: set = {".pdf", ".txt", ".docx"}
     DEBUG: bool = False
 
-    HF_ACCESS_TOKEN: str = ...
-    EMBEDDING_MODEL_ID: str = ...
-    RERANKER_MODEL_ID: str = ...
-    LLM_MODEL_ID: str = ...
-    EMBEDDING_DEVICE: str = ...
-    RERANKER_DEVICE: str = ...
-    LLM_DEVICE: str = ...
-    CACHE_DIR: str = ...
-    HF_DATASETS_CACHE: str = ...
-    MAX_TOKENS: int = ...
-    ENABLE_RERANK: bool = ...
-    TMP_FILE_PATH: str = ...
-
-    class Config:
-        env_file = join(dirname(abspath(__file__)), ".env")
+    HF_ACCESS_TOKEN: str = ""
+    EMBEDDING_MODEL_ID: str = ""
+    RERANKER_MODEL_ID: str = ""
+    LLM_MODEL_ID: str = ""
+    PROMPT_TEMPLATE: str = ""
+    EMBEDDING_DEVICE: str = "CPU"
+    RERANKER_DEVICE: str = "CPU"
+    LLM_DEVICE: str = "CPU"
+    CACHE_DIR: str = "/tmp/model_cache"
+    HF_DATASETS_CACHE: str = "/tmp/model_cache"
+    MAX_TOKENS: int = 1024
+    ENABLE_RERANK: bool = True
+    TMP_FILE_PATH: str = "/tmp/chatqna/documents"
+    MODEL_CONFIG_PATH: str = "/tmp/model_config/config.yaml"
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        config_file = self.MODEL_CONFIG_PATH
+
+        if os.path.isfile(config_file):
+            print(f"INFO - {config_file} exists. Loading configuration from {config_file}")
+            with open(config_file, 'r') as f:
+                config = yaml.safe_load(f)
+
+            for key, value in config.get("model_settings", {}).items():
+                if hasattr(self, key):
+                    setattr(self, key, value)
+
+            for key, value in config.get("device_settings", {}).items():
+                if hasattr(self, key):
+                    setattr(self, key, value)
+
+        else:
+            print(f"WARNING - Expected a file at {config_file}, but found a directory or nothing.")
+            print("INFO - Proceeding with default settings or previously loaded configurations from env variables.")
+
+        if not self.PROMPT_TEMPLATE:
+            print("INFO - PROMPT_TEMPLATE is not set. Get prompt template based on LLM_MODEL_ID.")
+            self.PROMPT_TEMPLATE = get_prompt_template(self.LLM_MODEL_ID)
+
+
+config = Settings()

sample-applications/chat-question-and-answer-core/app/config.py

@@ -1,5 +1,5 @@
 import os
-from .config import Settings
+from .config import config
 from .logger import logger
 from pathlib import Path
 from fastapi import UploadFile
@@ -9,8 +9,6 @@
     TextLoader
 )
 
-config = Settings()
-
 
 def validate_document(file_object: UploadFile):
     """
@@ -85,4 +83,4 @@ def load_file_document(file_path):
 
     docs = loader.load()
 
-    return docs
+    return docs

sample-applications/chat-question-and-answer-core/app/document.py

@@ -1,9 +1,8 @@
-from .config import Settings
+from .config import config
 from typing import Optional
 import logging
 import sys
 
-config = Settings()
 
 def initialize_logger(name: Optional[str] = None) -> logging.Logger:
     """
@@ -39,5 +38,4 @@ def initialize_logger(name: Optional[str] = None) -> logging.Logger:
     return logger
 
 
-# Optional: create a default logger instance for convenience
 logger = initialize_logger(config.APP_DISPLAY_NAME)

sample-applications/chat-question-and-answer-core/app/logger.py

@@ -0,0 +1,105 @@
+# Defaul prompt template for RAG (Retrieval-Augmented Generation)
+default_rag_prompt_template = """
+    Use the following pieces of context from retrieved
+    dataset to answer the question. Do not make up an answer if there is no
+    context provided to help answer it.
+
+    Context:
+    ---------
+    {context}
+
+    ---------
+    Question: {question}
+    ---------
+
+    Answer:
+    """
+
+# Define the prompt templates for validated model IDs
+model_prompt_templates = {
+    "microsoft/Phi-3.5-mini-instruct": """
+    <|system|>
+    Use the following pieces of context from retrieved
+    dataset to answer the question. Do not make up an answer if there is no
+    context provided to help answer it.<|end|>
+
+    <|context|>
+    Context:
+    ---------
+    {context}
+    <|end|>
+
+    <|user|>
+    ---------
+    Question: {question}
+    ---------
+    <|end|>
+
+    <|assistant|>
+    Answer:
+    """,
+    "Intel/neural-chat-7b-v3-3": """
+    ### System:
+    Use the following pieces of context from retrieved
+    dataset to answer the question. Do not make up an answer if there is no
+    context provided to help answer it.
+
+    ### Context:
+    {context}
+
+    ### User:
+    {question}
+
+    ### Assistant:
+    """,
+    "meta-llama/Llama-3.1-8B-Instruct":
+    """
+    <|start_header_id|>system<|end_header_id|>
+    Use the following pieces of context from retrieved
+    dataset to answer the question. Do not make up an answer if there is no
+    context provided to help answer it.
+
+    <context>
+    Context:
+    {context}
+    </context>
+
+    <|start_header_id|>user<|end_header_id|>
+    Question: {question}
+    <|eot_id|>
+
+    <|start_header_id|>assistant<|end_header_id|>
+    """,
+    "Qwen/Qwen2.5-7B-Instruct": """
+    <|im_start|>
+    system:
+    Use the following pieces of context from retrieved
+    dataset to answer the question. Do not make up an answer if there is no
+    context provided to help answer it.
+
+    context:
+    {context}
+    <|im_end|>
+
+    <|im_start|>
+    user:
+    {question}
+    <|im_end|>
+
+    <|im_start|>assistant
+    """
+}
+
+def get_prompt_template(llm_model_id: str) -> str:
+    """
+    Get the prompt template for a given model ID.
+
+    Args:
+        model_id (str): The model ID to get the prompt template for.
+
+    Returns:
+        str: The prompt template for the specified model ID.
+    """
+
+    # If the model ID is not in the predefined templates, return the default RAG prompt template
+    return model_prompt_templates.get(llm_model_id, default_rag_prompt_template)

sample-applications/chat-question-and-answer-core/app/prompt.py

@@ -8,7 +8,7 @@
 from http import HTTPStatus
 from pydantic import BaseModel
 from typing import Annotated
-from .config import Settings
+from .config import config
 from .logger import logger
 from .chain import (
     create_faiss_vectordb,
@@ -23,8 +23,6 @@
 
 app = FastAPI(root_path="/v1/chatqna")
 
-config = Settings()
-
 app.add_middleware(
     CORSMiddleware,
     allow_origins=os.getenv("CORS_ALLOW_ORIGINS", "*").split(

sample-applications/chat-question-and-answer-core/app/server.py

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: model-config
+data:
+  config.yaml: |
+    model_settings:
+      EMBEDDING_MODEL_ID: "BAAI/bge-small-en-v1.5"
+      RERANKER_MODEL_ID: "BAAI/bge-reranker-base"
+      LLM_MODEL_ID: "microsoft/Phi-3.5-mini-instruct"
+      PROMPT_TEMPLATE: "
+        <|system|>
+        Use the following pieces of context from retrieved
+        dataset to answer the question. Do not make up an answer if there is no
+        context provided to help answer it.<|end|>
+
+        <|context|>
+        Context:
+        ---------
+        {context}
+        <|end|>
+
+        <|user|>
+        ---------
+        Question: {question}
+        ---------
+        <|end|>
+
+        <|assistant|>
+        Answer:
+        "