Ai config (#358)

* Add assumerole

Author: piyushroshan

README.md

@@ -116,6 +116,8 @@ Visit [http://localhost:8888](http://localhost:8888)
 [http://localhost:8025](http://localhost:8025)
 You can change the smtp configuration if required however all emails with domain **example.com** will still go to mailhog.
 
+For chatbot LLM provider configuration, see [setup instructions](docs/setup.md#chatbot-llm-configuration).
+
 ### Vagrant
 
 This option allows you to run crAPI within a virtual machine, thus isolated from

deploy/docker/docker-compose.yml

@@ -172,11 +172,36 @@ services:
       - API_USER=admin@example.com
       - API_PASSWORD=Admin!123
       - OPENAPI_SPEC=/app/resources/crapi-openapi-spec.json
-      - DEFAULT_MODEL=gpt-4o-mini
+      - CHATBOT_LLM_PROVIDER=${CHATBOT_LLM_PROVIDER:-openai}
+      - CHATBOT_LLM_MODEL=${CHATBOT_LLM_MODEL:-}
+      - CHATBOT_EMBEDDINGS_MODEL=${CHATBOT_EMBEDDINGS_MODEL:-}
+      - CHATBOT_EMBEDDINGS_DIMENSIONS=${CHATBOT_EMBEDDINGS_DIMENSIONS:-1536}
+      - CHATBOT_OPENAI_API_KEY=${CHATBOT_OPENAI_API_KEY:-}
+      - CHATBOT_OPENAI_BASE_URL=${CHATBOT_OPENAI_BASE_URL:-}
+      - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY:-}
+      - AZURE_OPENAI_API_KEY=${AZURE_OPENAI_API_KEY:-}
+      - AZURE_AD_TOKEN=${AZURE_AD_TOKEN:-}
+      - AZURE_OPENAI_ENDPOINT=${AZURE_OPENAI_ENDPOINT:-}
+      - AZURE_OPENAI_API_VERSION=${AZURE_OPENAI_API_VERSION:-2024-02-15-preview}
+      - AZURE_OPENAI_CHAT_DEPLOYMENT=${AZURE_OPENAI_CHAT_DEPLOYMENT:-}
+      - AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT=${AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT:-}
+      - GROQ_API_KEY=${GROQ_API_KEY:-}
+      - MISTRAL_API_KEY=${MISTRAL_API_KEY:-}
+      - COHERE_API_KEY=${COHERE_API_KEY:-}
+      - AWS_BEARER_TOKEN_BEDROCK=${AWS_BEARER_TOKEN_BEDROCK:-}
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
+      - AWS_SESSION_TOKEN=${AWS_SESSION_TOKEN:-}
+      - AWS_REGION=${AWS_REGION:-}
+      - AWS_ASSUME_ROLE_ARN=${AWS_ASSUME_ROLE_ARN:-}
+      - AWS_EXTERNAL_ID=${AWS_EXTERNAL_ID:-}
+      - AWS_ROLE_SESSION_NAME=${AWS_ROLE_SESSION_NAME:-crapi-chatbot-session}
+      - GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS:-}
+      - VERTEX_PROJECT=${VERTEX_PROJECT:-}
+      - VERTEX_LOCATION=${VERTEX_LOCATION:-}
       - MAX_CONTENT_LENGTH=50000
       - CHROMA_HOST=chromadb
       - CHROMA_PORT=8000
-      # - CHATBOT_OPENAI_API_KEY=
     depends_on:
       mongodb:
         condition: service_healthy

deploy/helm/templates/chatbot/config.yaml

@@ -20,8 +20,33 @@ data:
     MONGO_DB_USER: {{ .Values.mongodb.config.mongoUser }}
     MONGO_DB_PASSWORD: {{ .Values.mongodb.config.mongoPassword }}
     MONGO_DB_NAME: {{ .Values.mongodb.config.mongoDbName }}
-    CHATBOT_OPENAI_API_KEY: {{ .Values.openAIApiKey }}
-    DEFAULT_MODEL: {{ .Values.chatbot.config.defaultModel | quote }}
+    CHATBOT_OPENAI_API_KEY: {{ .Values.chatbotOpenaiApiKey | default .Values.openAIApiKey | quote }}
+    CHATBOT_OPENAI_BASE_URL: {{ .Values.chatbotOpenaiBaseUrl | quote }}
+    CHATBOT_LLM_PROVIDER: {{ .Values.chatbotLlmProvider | quote }}
+    CHATBOT_LLM_MODEL: {{ .Values.chatbotLlmModel | quote }}
+    CHATBOT_EMBEDDINGS_MODEL: {{ .Values.chatbotEmbeddingsModel | quote }}
+    CHATBOT_EMBEDDINGS_DIMENSIONS: {{ .Values.chatbotEmbeddingsDimensions | quote }}
+    ANTHROPIC_API_KEY: {{ .Values.anthropicApiKey | quote }}
+    AZURE_OPENAI_API_KEY: {{ .Values.azureOpenaiApiKey | quote }}
+    AZURE_AD_TOKEN: {{ .Values.azureAdToken | quote }}
+    AZURE_OPENAI_ENDPOINT: {{ .Values.azureOpenaiEndpoint | quote }}
+    AZURE_OPENAI_API_VERSION: {{ .Values.azureOpenaiApiVersion | quote }}
+    AZURE_OPENAI_CHAT_DEPLOYMENT: {{ .Values.azureOpenaiChatDeployment | quote }}
+    AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT: {{ .Values.azureOpenaiEmbeddingsDeployment | quote }}
+    GROQ_API_KEY: {{ .Values.groqApiKey | quote }}
+    MISTRAL_API_KEY: {{ .Values.mistralApiKey | quote }}
+    COHERE_API_KEY: {{ .Values.cohereApiKey | quote }}
+    AWS_BEARER_TOKEN_BEDROCK: {{ .Values.awsBearerTokenBedrock | quote }}
+    AWS_ACCESS_KEY_ID: {{ .Values.awsAccessKeyId | quote }}
+    AWS_SECRET_ACCESS_KEY: {{ .Values.awsSecretAccessKey | quote }}
+    AWS_SESSION_TOKEN: {{ .Values.awsSessionToken | quote }}
+    AWS_REGION: {{ .Values.awsRegion | quote }}
+    AWS_ASSUME_ROLE_ARN: {{ .Values.awsAssumeRoleArn | quote }}
+    AWS_EXTERNAL_ID: {{ .Values.awsExternalId | quote }}
+    AWS_ROLE_SESSION_NAME: {{ .Values.awsRoleSessionName | quote }}
+    GOOGLE_APPLICATION_CREDENTIALS: {{ .Values.googleApplicationCredentials | quote }}
+    VERTEX_PROJECT: {{ .Values.vertexProject | quote }}
+    VERTEX_LOCATION: {{ .Values.vertexLocation | quote }}
     MAX_CONTENT_LENGTH: {{ .Values.chatbot.config.maxContentLength | quote }}
     CHROMA_HOST: {{ .Values.chromadb.service.name }}
     CHROMA_PORT: {{ .Values.chromadb.port | quote }}

deploy/helm/values.yaml

@@ -11,9 +11,57 @@ apiGatewayServiceInstall: true
 apiGatewayPassword:
 tlsEnabled: false
 jwtExpiration: 604800000
-openAIApiKey: ""
 logLevel: INFO
 
+# Chatbot LLM Configuration
+chatbotLlmProvider: openai
+chatbotLlmModel: ""
+chatbotEmbeddingsModel: ""
+chatbotEmbeddingsDimensions: 1536
+
+# OpenAI API Key
+chatbotOpenaiApiKey: ""
+openAIApiKey: "" # for backward compatibility
+
+# OpenAI Base URL (for DeepSeek or other OpenAI-compatible APIs)
+chatbotOpenaiBaseUrl: ""
+
+# Anthropic API Key
+anthropicApiKey: ""
+
+# Azure configuration
+azureOpenaiApiKey: ""
+azureAdToken: ""
+azureOpenaiEndpoint: ""
+azureOpenaiApiVersion: 2024-02-15-preview
+azureOpenaiChatDeployment: ""
+azureOpenaiEmbeddingsDeployment: ""
+
+# Groq API Key
+groqApiKey: ""
+
+# Mistral API Key
+mistralApiKey: ""
+
+# Cohere API Key
+cohereApiKey: ""
+
+# AWS Bedrock configuration
+awsBearerTokenBedrock: ""
+awsAccessKeyId: ""
+awsSecretAccessKey: ""
+awsSessionToken: ""
+awsRegion: ""
+# AWS IAM Assume Role configuration (optional - use instead of static credentials)
+awsAssumeRoleArn: ""
+awsExternalId: ""
+awsRoleSessionName: "crapi-chatbot-session"
+
+# Google Vertex AI configuration
+googleApplicationCredentials: ""
+vertexProject: ""
+vertexLocation: ""
+
 waitForK8sResources:
   enabled: True
   image: groundnuty/k8s-wait-for:v2.0
@@ -152,7 +200,6 @@ chatbot:
     postgresDbDriver: postgres
     mongoDbDriver: mongodb
     secretKey: crapi
-    defaultModel: gpt-4o-mini
     maxContentLength: 50000
     chromaPersistDirectory: /app/vectorstore
     apiUser: admin@example.com

deploy/k8s/base/chatbot/config.yaml

@@ -4,3 +4,51 @@ metadata:
   name: crapi-chatbot-configmap
   labels:
     app: crapi-chatbot
+data:
+  SERVER_PORT: "5002"
+  IDENTITY_SERVICE: "crapi-identity:8080"
+  WEB_SERVICE: "crapi-web"
+  TLS_ENABLED: "false"
+  DB_HOST: "postgresdb"
+  DB_USER: "admin"
+  DB_PASSWORD: "crapisecretpassword"
+  DB_NAME: "crapi"
+  DB_PORT: "5432"
+  MONGO_DB_HOST: "mongodb"
+  MONGO_DB_PORT: "27017"
+  MONGO_DB_USER: "admin"
+  MONGO_DB_PASSWORD: "crapisecretpassword"
+  MONGO_DB_NAME: "crapi"
+  CHATBOT_OPENAI_API_KEY: ""
+  CHATBOT_OPENAI_BASE_URL: ""
+  CHATBOT_LLM_PROVIDER: "openai"
+  CHATBOT_LLM_MODEL: ""
+  CHATBOT_EMBEDDINGS_MODEL: ""
+  CHATBOT_EMBEDDINGS_DIMENSIONS: "1536"
+  ANTHROPIC_API_KEY: ""
+  AZURE_OPENAI_API_KEY: ""
+  AZURE_AD_TOKEN: ""
+  AZURE_OPENAI_ENDPOINT: ""
+  AZURE_OPENAI_API_VERSION: "2024-02-15-preview"
+  AZURE_OPENAI_CHAT_DEPLOYMENT: ""
+  AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT: ""
+  GROQ_API_KEY: ""
+  MISTRAL_API_KEY: ""
+  COHERE_API_KEY: ""
+  AWS_BEARER_TOKEN_BEDROCK: ""
+  AWS_ACCESS_KEY_ID: ""
+  AWS_SECRET_ACCESS_KEY: ""
+  AWS_SESSION_TOKEN: ""
+  AWS_REGION: ""
+  AWS_ASSUME_ROLE_ARN: ""
+  AWS_EXTERNAL_ID: ""
+  AWS_ROLE_SESSION_NAME: "crapi-chatbot-session"
+  GOOGLE_APPLICATION_CREDENTIALS: ""
+  VERTEX_PROJECT: ""
+  VERTEX_LOCATION: ""
+  MAX_CONTENT_LENGTH: "50000"
+  CHROMA_HOST: "chromadb"
+  CHROMA_PORT: "8000"
+  API_USER: "admin@example.com"
+  API_PASSWORD: "Admin!123"
+  OPENAPI_SPEC: "/app/resources/crapi-openapi-spec.json"

docs/setup.md

@@ -101,6 +101,49 @@ You can use prebuilt images generated by our CI workflow by downloading the dock
 [http://localhost:8025](http://localhost:8025)
 You can change the smtp configuration if required however all emails with domain **example.com** will still go to mailhog.
 
+### Chatbot LLM configuration
+
+The chatbot supports multiple LLM providers. Provider selection and models are set via environment variables. OpenAI and Anthropic keys are set via the API (with env fallback); all other providers use env credentials only.
+
+Core settings:
+- `CHATBOT_LLM_PROVIDER`: `openai`, `anthropic`, `azure_openai`, `bedrock`, `vertex`, `groq`, `mistral`, `cohere`
+- `CHATBOT_LLM_MODEL`: optional, provider model name. Defaults per provider:
+  - openai: `gpt-4o-mini`
+  - anthropic: `claude-sonnet-4-20250514`
+  - bedrock: `anthropic.claude-3-sonnet-20240229-v1:0`
+  - vertex: `gemini-1.5-flash`
+  - groq: `llama-3.3-70b-versatile`
+  - mistral: `mistral-large-latest`
+  - cohere: `command-r-plus`
+  - azure_openai: uses `AZURE_OPENAI_CHAT_DEPLOYMENT`
+- `CHATBOT_EMBEDDINGS_MODEL`: optional embeddings model name (used across providers)
+- `CHATBOT_EMBEDDINGS_DIMENSIONS`: optional, defaults to `1536`
+
+OpenAI (API, optional env fallback):
+- POST `/genai/init` with `{"openai_api_key":"..."}` (per-session)
+- `CHATBOT_OPENAI_API_KEY` (optional env fallback)
+
+Anthropic (API, optional env fallback):
+- POST `/genai/init` with `{"anthropic_api_key":"..."}` (per-session)
+- `ANTHROPIC_API_KEY` (optional env fallback)
+
+Azure OpenAI (env only):
+- `AZURE_OPENAI_ENDPOINT`, `AZURE_OPENAI_CHAT_DEPLOYMENT`
+- Auth: `AZURE_OPENAI_API_KEY` or `AZURE_AD_TOKEN` (managed identity)
+- Optional: `AZURE_OPENAI_API_VERSION`, `AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT`
+
+AWS Bedrock (env only):
+- `AWS_REGION` and either:
+  - `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN` (optional), or
+  - `AWS_BEARER_TOKEN_BEDROCK`
+
+Google Vertex AI (env only):
+- `VERTEX_PROJECT`, `VERTEX_LOCATION`
+- `GOOGLE_APPLICATION_CREDENTIALS` (optional if using ADC in GCP environments)
+
+Groq / Mistral / Cohere (env only):
+- `GROQ_API_KEY`, `MISTRAL_API_KEY`, `COHERE_API_KEY`
+
 ### Build it yourself
 
 1. Clone crAPI repository

services/chatbot/.env

@@ -19,4 +19,6 @@ export DEFAULT_MODEL=gpt-4o-mini
 export MAX_CONTENT_LENGTH=50000
 export CHROMA_HOST=localhost
 export CHROMA_PORT=8000
+export CHATBOT_LLM_PROVIDER=openai
+export CHATBOT_LLM_MODEL=gpt-4o-mini
 export CHATBOT_OPENAI_API_KEY=

services/chatbot/requirements.txt

@@ -20,4 +20,10 @@ quart-cors==0.8.0
 motor==3.7.1
 faiss-cpu==1.12.0
 psycopg2-binary==2.9.11
-uvicorn==0.38.0
+uvicorn==0.38.0
+langchain-aws
+langchain-google-vertexai
+langchain-anthropic
+langchain-groq
+langchain-mistralai
+langchain-cohere