feat: embedding provider now defaults to vLLM (#148)

# What does this PR do?
this commit removes the inline::sentence-transformer provider as the default embedding mode provider as it was causing undesired load on the server CPU
process

now default to a new vLLM provider dedicated to
embedding or a different provider specified via
EMBEDDING_PROVIDER

inline::sentence-transformers must now be enabled by setting ENABLE_SENTENCE_TRANSFORMERS



## Summary by CodeRabbit

* **New Features**
  * Added support for a remote VLLM embedding provider with configurable endpoint, token and TLS options; embedding provider can be selected via environment variables.

* **Documentation**
  * Embedding defaults updated: sentence-transformers is now disabled by default and can be enabled via an environment flag.
  * Guidance added to configure the VLLM embedding URL alongside existing VLLM settings.

* **Tests**
  * Test startup adjusted to set env vars to exercise provider selection.

<sub>✏️ Tip: You can customize this high-level summary in your review settings.</sub>



Approved-by: VaishnaviHire

Approved-by: cdoern

Author: mergify[bot]

distribution/README.md

@@ -19,12 +19,13 @@ You can see an overview of the APIs and Providers the image ships with in the ta
 | eval | remote::trustyai_ragas | Yes (version 0.5.1) | ❌ | Set the `KUBEFLOW_LLAMA_STACK_URL` environment variable |
 | files | inline::localfs | No | ✅ | N/A |
 | files | remote::s3 | No | ❌ | Set the `ENABLE_S3` environment variable |
-| inference | inline::sentence-transformers | No | ✅ | N/A |
+| inference | inline::sentence-transformers | No | ❌ | Set the `ENABLE_SENTENCE_TRANSFORMERS` environment variable |
 | inference | remote::azure | No | ❌ | Set the `AZURE_API_KEY` environment variable |
 | inference | remote::bedrock | No | ❌ | Set the `AWS_ACCESS_KEY_ID` environment variable |
 | inference | remote::openai | No | ❌ | Set the `OPENAI_API_KEY` environment variable |
 | inference | remote::vertexai | No | ❌ | Set the `VERTEX_AI_PROJECT` environment variable |
 | inference | remote::vllm | No | ❌ | Set the `VLLM_URL` environment variable |
+| inference | remote::vllm | No | ❌ | Set the `VLLM_EMBEDDING_URL` environment variable |
 | inference | remote::watsonx | No | ❌ | Set the `WATSONX_API_KEY` environment variable |
 | safety | remote::trustyai_fms | Yes (version 0.3.1) | ✅ | N/A |
 | scoring | inline::basic | No | ✅ | N/A |

distribution/run.yaml

@@ -20,6 +20,13 @@ providers:
       max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
       api_token: ${env.VLLM_API_TOKEN:=fake}
       tls_verify: ${env.VLLM_TLS_VERIFY:=true}
+  - provider_id: ${env.VLLM_EMBEDDING_URL:+vllm-embedding}
+    provider_type: remote::vllm
+    config:
+      url: ${env.VLLM_EMBEDDING_URL:=}
+      max_tokens: ${env.VLLM_EMBEDDING_MAX_TOKENS:=4096}
+      api_token: ${env.VLLM_EMBEDDING_API_TOKEN:=fake}
+      tls_verify: ${env.VLLM_EMBEDDING_TLS_VERIFY:=true}
   - provider_id: ${env.AWS_ACCESS_KEY_ID:+bedrock}
     provider_type: remote::bedrock
     config:
@@ -33,7 +40,7 @@ providers:
       connect_timeout: ${env.AWS_CONNECT_TIMEOUT:=60}
       read_timeout: ${env.AWS_READ_TIMEOUT:=60}
       session_ttl: ${env.AWS_SESSION_TTL:=3600}
-  - provider_id: sentence-transformers
+  - provider_id: ${env.ENABLE_SENTENCE_TRANSFORMERS:+sentence-transformers}
     provider_type: inline::sentence-transformers
     config: {}
   - provider_id: ${env.WATSONX_API_KEY:+watsonx}
@@ -256,11 +263,10 @@ registered_resources:
     model_id: ${env.INFERENCE_MODEL}
     provider_id: vllm-inference
     model_type: llm
-
   - metadata:
       embedding_dimension: 768
     model_id: granite-embedding-125m
-    provider_id: sentence-transformers
+    provider_id: ${env.EMBEDDING_PROVIDER:=vllm-embedding}
     provider_model_id: ibm-granite/granite-embedding-125m-english
     model_type: embedding
   shields: []

tests/smoke.sh

@@ -12,6 +12,8 @@ function start_and_wait_for_llama_stack_container {
     --env INFERENCE_MODEL="$INFERENCE_MODEL" \
     --env EMBEDDING_MODEL="$EMBEDDING_MODEL" \
     --env VLLM_URL="$VLLM_URL" \
+    --env ENABLE_SENTENCE_TRANSFORMERS=True \
+    --env EMBEDDING_PROVIDER=sentence-transformers \
     --env TRUSTYAI_LMEVAL_USE_K8S=False \
     --name llama-stack \
     "$IMAGE_NAME:$GITHUB_SHA"