feat: add TrustyAI Ragas inline and remote providers (#52)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- New Features
- Added RAG-based evaluation support with both inline and remote
provider options.

- Configuration
- Introduced EMBEDDING_MODEL environment variable to select embeddings;
exposed to build, test, and runtime.

- Documentation
- Updated provider listings to include the new inline and remote RAG
evaluation entries.

- Chores
- Container build now installs the RAG evaluation provider dependency;
tests and run scripts forward EMBEDDING_MODEL.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Author: nathan-weinberg

.github/workflows/redhat-distro-container.yml

@@ -31,6 +31,7 @@ jobs:
     runs-on: ubuntu-latest
     env:
       INFERENCE_MODEL: meta-llama/Llama-3.2-1B-Instruct
+      EMBEDDING_MODEL: granite-embedding-125m
       VLLM_URL: http://localhost:8000/v1
     strategy:
       matrix:

distribution/Containerfile

@@ -41,6 +41,8 @@ RUN pip install \
     uvicorn
 RUN pip install \
     llama_stack_provider_lmeval==0.2.4
+RUN pip install \
+    llama_stack_provider_ragas[remote]==0.3.0
 RUN pip install \
     --extra-index-url https://test.pypi.org/simple/ llama_stack_provider_trustyai_fms==0.2.3
 RUN pip install --extra-index-url https://download.pytorch.org/whl/cpu torch 'torchao>=0.12.0' torchvision

distribution/README.md

@@ -13,7 +13,9 @@ You can see an overview of the APIs and Providers the image ships with in the ta
 | agents | inline::meta-reference | Yes | N/A |
 | datasetio | inline::localfs | Yes | N/A |
 | datasetio | remote::huggingface | Yes | N/A |
+| eval | inline::trustyai_ragas | No | Set the `EMBEDDING_MODEL` environment variable |
 | eval | remote::trustyai_lmeval | Yes | N/A |
+| eval | remote::trustyai_ragas | No | Set the `KUBEFLOW_LLAMA_STACK_URL` environment variable |
 | files | inline::localfs | Yes | N/A |
 | inference | inline::sentence-transformers | Yes | N/A |
 | inference | remote::azure | No | Set the `AZURE_API_KEY` environment variable |

distribution/build.yaml

@@ -21,6 +21,8 @@ distribution_spec:
     eval:
     - provider_type: remote::trustyai_lmeval
       module: llama_stack_provider_lmeval==0.2.4
+    - provider_type: inline::trustyai_ragas
+      module: llama_stack_provider_ragas[remote]==0.3.0
     datasetio:
     - provider_type: remote::huggingface
     - provider_type: inline::localfs

distribution/run.yaml

@@ -108,6 +108,23 @@ providers:
     config:
         use_k8s: ${env.TRUSTYAI_LMEVAL_USE_K8S:=true}
         base_url: ${env.VLLM_URL:=}
+  - provider_id: ${env.EMBEDDING_MODEL:+trustyai_ragas_inline}
+    provider_type: inline::trustyai_ragas
+    module: llama_stack_provider_ragas.inline
+    config:
+      embedding_model: ${env.EMBEDDING_MODEL:=}
+  - provider_id: ${env.KUBEFLOW_LLAMA_STACK_URL:+trustyai_ragas_remote}
+    provider_type: remote::trustyai_ragas
+    module: llama_stack_provider_ragas.remote
+    config:
+      embedding_model: ${env.EMBEDDING_MODEL:=}
+      kubeflow_config:
+        results_s3_prefix: ${env.KUBEFLOW_RESULTS_S3_PREFIX:=}
+        s3_credentials_secret_name: ${env.KUBEFLOW_S3_CREDENTIALS_SECRET_NAME:=}
+        pipelines_endpoint: ${env.KUBEFLOW_PIPELINES_ENDPOINT:=}
+        namespace: ${env.KUBEFLOW_NAMESPACE:=}
+        llama_stack_url: ${env.KUBEFLOW_LLAMA_STACK_URL:=}
+        base_image: ${env.KUBEFLOW_BASE_IMAGE:=}
   datasetio:
   - provider_id: huggingface
     provider_type: remote::huggingface

tests/smoke.sh

@@ -10,6 +10,7 @@ function start_and_wait_for_llama_stack_container {
     --net=host \
     -p 8321:8321 \
     --env INFERENCE_MODEL="$INFERENCE_MODEL" \
+    --env EMBEDDING_MODEL="$EMBEDDING_MODEL" \
     --env VLLM_URL="$VLLM_URL" \
     --env TRUSTYAI_LMEVAL_USE_K8S=False \
     --name llama-stack \