run.yaml

version: 2
image_name: rh
apis:
- agents
- batches
- datasetio
- eval
- inference
- safety
- scoring
- tool_runtime
- vector_io
- files
providers:
  inference:
  - provider_id: ${env.VLLM_URL:+vllm-inference}
    provider_type: remote::vllm
    config:
      url: ${env.VLLM_URL:=}
      max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
      api_token: ${env.VLLM_API_TOKEN:=fake}
      tls_verify: ${env.VLLM_TLS_VERIFY:=true}
  - provider_id: ${env.VLLM_EMBEDDING_URL:+vllm-embedding}
    provider_type: remote::vllm
    config:
      url: ${env.VLLM_EMBEDDING_URL:=}
      max_tokens: ${env.VLLM_EMBEDDING_MAX_TOKENS:=4096}
      api_token: ${env.VLLM_EMBEDDING_API_TOKEN:=fake}
      tls_verify: ${env.VLLM_EMBEDDING_TLS_VERIFY:=true}
  - provider_id: ${env.AWS_BEARER_TOKEN_BEDROCK:+bedrock}
    provider_type: remote::bedrock
    config:
      api_key: ${env.AWS_BEARER_TOKEN_BEDROCK:=}
      region_name: ${env.AWS_DEFAULT_REGION:=us-east-2}
  - provider_id: ${env.ENABLE_SENTENCE_TRANSFORMERS:+sentence-transformers}
    provider_type: inline::sentence-transformers
    config: {}
  - provider_id: ${env.WATSONX_API_KEY:+watsonx}
    provider_type: remote::watsonx
    config:
      url: ${env.WATSONX_BASE_URL:=https://us-south.ml.cloud.ibm.com}
      api_key: ${env.WATSONX_API_KEY:=}
      project_id: ${env.WATSONX_PROJECT_ID:=}
  - provider_id: ${env.AZURE_API_KEY:+azure}
    provider_type: remote::azure
    config:
      api_key: ${env.AZURE_API_KEY:=}
      api_base: ${env.AZURE_API_BASE:=}
      api_version: ${env.AZURE_API_VERSION:=}
      api_type: ${env.AZURE_API_TYPE:=}
  - provider_id: ${env.VERTEX_AI_PROJECT:+vertexai}
    provider_type: remote::vertexai
    config:
      project: ${env.VERTEX_AI_PROJECT:=}
      location: ${env.VERTEX_AI_LOCATION:=us-central1}
  - provider_id: ${env.OPENAI_API_KEY:+openai}
    provider_type: remote::openai
    config:
      api_key: ${env.OPENAI_API_KEY:=}
      base_url: ${env.OPENAI_BASE_URL:=https://api.openai.com/v1}
  vector_io:
  - provider_id: milvus
    provider_type: inline::milvus
    config:
      db_path: /opt/app-root/src/.llama/distributions/rh/milvus.db
      persistence:
        backend: kv_milvus_inline
        namespace: vector_io::milvus
  - provider_id: ${env.ENABLE_FAISS:+faiss}
    provider_type: inline::faiss
    config:
      persistence:
        backend: kv_faiss
        namespace: vector_io::faiss
  - provider_id: ${env.MILVUS_ENDPOINT:+milvus-remote}
    provider_type: remote::milvus
    config:
      uri: ${env.MILVUS_ENDPOINT:=}
      token: ${env.MILVUS_TOKEN:=}
      secure: ${env.MILVUS_SECURE:=}
      consistency_level: ${env.MILVUS_CONSISTENCY_LEVEL:=}
      ca_pem_path: ${env.MILVUS_CA_PEM_PATH:=}
      client_pem_path: ${env.MILVUS_CLIENT_PEM_PATH:=}
      client_key_path: ${env.MILVUS_CLIENT_KEY_PATH:=}
      persistence:
        backend: kv_milvus_remote
        namespace: vector_io::milvus_remote
  - provider_id: ${env.ENABLE_PGVECTOR:+pgvector}
    provider_type: remote::pgvector
    config:
      host: ${env.PGVECTOR_HOST:=localhost}
      port: ${env.PGVECTOR_PORT:=5432}
      db: ${env.PGVECTOR_DB:=}
      user: ${env.PGVECTOR_USER:=}
      password: ${env.PGVECTOR_PASSWORD:=}
      persistence:
        backend: kv_pgvector
        namespace: vector_io::pgvector
  safety:
    - provider_id: trustyai_fms
      provider_type: remote::trustyai_fms
      module: llama_stack_provider_trustyai_fms==0.2.2
      config:
        orchestrator_url: ${env.FMS_ORCHESTRATOR_URL:=}
        ssl_cert_path: ${env.FMS_SSL_CERT_PATH:=}
        shields: {}
  agents:
  - provider_id: meta-reference
    provider_type: inline::meta-reference
    config:
      persistence:
        agent_state:
          backend: kv_agents
          namespace: agents::meta_reference
        responses:
          backend: sql_agents
          table_name: agents_responses
          max_write_queue_size: 10000
          num_writers: 4
  eval:
  - provider_id: trustyai_lmeval
    provider_type: remote::trustyai_lmeval
    module: llama_stack_provider_lmeval==0.2.4
    config:
      use_k8s: ${env.TRUSTYAI_LMEVAL_USE_K8S:=true}
      base_url: ${env.VLLM_URL:=http://localhost:8000/v1}
  - provider_id: ${env.TRUSTYAI_EMBEDDING_MODEL:+trustyai_ragas_inline}
    provider_type: inline::trustyai_ragas
    module: llama_stack_provider_ragas.inline
    config:
      embedding_model: ${env.TRUSTYAI_EMBEDDING_MODEL:=}
  - provider_id: ${env.KUBEFLOW_LLAMA_STACK_URL:+trustyai_ragas_remote}
    provider_type: remote::trustyai_ragas
    module: llama_stack_provider_ragas.remote
    config:
      embedding_model: ${env.TRUSTYAI_EMBEDDING_MODEL:=}
      kubeflow_config:
        results_s3_prefix: ${env.KUBEFLOW_RESULTS_S3_PREFIX:=}
        s3_credentials_secret_name: ${env.KUBEFLOW_S3_CREDENTIALS_SECRET_NAME:=}
        pipelines_endpoint: ${env.KUBEFLOW_PIPELINES_ENDPOINT:=}
        namespace: ${env.KUBEFLOW_NAMESPACE:=}
        llama_stack_url: ${env.KUBEFLOW_LLAMA_STACK_URL:=}
        base_image: ${env.KUBEFLOW_BASE_IMAGE:=}
        pipelines_api_token: ${env.KUBEFLOW_PIPELINES_TOKEN:=}
  datasetio:
  - provider_id: huggingface
    provider_type: remote::huggingface
    config:
      kvstore:
        backend: kv_datasetio_huggingface
        namespace: datasetio::huggingface
  - provider_id: localfs
    provider_type: inline::localfs
    config:
      kvstore:
        backend: kv_datasetio_localfs
        namespace: datasetio::localfs
  scoring:
  - provider_id: basic
    provider_type: inline::basic
    config: {}
  - provider_id: llm-as-judge
    provider_type: inline::llm-as-judge
    config: {}
  - provider_id: braintrust
    provider_type: inline::braintrust
    config:
      openai_api_key: ${env.OPENAI_API_KEY:=}
  tool_runtime:
  - provider_id: brave-search
    provider_type: remote::brave-search
    config:
      api_key: ${env.BRAVE_SEARCH_API_KEY:=}
      max_results: 3
  - provider_id: tavily-search
    provider_type: remote::tavily-search
    config:
      api_key: ${env.TAVILY_SEARCH_API_KEY:=}
      max_results: 3
  - provider_id: rag-runtime
    provider_type: inline::rag-runtime
    config: {}
  - provider_id: model-context-protocol
    provider_type: remote::model-context-protocol
    config: {}
  files:
  - provider_id: meta-reference-files
    provider_type: inline::localfs
    config:
      storage_dir: /opt/app-root/src/.llama/distributions/rh/files
      metadata_store:
        backend: sql_files
        table_name: files_metadata
  - provider_id: ${env.ENABLE_S3:+s3}
    provider_type: remote::s3
    config:
      bucket_name: ${env.S3_BUCKET_NAME:=}
      region: ${env.AWS_DEFAULT_REGION:=us-east-1}
      aws_access_key_id: ${env.AWS_ACCESS_KEY_ID:=}
      aws_secret_access_key: ${env.AWS_SECRET_ACCESS_KEY:=}
      endpoint_url: ${env.S3_ENDPOINT_URL:=}
      auto_create_bucket: ${env.S3_AUTO_CREATE_BUCKET:=false}
      metadata_store:
        backend: sql_files
        table_name: files_metadata
  batches:
  - provider_id: reference
    provider_type: inline::reference
    config:
      kvstore:
        namespace: batches
        backend: kv_default
storage:
  backends:
    kv_default:
      type: kv_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/kvstore.db
    kv_agents:
      type: kv_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/agents_store.db
    kv_faiss:
      type: kv_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/faiss.db
    kv_milvus_inline:
      type: kv_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/milvus_inline_registry.db
    kv_milvus_remote:
      type: kv_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/milvus_remote_registry.db
    kv_pgvector:
      type: kv_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/pgvector_registry.db
    kv_datasetio_huggingface:
      type: kv_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/huggingface_datasetio.db
    kv_datasetio_localfs:
      type: kv_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/localfs_datasetio.db
    sql_inference:
      type: sql_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/inference_store.db
    sql_agents:
      type: sql_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/responses_store.db
    sql_files:
      type: sql_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/files_metadata.db
    sql_default:
      type: sql_sqlite
      db_path: /opt/app-root/src/.llama/distributions/rh/sql_store.db
  stores:
    metadata:
      backend: kv_default
      namespace: registry
    inference:
      backend: sql_inference
      table_name: inference_store
      max_write_queue_size: 10000
      num_writers: 4
    conversations:
      table_name: openai_conversations
      backend: sql_default
    prompts:
      namespace: prompts
      backend: kv_default
registered_resources:
  models:
  - metadata: {}
    model_id: ${env.INFERENCE_MODEL}
    provider_id: vllm-inference
    model_type: llm
  - metadata:
      embedding_dimension: ${env.EMBEDDING_DIMENSION:=768}
    model_id: ${env.EMBEDDING_MODEL:=granite-embedding-125m-english}
    provider_id: ${env.EMBEDDING_PROVIDER:=vllm-embedding}
    provider_model_id: ${env.EMBEDDING_PROVIDER_MODEL_ID:=ibm-granite/granite-embedding-125m-english}
    model_type: embedding
  shields: []
  vector_dbs: []
  datasets: []
  scoring_fns: []
  benchmarks: []
  tool_groups:
  - toolgroup_id: builtin::websearch
    provider_id: tavily-search
  - toolgroup_id: builtin::rag
    provider_id: rag-runtime
telemetry:
  enabled: true
server:
  port: 8321