|
| 1 | +# ════════════════════════════════════════════════════════════════════════════ |
| 2 | +# Inference SSOT — single source of truth for every model the demo serves. |
| 3 | +# |
| 4 | +# [endpoints.*] define each served model EXACTLY ONCE: which backend runs it, |
| 5 | +# on which host/port, and from which weights. `pixi run inference` reads these, |
| 6 | +# routes each endpoint to the right backend, and launches it: |
| 7 | +# |
| 8 | +# backend = "gpu" | "cpu" → llama.cpp (built in inference/llama.cpp) |
| 9 | +# backend = "npu" → FastFlowLM (`flm serve`, AMD Ryzen AI NPU) |
| 10 | +# backend = "openai" → remote API, NOT served locally (see cloud_config.toml) |
| 11 | +# |
| 12 | +# Agent sections below reference endpoints by name (e.g. `llm = "main_llm"`). |
| 13 | +# rai_app/initialization/llms.py resolves those references into model + base_url, |
| 14 | +# so changing a port, moving a model gpu↔npu, or swapping weights happens HERE, |
| 15 | +# in one place — not scattered across pixi tasks, smoke_test.sh, etc. |
| 16 | +# ════════════════════════════════════════════════════════════════════════════ |
| 17 | + |
| 18 | +# ─── Endpoints ────────────────────────────────────────────────────────────── |
| 19 | + |
| 20 | +[endpoints.main_llm] |
| 21 | +type = "llm" |
| 22 | +backend = "gpu" # llama.cpp, Vulkan build |
| 23 | +model = "gpt-oss-20b" # name advertised to OpenAI-compatible clients |
| 24 | +host = "localhost" |
| 25 | +port = 8080 |
| 26 | +model_path = "${DEMO_ROOT}/models/gpt-oss-20b-Q4_K_M.gguf" |
| 27 | +model_url = "https://huggingface.co/unsloth/gpt-oss-20b-GGUF/resolve/main/gpt-oss-20b-Q4_K_M.gguf?download=true" |
| 28 | +extra_args = ["--no-prefill-assistant"] |
| 29 | + |
| 30 | +# safety agent VLM — FastFlowLM (npu) |
| 31 | +[endpoints.vlm_safety] |
| 32 | +type = "vlm" |
| 33 | +backend = "npu" |
| 34 | +host = "localhost" |
| 35 | +port = 8081 |
| 36 | +model = "gemma3:4b" |
| 37 | +flm_model = "gemma3:4b" |
| 38 | + |
| 39 | +# inspection/general/megamind/condition VLM — LFM2-VL on llama.cpp (gpu) |
| 40 | +[endpoints.vlm_inspection] |
| 41 | +type = "vlm" |
| 42 | +backend = "gpu" |
| 43 | +host = "localhost" |
| 44 | +port = 8084 |
| 45 | +model = "LFM2-VL-3B" |
| 46 | +model_path = "${DEMO_ROOT}/models/LFM2-VL-3B-Q8_0.gguf" |
| 47 | +mmproj_path = "${DEMO_ROOT}/models/mmproj-LFM2-VL-3B-Q8_0.gguf" |
| 48 | +model_url = "https://huggingface.co/LiquidAI/LFM2-VL-3B-GGUF/resolve/main/LFM2-VL-3B-Q8_0.gguf?download=true" |
| 49 | +mmproj_url = "https://huggingface.co/LiquidAI/LFM2-VL-3B-GGUF/resolve/main/mmproj-LFM2-VL-3B-Q8_0.gguf?download=true" |
| 50 | + |
| 51 | +[endpoints.embeddings] |
| 52 | +type = "embedding" |
| 53 | +backend = "gpu" |
| 54 | +model = "qwen3-embedding:0.6b" |
| 55 | +host = "localhost" |
| 56 | +port = 8082 |
| 57 | +model_path = "${DEMO_ROOT}/models/Qwen3-Embedding-0.6B-Q8_0.gguf" |
| 58 | +model_url = "https://huggingface.co/Qwen/Qwen3-Embedding-0.6B-GGUF/resolve/main/Qwen3-Embedding-0.6B-Q8_0.gguf?download=true" |
| 59 | + |
| 60 | +[endpoints.reranker] |
| 61 | +type = "reranker" |
| 62 | +backend = "gpu" |
| 63 | +model = "qwen3-reranker:0.6b" |
| 64 | +host = "localhost" |
| 65 | +port = 8083 |
| 66 | +model_path = "${DEMO_ROOT}/models/qwen3-reranker-0.6b-q8_0.gguf" |
| 67 | +model_url = "https://huggingface.co/ggml-org/Qwen3-Reranker-0.6B-Q8_0-GGUF/resolve/main/qwen3-reranker-0.6b-q8_0.gguf?download=true" |
| 68 | + |
| 69 | +# ─── Agents ───────────────────────────────────────────────────────────────── |
| 70 | +# Each agent references endpoints by name. `*_reasoning` / `*_temperature` are |
| 71 | +# client-side decoding settings and stay with the agent, not the endpoint. |
| 72 | + |
1 | 73 | [general] |
2 | | -llm_model = "gpt-oss-20b" |
3 | | -llm_base_url = "http://localhost:8080/v1" |
| 74 | +llm = "main_llm" |
| 75 | +vlm = "vlm_inspection" |
4 | 76 | llm_reasoning = false |
5 | | -vlm_model = "lfm2-vl" |
6 | | -vlm_base_url = "http://localhost:8081/v1" |
7 | 77 | vlm_reasoning = false |
8 | 78 |
|
9 | 79 | # megamind agent utilizes llm for task delegation |
10 | 80 | # to agents using both llm and vlm for task execution |
11 | 81 | [megamind_agent] |
12 | | -llm_model = "gpt-oss-20b" |
13 | | -llm_base_url = "http://localhost:8080/v1" |
14 | | -vlm_model = "lfm2-vl" |
15 | | -vlm_base_url = "http://localhost:8081/v1" |
| 82 | +llm = "main_llm" |
| 83 | +vlm = "vlm_inspection" |
16 | 84 | llm_reasoning = false |
17 | 85 | vlm_reasoning = false |
18 | 86 |
|
19 | 87 | # inspection agent uses vlm to determine anomalies in the image |
20 | 88 | [inspection_agent] |
21 | | -vlm_model = "lfm2-vl" |
22 | | -vlm_base_url = "http://localhost:8081/v1" |
| 89 | +vlm = "vlm_inspection" |
23 | 90 | vlm_reasoning = false |
24 | 91 | vlm_temperature = 0.0 |
25 | 92 |
|
26 | 93 | [condition_agent] |
27 | | -vlm_model = "lfm2-vl" |
28 | | -vlm_base_url = "http://localhost:8081/v1" |
| 94 | +vlm = "vlm_inspection" |
29 | 95 | vlm_reasoning = false |
30 | 96 | vlm_temperature = 0.1 |
31 | 97 |
|
32 | 98 | # safety agent uses vlm to determine anomalies in the image |
33 | | -# and embeddings to determine OSHA safety violations |
| 99 | +# and embeddings + reranker to determine OSHA safety violations |
34 | 100 | [safety_agent] |
35 | | -vlm_model = "lfm2-vl" |
36 | | -vlm_base_url = "http://localhost:8081/v1" |
| 101 | +vlm = "vlm_safety" |
37 | 102 | vlm_reasoning = false |
38 | 103 | vlm_temperature = 0.0 |
39 | | - |
40 | | -embeddings_model = "qwen3-embedding:0.6b" |
41 | | -embeddings_base_url = "http://localhost:8082/v1" |
42 | | -# NOTE: The reranker and the embedding model are two separate models! |
43 | | -reranker_base_url = "http://localhost:8083/v1/reranking" |
44 | | - |
| 104 | +embeddings = "embeddings" |
| 105 | +reranker = "reranker" |
0 commit comments