diff --git a/templates/use-case-template/README.md b/templates/use-case-template/README.md index 12f921a..2948634 100644 --- a/templates/use-case-template/README.md +++ b/templates/use-case-template/README.md @@ -33,7 +33,7 @@ Or, with `uv` (recommended — this folder is a `uv` project): `uv sync`. | Environment variable | Required | Description | |---|---|---| -| `ANTHROPIC_API_KEY` | for `run`/`eval`/`optimize`/`promote` | Anthropic key; used via litellm for generation, judge metrics, and the optimizer | +| `ANTHROPIC_API_KEY` (or the key for your `OPIK_EXAMPLES_MODEL` provider) | for live `run`/`eval`/`optimize`/`promote` | Model-provider key used via litellm for generation, judge metrics, and the optimizer | | `OPIK_API_KEY` | for `eval`/`optimize`/`promote` | Your Opik API key. Unset → those commands run in DRY_RUN | | `OPIK_WORKSPACE` | for `eval`/`optimize`/`promote` | Your Opik workspace name | | `OPIK_PROJECT_NAME` | No | Opik project for traces/experiments (default `example-use-case`) | diff --git a/templates/use-case-template/src/example_use_case/cli.py b/templates/use-case-template/src/example_use_case/cli.py index 330e07d..94f0c56 100644 --- a/templates/use-case-template/src/example_use_case/cli.py +++ b/templates/use-case-template/src/example_use_case/cli.py @@ -8,9 +8,9 @@ @app.command() def run(input: str, context: list[str] = typer.Option(None, "--context", "-c")) -> None: - """Run the app on a single input (traced in Opik when an LLM key is set).""" - if not config.LLM_READY: - typer.echo("[DRY RUN] ANTHROPIC_API_KEY not set — would run the app on:") + """Run the app on a single input (traced in Opik when credentials are set).""" + if config.DRY_RUN: + typer.echo("[DRY RUN] Opik creds not set — would run the app on:") typer.echo(f" input: {input}") for line in context or []: typer.echo(f" context: {line}") diff --git a/templates/use-case-template/src/example_use_case/config.py b/templates/use-case-template/src/example_use_case/config.py index 4e44f8d..aa5650e 100644 --- a/templates/use-case-template/src/example_use_case/config.py +++ b/templates/use-case-template/src/example_use_case/config.py @@ -3,12 +3,9 @@ OPIK_API_KEY = os.environ.get("OPIK_API_KEY") OPIK_WORKSPACE = os.environ.get("OPIK_WORKSPACE") OPIK_PROJECT_NAME = os.environ.get("OPIK_PROJECT_NAME", "example-use-case") -ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY") # No Opik credentials -> every command still runs and prints locally instead of calling Opik. DRY_RUN = not (OPIK_API_KEY and OPIK_WORKSPACE) -# Generation/judging needs an Anthropic key; without it we describe what would happen. -LLM_READY = bool(ANTHROPIC_API_KEY) # litellm model strings (Anthropic provider). Swap for any litellm-supported model. # CI sets OPIK_EXAMPLES_MODEL to a cheap model; locally, leave it unset to use the full model. diff --git a/use-cases/f1_radio_rag/README.md b/use-cases/f1_radio_rag/README.md index 5a82d78..9e4d77b 100644 --- a/use-cases/f1_radio_rag/README.md +++ b/use-cases/f1_radio_rag/README.md @@ -38,10 +38,11 @@ Or, with `uv` (recommended — this folder is a `uv` project): `uv sync`. | Environment variable | Required | Description | |---|---|---| -| `ANTHROPIC_API_KEY` | for `ask`/`eval`/`optimize`/`promote` | Anthropic key; used via litellm for generation, the LLM-judge metrics, and the optimizer | +| `ANTHROPIC_API_KEY` (or the key for your `OPIK_EXAMPLES_MODEL` provider) | for live `ask`/`eval`/`optimize`/`promote` | Model-provider key used via litellm for generation, the LLM-judge metrics, and the optimizer | | `OPIK_API_KEY` | for `eval`/`optimize`/`promote` | Your Opik API key. Unset → those commands run in DRY_RUN | | `OPIK_WORKSPACE` | for `eval`/`optimize`/`promote` | Your Opik workspace name | | `OPIK_PROJECT_NAME` | No | Opik project for traces/experiments (default `f1-radio-rag`) | +| `OPIK_EXAMPLES_MODEL` | No | litellm model for generation/judging/optimising. Unset → `anthropic/claude-sonnet-4-6`; CI sets a cheap model (e.g. `openai/gpt-4o-mini`) | | `OPIK_URL_OVERRIDE` | No | Base URL for self-hosted Opik (default: Opik Cloud) | ## Running it @@ -66,13 +67,22 @@ uv run f1rag promote # optimised prompt saved to the Prompt Library (version uv run f1rag run-all # the whole loop in one shot ``` +`run.sh` is the entrypoint CI runs: it exports `OPIK_PROJECT_NAME`, then `uv sync` and runs +`ingest` + `ask`. With no credentials it stays in dry-run and exits 0 (the secrets-free CI check); +with credentials set it logs a live trace to Opik. + +```bash +bash run.sh +``` + ## How it works 1. **Ingest** (`rag.py`) — `chromadb.PersistentClient` stores one document per radio message with session/driver/lap metadata, using ChromaDB's default local embeddings (no embedding-API cost). 2. **Ask** (`rag.py`) — `answer()` retrieves the top-k messages, then calls - `litellm.completion(model="anthropic/claude-sonnet-4-6", ...)` with the summariser prompt from - `prompts.py`. It's decorated with `@opik.track`, so each call appears as a trace in Opik. + `litellm.completion(model=config.GEN_MODEL, ...)` with the summariser prompt from `prompts.py` + (`GEN_MODEL` defaults to `anthropic/claude-sonnet-4-6`, overridable via `OPIK_EXAMPLES_MODEL`). + It's decorated with `@opik.track`, so each call appears as a trace in Opik. 3. **Eval** (`evaluation.py`) — builds an Opik dataset and a test suite, then scores the live RAG task. The test suite checks plain-English **assertions**; `evaluate` runs the `ContextRecall` (retrieval quality) and `Hallucination` (faithfulness) metrics. The eval cases live in diff --git a/use-cases/f1_radio_rag/run.sh b/use-cases/f1_radio_rag/run.sh new file mode 100644 index 0000000..f4e8df4 --- /dev/null +++ b/use-cases/f1_radio_rag/run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +export OPIK_PROJECT_NAME="f1-radio-rag" + +uv sync + +uv run f1rag ingest +uv run f1rag ask "Why did Verstappen pit early?" diff --git a/use-cases/f1_radio_rag/src/f1_radio_rag/cli.py b/use-cases/f1_radio_rag/src/f1_radio_rag/cli.py index cb2ee47..76eab17 100644 --- a/use-cases/f1_radio_rag/src/f1_radio_rag/cli.py +++ b/use-cases/f1_radio_rag/src/f1_radio_rag/cli.py @@ -23,8 +23,8 @@ def ask(query: str, k: int = 5) -> None: typer.echo("Retrieved messages:") for message in context: typer.echo(f" - {message}") - if not config.LLM_READY: - typer.echo("\n[DRY RUN] ANTHROPIC_API_KEY not set — would summarise the above with Claude.") + if config.DRY_RUN: + typer.echo(f"\n[DRY RUN] Opik creds not set — would summarise with {config.GEN_MODEL}.") return result = rag.answer(query, k) typer.echo(f"\nSummary:\n{result['output']}") diff --git a/use-cases/f1_radio_rag/src/f1_radio_rag/config.py b/use-cases/f1_radio_rag/src/f1_radio_rag/config.py index 46aa71b..bcfef63 100644 --- a/use-cases/f1_radio_rag/src/f1_radio_rag/config.py +++ b/use-cases/f1_radio_rag/src/f1_radio_rag/config.py @@ -3,17 +3,15 @@ OPIK_API_KEY = os.environ.get("OPIK_API_KEY") OPIK_WORKSPACE = os.environ.get("OPIK_WORKSPACE") OPIK_PROJECT_NAME = os.environ.get("OPIK_PROJECT_NAME", "f1-radio-rag") -ANTHROPIC_API_KEY = os.environ.get("ANTHROPIC_API_KEY") # No Opik credentials -> every command still runs and prints locally instead of calling Opik. DRY_RUN = not (OPIK_API_KEY and OPIK_WORKSPACE) -# Generation/judging needs an Anthropic key; without it we describe what would happen. -LLM_READY = bool(ANTHROPIC_API_KEY) -# litellm model strings (Anthropic provider). -GEN_MODEL = "anthropic/claude-sonnet-4-6" # deployed summariser -JUDGE_MODEL = "anthropic/claude-sonnet-4-6" # LLM-as-judge for metrics -OPTIMIZER_MODEL = "anthropic/claude-sonnet-4-6" # meta-model that rewrites the prompt +# litellm model strings. CI sets OPIK_EXAMPLES_MODEL to a cheap model (e.g. openai/gpt-4o-mini); +# locally, leave it unset to use the full model. +GEN_MODEL = os.environ.get("OPIK_EXAMPLES_MODEL", "anthropic/claude-sonnet-4-6") # deployed summariser +JUDGE_MODEL = GEN_MODEL # LLM-as-judge for metrics +OPTIMIZER_MODEL = GEN_MODEL # meta-model that rewrites the prompt CHROMA_DIR = "chroma_db" COLLECTION = "f1_radio"