From f00263258e8764f2a2fbea37d9d19bfaa56c5e30 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 14:31:57 +0200 Subject: [PATCH 01/13] chore: scaffold prompt_agent_optimization guide folder --- guides/prompt_agent_optimization/.gitignore | 5 ++++ guides/prompt_agent_optimization/README.md | 3 ++ .../optimization_guide/__init__.py | 0 .../prompt_agent_optimization/pyproject.toml | 28 +++++++++++++++++++ 4 files changed, 36 insertions(+) create mode 100644 guides/prompt_agent_optimization/.gitignore create mode 100644 guides/prompt_agent_optimization/README.md create mode 100644 guides/prompt_agent_optimization/optimization_guide/__init__.py create mode 100644 guides/prompt_agent_optimization/pyproject.toml diff --git a/guides/prompt_agent_optimization/.gitignore b/guides/prompt_agent_optimization/.gitignore new file mode 100644 index 0000000..ffb4101 --- /dev/null +++ b/guides/prompt_agent_optimization/.gitignore @@ -0,0 +1,5 @@ +.venv/ +chroma_db/ +__pycache__/ +*.pyc +.ipynb_checkpoints/ diff --git a/guides/prompt_agent_optimization/README.md b/guides/prompt_agent_optimization/README.md new file mode 100644 index 0000000..35b0113 --- /dev/null +++ b/guides/prompt_agent_optimization/README.md @@ -0,0 +1,3 @@ +# Prompt & Agent Optimization with Opik + +Placeholder — see `prompt_agent_optimization.ipynb`. Full README written in Task 10. diff --git a/guides/prompt_agent_optimization/optimization_guide/__init__.py b/guides/prompt_agent_optimization/optimization_guide/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/guides/prompt_agent_optimization/pyproject.toml b/guides/prompt_agent_optimization/pyproject.toml new file mode 100644 index 0000000..e9b757a --- /dev/null +++ b/guides/prompt_agent_optimization/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "prompt-agent-optimization" +version = "0.1.0" +description = "Prompt & agent optimization with Opik, end-to-end over an escalating RAG-over-docs example." +readme = "README.md" +requires-python = ">=3.12,<3.14" +dependencies = [ + "opik>=2.0", + "opik-optimizer", + "chromadb", + "litellm", +] + +[dependency-groups] +dev = ["ruff", "pytest"] + +# WHY: notebook-only example — uv manages the env, no installable package. +[tool.uv] +package = false + +[tool.ruff] +line-length = 110 +target-version = "py312" +# WHY: ruff can't parse the notebook's cell schema; lint .py files only. +extend-exclude = ["*.ipynb"] + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B"] From f0a7229a713761746915c11cebfa9a0080b74dc3 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 14:35:24 +0200 Subject: [PATCH 02/13] feat: config module with fail-fast prerequisites check --- .../optimization_guide/config.py | 42 ++++++++++++++++ .../tests/conftest.py | 5 ++ .../tests/test_config.py | 49 +++++++++++++++++++ 3 files changed, 96 insertions(+) create mode 100644 guides/prompt_agent_optimization/optimization_guide/config.py create mode 100644 guides/prompt_agent_optimization/tests/conftest.py create mode 100644 guides/prompt_agent_optimization/tests/test_config.py diff --git a/guides/prompt_agent_optimization/optimization_guide/config.py b/guides/prompt_agent_optimization/optimization_guide/config.py new file mode 100644 index 0000000..19ee0c5 --- /dev/null +++ b/guides/prompt_agent_optimization/optimization_guide/config.py @@ -0,0 +1,42 @@ +import os +from pathlib import Path + +DEFAULT_MODEL = "anthropic/claude-sonnet-4-6" + +_MODEL = os.environ.get("OPIK_EXAMPLES_MODEL", DEFAULT_MODEL) +GEN_MODEL = _MODEL +JUDGE_MODEL = _MODEL +OPTIMIZER_MODEL = _MODEL + +PROJECT_NAME = os.environ.get("OPIK_PROJECT_NAME", "prompt-agent-optimization") + +DATA_DIR = Path(__file__).resolve().parent.parent / "data" +CHROMA_DIR = str(Path(__file__).resolve().parent.parent / "chroma_db") +COLLECTION = "product_docs" + +# Provider key env var expected for each litellm provider prefix. +_PROVIDER_KEYS = { + "anthropic/": "ANTHROPIC_API_KEY", + "openai/": "OPENAI_API_KEY", + "gemini/": "GEMINI_API_KEY", +} + + +def check_prerequisites() -> None: + """Raise RuntimeError listing every missing required env var. No DRY_RUN fallback.""" + missing = [] + for var in ("OPIK_API_KEY", "OPIK_WORKSPACE"): + if not os.environ.get(var): + missing.append(var) + provider_key = next( + (key for prefix, key in _PROVIDER_KEYS.items() if GEN_MODEL.startswith(prefix)), + None, + ) + if provider_key and not os.environ.get(provider_key): + missing.append(f"{provider_key} (for model {GEN_MODEL})") + if missing: + raise RuntimeError( + "Missing required environment variables: " + + ", ".join(missing) + + ". Set them before running this guide (see the README)." + ) diff --git a/guides/prompt_agent_optimization/tests/conftest.py b/guides/prompt_agent_optimization/tests/conftest.py new file mode 100644 index 0000000..f0e3b4b --- /dev/null +++ b/guides/prompt_agent_optimization/tests/conftest.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path + +# Add parent directory to sys.path so tests can import optimization_guide +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) diff --git a/guides/prompt_agent_optimization/tests/test_config.py b/guides/prompt_agent_optimization/tests/test_config.py new file mode 100644 index 0000000..f0abfa8 --- /dev/null +++ b/guides/prompt_agent_optimization/tests/test_config.py @@ -0,0 +1,49 @@ +import importlib + +import pytest + + +def _reload_config(monkeypatch, **env): + env_vars = [ + "OPIK_API_KEY", + "OPIK_WORKSPACE", + "OPIK_EXAMPLES_MODEL", + "OPIK_PROJECT_NAME", + "ANTHROPIC_API_KEY", + ] + for k in env_vars: + monkeypatch.delenv(k, raising=False) + for k, v in env.items(): + monkeypatch.setenv(k, v) + import optimization_guide.config as config + return importlib.reload(config) + + +def test_default_model_when_unset(monkeypatch): + config = _reload_config(monkeypatch) + assert config.GEN_MODEL == "anthropic/claude-sonnet-4-6" + assert config.JUDGE_MODEL == "anthropic/claude-sonnet-4-6" + assert config.OPTIMIZER_MODEL == "anthropic/claude-sonnet-4-6" + + +def test_model_override(monkeypatch): + config = _reload_config(monkeypatch, OPIK_EXAMPLES_MODEL="openai/gpt-4o-mini") + assert config.GEN_MODEL == "openai/gpt-4o-mini" + + +def test_project_name_default(monkeypatch): + config = _reload_config(monkeypatch) + assert config.PROJECT_NAME == "prompt-agent-optimization" + + +def test_check_prerequisites_raises_when_missing(monkeypatch): + config = _reload_config(monkeypatch) # no OPIK_API_KEY / OPIK_WORKSPACE + with pytest.raises(RuntimeError) as exc: + config.check_prerequisites() + assert "OPIK_API_KEY" in str(exc.value) + assert "OPIK_WORKSPACE" in str(exc.value) + + +def test_check_prerequisites_passes_when_present(monkeypatch): + config = _reload_config(monkeypatch, OPIK_API_KEY="x", OPIK_WORKSPACE="w", ANTHROPIC_API_KEY="k") + assert config.check_prerequisites() is None From ac8b1b174b9c8ad3e37ff1412aed41451ef9ad0b Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 14:39:42 +0200 Subject: [PATCH 03/13] feat: fictional Ledgerline corpus + exact and judge eval datasets --- .../prompt_agent_optimization/data/docs.json | 14 +++++++++++++ .../data/eval_cases_exact.json | 20 +++++++++++++++++++ .../data/eval_cases_judge.json | 14 +++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 guides/prompt_agent_optimization/data/docs.json create mode 100644 guides/prompt_agent_optimization/data/eval_cases_exact.json create mode 100644 guides/prompt_agent_optimization/data/eval_cases_judge.json diff --git a/guides/prompt_agent_optimization/data/docs.json b/guides/prompt_agent_optimization/data/docs.json new file mode 100644 index 0000000..4718ce6 --- /dev/null +++ b/guides/prompt_agent_optimization/data/docs.json @@ -0,0 +1,14 @@ +[ + {"id": "timeouts", "title": "Job timeouts", "text": "Every Ledgerline job has a default timeout of 30 seconds. Jobs exceeding the timeout are marked failed and eligible for retry. The maximum configurable timeout is 15 minutes."}, + {"id": "retries", "title": "Retries", "text": "Failed jobs are retried automatically. The default maximum number of retries is 3, using exponential backoff starting at 2 seconds. Set max_retries to 0 to disable retries."}, + {"id": "rate-limits", "title": "Rate limits", "text": "The API allows 1000 requests per minute per API key. Exceeding the limit returns HTTP 429. Rate limit headers are included on every response."}, + {"id": "auth", "title": "Authentication", "text": "Authenticate by sending your API key in the Authorization header as a Bearer token: 'Authorization: Bearer '. Keys are created in the dashboard."}, + {"id": "priorities", "title": "Queue priorities", "text": "Ledgerline supports three queue priorities: low, default, and high. High-priority jobs are dequeued before default and low. Priority is set per job at enqueue time."}, + {"id": "dead-letter", "title": "Dead-letter queue", "text": "After a job exhausts all retries it is moved to the dead-letter queue, where it is retained for 7 days before permanent deletion. Dead-letter jobs can be replayed from the dashboard."}, + {"id": "webhooks", "title": "Webhooks", "text": "When a job completes, Ledgerline POSTs a webhook to your configured URL. The payload includes job_id, status, and result fields. Webhook deliveries are signed with the X-Ledgerline-Signature header."}, + {"id": "install", "title": "SDK installation", "text": "Install the Python SDK with 'pip install ledgerline'. The SDK requires Python 3.9 or newer. Import it as 'import ledgerline'."}, + {"id": "concurrency", "title": "Concurrency", "text": "Each project runs up to 50 concurrent jobs by default. Contact support to raise the concurrency limit for your plan."}, + {"id": "regions", "title": "Regions", "text": "Ledgerline is available in three regions: us-east, eu-west, and ap-south. The default region is us-east. Set the region when initializing the client."}, + {"id": "batch", "title": "Batch enqueue", "text": "You can enqueue up to 500 jobs in a single batch request. Larger batches must be split. Each job in a batch is billed individually."}, + {"id": "idempotency", "title": "Idempotency", "text": "Pass an Idempotency-Key header to safely retry enqueue requests. Ledgerline deduplicates requests with the same key for 24 hours."} +] diff --git a/guides/prompt_agent_optimization/data/eval_cases_exact.json b/guides/prompt_agent_optimization/data/eval_cases_exact.json new file mode 100644 index 0000000..c423f51 --- /dev/null +++ b/guides/prompt_agent_optimization/data/eval_cases_exact.json @@ -0,0 +1,20 @@ +[ + {"query": "What is the default job timeout?", "expected_substring": "30 seconds"}, + {"query": "What is the maximum configurable timeout?", "expected_substring": "15 minutes"}, + {"query": "How many times are failed jobs retried by default?", "expected_substring": "3"}, + {"query": "How do I disable retries?", "expected_substring": "max_retries to 0"}, + {"query": "What backoff does retry use, and starting at what delay?", "expected_substring": "2 seconds"}, + {"query": "How many requests per minute per API key are allowed?", "expected_substring": "1000 requests per minute"}, + {"query": "What HTTP status is returned when the rate limit is exceeded?", "expected_substring": "429"}, + {"query": "Which header carries the API key?", "expected_substring": "Authorization"}, + {"query": "What token scheme is used for auth?", "expected_substring": "Bearer"}, + {"query": "What queue priorities are supported?", "expected_substring": "low, default, and high"}, + {"query": "How long are dead-letter jobs retained?", "expected_substring": "7 days"}, + {"query": "Which header signs webhook deliveries?", "expected_substring": "X-Ledgerline-Signature"}, + {"query": "How do I install the Python SDK?", "expected_substring": "pip install ledgerline"}, + {"query": "What Python version does the SDK require?", "expected_substring": "3.9"}, + {"query": "How many concurrent jobs run per project by default?", "expected_substring": "50 concurrent jobs"}, + {"query": "What is the default region?", "expected_substring": "us-east"}, + {"query": "How many jobs can I enqueue in one batch?", "expected_substring": "500 jobs"}, + {"query": "How long are idempotency keys deduplicated?", "expected_substring": "24 hours"} +] diff --git a/guides/prompt_agent_optimization/data/eval_cases_judge.json b/guides/prompt_agent_optimization/data/eval_cases_judge.json new file mode 100644 index 0000000..4b641e2 --- /dev/null +++ b/guides/prompt_agent_optimization/data/eval_cases_judge.json @@ -0,0 +1,14 @@ +[ + {"query": "How should I handle a job that keeps failing?", "reference": "Explain retries with exponential backoff, the default of 3 retries, and that exhausted jobs move to the dead-letter queue (retained 7 days, replayable from the dashboard)."}, + {"query": "How do I make sure I don't enqueue the same job twice if my request retries?", "reference": "Use an Idempotency-Key header; Ledgerline deduplicates same-key requests for 24 hours."}, + {"query": "What's the best way to authenticate my requests?", "reference": "Send the API key as a Bearer token in the Authorization header; create keys in the dashboard."}, + {"query": "How do I get notified when a job finishes?", "reference": "Configure a webhook URL; Ledgerline POSTs job_id, status, and result, signed with X-Ledgerline-Signature."}, + {"query": "How can I prioritise urgent work?", "reference": "Set the job priority to high at enqueue time; high-priority jobs are dequeued before default and low."}, + {"query": "How do I run more jobs at the same time?", "reference": "Default concurrency is 50 concurrent jobs per project; contact support to raise the limit."}, + {"query": "How do I choose where my jobs run?", "reference": "Set the region (us-east, eu-west, ap-south) when initializing the client; default is us-east."}, + {"query": "What happens when I hit the rate limit?", "reference": "Requests over 1000/min per key return HTTP 429; rate-limit headers are on every response."}, + {"query": "How do I submit many jobs efficiently?", "reference": "Use batch enqueue, up to 500 jobs per request; split larger batches; each job billed individually."}, + {"query": "How long do I have to recover a permanently failing job?", "reference": "Dead-letter jobs are retained 7 days before permanent deletion and can be replayed from the dashboard."}, + {"query": "Can I make jobs run longer than the default?", "reference": "Yes; the default timeout is 30 seconds and the maximum configurable timeout is 15 minutes."}, + {"query": "How do I start using the SDK in Python?", "reference": "Install with pip install ledgerline (Python 3.9+), then import ledgerline."} +] From ea5e508430828e03f1aec98aaf0b0f0668e27a46 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 14:42:30 +0200 Subject: [PATCH 04/13] feat: data loaders and Opik dataset builder --- .../optimization_guide/data.py | 30 ++++++++++++ .../tests/test_data.py | 49 +++++++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 guides/prompt_agent_optimization/optimization_guide/data.py create mode 100644 guides/prompt_agent_optimization/tests/test_data.py diff --git a/guides/prompt_agent_optimization/optimization_guide/data.py b/guides/prompt_agent_optimization/optimization_guide/data.py new file mode 100644 index 0000000..02dcd09 --- /dev/null +++ b/guides/prompt_agent_optimization/optimization_guide/data.py @@ -0,0 +1,30 @@ +import json +from typing import Any + +from . import config + + +def _load(name: str) -> list[dict]: + return json.loads((config.DATA_DIR / name).read_text()) + + +def load_docs() -> list[dict]: + return _load("docs.json") + + +def load_exact_cases() -> list[dict]: + return _load("eval_cases_exact.json") + + +def load_judge_cases() -> list[dict]: + return _load("eval_cases_judge.json") + + +def build_dataset(client: Any, name: str, cases: list[dict]) -> Any: + """Get-or-create an Opik dataset and insert cases. + + Opik dedups identical items on insert, so re-running is safe (idempotent). + """ + dataset = client.get_or_create_dataset(name) + dataset.insert(cases) + return dataset diff --git a/guides/prompt_agent_optimization/tests/test_data.py b/guides/prompt_agent_optimization/tests/test_data.py new file mode 100644 index 0000000..980c9fa --- /dev/null +++ b/guides/prompt_agent_optimization/tests/test_data.py @@ -0,0 +1,49 @@ +from optimization_guide import data + + +def test_load_docs_shape(): + docs = data.load_docs() + assert len(docs) >= 10 + assert all({"id", "title", "text"} <= set(d) for d in docs) + + +def test_load_exact_cases_grounded(): + docs = data.load_docs() + corpus = " ".join(d["text"] for d in docs) + cases = data.load_exact_cases() + assert len(cases) >= 15 + for c in cases: + assert c["expected_substring"] in corpus + + +def test_load_judge_cases_shape(): + cases = data.load_judge_cases() + assert len(cases) >= 10 + assert all({"query", "reference"} <= set(c) for c in cases) + + +class _FakeDataset: + def __init__(self, name): + self.name = name + self.items = [] + + def insert(self, items): + self.items.extend(items) + + +class _FakeClient: + def __init__(self): + self.created = {} + + def get_or_create_dataset(self, name): + ds = self.created.setdefault(name, _FakeDataset(name)) + return ds + + +def test_build_dataset_inserts_cases(): + client = _FakeClient() + cases = [{"query": "q1", "expected_substring": "a"}, {"query": "q2", "expected_substring": "b"}] + ds = data.build_dataset(client, "exact-eval", cases) + assert ds.name == "exact-eval" + assert len(ds.items) == 2 + assert ds.items[0]["query"] == "q1" From 4ca295ea3c91edada610def019dbf937da0b9685 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 14:45:35 +0200 Subject: [PATCH 05/13] feat: RAG retriever, answer function, and agent retrieval gate --- .../optimization_guide/rag_app.py | 68 +++++++++++++++++++ .../tests/test_rag_app.py | 67 ++++++++++++++++++ 2 files changed, 135 insertions(+) create mode 100644 guides/prompt_agent_optimization/optimization_guide/rag_app.py create mode 100644 guides/prompt_agent_optimization/tests/test_rag_app.py diff --git a/guides/prompt_agent_optimization/optimization_guide/rag_app.py b/guides/prompt_agent_optimization/optimization_guide/rag_app.py new file mode 100644 index 0000000..dc61848 --- /dev/null +++ b/guides/prompt_agent_optimization/optimization_guide/rag_app.py @@ -0,0 +1,68 @@ +import threading + +import chromadb +import litellm +import opik + +from . import config + +_collection = None +_collection_lock = threading.Lock() + + +def get_collection(): + # WHY: cache one PersistentClient. Optimizer/evaluate call the task across worker + # threads; concurrent PersistentClient creation races on tenant validation. + global _collection + if _collection is None: + with _collection_lock: + if _collection is None: + client = chromadb.PersistentClient(path=config.CHROMA_DIR) + _collection = client.get_or_create_collection( + name=config.COLLECTION, metadata={"hnsw:space": "cosine"} + ) + return _collection + + +def ingest(docs: list[dict]) -> int: + collection = get_collection() + collection.upsert( + ids=[d["id"] for d in docs], + documents=[d["text"] for d in docs], + metadatas=[{"title": d["title"]} for d in docs], + ) + return collection.count() + + +def retrieve(query: str, n_results: int = 3) -> list[str]: + collection = get_collection() + result = collection.query(query_texts=[query], n_results=n_results) + return result["documents"][0] + + +@opik.track +def answer(query: str, system_prompt: str, model: str | None = None) -> str: + context = "\n\n".join(retrieve(query)) + messages = [ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": f"Context:\n{context}\n\nQuestion: {query}"}, + ] + response = litellm.completion(model=model or config.GEN_MODEL, messages=messages) + return response.choices[0].message.content + + +@opik.track +def should_retrieve(query: str, model: str | None = None) -> bool: + """Part 3 agent gate: decide whether this query needs a docs lookup.""" + messages = [ + { + "role": "system", + "content": ( + "You decide whether a user question about the Ledgerline product needs a " + "documentation lookup. Answer with exactly YES or NO." + ), + }, + {"role": "user", "content": query}, + ] + response = litellm.completion(model=model or config.GEN_MODEL, messages=messages) + return response.choices[0].message.content.strip().upper().startswith("YES") diff --git a/guides/prompt_agent_optimization/tests/test_rag_app.py b/guides/prompt_agent_optimization/tests/test_rag_app.py new file mode 100644 index 0000000..dfe3c1d --- /dev/null +++ b/guides/prompt_agent_optimization/tests/test_rag_app.py @@ -0,0 +1,67 @@ +import pytest + + +@pytest.fixture +def rag(monkeypatch, tmp_path): + # Point Chroma at a temp dir before importing the module-level singleton. + import importlib + + monkeypatch.setenv("OPIK_EXAMPLES_MODEL", "anthropic/claude-sonnet-4-6") + import optimization_guide.config as config + + importlib.reload(config) + monkeypatch.setattr(config, "CHROMA_DIR", str(tmp_path / "chroma")) + monkeypatch.setattr(config, "COLLECTION", "test_docs") + import optimization_guide.rag_app as rag_app + + importlib.reload(rag_app) + # reset cached singleton + rag_app._collection = None + return rag_app + + +def test_ingest_and_retrieve(rag): + docs = [ + {"id": "a", "title": "Timeouts", "text": "The default job timeout is 30 seconds."}, + {"id": "b", "title": "Regions", "text": "The default region is us-east."}, + ] + count = rag.ingest(docs) + assert count == 2 + hits = rag.retrieve("what is the default timeout", n_results=1) + assert len(hits) == 1 + assert "30 seconds" in hits[0] + + +def test_answer_uses_context_and_prompt(rag, monkeypatch): + rag.ingest([{"id": "a", "title": "Timeouts", "text": "The default job timeout is 30 seconds."}]) + captured = {} + + def fake_completion(model, messages, **kwargs): + captured["model"] = model + captured["messages"] = messages + + class R: + choices = [type("C", (), {"message": type("M", (), {"content": "It is 30 seconds."})()})()] + return R() + + monkeypatch.setattr(rag.litellm, "completion", fake_completion) + out = rag.answer( + "what is the default timeout", + system_prompt="You are helpful.", + model="anthropic/claude-sonnet-4-6", + ) + assert out == "It is 30 seconds." + # system prompt propagated, context injected + assert captured["messages"][0]["role"] == "system" + assert "You are helpful." in captured["messages"][0]["content"] + assert any("30 seconds" in m["content"] for m in captured["messages"]) + + +def test_should_retrieve_parses_yes(rag, monkeypatch): + def fake_completion(model, messages, **kwargs): + class R: + choices = [type("C", (), {"message": type("M", (), {"content": "YES"})()})()] + return R() + + monkeypatch.setattr(rag.litellm, "completion", fake_completion) + assert rag.should_retrieve("how do I configure retries?") is True From 35a83fb7a09c7645cc64caee2ca71cfcade382d7 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 14:50:43 +0200 Subject: [PATCH 06/13] chore: add jupyter tooling deps + silence Opik tracking in tests Notebook authoring (nbformat) and headless execution (jupyter nbconvert) need Jupyter in the env; the plan's pyproject omitted it. Also disable Opik tracking in tests so credential-less @opik.track calls stop emitting 401 teardown noise, keeping test output pristine. --- guides/prompt_agent_optimization/pyproject.toml | 7 +++++++ guides/prompt_agent_optimization/tests/conftest.py | 6 ++++++ 2 files changed, 13 insertions(+) diff --git a/guides/prompt_agent_optimization/pyproject.toml b/guides/prompt_agent_optimization/pyproject.toml index e9b757a..30983ad 100644 --- a/guides/prompt_agent_optimization/pyproject.toml +++ b/guides/prompt_agent_optimization/pyproject.toml @@ -9,6 +9,13 @@ dependencies = [ "opik-optimizer", "chromadb", "litellm", + # WHY: this guide's deliverable is a notebook — Jupyter + nbconvert/nbformat + # are runtime deps so `uv sync` gives users a working kernel and lets the + # notebook be validated and executed headlessly. + "jupyterlab", + "nbconvert", + "nbformat", + "ipykernel", ] [dependency-groups] diff --git a/guides/prompt_agent_optimization/tests/conftest.py b/guides/prompt_agent_optimization/tests/conftest.py index f0e3b4b..cc97ef0 100644 --- a/guides/prompt_agent_optimization/tests/conftest.py +++ b/guides/prompt_agent_optimization/tests/conftest.py @@ -1,5 +1,11 @@ +import os import sys from pathlib import Path +# Disable Opik tracking during tests: the @opik.track-decorated rag_app +# functions would otherwise flush spans to the backend at teardown and, with no +# credentials in the test env, emit 401 noise after the pytest summary. +os.environ.setdefault("OPIK_TRACK_DISABLE", "true") + # Add parent directory to sys.path so tests can import optimization_guide sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) From c174551483b2d66e6c8b8645846e670128157604 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 14:54:32 +0200 Subject: [PATCH 07/13] =?UTF-8?q?feat:=20notebook=20Part=200=20(framing)?= =?UTF-8?q?=20and=20Part=201=20(workshop)=20=E2=80=94=20exact-match=20opti?= =?UTF-8?q?mization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../prompt_agent_optimization.ipynb | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 guides/prompt_agent_optimization/prompt_agent_optimization.ipynb diff --git a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb new file mode 100644 index 0000000..033c5b6 --- /dev/null +++ b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb @@ -0,0 +1,210 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "169afc4d", + "metadata": {}, + "source": [ + "# Prompt & Agent Optimization with Opik — an A-to-Z Guide\n", + "\n", + "This notebook takes you from *\"I have a prompt that works okay\"* to *\"my prompt and my agent are measurably better, and every improvement is a comparable run in Opik.\"*\n", + "\n", + "**Two ways to use it:**\n", + "- **Live workshop (≈20 min):** run **Part 1** top to bottom. You'll optimize a RAG answer prompt against an exact-match metric and see the improvement in Opik.\n", + "- **Take-home guide:** continue through Parts 2–5 — LLM-judge metrics (and how to *trust* them), multi-objective optimization, and optimizing an agent's tool use.\n", + "\n", + "We optimize a **documentation assistant** for a fictional product, **Ledgerline** (a task-queue API), so the corpus is clean and the lesson is about *optimization*, not about parsing messy docs." + ] + }, + { + "cell_type": "markdown", + "id": "2b26fbe8", + "metadata": {}, + "source": [ + "## Part 0 — How to think about prompt optimization\n", + "\n", + "**When do you start?** When you have (1) a prompt that works *okay*, (2) a dataset of representative inputs, and (3) a metric that says how good an output is — and hand-tuning has plateaued.\n", + "\n", + "**The mental shift.** Classic optimization gives you an objective and a gradient. Prompt optimization is different: the **search space is prompt text**, and the **objective is a metric computed over a dataset**. You can't differentiate it, so optimizers *propose* candidate prompts, *evaluate* them on your dataset, keep the best, and repeat.\n", + "\n", + "The three ingredients map exactly to three objects you'll build:\n", + "\n", + "| Ingredient | Opik object |\n", + "|---|---|\n", + "| The prompt | `ChatPrompt` |\n", + "| The dataset | Opik `Dataset` |\n", + "| The metric | a callable `(dataset_item, llm_output) -> float` |\n", + "\n", + "The loop, once, looks like: **propose candidate → evaluate on dataset → keep best → repeat.** Everything below is that loop, escalating in complexity." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33b93828", + "metadata": {}, + "outputs": [], + "source": [ + "import opik\n", + "from optimization_guide import config, data, rag_app\n", + "\n", + "# Fail fast with a clear message if credentials are missing.\n", + "config.check_prerequisites()\n", + "\n", + "client = opik.Opik(project_name=config.PROJECT_NAME)\n", + "print(\"Using models:\", config.GEN_MODEL)" + ] + }, + { + "cell_type": "markdown", + "id": "682398ff", + "metadata": {}, + "source": [ + "### Part 1 — Your first optimization ⭐ (workshop)\n", + "\n", + "We'll ingest the Ledgerline docs, build an evaluation dataset with **checkable answers**, score a baseline prompt with an **exact-match metric** (no LLM judge needed), then let an optimizer improve the prompt." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e699b31", + "metadata": {}, + "outputs": [], + "source": [ + "docs = data.load_docs()\n", + "count = rag_app.ingest(docs)\n", + "print(f\"Ingested {count} doc snippets into ChromaDB\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "18689624", + "metadata": {}, + "outputs": [], + "source": [ + "exact_cases = data.load_exact_cases()\n", + "exact_dataset = data.build_dataset(client, \"ledgerline-exact\", exact_cases)\n", + "print(f\"Dataset 'ledgerline-exact' has {len(exact_cases)} cases\")" + ] + }, + { + "cell_type": "markdown", + "id": "f654fca1", + "metadata": {}, + "source": [ + "#### The metric: exact-match, no judge\n", + "\n", + "Our first metric is deterministic and cheap: **does the answer contain the expected fact?** Opik ships `Contains` for exactly this. Optimizer metrics are plain callables `(dataset_item, llm_output) -> float`, so we wrap `Contains` in one. *Not every metric needs an LLM.*" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "82e28ed4", + "metadata": {}, + "outputs": [], + "source": [ + "from opik.evaluation.metrics import Contains\n", + "\n", + "\n", + "def exact_match(dataset_item: dict, llm_output: str) -> float:\n", + " # Contains returns 1.0 if expected_substring is in the output, else 0.0.\n", + " result = Contains(case_sensitive=False).score(\n", + " output=llm_output,\n", + " reference=dataset_item[\"expected_substring\"],\n", + " )\n", + " return result.value\n", + "\n", + "\n", + "exact_match.__name__ = \"exact_match\"" + ] + }, + { + "cell_type": "markdown", + "id": "38d98de9", + "metadata": {}, + "source": [ + "#### The starting prompt\n", + "\n", + "Here is our baseline system prompt — deliberately mediocre, so there's room to improve. This is the `ChatPrompt` the optimizer will rewrite. `{query}` is filled from each dataset row." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "beea886d", + "metadata": {}, + "outputs": [], + "source": [ + "from opik_optimizer import ChatPrompt\n", + "\n", + "BASELINE_SYSTEM = \"You are a support bot. Answer the question.\"\n", + "\n", + "prompt = ChatPrompt(\n", + " name=\"ledgerline-answer\",\n", + " system=BASELINE_SYSTEM,\n", + " user=\"{query}\",\n", + " model=config.GEN_MODEL,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "b4a0a0f2", + "metadata": {}, + "source": [ + "#### Run the optimizer\n", + "\n", + "We use **`MetaPromptOptimizer`** — it uses a reasoning LLM to critique and rewrite the prompt. It's the docs' recommended general-purpose starting point for prompt wording. Watch the params:\n", + "- `max_trials` — how many candidate prompts to try.\n", + "- `n_samples` — dataset rows evaluated per candidate (smaller = cheaper/faster for a live run).\n", + "- `skip_perfect_score=False` — keep optimizing even if the baseline already scores high." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91592a75", + "metadata": {}, + "outputs": [], + "source": [ + "from opik_optimizer import MetaPromptOptimizer\n", + "\n", + "optimizer = MetaPromptOptimizer(\n", + " model=config.OPTIMIZER_MODEL,\n", + " n_threads=4,\n", + " skip_perfect_score=False,\n", + ")\n", + "\n", + "result = optimizer.optimize_prompt(\n", + " prompt=prompt,\n", + " dataset=exact_dataset,\n", + " metric=exact_match,\n", + " max_trials=8,\n", + " n_samples=8,\n", + ")\n", + "\n", + "print(\"Baseline score:\", result.initial_score)\n", + "print(\"Best score: \", result.score)\n", + "print(\"\\nOptimized system prompt:\\n\", result.prompt)" + ] + }, + { + "cell_type": "markdown", + "id": "8aa5a588", + "metadata": {}, + "source": [ + "#### See it in Opik\n", + "\n", + "Open **Evaluation → Optimization runs** in your Opik workspace. You'll see this run with every candidate prompt, its score, and the trace for each trial. Compare the baseline row to the best row — that delta is your improvement.\n", + "\n", + "🎓 **This is where the live workshop ends.** You've run a real optimization and improved a prompt, measured against a dataset, stored in Opik. Everything below builds on exactly this loop." + ] + } + ], + "metadata": {}, + "nbformat": 4, + "nbformat_minor": 5 +} From 167ce3b3e072a6c2c9ae9b435409d8122983ed08 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 14:58:46 +0200 Subject: [PATCH 08/13] =?UTF-8?q?feat:=20notebook=20Part=202=20=E2=80=94?= =?UTF-8?q?=20judge=20metric,=20trust-the-judge,=20multi-objective?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../prompt_agent_optimization.ipynb | 162 ++++++++++++++++++ 1 file changed, 162 insertions(+) diff --git a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb index 033c5b6..48c25c8 100644 --- a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb +++ b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb @@ -202,6 +202,168 @@ "\n", "🎓 **This is where the live workshop ends.** You've run a real optimization and improved a prompt, measured against a dataset, stored in Opik. Everything below builds on exactly this loop." ] + }, + { + "cell_type": "markdown", + "id": "693e9769", + "metadata": {}, + "source": [ + "## Part 2 — Metrics done right\n", + "\n", + "Exact-match got us far because our questions had crisp answers. But real docs questions are open-ended — *\"How should I handle a job that keeps failing?\"* has no single substring. For those you need a metric that judges **meaning**: an **LLM-as-judge**." + ] + }, + { + "cell_type": "markdown", + "id": "18467d7d", + "metadata": {}, + "source": [ + "#### The LLM-judge metric\n", + "\n", + "Opik ships judge metrics like `AnswerRelevance` (is the answer relevant to the question, given context?) and `Hallucination` (is it unsupported by context?). We wrap `AnswerRelevance` as an optimizer metric, exactly like we wrapped `Contains` — same callable shape, different scorer." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37b9ee3d", + "metadata": {}, + "outputs": [], + "source": [ + "from opik.evaluation.metrics import AnswerRelevance\n", + "\n", + "\n", + "def answer_relevance(dataset_item: dict, llm_output: str) -> float:\n", + " result = AnswerRelevance(model=config.JUDGE_MODEL).score(\n", + " input=dataset_item[\"query\"],\n", + " output=llm_output,\n", + " context=[dataset_item[\"reference\"]],\n", + " )\n", + " return result.value\n", + "\n", + "\n", + "answer_relevance.__name__ = \"answer_relevance\"" + ] + }, + { + "cell_type": "markdown", + "id": "12c87e9b", + "metadata": {}, + "source": [ + "#### How do we *trust* a judge?\n", + "\n", + "An LLM-judge is itself a prompt — it can be wrong. Before you optimize *against* it, sanity-check it:\n", + "\n", + "1. **Spot-check against your own labels.** Take 3–5 rows, decide the score yourself, and compare. If you and the judge disagree wildly, fix the judge before trusting its numbers.\n", + "2. **Read the *reason*, not just the number.** Opik judge metrics return a `reason`. A right score for the wrong reason is a red flag.\n", + "3. **Watch for drift and bias.** Judges favor longer, confident-sounding answers. If your metric rewards verbosity, your \"optimized\" prompt may just be wordier — which is exactly why Part 2 ends with a *cost/length* objective.\n", + "\n", + "Run the cell below to inspect a judge score **and its reasoning** on one example." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0b0cc2e0", + "metadata": {}, + "outputs": [], + "source": [ + "sample = data.load_judge_cases()[0]\n", + "sample_output = rag_app.answer(sample[\"query\"], system_prompt=result.prompt.system) # result.prompt is a ChatPrompt; .system is the optimized system text\n", + "judged = AnswerRelevance(model=config.JUDGE_MODEL).score(\n", + " input=sample[\"query\"],\n", + " output=sample_output,\n", + " context=[sample[\"reference\"]],\n", + ")\n", + "print(\"Question:\", sample[\"query\"])\n", + "print(\"Answer: \", sample_output)\n", + "print(\"Score: \", judged.value)\n", + "print(\"Reason: \", judged.reason)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44e455db", + "metadata": {}, + "outputs": [], + "source": [ + "judge_cases = data.load_judge_cases()\n", + "judge_dataset = data.build_dataset(client, \"ledgerline-judge\", judge_cases)\n", + "\n", + "judge_prompt = ChatPrompt(\n", + " name=\"ledgerline-answer-judge\",\n", + " system=BASELINE_SYSTEM,\n", + " user=\"{query}\",\n", + " model=config.GEN_MODEL,\n", + ")\n", + "\n", + "judge_result = optimizer.optimize_prompt(\n", + " prompt=judge_prompt,\n", + " dataset=judge_dataset,\n", + " metric=answer_relevance,\n", + " max_trials=8,\n", + " n_samples=8,\n", + ")\n", + "print(\"Judge-metric baseline:\", judge_result.initial_score, \"-> best:\", judge_result.score)" + ] + }, + { + "cell_type": "markdown", + "id": "e6b6e231", + "metadata": {}, + "source": [ + "#### Multi-objective: quality *and* cost\n", + "\n", + "Optimizing purely for a judge can inflate answer length. Often you want **quality high *and* answers short**. `MultiMetricObjective` combines metrics into one weighted composite the optimizer maximizes — this is how Opik does multi-objective optimization.\n", + "\n", + "Below we combine `answer_relevance` (weight 0.7) with a length penalty (weight 0.3). The length metric is a plain callable that returns a normalized \"shorter is better\" score." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a14ea1f7", + "metadata": {}, + "outputs": [], + "source": [ + "from opik_optimizer import MultiMetricObjective\n", + "\n", + "\n", + "def brevity(dataset_item: dict, llm_output: str) -> float:\n", + " # Normalized \"shorter is better\": 1.0 for <=200 chars, decaying to 0 at 1000 chars.\n", + " length = len(llm_output)\n", + " return max(0.0, min(1.0, (1000 - length) / 800))\n", + "\n", + "\n", + "brevity.__name__ = \"brevity\"\n", + "\n", + "composite = MultiMetricObjective(\n", + " metrics=[answer_relevance, brevity],\n", + " weights=[0.7, 0.3],\n", + " name=\"relevance_and_brevity\",\n", + ")\n", + "\n", + "multi_result = optimizer.optimize_prompt(\n", + " prompt=judge_prompt,\n", + " dataset=judge_dataset,\n", + " metric=composite,\n", + " max_trials=8,\n", + " n_samples=8,\n", + ")\n", + "print(\"Multi-objective best score:\", multi_result.score)\n", + "print(\"\\nOptimized prompt:\\n\", multi_result.prompt)" + ] + }, + { + "cell_type": "markdown", + "id": "1da5064e", + "metadata": {}, + "source": [ + "#### Compare your runs\n", + "\n", + "You now have three optimization runs in Opik: exact-match, judge, and multi-objective. In **Evaluation → Optimization runs**, put them side by side. Notice how the multi-objective prompt trades a little relevance for much shorter answers — that trade-off is the whole point of naming your objectives explicitly." + ] } ], "metadata": {}, From 08c10471533b998749016137f830ade2814bf676 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 15:04:34 +0200 Subject: [PATCH 09/13] =?UTF-8?q?feat:=20notebook=20Parts=203-5=20?= =?UTF-8?q?=E2=80=94=20agent=20optimization,=20optimizer=20selection,=20pr?= =?UTF-8?q?ompt=20library?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../prompt_agent_optimization.ipynb | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb index 48c25c8..0e857ff 100644 --- a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb +++ b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb @@ -364,6 +364,150 @@ "\n", "You now have three optimization runs in Opik: exact-match, judge, and multi-objective. In **Evaluation → Optimization runs**, put them side by side. Notice how the multi-objective prompt trades a little relevance for much shorter answers — that trade-off is the whole point of naming your objectives explicitly." ] + }, + { + "cell_type": "markdown", + "id": "b00aec82", + "metadata": {}, + "source": [ + "## Part 3 — From prompt to agent\n", + "\n", + "So far we optimized a single answer prompt. Real systems are agents: they *decide* what to do. Our RAG app can grow a **retrieval gate** — decide whether a question even needs a docs lookup (cheap questions skip retrieval). That decision is itself a prompt, and the same optimizer loop tunes it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4cf1853a", + "metadata": {}, + "outputs": [], + "source": [ + "@opik.track\n", + "def agentic_answer(query: str, system_prompt: str) -> str:\n", + " # Agent step: decide whether to retrieve, then answer accordingly.\n", + " if rag_app.should_retrieve(query):\n", + " return rag_app.answer(query, system_prompt=system_prompt)\n", + " messages = [\n", + " {\"role\": \"system\", \"content\": system_prompt},\n", + " {\"role\": \"user\", \"content\": query},\n", + " ]\n", + " import litellm\n", + " resp = litellm.completion(model=config.GEN_MODEL, messages=messages)\n", + " return resp.choices[0].message.content" + ] + }, + { + "cell_type": "markdown", + "id": "373db4fb", + "metadata": {}, + "source": [ + "#### When demonstrations matter: Few-Shot Bayesian\n", + "\n", + "If the win comes from *showing examples* rather than rewording instructions, reach for `FewShotBayesianOptimizer` — it uses Bayesian search (Optuna) to pick the best set and order of few-shot demonstrations to attach to your prompt. Same API as before." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5c7a51e4", + "metadata": {}, + "outputs": [], + "source": [ + "from opik_optimizer import FewShotBayesianOptimizer\n", + "\n", + "fewshot_optimizer = FewShotBayesianOptimizer(model=config.OPTIMIZER_MODEL, n_threads=4)\n", + "\n", + "fewshot_result = fewshot_optimizer.optimize_prompt(\n", + " prompt=judge_prompt,\n", + " dataset=judge_dataset,\n", + " metric=answer_relevance,\n", + " n_samples=8,\n", + ")\n", + "print(\"Few-shot best score:\", fewshot_result.score)" + ] + }, + { + "cell_type": "markdown", + "id": "d899aa41", + "metadata": {}, + "source": [ + "#### Tuning the model, not the prompt: Parameter optimizer\n", + "\n", + "Sometimes the prompt is fine and you just need better sampling settings. `ParameterOptimizer` leaves the prompt alone and searches temperature / top_p with Bayesian optimization. It's the right reach when behavior — not wording — is the problem. See the [Parameter optimizer docs](https://www.comet.com/docs/opik/agent_optimization/algorithms/parameter_optimizer) for the search-space API." + ] + }, + { + "cell_type": "markdown", + "id": "5668c2f1", + "metadata": {}, + "source": [ + "## Part 4 — Choosing an optimizer\n", + "\n", + "You've now *used* several optimizers at the moment each was the right tool. Here's the consolidated map:\n", + "\n", + "| Optimizer | Best for | You saw it in |\n", + "|---|---|---|\n", + "| **MetaPrompt** | General prompt rewording & clarity | Part 1 |\n", + "| **HRPO** | Systematic fixes from *why* prompts fail (failure-mode analysis) | (try on your own) |\n", + "| **Few-Shot Bayesian** | Picking the best demonstrations | Part 3 |\n", + "| **Evolutionary** | Exploring diverse structures; multi-objective | (see multi-objective, Part 2) |\n", + "| **GEPA** | Single-turn, reflection-heavy tasks (`pip install gepa`) | (try on your own) |\n", + "| **Parameter** | Temperature / top_p, prompt unchanged | Part 3 |\n", + "\n", + "**How to choose, in four questions:**\n", + "1. **What's the constraint** — wording, examples, tool use, or sampling params?\n", + "2. **Is the dataset ready** — reflective optimizers (HRPO) need metrics with detailed *reasons*. Split train/validation to avoid overfitting.\n", + "3. **What's the budget** — Evolutionary/GEPA burn more tokens than MetaPrompt.\n", + "4. **Can you chain?** — e.g. MetaPrompt to fix wording, then Parameter to tune sampling.\n", + "\n", + "The docs' own advice: **start with GEPA or HRPO** for a new task, then specialize." + ] + }, + { + "cell_type": "markdown", + "id": "0d0e2885", + "metadata": {}, + "source": [ + "#### Chaining optimizers\n", + "\n", + "Because every optimizer shares the same API and returns an `OptimizationResult` whose `.prompt` you can feed into the next, you can chain them: optimize wording, then feed the winner into a Parameter run. See [Chaining optimizers](https://www.comet.com/docs/opik/agent_optimization/advanced/chaining_optimizers)." + ] + }, + { + "cell_type": "markdown", + "id": "9f5990ba", + "metadata": {}, + "source": [ + "## Part 5 — Take it further\n", + "\n", + "**Version the winner.** Promote your best prompt to the Opik **Prompt Library** so it's versioned and reusable." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c9be51dd", + "metadata": {}, + "outputs": [], + "source": [ + "# multi_result.prompt is a ChatPrompt; .system is the optimized system text\n", + "best_prompt = opik.Prompt(name=\"ledgerline-answer\", prompt=multi_result.prompt.system)\n", + "print(\"Saved prompt version:\", best_prompt.commit)" + ] + }, + { + "cell_type": "markdown", + "id": "d6d2358a", + "metadata": {}, + "source": [ + "**Where to go next:**\n", + "- **[Optimization Studio](https://www.comet.com/docs/opik/agent_optimization/optimization_studio)** — run all of this from the Opik UI, no code.\n", + "- **[Optimizer benchmarks](https://www.comet.com/docs/opik/agent_optimization/algorithms/benchmarks)** — numbers per algorithm.\n", + "- **[Agent optimization overview](https://www.comet.com/docs/opik/agent_optimization/overview)** — the full reference.\n", + "- **Wrap this in a CLI** — the plumbing (`optimization_guide/`) is import-ready; turning the notebook into a repeatable CLI is a natural next project (out of scope here).\n", + "\n", + "You've gone A-to-Z: framing → first optimization → trustworthy judge metrics → multi-objective → agent tuning → optimizer selection → versioned prompt. Every step is a comparable run in Opik." + ] } ], "metadata": {}, From ecd3062ec5965a624ce491f24ec414346f7c8b7e Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 15:08:33 +0200 Subject: [PATCH 10/13] docs: README for prompt_agent_optimization guide + index row --- guides/README.md | 1 + guides/prompt_agent_optimization/README.md | 57 +++++++++++++++++++++- 2 files changed, 56 insertions(+), 2 deletions(-) diff --git a/guides/README.md b/guides/README.md index 12605ba..ff57e4a 100644 --- a/guides/README.md +++ b/guides/README.md @@ -6,6 +6,7 @@ Task-oriented examples for doing something specific with Opik — combining Opik |---|---| | [annotation_queues_with_context/](./annotation_queues_with_context/) | Structure RAG traces for Opik annotation queues — clean answer in output, context in metadata, full detail in child spans | | [multimodal_online_evaluation/](./multimodal_online_evaluation/) | Run an online LLM-as-judge eval over multimodal (text + image) traces — create the rule in the UI and with the SDK | +| [prompt_agent_optimization/](./prompt_agent_optimization/) | A-to-Z guide to prompt & agent optimization with Opik — one escalating RAG example, from an exact-match metric to LLM-judge, multi-objective, and agent/tool optimization; doubles as a live workshop (Part 1) | | [tracing_finetuned_models/](./tracing_finetuned_models/) | Fine-tune a model, register it to the CometML Model Registry, then fetch and trace inference in Opik | [Contribute one](../CONTRIBUTING.md). diff --git a/guides/prompt_agent_optimization/README.md b/guides/prompt_agent_optimization/README.md index 35b0113..c9887e5 100644 --- a/guides/prompt_agent_optimization/README.md +++ b/guides/prompt_agent_optimization/README.md @@ -1,3 +1,56 @@ -# Prompt & Agent Optimization with Opik +# Prompt & Agent Optimization with Opik — an A-to-Z guide -Placeholder — see `prompt_agent_optimization.ipynb`. Full README written in Task 10. +A single notebook that teaches prompt and agent optimization end-to-end, over one +escalating RAG-over-docs example (a documentation assistant for a fictional +product, **Ledgerline**). It doubles as: + +- a **live workshop** — run **Part 1** (~20 min) to optimize a prompt against an + exact-match metric and see it in Opik; and +- a **take-home guide** — Parts 2–5 cover LLM-judge metrics (and how to *trust* + them), multi-objective optimization, agent/tool optimization, and choosing an + optimizer. + +Every optimization logs to Opik under **Evaluation → Optimization runs**, so each +step is a comparable run. + +## What it covers + +- **Part 0** — how to think about prompt optimization (prompt + dataset + metric). +- **Part 1** ⭐ — your first optimization: exact-match metric + `MetaPromptOptimizer`. +- **Part 2** — LLM-judge metrics, *how to trust a judge*, and multi-objective + optimization with `MultiMetricObjective`. +- **Part 3** — from prompt to agent: a retrieval gate, `FewShotBayesianOptimizer`, + and `ParameterOptimizer`. +- **Part 4** — choosing an optimizer (selection table + how to choose + chaining). +- **Part 5** — promote the winner to the Prompt Library; pointers to Optimization + Studio and the docs. + +## Prerequisites + +```bash +uv sync +``` + +| Environment variable | Required | Description | +|---|---|---| +| `OPIK_API_KEY` | yes | Your Opik API key. | +| `OPIK_WORKSPACE` | yes | Your Opik workspace name. | +| `ANTHROPIC_API_KEY` (or the key for your `OPIK_EXAMPLES_MODEL` provider) | yes | Model-provider key used via litellm for generation, judging, and optimizing. | +| `OPIK_PROJECT_NAME` | no | Opik project for traces/runs (default `prompt-agent-optimization`). | +| `OPIK_EXAMPLES_MODEL` | no | litellm model (default `anthropic/claude-sonnet-4-6`). Use a cheap model to run fast. | +| `OPIK_URL_OVERRIDE` | no | Base URL for self-hosted Opik. | + +There is **no dry-run** — optimization requires running real evaluations. The +notebook's first cell fails fast if a required variable is missing. + +## Running it + +Open `prompt_agent_optimization.ipynb` in Jupyter and run cells top to bottom. +For the workshop, stop at the end of Part 1. You can launch JupyterLab directly +with `uv run jupyter lab` (it's included as a project dependency). + +## How the code is organized + +Optimization code (`ChatPrompt`, metrics, optimizer calls) lives **inline in the +notebook** — it's the lesson. Repeated plumbing (retriever, data loading) lives in +`optimization_guide/` so it stays out of the way and could back a future CLI. From ff2f88564b737abdf4e0b4acb3a6a848552d3642 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 24 Jul 2026 15:56:07 +0200 Subject: [PATCH 11/13] fix: wire real retrieval into optimized prompts + make Part 3 a genuine tool-calling agent Final review found the optimizer never saw the (fictional) docs, so the exact-match RAG lesson could not work, and Part 3's agent was dead code. Part 1-2: attach retrieved context (rag_app.retrieve) to each eval row and inject it via the ChatPrompt user template (retrieve-then-generate); retrieval is fixed, the prompt is optimized. Print baseline vs optimized .system so the refinement is visible. Verified offline: 18/18 exact answers appear in the retrieved top-3 context. Part 3: replace the never-optimized agentic_answer gate with a genuine tool-calling agent (search_docs -> rag_app.retrieve, allow_tool_use) whose system prompt is actually optimized; note optimize_tools=True as the one-line tool-description add; FewShot tunes the same agent. Tool binding verified against the optimizer's own resolve_toolcalling_tools. README Part 3 + the Part 4 selection table updated to match. --- guides/prompt_agent_optimization/README.md | 5 +- .../prompt_agent_optimization.ipynb | 105 ++++++++++++++---- 2 files changed, 85 insertions(+), 25 deletions(-) diff --git a/guides/prompt_agent_optimization/README.md b/guides/prompt_agent_optimization/README.md index c9887e5..e95c362 100644 --- a/guides/prompt_agent_optimization/README.md +++ b/guides/prompt_agent_optimization/README.md @@ -19,8 +19,9 @@ step is a comparable run. - **Part 1** ⭐ — your first optimization: exact-match metric + `MetaPromptOptimizer`. - **Part 2** — LLM-judge metrics, *how to trust a judge*, and multi-objective optimization with `MultiMetricObjective`. -- **Part 3** — from prompt to agent: a retrieval gate, `FewShotBayesianOptimizer`, - and `ParameterOptimizer`. +- **Part 3** — from prompt to agent: a tool-calling `search_docs` agent optimized + end-to-end, then `FewShotBayesianOptimizer` on the same agent (with a pointer to + `ParameterOptimizer`). - **Part 4** — choosing an optimizer (selection table + how to choose + chaining). - **Part 5** — promote the winner to the Prompt Library; pointers to Optimization Studio and the docs. diff --git a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb index 0e857ff..ea5d558 100644 --- a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb +++ b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb @@ -85,8 +85,17 @@ "outputs": [], "source": [ "exact_cases = data.load_exact_cases()\n", + "\n", + "# RAG step — retrieve. For each question, pull the most relevant docs with our\n", + "# real ChromaDB retriever and attach them as `context`. A production RAG system\n", + "# retrieves per query at answer time; we do it once here so every optimizer trial\n", + "# answers the SAME question from the SAME context. What we optimize is the\n", + "# *prompt*, not the retriever.\n", + "for case in exact_cases:\n", + " case[\"context\"] = \"\\n\\n\".join(rag_app.retrieve(case[\"query\"]))\n", + "\n", "exact_dataset = data.build_dataset(client, \"ledgerline-exact\", exact_cases)\n", - "print(f\"Dataset 'ledgerline-exact' has {len(exact_cases)} cases\")" + "print(f\"Dataset 'ledgerline-exact' has {len(exact_cases)} cases (each with retrieved context)\")" ] }, { @@ -128,7 +137,9 @@ "source": [ "#### The starting prompt\n", "\n", - "Here is our baseline system prompt — deliberately mediocre, so there's room to improve. This is the `ChatPrompt` the optimizer will rewrite. `{query}` is filled from each dataset row." + "Here is our baseline system prompt — deliberately mediocre, so there's room to improve. This is the `ChatPrompt` the optimizer will rewrite.\n", + "\n", + "Look at the **user template**: `{context}` is filled with the docs we just retrieved and `{query}` with the question — both come from each dataset row. That's the **retrieve-then-generate** shape of a real RAG system. Retrieval is held fixed; what we optimize is how the **system prompt** tells the model to turn that context into a correct, concise answer." ] }, { @@ -145,7 +156,7 @@ "prompt = ChatPrompt(\n", " name=\"ledgerline-answer\",\n", " system=BASELINE_SYSTEM,\n", - " user=\"{query}\",\n", + " user=\"Context:\\n{context}\\n\\nQuestion: {query}\",\n", " model=config.GEN_MODEL,\n", ")" ] @@ -188,7 +199,13 @@ "\n", "print(\"Baseline score:\", result.initial_score)\n", "print(\"Best score: \", result.score)\n", - "print(\"\\nOptimized system prompt:\\n\", result.prompt)" + "\n", + "# See HOW the prompt was refined: the optimizer rewrote the *system* instructions.\n", + "# result.prompt is a ChatPrompt; .system is the optimized system text.\n", + "print(\"\\n--- Baseline system prompt ---\")\n", + "print(BASELINE_SYSTEM)\n", + "print(\"\\n--- Optimized system prompt ---\")\n", + "print(result.prompt.system)" ] }, { @@ -198,7 +215,7 @@ "source": [ "#### See it in Opik\n", "\n", - "Open **Evaluation → Optimization runs** in your Opik workspace. You'll see this run with every candidate prompt, its score, and the trace for each trial. Compare the baseline row to the best row — that delta is your improvement.\n", + "The cell above printed the **baseline vs optimized system prompt** side by side — that rewrite is the concrete refinement the optimizer found. Now open **Evaluation → Optimization runs** in your Opik workspace: you'll see this run with every candidate prompt, its score, and the trace for each trial. Compare the baseline row to the best row — that delta is your improvement.\n", "\n", "🎓 **This is where the live workshop ends.** You've run a real optimization and improved a prompt, measured against a dataset, stored in Opik. Everything below builds on exactly this loop." ] @@ -289,12 +306,14 @@ "outputs": [], "source": [ "judge_cases = data.load_judge_cases()\n", + "for case in judge_cases:\n", + " case[\"context\"] = \"\\n\\n\".join(rag_app.retrieve(case[\"query\"]))\n", "judge_dataset = data.build_dataset(client, \"ledgerline-judge\", judge_cases)\n", "\n", "judge_prompt = ChatPrompt(\n", " name=\"ledgerline-answer-judge\",\n", " system=BASELINE_SYSTEM,\n", - " user=\"{query}\",\n", + " user=\"Context:\\n{context}\\n\\nQuestion: {query}\",\n", " model=config.GEN_MODEL,\n", ")\n", "\n", @@ -352,7 +371,7 @@ " n_samples=8,\n", ")\n", "print(\"Multi-objective best score:\", multi_result.score)\n", - "print(\"\\nOptimized prompt:\\n\", multi_result.prompt)" + "print(\"\\nOptimized system prompt:\\n\", multi_result.prompt.system)" ] }, { @@ -372,7 +391,11 @@ "source": [ "## Part 3 — From prompt to agent\n", "\n", - "So far we optimized a single answer prompt. Real systems are agents: they *decide* what to do. Our RAG app can grow a **retrieval gate** — decide whether a question even needs a docs lookup (cheap questions skip retrieval). That decision is itself a prompt, and the same optimizer loop tunes it." + "So far retrieval was **fixed**: we retrieved once, put the docs in the prompt, and optimized the wording. Real systems are agents — they *decide* what to do. Here we hand the model a **`search_docs` tool** wired to our retriever and let it choose when to call it. That turns the prompt into an **agent**, and the same optimizer loop tunes it.\n", + "\n", + "**What \"optimizing an agent\" means:** not rewriting the tool's code — the retriever is fixed. It means optimizing the natural-language surface the agent reasons over: its **system prompt** (when to search, how to answer from results) and, optionally, its **tool descriptions** (`optimize_tools=True`) so it calls the tool at the right moments.\n", + "\n", + "*(A lighter alternative to a tool is a yes/no retrieval gate — see `rag_app.should_retrieve` — optimized with the same loop. We use a real tool here because it better reflects a production agent.)*" ] }, { @@ -382,18 +405,54 @@ "metadata": {}, "outputs": [], "source": [ - "@opik.track\n", - "def agentic_answer(query: str, system_prompt: str) -> str:\n", - " # Agent step: decide whether to retrieve, then answer accordingly.\n", - " if rag_app.should_retrieve(query):\n", - " return rag_app.answer(query, system_prompt=system_prompt)\n", - " messages = [\n", - " {\"role\": \"system\", \"content\": system_prompt},\n", - " {\"role\": \"user\", \"content\": query},\n", - " ]\n", - " import litellm\n", - " resp = litellm.completion(model=config.GEN_MODEL, messages=messages)\n", - " return resp.choices[0].message.content" + "# The tool the agent may call. It wraps our real ChromaDB retriever; the *agent*\n", + "# decides when to call it. Returning one string keeps the tool result clean.\n", + "def search_docs(query: str) -> str:\n", + " \"\"\"Search the Ledgerline documentation and return the most relevant snippets.\"\"\"\n", + " return \"\\n\\n\".join(rag_app.retrieve(query))\n", + "\n", + "\n", + "SEARCH_DOCS_TOOL = {\n", + " \"type\": \"function\",\n", + " \"function\": {\n", + " \"name\": \"search_docs\",\n", + " \"description\": \"Search the Ledgerline product documentation for relevant snippets.\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"query\": {\"type\": \"string\", \"description\": \"What to look up in the docs.\"},\n", + " },\n", + " \"required\": [\"query\"],\n", + " },\n", + " },\n", + "}\n", + "\n", + "AGENT_SYSTEM = \"You are a Ledgerline support agent. Use tools when they help.\"\n", + "\n", + "# tools + function_map make this ChatPrompt an agent: on a tool call, the\n", + "# optimizer executes search_docs and feeds the result back to the model.\n", + "agent_prompt = ChatPrompt(\n", + " name=\"ledgerline-agent\",\n", + " system=AGENT_SYSTEM,\n", + " user=\"{query}\",\n", + " tools=[SEARCH_DOCS_TOOL],\n", + " function_map={\"search_docs\": search_docs},\n", + " model=config.GEN_MODEL,\n", + ")\n", + "\n", + "# optimize_prompts defaults to \"system\": we tune the agent's instructions.\n", + "# (Flip optimize_tools=True to ALSO let the optimizer refine the tool description.)\n", + "agent_result = optimizer.optimize_prompt(\n", + " prompt=agent_prompt,\n", + " dataset=judge_dataset,\n", + " metric=answer_relevance,\n", + " max_trials=8,\n", + " n_samples=8,\n", + " allow_tool_use=True,\n", + ")\n", + "print(\"Agent baseline:\", agent_result.initial_score, \"-> best:\", agent_result.score)\n", + "print(\"\\n--- Optimized agent system prompt ---\")\n", + "print(agent_result.prompt.system)" ] }, { @@ -403,7 +462,7 @@ "source": [ "#### When demonstrations matter: Few-Shot Bayesian\n", "\n", - "If the win comes from *showing examples* rather than rewording instructions, reach for `FewShotBayesianOptimizer` — it uses Bayesian search (Optuna) to pick the best set and order of few-shot demonstrations to attach to your prompt. Same API as before." + "If the win comes from *showing examples* rather than rewording instructions, reach for `FewShotBayesianOptimizer` — it uses Bayesian search (Optuna) to pick the best set and order of few-shot demonstrations to attach. We point it at the **same agent**, so it tunes the agent's examples rather than a fresh prompt." ] }, { @@ -418,7 +477,7 @@ "fewshot_optimizer = FewShotBayesianOptimizer(model=config.OPTIMIZER_MODEL, n_threads=4)\n", "\n", "fewshot_result = fewshot_optimizer.optimize_prompt(\n", - " prompt=judge_prompt,\n", + " prompt=agent_prompt,\n", " dataset=judge_dataset,\n", " metric=answer_relevance,\n", " n_samples=8,\n", @@ -452,7 +511,7 @@ "| **Few-Shot Bayesian** | Picking the best demonstrations | Part 3 |\n", "| **Evolutionary** | Exploring diverse structures; multi-objective | (see multi-objective, Part 2) |\n", "| **GEPA** | Single-turn, reflection-heavy tasks (`pip install gepa`) | (try on your own) |\n", - "| **Parameter** | Temperature / top_p, prompt unchanged | Part 3 |\n", + "| **Parameter** | Temperature / top_p, prompt unchanged | Part 3 (described) |\n", "\n", "**How to choose, in four questions:**\n", "1. **What's the constraint** — wording, examples, tool use, or sampling params?\n", From 3553f760f05bab961cb6f2b2f1a1a794e230ab86 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Wed, 29 Jul 2026 10:26:29 +0200 Subject: [PATCH 12/13] docs: add example dataset-item and optimized-prompt cells to Part 1 (reviewer feedback) Colleagues asked to see, not just be told: what a dataset item looks like and what an optimized prompt looks like. Adds to Part 1 (1) a cell that prints a real dataset item plus a markdown showing its shape (query / expected_substring / retrieved context), and (2) an illustrative baseline->optimized system-prompt before/after (the live run prints the real one). No screenshots by request; the 'See it in Opik' cell already describes the UI in prose. --- .../prompt_agent_optimization.ipynb | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb index ea5d558..4b8bf99 100644 --- a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb +++ b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb @@ -98,6 +98,38 @@ "print(f\"Dataset 'ledgerline-exact' has {len(exact_cases)} cases (each with retrieved context)\")" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3ad2617", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "# Peek at one dataset item. The optimizer fills {query}/{context} in the prompt\n", + "# from these fields; the exact-match metric checks that expected_substring appears.\n", + "print(json.dumps(exact_cases[0], indent=2))" + ] + }, + { + "cell_type": "markdown", + "id": "a269b356", + "metadata": {}, + "source": [ + "A dataset item looks like this — the question, the fact we check for, and the docs our retriever pulled for it (context abridged; the cell above prints it in full):\n", + "\n", + "```json\n", + "{\n", + " \"query\": \"What is the default job timeout?\",\n", + " \"expected_substring\": \"30 seconds\",\n", + " \"context\": \"Every Ledgerline job has a default timeout of 30 seconds. ... The maximum configurable timeout is 15 minutes.\\n\\nFailed jobs are retried automatically. The default maximum number of retries is 3 ...\\n\\nAfter a job exhausts all retries it is moved to the dead-letter queue, ... retained for 7 days ...\"\n", + "}\n", + "```\n", + "\n", + "`{query}` and `{context}` are filled into the prompt from these fields; the metric checks that `expected_substring` shows up in the answer." + ] + }, { "cell_type": "markdown", "id": "f654fca1", @@ -208,6 +240,22 @@ "print(result.prompt.system)" ] }, + { + "cell_type": "markdown", + "id": "4795bd62", + "metadata": {}, + "source": [ + "#### What does an optimized prompt look like?\n", + "\n", + "The cell above prints *your* run's actual result. To set expectations, here's the kind of rewrite `MetaPromptOptimizer` typically produces — turning the terse baseline into explicit, context-grounded instructions (**illustrative; your exact wording will differ**):\n", + "\n", + "> **Baseline:** `You are a support bot. Answer the question.`\n", + ">\n", + "> **Optimized (illustrative):** `You are a Ledgerline support assistant. Answer using only the provided context. Quote exact values — durations, limits, header names — verbatim, and keep the answer to one or two sentences. If the context doesn't contain the answer, say so.`\n", + "\n", + "Notice what optimization *found*: lean on the context, quote exact values (exactly what the exact-match metric rewards), and stay concise." + ] + }, { "cell_type": "markdown", "id": "8aa5a588", From 2548f4aee16b878da254ecd320b341ec79686889 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Thu, 20 Aug 2026 13:46:47 +0200 Subject: [PATCH 13/13] refactor: make the notebook fully self-contained (Colab-ready) + fix optimized-prompt extraction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guide now runs top-to-bottom in a fresh kernel (Colab or local): a %pip install cell, in-notebook credentials via the repo-standard opik.configure(project_name=..., install_mcp=False), and the corpus + tiny RAG app (ChromaDB retriever + answer function) inlined. Removes the optimization_guide/ package, its unit tests, and data/*.json — everything the lesson needs is in the notebook, so it can be shared as a single file. Also fixes the crash CI's live notebook run surfaced: an optimizer-returned ChatPrompt stores its text in messages (get_messages()), not the .system scalar (which is None), so reading result.prompt.system passed None as a system message and litellm's Anthropic transform raised KeyError: 'content'. A new optimized_system(result) helper reads the system message from get_messages(). pyproject slimmed (drop ruff/pytest; keep deps + Jupyter tooling); README rewritten for Colab-or-local + opik.configure credentials. --- guides/prompt_agent_optimization/README.md | 56 +-- .../prompt_agent_optimization/data/docs.json | 14 - .../data/eval_cases_exact.json | 20 - .../data/eval_cases_judge.json | 14 - .../optimization_guide/__init__.py | 0 .../optimization_guide/config.py | 42 --- .../optimization_guide/data.py | 30 -- .../optimization_guide/rag_app.py | 68 ---- .../prompt_agent_optimization.ipynb | 352 ++++++++++++++---- .../prompt_agent_optimization/pyproject.toml | 19 +- .../tests/conftest.py | 11 - .../tests/test_config.py | 49 --- .../tests/test_data.py | 49 --- .../tests/test_rag_app.py | 67 ---- 14 files changed, 307 insertions(+), 484 deletions(-) delete mode 100644 guides/prompt_agent_optimization/data/docs.json delete mode 100644 guides/prompt_agent_optimization/data/eval_cases_exact.json delete mode 100644 guides/prompt_agent_optimization/data/eval_cases_judge.json delete mode 100644 guides/prompt_agent_optimization/optimization_guide/__init__.py delete mode 100644 guides/prompt_agent_optimization/optimization_guide/config.py delete mode 100644 guides/prompt_agent_optimization/optimization_guide/data.py delete mode 100644 guides/prompt_agent_optimization/optimization_guide/rag_app.py delete mode 100644 guides/prompt_agent_optimization/tests/conftest.py delete mode 100644 guides/prompt_agent_optimization/tests/test_config.py delete mode 100644 guides/prompt_agent_optimization/tests/test_data.py delete mode 100644 guides/prompt_agent_optimization/tests/test_rag_app.py diff --git a/guides/prompt_agent_optimization/README.md b/guides/prompt_agent_optimization/README.md index e95c362..155b39c 100644 --- a/guides/prompt_agent_optimization/README.md +++ b/guides/prompt_agent_optimization/README.md @@ -1,8 +1,8 @@ # Prompt & Agent Optimization with Opik — an A-to-Z guide -A single notebook that teaches prompt and agent optimization end-to-end, over one -escalating RAG-over-docs example (a documentation assistant for a fictional -product, **Ledgerline**). It doubles as: +A single, **self-contained** notebook that teaches prompt and agent optimization +end-to-end, over one escalating RAG-over-docs example (a documentation assistant +for a fictional product, **Ledgerline**). It doubles as: - a **live workshop** — run **Part 1** (~20 min) to optimize a prompt against an exact-match metric and see it in Opik; and @@ -26,32 +26,40 @@ step is a comparable run. - **Part 5** — promote the winner to the Prompt Library; pointers to Optimization Studio and the docs. -## Prerequisites +## Running it -```bash -uv sync -``` +The notebook is self-contained — it installs its dependencies and configures its +credentials in the first few cells, and defines its corpus + RAG app inline. Run +the cells top to bottom; for the workshop, stop at the end of Part 1. -| Environment variable | Required | Description | -|---|---|---| -| `OPIK_API_KEY` | yes | Your Opik API key. | -| `OPIK_WORKSPACE` | yes | Your Opik workspace name. | -| `ANTHROPIC_API_KEY` (or the key for your `OPIK_EXAMPLES_MODEL` provider) | yes | Model-provider key used via litellm for generation, judging, and optimizing. | -| `OPIK_PROJECT_NAME` | no | Opik project for traces/runs (default `prompt-agent-optimization`). | -| `OPIK_EXAMPLES_MODEL` | no | litellm model (default `anthropic/claude-sonnet-4-6`). Use a cheap model to run fast. | -| `OPIK_URL_OVERRIDE` | no | Base URL for self-hosted Opik. | +- **Google Colab** — upload/open the notebook and run it; the first cell + `%pip install`s everything. +- **Locally** — `uv sync` then `uv run jupyter lab` (or open the notebook in your + editor's Jupyter). `uv` and the `pyproject.toml` are here for convenience; the + notebook's own `%pip install` cell means it also runs in a bare environment. -There is **no dry-run** — optimization requires running real evaluations. The -notebook's first cell fails fast if a required variable is missing. +## Credentials -## Running it +The **Credentials** cell walks you through setup — no external environment dance +required: + +- **Opik** — it calls `opik.configure()`, which prompts for your API key and + workspace (get them free at [comet.com/opik](https://www.comet.com/opik)). +- **A model provider key** — the guide calls models through litellm. It defaults + to a small Anthropic Claude model and prompts for your `ANTHROPIC_API_KEY`. To + use another provider, set `OPIK_EXAMPLES_MODEL` (e.g. `openai/gpt-4o-mini`) and + you'll be prompted for that provider's key instead. -Open `prompt_agent_optimization.ipynb` in Jupyter and run cells top to bottom. -For the workshop, stop at the end of Part 1. You can launch JupyterLab directly -with `uv run jupyter lab` (it's included as a project dependency). +If the relevant variables are already set in your environment (`OPIK_API_KEY`, +`OPIK_WORKSPACE`, `OPIK_EXAMPLES_MODEL`, the provider key, and optional +`OPIK_PROJECT_NAME`), the cell skips the prompts — which is how it runs +non-interactively in CI. There is **no dry-run**: optimization runs real +evaluations against your Opik workspace. ## How the code is organized -Optimization code (`ChatPrompt`, metrics, optimizer calls) lives **inline in the -notebook** — it's the lesson. Repeated plumbing (retriever, data loading) lives in -`optimization_guide/` so it stays out of the way and could back a future CLI. +Everything lives **in the notebook** — the corpus, the tiny RAG app (a ChromaDB +retriever + an `answer()` function), the metrics, and every optimizer call. That's +deliberate: you can read it top to bottom, run it anywhere, and share it as a +single file with no external dependencies. Lifting the inline retriever/answer +helpers into a module to back a repeatable CLI is a natural next step. diff --git a/guides/prompt_agent_optimization/data/docs.json b/guides/prompt_agent_optimization/data/docs.json deleted file mode 100644 index 4718ce6..0000000 --- a/guides/prompt_agent_optimization/data/docs.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - {"id": "timeouts", "title": "Job timeouts", "text": "Every Ledgerline job has a default timeout of 30 seconds. Jobs exceeding the timeout are marked failed and eligible for retry. The maximum configurable timeout is 15 minutes."}, - {"id": "retries", "title": "Retries", "text": "Failed jobs are retried automatically. The default maximum number of retries is 3, using exponential backoff starting at 2 seconds. Set max_retries to 0 to disable retries."}, - {"id": "rate-limits", "title": "Rate limits", "text": "The API allows 1000 requests per minute per API key. Exceeding the limit returns HTTP 429. Rate limit headers are included on every response."}, - {"id": "auth", "title": "Authentication", "text": "Authenticate by sending your API key in the Authorization header as a Bearer token: 'Authorization: Bearer '. Keys are created in the dashboard."}, - {"id": "priorities", "title": "Queue priorities", "text": "Ledgerline supports three queue priorities: low, default, and high. High-priority jobs are dequeued before default and low. Priority is set per job at enqueue time."}, - {"id": "dead-letter", "title": "Dead-letter queue", "text": "After a job exhausts all retries it is moved to the dead-letter queue, where it is retained for 7 days before permanent deletion. Dead-letter jobs can be replayed from the dashboard."}, - {"id": "webhooks", "title": "Webhooks", "text": "When a job completes, Ledgerline POSTs a webhook to your configured URL. The payload includes job_id, status, and result fields. Webhook deliveries are signed with the X-Ledgerline-Signature header."}, - {"id": "install", "title": "SDK installation", "text": "Install the Python SDK with 'pip install ledgerline'. The SDK requires Python 3.9 or newer. Import it as 'import ledgerline'."}, - {"id": "concurrency", "title": "Concurrency", "text": "Each project runs up to 50 concurrent jobs by default. Contact support to raise the concurrency limit for your plan."}, - {"id": "regions", "title": "Regions", "text": "Ledgerline is available in three regions: us-east, eu-west, and ap-south. The default region is us-east. Set the region when initializing the client."}, - {"id": "batch", "title": "Batch enqueue", "text": "You can enqueue up to 500 jobs in a single batch request. Larger batches must be split. Each job in a batch is billed individually."}, - {"id": "idempotency", "title": "Idempotency", "text": "Pass an Idempotency-Key header to safely retry enqueue requests. Ledgerline deduplicates requests with the same key for 24 hours."} -] diff --git a/guides/prompt_agent_optimization/data/eval_cases_exact.json b/guides/prompt_agent_optimization/data/eval_cases_exact.json deleted file mode 100644 index c423f51..0000000 --- a/guides/prompt_agent_optimization/data/eval_cases_exact.json +++ /dev/null @@ -1,20 +0,0 @@ -[ - {"query": "What is the default job timeout?", "expected_substring": "30 seconds"}, - {"query": "What is the maximum configurable timeout?", "expected_substring": "15 minutes"}, - {"query": "How many times are failed jobs retried by default?", "expected_substring": "3"}, - {"query": "How do I disable retries?", "expected_substring": "max_retries to 0"}, - {"query": "What backoff does retry use, and starting at what delay?", "expected_substring": "2 seconds"}, - {"query": "How many requests per minute per API key are allowed?", "expected_substring": "1000 requests per minute"}, - {"query": "What HTTP status is returned when the rate limit is exceeded?", "expected_substring": "429"}, - {"query": "Which header carries the API key?", "expected_substring": "Authorization"}, - {"query": "What token scheme is used for auth?", "expected_substring": "Bearer"}, - {"query": "What queue priorities are supported?", "expected_substring": "low, default, and high"}, - {"query": "How long are dead-letter jobs retained?", "expected_substring": "7 days"}, - {"query": "Which header signs webhook deliveries?", "expected_substring": "X-Ledgerline-Signature"}, - {"query": "How do I install the Python SDK?", "expected_substring": "pip install ledgerline"}, - {"query": "What Python version does the SDK require?", "expected_substring": "3.9"}, - {"query": "How many concurrent jobs run per project by default?", "expected_substring": "50 concurrent jobs"}, - {"query": "What is the default region?", "expected_substring": "us-east"}, - {"query": "How many jobs can I enqueue in one batch?", "expected_substring": "500 jobs"}, - {"query": "How long are idempotency keys deduplicated?", "expected_substring": "24 hours"} -] diff --git a/guides/prompt_agent_optimization/data/eval_cases_judge.json b/guides/prompt_agent_optimization/data/eval_cases_judge.json deleted file mode 100644 index 4b641e2..0000000 --- a/guides/prompt_agent_optimization/data/eval_cases_judge.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - {"query": "How should I handle a job that keeps failing?", "reference": "Explain retries with exponential backoff, the default of 3 retries, and that exhausted jobs move to the dead-letter queue (retained 7 days, replayable from the dashboard)."}, - {"query": "How do I make sure I don't enqueue the same job twice if my request retries?", "reference": "Use an Idempotency-Key header; Ledgerline deduplicates same-key requests for 24 hours."}, - {"query": "What's the best way to authenticate my requests?", "reference": "Send the API key as a Bearer token in the Authorization header; create keys in the dashboard."}, - {"query": "How do I get notified when a job finishes?", "reference": "Configure a webhook URL; Ledgerline POSTs job_id, status, and result, signed with X-Ledgerline-Signature."}, - {"query": "How can I prioritise urgent work?", "reference": "Set the job priority to high at enqueue time; high-priority jobs are dequeued before default and low."}, - {"query": "How do I run more jobs at the same time?", "reference": "Default concurrency is 50 concurrent jobs per project; contact support to raise the limit."}, - {"query": "How do I choose where my jobs run?", "reference": "Set the region (us-east, eu-west, ap-south) when initializing the client; default is us-east."}, - {"query": "What happens when I hit the rate limit?", "reference": "Requests over 1000/min per key return HTTP 429; rate-limit headers are on every response."}, - {"query": "How do I submit many jobs efficiently?", "reference": "Use batch enqueue, up to 500 jobs per request; split larger batches; each job billed individually."}, - {"query": "How long do I have to recover a permanently failing job?", "reference": "Dead-letter jobs are retained 7 days before permanent deletion and can be replayed from the dashboard."}, - {"query": "Can I make jobs run longer than the default?", "reference": "Yes; the default timeout is 30 seconds and the maximum configurable timeout is 15 minutes."}, - {"query": "How do I start using the SDK in Python?", "reference": "Install with pip install ledgerline (Python 3.9+), then import ledgerline."} -] diff --git a/guides/prompt_agent_optimization/optimization_guide/__init__.py b/guides/prompt_agent_optimization/optimization_guide/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/guides/prompt_agent_optimization/optimization_guide/config.py b/guides/prompt_agent_optimization/optimization_guide/config.py deleted file mode 100644 index 19ee0c5..0000000 --- a/guides/prompt_agent_optimization/optimization_guide/config.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -from pathlib import Path - -DEFAULT_MODEL = "anthropic/claude-sonnet-4-6" - -_MODEL = os.environ.get("OPIK_EXAMPLES_MODEL", DEFAULT_MODEL) -GEN_MODEL = _MODEL -JUDGE_MODEL = _MODEL -OPTIMIZER_MODEL = _MODEL - -PROJECT_NAME = os.environ.get("OPIK_PROJECT_NAME", "prompt-agent-optimization") - -DATA_DIR = Path(__file__).resolve().parent.parent / "data" -CHROMA_DIR = str(Path(__file__).resolve().parent.parent / "chroma_db") -COLLECTION = "product_docs" - -# Provider key env var expected for each litellm provider prefix. -_PROVIDER_KEYS = { - "anthropic/": "ANTHROPIC_API_KEY", - "openai/": "OPENAI_API_KEY", - "gemini/": "GEMINI_API_KEY", -} - - -def check_prerequisites() -> None: - """Raise RuntimeError listing every missing required env var. No DRY_RUN fallback.""" - missing = [] - for var in ("OPIK_API_KEY", "OPIK_WORKSPACE"): - if not os.environ.get(var): - missing.append(var) - provider_key = next( - (key for prefix, key in _PROVIDER_KEYS.items() if GEN_MODEL.startswith(prefix)), - None, - ) - if provider_key and not os.environ.get(provider_key): - missing.append(f"{provider_key} (for model {GEN_MODEL})") - if missing: - raise RuntimeError( - "Missing required environment variables: " - + ", ".join(missing) - + ". Set them before running this guide (see the README)." - ) diff --git a/guides/prompt_agent_optimization/optimization_guide/data.py b/guides/prompt_agent_optimization/optimization_guide/data.py deleted file mode 100644 index 02dcd09..0000000 --- a/guides/prompt_agent_optimization/optimization_guide/data.py +++ /dev/null @@ -1,30 +0,0 @@ -import json -from typing import Any - -from . import config - - -def _load(name: str) -> list[dict]: - return json.loads((config.DATA_DIR / name).read_text()) - - -def load_docs() -> list[dict]: - return _load("docs.json") - - -def load_exact_cases() -> list[dict]: - return _load("eval_cases_exact.json") - - -def load_judge_cases() -> list[dict]: - return _load("eval_cases_judge.json") - - -def build_dataset(client: Any, name: str, cases: list[dict]) -> Any: - """Get-or-create an Opik dataset and insert cases. - - Opik dedups identical items on insert, so re-running is safe (idempotent). - """ - dataset = client.get_or_create_dataset(name) - dataset.insert(cases) - return dataset diff --git a/guides/prompt_agent_optimization/optimization_guide/rag_app.py b/guides/prompt_agent_optimization/optimization_guide/rag_app.py deleted file mode 100644 index dc61848..0000000 --- a/guides/prompt_agent_optimization/optimization_guide/rag_app.py +++ /dev/null @@ -1,68 +0,0 @@ -import threading - -import chromadb -import litellm -import opik - -from . import config - -_collection = None -_collection_lock = threading.Lock() - - -def get_collection(): - # WHY: cache one PersistentClient. Optimizer/evaluate call the task across worker - # threads; concurrent PersistentClient creation races on tenant validation. - global _collection - if _collection is None: - with _collection_lock: - if _collection is None: - client = chromadb.PersistentClient(path=config.CHROMA_DIR) - _collection = client.get_or_create_collection( - name=config.COLLECTION, metadata={"hnsw:space": "cosine"} - ) - return _collection - - -def ingest(docs: list[dict]) -> int: - collection = get_collection() - collection.upsert( - ids=[d["id"] for d in docs], - documents=[d["text"] for d in docs], - metadatas=[{"title": d["title"]} for d in docs], - ) - return collection.count() - - -def retrieve(query: str, n_results: int = 3) -> list[str]: - collection = get_collection() - result = collection.query(query_texts=[query], n_results=n_results) - return result["documents"][0] - - -@opik.track -def answer(query: str, system_prompt: str, model: str | None = None) -> str: - context = "\n\n".join(retrieve(query)) - messages = [ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": f"Context:\n{context}\n\nQuestion: {query}"}, - ] - response = litellm.completion(model=model or config.GEN_MODEL, messages=messages) - return response.choices[0].message.content - - -@opik.track -def should_retrieve(query: str, model: str | None = None) -> bool: - """Part 3 agent gate: decide whether this query needs a docs lookup.""" - messages = [ - { - "role": "system", - "content": ( - "You decide whether a user question about the Ledgerline product needs a " - "documentation lookup. Answer with exactly YES or NO." - ), - }, - {"role": "user", "content": query}, - ] - response = litellm.completion(model=model or config.GEN_MODEL, messages=messages) - return response.choices[0].message.content.strip().upper().startswith("YES") diff --git a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb index 4b8bf99..b34a840 100644 --- a/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb +++ b/guides/prompt_agent_optimization/prompt_agent_optimization.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "169afc4d", + "id": "4da2e303", "metadata": {}, "source": [ "# Prompt & Agent Optimization with Opik — an A-to-Z Guide\n", @@ -13,12 +13,14 @@ "- **Live workshop (≈20 min):** run **Part 1** top to bottom. You'll optimize a RAG answer prompt against an exact-match metric and see the improvement in Opik.\n", "- **Take-home guide:** continue through Parts 2–5 — LLM-judge metrics (and how to *trust* them), multi-objective optimization, and optimizing an agent's tool use.\n", "\n", - "We optimize a **documentation assistant** for a fictional product, **Ledgerline** (a task-queue API), so the corpus is clean and the lesson is about *optimization*, not about parsing messy docs." + "We optimize a **documentation assistant** for a fictional product, **Ledgerline** (a task-queue API), so the corpus is clean and the lesson is about *optimization*, not about parsing messy docs.\n", + "\n", + "**Runs anywhere.** The notebook installs its own dependencies and configures credentials in the first few cells and defines its tiny RAG app inline, so you can run it top-to-bottom in **Google Colab** or locally — nothing else to set up." ] }, { "cell_type": "markdown", - "id": "2b26fbe8", + "id": "41618004", "metadata": {}, "source": [ "## Part 0 — How to think about prompt optimization\n", @@ -38,26 +40,210 @@ "The loop, once, looks like: **propose candidate → evaluate on dataset → keep best → repeat.** Everything below is that loop, escalating in complexity." ] }, + { + "cell_type": "markdown", + "id": "8f9dc65b", + "metadata": {}, + "source": [ + "## Setup — run these first\n", + "\n", + "The next few cells make the notebook self-contained: install dependencies, configure your credentials, and define a tiny RAG app over the Ledgerline docs. Run them once, top to bottom." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a88ae85", + "metadata": {}, + "outputs": [], + "source": [ + "# Install dependencies. In Colab this installs them; locally (after `uv sync`)\n", + "# they're already present. --upgrade keeps you on current SDKs.\n", + "%pip install --quiet --upgrade opik opik-optimizer chromadb litellm" + ] + }, + { + "cell_type": "markdown", + "id": "f2d2833c", + "metadata": {}, + "source": [ + "### Credentials\n", + "\n", + "Run the cell below. `opik.configure()` reads your **Opik API key** and **workspace** from the environment, or prompts for them (get them free at [comet.com/opik](https://www.comet.com/opik)) — the same one-call setup used across these guides. `install_mcp=False` keeps it non-interactive, so the same cell also runs unattended in CI. `project_name=` pins the **project** every trace and optimization run logs to.\n", + "\n", + "You also need a **model provider key**: the guide calls models through litellm and defaults to a small Anthropic Claude model, so it'll use (or prompt for) your `ANTHROPIC_API_KEY`. To use another provider, set `OPIK_EXAMPLES_MODEL` (e.g. `openai/gpt-4o-mini`) and it'll use that provider's key instead." + ] + }, { "cell_type": "code", "execution_count": null, - "id": "33b93828", + "id": "2ddf88c4", "metadata": {}, "outputs": [], "source": [ + "import getpass\n", + "import os\n", + "\n", "import opik\n", - "from optimization_guide import config, data, rag_app\n", "\n", - "# Fail fast with a clear message if credentials are missing.\n", - "config.check_prerequisites()\n", + "OPIK_PROJECT_NAME = \"prompt-agent-optimization\"\n", + "\n", + "# Reads OPIK_API_KEY / OPIK_WORKSPACE from the environment, or prompts for them.\n", + "# install_mcp=False keeps it non-interactive so it also runs unattended in CI.\n", + "opik.configure(project_name=OPIK_PROJECT_NAME, install_mcp=False)\n", + "\n", + "# The model, called via litellm. Default is a small Anthropic Claude model; set\n", + "# OPIK_EXAMPLES_MODEL to switch providers (e.g. \"openai/gpt-4o-mini\").\n", + "MODEL = os.environ.get(\"OPIK_EXAMPLES_MODEL\", \"anthropic/claude-haiku-4-5-20251001\")\n", + "\n", + "# Its provider key (ANTHROPIC_API_KEY / OPENAI_API_KEY / ...): from env, or prompted.\n", + "key_var = f\"{MODEL.split('/')[0].upper()}_API_KEY\"\n", + "if not os.environ.get(key_var):\n", + " os.environ[key_var] = getpass.getpass(f\"Enter {key_var}: \")\n", + "\n", + "print(\"Using model:\", MODEL, \"| logging to project:\", OPIK_PROJECT_NAME)" + ] + }, + { + "cell_type": "markdown", + "id": "07025541", + "metadata": {}, + "source": [ + "### The app: a tiny RAG over the Ledgerline docs\n", + "\n", + "Everything the guide needs is defined right here in the notebook — no external files. First the corpus and evaluation cases (for a fictional task-queue product, **Ledgerline**), then a small RAG app: a ChromaDB retriever and an `answer()` function. Swap this corpus for your own product's docs and the rest of the guide still applies." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc121bb4", + "metadata": {}, + "outputs": [], + "source": [ + "# --- Corpus: ~12 short Ledgerline doc snippets. ---\n", + "DOCS = [\n", + " {\"id\": \"timeouts\", \"title\": \"Job timeouts\", \"text\": \"Every Ledgerline job has a default timeout of 30 seconds. Jobs exceeding the timeout are marked failed and eligible for retry. The maximum configurable timeout is 15 minutes.\"},\n", + " {\"id\": \"retries\", \"title\": \"Retries\", \"text\": \"Failed jobs are retried automatically. The default maximum number of retries is 3, using exponential backoff starting at 2 seconds. Set max_retries to 0 to disable retries.\"},\n", + " {\"id\": \"rate-limits\", \"title\": \"Rate limits\", \"text\": \"The API allows 1000 requests per minute per API key. Exceeding the limit returns HTTP 429. Rate limit headers are included on every response.\"},\n", + " {\"id\": \"auth\", \"title\": \"Authentication\", \"text\": \"Authenticate by sending your API key in the Authorization header as a Bearer token: 'Authorization: Bearer '. Keys are created in the dashboard.\"},\n", + " {\"id\": \"priorities\", \"title\": \"Queue priorities\", \"text\": \"Ledgerline supports three queue priorities: low, default, and high. High-priority jobs are dequeued before default and low. Priority is set per job at enqueue time.\"},\n", + " {\"id\": \"dead-letter\", \"title\": \"Dead-letter queue\", \"text\": \"After a job exhausts all retries it is moved to the dead-letter queue, where it is retained for 7 days before permanent deletion. Dead-letter jobs can be replayed from the dashboard.\"},\n", + " {\"id\": \"webhooks\", \"title\": \"Webhooks\", \"text\": \"When a job completes, Ledgerline POSTs a webhook to your configured URL. The payload includes job_id, status, and result fields. Webhook deliveries are signed with the X-Ledgerline-Signature header.\"},\n", + " {\"id\": \"install\", \"title\": \"SDK installation\", \"text\": \"Install the Python SDK with 'pip install ledgerline'. The SDK requires Python 3.9 or newer. Import it as 'import ledgerline'.\"},\n", + " {\"id\": \"concurrency\", \"title\": \"Concurrency\", \"text\": \"Each project runs up to 50 concurrent jobs by default. Contact support to raise the concurrency limit for your plan.\"},\n", + " {\"id\": \"regions\", \"title\": \"Regions\", \"text\": \"Ledgerline is available in three regions: us-east, eu-west, and ap-south. The default region is us-east. Set the region when initializing the client.\"},\n", + " {\"id\": \"batch\", \"title\": \"Batch enqueue\", \"text\": \"You can enqueue up to 500 jobs in a single batch request. Larger batches must be split. Each job in a batch is billed individually.\"},\n", + " {\"id\": \"idempotency\", \"title\": \"Idempotency\", \"text\": \"Pass an Idempotency-Key header to safely retry enqueue requests. Ledgerline deduplicates requests with the same key for 24 hours.\"},\n", + "]\n", + "\n", + "# --- Part 1 eval: exact-match cases. Each expected_substring appears verbatim in a doc. ---\n", + "EXACT_CASES = [\n", + " {\"query\": \"What is the default job timeout?\", \"expected_substring\": \"30 seconds\"},\n", + " {\"query\": \"What is the maximum configurable timeout?\", \"expected_substring\": \"15 minutes\"},\n", + " {\"query\": \"How many times are failed jobs retried by default?\", \"expected_substring\": \"3\"},\n", + " {\"query\": \"How do I disable retries?\", \"expected_substring\": \"max_retries to 0\"},\n", + " {\"query\": \"What backoff does retry use, and starting at what delay?\", \"expected_substring\": \"2 seconds\"},\n", + " {\"query\": \"How many requests per minute per API key are allowed?\", \"expected_substring\": \"1000 requests per minute\"},\n", + " {\"query\": \"What HTTP status is returned when the rate limit is exceeded?\", \"expected_substring\": \"429\"},\n", + " {\"query\": \"Which header carries the API key?\", \"expected_substring\": \"Authorization\"},\n", + " {\"query\": \"What token scheme is used for auth?\", \"expected_substring\": \"Bearer\"},\n", + " {\"query\": \"What queue priorities are supported?\", \"expected_substring\": \"low, default, and high\"},\n", + " {\"query\": \"How long are dead-letter jobs retained?\", \"expected_substring\": \"7 days\"},\n", + " {\"query\": \"Which header signs webhook deliveries?\", \"expected_substring\": \"X-Ledgerline-Signature\"},\n", + " {\"query\": \"How do I install the Python SDK?\", \"expected_substring\": \"pip install ledgerline\"},\n", + " {\"query\": \"What Python version does the SDK require?\", \"expected_substring\": \"3.9\"},\n", + " {\"query\": \"How many concurrent jobs run per project by default?\", \"expected_substring\": \"50 concurrent jobs\"},\n", + " {\"query\": \"What is the default region?\", \"expected_substring\": \"us-east\"},\n", + " {\"query\": \"How many jobs can I enqueue in one batch?\", \"expected_substring\": \"500 jobs\"},\n", + " {\"query\": \"How long are idempotency keys deduplicated?\", \"expected_substring\": \"24 hours\"},\n", + "]\n", + "\n", + "# --- Part 2 eval: open-ended cases with a reference answer (for an LLM judge). ---\n", + "JUDGE_CASES = [\n", + " {\"query\": \"How should I handle a job that keeps failing?\", \"reference\": \"Explain retries with exponential backoff, the default of 3 retries, and that exhausted jobs move to the dead-letter queue (retained 7 days, replayable from the dashboard).\"},\n", + " {\"query\": \"How do I make sure I don't enqueue the same job twice if my request retries?\", \"reference\": \"Use an Idempotency-Key header; Ledgerline deduplicates same-key requests for 24 hours.\"},\n", + " {\"query\": \"What's the best way to authenticate my requests?\", \"reference\": \"Send the API key as a Bearer token in the Authorization header; create keys in the dashboard.\"},\n", + " {\"query\": \"How do I get notified when a job finishes?\", \"reference\": \"Configure a webhook URL; Ledgerline POSTs job_id, status, and result, signed with X-Ledgerline-Signature.\"},\n", + " {\"query\": \"How can I prioritise urgent work?\", \"reference\": \"Set the job priority to high at enqueue time; high-priority jobs are dequeued before default and low.\"},\n", + " {\"query\": \"How do I run more jobs at the same time?\", \"reference\": \"Default concurrency is 50 concurrent jobs per project; contact support to raise the limit.\"},\n", + " {\"query\": \"How do I choose where my jobs run?\", \"reference\": \"Set the region (us-east, eu-west, ap-south) when initializing the client; default is us-east.\"},\n", + " {\"query\": \"What happens when I hit the rate limit?\", \"reference\": \"Requests over 1000/min per key return HTTP 429; rate-limit headers are on every response.\"},\n", + " {\"query\": \"How do I submit many jobs efficiently?\", \"reference\": \"Use batch enqueue, up to 500 jobs per request; split larger batches; each job billed individually.\"},\n", + " {\"query\": \"How long do I have to recover a permanently failing job?\", \"reference\": \"Dead-letter jobs are retained 7 days before permanent deletion and can be replayed from the dashboard.\"},\n", + " {\"query\": \"Can I make jobs run longer than the default?\", \"reference\": \"Yes; the default timeout is 30 seconds and the maximum configurable timeout is 15 minutes.\"},\n", + " {\"query\": \"How do I start using the SDK in Python?\", \"reference\": \"Install with pip install ledgerline (Python 3.9+), then import ledgerline.\"},\n", + "]\n", + "\n", + "print(f\"{len(DOCS)} docs, {len(EXACT_CASES)} exact cases, {len(JUDGE_CASES)} judge cases\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f971cba4", + "metadata": {}, + "outputs": [], + "source": [ + "import threading\n", + "\n", + "import chromadb\n", + "import litellm\n", + "\n", + "client = opik.Opik()\n", + "\n", + "# --- A tiny RAG app: an in-memory ChromaDB retriever + an answer function. ---\n", + "_collection = chromadb.Client().get_or_create_collection(\n", + " \"ledgerline_docs\", metadata={\"hnsw:space\": \"cosine\"}\n", + ")\n", + "# The optimizer calls retrieve() across worker threads; serialize reads for safety.\n", + "_retrieve_lock = threading.Lock()\n", + "\n", + "\n", + "def ingest(docs):\n", + " _collection.upsert(\n", + " ids=[d[\"id\"] for d in docs],\n", + " documents=[d[\"text\"] for d in docs],\n", + " metadatas=[{\"title\": d[\"title\"]} for d in docs],\n", + " )\n", + " return _collection.count()\n", + "\n", "\n", - "client = opik.Opik(project_name=config.PROJECT_NAME)\n", - "print(\"Using models:\", config.GEN_MODEL)" + "def retrieve(query, n_results=3):\n", + " with _retrieve_lock:\n", + " result = _collection.query(query_texts=[query], n_results=n_results)\n", + " return result[\"documents\"][0]\n", + "\n", + "\n", + "@opik.track(project_name=OPIK_PROJECT_NAME)\n", + "def answer(query, system_prompt, model=None):\n", + " context = \"\\n\\n\".join(retrieve(query))\n", + " messages = [\n", + " {\"role\": \"system\", \"content\": system_prompt},\n", + " {\"role\": \"user\", \"content\": f\"Context:\\n{context}\\n\\nQuestion: {query}\"},\n", + " ]\n", + " response = litellm.completion(model=model or MODEL, messages=messages)\n", + " return response.choices[0].message.content\n", + "\n", + "\n", + "def build_dataset(name, cases):\n", + " dataset = client.get_or_create_dataset(name)\n", + " dataset.insert(cases)\n", + " return dataset\n", + "\n", + "\n", + "def optimized_system(result):\n", + " \"\"\"The optimized system text. An optimizer-returned ChatPrompt stores its text\n", + " in messages (get_messages()), not the .system scalar (which stays None).\"\"\"\n", + " for message in result.prompt.get_messages():\n", + " if message[\"role\"] == \"system\":\n", + " return message[\"content\"]\n", + " return None" ] }, { "cell_type": "markdown", - "id": "682398ff", + "id": "5474ff73", "metadata": {}, "source": [ "### Part 1 — Your first optimization ⭐ (workshop)\n", @@ -68,40 +254,37 @@ { "cell_type": "code", "execution_count": null, - "id": "1e699b31", + "id": "e0b47f79", "metadata": {}, "outputs": [], "source": [ - "docs = data.load_docs()\n", - "count = rag_app.ingest(docs)\n", + "count = ingest(DOCS)\n", "print(f\"Ingested {count} doc snippets into ChromaDB\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "18689624", + "id": "6a5bb899", "metadata": {}, "outputs": [], "source": [ - "exact_cases = data.load_exact_cases()\n", - "\n", "# RAG step — retrieve. For each question, pull the most relevant docs with our\n", - "# real ChromaDB retriever and attach them as `context`. A production RAG system\n", - "# retrieves per query at answer time; we do it once here so every optimizer trial\n", - "# answers the SAME question from the SAME context. What we optimize is the\n", - "# *prompt*, not the retriever.\n", - "for case in exact_cases:\n", - " case[\"context\"] = \"\\n\\n\".join(rag_app.retrieve(case[\"query\"]))\n", + "# retriever and attach them as `context`. A production RAG system retrieves per\n", + "# query at answer time; we do it once here so every optimizer trial answers the\n", + "# SAME question from the SAME context. What we optimize is the *prompt*, not the\n", + "# retriever.\n", + "for case in EXACT_CASES:\n", + " case[\"context\"] = \"\\n\\n\".join(retrieve(case[\"query\"]))\n", "\n", - "exact_dataset = data.build_dataset(client, \"ledgerline-exact\", exact_cases)\n", - "print(f\"Dataset 'ledgerline-exact' has {len(exact_cases)} cases (each with retrieved context)\")" + "exact_dataset = build_dataset(\"ledgerline-exact\", EXACT_CASES)\n", + "print(f\"Dataset 'ledgerline-exact' has {len(EXACT_CASES)} cases (each with retrieved context)\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "d3ad2617", + "id": "fc73beb9", "metadata": {}, "outputs": [], "source": [ @@ -109,12 +292,12 @@ "\n", "# Peek at one dataset item. The optimizer fills {query}/{context} in the prompt\n", "# from these fields; the exact-match metric checks that expected_substring appears.\n", - "print(json.dumps(exact_cases[0], indent=2))" + "print(json.dumps(EXACT_CASES[0], indent=2))" ] }, { "cell_type": "markdown", - "id": "a269b356", + "id": "fcd354dd", "metadata": {}, "source": [ "A dataset item looks like this — the question, the fact we check for, and the docs our retriever pulled for it (context abridged; the cell above prints it in full):\n", @@ -132,7 +315,7 @@ }, { "cell_type": "markdown", - "id": "f654fca1", + "id": "7493d07b", "metadata": {}, "source": [ "#### The metric: exact-match, no judge\n", @@ -143,7 +326,7 @@ { "cell_type": "code", "execution_count": null, - "id": "82e28ed4", + "id": "b441d305", "metadata": {}, "outputs": [], "source": [ @@ -164,7 +347,7 @@ }, { "cell_type": "markdown", - "id": "38d98de9", + "id": "8dbbc319", "metadata": {}, "source": [ "#### The starting prompt\n", @@ -177,7 +360,7 @@ { "cell_type": "code", "execution_count": null, - "id": "beea886d", + "id": "a3dbb793", "metadata": {}, "outputs": [], "source": [ @@ -189,13 +372,13 @@ " name=\"ledgerline-answer\",\n", " system=BASELINE_SYSTEM,\n", " user=\"Context:\\n{context}\\n\\nQuestion: {query}\",\n", - " model=config.GEN_MODEL,\n", + " model=MODEL,\n", ")" ] }, { "cell_type": "markdown", - "id": "b4a0a0f2", + "id": "3f64ec73", "metadata": {}, "source": [ "#### Run the optimizer\n", @@ -209,14 +392,14 @@ { "cell_type": "code", "execution_count": null, - "id": "91592a75", + "id": "d46de3a0", "metadata": {}, "outputs": [], "source": [ "from opik_optimizer import MetaPromptOptimizer\n", "\n", "optimizer = MetaPromptOptimizer(\n", - " model=config.OPTIMIZER_MODEL,\n", + " model=MODEL,\n", " n_threads=4,\n", " skip_perfect_score=False,\n", ")\n", @@ -233,16 +416,15 @@ "print(\"Best score: \", result.score)\n", "\n", "# See HOW the prompt was refined: the optimizer rewrote the *system* instructions.\n", - "# result.prompt is a ChatPrompt; .system is the optimized system text.\n", "print(\"\\n--- Baseline system prompt ---\")\n", "print(BASELINE_SYSTEM)\n", "print(\"\\n--- Optimized system prompt ---\")\n", - "print(result.prompt.system)" + "print(optimized_system(result))" ] }, { "cell_type": "markdown", - "id": "4795bd62", + "id": "2009c5ae", "metadata": {}, "source": [ "#### What does an optimized prompt look like?\n", @@ -258,7 +440,7 @@ }, { "cell_type": "markdown", - "id": "8aa5a588", + "id": "e3a37eb5", "metadata": {}, "source": [ "#### See it in Opik\n", @@ -270,7 +452,7 @@ }, { "cell_type": "markdown", - "id": "693e9769", + "id": "9e7c1ee4", "metadata": {}, "source": [ "## Part 2 — Metrics done right\n", @@ -280,7 +462,7 @@ }, { "cell_type": "markdown", - "id": "18467d7d", + "id": "214c46a7", "metadata": {}, "source": [ "#### The LLM-judge metric\n", @@ -291,7 +473,7 @@ { "cell_type": "code", "execution_count": null, - "id": "37b9ee3d", + "id": "4a271870", "metadata": {}, "outputs": [], "source": [ @@ -299,7 +481,7 @@ "\n", "\n", "def answer_relevance(dataset_item: dict, llm_output: str) -> float:\n", - " result = AnswerRelevance(model=config.JUDGE_MODEL).score(\n", + " result = AnswerRelevance(model=MODEL).score(\n", " input=dataset_item[\"query\"],\n", " output=llm_output,\n", " context=[dataset_item[\"reference\"]],\n", @@ -312,7 +494,7 @@ }, { "cell_type": "markdown", - "id": "12c87e9b", + "id": "6fe27303", "metadata": {}, "source": [ "#### How do we *trust* a judge?\n", @@ -329,13 +511,13 @@ { "cell_type": "code", "execution_count": null, - "id": "0b0cc2e0", + "id": "a96a5df9", "metadata": {}, "outputs": [], "source": [ - "sample = data.load_judge_cases()[0]\n", - "sample_output = rag_app.answer(sample[\"query\"], system_prompt=result.prompt.system) # result.prompt is a ChatPrompt; .system is the optimized system text\n", - "judged = AnswerRelevance(model=config.JUDGE_MODEL).score(\n", + "sample = JUDGE_CASES[0]\n", + "sample_output = answer(sample[\"query\"], system_prompt=optimized_system(result))\n", + "judged = AnswerRelevance(model=MODEL).score(\n", " input=sample[\"query\"],\n", " output=sample_output,\n", " context=[sample[\"reference\"]],\n", @@ -349,20 +531,19 @@ { "cell_type": "code", "execution_count": null, - "id": "44e455db", + "id": "0416e070", "metadata": {}, "outputs": [], "source": [ - "judge_cases = data.load_judge_cases()\n", - "for case in judge_cases:\n", - " case[\"context\"] = \"\\n\\n\".join(rag_app.retrieve(case[\"query\"]))\n", - "judge_dataset = data.build_dataset(client, \"ledgerline-judge\", judge_cases)\n", + "for case in JUDGE_CASES:\n", + " case[\"context\"] = \"\\n\\n\".join(retrieve(case[\"query\"]))\n", + "judge_dataset = build_dataset(\"ledgerline-judge\", JUDGE_CASES)\n", "\n", "judge_prompt = ChatPrompt(\n", " name=\"ledgerline-answer-judge\",\n", " system=BASELINE_SYSTEM,\n", " user=\"Context:\\n{context}\\n\\nQuestion: {query}\",\n", - " model=config.GEN_MODEL,\n", + " model=MODEL,\n", ")\n", "\n", "judge_result = optimizer.optimize_prompt(\n", @@ -377,7 +558,7 @@ }, { "cell_type": "markdown", - "id": "e6b6e231", + "id": "7350cf6b", "metadata": {}, "source": [ "#### Multi-objective: quality *and* cost\n", @@ -390,7 +571,7 @@ { "cell_type": "code", "execution_count": null, - "id": "a14ea1f7", + "id": "dea15376", "metadata": {}, "outputs": [], "source": [ @@ -419,12 +600,12 @@ " n_samples=8,\n", ")\n", "print(\"Multi-objective best score:\", multi_result.score)\n", - "print(\"\\nOptimized system prompt:\\n\", multi_result.prompt.system)" + "print(\"\\nOptimized system prompt:\\n\", optimized_system(multi_result))" ] }, { "cell_type": "markdown", - "id": "1da5064e", + "id": "cc9c9c42", "metadata": {}, "source": [ "#### Compare your runs\n", @@ -434,7 +615,7 @@ }, { "cell_type": "markdown", - "id": "b00aec82", + "id": "2ad67d28", "metadata": {}, "source": [ "## Part 3 — From prompt to agent\n", @@ -443,21 +624,21 @@ "\n", "**What \"optimizing an agent\" means:** not rewriting the tool's code — the retriever is fixed. It means optimizing the natural-language surface the agent reasons over: its **system prompt** (when to search, how to answer from results) and, optionally, its **tool descriptions** (`optimize_tools=True`) so it calls the tool at the right moments.\n", "\n", - "*(A lighter alternative to a tool is a yes/no retrieval gate — see `rag_app.should_retrieve` — optimized with the same loop. We use a real tool here because it better reflects a production agent.)*" + "*(A lighter alternative to a tool is a yes/no retrieval gate — a small prompt that decides whether to look up docs at all — optimized with the same loop. We use a real tool here because it better reflects a production agent.)*" ] }, { "cell_type": "code", "execution_count": null, - "id": "4cf1853a", + "id": "5387f4e6", "metadata": {}, "outputs": [], "source": [ - "# The tool the agent may call. It wraps our real ChromaDB retriever; the *agent*\n", - "# decides when to call it. Returning one string keeps the tool result clean.\n", + "# The tool the agent may call. It wraps our retriever; the *agent* decides when to\n", + "# call it. Returning one string keeps the tool result clean.\n", "def search_docs(query: str) -> str:\n", " \"\"\"Search the Ledgerline documentation and return the most relevant snippets.\"\"\"\n", - " return \"\\n\\n\".join(rag_app.retrieve(query))\n", + " return \"\\n\\n\".join(retrieve(query))\n", "\n", "\n", "SEARCH_DOCS_TOOL = {\n", @@ -477,15 +658,15 @@ "\n", "AGENT_SYSTEM = \"You are a Ledgerline support agent. Use tools when they help.\"\n", "\n", - "# tools + function_map make this ChatPrompt an agent: on a tool call, the\n", - "# optimizer executes search_docs and feeds the result back to the model.\n", + "# tools + function_map make this ChatPrompt an agent: on a tool call, the optimizer\n", + "# executes search_docs and feeds the result back to the model.\n", "agent_prompt = ChatPrompt(\n", " name=\"ledgerline-agent\",\n", " system=AGENT_SYSTEM,\n", " user=\"{query}\",\n", " tools=[SEARCH_DOCS_TOOL],\n", " function_map={\"search_docs\": search_docs},\n", - " model=config.GEN_MODEL,\n", + " model=MODEL,\n", ")\n", "\n", "# optimize_prompts defaults to \"system\": we tune the agent's instructions.\n", @@ -500,12 +681,12 @@ ")\n", "print(\"Agent baseline:\", agent_result.initial_score, \"-> best:\", agent_result.score)\n", "print(\"\\n--- Optimized agent system prompt ---\")\n", - "print(agent_result.prompt.system)" + "print(optimized_system(agent_result))" ] }, { "cell_type": "markdown", - "id": "373db4fb", + "id": "da6d814d", "metadata": {}, "source": [ "#### When demonstrations matter: Few-Shot Bayesian\n", @@ -516,13 +697,13 @@ { "cell_type": "code", "execution_count": null, - "id": "5c7a51e4", + "id": "b4245ab8", "metadata": {}, "outputs": [], "source": [ "from opik_optimizer import FewShotBayesianOptimizer\n", "\n", - "fewshot_optimizer = FewShotBayesianOptimizer(model=config.OPTIMIZER_MODEL, n_threads=4)\n", + "fewshot_optimizer = FewShotBayesianOptimizer(model=MODEL, n_threads=4)\n", "\n", "fewshot_result = fewshot_optimizer.optimize_prompt(\n", " prompt=agent_prompt,\n", @@ -535,7 +716,7 @@ }, { "cell_type": "markdown", - "id": "d899aa41", + "id": "0bb4ec29", "metadata": {}, "source": [ "#### Tuning the model, not the prompt: Parameter optimizer\n", @@ -545,7 +726,7 @@ }, { "cell_type": "markdown", - "id": "5668c2f1", + "id": "643a2c1c", "metadata": {}, "source": [ "## Part 4 — Choosing an optimizer\n", @@ -572,7 +753,7 @@ }, { "cell_type": "markdown", - "id": "0d0e2885", + "id": "c3dc750c", "metadata": {}, "source": [ "#### Chaining optimizers\n", @@ -582,7 +763,7 @@ }, { "cell_type": "markdown", - "id": "9f5990ba", + "id": "9d19f6d7", "metadata": {}, "source": [ "## Part 5 — Take it further\n", @@ -593,31 +774,40 @@ { "cell_type": "code", "execution_count": null, - "id": "c9be51dd", + "id": "2d189ce0", "metadata": {}, "outputs": [], "source": [ - "# multi_result.prompt is a ChatPrompt; .system is the optimized system text\n", - "best_prompt = opik.Prompt(name=\"ledgerline-answer\", prompt=multi_result.prompt.system)\n", + "best_prompt = opik.Prompt(name=\"ledgerline-answer\", prompt=optimized_system(multi_result))\n", "print(\"Saved prompt version:\", best_prompt.commit)" ] }, { "cell_type": "markdown", - "id": "d6d2358a", + "id": "2c4399cd", "metadata": {}, "source": [ "**Where to go next:**\n", "- **[Optimization Studio](https://www.comet.com/docs/opik/agent_optimization/optimization_studio)** — run all of this from the Opik UI, no code.\n", "- **[Optimizer benchmarks](https://www.comet.com/docs/opik/agent_optimization/algorithms/benchmarks)** — numbers per algorithm.\n", "- **[Agent optimization overview](https://www.comet.com/docs/opik/agent_optimization/overview)** — the full reference.\n", - "- **Wrap this in a CLI** — the plumbing (`optimization_guide/`) is import-ready; turning the notebook into a repeatable CLI is a natural next project (out of scope here).\n", + "- **Wrap this in a CLI** — the retriever + `answer()` helpers are defined inline above; lifting them into a module to back a repeatable CLI is a natural next project (out of scope here).\n", "\n", "You've gone A-to-Z: framing → first optimization → trustworthy judge metrics → multi-objective → agent tuning → optimizer selection → versioned prompt. Every step is a comparable run in Opik." ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "opik-examples (3.12.13)", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.13" + } + }, "nbformat": 4, "nbformat_minor": 5 } diff --git a/guides/prompt_agent_optimization/pyproject.toml b/guides/prompt_agent_optimization/pyproject.toml index 30983ad..07f3bdb 100644 --- a/guides/prompt_agent_optimization/pyproject.toml +++ b/guides/prompt_agent_optimization/pyproject.toml @@ -9,27 +9,16 @@ dependencies = [ "opik-optimizer", "chromadb", "litellm", - # WHY: this guide's deliverable is a notebook — Jupyter + nbconvert/nbformat - # are runtime deps so `uv sync` gives users a working kernel and lets the - # notebook be validated and executed headlessly. + # WHY: this guide's deliverable is a notebook. Jupyter lets `uv run jupyter lab` + # open it locally, and nbconvert/nbformat let it be validated and executed + # headlessly (CI). In Colab (or any bare env) the notebook's own %pip cell + # installs the runtime deps instead. "jupyterlab", "nbconvert", "nbformat", "ipykernel", ] -[dependency-groups] -dev = ["ruff", "pytest"] - # WHY: notebook-only example — uv manages the env, no installable package. [tool.uv] package = false - -[tool.ruff] -line-length = 110 -target-version = "py312" -# WHY: ruff can't parse the notebook's cell schema; lint .py files only. -extend-exclude = ["*.ipynb"] - -[tool.ruff.lint] -select = ["E", "F", "I", "UP", "B"] diff --git a/guides/prompt_agent_optimization/tests/conftest.py b/guides/prompt_agent_optimization/tests/conftest.py deleted file mode 100644 index cc97ef0..0000000 --- a/guides/prompt_agent_optimization/tests/conftest.py +++ /dev/null @@ -1,11 +0,0 @@ -import os -import sys -from pathlib import Path - -# Disable Opik tracking during tests: the @opik.track-decorated rag_app -# functions would otherwise flush spans to the backend at teardown and, with no -# credentials in the test env, emit 401 noise after the pytest summary. -os.environ.setdefault("OPIK_TRACK_DISABLE", "true") - -# Add parent directory to sys.path so tests can import optimization_guide -sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) diff --git a/guides/prompt_agent_optimization/tests/test_config.py b/guides/prompt_agent_optimization/tests/test_config.py deleted file mode 100644 index f0abfa8..0000000 --- a/guides/prompt_agent_optimization/tests/test_config.py +++ /dev/null @@ -1,49 +0,0 @@ -import importlib - -import pytest - - -def _reload_config(monkeypatch, **env): - env_vars = [ - "OPIK_API_KEY", - "OPIK_WORKSPACE", - "OPIK_EXAMPLES_MODEL", - "OPIK_PROJECT_NAME", - "ANTHROPIC_API_KEY", - ] - for k in env_vars: - monkeypatch.delenv(k, raising=False) - for k, v in env.items(): - monkeypatch.setenv(k, v) - import optimization_guide.config as config - return importlib.reload(config) - - -def test_default_model_when_unset(monkeypatch): - config = _reload_config(monkeypatch) - assert config.GEN_MODEL == "anthropic/claude-sonnet-4-6" - assert config.JUDGE_MODEL == "anthropic/claude-sonnet-4-6" - assert config.OPTIMIZER_MODEL == "anthropic/claude-sonnet-4-6" - - -def test_model_override(monkeypatch): - config = _reload_config(monkeypatch, OPIK_EXAMPLES_MODEL="openai/gpt-4o-mini") - assert config.GEN_MODEL == "openai/gpt-4o-mini" - - -def test_project_name_default(monkeypatch): - config = _reload_config(monkeypatch) - assert config.PROJECT_NAME == "prompt-agent-optimization" - - -def test_check_prerequisites_raises_when_missing(monkeypatch): - config = _reload_config(monkeypatch) # no OPIK_API_KEY / OPIK_WORKSPACE - with pytest.raises(RuntimeError) as exc: - config.check_prerequisites() - assert "OPIK_API_KEY" in str(exc.value) - assert "OPIK_WORKSPACE" in str(exc.value) - - -def test_check_prerequisites_passes_when_present(monkeypatch): - config = _reload_config(monkeypatch, OPIK_API_KEY="x", OPIK_WORKSPACE="w", ANTHROPIC_API_KEY="k") - assert config.check_prerequisites() is None diff --git a/guides/prompt_agent_optimization/tests/test_data.py b/guides/prompt_agent_optimization/tests/test_data.py deleted file mode 100644 index 980c9fa..0000000 --- a/guides/prompt_agent_optimization/tests/test_data.py +++ /dev/null @@ -1,49 +0,0 @@ -from optimization_guide import data - - -def test_load_docs_shape(): - docs = data.load_docs() - assert len(docs) >= 10 - assert all({"id", "title", "text"} <= set(d) for d in docs) - - -def test_load_exact_cases_grounded(): - docs = data.load_docs() - corpus = " ".join(d["text"] for d in docs) - cases = data.load_exact_cases() - assert len(cases) >= 15 - for c in cases: - assert c["expected_substring"] in corpus - - -def test_load_judge_cases_shape(): - cases = data.load_judge_cases() - assert len(cases) >= 10 - assert all({"query", "reference"} <= set(c) for c in cases) - - -class _FakeDataset: - def __init__(self, name): - self.name = name - self.items = [] - - def insert(self, items): - self.items.extend(items) - - -class _FakeClient: - def __init__(self): - self.created = {} - - def get_or_create_dataset(self, name): - ds = self.created.setdefault(name, _FakeDataset(name)) - return ds - - -def test_build_dataset_inserts_cases(): - client = _FakeClient() - cases = [{"query": "q1", "expected_substring": "a"}, {"query": "q2", "expected_substring": "b"}] - ds = data.build_dataset(client, "exact-eval", cases) - assert ds.name == "exact-eval" - assert len(ds.items) == 2 - assert ds.items[0]["query"] == "q1" diff --git a/guides/prompt_agent_optimization/tests/test_rag_app.py b/guides/prompt_agent_optimization/tests/test_rag_app.py deleted file mode 100644 index dfe3c1d..0000000 --- a/guides/prompt_agent_optimization/tests/test_rag_app.py +++ /dev/null @@ -1,67 +0,0 @@ -import pytest - - -@pytest.fixture -def rag(monkeypatch, tmp_path): - # Point Chroma at a temp dir before importing the module-level singleton. - import importlib - - monkeypatch.setenv("OPIK_EXAMPLES_MODEL", "anthropic/claude-sonnet-4-6") - import optimization_guide.config as config - - importlib.reload(config) - monkeypatch.setattr(config, "CHROMA_DIR", str(tmp_path / "chroma")) - monkeypatch.setattr(config, "COLLECTION", "test_docs") - import optimization_guide.rag_app as rag_app - - importlib.reload(rag_app) - # reset cached singleton - rag_app._collection = None - return rag_app - - -def test_ingest_and_retrieve(rag): - docs = [ - {"id": "a", "title": "Timeouts", "text": "The default job timeout is 30 seconds."}, - {"id": "b", "title": "Regions", "text": "The default region is us-east."}, - ] - count = rag.ingest(docs) - assert count == 2 - hits = rag.retrieve("what is the default timeout", n_results=1) - assert len(hits) == 1 - assert "30 seconds" in hits[0] - - -def test_answer_uses_context_and_prompt(rag, monkeypatch): - rag.ingest([{"id": "a", "title": "Timeouts", "text": "The default job timeout is 30 seconds."}]) - captured = {} - - def fake_completion(model, messages, **kwargs): - captured["model"] = model - captured["messages"] = messages - - class R: - choices = [type("C", (), {"message": type("M", (), {"content": "It is 30 seconds."})()})()] - return R() - - monkeypatch.setattr(rag.litellm, "completion", fake_completion) - out = rag.answer( - "what is the default timeout", - system_prompt="You are helpful.", - model="anthropic/claude-sonnet-4-6", - ) - assert out == "It is 30 seconds." - # system prompt propagated, context injected - assert captured["messages"][0]["role"] == "system" - assert "You are helpful." in captured["messages"][0]["content"] - assert any("30 seconds" in m["content"] for m in captured["messages"]) - - -def test_should_retrieve_parses_yes(rag, monkeypatch): - def fake_completion(model, messages, **kwargs): - class R: - choices = [type("C", (), {"message": type("M", (), {"content": "YES"})()})()] - return R() - - monkeypatch.setattr(rag.litellm, "completion", fake_completion) - assert rag.should_retrieve("how do I configure retries?") is True