[guide] Prompt versioning guide

guides/prompt_versioning/.env.example

@@ -0,0 +1,10 @@
+# OpenAI (used by inference.py and evaluate.py to run the latest prompt version)
+OPENAI_API_KEY=your_openai_api_key_here
+
+# Opik (Comet-hosted). Leave OPIK_API_KEY/OPIK_WORKSPACE unset to run every script in DRY_RUN.
+OPIK_API_KEY=your_opik_api_key_here
+OPIK_WORKSPACE=your_workspace
+OPIK_PROJECT_NAME=prompt-versioning
+
+# litellm model used by inference.py and evaluate.py
+OPIK_EXAMPLES_MODEL=openai/gpt-5-mini

guides/prompt_versioning/README.md

@@ -0,0 +1,74 @@
+# Prompt Versioning
+
+Version prompts in the Opik Prompt Library, compare versions for hallucination before you
+ship one, then run inference against whichever version is currently the latest commit
+without hardcoding the prompt text into your application.
+
+## What this does
+
+Every call to `client.create_prompt(name=..., prompt=...)` with the same `name` creates a new,
+immutable **commit** rather than overwriting the previous one — so you always have a history of
+what a prompt used to say, and can fetch any specific version by its commit hash. This guide
+walks through the full loop: commit two versions of a prompt, score them against each other with
+an LLM-as-judge `Hallucination` metric, then fetch whichever version is newest and run it through
+a real OpenAI call.
+
+- **`prompts.py`** — the raw prompt strings, versioned in this guide (a "loose" baseline and a
+  stricter, compliance-reviewed rewrite for two use cases: a fintech assistant and an earnings-call
+  summarizer).
+- **`version.py`** — the Opik logic: commit new prompt versions and fetch a version by commit or by
+  "latest".
+- **`evaluate.py`** — builds a small Opik dataset and scores both versions of the summarizer prompt
+  with the `Hallucination` metric via `evaluate_prompt`, so you can compare them as experiments in
+  the Opik UI.
+- **`inference.py`** — fetches the latest committed version of the fintech-assistant prompt from
+  Opik and runs it through the OpenAI SDK, traced with `@opik.track`.
+
+## Prerequisites
+
+```bash
+uv sync    # or: uv pip install "opik>=2.0.74" "openai>=1.0"
+```
+
+| Variable | Required for | Description |
+|---|---|---|
+| `OPIK_API_KEY` | `version.py`, `evaluate.py`, `inference.py` | Your Opik API key. Unset → every script runs in DRY_RUN |
+| `OPIK_WORKSPACE` | `version.py`, `evaluate.py`, `inference.py` | Your Opik workspace name |
+| `OPIK_PROJECT_NAME` | `version.py`, `evaluate.py`, `inference.py` | Opik project for traces/experiments |
+| `OPENAI_API_KEY` | `inference.py`, `evaluate.py` | OpenAI key used to run the fetched prompt |
+
+## Running it
+
+```bash
+# Dry-run first — no credentials needed.
+uv run python version.py     # prints the prompt versions it would create
+uv run python evaluate.py    # prints the versions it would score
+uv run python inference.py   # prints the query it would run
+
+# Full run — set credentials, then the same scripts talk to Opik (+ OpenAI for inference.py).
+export OPIK_API_KEY="<your-key>"
+export OPIK_WORKSPACE="<your-workspace>"
+export OPENAI_API_KEY="<your-key>"
+
+uv run python version.py     # commits 2 versions of 'fintech-assistant', prints their commits
+uv run python evaluate.py    # commits 2 versions of 'summarizer-fintech', logs 2 experiments
+uv run python inference.py   # fetches the latest 'fintech-assistant' commit, runs it via OpenAI
+```
+
+Or just run `./run.sh` to do all three in sequence
+
+## How it works
+
+1. **`prompts.py`** holds plain prompt strings — no Opik calls. Two pairs of versions: a loose
+   baseline and a stricter rewrite, for a fintech assistant and for an earnings-call summarizer.
+2. **`version.py`** wraps `client.create_prompt` (commit a new version) and `client.get_prompt`
+   (fetch a version — by `commit`, or the newest one when `commit` is omitted). Running it commits
+   both fintech-assistant versions and prints each resulting commit hash.
+3. **`evaluate.py`** commits both summarizer versions, builds a one-item Opik dataset from a sample
+   earnings-call transcript, and runs `evaluate_prompt` for each version with the `Hallucination`
+   metric as the scorer. Each version becomes its own experiment in Opik, so you can compare
+   hallucination scores side by side — the stricter version should score lower.
+4. **`inference.py`** calls `version.get_latest` to resolve whichever fintech-assistant commit is
+   newest, then sends it as the system prompt to the OpenAI SDK. The call is wrapped in
+   `@opik.track`, so it shows up as a trace in Opik. Because it always asks for the latest commit,
+   promoting a new prompt version in Opik changes what this script runs without any code changes.

guides/prompt_versioning/config.py

@@ -0,0 +1,14 @@
+import os
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+OPIK_API_KEY = os.environ.get("OPIK_API_KEY")
+OPIK_WORKSPACE = os.environ.get("OPIK_WORKSPACE")
+OPIK_PROJECT_NAME = os.environ.get("OPIK_PROJECT_NAME", "prompt-versioning")
+
+DRY_RUN = not (OPIK_API_KEY and OPIK_WORKSPACE)
+
+LLM_MODEL = os.environ.get("OPIK_EXAMPLES_MODEL", "openai/gpt-5-mini")
+JUDGE_MODEL = LLM_MODEL

guides/prompt_versioning/evaluate.py

@@ -0,0 +1,66 @@
+"""Score two prompt versions for hallucination using an Opik dataset + evaluate_prompt"""
+
+import opik
+from opik.evaluation import evaluate_prompt
+from opik.evaluation.metrics import Hallucination
+
+from config import DRY_RUN, JUDGE_MODEL, OPIK_PROJECT_NAME
+from prompts import SUMMARIZER_V1, SUMMARIZER_V2
+
+SUMMARIZER_PROMPT_NAME = "summarizerfintechv1"
+DATASET_NAME = "prompt-versioning-summary-eg"
+
+TRANSCRIPT = (
+    "Apple reported Q4 revenue of $89.5 billion, up 6% year-over-year. iPhone revenue grew "
+    "10% to $43.8 billion. CEO Tim Cook said 'We're thrilled with the strong demand for "
+    "iPhone 15 Pro.'"
+)
+CONTEXT = "Q4 revenue: $89.5B, +6% YoY. iPhone: $43.8B, +10%. Tim Cook commented on iPhone 15 Pro demand."
+QUERY = f"Summarize this earnings call:\n{TRANSCRIPT}"
+
+
+def build_dataset(client: opik.Opik) -> opik.Dataset:
+    dataset = client.get_or_create_dataset(name=DATASET_NAME, project_name=OPIK_PROJECT_NAME)
+    dataset.insert([{"input": QUERY, "context": CONTEXT}])
+    return dataset
+
+
+def score_version(dataset: opik.Dataset, prompt: opik.Prompt, experiment_name: str):
+    return evaluate_prompt(
+        dataset=dataset,
+        messages=[
+            {"role": "system", "content": prompt.prompt},
+            {"role": "user", "content": "{{input}}"},
+        ],
+        model=JUDGE_MODEL,
+        scoring_metrics=[Hallucination(model=JUDGE_MODEL)],
+        experiment_name=experiment_name,
+        prompt=prompt,
+    )
+
+
+def main() -> None:
+    if DRY_RUN:
+        print(
+            "[DRY RUN] Opik creds not set — would score 2 versions of "
+            f"'{SUMMARIZER_PROMPT_NAME}' on dataset '{DATASET_NAME}' with "
+            f"Hallucination(model={JUDGE_MODEL}):"
+        )
+        print(f"  v1 (basic):  {SUMMARIZER_V1[:60]}...")
+        print(f"  v2 (strict): {SUMMARIZER_V2[:60]}...")
+        return
+
+    client = opik.Opik()
+    dataset = build_dataset(client)
+
+    v1 = client.create_prompt(name=SUMMARIZER_PROMPT_NAME, prompt=SUMMARIZER_V1)
+    v2 = client.create_prompt(name=SUMMARIZER_PROMPT_NAME, prompt=SUMMARIZER_V2)
+
+    score_version(dataset, v1, "summarizer-v1-basic")
+    score_version(dataset, v2, "summarizer-v2-strict")
+
+    print("Experiments logged to Opik — compare the Hallucination scores for v1 vs v2 in the UI.")
+
+
+if __name__ == "__main__":
+    main()

guides/prompt_versioning/inference.py

@@ -0,0 +1,42 @@
+"""Fetch the latest committed prompt version from Opik and run it via litellm"""
+
+import litellm
+import opik
+
+from config import DRY_RUN, LLM_MODEL, OPIK_PROJECT_NAME
+from version import PROMPT_NAME, get_latest
+
+USER_QUERY = "Should I put my savings in Bitcoin or index funds?"
+
+
+@opik.track(project_name=OPIK_PROJECT_NAME)
+def run_inference(system_prompt: str, user_query: str) -> str:
+    response = litellm.completion(
+        model=LLM_MODEL,
+        messages=[
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": user_query},
+        ],
+    )
+    return response.choices[0].message.content
+
+
+def main() -> None:
+    if DRY_RUN:
+        print(
+            f"[DRY RUN] Opik creds not set — would fetch the latest '{PROMPT_NAME}' commit and "
+            f"run it via litellm ({LLM_MODEL}) on:"
+        )
+        print(f"  {USER_QUERY}")
+        return
+
+    opik_client = opik.Opik()
+    prompt = get_latest(opik_client)
+    print(f"Using '{PROMPT_NAME}' commit {prompt.commit}")
+
+    answer = run_inference(prompt.prompt, USER_QUERY)
+    print(f"\n{answer}")
+
+
+if __name__ == "__main__":
+    main()

guides/prompt_versioning/prompts.py

@@ -0,0 +1,62 @@
+"""Sample Prompts used across the prompt-versioning guide."""
+
+FINTECH_ASSISTANT_V1 = """
+You are a financial advisor assistant for Finance with Tarun.
+
+Your role is to help users understand investment concepts and retirement planning.
+Be helpful and informative. Answer questions about stocks, bonds, ETFs, and retirement accounts.
+"""
+
+FINTECH_ASSISTANT_V2 = (
+    """
+You are a licensed financial advisor assistant for Finance with Tarun, a registered investment advisor (RIA).
+
+## Your Expertise
+- Retirement planning (401k, IRA, Roth IRA)
+- Asset allocation strategies
+- Risk assessment and portfolio diversification
+- Tax-efficient investing basics
+
+## Response Guidelines
+1. Start with a direct answer to the user's question
+2. Provide educational context (2-3 sentences)
+3. Include a risk consideration when relevant
+4. End with a follow-up question to understand their situation better
+
+## Compliance Rules (MUST FOLLOW)
+- Never recommend specific stocks or securities by name
+"""
+    '- Always include: "This is educational information, not personalized financial advice. '
+    'Consult a licensed advisor for your specific situation."\n'
+    """- If asked about market timing or "hot tips", redirect to long-term investing principles
+- Never guarantee returns or make performance predictions
+- For tax questions, recommend consulting a CPA
+
+## Tone
+Professional yet approachable. Use clear language, avoid jargon unless explained.
+"""
+)
+
+SUMMARIZER_V1 = """You are a financial analyst summarizing earnings calls.
+Provide a comprehensive summary including key metrics, guidance, and management commentary."""
+
+SUMMARIZER_V2 = (
+    "You are a financial analyst creating earnings call summaries for compliance-reviewed "
+    "reports.\n"
+    """
+
+## Strict Rules
+- ONLY include facts explicitly stated in the provided transcript
+- Use EXACT numbers - never round or approximate
+- Never infer sentiment not directly expressed by management
+- If guidance wasn't mentioned, state "No guidance provided"
+- Attribute all quotes: "CEO [Name] stated..."
+
+## Output Format
+**Reported Metrics**: [Only numbers explicitly stated]
+**Management Commentary**: [Direct quotes or close paraphrases only]
+**Forward Guidance**: [Only if explicitly provided]
+**NOT MENTIONED**: [List key items not covered]
+
+If uncertain whether something was stated, DO NOT include it."""
+)

guides/prompt_versioning/pyproject.toml

@@ -0,0 +1,24 @@
+[project]
+name = "prompt-versioning"
+version = "0.1.0"
+description = "Version prompts in the Opik Prompt Library, evaluate versions for hallucination, and run inference against the latest committed version via the OpenAI SDK."
+readme = "README.md"
+requires-python = ">=3.12,<3.14"
+dependencies = [
+    "opik>=2.0.74",
+    "openai>=1.0",
+]
+
+[dependency-groups]
+dev = ["ruff"]
+
+# WHY: loose runnable scripts, not an installable package — uv manages the env only.
+[tool.uv]
+package = false
+
+[tool.ruff]
+line-length = 110
+target-version = "py312"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B"]

guides/prompt_versioning/run.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -e
+
+# Entry point CI runs for this example. With no Opik/OpenAI credentials each script falls
+# back to DRY_RUN and exits 0 (the secrets-free check); with credentials set it runs live —
+# versioning prompts in Opik, evaluating versions for hallucination, then running inference
+# against the latest committed version via the OpenAI SDK.
+uv sync
+uv run python version.py
+uv run python inference.py
+uv run python evaluate.py

guides/prompt_versioning/version.py

@@ -0,0 +1,49 @@
+"""Prompt Versioning tracing using OPIK
+Every call to `client.create_prompt` with the same `name` creates a new, immutable
+version ("commit") of that prompt nothing is overwritten. `client.get_prompt` fetches a specific commit"""
+
+import opik
+
+from config import DRY_RUN
+from prompts import FINTECH_ASSISTANT_V1, FINTECH_ASSISTANT_V2
+
+PROMPT_NAME = "fintechassistv1"
+
+
+def create_version(client: opik.Opik, prompt_text: str, tag: str) -> opik.Prompt:
+    """Save a new version of PROMPT_NAME to the Prompt Library."""
+    return client.create_prompt(name=PROMPT_NAME, prompt=prompt_text, metadata={"tag": tag})
+
+
+def get_latest(client: opik.Opik) -> opik.Prompt:
+    """Fetch the most recently committed version of PROMPT_NAME."""
+    return client.get_prompt(name=PROMPT_NAME)
+
+
+def get_version(client: opik.Opik, commit: str) -> opik.Prompt:
+    """Fetch a specific commit of PROMPT_NAME."""
+    return client.get_prompt(name=PROMPT_NAME, commit=commit)
+
+
+def main() -> None:
+    if DRY_RUN:
+        print("[DRY RUN] Opik creds not set — would create 2 versions of prompt:")
+        print(f"  name: {PROMPT_NAME}")
+        print(f"  v1 tag=baseline             ({len(FINTECH_ASSISTANT_V1)} chars)")
+        print(f"  v2 tag=compliance-reviewed  ({len(FINTECH_ASSISTANT_V2)} chars)")
+        return
+
+    client = opik.Opik()
+
+    v1 = create_version(client, FINTECH_ASSISTANT_V1, tag="baseline")
+    print(f"Created '{PROMPT_NAME}' commit {v1.commit} (tag=baseline)")
+
+    v2 = create_version(client, FINTECH_ASSISTANT_V2, tag="compliance-reviewed")
+    print(f"Created '{PROMPT_NAME}' commit {v2.commit} (tag=compliance-reviewed)")
+
+    latest = get_latest(client)
+    print(f"Latest version of '{PROMPT_NAME}' is commit {latest.commit}")
+
+
+if __name__ == "__main__":
+    main()