diff --git a/.claude/skills/scaffold-example/SKILL.md b/.claude/skills/scaffold-example/SKILL.md index 831a038..e654c5d 100644 --- a/.claude/skills/scaffold-example/SKILL.md +++ b/.claude/skills/scaffold-example/SKILL.md @@ -87,11 +87,14 @@ uv run eval # use-case: [DRY RUN] lists the dataset items + ass uv run optimize # use-case: [DRY RUN] line uv run --dry-run # script: [DRY RUN] line uv run ruff check . # clean +bash run.sh # the exact dry-run smoke test CI runs (exits 0 without creds) ``` -With credentials set (`ANTHROPIC_API_KEY`, `OPIK_API_KEY`, `OPIK_WORKSPACE`), the same commands -talk to Claude + Opik and `eval` prints a pass rate + experiment URL. Running the eval and -optimization workflows in depth is covered by the `run-evals` and `run-optimizations` skills. +The scaffold includes a working `run.sh` at the project root — this is what the CI `dry-run` and +`live-run` jobs execute. With credentials set (`ANTHROPIC_API_KEY`, `OPIK_API_KEY`, +`OPIK_WORKSPACE`), the same commands talk to Claude + Opik and `eval` prints a pass rate + +experiment URL. Running the eval and optimization workflows in depth is covered by the `run-evals` +and `run-optimizations` skills. ## What this skill does NOT do diff --git a/.claude/skills/scaffold-example/scripts/scaffold.py b/.claude/skills/scaffold-example/scripts/scaffold.py index ce279e4..ed572ed 100644 --- a/.claude/skills/scaffold-example/scripts/scaffold.py +++ b/.claude/skills/scaffold-example/scripts/scaffold.py @@ -150,6 +150,14 @@ def main() -> int: text = text.replace(f'{kebab} = "{entry}"', f'{command} = "{entry}"') pyproject.write_text(text, encoding="utf-8") + # run.sh invokes the CLI by the kebab name (from the sentinel rewrite); point it + # at the overridden command so `bash run.sh` matches [project.scripts]. + run_sh = dest / "run.sh" + if run_sh.is_file(): + text = run_sh.read_text(encoding="utf-8") + text = text.replace(f"uv run {kebab}", f"uv run {command}") + run_sh.write_text(text, encoding="utf-8") + # Description. if args.description: pyproject = dest / "pyproject.toml" @@ -173,11 +181,11 @@ def main() -> int: if kind == "package": print(f" 2. uv run {command} eval # DRY_RUN smoke test (no creds needed)") print(" 3. Fill the TODO stubs: app.py (real logic), prompts.py, data/cases.json, README.md") - print(f" 4. uv run ruff check . && uv run {command} run-all") + print(" 4. uv run ruff check . && bash run.sh # what CI runs (dry-run without creds)") else: print(f" 2. uv run {command} --dry-run # DRY_RUN smoke test (no creds needed)") print(f" 3. Fill the TODO in {pkg}.py (real logic) and the README.md sections") - print(" 4. uv run ruff check .") + print(" 4. uv run ruff check . && bash run.sh # what CI runs (dry-run without creds)") return 0 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..318471f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# Maintainer-owned CI + automation. Edits to these paths require a maintainer review. +# The scheduled-run curation list and the workflows are intentionally not contributor-editable. +/.github/workflows/ @fschlz @LeoRoccoBreedt +/.github/scheduled.json @fschlz @LeoRoccoBreedt +/.github/CODEOWNERS @fschlz @LeoRoccoBreedt +/.github/dependabot.yml @fschlz @LeoRoccoBreedt diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a9bd567 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + # Keep the GitHub Actions pinned in .github/workflows/ (checkout, setup-uv, …) current, + # so a "stay-working" examples repo doesn't silently drift onto stale/abandoned action versions. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + commit-message: + prefix: "ci" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..e4c26b4 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,18 @@ +## What & why + + + +## Checklist + + + +- [ ] Example is in the right bucket (`integrations` / `guides` / `use-cases` / `scripts`) +- [ ] Folder name is `lowercase_with_underscores` +- [ ] `README.md` has all required sections; index tables updated if examples were added/renamed/removed +- [ ] Dry-run works with no credentials — `bash run.sh` exits cleanly (this is what CI's secrets-free job runs) +- [ ] `uv run ruff check .` and `uv run ruff format --check .` are clean +- [ ] No credentials or `.env` files committed +- [ ] Dependencies declared in `pyproject.toml` (uv project); no `requirements.txt`, no committed `uv.lock` +- [ ] `run.sh` exists and starts with `set -e` +- [ ] `OPIK_PROJECT_NAME` is set — exported in `run.sh` (scripts) or defined in `config.py` (use-cases/guides) +- [ ] Examples that call LLMs use litellm and read `OPIK_EXAMPLES_MODEL` diff --git a/.github/scheduled.json b/.github/scheduled.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/.github/scheduled.json @@ -0,0 +1 @@ +[] diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml new file mode 100644 index 0000000..993c36c --- /dev/null +++ b/.github/workflows/compliance.yml @@ -0,0 +1,171 @@ +name: Example Compliance Check + +# Verifies that new example folders added in a PR follow repo conventions. +# Only checks folders that are new (not previously on the base branch). +# Rules enforced: +# 1. run.sh must exist and contain set -e +# 2. OPIK_PROJECT_NAME must be set — either exported in run.sh (scripts) +# or referenced in a .py file (use-cases/guides with a config module) +# 3. README.md must exist +# 4. pyproject.toml must exist; requirements.txt must not +# 5. If litellm is a dependency, OPIK_EXAMPLES_MODEL must be referenced in a .py file +# +# This check is secrets-free so it runs safely on PRs from forks. + +on: + pull_request: + paths: + - "examples/**" + - "integrations/**" + - "scripts/**" + - "use-cases/**" + - "guides/**" + +permissions: + contents: read + +jobs: + compliance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Find changed example folders + id: changed-folders + run: | + changed_files=$(git diff --name-only "origin/${{ github.base_ref }}...HEAD") + + folders="" + while IFS= read -r file; do + top=$(echo "$file" | cut -d'/' -f1) + case "$top" in + integrations) + parent=$(echo "$file" | cut -d'/' -f1,2,3) + ;; + examples|scripts|use-cases|guides) + parent=$(echo "$file" | cut -d'/' -f1,2) + ;; + *) + continue + ;; + esac + folders="$folders $parent" + done <<< "$changed_files" + + # Deduplicate and trim + folders=$(echo "$folders" | tr ' ' '\n' | sort -u | grep -v '^$' | tr '\n' ' ' | xargs) + echo "folders=$folders" >> "$GITHUB_OUTPUT" + + if [[ -z "$folders" ]]; then + echo "No example folders changed." + else + echo "Changed example folders: $folders" + fi + + - name: Check compliance + if: steps.changed-folders.outputs.folders != '' + run: | + FAILED=0 + + for folder in ${{ steps.changed-folders.outputs.folders }}; do + # Skip if the folder was deleted entirely + if [[ ! -d "$folder" ]]; then + echo "Skipping deleted folder: $folder" + continue + fi + + # Only enforce on NEW folders (not yet on the base branch). + # Modifying an existing example does not require adding run.sh. + if git ls-tree "origin/${{ github.base_ref }}" "$folder" 2>/dev/null | grep -q .; then + echo " Skipping '$folder' (existing folder — compliance enforced on new examples only)" + continue + fi + + echo "" + echo "Checking: $folder" + + if [[ ! -f "$folder/run.sh" ]]; then + echo "::error file=$folder/run.sh::$folder is missing run.sh" + FAILED=1 + else + echo " OK run.sh" + fi + + if [[ ! -f "$folder/README.md" ]]; then + echo "::error file=$folder/README.md::$folder is missing README.md" + FAILED=1 + else + echo " OK README.md" + fi + + if [[ ! -f "$folder/pyproject.toml" ]]; then + echo "::error::$folder is missing pyproject.toml (uv projects only — no requirements.txt)" + FAILED=1 + else + echo " OK pyproject.toml" + fi + + if [[ -f "$folder/requirements.txt" ]]; then + echo "::error file=$folder/requirements.txt::$folder has requirements.txt — declare deps in pyproject.toml instead" + FAILED=1 + fi + + if [[ -f "$folder/run.sh" ]]; then + if ! grep -q "set -e" "$folder/run.sh"; then + echo "::error file=$folder/run.sh::$folder/run.sh must contain 'set -e'" + FAILED=1 + else + echo " OK set -e in run.sh" + fi + + # OPIK_PROJECT_NAME may be exported in run.sh (scripts) + # or defined as a constant in a .py file (use-cases/guides with config.py). + if grep -q "export OPIK_PROJECT_NAME" "$folder/run.sh"; then + echo " OK OPIK_PROJECT_NAME in run.sh" + elif grep -rq "OPIK_PROJECT_NAME" "$folder" --include="*.py" 2>/dev/null; then + echo " OK OPIK_PROJECT_NAME in .py config" + else + echo "::error::$folder must set OPIK_PROJECT_NAME — either export it in run.sh or define it in a .py config file" + FAILED=1 + fi + fi + + # If litellm is declared as a dependency, OPIK_EXAMPLES_MODEL must be referenced. + # Match a quoted dependency entry ("litellm", "litellm>=...", "litellm[extra]") + # so a bare mention in a comment doesn't trip the check. + if grep -qE '"litellm' "$folder/pyproject.toml" 2>/dev/null; then + if grep -rq "OPIK_EXAMPLES_MODEL" "$folder" --include="*.py" 2>/dev/null; then + echo " OK OPIK_EXAMPLES_MODEL referenced (litellm dep detected)" + else + echo "::error::$folder uses litellm but no .py file references OPIK_EXAMPLES_MODEL" + echo "::error::Add: os.environ.get(\"OPIK_EXAMPLES_MODEL\", \"\") to your config" + FAILED=1 + fi + fi + done + + echo "" + if [[ "$FAILED" -ne 0 ]]; then + echo "Compliance check failed. Fix the errors above before merging." + exit 1 + fi + + echo "All compliance checks passed." + + - name: Write step summary + if: always() && steps.changed-folders.outputs.folders != '' + run: | + { + echo "## Example Compliance Check" + echo "" + echo "Folders checked: \`${{ steps.changed-folders.outputs.folders }}\`" + echo "" + echo "**Rules:**" + echo "- \`run.sh\` must exist and contain \`set -e\`" + echo "- \`OPIK_PROJECT_NAME\` must be set — via \`export\` in \`run.sh\` (scripts) or as a constant in a \`.py\` config file (use-cases/guides)" + echo "- \`README.md\` must exist" + echo "- \`pyproject.toml\` must exist; \`requirements.txt\` is not allowed" + echo "- If \`litellm\` is a dependency, a \`.py\` file must reference \`OPIK_EXAMPLES_MODEL\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml new file mode 100644 index 0000000..2beaf93 --- /dev/null +++ b/.github/workflows/pr-test.yml @@ -0,0 +1,183 @@ +name: PR Example Tests + +# Runs the example folders changed in this PR. +# - lint + dry-run are secrets-free and run on every PR, including forks. +# - live-run uses real Opik credentials and runs only when secrets are available +# (same-repo PRs); it confirms the example executes and logs traces to the +# opik-examples workspace. +# Any job failure blocks the PR. + +on: + pull_request: + paths: + - "examples/**" + - "integrations/**" + - "scripts/**" + - "use-cases/**" + - "guides/**" + +permissions: + contents: read + +concurrency: + group: pr-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + detect-changes: + runs-on: ubuntu-latest + outputs: + folders: ${{ steps.detect.outputs.folders }} + has_changes: ${{ steps.detect.outputs.has_changes }} + has_secrets: ${{ steps.secrets.outputs.has_secrets }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed runnable example folders + id: detect + run: | + changed_files=$(git diff --name-only "origin/${{ github.base_ref }}...HEAD") + + seen=() + folders_json="[" + + while IFS= read -r file; do + top=$(echo "$file" | cut -d'/' -f1) + case "$top" in + integrations) + folder=$(echo "$file" | cut -d'/' -f1,2,3) + ;; + examples|scripts|use-cases|guides) + folder=$(echo "$file" | cut -d'/' -f1,2) + ;; + *) + continue + ;; + esac + + if [[ -z "$folder" ]] || [[ ! -d "$folder" ]] || [[ ! -f "$folder/run.sh" ]]; then + continue + fi + + already_seen=false + for s in "${seen[@]}"; do + [[ "$s" == "$folder" ]] && already_seen=true && break + done + + if [[ "$already_seen" == false ]]; then + seen+=("$folder") + [[ "${#seen[@]}" -gt 1 ]] && folders_json="$folders_json," + folders_json="$folders_json\"$folder\"" + fi + done <<< "$changed_files" + + folders_json="$folders_json]" + + if [[ "${#seen[@]}" -eq 0 ]]; then + echo "folders=[]" >> "$GITHUB_OUTPUT" + echo "has_changes=false" >> "$GITHUB_OUTPUT" + echo "No runnable example folders changed." + else + echo "folders=$folders_json" >> "$GITHUB_OUTPUT" + echo "has_changes=true" >> "$GITHUB_OUTPUT" + echo "Runnable folders to test: ${seen[*]}" + fi + + - name: Check whether Opik secrets are available + id: secrets + # secrets.* is not available in job-level `if:`, so resolve it here and + # expose a boolean output the live-run job can gate on. Fork PRs get no + # secrets, so this is false and only lint + dry-run run. + env: + OPIK_API_KEY: ${{ secrets.OPIK_API_KEY }} + run: | + if [[ -n "$OPIK_API_KEY" ]]; then + echo "has_secrets=true" >> "$GITHUB_OUTPUT" + echo "Opik secrets available — live run will execute." + else + echo "has_secrets=false" >> "$GITHUB_OUTPUT" + echo "No Opik secrets (fork PR) — lint + dry-run only." + fi + + lint: + needs: detect-changes + if: needs.detect-changes.outputs.has_changes == 'true' + runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + folder: ${{ fromJson(needs.detect-changes.outputs.folders) }} + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + version: "latest" + enable-cache: true + python-version: "3.12" + + - name: Ruff check + format + run: | + cd "${{ matrix.folder }}" + uv sync + uv run ruff check . + uv run ruff format --check . + + dry-run: + needs: detect-changes + if: needs.detect-changes.outputs.has_changes == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + folder: ${{ fromJson(needs.detect-changes.outputs.folders) }} + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + version: "latest" + enable-cache: true + python-version: "3.12" + + - name: Run example (dry-run, no credentials) + # No Opik/LLM secrets in env: the example must detect missing credentials, + # fall back to DRY_RUN, and exit 0. This is the only execution signal fork + # PRs receive, and it enforces that every example's dry-run path works. + run: | + cd "${{ matrix.folder }}" + bash run.sh + + live-run: + needs: detect-changes + if: needs.detect-changes.outputs.has_changes == 'true' && needs.detect-changes.outputs.has_secrets == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + folder: ${{ fromJson(needs.detect-changes.outputs.folders) }} + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + version: "latest" + enable-cache: true + python-version: "3.12" + + - name: Run example (live, real Opik credentials) + env: + OPIK_API_KEY: ${{ secrets.OPIK_API_KEY }} + OPIK_WORKSPACE: ${{ vars.OPIK_WORKSPACE }} + OPIK_ENVIRONMENT: ${{ vars.OPIK_ENVIRONMENT }} + OPIK_EXAMPLES_MODEL: ${{ vars.OPIK_EXAMPLES_MODEL }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + cd "${{ matrix.folder }}" + bash run.sh diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml new file mode 100644 index 0000000..9296956 --- /dev/null +++ b/.github/workflows/scheduled.yml @@ -0,0 +1,64 @@ +name: Scheduled Example Tests + +# Runs a curated set of examples on a schedule to verify they stay working. +# The list is maintainer-managed — contributors do not need to add their example here. +# Criteria for inclusion: self-contained (no GPU, no pre-existing Opik data required), +# cheap to run, and stable enough for unattended weekly execution. +# +# To change the schedule: edit the cron expression below. +# To add or remove folders: edit .github/scheduled.json. + +on: + schedule: + # Every Monday at 06:00 UTC. Edit this line to change the cadence. + - cron: "0 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + load-examples: + runs-on: ubuntu-latest + outputs: + folders: ${{ steps.load.outputs.folders }} + steps: + - uses: actions/checkout@v4 + + - name: Load curated example list + id: load + run: | + folders=$(cat .github/scheduled.json) + echo "folders=$folders" >> "$GITHUB_OUTPUT" + + run-examples: + needs: load-examples + # Skip cleanly when the curated list is empty — an empty matrix array otherwise + # errors with "Matrix vector 'folder' does not contain any values". + if: needs.load-examples.outputs.folders != '[]' + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + folder: ${{ fromJson(needs.load-examples.outputs.folders) }} + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + version: "latest" + enable-cache: true + python-version: "3.12" + + - name: Run example + env: + OPIK_API_KEY: ${{ secrets.OPIK_API_KEY }} + OPIK_WORKSPACE: ${{ vars.OPIK_WORKSPACE }} + OPIK_ENVIRONMENT: ${{ vars.OPIK_ENVIRONMENT }} + OPIK_EXAMPLES_MODEL: ${{ vars.OPIK_EXAMPLES_MODEL }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + cd "${{ matrix.folder }}" + bash run.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 176a35a..1f83357 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,6 +86,7 @@ Then, either way: - A `README.md` that explains what problem it solves, what the example does, and how to run it - A working dry-run mode so anyone can run it locally without an Opik account - Credentials loaded from environment variables only — no hardcoded keys +- A `run.sh` that exports `OPIK_PROJECT_NAME` and can run the example end-to-end ### README structure @@ -98,7 +99,7 @@ Use the template's README as a guide. Required sections: ### Dry-run mode -Every runnable script should detect missing credentials and fall back to a local mode that prints output to the console instead of sending it to Opik. The standard pattern: +Every runnable example must work without credentials. The secrets-free CI `dry-run` job runs `bash run.sh` with no Opik or LLM keys set and expects a clean exit — it is the only execution signal a fork PR receives, so a working dry-run path is required, not optional. The standard pattern: ```python DRY_RUN = not (os.environ.get("OPIK_API_KEY") and os.environ.get("OPIK_WORKSPACE")) @@ -109,7 +110,7 @@ else: # send to Opik ``` -The dry-run output should be meaningful enough to verify the example is working correctly. +The dry-run output should be meaningful enough to verify the example is working. To see real traces, set your own Opik credentials locally — the same way CI's live run does. ### Credentials @@ -128,6 +129,96 @@ Never commit `.env` files or hardcoded keys. Add `.env` to the example's `.gitig Each example is a `uv` project: declare dependencies in its `pyproject.toml` (the single source of truth) and run with `uv sync` / `uv run`. No `requirements.txt`, no Poetry, and don't commit `uv.lock`. The README may still show an optional `pip install` fallback line. Do not assume the user has the repo's root virtualenv set up. +### run.sh + +Every testable example must include a `run.sh` at its root. This file is what the CI matrix runs. Requirements: + +- Start with `set -e` (fail fast on any error) +- Call `uv sync` before invoking any Python +- `OPIK_PROJECT_NAME` must be set — see below for where + +```bash +#!/usr/bin/env bash +set -e + +uv sync +uv run my-example eval # the command(s) that run the example end-to-end +``` + +The scaffold tool adds a working `run.sh` automatically when you create a new example. + +### Setting OPIK_PROJECT_NAME + +Every example must have a unique project name so its traces are identifiable in Opik. Where you set it depends on the example type: + +**Scripts** (single `.py` file, no config module) — export it in `run.sh`: + +```bash +export OPIK_PROJECT_NAME="my-script" +``` + +**Use-cases and guides** (examples with a `config.py`) — define it as a Python constant and pass it explicitly. This keeps all configuration in one place and works even when running Python directly: + +```python +# config.py +OPIK_PROJECT_NAME = os.environ.get("OPIK_PROJECT_NAME", "my-use-case") +``` + +```python +# app.py +@opik.track(project_name=config.OPIK_PROJECT_NAME) +def my_function(): ... +``` + +The compliance check accepts either pattern — it looks for `OPIK_PROJECT_NAME` in `run.sh` or in any `.py` file in the folder. + +### Opik workspace + +Live CI runs log to the workspace set in the `OPIK_WORKSPACE` GitHub Actions variable (currently `opik-examples`). Locally, set both vars explicitly to avoid accidentally writing to the shared workspace: + +```bash +export OPIK_API_KEY= +export OPIK_WORKSPACE= +``` + +`DRY_RUN` gates on both being set, so forgetting either keeps you in dry-run mode — safe by default. The `OPIK_API_KEY` Actions secret must be a service account key with write access to `opik-examples`. + +### CI model convention + +CI uses a cheap model to keep costs low. The `OPIK_EXAMPLES_MODEL` GitHub Actions variable controls which model is used (currently `anthropic/claude-haiku-4-5-20251001`). All examples that make LLM calls must read this variable. + +Use [litellm](https://github.com/BerriAI/litellm) as the model call layer — it routes to any provider via a single unified model name: + +```python +import litellm +response = litellm.completion(model=config.GEN_MODEL, messages=[...]) +``` + +In `config.py`, read `OPIK_EXAMPLES_MODEL` with a sensible default for local development: + +```python +# CI sets OPIK_EXAMPLES_MODEL to a cheap model (e.g. anthropic/claude-haiku-4-5-20251001). +# Locally, leave it unset to use the full model. +GEN_MODEL = os.environ.get("OPIK_EXAMPLES_MODEL", "anthropic/claude-sonnet-4-6") +JUDGE_MODEL = GEN_MODEL +OPTIMIZER_MODEL = GEN_MODEL +``` + +Model names use litellm's provider-prefixed format: `openai/gpt-4o-mini`, `anthropic/claude-haiku-4-5-20251001`, `google/gemini-1.5-flash`, etc. The compliance check will fail if `litellm` is declared as a dependency but `OPIK_EXAMPLES_MODEL` is not referenced. + +### Opik logging + +Same-repo PRs and scheduled runs have real Opik credentials (`OPIK_API_KEY`, `OPIK_WORKSPACE`, `OPIK_ENVIRONMENT`), and a `live-run` job logs traces to the `opik-examples` workspace — this is how we verify an example is working, not just that it exits 0. Fork PRs don't receive secrets (GitHub withholds them from forks), so they run only the secrets-free `lint` + `dry-run` jobs; a maintainer runs the live job after review. + +`OPIK_ENVIRONMENT` is set automatically in CI; you do not need to set it locally. It tags traces so CI runs are distinguishable from local runs in the Opik UI. + +Locally, set your own credentials to log to your personal workspace: + +```bash +export OPIK_API_KEY= +export OPIK_WORKSPACE= +``` + ### Code style - No comments that explain what the code does — well-named variables and functions do that @@ -142,9 +233,13 @@ Before opening a PR, verify: - [ ] Folder name is lowercase with underscores (e.g. `my_example`, not `MyExample` or `my-example`) - [ ] `README.md` has all required sections - [ ] READMEs updated — the example's `README.md`, and for added/renamed/removed examples the bucket index and the root `README.md` table -- [ ] Dry-run mode works (no env vars set) prints useful output without errors — e.g. `uv run eval` +- [ ] Dry-run works with no credentials set — `bash run.sh` exits cleanly (this is exactly what CI's secrets-free job runs) +- [ ] `uv run ruff check .` and `uv run ruff format --check .` are clean - [ ] No credentials or `.env` files committed - [ ] Dependencies declared in `pyproject.toml` (uv project); no `requirements.txt` +- [ ] `run.sh` exists and starts with `set -e` +- [ ] `OPIK_PROJECT_NAME` is set — exported in `run.sh` (scripts) or defined in `config.py` (use-cases/guides) +- [ ] Examples that call LLMs use litellm and read `OPIK_EXAMPLES_MODEL` in `config.py` ## Questions diff --git a/templates/script-template/run.sh b/templates/script-template/run.sh new file mode 100755 index 0000000..a914ac7 --- /dev/null +++ b/templates/script-template/run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +# Entry point CI runs for this example. With no Opik credentials it falls back to +# DRY_RUN and exits 0 (the secrets-free check); with credentials set it runs live +# and logs traces to Opik. +uv sync +export OPIK_PROJECT_NAME="example-script" +uv run example-script diff --git a/templates/use-case-template/run.sh b/templates/use-case-template/run.sh new file mode 100755 index 0000000..56706de --- /dev/null +++ b/templates/use-case-template/run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +# Entry point CI runs for this example. `eval` exercises the full Opik loop +# (dataset + test suite + evaluation). With no Opik credentials it falls back to +# DRY_RUN and exits 0 (the secrets-free check); with credentials set it runs live +# and logs traces to Opik. OPIK_PROJECT_NAME is defined in config.py. +uv sync +uv run example-use-case eval diff --git a/templates/use-case-template/src/example_use_case/config.py b/templates/use-case-template/src/example_use_case/config.py index 870267d..4e44f8d 100644 --- a/templates/use-case-template/src/example_use_case/config.py +++ b/templates/use-case-template/src/example_use_case/config.py @@ -11,9 +11,10 @@ LLM_READY = bool(ANTHROPIC_API_KEY) # litellm model strings (Anthropic provider). Swap for any litellm-supported model. -GEN_MODEL = "anthropic/claude-sonnet-4-6" # the deployed app -JUDGE_MODEL = "anthropic/claude-sonnet-4-6" # LLM-as-judge for metrics -OPTIMIZER_MODEL = "anthropic/claude-sonnet-4-6" # meta-model that rewrites the prompt +# CI sets OPIK_EXAMPLES_MODEL to a cheap model; locally, leave it unset to use the full model. +GEN_MODEL = os.environ.get("OPIK_EXAMPLES_MODEL", "anthropic/claude-sonnet-4-6") # the deployed app +JUDGE_MODEL = GEN_MODEL # LLM-as-judge for metrics +OPTIMIZER_MODEL = GEN_MODEL # meta-model that rewrites the prompt DATASET_NAME = "example-use-case-eval" SUITE_NAME = "example-use-case-suite"