From 5dfe36c80fc6640dc9e7030a686ccc7e62e0422d Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Thu, 25 Jun 2026 12:35:48 +0200 Subject: [PATCH 1/7] ci: add PR test, scheduled runs, and compliance check workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces legacy e2e-tests.yml and test-scripts-matrix.yml with three focused workflows: - pr-examples-test.yml: detects changed example folders that have run_examples.sh, runs each in a matrix job using real Opik credentials and a cheap model (OPIK_EXAMPLES_MODEL). Blocks the PR on failure. - scheduled-examples.yml: weekly curated run (Monday 06:00 UTC) across Python 3.12 and 3.13. Example list lives in .github/ci-examples.json and is easy to edit. Also supports workflow_dispatch. - example-compliance-check.yml: secrets-free check on new example folders — enforces run_examples.sh with set -e and OPIK_PROJECT_NAME, pyproject.toml (no requirements.txt), README.md, and OPIK_EXAMPLES_MODEL if litellm is a dependency. Updates CONTRIBUTING.md to document the run_examples.sh convention, the litellm/OPIK_EXAMPLES_MODEL pattern for LLM-calling examples, and CI credential expectations. Required GitHub secrets/variables before workflows go live: Secrets: OPIK_API_KEY, OPENAI_API_KEY, ANTHROPIC_API_KEY Variables: OPIK_WORKSPACE, OPIK_EXAMPLES_MODEL, OPIK_ENVIRONMENT Co-Authored-By: Claude Sonnet 4.6 --- .github/ci-examples.json | 1 + .../workflows/example-compliance-check.yml | 160 ++++++++++++++++++ .github/workflows/pr-examples-test.yml | 107 ++++++++++++ .github/workflows/scheduled-examples.yml | 59 +++++++ CONTRIBUTING.md | 76 ++++++++- 5 files changed, 400 insertions(+), 3 deletions(-) create mode 100644 .github/ci-examples.json create mode 100644 .github/workflows/example-compliance-check.yml create mode 100644 .github/workflows/pr-examples-test.yml create mode 100644 .github/workflows/scheduled-examples.yml diff --git a/.github/ci-examples.json b/.github/ci-examples.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/.github/ci-examples.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/.github/workflows/example-compliance-check.yml b/.github/workflows/example-compliance-check.yml new file mode 100644 index 0000000..0aa6b7e --- /dev/null +++ b/.github/workflows/example-compliance-check.yml @@ -0,0 +1,160 @@ +name: Example Compliance Check + +# Verifies that new example folders added in a PR follow repo conventions. +# Only checks folders that are new (not previously on the base branch). +# Rules enforced: +# 1. run_examples.sh must exist, have set -e, and export OPIK_PROJECT_NAME +# 2. README.md must exist +# 3. pyproject.toml must exist; requirements.txt must not +# 4. If litellm is a dependency, OPIK_EXAMPLES_MODEL must be referenced in a .py file +# +# This check is secrets-free so it runs safely on PRs from forks. + +on: + pull_request: + paths: + - "examples/**" + - "integrations/**" + - "scripts/**" + - "use-cases/**" + - "guides/**" + +permissions: + contents: read + +jobs: + compliance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Find changed example folders + id: changed-folders + run: | + changed_files=$(git diff --name-only "origin/${{ github.base_ref }}...HEAD") + + folders="" + while IFS= read -r file; do + top=$(echo "$file" | cut -d'/' -f1) + case "$top" in + integrations) + parent=$(echo "$file" | cut -d'/' -f1,2,3) + ;; + examples|scripts|use-cases|guides) + parent=$(echo "$file" | cut -d'/' -f1,2) + ;; + *) + continue + ;; + esac + folders="$folders $parent" + done <<< "$changed_files" + + # Deduplicate and trim + folders=$(echo "$folders" | tr ' ' '\n' | sort -u | grep -v '^$' | tr '\n' ' ' | xargs) + echo "folders=$folders" >> "$GITHUB_OUTPUT" + + if [[ -z "$folders" ]]; then + echo "No example folders changed." + else + echo "Changed example folders: $folders" + fi + + - name: Check compliance + if: steps.changed-folders.outputs.folders != '' + run: | + FAILED=0 + + for folder in ${{ steps.changed-folders.outputs.folders }}; do + # Skip if the folder was deleted entirely + if [[ ! -d "$folder" ]]; then + echo "Skipping deleted folder: $folder" + continue + fi + + # Only enforce on NEW folders (not yet on the base branch). + # Modifying an existing example does not require adding run_examples.sh. + if git ls-tree "origin/${{ github.base_ref }}" "$folder" 2>/dev/null | grep -q .; then + echo " Skipping '$folder' (existing folder — compliance enforced on new examples only)" + continue + fi + + echo "" + echo "Checking: $folder" + + if [[ ! -f "$folder/run_examples.sh" ]]; then + echo "::error file=$folder/run_examples.sh::$folder is missing run_examples.sh" + FAILED=1 + else + echo " OK run_examples.sh" + fi + + if [[ ! -f "$folder/README.md" ]]; then + echo "::error file=$folder/README.md::$folder is missing README.md" + FAILED=1 + else + echo " OK README.md" + fi + + if [[ ! -f "$folder/pyproject.toml" ]]; then + echo "::error::$folder is missing pyproject.toml (uv projects only — no requirements.txt)" + FAILED=1 + else + echo " OK pyproject.toml" + fi + + if [[ -f "$folder/requirements.txt" ]]; then + echo "::error file=$folder/requirements.txt::$folder has requirements.txt — declare deps in pyproject.toml instead" + FAILED=1 + fi + + if [[ -f "$folder/run_examples.sh" ]]; then + if grep -q "export OPIK_PROJECT_NAME" "$folder/run_examples.sh"; then + echo " OK OPIK_PROJECT_NAME in run_examples.sh" + else + echo "::error file=$folder/run_examples.sh::$folder/run_examples.sh must export OPIK_PROJECT_NAME" + FAILED=1 + fi + + if ! grep -q "set -e" "$folder/run_examples.sh"; then + echo "::error file=$folder/run_examples.sh::$folder/run_examples.sh must contain 'set -e'" + FAILED=1 + else + echo " OK set -e in run_examples.sh" + fi + fi + + # If litellm is declared as a dependency, OPIK_EXAMPLES_MODEL must be referenced. + if grep -q "litellm" "$folder/pyproject.toml" 2>/dev/null; then + if grep -rq "OPIK_EXAMPLES_MODEL" "$folder" --include="*.py" 2>/dev/null; then + echo " OK OPIK_EXAMPLES_MODEL referenced (litellm dep detected)" + else + echo "::error::$folder uses litellm but no .py file references OPIK_EXAMPLES_MODEL" + echo "::error::Add: os.environ.get(\"OPIK_EXAMPLES_MODEL\", \"\") to your config" + FAILED=1 + fi + fi + done + + echo "" + if [[ "$FAILED" -ne 0 ]]; then + echo "Compliance check failed. Fix the errors above before merging." + exit 1 + fi + + echo "All compliance checks passed." + + - name: Write step summary + if: always() && steps.changed-folders.outputs.folders != '' + run: | + echo "## Example Compliance Check" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "Folders checked: \`${{ steps.changed-folders.outputs.folders }}\`" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**Rules:**" >> "$GITHUB_STEP_SUMMARY" + echo "- \`run_examples.sh\` must exist, contain \`set -e\`, and export \`OPIK_PROJECT_NAME\`" >> "$GITHUB_STEP_SUMMARY" + echo "- \`README.md\` must exist" >> "$GITHUB_STEP_SUMMARY" + echo "- \`pyproject.toml\` must exist; \`requirements.txt\` is not allowed" >> "$GITHUB_STEP_SUMMARY" + echo "- If \`litellm\` is a dependency, a \`.py\` file must reference \`OPIK_EXAMPLES_MODEL\`" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml new file mode 100644 index 0000000..6157629 --- /dev/null +++ b/.github/workflows/pr-examples-test.yml @@ -0,0 +1,107 @@ +name: PR Example Tests + +# Runs only the example folders changed in this PR using a cheap model and real Opik +# credentials. A passing run means the code executes correctly and traces are visible +# in the opik-examples workspace. Failures block the PR. + +on: + pull_request: + paths: + - "examples/**" + - "integrations/**" + - "scripts/**" + - "use-cases/**" + - "guides/**" + +permissions: + contents: read + +jobs: + detect-changes: + runs-on: ubuntu-latest + outputs: + folders: ${{ steps.detect.outputs.folders }} + has_changes: ${{ steps.detect.outputs.has_changes }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Detect changed runnable example folders + id: detect + run: | + changed_files=$(git diff --name-only "origin/${{ github.base_ref }}...HEAD") + + seen=() + folders_json="[" + + while IFS= read -r file; do + top=$(echo "$file" | cut -d'/' -f1) + case "$top" in + integrations) + folder=$(echo "$file" | cut -d'/' -f1,2,3) + ;; + examples|scripts|use-cases|guides) + folder=$(echo "$file" | cut -d'/' -f1,2) + ;; + *) + continue + ;; + esac + + if [[ -z "$folder" ]] || [[ ! -d "$folder" ]] || [[ ! -f "$folder/run_examples.sh" ]]; then + continue + fi + + already_seen=false + for s in "${seen[@]}"; do + [[ "$s" == "$folder" ]] && already_seen=true && break + done + + if [[ "$already_seen" == false ]]; then + seen+=("$folder") + [[ "${#seen[@]}" -gt 1 ]] && folders_json="$folders_json," + folders_json="$folders_json\"$folder\"" + fi + done <<< "$changed_files" + + folders_json="$folders_json]" + + if [[ "${#seen[@]}" -eq 0 ]]; then + echo "folders=[]" >> "$GITHUB_OUTPUT" + echo "has_changes=false" >> "$GITHUB_OUTPUT" + echo "No runnable example folders changed." + else + echo "folders=$folders_json" >> "$GITHUB_OUTPUT" + echo "has_changes=true" >> "$GITHUB_OUTPUT" + echo "Runnable folders to test: ${seen[*]}" + fi + + test-changed: + needs: detect-changes + if: needs.detect-changes.outputs.has_changes == 'true' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + folder: ${{ fromJson(needs.detect-changes.outputs.folders) }} + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + version: "latest" + enable-cache: true + python-version: "3.12" + + - name: Run example + env: + OPIK_API_KEY: ${{ secrets.OPIK_API_KEY }} + OPIK_WORKSPACE: ${{ vars.OPIK_WORKSPACE }} + OPIK_ENVIRONMENT: ${{ vars.OPIK_ENVIRONMENT }} + OPIK_EXAMPLES_MODEL: ${{ vars.OPIK_EXAMPLES_MODEL }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + cd ${{ matrix.folder }} + bash run_examples.sh diff --git a/.github/workflows/scheduled-examples.yml b/.github/workflows/scheduled-examples.yml new file mode 100644 index 0000000..7f3b3a8 --- /dev/null +++ b/.github/workflows/scheduled-examples.yml @@ -0,0 +1,59 @@ +name: Scheduled Example Tests + +# Runs a curated set of examples on a schedule to verify they stay working. +# The list of examples is in .github/ci-examples.json. +# +# To change the schedule: edit the cron expression below. +# To add or remove examples: edit .github/ci-examples.json. + +on: + schedule: + # Every Monday at 06:00 UTC. Edit this line to change the cadence. + - cron: "0 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + load-examples: + runs-on: ubuntu-latest + outputs: + folders: ${{ steps.load.outputs.folders }} + steps: + - uses: actions/checkout@v4 + + - name: Load curated example list + id: load + run: | + folders=$(cat .github/ci-examples.json) + echo "folders=$folders" >> "$GITHUB_OUTPUT" + + run-examples: + needs: load-examples + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + folder: ${{ fromJson(needs.load-examples.outputs.folders) }} + python-version: ["3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + version: "latest" + enable-cache: true + python-version: ${{ matrix.python-version }} + + - name: Run example + env: + OPIK_API_KEY: ${{ secrets.OPIK_API_KEY }} + OPIK_WORKSPACE: ${{ vars.OPIK_WORKSPACE }} + OPIK_ENVIRONMENT: ${{ vars.OPIK_ENVIRONMENT }} + OPIK_EXAMPLES_MODEL: ${{ vars.OPIK_EXAMPLES_MODEL }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + cd ${{ matrix.folder }} + bash run_examples.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 176a35a..e0196e1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,6 +86,7 @@ Then, either way: - A `README.md` that explains what problem it solves, what the example does, and how to run it - A working dry-run mode so anyone can run it locally without an Opik account - Credentials loaded from environment variables only — no hardcoded keys +- A `run_examples.sh` that exports `OPIK_PROJECT_NAME` and can run the example end-to-end ### README structure @@ -96,9 +97,9 @@ Use the template's README as a guide. Required sections: - **Running it** — exact commands, including dry-run - **How it works** — brief walkthrough of the key steps -### Dry-run mode +### Running locally without credentials -Every runnable script should detect missing credentials and fall back to a local mode that prints output to the console instead of sending it to Opik. The standard pattern: +CI always has real Opik credentials. If you want to smoke-test an example locally before obtaining credentials, you can add a dry-run guard — but this is optional and not enforced: ```python DRY_RUN = not (os.environ.get("OPIK_API_KEY") and os.environ.get("OPIK_WORKSPACE")) @@ -109,7 +110,7 @@ else: # send to Opik ``` -The dry-run output should be meaningful enough to verify the example is working correctly. +The recommended path is to set your own Opik credentials locally so you see real traces, the same way CI does. ### Credentials @@ -128,6 +129,73 @@ Never commit `.env` files or hardcoded keys. Add `.env` to the example's `.gitig Each example is a `uv` project: declare dependencies in its `pyproject.toml` (the single source of truth) and run with `uv sync` / `uv run`. No `requirements.txt`, no Poetry, and don't commit `uv.lock`. The README may still show an optional `pip install` fallback line. Do not assume the user has the repo's root virtualenv set up. +### run_examples.sh + +Every testable example must include a `run_examples.sh` at its root. This file is what the CI matrix runs. Requirements: + +- Start with `set -e` (fail fast on any error) +- Export `OPIK_PROJECT_NAME` to a unique kebab-case name (enforced by the compliance check) +- Call `uv sync` before invoking any Python + +```bash +#!/usr/bin/env bash +set -e + +export OPIK_PROJECT_NAME="my-example" + +uv sync +uv run my-example run-all +``` + +The scaffold tool adds a working `run_examples.sh` automatically when you create a new example. + +### Opik workspace + +All CI runs log to the workspace set in the `OPIK_WORKSPACE` GitHub Actions variable (currently `opik-examples`). Locally, set both vars explicitly to avoid accidentally writing to the shared workspace: + +```bash +export OPIK_API_KEY= +export OPIK_WORKSPACE= +``` + +`DRY_RUN` gates on both being set, so forgetting either keeps you in dry-run mode — safe by default. The `OPIK_API_KEY` Actions secret must be a service account key with write access to `opik-examples`. + +### CI model convention + +CI uses a cheap model to keep costs low. The `OPIK_EXAMPLES_MODEL` GitHub Actions variable controls which model is used (currently `openai/gpt-4o-mini`). All examples that make LLM calls must read this variable. + +Use [litellm](https://github.com/BerriAI/litellm) as the model call layer — it routes to any provider via a single unified model name: + +```python +import litellm +response = litellm.completion(model=config.GEN_MODEL, messages=[...]) +``` + +In `config.py`, read `OPIK_EXAMPLES_MODEL` with a sensible default for local development: + +```python +# CI sets OPIK_EXAMPLES_MODEL to a cheap model (e.g. openai/gpt-4o-mini). +# Locally, leave it unset to use the full model. +GEN_MODEL = os.environ.get("OPIK_EXAMPLES_MODEL", "anthropic/claude-sonnet-4-6") +JUDGE_MODEL = GEN_MODEL +OPTIMIZER_MODEL = GEN_MODEL +``` + +Model names use litellm's provider-prefixed format: `openai/gpt-4o-mini`, `anthropic/claude-haiku-4-5-20251001`, `google/gemini-1.5-flash`, etc. The compliance check will fail if `litellm` is declared as a dependency but `OPIK_EXAMPLES_MODEL` is not referenced. + +### Opik logging + +CI always has real Opik credentials (`OPIK_API_KEY`, `OPIK_WORKSPACE`, `OPIK_ENVIRONMENT`). Every CI run logs traces to the `opik-examples` workspace — this is how we verify an example is working, not just that it exits 0. + +`OPIK_ENVIRONMENT` is set automatically in CI; you do not need to set it locally. It tags traces so CI runs are distinguishable from local runs in the Opik UI. + +Locally, set your own credentials to log to your personal workspace: + +```bash +export OPIK_API_KEY= +export OPIK_WORKSPACE= +``` + ### Code style - No comments that explain what the code does — well-named variables and functions do that @@ -145,6 +213,8 @@ Before opening a PR, verify: - [ ] Dry-run mode works (no env vars set) prints useful output without errors — e.g. `uv run eval` - [ ] No credentials or `.env` files committed - [ ] Dependencies declared in `pyproject.toml` (uv project); no `requirements.txt` +- [ ] `run_examples.sh` exists, starts with `set -e`, and exports `OPIK_PROJECT_NAME` +- [ ] Examples that call LLMs use litellm and read `OPIK_EXAMPLES_MODEL` in `config.py` ## Questions From 50eeb777cd64301b5a37db77ae949f1700827a1b Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Thu, 25 Jun 2026 15:28:33 +0200 Subject: [PATCH 2/7] ci: allow OPIK_PROJECT_NAME in config.py as alternative to run_examples.sh export Scripts (single .py, no config module) set it via export in run_examples.sh. Use-cases and guides with a config.py define it as a Python constant and pass it explicitly to opik.track(). The compliance check accepts either pattern. Updates CONTRIBUTING.md and the step summary to document both approaches. Co-Authored-By: Claude Sonnet 4.6 --- .../workflows/example-compliance-check.yml | 31 +++++++++++------- CONTRIBUTING.md | 32 ++++++++++++++++--- 2 files changed, 47 insertions(+), 16 deletions(-) diff --git a/.github/workflows/example-compliance-check.yml b/.github/workflows/example-compliance-check.yml index 0aa6b7e..32dfa13 100644 --- a/.github/workflows/example-compliance-check.yml +++ b/.github/workflows/example-compliance-check.yml @@ -3,10 +3,12 @@ name: Example Compliance Check # Verifies that new example folders added in a PR follow repo conventions. # Only checks folders that are new (not previously on the base branch). # Rules enforced: -# 1. run_examples.sh must exist, have set -e, and export OPIK_PROJECT_NAME -# 2. README.md must exist -# 3. pyproject.toml must exist; requirements.txt must not -# 4. If litellm is a dependency, OPIK_EXAMPLES_MODEL must be referenced in a .py file +# 1. run_examples.sh must exist and contain set -e +# 2. OPIK_PROJECT_NAME must be set — either exported in run_examples.sh (scripts) +# or referenced in a .py file (use-cases/guides with a config module) +# 3. README.md must exist +# 4. pyproject.toml must exist; requirements.txt must not +# 5. If litellm is a dependency, OPIK_EXAMPLES_MODEL must be referenced in a .py file # # This check is secrets-free so it runs safely on PRs from forks. @@ -111,19 +113,23 @@ jobs: fi if [[ -f "$folder/run_examples.sh" ]]; then - if grep -q "export OPIK_PROJECT_NAME" "$folder/run_examples.sh"; then - echo " OK OPIK_PROJECT_NAME in run_examples.sh" - else - echo "::error file=$folder/run_examples.sh::$folder/run_examples.sh must export OPIK_PROJECT_NAME" - FAILED=1 - fi - if ! grep -q "set -e" "$folder/run_examples.sh"; then echo "::error file=$folder/run_examples.sh::$folder/run_examples.sh must contain 'set -e'" FAILED=1 else echo " OK set -e in run_examples.sh" fi + + # OPIK_PROJECT_NAME may be exported in run_examples.sh (scripts) + # or defined as a constant in a .py file (use-cases/guides with config.py). + if grep -q "export OPIK_PROJECT_NAME" "$folder/run_examples.sh"; then + echo " OK OPIK_PROJECT_NAME in run_examples.sh" + elif grep -rq "OPIK_PROJECT_NAME" "$folder" --include="*.py" 2>/dev/null; then + echo " OK OPIK_PROJECT_NAME in .py config" + else + echo "::error::$folder must set OPIK_PROJECT_NAME — either export it in run_examples.sh or define it in a .py config file" + FAILED=1 + fi fi # If litellm is declared as a dependency, OPIK_EXAMPLES_MODEL must be referenced. @@ -154,7 +160,8 @@ jobs: echo "Folders checked: \`${{ steps.changed-folders.outputs.folders }}\`" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" echo "**Rules:**" >> "$GITHUB_STEP_SUMMARY" - echo "- \`run_examples.sh\` must exist, contain \`set -e\`, and export \`OPIK_PROJECT_NAME\`" >> "$GITHUB_STEP_SUMMARY" + echo "- \`run_examples.sh\` must exist and contain \`set -e\`" >> "$GITHUB_STEP_SUMMARY" + echo "- \`OPIK_PROJECT_NAME\` must be set — via \`export\` in \`run_examples.sh\` (scripts) or as a constant in a \`.py\` config file (use-cases/guides)" >> "$GITHUB_STEP_SUMMARY" echo "- \`README.md\` must exist" >> "$GITHUB_STEP_SUMMARY" echo "- \`pyproject.toml\` must exist; \`requirements.txt\` is not allowed" >> "$GITHUB_STEP_SUMMARY" echo "- If \`litellm\` is a dependency, a \`.py\` file must reference \`OPIK_EXAMPLES_MODEL\`" >> "$GITHUB_STEP_SUMMARY" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e0196e1..09d3465 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -134,21 +134,44 @@ Each example is a `uv` project: declare dependencies in its `pyproject.toml` (th Every testable example must include a `run_examples.sh` at its root. This file is what the CI matrix runs. Requirements: - Start with `set -e` (fail fast on any error) -- Export `OPIK_PROJECT_NAME` to a unique kebab-case name (enforced by the compliance check) - Call `uv sync` before invoking any Python +- `OPIK_PROJECT_NAME` must be set — see below for where ```bash #!/usr/bin/env bash set -e -export OPIK_PROJECT_NAME="my-example" - uv sync uv run my-example run-all ``` The scaffold tool adds a working `run_examples.sh` automatically when you create a new example. +### Setting OPIK_PROJECT_NAME + +Every example must have a unique project name so its traces are identifiable in Opik. Where you set it depends on the example type: + +**Scripts** (single `.py` file, no config module) — export it in `run_examples.sh`: + +```bash +export OPIK_PROJECT_NAME="my-script" +``` + +**Use-cases and guides** (examples with a `config.py`) — define it as a Python constant and pass it explicitly. This keeps all configuration in one place and works even when running Python directly: + +```python +# config.py +OPIK_PROJECT_NAME = os.environ.get("OPIK_PROJECT_NAME", "my-use-case") +``` + +```python +# app.py +@opik.track(project_name=config.OPIK_PROJECT_NAME) +def my_function(): ... +``` + +The compliance check accepts either pattern — it looks for `OPIK_PROJECT_NAME` in `run_examples.sh` or in any `.py` file in the folder. + ### Opik workspace All CI runs log to the workspace set in the `OPIK_WORKSPACE` GitHub Actions variable (currently `opik-examples`). Locally, set both vars explicitly to avoid accidentally writing to the shared workspace: @@ -213,7 +236,8 @@ Before opening a PR, verify: - [ ] Dry-run mode works (no env vars set) prints useful output without errors — e.g. `uv run eval` - [ ] No credentials or `.env` files committed - [ ] Dependencies declared in `pyproject.toml` (uv project); no `requirements.txt` -- [ ] `run_examples.sh` exists, starts with `set -e`, and exports `OPIK_PROJECT_NAME` +- [ ] `run_examples.sh` exists and starts with `set -e` +- [ ] `OPIK_PROJECT_NAME` is set — exported in `run_examples.sh` (scripts) or defined in `config.py` (use-cases/guides) - [ ] Examples that call LLMs use litellm and read `OPIK_EXAMPLES_MODEL` in `config.py` ## Questions From 224ffc74a83fdb5e5cb9f6328371f8cb98971053 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Thu, 25 Jun 2026 18:39:21 +0200 Subject: [PATCH 3/7] ci: document that ci-examples.json is maintainer-managed Clarifies inclusion criteria (self-contained, cheap, no GPU, no seed data) and that contributors do not need to add their example to the scheduled list. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/scheduled-examples.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scheduled-examples.yml b/.github/workflows/scheduled-examples.yml index 7f3b3a8..9e00433 100644 --- a/.github/workflows/scheduled-examples.yml +++ b/.github/workflows/scheduled-examples.yml @@ -1,7 +1,9 @@ name: Scheduled Example Tests # Runs a curated set of examples on a schedule to verify they stay working. -# The list of examples is in .github/ci-examples.json. +# The list is maintainer-managed — contributors do not need to add their example here. +# Criteria for inclusion: self-contained (no GPU, no pre-existing Opik data required), +# cheap to run, and stable enough for unattended weekly execution. # # To change the schedule: edit the cron expression below. # To add or remove examples: edit .github/ci-examples.json. From f80fc12317d05d252af32ccd9331d793e568a8d2 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 26 Jun 2026 11:26:02 +0200 Subject: [PATCH 4/7] ci: rename run_examples.sh to run.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clearer name — each example folder has exactly one run.sh scoped to that folder, so there is no ambiguity about what it runs. Co-Authored-By: Claude Sonnet 4.6 --- .../workflows/example-compliance-check.yml | 32 +++++++++---------- .github/workflows/pr-examples-test.yml | 4 +-- .github/workflows/scheduled-examples.yml | 2 +- CONTRIBUTING.md | 16 +++++----- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/example-compliance-check.yml b/.github/workflows/example-compliance-check.yml index 32dfa13..7ec5687 100644 --- a/.github/workflows/example-compliance-check.yml +++ b/.github/workflows/example-compliance-check.yml @@ -3,8 +3,8 @@ name: Example Compliance Check # Verifies that new example folders added in a PR follow repo conventions. # Only checks folders that are new (not previously on the base branch). # Rules enforced: -# 1. run_examples.sh must exist and contain set -e -# 2. OPIK_PROJECT_NAME must be set — either exported in run_examples.sh (scripts) +# 1. run.sh must exist and contain set -e +# 2. OPIK_PROJECT_NAME must be set — either exported in run.sh (scripts) # or referenced in a .py file (use-cases/guides with a config module) # 3. README.md must exist # 4. pyproject.toml must exist; requirements.txt must not @@ -77,7 +77,7 @@ jobs: fi # Only enforce on NEW folders (not yet on the base branch). - # Modifying an existing example does not require adding run_examples.sh. + # Modifying an existing example does not require adding run.sh. if git ls-tree "origin/${{ github.base_ref }}" "$folder" 2>/dev/null | grep -q .; then echo " Skipping '$folder' (existing folder — compliance enforced on new examples only)" continue @@ -86,11 +86,11 @@ jobs: echo "" echo "Checking: $folder" - if [[ ! -f "$folder/run_examples.sh" ]]; then - echo "::error file=$folder/run_examples.sh::$folder is missing run_examples.sh" + if [[ ! -f "$folder/run.sh" ]]; then + echo "::error file=$folder/run.sh::$folder is missing run.sh" FAILED=1 else - echo " OK run_examples.sh" + echo " OK run.sh" fi if [[ ! -f "$folder/README.md" ]]; then @@ -112,22 +112,22 @@ jobs: FAILED=1 fi - if [[ -f "$folder/run_examples.sh" ]]; then - if ! grep -q "set -e" "$folder/run_examples.sh"; then - echo "::error file=$folder/run_examples.sh::$folder/run_examples.sh must contain 'set -e'" + if [[ -f "$folder/run.sh" ]]; then + if ! grep -q "set -e" "$folder/run.sh"; then + echo "::error file=$folder/run.sh::$folder/run.sh must contain 'set -e'" FAILED=1 else - echo " OK set -e in run_examples.sh" + echo " OK set -e in run.sh" fi - # OPIK_PROJECT_NAME may be exported in run_examples.sh (scripts) + # OPIK_PROJECT_NAME may be exported in run.sh (scripts) # or defined as a constant in a .py file (use-cases/guides with config.py). - if grep -q "export OPIK_PROJECT_NAME" "$folder/run_examples.sh"; then - echo " OK OPIK_PROJECT_NAME in run_examples.sh" + if grep -q "export OPIK_PROJECT_NAME" "$folder/run.sh"; then + echo " OK OPIK_PROJECT_NAME in run.sh" elif grep -rq "OPIK_PROJECT_NAME" "$folder" --include="*.py" 2>/dev/null; then echo " OK OPIK_PROJECT_NAME in .py config" else - echo "::error::$folder must set OPIK_PROJECT_NAME — either export it in run_examples.sh or define it in a .py config file" + echo "::error::$folder must set OPIK_PROJECT_NAME — either export it in run.sh or define it in a .py config file" FAILED=1 fi fi @@ -160,8 +160,8 @@ jobs: echo "Folders checked: \`${{ steps.changed-folders.outputs.folders }}\`" >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" echo "**Rules:**" >> "$GITHUB_STEP_SUMMARY" - echo "- \`run_examples.sh\` must exist and contain \`set -e\`" >> "$GITHUB_STEP_SUMMARY" - echo "- \`OPIK_PROJECT_NAME\` must be set — via \`export\` in \`run_examples.sh\` (scripts) or as a constant in a \`.py\` config file (use-cases/guides)" >> "$GITHUB_STEP_SUMMARY" + echo "- \`run.sh\` must exist and contain \`set -e\`" >> "$GITHUB_STEP_SUMMARY" + echo "- \`OPIK_PROJECT_NAME\` must be set — via \`export\` in \`run.sh\` (scripts) or as a constant in a \`.py\` config file (use-cases/guides)" >> "$GITHUB_STEP_SUMMARY" echo "- \`README.md\` must exist" >> "$GITHUB_STEP_SUMMARY" echo "- \`pyproject.toml\` must exist; \`requirements.txt\` is not allowed" >> "$GITHUB_STEP_SUMMARY" echo "- If \`litellm\` is a dependency, a \`.py\` file must reference \`OPIK_EXAMPLES_MODEL\`" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-examples-test.yml index 6157629..9a3c8ed 100644 --- a/.github/workflows/pr-examples-test.yml +++ b/.github/workflows/pr-examples-test.yml @@ -49,7 +49,7 @@ jobs: ;; esac - if [[ -z "$folder" ]] || [[ ! -d "$folder" ]] || [[ ! -f "$folder/run_examples.sh" ]]; then + if [[ -z "$folder" ]] || [[ ! -d "$folder" ]] || [[ ! -f "$folder/run.sh" ]]; then continue fi @@ -104,4 +104,4 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: | cd ${{ matrix.folder }} - bash run_examples.sh + bash run.sh diff --git a/.github/workflows/scheduled-examples.yml b/.github/workflows/scheduled-examples.yml index 9e00433..3ff762c 100644 --- a/.github/workflows/scheduled-examples.yml +++ b/.github/workflows/scheduled-examples.yml @@ -58,4 +58,4 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: | cd ${{ matrix.folder }} - bash run_examples.sh + bash run.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 09d3465..9311541 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,7 +86,7 @@ Then, either way: - A `README.md` that explains what problem it solves, what the example does, and how to run it - A working dry-run mode so anyone can run it locally without an Opik account - Credentials loaded from environment variables only — no hardcoded keys -- A `run_examples.sh` that exports `OPIK_PROJECT_NAME` and can run the example end-to-end +- A `run.sh` that exports `OPIK_PROJECT_NAME` and can run the example end-to-end ### README structure @@ -129,9 +129,9 @@ Never commit `.env` files or hardcoded keys. Add `.env` to the example's `.gitig Each example is a `uv` project: declare dependencies in its `pyproject.toml` (the single source of truth) and run with `uv sync` / `uv run`. No `requirements.txt`, no Poetry, and don't commit `uv.lock`. The README may still show an optional `pip install` fallback line. Do not assume the user has the repo's root virtualenv set up. -### run_examples.sh +### run.sh -Every testable example must include a `run_examples.sh` at its root. This file is what the CI matrix runs. Requirements: +Every testable example must include a `run.sh` at its root. This file is what the CI matrix runs. Requirements: - Start with `set -e` (fail fast on any error) - Call `uv sync` before invoking any Python @@ -145,13 +145,13 @@ uv sync uv run my-example run-all ``` -The scaffold tool adds a working `run_examples.sh` automatically when you create a new example. +The scaffold tool adds a working `run.sh` automatically when you create a new example. ### Setting OPIK_PROJECT_NAME Every example must have a unique project name so its traces are identifiable in Opik. Where you set it depends on the example type: -**Scripts** (single `.py` file, no config module) — export it in `run_examples.sh`: +**Scripts** (single `.py` file, no config module) — export it in `run.sh`: ```bash export OPIK_PROJECT_NAME="my-script" @@ -170,7 +170,7 @@ OPIK_PROJECT_NAME = os.environ.get("OPIK_PROJECT_NAME", "my-use-case") def my_function(): ... ``` -The compliance check accepts either pattern — it looks for `OPIK_PROJECT_NAME` in `run_examples.sh` or in any `.py` file in the folder. +The compliance check accepts either pattern — it looks for `OPIK_PROJECT_NAME` in `run.sh` or in any `.py` file in the folder. ### Opik workspace @@ -236,8 +236,8 @@ Before opening a PR, verify: - [ ] Dry-run mode works (no env vars set) prints useful output without errors — e.g. `uv run eval` - [ ] No credentials or `.env` files committed - [ ] Dependencies declared in `pyproject.toml` (uv project); no `requirements.txt` -- [ ] `run_examples.sh` exists and starts with `set -e` -- [ ] `OPIK_PROJECT_NAME` is set — exported in `run_examples.sh` (scripts) or defined in `config.py` (use-cases/guides) +- [ ] `run.sh` exists and starts with `set -e` +- [ ] `OPIK_PROJECT_NAME` is set — exported in `run.sh` (scripts) or defined in `config.py` (use-cases/guides) - [ ] Examples that call LLMs use litellm and read `OPIK_EXAMPLES_MODEL` in `config.py` ## Questions From b287e9994b7f81613d67d96082c5f0699474b57b Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 26 Jun 2026 12:11:53 +0200 Subject: [PATCH 5/7] docs: update CI model example to anthropic/claude-haiku-4-5-20251001 --- CONTRIBUTING.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9311541..6f45dc4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -185,7 +185,7 @@ export OPIK_WORKSPACE= ### CI model convention -CI uses a cheap model to keep costs low. The `OPIK_EXAMPLES_MODEL` GitHub Actions variable controls which model is used (currently `openai/gpt-4o-mini`). All examples that make LLM calls must read this variable. +CI uses a cheap model to keep costs low. The `OPIK_EXAMPLES_MODEL` GitHub Actions variable controls which model is used (currently `anthropic/claude-haiku-4-5-20251001`). All examples that make LLM calls must read this variable. Use [litellm](https://github.com/BerriAI/litellm) as the model call layer — it routes to any provider via a single unified model name: @@ -197,7 +197,7 @@ response = litellm.completion(model=config.GEN_MODEL, messages=[...]) In `config.py`, read `OPIK_EXAMPLES_MODEL` with a sensible default for local development: ```python -# CI sets OPIK_EXAMPLES_MODEL to a cheap model (e.g. openai/gpt-4o-mini). +# CI sets OPIK_EXAMPLES_MODEL to a cheap model (e.g. anthropic/claude-haiku-4-5-20251001). # Locally, leave it unset to use the full model. GEN_MODEL = os.environ.get("OPIK_EXAMPLES_MODEL", "anthropic/claude-sonnet-4-6") JUDGE_MODEL = GEN_MODEL From 4148f29dd7a40254474fd44aef749a00fca70e25 Mon Sep 17 00:00:00 2001 From: LeoRoccoBreedt Date: Fri, 26 Jun 2026 12:43:25 +0200 Subject: [PATCH 6/7] chore: rename workflows to drop 'example' prefix, single Python version in scheduled runs --- .github/{ci-examples.json => scheduled.json} | 0 .../{example-compliance-check.yml => compliance.yml} | 0 .github/workflows/{pr-examples-test.yml => pr-test.yml} | 0 .../workflows/{scheduled-examples.yml => scheduled.yml} | 7 +++---- 4 files changed, 3 insertions(+), 4 deletions(-) rename .github/{ci-examples.json => scheduled.json} (100%) rename .github/workflows/{example-compliance-check.yml => compliance.yml} (100%) rename .github/workflows/{pr-examples-test.yml => pr-test.yml} (100%) rename .github/workflows/{scheduled-examples.yml => scheduled.yml} (88%) diff --git a/.github/ci-examples.json b/.github/scheduled.json similarity index 100% rename from .github/ci-examples.json rename to .github/scheduled.json diff --git a/.github/workflows/example-compliance-check.yml b/.github/workflows/compliance.yml similarity index 100% rename from .github/workflows/example-compliance-check.yml rename to .github/workflows/compliance.yml diff --git a/.github/workflows/pr-examples-test.yml b/.github/workflows/pr-test.yml similarity index 100% rename from .github/workflows/pr-examples-test.yml rename to .github/workflows/pr-test.yml diff --git a/.github/workflows/scheduled-examples.yml b/.github/workflows/scheduled.yml similarity index 88% rename from .github/workflows/scheduled-examples.yml rename to .github/workflows/scheduled.yml index 3ff762c..176de83 100644 --- a/.github/workflows/scheduled-examples.yml +++ b/.github/workflows/scheduled.yml @@ -6,7 +6,7 @@ name: Scheduled Example Tests # cheap to run, and stable enough for unattended weekly execution. # # To change the schedule: edit the cron expression below. -# To add or remove examples: edit .github/ci-examples.json. +# To add or remove folders: edit .github/scheduled.json. on: schedule: @@ -28,7 +28,7 @@ jobs: - name: Load curated example list id: load run: | - folders=$(cat .github/ci-examples.json) + folders=$(cat .github/scheduled.json) echo "folders=$folders" >> "$GITHUB_OUTPUT" run-examples: @@ -38,7 +38,6 @@ jobs: fail-fast: false matrix: folder: ${{ fromJson(needs.load-examples.outputs.folders) }} - python-version: ["3.12", "3.13"] steps: - uses: actions/checkout@v4 @@ -46,7 +45,7 @@ jobs: with: version: "latest" enable-cache: true - python-version: ${{ matrix.python-version }} + python-version: "3.12" - name: Run example env: From 57f4b863da2de390ff30ba418d193d32ea8d98af Mon Sep 17 00:00:00 2001 From: Francisco Schulz <22344801+fschlz@users.noreply.github.com> Date: Mon, 29 Jun 2026 18:03:38 -0700 Subject: [PATCH 7/7] ci: harden PR workflows + align templates/scaffold with conventions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review of #16 (high/medium/low) and make the templates + scaffold consistent with the conventions this PR introduces, so a freshly scaffolded example passes its own compliance check. pr-test.yml: split the single test job into detect-changes (+has_secrets) / lint / dry-run / live-run. lint + dry-run are secrets-free, so fork PRs (which GitHub withholds secrets from) get real signal; live-run is gated on secrets being present. Add a concurrency group + per-job timeouts; quote the matrix folder. scheduled.yml: guard run-examples on a non-empty list — an empty matrix array errored ("Matrix vector 'folder' does not contain any values"), so the weekly cron and workflow_dispatch failed with the empty scheduled.json. Add a timeout; quote the folder. compliance.yml: match a quoted litellm dependency instead of a bare substring; group the step-summary redirects (shellcheck SC2129). actionlint now clean. scheduled.json: add a trailing newline. templates: add run.sh to both templates — it doubles as the secrets-free dry-run smoke test CI runs. use-case config.py now reads OPIK_EXAMPLES_MODEL, so a fresh use-case scaffold satisfies the new litellm compliance rule. scaffold.py rewrites run.sh's command on --command override and no longer prints a phantom "run-all" next step. CONTRIBUTING.md: dry-run is required again (the secrets-free CI job enforces it); fix the "CI always has real Opik credentials" claim (false for forks); fix the run.sh snippet; add ruff check/format to the checklist. .github: add pull_request_template.md (embeds the checklist), CODEOWNERS (workflows + scheduled.json are maintainer-managed), dependabot for actions. Verified: actionlint clean on all three workflows; scaffolded use-case and script projects pass ruff check + format and `bash run.sh` exits 0 in DRY_RUN with no credentials; compliance positive and negative cases behave as expected. Co-Authored-By: Claude Opus 4.8 (1M context) --- .claude/skills/scaffold-example/SKILL.md | 9 +- .../scaffold-example/scripts/scaffold.py | 12 ++- .github/CODEOWNERS | 6 ++ .github/dependabot.yml | 11 +++ .github/pull_request_template.md | 18 ++++ .github/scheduled.json | 2 +- .github/workflows/compliance.yml | 26 +++--- .github/workflows/pr-test.yml | 88 +++++++++++++++++-- .github/workflows/scheduled.yml | 6 +- CONTRIBUTING.md | 15 ++-- templates/script-template/run.sh | 9 ++ templates/use-case-template/run.sh | 9 ++ .../src/example_use_case/config.py | 7 +- 13 files changed, 184 insertions(+), 34 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 .github/dependabot.yml create mode 100644 .github/pull_request_template.md create mode 100755 templates/script-template/run.sh create mode 100755 templates/use-case-template/run.sh diff --git a/.claude/skills/scaffold-example/SKILL.md b/.claude/skills/scaffold-example/SKILL.md index 831a038..e654c5d 100644 --- a/.claude/skills/scaffold-example/SKILL.md +++ b/.claude/skills/scaffold-example/SKILL.md @@ -87,11 +87,14 @@ uv run eval # use-case: [DRY RUN] lists the dataset items + ass uv run optimize # use-case: [DRY RUN] line uv run --dry-run # script: [DRY RUN] line uv run ruff check . # clean +bash run.sh # the exact dry-run smoke test CI runs (exits 0 without creds) ``` -With credentials set (`ANTHROPIC_API_KEY`, `OPIK_API_KEY`, `OPIK_WORKSPACE`), the same commands -talk to Claude + Opik and `eval` prints a pass rate + experiment URL. Running the eval and -optimization workflows in depth is covered by the `run-evals` and `run-optimizations` skills. +The scaffold includes a working `run.sh` at the project root — this is what the CI `dry-run` and +`live-run` jobs execute. With credentials set (`ANTHROPIC_API_KEY`, `OPIK_API_KEY`, +`OPIK_WORKSPACE`), the same commands talk to Claude + Opik and `eval` prints a pass rate + +experiment URL. Running the eval and optimization workflows in depth is covered by the `run-evals` +and `run-optimizations` skills. ## What this skill does NOT do diff --git a/.claude/skills/scaffold-example/scripts/scaffold.py b/.claude/skills/scaffold-example/scripts/scaffold.py index ce279e4..ed572ed 100644 --- a/.claude/skills/scaffold-example/scripts/scaffold.py +++ b/.claude/skills/scaffold-example/scripts/scaffold.py @@ -150,6 +150,14 @@ def main() -> int: text = text.replace(f'{kebab} = "{entry}"', f'{command} = "{entry}"') pyproject.write_text(text, encoding="utf-8") + # run.sh invokes the CLI by the kebab name (from the sentinel rewrite); point it + # at the overridden command so `bash run.sh` matches [project.scripts]. + run_sh = dest / "run.sh" + if run_sh.is_file(): + text = run_sh.read_text(encoding="utf-8") + text = text.replace(f"uv run {kebab}", f"uv run {command}") + run_sh.write_text(text, encoding="utf-8") + # Description. if args.description: pyproject = dest / "pyproject.toml" @@ -173,11 +181,11 @@ def main() -> int: if kind == "package": print(f" 2. uv run {command} eval # DRY_RUN smoke test (no creds needed)") print(" 3. Fill the TODO stubs: app.py (real logic), prompts.py, data/cases.json, README.md") - print(f" 4. uv run ruff check . && uv run {command} run-all") + print(" 4. uv run ruff check . && bash run.sh # what CI runs (dry-run without creds)") else: print(f" 2. uv run {command} --dry-run # DRY_RUN smoke test (no creds needed)") print(f" 3. Fill the TODO in {pkg}.py (real logic) and the README.md sections") - print(" 4. uv run ruff check .") + print(" 4. uv run ruff check . && bash run.sh # what CI runs (dry-run without creds)") return 0 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..318471f --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# Maintainer-owned CI + automation. Edits to these paths require a maintainer review. +# The scheduled-run curation list and the workflows are intentionally not contributor-editable. +/.github/workflows/ @fschlz @LeoRoccoBreedt +/.github/scheduled.json @fschlz @LeoRoccoBreedt +/.github/CODEOWNERS @fschlz @LeoRoccoBreedt +/.github/dependabot.yml @fschlz @LeoRoccoBreedt diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..a9bd567 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + # Keep the GitHub Actions pinned in .github/workflows/ (checkout, setup-uv, …) current, + # so a "stay-working" examples repo doesn't silently drift onto stale/abandoned action versions. + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: "monday" + commit-message: + prefix: "ci" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..e4c26b4 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,18 @@ +## What & why + + + +## Checklist + + + +- [ ] Example is in the right bucket (`integrations` / `guides` / `use-cases` / `scripts`) +- [ ] Folder name is `lowercase_with_underscores` +- [ ] `README.md` has all required sections; index tables updated if examples were added/renamed/removed +- [ ] Dry-run works with no credentials — `bash run.sh` exits cleanly (this is what CI's secrets-free job runs) +- [ ] `uv run ruff check .` and `uv run ruff format --check .` are clean +- [ ] No credentials or `.env` files committed +- [ ] Dependencies declared in `pyproject.toml` (uv project); no `requirements.txt`, no committed `uv.lock` +- [ ] `run.sh` exists and starts with `set -e` +- [ ] `OPIK_PROJECT_NAME` is set — exported in `run.sh` (scripts) or defined in `config.py` (use-cases/guides) +- [ ] Examples that call LLMs use litellm and read `OPIK_EXAMPLES_MODEL` diff --git a/.github/scheduled.json b/.github/scheduled.json index 0637a08..fe51488 100644 --- a/.github/scheduled.json +++ b/.github/scheduled.json @@ -1 +1 @@ -[] \ No newline at end of file +[] diff --git a/.github/workflows/compliance.yml b/.github/workflows/compliance.yml index 7ec5687..993c36c 100644 --- a/.github/workflows/compliance.yml +++ b/.github/workflows/compliance.yml @@ -133,7 +133,9 @@ jobs: fi # If litellm is declared as a dependency, OPIK_EXAMPLES_MODEL must be referenced. - if grep -q "litellm" "$folder/pyproject.toml" 2>/dev/null; then + # Match a quoted dependency entry ("litellm", "litellm>=...", "litellm[extra]") + # so a bare mention in a comment doesn't trip the check. + if grep -qE '"litellm' "$folder/pyproject.toml" 2>/dev/null; then if grep -rq "OPIK_EXAMPLES_MODEL" "$folder" --include="*.py" 2>/dev/null; then echo " OK OPIK_EXAMPLES_MODEL referenced (litellm dep detected)" else @@ -155,13 +157,15 @@ jobs: - name: Write step summary if: always() && steps.changed-folders.outputs.folders != '' run: | - echo "## Example Compliance Check" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Folders checked: \`${{ steps.changed-folders.outputs.folders }}\`" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "**Rules:**" >> "$GITHUB_STEP_SUMMARY" - echo "- \`run.sh\` must exist and contain \`set -e\`" >> "$GITHUB_STEP_SUMMARY" - echo "- \`OPIK_PROJECT_NAME\` must be set — via \`export\` in \`run.sh\` (scripts) or as a constant in a \`.py\` config file (use-cases/guides)" >> "$GITHUB_STEP_SUMMARY" - echo "- \`README.md\` must exist" >> "$GITHUB_STEP_SUMMARY" - echo "- \`pyproject.toml\` must exist; \`requirements.txt\` is not allowed" >> "$GITHUB_STEP_SUMMARY" - echo "- If \`litellm\` is a dependency, a \`.py\` file must reference \`OPIK_EXAMPLES_MODEL\`" >> "$GITHUB_STEP_SUMMARY" + { + echo "## Example Compliance Check" + echo "" + echo "Folders checked: \`${{ steps.changed-folders.outputs.folders }}\`" + echo "" + echo "**Rules:**" + echo "- \`run.sh\` must exist and contain \`set -e\`" + echo "- \`OPIK_PROJECT_NAME\` must be set — via \`export\` in \`run.sh\` (scripts) or as a constant in a \`.py\` config file (use-cases/guides)" + echo "- \`README.md\` must exist" + echo "- \`pyproject.toml\` must exist; \`requirements.txt\` is not allowed" + echo "- If \`litellm\` is a dependency, a \`.py\` file must reference \`OPIK_EXAMPLES_MODEL\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/pr-test.yml b/.github/workflows/pr-test.yml index 9a3c8ed..2beaf93 100644 --- a/.github/workflows/pr-test.yml +++ b/.github/workflows/pr-test.yml @@ -1,8 +1,11 @@ name: PR Example Tests -# Runs only the example folders changed in this PR using a cheap model and real Opik -# credentials. A passing run means the code executes correctly and traces are visible -# in the opik-examples workspace. Failures block the PR. +# Runs the example folders changed in this PR. +# - lint + dry-run are secrets-free and run on every PR, including forks. +# - live-run uses real Opik credentials and runs only when secrets are available +# (same-repo PRs); it confirms the example executes and logs traces to the +# opik-examples workspace. +# Any job failure blocks the PR. on: pull_request: @@ -16,12 +19,17 @@ on: permissions: contents: read +concurrency: + group: pr-test-${{ github.ref }} + cancel-in-progress: true + jobs: detect-changes: runs-on: ubuntu-latest outputs: folders: ${{ steps.detect.outputs.folders }} has_changes: ${{ steps.detect.outputs.has_changes }} + has_secrets: ${{ steps.secrets.outputs.has_secrets }} steps: - uses: actions/checkout@v4 with: @@ -77,10 +85,78 @@ jobs: echo "Runnable folders to test: ${seen[*]}" fi - test-changed: + - name: Check whether Opik secrets are available + id: secrets + # secrets.* is not available in job-level `if:`, so resolve it here and + # expose a boolean output the live-run job can gate on. Fork PRs get no + # secrets, so this is false and only lint + dry-run run. + env: + OPIK_API_KEY: ${{ secrets.OPIK_API_KEY }} + run: | + if [[ -n "$OPIK_API_KEY" ]]; then + echo "has_secrets=true" >> "$GITHUB_OUTPUT" + echo "Opik secrets available — live run will execute." + else + echo "has_secrets=false" >> "$GITHUB_OUTPUT" + echo "No Opik secrets (fork PR) — lint + dry-run only." + fi + + lint: needs: detect-changes if: needs.detect-changes.outputs.has_changes == 'true' runs-on: ubuntu-latest + timeout-minutes: 10 + strategy: + fail-fast: false + matrix: + folder: ${{ fromJson(needs.detect-changes.outputs.folders) }} + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + version: "latest" + enable-cache: true + python-version: "3.12" + + - name: Ruff check + format + run: | + cd "${{ matrix.folder }}" + uv sync + uv run ruff check . + uv run ruff format --check . + + dry-run: + needs: detect-changes + if: needs.detect-changes.outputs.has_changes == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + folder: ${{ fromJson(needs.detect-changes.outputs.folders) }} + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v5 + with: + version: "latest" + enable-cache: true + python-version: "3.12" + + - name: Run example (dry-run, no credentials) + # No Opik/LLM secrets in env: the example must detect missing credentials, + # fall back to DRY_RUN, and exit 0. This is the only execution signal fork + # PRs receive, and it enforces that every example's dry-run path works. + run: | + cd "${{ matrix.folder }}" + bash run.sh + + live-run: + needs: detect-changes + if: needs.detect-changes.outputs.has_changes == 'true' && needs.detect-changes.outputs.has_secrets == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -94,7 +170,7 @@ jobs: enable-cache: true python-version: "3.12" - - name: Run example + - name: Run example (live, real Opik credentials) env: OPIK_API_KEY: ${{ secrets.OPIK_API_KEY }} OPIK_WORKSPACE: ${{ vars.OPIK_WORKSPACE }} @@ -103,5 +179,5 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: | - cd ${{ matrix.folder }} + cd "${{ matrix.folder }}" bash run.sh diff --git a/.github/workflows/scheduled.yml b/.github/workflows/scheduled.yml index 176de83..9296956 100644 --- a/.github/workflows/scheduled.yml +++ b/.github/workflows/scheduled.yml @@ -33,7 +33,11 @@ jobs: run-examples: needs: load-examples + # Skip cleanly when the curated list is empty — an empty matrix array otherwise + # errors with "Matrix vector 'folder' does not contain any values". + if: needs.load-examples.outputs.folders != '[]' runs-on: ubuntu-latest + timeout-minutes: 20 strategy: fail-fast: false matrix: @@ -56,5 +60,5 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} run: | - cd ${{ matrix.folder }} + cd "${{ matrix.folder }}" bash run.sh diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6f45dc4..1f83357 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -97,9 +97,9 @@ Use the template's README as a guide. Required sections: - **Running it** — exact commands, including dry-run - **How it works** — brief walkthrough of the key steps -### Running locally without credentials +### Dry-run mode -CI always has real Opik credentials. If you want to smoke-test an example locally before obtaining credentials, you can add a dry-run guard — but this is optional and not enforced: +Every runnable example must work without credentials. The secrets-free CI `dry-run` job runs `bash run.sh` with no Opik or LLM keys set and expects a clean exit — it is the only execution signal a fork PR receives, so a working dry-run path is required, not optional. The standard pattern: ```python DRY_RUN = not (os.environ.get("OPIK_API_KEY") and os.environ.get("OPIK_WORKSPACE")) @@ -110,7 +110,7 @@ else: # send to Opik ``` -The recommended path is to set your own Opik credentials locally so you see real traces, the same way CI does. +The dry-run output should be meaningful enough to verify the example is working. To see real traces, set your own Opik credentials locally — the same way CI's live run does. ### Credentials @@ -142,7 +142,7 @@ Every testable example must include a `run.sh` at its root. This file is what th set -e uv sync -uv run my-example run-all +uv run my-example eval # the command(s) that run the example end-to-end ``` The scaffold tool adds a working `run.sh` automatically when you create a new example. @@ -174,7 +174,7 @@ The compliance check accepts either pattern — it looks for `OPIK_PROJECT_NAME` ### Opik workspace -All CI runs log to the workspace set in the `OPIK_WORKSPACE` GitHub Actions variable (currently `opik-examples`). Locally, set both vars explicitly to avoid accidentally writing to the shared workspace: +Live CI runs log to the workspace set in the `OPIK_WORKSPACE` GitHub Actions variable (currently `opik-examples`). Locally, set both vars explicitly to avoid accidentally writing to the shared workspace: ```bash export OPIK_API_KEY= @@ -208,7 +208,7 @@ Model names use litellm's provider-prefixed format: `openai/gpt-4o-mini`, `anthr ### Opik logging -CI always has real Opik credentials (`OPIK_API_KEY`, `OPIK_WORKSPACE`, `OPIK_ENVIRONMENT`). Every CI run logs traces to the `opik-examples` workspace — this is how we verify an example is working, not just that it exits 0. +Same-repo PRs and scheduled runs have real Opik credentials (`OPIK_API_KEY`, `OPIK_WORKSPACE`, `OPIK_ENVIRONMENT`), and a `live-run` job logs traces to the `opik-examples` workspace — this is how we verify an example is working, not just that it exits 0. Fork PRs don't receive secrets (GitHub withholds them from forks), so they run only the secrets-free `lint` + `dry-run` jobs; a maintainer runs the live job after review. `OPIK_ENVIRONMENT` is set automatically in CI; you do not need to set it locally. It tags traces so CI runs are distinguishable from local runs in the Opik UI. @@ -233,7 +233,8 @@ Before opening a PR, verify: - [ ] Folder name is lowercase with underscores (e.g. `my_example`, not `MyExample` or `my-example`) - [ ] `README.md` has all required sections - [ ] READMEs updated — the example's `README.md`, and for added/renamed/removed examples the bucket index and the root `README.md` table -- [ ] Dry-run mode works (no env vars set) prints useful output without errors — e.g. `uv run eval` +- [ ] Dry-run works with no credentials set — `bash run.sh` exits cleanly (this is exactly what CI's secrets-free job runs) +- [ ] `uv run ruff check .` and `uv run ruff format --check .` are clean - [ ] No credentials or `.env` files committed - [ ] Dependencies declared in `pyproject.toml` (uv project); no `requirements.txt` - [ ] `run.sh` exists and starts with `set -e` diff --git a/templates/script-template/run.sh b/templates/script-template/run.sh new file mode 100755 index 0000000..a914ac7 --- /dev/null +++ b/templates/script-template/run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +# Entry point CI runs for this example. With no Opik credentials it falls back to +# DRY_RUN and exits 0 (the secrets-free check); with credentials set it runs live +# and logs traces to Opik. +uv sync +export OPIK_PROJECT_NAME="example-script" +uv run example-script diff --git a/templates/use-case-template/run.sh b/templates/use-case-template/run.sh new file mode 100755 index 0000000..56706de --- /dev/null +++ b/templates/use-case-template/run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -e + +# Entry point CI runs for this example. `eval` exercises the full Opik loop +# (dataset + test suite + evaluation). With no Opik credentials it falls back to +# DRY_RUN and exits 0 (the secrets-free check); with credentials set it runs live +# and logs traces to Opik. OPIK_PROJECT_NAME is defined in config.py. +uv sync +uv run example-use-case eval diff --git a/templates/use-case-template/src/example_use_case/config.py b/templates/use-case-template/src/example_use_case/config.py index 870267d..4e44f8d 100644 --- a/templates/use-case-template/src/example_use_case/config.py +++ b/templates/use-case-template/src/example_use_case/config.py @@ -11,9 +11,10 @@ LLM_READY = bool(ANTHROPIC_API_KEY) # litellm model strings (Anthropic provider). Swap for any litellm-supported model. -GEN_MODEL = "anthropic/claude-sonnet-4-6" # the deployed app -JUDGE_MODEL = "anthropic/claude-sonnet-4-6" # LLM-as-judge for metrics -OPTIMIZER_MODEL = "anthropic/claude-sonnet-4-6" # meta-model that rewrites the prompt +# CI sets OPIK_EXAMPLES_MODEL to a cheap model; locally, leave it unset to use the full model. +GEN_MODEL = os.environ.get("OPIK_EXAMPLES_MODEL", "anthropic/claude-sonnet-4-6") # the deployed app +JUDGE_MODEL = GEN_MODEL # LLM-as-judge for metrics +OPTIMIZER_MODEL = GEN_MODEL # meta-model that rewrites the prompt DATASET_NAME = "example-use-case-eval" SUITE_NAME = "example-use-case-suite"