Merge pull request #33 from agentevals-dev/peterj/customevals

add support for custom graders (eval metrics)

Author: peterj

.github/workflows/publish-evaluator-sdk.yml

@@ -0,0 +1,60 @@
+name: Publish evaluator SDK
+
+on:
+  push:
+    tags:
+      - 'evaluator-sdk-v*'
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'Release tag (e.g. evaluator-sdk-v0.1.0)'
+        required: true
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Verify tag matches pyproject.toml version
+        run: |
+          REF=${{ github.event.inputs.tag || github.ref_name }}
+          TOML_VERSION=$(grep '^version' packages/evaluator-sdk-py/pyproject.toml | cut -d'"' -f2)
+          TAG_VERSION=${REF#evaluator-sdk-v}
+          if [ "$TOML_VERSION" != "$TAG_VERSION" ]; then
+            echo "Error: $REF does not match packages/evaluator-sdk-py/pyproject.toml version $TOML_VERSION"
+            exit 1
+          fi
+
+      - name: Install build tool
+        run: pip install build
+
+      - name: Build wheel and sdist
+        run: python -m build packages/evaluator-sdk-py/ --outdir dist/
+
+      - uses: actions/upload-artifact@v7
+        with:
+          name: evaluator-sdk-dist
+          path: dist/
+
+  publish:
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      id-token: write
+
+    steps:
+      - uses: actions/download-artifact@v8
+        with:
+          name: evaluator-sdk-dist
+          path: dist/
+
+      - uses: pypa/gh-action-pypi-publish@release/v1

README.md

@@ -58,7 +58,7 @@ Point any OTel-instrumented agent at the receiver. No SDK, no code changes:
 
 ```bash
 # Terminal 1
-agentevals serve --dev
+uv run agentevals serve --dev
 
 # Terminal 2
 export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
@@ -116,7 +116,7 @@ agentevals serve
 
 ```bash
 uv run agentevals serve --dev    # Terminal 1
-cd ui && npm run dev             # Terminal 2 → http://localhost:5173
+cd ui && npm install && npm run dev             # Terminal 2 → http://localhost:5173
 ```
 
 Upload traces and eval sets, select metrics, and view results with interactive span trees. Live-streamed traces appear in the "Local Dev" tab, grouped by session ID.