feat: add LabelModelGrader support for OpenAI Evals backend

Adds label_model grader type, validates passing_labels against labels,
moves OpenAI grader example to a separate file.

Author: mesutoezdil

docs/custom-evaluators.md

@@ -317,6 +317,26 @@ The `grader.evaluation_metric` field selects the similarity algorithm:
 | `rouge_1` through `rouge_5` | Unigram through 5-gram overlap (F-measure) |
 | `rouge_l` | Longest common subsequence overlap (F-measure) |
 
+### Label Model Grader
+
+Scores responses without a golden set. The model reads each response and assigns a label from a fixed list. Passing labels are defined in the config.
+
+```yaml
+evaluators:
+  - name: quality_check
+    type: openai_eval
+    grader:
+      type: label_model
+      model: gpt-4o-mini
+      input:
+        - role: user
+          content: "Rate this response: {{ item.actual_response }}"
+      labels: [good, bad]
+      passing_labels: [good]
+```
+
+The `threshold` field is not used for `label_model`. A response passes if its assigned label is in `passing_labels`.
+
 ### How it works
 
 Under the hood, agentevals creates an ephemeral eval on OpenAI, submits the actual and expected responses as JSONL items, polls for results, and cleans up. The agent's response and the golden reference are both placed in the `item` namespace (with `include_sample_schema: false`), so OpenAI only grades the provided text without generating any model outputs.

examples/custom_evaluators/eval_config.yaml

@@ -32,3 +32,4 @@ evaluators:
     ref: evaluators/random_evaluator/random_evaluator.py
     threshold: 0.110
     executor: local
+

examples/custom_evaluators/eval_config_openai_eval.yaml

@@ -0,0 +1,18 @@
+# Eval config using OpenAI Evals API graders.
+# Requires OPENAI_API_KEY to be set.
+#
+# Run with:
+#   agentevals run samples/helm.json \
+#     --config examples/custom_evaluators/eval_config_openai_eval.yaml
+
+evaluators:
+  - name: quality_check
+    type: openai_eval
+    grader:
+      type: label_model
+      model: gpt-4o-mini
+      input:
+        - role: user
+          content: "Rate this response: {{ item.actual_response }}"
+      labels: [good, bad]
+      passing_labels: [good]

src/agentevals/config.py

@@ -84,13 +84,21 @@ class OpenAIEvalDef(BaseModel):
     @classmethod
     def _validate_grader(cls, v: dict[str, Any]) -> dict[str, Any]:
         grader_type = v.get("type")
-        if grader_type != "text_similarity":
-            raise ValueError(f"Only 'text_similarity' grader type is currently supported, got '{grader_type}'")
-        metric = v.get("evaluation_metric")
-        if not metric:
-            raise ValueError("'evaluation_metric' is required for text_similarity grader")
-        if metric not in _VALID_SIMILARITY_METRICS:
-            raise ValueError(f"Unknown evaluation_metric '{metric}'. Valid: {sorted(_VALID_SIMILARITY_METRICS)}")
+        if grader_type == "text_similarity":
+            metric = v.get("evaluation_metric")
+            if not metric:
+                raise ValueError("'evaluation_metric' is required for text_similarity grader")
+            if metric not in _VALID_SIMILARITY_METRICS:
+                raise ValueError(f"Unknown evaluation_metric '{metric}'. Valid: {sorted(_VALID_SIMILARITY_METRICS)}")
+        elif grader_type == "label_model":
+            for field in ("model", "input", "labels", "passing_labels"):
+                if not v.get(field):
+                    raise ValueError(f"'{field}' is required for label_model grader")
+            invalid = [lbl for lbl in v["passing_labels"] if lbl not in v["labels"]]
+            if invalid:
+                raise ValueError(f"passing_labels contains labels not declared in labels: {invalid}")
+        else:
+            raise ValueError(f"Unsupported grader type: '{grader_type}'. Supported: label_model, text_similarity")
         return v
 
 

src/agentevals/openai_eval_backend.py

@@ -31,6 +31,12 @@
     "required": ["actual_response", "expected_response"],
 }
 
+_ACTUAL_ONLY_SCHEMA = {
+    "type": "object",
+    "properties": {"actual_response": {"type": "string"}},
+    "required": ["actual_response"],
+}
+
 
 def _build_testing_criteria(evaluator_def: OpenAIEvalDef) -> dict[str, Any]:
     """Build the OpenAI testing_criteria dict from the evaluator config.
@@ -51,28 +57,31 @@ def _build_testing_criteria(evaluator_def: OpenAIEvalDef) -> dict[str, Any]:
             "pass_threshold": evaluator_def.threshold,
         }
 
+    if grader_type == "label_model":
+        return {
+            "type": "label_model",
+            "name": evaluator_def.name,
+            "model": grader["model"],
+            "input": grader["input"],
+            "labels": grader["labels"],
+            "passing_labels": grader["passing_labels"],
+        }
+
     raise ValueError(f"Unsupported grader type: {grader_type}")
 
 
 def _build_jsonl_items(
     actual_invocations: list[Invocation],
     expected_invocations: list[Invocation],
+    include_expected: bool = True,
 ) -> list[dict[str, Any]]:
     items = []
     for i, actual_inv in enumerate(actual_invocations):
-        actual_text = _content_to_text(actual_inv.final_response)
-        if i < len(expected_invocations):
-            expected_text = _content_to_text(expected_invocations[i].final_response)
-        else:
-            expected_text = ""
-        items.append(
-            {
-                "item": {
-                    "actual_response": actual_text,
-                    "expected_response": expected_text,
-                }
-            }
-        )
+        entry: dict[str, Any] = {"actual_response": _content_to_text(actual_inv.final_response)}
+        if include_expected:
+            expected_text = _content_to_text(expected_invocations[i].final_response) if i < len(expected_invocations) else ""
+            entry["expected_response"] = expected_text
+        items.append({"item": entry})
     return items
 
 
@@ -111,13 +120,15 @@ async def evaluate_openai_eval(
             error="OPENAI_API_KEY environment variable is not set.",
         )
 
-    if expected_invocations is None:
+    grader_type = evaluator_def.grader["type"]
+
+    if grader_type == "text_similarity" and expected_invocations is None:
         return MetricResult(
             metric_name=evaluator_def.name,
             error="OpenAI text_similarity grader requires expected invocations (golden eval set).",
         )
 
-    items = _build_jsonl_items(actual_invocations, expected_invocations)
+    items = _build_jsonl_items(actual_invocations, expected_invocations or [], include_expected=(grader_type != "label_model"))
     if not items:
         return MetricResult(
             metric_name=evaluator_def.name,
@@ -130,12 +141,13 @@ async def evaluate_openai_eval(
     try:
         client = await asyncio.to_thread(_get_openai_client)
 
+        item_schema = _ACTUAL_ONLY_SCHEMA if grader_type == "label_model" else _TEXT_PAIR_SCHEMA
         eval_obj = await asyncio.to_thread(
             client.evals.create,
             name=f"agentevals-{evaluator_def.name}",
             data_source_config={
                 "type": "custom",
-                "item_schema": _TEXT_PAIR_SCHEMA,
+                "item_schema": item_schema,
                 "include_sample_schema": False,
             },
             testing_criteria=[testing_criteria],
@@ -225,12 +237,17 @@ async def _collect_results(client: Any, eval_id: str, run_id: str, run: Any, eva
     total = result_counts.total if result_counts else 0
     eval_status = "PASSED" if failed == 0 and total > 0 else "FAILED"
 
+    grader = evaluator_def.grader
     details: dict[str, Any] = {
         "openai_eval_id": eval_id,
         "openai_run_id": run_id,
-        "evaluation_metric": evaluator_def.grader.get("evaluation_metric"),
         "result_counts": {"passed": passed, "failed": failed, "total": total},
     }
+    if grader["type"] == "text_similarity":
+        details["evaluation_metric"] = grader.get("evaluation_metric")
+    elif grader["type"] == "label_model":
+        details["model"] = grader.get("model")
+        details["passing_labels"] = grader.get("passing_labels")
     per_criteria = getattr(run, "per_testing_criteria_results", None)
     if per_criteria:
         details["per_testing_criteria"] = [

tests/test_openai_eval_backend.py

@@ -0,0 +1,116 @@
+import pytest
+from unittest.mock import MagicMock
+
+from agentevals.config import OpenAIEvalDef
+from agentevals.openai_eval_backend import (
+    _build_jsonl_items,
+    _build_testing_criteria,
+    evaluate_openai_eval,
+)
+
+
+def _label_grader(**overrides):
+    base = {
+        "type": "label_model",
+        "model": "gpt-4o-mini",
+        "input": [{"role": "user", "content": "Rate: {{ item.actual_response }}"}],
+        "labels": ["good", "bad"],
+        "passing_labels": ["good"],
+    }
+    base.update(overrides)
+    return base
+
+
+def _invocation(text: str):
+    inv = MagicMock()
+    inv.final_response.parts = [MagicMock(text=text)]
+    return inv
+
+
+class TestOpenAIEvalDefValidation:
+    def test_text_similarity_valid(self):
+        d = OpenAIEvalDef(name="sim", grader={"type": "text_similarity", "evaluation_metric": "bleu"})
+        assert d.grader["type"] == "text_similarity"
+
+    def test_text_similarity_missing_metric(self):
+        with pytest.raises(Exception, match="evaluation_metric"):
+            OpenAIEvalDef(name="sim", grader={"type": "text_similarity"})
+
+    def test_text_similarity_bad_metric(self):
+        with pytest.raises(Exception, match="Unknown evaluation_metric"):
+            OpenAIEvalDef(name="sim", grader={"type": "text_similarity", "evaluation_metric": "invalid"})
+
+    def test_label_model_valid(self):
+        d = OpenAIEvalDef(name="lm", grader=_label_grader())
+        assert d.grader["type"] == "label_model"
+
+    @pytest.mark.parametrize("field", ["model", "input", "labels", "passing_labels"])
+    def test_label_model_missing_required_field(self, field):
+        with pytest.raises(Exception, match=field):
+            OpenAIEvalDef(name="lm", grader=_label_grader(**{field: None}))
+
+    def test_label_model_passing_labels_not_in_labels(self):
+        grader = _label_grader()
+        grader["passing_labels"] = ["unknown"]
+        with pytest.raises(Exception, match="passing_labels"):
+            OpenAIEvalDef(name="lm", grader=grader)
+
+    def test_unsupported_grader_type(self):
+        with pytest.raises(Exception, match="Unsupported grader type"):
+            OpenAIEvalDef(name="x", grader={"type": "unknown"})
+
+
+class TestBuildTestingCriteria:
+    def test_text_similarity_shape(self):
+        d = OpenAIEvalDef(name="sim", grader={"type": "text_similarity", "evaluation_metric": "bleu"}, threshold=0.7)
+        c = _build_testing_criteria(d)
+        assert c["type"] == "text_similarity"
+        assert c["evaluation_metric"] == "bleu"
+        assert c["pass_threshold"] == 0.7
+        assert "{{ item.actual_response }}" in c["input"]
+        assert "{{ item.expected_response }}" in c["reference"]
+
+    def test_label_model_shape(self):
+        grader = _label_grader()
+        d = OpenAIEvalDef(name="quality", grader=grader)
+        c = _build_testing_criteria(d)
+        assert c["type"] == "label_model"
+        assert c["model"] == "gpt-4o-mini"
+        assert c["labels"] == ["good", "bad"]
+        assert c["passing_labels"] == ["good"]
+        assert c["input"] == grader["input"]
+
+
+class TestBuildJsonlItems:
+    def test_text_similarity_includes_expected(self):
+        items = _build_jsonl_items([_invocation("hello")], [_invocation("world")], include_expected=True)
+        assert "expected_response" in items[0]["item"]
+
+    def test_label_model_excludes_expected(self):
+        items = _build_jsonl_items([_invocation("hello")], [], include_expected=False)
+        assert "expected_response" not in items[0]["item"]
+
+    def test_missing_expected_falls_back_to_empty(self):
+        items = _build_jsonl_items([_invocation("hello")], [], include_expected=True)
+        assert items[0]["item"]["expected_response"] == ""
+
+
+class TestEvaluateOpenAIEval:
+    async def test_no_api_key_returns_error(self, monkeypatch):
+        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+        d = OpenAIEvalDef(name="sim", grader={"type": "text_similarity", "evaluation_metric": "bleu"})
+        result = await evaluate_openai_eval(d, [], [])
+        assert "OPENAI_API_KEY" in (result.error or "")
+
+    async def test_text_similarity_requires_expected(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-key")
+        d = OpenAIEvalDef(name="sim", grader={"type": "text_similarity", "evaluation_metric": "bleu"})
+        result = await evaluate_openai_eval(d, [_invocation("hi")], None)
+        assert "expected invocations" in (result.error or "")
+
+    async def test_label_model_does_not_require_expected(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-key")
+        monkeypatch.setattr("agentevals.openai_eval_backend._get_openai_client", lambda: None)
+        d = OpenAIEvalDef(name="lm", grader=_label_grader())
+        result = await evaluate_openai_eval(d, [_invocation("hi")], None)
+        assert "expected invocations" not in (result.error or "")