add EvalStatus enum

Signed-off-by: Peter Jausovec <peter.jausovec@solo.io>

Author: peterj

packages/evaluator-sdk-py/src/agentevals_evaluator_sdk/__init__.py

@@ -24,6 +24,7 @@ def my_evaluator(input: EvalInput) -> EvalResult:
 from .types import (
     EvalInput,
     EvalResult,
+    EvalStatus,
     IntermediateStepData,
     InvocationData,
     ToolCallData,
@@ -34,6 +35,7 @@ def my_evaluator(input: EvalInput) -> EvalResult:
     "evaluator",
     "EvalInput",
     "EvalResult",
+    "EvalStatus",
     "IntermediateStepData",
     "InvocationData",
     "ToolCallData",

packages/evaluator-sdk-py/src/agentevals_evaluator_sdk/types.py

@@ -7,6 +7,7 @@
 
 from __future__ import annotations
 
+from enum import Enum
 from typing import Any, Optional
 
 from pydantic import BaseModel, Field
@@ -63,13 +64,21 @@ class EvalInput(BaseModel):
     expected_invocations: Optional[list[InvocationData]] = None
 
 
+class EvalStatus(str, Enum):
+    """Wire JSON uses the string values (stable protocol with agentevals CLI)."""
+
+    PASSED = "PASSED"
+    FAILED = "FAILED"
+    NOT_EVALUATED = "NOT_EVALUATED"
+
 class EvalResult(BaseModel):
     """Output payload expected from a custom evaluator script/container on stdout."""
 
     score: float = Field(ge=0.0, le=1.0)
-    status: Optional[str] = Field(
+    status: Optional[EvalStatus] = Field(
         default=None,
-        description='One of "PASSED", "FAILED", "NOT_EVALUATED". Derived from score vs threshold if omitted.',
+        description="One of EvalStatus.PASSED, EvalStatus.FAILED, EvalStatus.NOT_EVALUATED. Derived from score vs threshold if omitted.",
+        enum=EvalStatus,
     )
     per_invocation_scores: list[Optional[float]] = Field(default_factory=list)
     details: Optional[dict[str, Any]] = None