cleanup

Author: krisztianfekete

src/agentevals/api/streaming_routes.py

@@ -9,7 +9,7 @@
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import FileResponse
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, ConfigDict, Field
 
 from ..config import BuiltinMetricDef, EvalRunConfig, EvaluatorDef
 from ..converter import convert_traces
@@ -42,6 +42,8 @@ class CreateEvalSetRequest(BaseModel):
 
 
 class EvaluateSessionsRequest(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
     golden_session_id: str
     eval_set_id: str
     evaluators: list[EvaluatorDef] = Field(default_factory=lambda: [BuiltinMetricDef(name="tool_trajectory_avg_score")])

src/agentevals/cli.py

@@ -52,23 +52,6 @@ def _relative_time(iso_str: str | None) -> str:
         return ""
 
 
-def _apply_builtin_overrides(evaluators, *, judge_model, threshold, trajectory_match_type):
-    updated = []
-    for evaluator in evaluators:
-        if getattr(evaluator, "type", None) == "builtin":
-            payload = evaluator.model_dump(by_alias=False)
-            if judge_model is not None:
-                payload["judge_model"] = judge_model
-            if threshold is not None:
-                payload["threshold"] = threshold
-            if trajectory_match_type is not None:
-                payload["trajectory_match_type"] = trajectory_match_type
-            updated.append(type(evaluator).model_validate(payload))
-        else:
-            updated.append(evaluator)
-    return updated
-
-
 @click.group()
 @click.version_option(version=__version__, prog_name="agentevals")
 @click.option(
@@ -160,61 +143,54 @@ def run(
     config_file: str | None,
 ) -> None:
     """Evaluate trace file(s) against the configured evaluators."""
-    from .config import EvalRunConfig, make_builtin_evaluator_entries
+    from .config import EvalRunConfig, apply_builtin_overrides, make_builtin_evaluator_entries
     from .output import format_results
     from .runner import run_evaluation
 
     explicit_metrics = list(metric) if metric else []
 
     if config_file:
-        from .eval_config_loader import load_eval_config, merge_configs
+        from .eval_config_loader import load_eval_config
 
-        file_config = load_eval_config(config_file)
-        config = file_config
+        config = load_eval_config(config_file)
         if explicit_metrics:
-            cli_config = EvalRunConfig(
-                trace_files=[],
-                evaluators=make_builtin_evaluator_entries(
-                    explicit_metrics,
-                    judge_model=judge_model,
-                    threshold=threshold,
-                    trajectory_match_type=trajectory_match_type,
-                ),
+            cli_evaluators = make_builtin_evaluator_entries(
+                explicit_metrics,
+                judge_model=judge_model,
+                threshold=threshold,
+                trajectory_match_type=trajectory_match_type,
             )
-            config = merge_configs(file_config, cli_config)
+            by_name = {e.name: e for e in config.evaluators}
+            for ev in cli_evaluators:
+                by_name[ev.name] = ev
+            config.evaluators = list(by_name.values())
         elif judge_model is not None or threshold is not None or trajectory_match_type is not None:
-            config = config.model_copy(
-                update={
-                    "evaluators": _apply_builtin_overrides(
-                        config.evaluators,
-                        judge_model=judge_model,
-                        threshold=threshold,
-                        trajectory_match_type=trajectory_match_type,
-                    )
-                }
+            config.evaluators = apply_builtin_overrides(
+                config.evaluators,
+                judge_model=judge_model,
+                threshold=threshold,
+                trajectory_match_type=trajectory_match_type,
             )
-        if trace_files:
-            config.trace_files = list(trace_files)
-        if eval_set is not None:
-            config.eval_set_file = eval_set
-        if trace_format is not None:
-            config.trace_format = trace_format
-        if output != "table":
-            config.output_format = output
     else:
         config = EvalRunConfig(
-            trace_files=list(trace_files),
-            eval_set_file=eval_set,
+            trace_files=[],
             evaluators=make_builtin_evaluator_entries(
-                explicit_metrics if explicit_metrics else None,
+                explicit_metrics or None,
                 judge_model=judge_model,
                 threshold=threshold,
                 trajectory_match_type=trajectory_match_type,
             ),
-            trace_format=trace_format,
-            output_format=output,
         )
 
+    if trace_files:
+        config.trace_files = list(trace_files)
+    if eval_set is not None:
+        config.eval_set_file = eval_set
+    if trace_format is not None:
+        config.trace_format = trace_format
+    if output != "table":
+        config.output_format = output
+
     result = asyncio.run(run_evaluation(config))
     formatted = format_results(result, fmt=config.output_format)
     click.echo(formatted)

src/agentevals/config.py

@@ -138,6 +138,34 @@ def make_builtin_evaluator_entries(
     return evaluators
 
 
+def apply_builtin_overrides(
+    evaluators: list[EvaluatorDef],
+    *,
+    judge_model: str | None = None,
+    threshold: float | None = None,
+    trajectory_match_type: str | None = None,
+) -> list[EvaluatorDef]:
+    """Return a new evaluator list with run-level overrides applied to built-ins.
+
+    Non-builtin entries pass through unchanged. Each override is only applied
+    when the corresponding argument is not None, so callers can pass any subset.
+    """
+    updated: list[EvaluatorDef] = []
+    for evaluator in evaluators:
+        if isinstance(evaluator, BuiltinMetricDef):
+            payload = evaluator.model_dump(by_alias=False)
+            if judge_model is not None:
+                payload["judge_model"] = judge_model
+            if threshold is not None:
+                payload["threshold"] = threshold
+            if trajectory_match_type is not None:
+                payload["trajectory_match_type"] = trajectory_match_type
+            updated.append(BuiltinMetricDef.model_validate(payload))
+        else:
+            updated.append(evaluator)
+    return updated
+
+
 class EvalParams(BaseModel):
     """Evaluation parameters independent of how traces are provided.
 

src/agentevals/eval_config_loader.py

@@ -66,7 +66,7 @@ def load_eval_config(path: str | Path) -> EvalRunConfig:
 
     legacy_keys = {
         "metrics",
-        "custom_graders",
+        "custom_evaluators",
         "judge_model",
         "threshold",
         "trajectory_match_type",

src/agentevals/mcp_server.py

@@ -8,7 +8,7 @@
 from mcp.server import FastMCP
 from pydantic import BaseModel, Field
 
-from agentevals.config import EvalRunConfig, make_builtin_evaluator_entries
+from agentevals.config import EvalRunConfig, apply_builtin_overrides, make_builtin_evaluator_entries
 from agentevals.runner import run_evaluation
 
 _DEFAULT_SERVER_URL = "http://localhost:8001"
@@ -89,23 +89,6 @@ class EvaluateSessionsResponse(BaseModel):
     results: list[SessionEvalResultResponse]
 
 
-def _apply_builtin_overrides(evaluators, *, judge_model=None, threshold=None, trajectory_match_type=None):
-    updated = []
-    for evaluator in evaluators:
-        if getattr(evaluator, "type", None) == "builtin":
-            payload = evaluator.model_dump(by_alias=False)
-            if judge_model is not None:
-                payload["judge_model"] = judge_model
-            if threshold is not None:
-                payload["threshold"] = threshold
-            if trajectory_match_type is not None:
-                payload["trajectory_match_type"] = trajectory_match_type
-            updated.append(type(evaluator).model_validate(payload))
-        else:
-            updated.append(evaluator)
-    return updated
-
-
 # ---------------------------------------------------------------------------
 # Result transformation
 # ---------------------------------------------------------------------------
@@ -301,7 +284,7 @@ async def evaluate_traces(
             elif judge_model is not None or threshold is not None:
                 config = config.model_copy(
                     update={
-                        "evaluators": _apply_builtin_overrides(
+                        "evaluators": apply_builtin_overrides(
                             config.evaluators,
                             judge_model=judge_model,
                             threshold=threshold,

tests/test_eval_config_loader.py

@@ -15,7 +15,7 @@ def test_load_eval_config_rejects_legacy_keys(tmp_path):
             """
             metrics:
               - tool_trajectory_avg_score
-            custom_graders:
+            custom_evaluators:
               - name: tool_call_checker
                 type: code
                 path: ./examples/custom_evaluators/tool_call_checker.py

ui/src/components/upload/MetricSelector.tsx

@@ -146,18 +146,13 @@ const selectorStyle = css`
 
 let cachedMetrics: MetricMetadata[] | null = null;
 
-function isSupportedBuiltinMetric(metric: MetricMetadata): boolean {
-  return metric.working !== false && metric.requiresRubrics !== true;
-}
-
 export const MetricSelector: React.FC<MetricSelectorProps> = ({
   selectedEvaluatorNames,
   onToggleEvaluatorName,
   loadFromAPI = false,
 }) => {
   const [metrics, setMetrics] = useState<MetricMetadata[]>(cachedMetrics ?? AVAILABLE_METRICS);
-  const supportedMetrics = metrics.filter(isSupportedBuiltinMetric);
-  const unsupportedCount = metrics.length - supportedMetrics.length;
+  const hasCaveatedMetrics = metrics.some((m) => m.requiresRubrics === true || m.working === false);
 
   useEffect(() => {
     if (!loadFromAPI || cachedMetrics) return;
@@ -175,7 +170,7 @@ export const MetricSelector: React.FC<MetricSelectorProps> = ({
     return () => { cancelled = true; };
   }, [loadFromAPI]);
 
-  const categorizedSupportedMetrics = supportedMetrics.reduce(
+  const categorizedMetrics = metrics.reduce(
     (acc, metric) => {
       if (!acc[metric.category]) {
         acc[metric.category] = [];
@@ -187,7 +182,7 @@ export const MetricSelector: React.FC<MetricSelectorProps> = ({
   );
 
   const handleSelectAll = () => {
-    supportedMetrics.forEach((metric) => {
+    metrics.forEach((metric) => {
       if (!selectedEvaluatorNames.includes(metric.name)) {
         onToggleEvaluatorName(metric.name);
       }
@@ -203,7 +198,7 @@ export const MetricSelector: React.FC<MetricSelectorProps> = ({
   return (
     <div css={selectorStyle}>
       <div className="metric-categories">
-        {Object.entries(categorizedSupportedMetrics).map(([category, metrics]) => (
+        {Object.entries(categorizedMetrics).map(([category, metrics]) => (
           <div key={category} className="metric-category">
             <div className="category-title">{category}</div>
             <div className="metric-list">
@@ -250,10 +245,9 @@ export const MetricSelector: React.FC<MetricSelectorProps> = ({
           Clear All
         </Button>
       </div>
-      {unsupportedCount > 0 && (
+      {hasCaveatedMetrics && (
         <div className="selector-note">
-          Hidden {unsupportedCount} unsupported built-in evaluator{unsupportedCount === 1 ? '' : 's'} that require
-          rubric configuration or are marked incomplete.
+          Some evaluators require rubric configuration or are work-in-progress; see badges.
         </div>
       )}
     </div>