diff --git a/agentplatform.go b/agentplatform.go index 80bc7ceb..78586c1e 100644 --- a/agentplatform.go +++ b/agentplatform.go @@ -3103,18 +3103,26 @@ type ModelEvaluationPreset struct { // MetricResultSummary represents per-metric aggregated pass/fail statistics // across all prompts in an evaluation run. type MetricResultSummary struct { - AvgAtKPercent float64 `json:"avg_at_k_percent,omitempty"` - ConsAtKPercent float64 `json:"cons_at_k_percent,omitempty"` - Description string `json:"description,omitempty"` - FailCount uint32 `json:"fail_count,omitempty"` - FailPercent float64 `json:"fail_percent,omitempty"` - MetricName string `json:"metric_name,omitempty"` - MetricUuid string `json:"metric_uuid,omitempty"` + AvgAtKPercent float64 `json:"avg_at_k_percent,omitempty"` + ConsAtKPercent float64 `json:"cons_at_k_percent,omitempty"` + Description string `json:"description,omitempty"` + FailCount uint32 `json:"fail_count,omitempty"` + FailPercent float64 `json:"fail_percent,omitempty"` + MetricName string `json:"metric_name,omitempty"` + MetricUuid string `json:"metric_uuid,omitempty"` + PassAtKCurve []*PassAtKPoint `json:"pass_at_k_curve,omitempty"` + PassAtKPercent float64 `json:"pass_at_k_percent,omitempty"` + PassCount uint32 `json:"pass_count,omitempty"` + PassPercent float64 `json:"pass_percent,omitempty"` + SkipPercent float64 `json:"skip_percent,omitempty"` + SkippedCount uint32 `json:"skipped_count,omitempty"` +} + +// PassAtKPoint is one point on a metric's pass@k curve, for k from 1 up to +// the run's epochs. Only populated when epochs > 1. +type PassAtKPoint struct { + K uint32 `json:"k,omitempty"` PassAtKPercent float64 `json:"pass_at_k_percent,omitempty"` - PassCount uint32 `json:"pass_count,omitempty"` - PassPercent float64 `json:"pass_percent,omitempty"` - SkipPercent float64 `json:"skip_percent,omitempty"` - SkippedCount uint32 `json:"skipped_count,omitempty"` } // LatencyMetrics contains latency metrics for candidate model invocations, diff --git a/agentplatform_test.go b/agentplatform_test.go index fe6d922f..191ce8d4 100644 --- a/agentplatform_test.go +++ b/agentplatform_test.go @@ -4016,7 +4016,11 @@ var getModelEvaluationRunResponse = ` "metric_uuid": "44444444-4444-4444-4444-444444444444", "metric_name": "accuracy", "pass_percent": 90.0, - "fail_percent": 10.0 + "fail_percent": 10.0, + "pass_at_k_curve": [ + {"k": 1, "pass_at_k_percent": 90.0}, + {"k": 2, "pass_at_k_percent": 100.0} + ] } ], "performance_metrics": { @@ -4091,6 +4095,10 @@ func TestGetModelEvaluationRun(t *testing.T) { assert.Len(t, out.Run.ResultSummary.MetricSummaries, 1) assert.Equal(t, "accuracy", out.Run.ResultSummary.MetricSummaries[0].MetricName) assert.Equal(t, 90.0, out.Run.ResultSummary.MetricSummaries[0].PassPercent) + assert.Equal(t, []*PassAtKPoint{ + {K: 1, PassAtKPercent: 90.0}, + {K: 2, PassAtKPercent: 100.0}, + }, out.Run.ResultSummary.MetricSummaries[0].PassAtKCurve) assert.NotNil(t, out.Run.ResultSummary.PerformanceMetrics) assert.NotNil(t, out.Run.ResultSummary.PerformanceMetrics.CandidateLatency) assert.Equal(t, 250.5, out.Run.ResultSummary.PerformanceMetrics.CandidateLatency.AvgE2ELatencyMs)