Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions agentplatform.go
Original file line number Diff line number Diff line change
Expand Up @@ -3103,18 +3103,26 @@ type ModelEvaluationPreset struct {
// MetricResultSummary represents per-metric aggregated pass/fail statistics
// across all prompts in an evaluation run.
type MetricResultSummary struct {
AvgAtKPercent float64 `json:"avg_at_k_percent,omitempty"`
ConsAtKPercent float64 `json:"cons_at_k_percent,omitempty"`
Description string `json:"description,omitempty"`
FailCount uint32 `json:"fail_count,omitempty"`
FailPercent float64 `json:"fail_percent,omitempty"`
MetricName string `json:"metric_name,omitempty"`
MetricUuid string `json:"metric_uuid,omitempty"`
AvgAtKPercent float64 `json:"avg_at_k_percent,omitempty"`
ConsAtKPercent float64 `json:"cons_at_k_percent,omitempty"`
Description string `json:"description,omitempty"`
FailCount uint32 `json:"fail_count,omitempty"`
FailPercent float64 `json:"fail_percent,omitempty"`
MetricName string `json:"metric_name,omitempty"`
MetricUuid string `json:"metric_uuid,omitempty"`
PassAtKCurve []*PassAtKPoint `json:"pass_at_k_curve,omitempty"`
PassAtKPercent float64 `json:"pass_at_k_percent,omitempty"`
PassCount uint32 `json:"pass_count,omitempty"`
PassPercent float64 `json:"pass_percent,omitempty"`
SkipPercent float64 `json:"skip_percent,omitempty"`
SkippedCount uint32 `json:"skipped_count,omitempty"`
}

// PassAtKPoint is one point on a metric's pass@k curve, for k from 1 up to
// the run's epochs. Only populated when epochs > 1.
type PassAtKPoint struct {
K uint32 `json:"k,omitempty"`
PassAtKPercent float64 `json:"pass_at_k_percent,omitempty"`
PassCount uint32 `json:"pass_count,omitempty"`
PassPercent float64 `json:"pass_percent,omitempty"`
SkipPercent float64 `json:"skip_percent,omitempty"`
SkippedCount uint32 `json:"skipped_count,omitempty"`
}

// LatencyMetrics contains latency metrics for candidate model invocations,
Expand Down
10 changes: 9 additions & 1 deletion agentplatform_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4016,7 +4016,11 @@ var getModelEvaluationRunResponse = `
"metric_uuid": "44444444-4444-4444-4444-444444444444",
"metric_name": "accuracy",
"pass_percent": 90.0,
"fail_percent": 10.0
"fail_percent": 10.0,
"pass_at_k_curve": [
{"k": 1, "pass_at_k_percent": 90.0},
{"k": 2, "pass_at_k_percent": 100.0}
]
}
],
"performance_metrics": {
Expand Down Expand Up @@ -4091,6 +4095,10 @@ func TestGetModelEvaluationRun(t *testing.T) {
assert.Len(t, out.Run.ResultSummary.MetricSummaries, 1)
assert.Equal(t, "accuracy", out.Run.ResultSummary.MetricSummaries[0].MetricName)
assert.Equal(t, 90.0, out.Run.ResultSummary.MetricSummaries[0].PassPercent)
assert.Equal(t, []*PassAtKPoint{
{K: 1, PassAtKPercent: 90.0},
{K: 2, PassAtKPercent: 100.0},
}, out.Run.ResultSummary.MetricSummaries[0].PassAtKCurve)
assert.NotNil(t, out.Run.ResultSummary.PerformanceMetrics)
assert.NotNil(t, out.Run.ResultSummary.PerformanceMetrics.CandidateLatency)
assert.Equal(t, 250.5, out.Run.ResultSummary.PerformanceMetrics.CandidateLatency.AvgE2ELatencyMs)
Expand Down
Loading