Skip to content

Commit 2e6cce1

Browse files
CornWorldCornWorld
committed
feat(lab): 数值可解释性 — testcase 构成、指标定义、评测分类
Phase 1 — testcase 构成可视化 + incomplete 状态解释: - 新建 testcase-info.ts: EVAL_STRUCTURE(16静态+8运行时=24), TESTCASE_DESCRIPTIONS(24项中文描述), parseCheckId/getCheckDesc - RunList: score 列 incomplete 显示 (仅静态) + title 提示, eval badge incomplete 加 title - RunDetail: 评测明细顶部构成摘要, incomplete alert 解释块 (含 piExitReason/piTimedOut) Phase 2 — 指标定义说明: - metrics.ts: 导出 METRIC_DEFINITIONS (5项指标口径) - RunDetail + SessionDetail: 每个 stat 下方加定义说明, 详情页标注指标由前端实时计算可能与列表预计算值有差异 Phase 3 — 评测分类说明 + testcase 可读描述: - RunDetail: 4个子区域标题加分类后缀 (静态通过/失败·文件结构与内容验证, 运行时通过/失败·Docker HTTP 调用验证) - list() 每个 testcase 条目后加中文描述 - RunList: 表头 req/tokens/wall 加 title 属性
1 parent ab37ad6 commit 2e6cce1

5 files changed

Lines changed: 104 additions & 11 deletions

File tree

‎lab/src/lib/metrics.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,14 @@ import type { Entry, RunDetail } from "../api";
22
import { fmtSec, fmtNum } from "./format";
33
import type { AgentMessage } from "@earendil-works/pi-agent-core";
44

5+
export const METRIC_DEFINITIONS = {
6+
wall: "首条→末条消息时间戳跨度，非实际运行时长",
7+
tokens: "input+output+cacheRead+cacheWrite+reasoning 总和",
8+
requests: "模型请求次数 (assistant 消息数)",
9+
toolCalls: "工具调用总次数",
10+
errors: "工具执行报错 (isError) 计数",
11+
} as const;
12+
513
// ── session metrics (computed client-side from raw session events) ──
614
// NOTE on timestamps: an assistant message's timestamp is when the model request
715
// was SENT (right after the previous tool result); a toolResult's timestamp is when
@@ -156,6 +164,7 @@ export function computeSessionMetrics(events: Entry[]): SessionMetrics {
156164
rounds,
157165
};
158166
}
167+
159168
export interface Insight {
160169
level: "ok" | "warn" | "err";
161170
text: string;

‎lab/src/lib/testcase-info.ts‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
export const EVAL_STRUCTURE = {
2+
static: { total: 16, label: "静态检查", desc: "文件结构与内容验证" },
3+
runtime: { total: 8, label: "运行时验证", desc: "Docker 容器 HTTP 调用" },
4+
fullTotal: 24,
5+
} as const;
6+
7+
export const INCOMPLETE_CHECKS = ["artifact-dir-exists", "pack-dir-found"] as const;
8+
9+
// 24 个 check ID → 中文描述
10+
export const TESTCASE_DESCRIPTIONS: Record<string, string> = {
11+
"artifact-dir-exists": "artifact 目录存在",
12+
"pack-dir-found": "pack 目录（含 pack.json）存在",
13+
"pack-json-valid": "pack.json 格式有效",
14+
"pack-name-match": "pack 名称匹配目录",
15+
"pack-version-valid": "pack 版本号有效",
16+
"frontend-scope-public": "前端 scope 为 public",
17+
"frontend-script-pow-guard-js": "前端包含 pow-guard.js 引用",
18+
"hook-file-exists": "hook 文件存在",
19+
"frontend-file-exists": "前端文件存在",
20+
"hook-has-challenge-route": "hook 包含 challenge 路由",
21+
"hook-has-verify-route": "hook 包含 verify 路由",
22+
"hook-has-crypto-ref": "hook 引用了 crypto/sha256",
23+
"hook-has-pow-validation": "hook 包含工作量验证逻辑",
24+
"frontend-uses-localStorage": "前端使用 localStorage",
25+
"frontend-has-overlay": "前端包含 overlay UI",
26+
"frontend-has-cache-duration": "前端包含缓存时长配置",
27+
"challenge-returns-200": "challenge 端点返回 200",
28+
"challenge-has-nonce": "challenge 响应含 nonce",
29+
"challenge-has-difficulty": "challenge 响应含 difficulty",
30+
"pow-solved": "PoW 已求解",
31+
"verify-negative": "负样本验证：错误 nonce 被拒绝",
32+
"verify-positive": "正样本验证：正确 nonce 返回 token",
33+
"homepage-injection": "主页注入 pow-guard.js",
34+
"frontend-asset-served": "前端资源正常返回",
35+
};
36+
37+
export function parseCheckId(entry: string): string {
38+
const idx = entry.indexOf(":");
39+
return idx > 0 ? entry.slice(0, idx).trim() : entry.trim();
40+
}
41+
42+
export function getCheckDesc(entry: string): string {
43+
return TESTCASE_DESCRIPTIONS[parseCheckId(entry)] ?? "";
44+
}

‎lab/src/views/RunDetail.tsx‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import { createResource, createMemo, Show, For } from "solid-js";
22
import { api, type RunDetail as RunDetailT } from "../api";
33
import { esc, fmtTime, fmtSec, fmtNum, evalBadge } from "../lib/format";
4-
import { computeSessionMetrics, generateInsights } from "../lib/metrics";
4+
import { computeSessionMetrics, generateInsights, METRIC_DEFINITIONS } from "../lib/metrics";
5+
import { EVAL_STRUCTURE, getCheckDesc } from "../lib/testcase-info";
56
import { Evidence } from "./Evidence";
67

78
export function RunDetail({
@@ -60,6 +61,9 @@ export function RunDetail({
6061
<li>
6162
<span class={"badge badge-sm mr-2 " + cls}>✓</span>
6263
{esc(x)}
64+
{getCheckDesc(x) && (
65+
<span class="text-xs opacity-50 ml-2">{getCheckDesc(x)}</span>
66+
)}
6367
</li>
6468
)}
6569
</For>
@@ -114,43 +118,58 @@ export function RunDetail({
114118
<span class="badge badge-ghost">{esc(d().closedNote)}</span>
115119
)}
116120
</div>
121+
<Show when={score()?.status === "incomplete"}>
122+
<div class="alert alert-warning my-2">
123+
<span class="text-sm">
124+
评测不完整 — agent {run().piTimedOut ? "超时" : "异常退出"}
125+
{run().piExitReason ? ` (${run().piExitReason})` : ""}，pack 未产出，仅执行了{" "}
126+
{score()?.score?.total ?? 0} 项静态检查
127+
</span>
128+
</div>
129+
</Show>
117130
{score()?.summary && (
118131
<p class="text-sm mt-2">
119132
<span class="font-bold">结论: </span>
120133
{esc(score()!.summary)}
121134
</p>
122135
)}
123136

137+
<p class="text-xs opacity-50 mb-1">指标由前端从 session JSONL 实时计算，可能与列表页预计算值略有差异</p>
124138
<div class="stats stats-shadow w-full my-3">
125139
<div class="stat">
126140
<div class="stat-value text-xl">
127141
{fmtSec(sessionMetrics()?.timeline?.wallSeconds)}
128142
</div>
129143
<div class="stat-title text-xs">wall (session observed)</div>
144+
<div class="text-xs opacity-50">{METRIC_DEFINITIONS.wall}</div>
130145
</div>
131146
<div class="stat">
132147
<div class="stat-value text-xl">
133148
{fmtNum(sessionMetrics()?.tokens?.totalTokens)}
134149
</div>
135150
<div class="stat-title text-xs">tokens</div>
151+
<div class="text-xs opacity-50">{METRIC_DEFINITIONS.tokens}</div>
136152
</div>
137153
<div class="stat">
138154
<div class="stat-value text-xl">
139155
{esc(sessionMetrics()?.agent?.modelRequestCount)}
140156
</div>
141157
<div class="stat-title text-xs">requests</div>
158+
<div class="text-xs opacity-50">{METRIC_DEFINITIONS.requests}</div>
142159
</div>
143160
<div class="stat">
144161
<div class="stat-value text-xl">
145162
{esc(sessionMetrics()?.agent?.toolCallCount)}
146163
</div>
147164
<div class="stat-title text-xs">toolCalls</div>
165+
<div class="text-xs opacity-50">{METRIC_DEFINITIONS.toolCalls}</div>
148166
</div>
149167
<div class="stat">
150168
<div class="stat-value text-xl">
151169
{esc(sessionMetrics()?.agent?.toolResultErrorCount ?? 0)}
152170
</div>
153171
<div class="stat-title text-xs">errors</div>
172+
<div class="text-xs opacity-50">{METRIC_DEFINITIONS.errors}</div>
154173
</div>
155174
</div>
156175

@@ -202,6 +221,11 @@ export function RunDetail({
202221
<div class="card bg-base-200 mt-4">
203222
<div class="card-body">
204223
<h3 class="text-sm font-bold mb-1">评测明细</h3>
224+
<p class="text-xs opacity-60 mb-2">
225+
{score()?.status === "incomplete"
226+
? `仅 ${score()?.score?.total ?? 0} 项静态检查（pack 未产出，运行时检查未执行）`
227+
: `${EVAL_STRUCTURE.static.total} ${EVAL_STRUCTURE.static.label} + ${EVAL_STRUCTURE.runtime.total} ${EVAL_STRUCTURE.runtime.label} = ${EVAL_STRUCTURE.fullTotal} 项总检查`}
228+
</p>
205229
<Show
206230
when={scoreCheck()}
207231
fallback={
@@ -215,25 +239,25 @@ export function RunDetail({
215239
<div class="grid grid-cols-1 md:grid-cols-2 gap-2">
216240
<div class="bg-base-300 rounded-lg p-3">
217241
<h4 class="text-xs font-bold text-success">
218-
静态通过 ({score()?.static?.passed?.length ?? 0})
242+
静态通过 · 文件结构与内容验证 ({score()?.static?.passed?.length ?? 0})
219243
</h4>
220244
{list(score()?.static?.passed, "badge-success")}
221245
</div>
222246
<div class="bg-base-300 rounded-lg p-3">
223247
<h4 class="text-xs font-bold text-error">
224-
静态失败 ({score()?.static?.failed?.length ?? 0})
248+
静态失败 · 文件结构与内容验证 ({score()?.static?.failed?.length ?? 0})
225249
</h4>
226250
{list(score()?.static?.failed, "badge-error")}
227251
</div>
228252
<div class="bg-base-300 rounded-lg p-3">
229253
<h4 class="text-xs font-bold text-success">
230-
运行时通过 ({score()?.runtime?.passed?.length ?? 0})
254+
运行时通过 · Docker HTTP 调用验证 ({score()?.runtime?.passed?.length ?? 0})
231255
</h4>
232256
{list(score()?.runtime?.passed, "badge-success")}
233257
</div>
234258
<div class="bg-base-300 rounded-lg p-3">
235259
<h4 class="text-xs font-bold text-error">
236-
运行时失败 ({score()?.runtime?.failed?.length ?? 0})
260+
运行时失败 · Docker HTTP 调用验证 ({score()?.runtime?.failed?.length ?? 0})
237261
</h4>
238262
{list(score()?.runtime?.failed, "badge-error")}
239263
{score()?.runtime?.blocked?.length ? (

‎lab/src/views/RunList.tsx‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -130,9 +130,9 @@ export function RunList(props: {
130130
<th>eval</th>
131131
<th>score</th>
132132
<th>model</th>
133-
<th class="text-right">req</th>
134-
<th class="text-right">tokens</th>
135-
<th class="text-right">wall</th>
133+
<th class="text-right" title="模型请求次数">req</th>
134+
<th class="text-right" title="input+output+cache+reasoning 总和">tokens</th>
135+
<th class="text-right" title="session 时间戳跨度">wall</th>
136136
<th>artifact modified (UTC)</th>
137137
</tr>
138138
</thead>
@@ -177,11 +177,22 @@ export function RunList(props: {
177177
</span>
178178
</td>
179179
<td>
180-
<span class={"badge badge-sm " + evalBadge(r.evalStatus)}>
180+
<span
181+
class={"badge badge-sm " + evalBadge(r.evalStatus)}
182+
title={r.evalStatus === "incomplete" ? "agent 超时或崩溃，pack 未产出，评测提前终止" : undefined}
183+
>
181184
{esc(r.evalStatus)}
182185
</span>
183186
</td>
184-
<td class="font-mono text-xs">{esc(r.evalScore ?? "-")}</td>
187+
<td
188+
class="font-mono text-xs"
189+
title={r.evalStatus === "incomplete" ? "pack 未产出，仅执行 2 项静态检查，运行时检查未执行" : "16 项静态检查 + 8 项运行时验证 = 24 项"}
190+
>
191+
{esc(r.evalScore ?? "-")}
192+
{r.evalStatus === "incomplete" && r.evalScore && (
193+
<span class="text-xs opacity-50 ml-1">(仅静态)</span>
194+
)}
195+
</td>
185196
<td class="text-xs opacity-70">{esc(r.model)}</td>
186197
<td class="text-right">{esc(r.requests)}</td>
187198
<td class="text-right">{esc(r.tokens)}</td>

‎lab/src/views/SessionDetail.tsx‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { createResource, createMemo, For } from "solid-js";
22
import { api } from "../api";
33
import { esc, fmtSec, fmtNum } from "../lib/format";
4-
import { computeSessionMetrics } from "../lib/metrics";
4+
import { computeSessionMetrics, METRIC_DEFINITIONS } from "../lib/metrics";
55

66
export function SessionDetail({
77
id,
@@ -62,30 +62,35 @@ export function SessionDetail({
6262
{fmtSec(sm()?.timeline?.wallSeconds)}
6363
</div>
6464
<div class="stat-title text-xs">wall (session observed)</div>
65+
<div class="text-xs opacity-50">{METRIC_DEFINITIONS.wall}</div>
6566
</div>
6667
<div class="stat">
6768
<div class="stat-value text-xl">
6869
{fmtNum(sm()?.tokens?.totalTokens)}
6970
</div>
7071
<div class="stat-title text-xs">tokens</div>
72+
<div class="text-xs opacity-50">{METRIC_DEFINITIONS.tokens}</div>
7173
</div>
7274
<div class="stat">
7375
<div class="stat-value text-xl">
7476
{esc(sm()?.agent?.modelRequestCount)}
7577
</div>
7678
<div class="stat-title text-xs">requests</div>
79+
<div class="text-xs opacity-50">{METRIC_DEFINITIONS.requests}</div>
7780
</div>
7881
<div class="stat">
7982
<div class="stat-value text-xl">
8083
{esc(sm()?.agent?.toolCallCount)}
8184
</div>
8285
<div class="stat-title text-xs">toolCalls</div>
86+
<div class="text-xs opacity-50">{METRIC_DEFINITIONS.toolCalls}</div>
8387
</div>
8488
<div class="stat">
8589
<div class="stat-value text-xl">
8690
{esc(sm()?.agent?.toolResultErrorCount ?? 0)}
8791
</div>
8892
<div class="stat-title text-xs">errors</div>
93+
<div class="text-xs opacity-50">{METRIC_DEFINITIONS.errors}</div>
8994
</div>
9095
</div>
9196

0 commit comments

Comments
 (0)