examples/rlm: Show rejected executor output

Keep successful JavaScript executions visible when their returned shape cannot be scored. Sanitize invalid output into a metadata-only summary and explain the semantic capability mismatch in the comparison UI.

Author: scuffi

examples/rlm/src/App.tsx

@@ -388,6 +388,11 @@ function Lane({
   const run = state.run;
   const metric = run?.metrics;
   const execution = latestExecution(messages);
+  const showsSemanticMismatch =
+    state.lane === "executor" &&
+    run?.sample.dataset === "oolongbench/oolong-synth" &&
+    run.sample.row === 1200 &&
+    run.score?.score === 0;
   const running =
     state.status === "loading" ||
     state.status === "running" ||
@@ -548,20 +553,36 @@ function Lane({
           </section>
         ) : null}
         {execution ? (
-          <details className="execution">
-            <summary>
-              {state.lane === "rlm" ? "Generated map/reduce module" : "Generated executor module"}
-              {" · "}
-              {formatBytes(new TextEncoder().encode(execution.source).byteLength)}
-            </summary>
-            <pre>{execution.source}</pre>
+          <>
+            <details className="execution">
+              <summary>
+                {state.lane === "rlm" ? "Generated map/reduce module" : "Generated executor module"}
+                {" · "}
+                {formatBytes(new TextEncoder().encode(execution.source).byteLength)}
+              </summary>
+              <pre>{execution.source}</pre>
+            </details>
             {execution.result !== undefined ? (
-              <div className="result">
-                <span>{state.lane === "rlm" ? "Reducer output" : "Execution output"}</span>
+              <section className="result">
+                <span>
+                  {state.lane === "rlm"
+                    ? "JavaScript reducer output"
+                    : "JavaScript execution output"}
+                </span>
                 <pre>{formatValue(execution.result)}</pre>
-              </div>
+              </section>
             ) : null}
-          </details>
+          </>
+        ) : null}
+        {showsSemanticMismatch ? (
+          <section className="capability-note">
+            <span>Capability mismatch</span>
+            <strong>JavaScript completed, but its output does not fit this task.</strong>
+            <p>
+              The records need semantic labels before code can count them. This lane has no model
+              capability; Structured RLM adds bounded <code>ws:model</code> map calls.
+            </p>
+          </section>
         ) : null}
         {state.error ? (
           <div className="error">

examples/rlm/src/layout.test.tsx

@@ -12,12 +12,19 @@ describe("RLM trace layout", () => {
   it("uses RLM map/reduce language instead of stale transcript and chat language", () => {
     expect(appSource).toContain("Structured RLM");
     expect(appSource).toContain("Model map");
-    expect(appSource).toContain("Reducer output");
+    expect(appSource).toContain("JavaScript reducer output");
     expect(appSource).toContain("Cancel comparison");
     expect(appSource).toContain("Waiting for the RLM map");
     expect(appSource).toContain("All three strategies are overlapping now");
     expect(appSource).toContain("Workers AI charges may apply");
     expect(appSource).not.toContain("Model response");
     expect(appSource).not.toContain("transcript");
   });
+
+  it("shows executor output separately from semantic capability mismatches", () => {
+    expect(appSource).toContain("JavaScript execution output");
+    expect(appSource).toContain("Capability mismatch");
+    expect(appSource).toContain("JavaScript completed, but its output does not fit this task.");
+    expect(appSource).toContain("bounded <code>ws:model</code> map calls");
+  });
 });

examples/rlm/src/styles.css

@@ -73,7 +73,7 @@ button { cursor:pointer; }
 .working span { color:var(--muted); font-size:7px; }
 .working button { display:flex; align-items:center; gap:4px; border:0; color:var(--red); background:transparent; font-size:8px; }
 .working button svg { width:9px; }
-.sample-card,.answer,.children,.execution { border:1px solid var(--line); border-radius:9px; background:white; }
+.sample-card,.answer,.children,.execution,.result,.capability-note { border:1px solid var(--line); border-radius:9px; background:white; }
 .sample-card { display:flex; flex-direction:column; gap:6px; padding:11px; }
 .sample-card>span,.answer>span { color:var(--orange); font-family:var(--mono); font-size:7px; font-weight:600; text-transform:uppercase; }
 .sample-card>strong { font-size:10px; line-height:1.45; }
@@ -101,9 +101,14 @@ button { cursor:pointer; }
 .execution { overflow:hidden; }
 .execution summary { padding:9px 10px; font-family:var(--mono); font-size:8px; cursor:pointer; }
 .execution>pre { max-height:320px; margin:0; padding:11px; overflow:auto; color:#e6e6de; background:#20201e; font-family:var(--mono); font-size:7.5px; line-height:1.55; white-space:pre-wrap; }
-.result { border-top:1px solid var(--line); }
-.result span { display:block; padding:7px 10px 0; color:var(--orange); font-family:var(--mono); font-size:7px; text-transform:uppercase; }
-.result pre { margin:0; padding:8px 10px 10px; overflow:auto; font-family:var(--mono); font-size:8px; }
+.result { overflow:hidden; }
+.result span { display:block; padding:8px 10px; border-bottom:1px solid var(--line); color:var(--orange); font-family:var(--mono); font-size:7px; text-transform:uppercase; }
+.result pre { max-height:180px; margin:0; padding:10px; overflow:auto; background:var(--soft); font-family:var(--mono); font-size:8px; line-height:1.5; white-space:pre-wrap; }
+.capability-note { padding:11px; border-color:#f1c69d; background:#fff9f3; }
+.capability-note>span { color:var(--orange); font-family:var(--mono); font-size:7px; font-weight:600; text-transform:uppercase; }
+.capability-note>strong { display:block; margin-top:5px; font-size:9px; }
+.capability-note p { margin:5px 0 0; color:var(--muted); font-size:8px; line-height:1.5; }
+.capability-note code { color:#333; font-family:var(--mono); font-size:7.5px; }
 .answer { display:grid; grid-template-columns:1fr auto; gap:8px; padding:11px; }
 .answer>strong { color:var(--green); font-family:var(--mono); font-size:18px; }
 .answer>div { grid-column:1/-1; display:grid; grid-template-columns:78px minmax(0,1fr); gap:7px; align-items:start; }

examples/rlm/worker/executor-agent.ts

@@ -165,7 +165,11 @@ export class ExecutorAgent extends AIChatAgent<ModelEnv, BenchmarkAgentState> {
           ...this.state,
           status: hasExecutionResult ? "completed" : "failed",
           finalAnswer: text,
-          error: hasExecutionResult ? null : "The model did not return an execution result.",
+          error: hasExecutionResult
+            ? null
+            : metrics.executionAttempts > 0
+              ? "Generated JavaScript execution failed."
+              : "The model did not call the JavaScript executor.",
           finishedAt: new Date().toISOString(),
           run,
         });

examples/rlm/worker/executor-tool.test.ts

@@ -30,7 +30,7 @@ describe("executor output sanitation", () => {
     });
   });
 
-  it("drops untyped, oversized, and error output", () => {
+  it("summarizes rejected output without exposing its values", () => {
     expect(
       sanitizeExecutorOutput({
         ...base,
@@ -46,9 +46,16 @@ describe("executor output sanitation", () => {
         exitCode: 0,
         stdout: "raw corpus",
         stderr: "",
-        result: { evidence: "raw corpus evidence" },
+        result: { diagnostic: "raw corpus evidence" },
       }),
-    ).not.toHaveProperty("result");
+    ).toMatchObject({
+      stdout: "",
+      result: {
+        accepted: false,
+        reason: "Expected a bounded scalar/list or an object containing answer.",
+        received: "object without answer",
+      },
+    });
     expect(
       sanitizeExecutorOutput({
         ...base,
@@ -57,7 +64,12 @@ describe("executor output sanitation", () => {
         stderr: "",
         result: { answer: "x".repeat(17 * 1024) },
       }),
-    ).not.toHaveProperty("result");
+    ).toMatchObject({
+      result: {
+        accepted: false,
+        received: "object with answer",
+      },
+    });
     expect(
       sanitizeExecutorOutput({
         ...base,

examples/rlm/worker/executor-tool.ts

@@ -5,7 +5,7 @@ export const EXECUTOR_BACKEND = "oolong-javascript";
 export const RLM_BACKEND = "oolong-rlm-javascript";
 
 type ExecWorkspace = Parameters<typeof createExecTool>[0]["workspace"];
-const MAX_SAFE_ANSWER_BYTES = 16 * 1024;
+const MAX_SAFE_RESULT_BYTES = 16 * 1024;
 
 export function createExecutorTool(
   workspace: ExecWorkspace,
@@ -45,17 +45,16 @@ export function sanitizeExecutorOutput(output: ExecToolOutput): ExecToolOutput {
   const base = { command: output.command, cwd: output.cwd, backend: output.backend };
   if ("error" in output) return { ...base, error: "Generated execution failed." };
 
-  const answer = safeAnswer(output.result);
   return {
     ...base,
     exitCode: output.exitCode,
     stdout: "",
     stderr: "",
-    ...(answer === undefined ? {} : { result: { answer } }),
+    result: safeResult(output.result),
   };
 }
 
-function safeAnswer(value: unknown): string | number | Array<string | number> | undefined {
+function safeResult(value: unknown): Record<string, unknown> {
   const answer = isRecord(value) && Object.hasOwn(value, "answer") ? value.answer : value;
   const valid =
     typeof answer === "string" ||
@@ -65,8 +64,23 @@ function safeAnswer(value: unknown): string | number | Array<string | number> |
         (item): item is string | number =>
           typeof item === "string" || (typeof item === "number" && Number.isFinite(item)),
       ));
-  if (!valid || encodedBytes(answer) > MAX_SAFE_ANSWER_BYTES) return undefined;
-  return answer;
+
+  if (valid && encodedBytes(answer) <= MAX_SAFE_RESULT_BYTES) return { answer };
+
+  return {
+    accepted: false,
+    reason: "Expected a bounded scalar/list or an object containing answer.",
+    received: describeResult(value),
+  };
+}
+
+function describeResult(value: unknown): string {
+  if (Array.isArray(value)) return `array with ${value.length} items`;
+  if (isRecord(value)) {
+    return Object.hasOwn(value, "answer") ? "object with answer" : "object without answer";
+  }
+  if (value === null) return "null";
+  return typeof value;
 }
 
 function isAsyncIterable(value: unknown): value is AsyncIterable<ExecToolOutput> {