Plugin page

PRODUCT.md

@@ -0,0 +1,36 @@
+# Product
+
+## Register
+
+product
+
+## Users
+
+Model and evaluation engineers, plugin maintainers, and technically informed consumers inspecting Ask Gina evaluation results. They need to compare candidate behavior, understand benchmark conditions, and trace every displayed value to a reviewed public artifact without reading private evaluator data.
+
+## Product Purpose
+
+Present Ask Gina's public evaluation results as an inspectable record of conformance. Success means a user can understand what ran, which checks passed or failed, how complete the evidence is, and whether two results are comparable without the interface inventing scores, availability, provenance, or ranking policy.
+
+## Brand Personality
+
+Precise, transparent, and quietly editorial. The interface should feel like a trustworthy technical ledger with the warmth of the Ask Gina brand, not a promotional model race.
+
+## Anti-references
+
+- Rank-first AI leaderboards that imply authority without comparable cohorts or declared policy.
+- Finance dashboards that use trading aesthetics, hype, or decorative precision to overstate evidence.
+- Opaque aggregate scores, zero-filled missing data, and charts derived from synthetic assumptions.
+- Dense evaluator tooling that exposes implementation details instead of the reviewed public contract.
+
+## Design Principles
+
+- Show the evidence behind every comparison.
+- Preserve unavailable, withheld, incomplete, synthetic, and unranked states as first-class information.
+- Compare only results with matching declared benchmark conditions.
+- Keep public presentation separate from grading, publication policy, and private evaluator inputs.
+- Prefer legible counts, units, and provenance over ornamental scoring.
+
+## Accessibility & Inclusion
+
+Support keyboard navigation, visible focus, semantic headings and tables, screen-reader descriptions for data graphics, color-independent status communication, reduced-motion preferences, and responsive use on narrow screens. Missing or adverse states must always be expressed in text rather than by color alone.

README.md

@@ -21,9 +21,10 @@ The contracts and SDK packages have no CommonJS, browser, edge, or subpath entry
 ## Public evals app
 
 `apps/evals` is a private React/Vite app with the Ask Gina landing-page typography,
-watercolor artwork, UI components, and Storybook. It has a leaderboard, model
-profiles, task evidence explorer, and methodology page. All scores and traces are
-illustrative fixtures, not measured benchmarks. It makes no live tool or wallet calls.
+watercolor artwork, UI components, and Storybook. Its unranked comparison and model
+pages consume a canonical synthetic publication through the verified public-results
+boundary; the task evidence explorer remains an illustrative fixture. Synthetic data
+is not a measured benchmark. The app makes no live tool or wallet calls.
 
 ```sh
 bun run evals:dev       # App on port 5173

apps/evals/__tests__/public-comparison.test.ts

@@ -0,0 +1,83 @@
+import publicationRaw from "../../../ai_docs/evals-handoff/planning/fixtures/synthetic-publication-correction-rev2.json?raw";
+import indexRaw from "../../../ai_docs/evals-handoff/planning/fixtures/synthetic-index.json?raw";
+import { describe, expect, it } from "vitest";
+import {
+  buildPublicComparisonCatalog,
+  loadPublicComparisonCatalog,
+} from "../src/lib/public-comparison";
+import { parsePublicArtifact } from "../src/lib/public-results";
+
+const encoded = (value: string): ArrayBuffer => new TextEncoder().encode(value).buffer;
+
+describe("public comparison adapter", () => {
+  it("verifies the canonical publication and preserves public metric availability", () =>
+    loadPublicComparisonCatalog({ publicationRaw, indexRaw }).then((catalog) => {
+      const row = catalog.cohorts[0]?.rows[0];
+
+      expect(catalog.cohorts).toHaveLength(1);
+      expect(catalog.withdrawnCount).toBe(1);
+      expect(row?.metrics.passRate).toMatchObject({
+        availability: "available",
+        value: 0.875,
+        numerator: 7,
+        denominator: 8,
+      });
+      expect(row?.metrics.latencyP50).toMatchObject({ availability: "available", value: 1200 });
+      expect(row?.metrics.tokenUsage).toMatchObject({ availability: "available", value: 9280 });
+      expect(row?.metrics.answerAccuracy).toEqual({
+        availability: "not_evaluated",
+        reason: "no_declared_method",
+        unit: "unavailable",
+      });
+      expect(row?.unrankedReasons).toEqual(["pilot", "synthetic"]);
+    }));
+
+  it("rejects publication bytes that no longer match the index", () => {
+    const changed = publicationRaw.replace(
+      "synthetic-reasoning-medium",
+      "synthetic-reasoning-high",
+    );
+    return expect(
+      loadPublicComparisonCatalog({ publicationRaw: changed, indexRaw }),
+    ).rejects.toThrow("SHA-256");
+  });
+
+  it("separates publications whose declared benchmark conditions differ", () => {
+    const publicationArtifact = parsePublicArtifact(encoded(publicationRaw));
+    const indexArtifact = parsePublicArtifact(encoded(indexRaw));
+    if (publicationArtifact.kind !== "publication" || indexArtifact.kind !== "index") {
+      throw new Error("Expected canonical public fixtures");
+    }
+    if (publicationArtifact.publication.content.kind !== "result") {
+      throw new Error("Expected a result publication");
+    }
+    const second = {
+      ...publicationArtifact.publication,
+      publicationId: "synthetic-publication-other-target",
+      revisionId: "synthetic-publication-other-target-rev1",
+      revision: 1,
+      supersedes: null,
+      content: {
+        kind: "result" as const,
+        result: {
+          ...publicationArtifact.publication.content.result,
+          resultId: "synthetic-result-other-target",
+          benchmark: {
+            ...publicationArtifact.publication.content.result.benchmark,
+            target: "synthetic-target-other",
+          },
+        },
+      },
+    };
+
+    const catalog = buildPublicComparisonCatalog(
+      [publicationArtifact.publication, second],
+      indexArtifact.index,
+    );
+    expect(catalog.cohorts).toHaveLength(2);
+    expect(catalog.cohorts.map((cohort) => cohort.conditions.target)).toEqual([
+      "synthetic-target-gina-mcp",
+      "synthetic-target-other",
+    ]);
+  });
+});

apps/evals/index.html

@@ -5,7 +5,7 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <meta name="theme-color" content="#ffffff" />
     <meta name="description" content="Ask Gina public evaluation results and methodology." />
-    <title>Ask Gina Evals</title>
+    <title>Ask Gina</title>
   </head>
   <body>
     <div id="root"></div>

apps/evals/src/App.stories.tsx

@@ -1,5 +1,6 @@
 import type { Meta, StoryObj } from "@storybook/react-vite";
 import App, { MethodologyPage } from "./App";
+import { syntheticFieldCatalog } from "./stories/public-comparison-fixtures";
 
 const meta = {
   title: "Evals/Public pages",
@@ -12,7 +13,7 @@ export default meta;
 type Story = StoryObj<typeof meta>;
 
 export const RoutedApp: Story = {
-  render: () => <App />,
+  render: () => <App catalog={syntheticFieldCatalog} />,
 };
 
 export const Methodology: Story = {

apps/evals/src/App.tsx

@@ -1,11 +1,11 @@
 import { useEffect, useState } from "react";
 import { ArrowUpRight } from "lucide-react";
-import { dataset } from "./data";
 import { PageShell, Panel } from "./components/eval-ui";
 import { LeaderboardPage } from "./pages/leaderboard";
 import { ModelProfilePage } from "./pages/model-profile";
 import { TaskExplorerPage } from "./pages/task-explorer";
 import { HandoffPage } from "./pages/handoff";
+import type { PublicComparisonCatalog } from "./lib/public-comparison";
 
 export function MethodologyPage() {
   return (
@@ -26,21 +26,21 @@ export function MethodologyPage() {
           <Panel title="First, a note on the data">
             <div className="eval-method-body">
               <p>
-                Every model score, timing, price, tool trace, and distribution on these pages is a
-                synthetic design fixture. Model names identify the intended comparison layout, not
-                an evaluation that has taken place.
+                The comparison and model pages consume a verified synthetic publication through the
+                public v1 result contract. The task explorer remains an invented design fixture.
               </p>
               <p>
-                The dataset and run labels are illustrative too. These results should not inform
-                model selection or financial decisions.
+                Synthetic values demonstrate the consumer boundary. They should not inform model
+                selection or financial decisions.
               </p>
             </div>
           </Panel>
           <Panel title="What a task measures">
             <div className="eval-method-body">
               <p>
-                Tasks cover portfolio analysis, spot markets, perpetuals, and prediction markets.
-                Each asks the agent to answer a financial question using read-only tools.
+                The public result measures conformance across routing, arguments, safety,
+                completion, and skill activation. Each dimension retains passed, failed, and
+                not-applicable counts.
               </p>
               <ul>
                 <li>Choose the right tools for the question.</li>
@@ -53,27 +53,25 @@ export function MethodologyPage() {
           <Panel title="Reading the scores">
             <div className="eval-method-body">
               <p>
-                Pass rate is the proportion of tasks that meet the rubric. Tool selection accuracy
-                measures whether the agent chose the expected tools. Task scores summarize the
-                individual rubric checks.
+                Pass rate is the proportion of observed attempts that pass with complete coverage.
+                Attempt counts, unique cases, retained evidence, and coverage remain separate.
               </p>
               <p>
-                The preview uses {dataset.tasks.toLocaleString("en-US")} tasks split equally across
-                four families. Displayed uncertainty ranges and histogram counts demonstrate the
-                proposed visual treatment. They are not computed confidence intervals.
+                Answer accuracy, USD cost, uncertainty, task-family groupings, score distributions,
+                and ordinal rankings are unavailable until a separately versioned method declares
+                them.
               </p>
             </div>
           </Panel>
           <Panel title="Latency, cost, and reproducibility">
             <div className="eval-method-body">
               <p>
-                Median latency is elapsed time per task. Cost is the estimated model cost per task
-                in USD. A published benchmark would need pinned model versions, tool definitions,
-                dataset, repetitions, and pricing assumptions.
+                Latency uses exported p50, p95, and maximum attempt durations. Token totals report
+                only retained observations. Missing samples never mean a free or zero-cost run.
               </p>
               <p>
-                The open-source runner is separate from this preview. The task explorer contains
-                sanitized synthetic examples, not exported production conversations.
+                Comparisons require matching suite, fixture, catalog, target, account class,
+                clean-chat setting, and repetitions. Model labels alone are not sufficient.
               </p>
               <a
                 className="eval-text-link"
@@ -95,7 +93,7 @@ export function MethodologyPage() {
   );
 }
 
-export default function App() {
+export default function App({ catalog }: { catalog?: PublicComparisonCatalog }) {
   const [route, setRoute] = useState(() => window.location.hash.slice(1) || "/leaderboard");
   useEffect(() => {
     const handleRoute = () => {
@@ -117,14 +115,14 @@ export default function App() {
           : route === "/handoff"
             ? "Public results"
             : "Leaderboard";
-    document.title = `${section} · Ask Gina Evals`;
+    document.title = `${section} · Ask Gina`;
   }, [route]);
   if (route === "/models" || route.startsWith("/models/"))
-    return <ModelProfilePage key={route} modelId={route.split("/")[2] || "kimi-k3"} />;
+    return <ModelProfilePage key={route} modelId={route.split("/")[2]} catalog={catalog} />;
   if (route === "/tasks") return <TaskExplorerPage />;
   if (route === "/methodology") return <MethodologyPage />;
   if (route === "/handoff") return <HandoffPage />;
-  if (route === "/leaderboard" || route === "/") return <LeaderboardPage />;
+  if (route === "/leaderboard" || route === "/") return <LeaderboardPage catalog={catalog} />;
   return (
     <PageShell active="leaderboard">
       <div className="eval-container eval-hero">

apps/evals/src/components/eval-ui.tsx

@@ -1,5 +1,6 @@
-import { useState, type CSSProperties, type ReactNode } from "react";
-import { ArrowUpRight, FlaskConical, X } from "lucide-react";
+import { type CSSProperties, type ReactNode } from "react";
+import { House, X } from "lucide-react";
+import askGinaLogoUrl from "../../../../docs/logo/light.svg";
 import { dataset, families, type EvalModel, type FamilyFilter, type PageId } from "../data";
 import { Button } from "./ui/button";
 import { DialogRoot, DialogContent, DialogTitle, DialogDescription } from "./ui/dialog";
@@ -8,25 +9,21 @@ type ShellPageId = PageId | "handoff";
 
 const navigation: readonly { id: ShellPageId; label: string; href: string }[] = [
   { id: "leaderboard", label: "Leaderboard", href: "#/leaderboard" },
-  { id: "models", label: "Models", href: "#/models/kimi-k3" },
+  { id: "models", label: "Models", href: "#/models" },
   { id: "tasks", label: "Tasks", href: "#/tasks" },
   { id: "methodology", label: "Methodology", href: "#/methodology" },
   { id: "handoff", label: "Public results", href: "#/handoff" },
 ];
 
 export function PageShell({ active, children }: { active: ShellPageId; children: ReactNode }) {
-  const [runOpen, setRunOpen] = useState(false);
   return (
     <div className="eval-app">
       <a className="eval-skip-link" href="#eval-main">
         Skip to content
       </a>
       <header className="eval-header">
-        <a className="eval-wordmark" href="#/leaderboard" aria-label="Ask Gina Evals home">
-          <strong>
-            Ask Gina<span className="eval-brand-dot">·</span>
-          </strong>
-          <span>Evals</span>
+        <a className="eval-wordmark" href="https://www.askgina.ai" aria-label="Ask Gina home">
+          <img src={askGinaLogoUrl} alt="Ask Gina" />
         </a>
         <nav className="eval-nav" aria-label="Main navigation">
           {navigation.map((item) => (
@@ -40,14 +37,10 @@ export function PageShell({ active, children }: { active: ShellPageId; children:
           ))}
         </nav>
         <div className="eval-header-actions">
-          <span className="eval-demo-label">
-            {active === "handoff" ? "Public JSON handoff" : "Design concept · Illustrative data"}
-          </span>
-          {active !== "handoff" && (
-            <Button className="eval-run-button" onClick={() => setRunOpen(true)}>
-              Run an evaluation <ArrowUpRight size={14} aria-hidden="true" />
-            </Button>
-          )}
+          <a className="eval-home-link" href="https://www.askgina.ai">
+            <House size={17} fill="currentColor" aria-hidden="true" />
+            <span>Home</span>
+          </a>
         </div>
       </header>
       <main id="eval-main" className="eval-main" tabIndex={-1}>
@@ -57,6 +50,8 @@ export function PageShell({ active, children }: { active: ShellPageId; children:
         <span>
           {active === "handoff" ? (
             "Public exports measure conformance, not answer accuracy or financial outcomes. Other pages use illustrative fixtures."
+          ) : active === "leaderboard" || active === "models" ? (
+            "Verified synthetic publication. Public v1 results are unranked and measure conformance only."
           ) : (
             <>
               {dataset.disclaimer} <a href="#/methodology">See methodology.</a>
@@ -65,34 +60,6 @@ export function PageShell({ active, children }: { active: ShellPageId; children:
         </span>
         <span>Open tools. Transparent results.</span>
       </footer>
-      <Modal title="Run an evaluation" open={runOpen} onClose={() => setRunOpen(false)}>
-        <div className="eval-run-intro">
-          <FlaskConical size={25} aria-hidden="true" />
-          <p>
-            This page is a design preview. It does not start live evaluations or connect to a
-            wallet.
-          </p>
-        </div>
-        <p>
-          The open-source eval runner lives in this repository. Replay its fixtures locally, or
-          follow the runner instructions to configure a live evaluation.
-        </p>
-        <pre className="eval-code" role="region" aria-label="Run instructions" tabIndex={0}>
-          <code>bun install --frozen-lockfile{"\n"}bun run eval:replay</code>
-        </pre>
-        <p className="eval-muted">
-          The replay command checks the repository's own fixtures. It does not produce the
-          illustrative model scores shown here.
-        </p>
-        <a
-          className="eval-text-link"
-          href="https://github.com/askgina/plugins/tree/main/packages/evals"
-          target="_blank"
-          rel="noreferrer"
-        >
-          Read eval runner instructions <ArrowUpRight size={14} aria-hidden="true" />
-        </a>
-      </Modal>
     </div>
   );
 }