Merge pull request #70 from getlantern/reflog/promoted-comparison-graph

reflog · web-flow · commit 6a6093269fca · 2026-07-22T11:51:07.000+02:00
experiments: promoted-vs-original goodput scatter
diff --git a/src/api/client.ts b/src/api/client.ts
@@ -711,10 +711,43 @@ export interface ExperimentSettingsResponse {
   applyDelaySeconds: number;
 }
 
+// One promoted track (an experiment's challenger) paired with its original (the
+// control), both measured over the same now-relative window in the experiment's
+// target market. Backs the promoted-vs-original scatter.
+export interface PromotedComparisonPoint {
+  experimentId: number;
+  targetCountry: string;
+  protocolName: string;
+  providerName: string;
+  locationName: string;
+  promotedTrackName: string;
+  originalTrackName: string;
+  promotedGoodput: number;   // median bytes/sec
+  originalGoodput: number;   // median bytes/sec
+  promotedSamples: number;
+  originalSamples: number;
+  promotedAt?: string;       // decided_at (when the experiment was promoted)
+}
+
+export interface PromotedComparisonResponse {
+  points: PromotedComparisonPoint[];
+  windowStart: string;
+  windowEnd: string;
+  windowHours: number;
+  statsError?: string;
+}
+
 export function fetchExperiments(): Promise<ExperimentsResponse> {
   return apiFetch("/experiments");
 }
 
+// fetchPromotedComparison returns every promoted track vs its original over a
+// now-relative window (hours). Server-side single SigNoz query, so the whole
+// scatter loads in one round-trip.
+export function fetchPromotedComparison(hours: number): Promise<PromotedComparisonResponse> {
+  return apiFetch("/experiments/promoted-comparison", { hours: String(hours) });
+}
+
 export function fetchExperimentDetail(id: number): Promise<ExperimentDetail> {
   return apiFetch("/experiments/detail", { id: String(id) });
 }
diff --git a/src/components/ExperimentsOverview.tsx b/src/components/ExperimentsOverview.tsx
@@ -1,15 +1,19 @@
-import { useEffect, useMemo, useState, type CSSProperties } from "react";
+import { useCallback, useEffect, useMemo, useState, type CSSProperties } from "react";
 import {
   Bar,
   BarChart,
   CartesianGrid,
   Legend,
   Line,
   LineChart,
+  ReferenceLine,
   ResponsiveContainer,
+  Scatter,
+  ScatterChart,
   Tooltip,
   XAxis,
   YAxis,
+  ZAxis,
 } from "recharts";
 import {
   abortExperiment,
@@ -20,13 +24,26 @@ import {
   type ExperimentStratum,
   type ExperimentSummary,
   type ExperimentPipeline,
+  type PromotedComparisonPoint,
 } from "../api/client";
-import { useExperiments, useExperimentDetail, useExperimentSettings } from "../hooks/useExperiments";
+import { useExperiments, useExperimentDetail, useExperimentSettings, usePromotedComparison } from "../hooks/useExperiments";
 import { useAuth } from "../hooks/useAuth";
 import ExperimentSettings from "./ExperimentSettings";
 
 const CHALLENGER_COLOR = "#00e5c8";
 const CONTROL_COLOR = "#f0a030";
+// Promoted-vs-original scatter: a promotion still beating its original is a win
+// (green, matching the "promoted" lifecycle color), still trailing is a loss.
+const WIN_COLOR = "#20e070";
+const LOSS_COLOR = "#ff4060";
+
+// Selectable now-relative windows for the promoted-vs-original scatter.
+const COMPARISON_WINDOWS: Array<{ label: string; hours: number }> = [
+  { label: "6h", hours: 6 },
+  { label: "24h", hours: 24 },
+  { label: "7d", hours: 168 },
+  { label: "30d", hours: 720 },
+];
 
 // Lifecycle status → display color, matching the dashboard's accent palette.
 const STATUS_COLORS: Record<string, string> = {
@@ -92,27 +109,47 @@ function StatusBadge({ status }: { status: string }) {
 
 // ── Lifecycle pipeline strip ──
 
-function PipelineStrip({ pipeline }: { pipeline: ExperimentPipeline | null }) {
+// PipelineStrip doubles as the table's status filter: each stage card is a toggle
+// button. A stage that's toggled off (dimmed + struck through) is hidden from the
+// experiments list below. Counts always reflect the full pipeline, not the filter.
+function PipelineStrip({ pipeline, hiddenStatuses, onToggle }: {
+  pipeline: ExperimentPipeline | null;
+  hiddenStatuses: Set<string>;
+  onToggle: (status: string) => void;
+}) {
   if (!pipeline) return null;
   return (
     <div style={card}>
-      <div style={sectionLabel}>Lifecycle pipeline</div>
+      <div style={{ ...sectionLabel, display: "flex", justifyContent: "space-between", gap: "1rem" }}>
+        <span>Lifecycle pipeline</span>
+        <span style={{ textTransform: "none", letterSpacing: 0 }}>click a stage to show / hide it in the list</span>
+      </div>
       <div style={{ display: "flex", gap: "0.4rem", flexWrap: "wrap" }}>
         {pipeline.order.map((status) => {
           const count = pipeline.counts[status] ?? 0;
           const color = STATUS_COLORS[status] || "#8890a0";
-          const active = count > 0;
+          const hasCount = count > 0;
+          const on = !hiddenStatuses.has(status);
           return (
-            <div key={status} style={{
-              flex: "1 1 6rem", minWidth: "5.5rem",
-              borderRadius: "var(--radius-sm)",
-              border: `1px solid ${active ? `${color}40` : "#ffffff0d"}`,
-              background: active ? `${color}12` : "#ffffff05",
-              padding: "0.5rem 0.6rem",
-            }}>
-              <div style={{ ...mono, fontSize: "1.3rem", fontWeight: 600, color: active ? color : "#5a6472" }}>{count}</div>
-              <div style={{ ...mono, fontSize: "0.5rem", textTransform: "uppercase", letterSpacing: "0.05em", color: "var(--text-muted)" }}>{status}</div>
-            </div>
+            <button
+              type="button"
+              key={status}
+              onClick={() => onToggle(status)}
+              aria-pressed={on}
+              title={on ? `Hide ${status} from the list` : `Show ${status} in the list`}
+              style={{
+                flex: "1 1 6rem", minWidth: "5.5rem", textAlign: "left",
+                appearance: "none", cursor: "pointer",
+                borderRadius: "var(--radius-sm)",
+                border: `1px solid ${on && hasCount ? `${color}40` : "#ffffff0d"}`,
+                background: on && hasCount ? `${color}12` : "#ffffff05",
+                padding: "0.5rem 0.6rem",
+                opacity: on ? 1 : 0.45,
+              }}
+            >
+              <div style={{ ...mono, fontSize: "1.3rem", fontWeight: 600, color: on && hasCount ? color : "#5a6472" }}>{count}</div>
+              <div style={{ ...mono, fontSize: "0.5rem", textTransform: "uppercase", letterSpacing: "0.05em", color: "var(--text-muted)", textDecoration: on ? "none" : "line-through" }}>{status}</div>
+            </button>
           );
         })}
       </div>
@@ -464,7 +501,7 @@ function ExperimentsTable({ experiments, selectedId, onSelect, onChanged }: {
   experiments: ExperimentSummary[]; selectedId: number | null; onSelect: (id: number | null) => void; onChanged: () => void;
 }) {
   if (experiments.length === 0) {
-    return <div style={{ ...mono, color: "var(--text-muted)", padding: "1rem" }}>No experiments yet.</div>;
+    return <div style={{ ...mono, color: "var(--text-muted)", padding: "1rem" }}>No experiments in the selected stages.</div>;
   }
   const headerStyle: CSSProperties = {
     display: "grid", gridTemplateColumns: colTemplate, gap: "0.5rem",
@@ -515,14 +552,204 @@ function ExperimentsTable({ experiments, selectedId, onSelect, onChanged }: {
   );
 }
 
+// ── Promoted vs original scatter ──
+
+const filterLabel: CSSProperties = {
+  ...mono, fontSize: "0.5rem", textTransform: "uppercase", letterSpacing: "0.05em",
+  color: "var(--text-muted)", marginBottom: "0.2rem",
+};
+
+const filterSelect: CSSProperties = {
+  ...mono, fontSize: "0.65rem", color: "var(--text-primary)",
+  background: "#ffffff08", border: "1px solid #ffffff10",
+  borderRadius: "var(--radius-sm)", padding: "0.35rem 0.55rem",
+};
+
+function chip(active: boolean): CSSProperties {
+  return {
+    ...mono, fontSize: "0.6rem", padding: "0.3rem 0.7rem", borderRadius: "var(--radius-sm)",
+    cursor: "pointer", userSelect: "none", appearance: "none",
+    background: active ? "var(--accent-primary-dim)" : "#ffffff08",
+    color: active ? "var(--accent-primary)" : "var(--text-muted)",
+    border: `1px solid ${active ? "#00e5c830" : "#ffffff10"}`,
+  };
+}
+
+function ComparisonTooltip({ active, payload }: {
+  active?: boolean;
+  payload?: Array<{ payload: PromotedComparisonPoint }>;
+}) {
+  if (!active || !payload || payload.length === 0) return null;
+  const p = payload[0].payload;
+  const delta = p.originalGoodput > 0 ? ((p.promotedGoodput - p.originalGoodput) / p.originalGoodput) * 100 : 0;
+  const deltaColor = delta >= 0 ? WIN_COLOR : LOSS_COLOR;
+  return (
+    <div style={{ ...mono, fontSize: "0.62rem", background: "var(--bg-secondary)", border: "1px solid #ffffff14", borderRadius: "var(--radius-sm)", padding: "0.5rem 0.6rem", lineHeight: 1.5 }}>
+      <div style={{ color: "var(--text-muted)" }}>#{p.experimentId} · {p.targetCountry} · {p.protocolName || "—"}{p.providerName ? ` · ${p.providerName}` : ""}</div>
+      <div><span style={{ color: WIN_COLOR }}>{p.promotedTrackName}</span> <span style={{ color: "var(--text-muted)" }}>(promoted)</span></div>
+      <div><span style={{ color: CONTROL_COLOR }}>{p.originalTrackName}</span> <span style={{ color: "var(--text-muted)" }}>(original)</span></div>
+      <div style={{ marginTop: "0.25rem" }}>promoted: {formatBytesPerSec(p.promotedGoodput)} <span style={{ color: "var(--text-muted)" }}>({p.promotedSamples} sess)</span></div>
+      <div>original: {formatBytesPerSec(p.originalGoodput)} <span style={{ color: "var(--text-muted)" }}>({p.originalSamples} sess)</span></div>
+      <div style={{ color: deltaColor, marginTop: "0.15rem" }}>{delta >= 0 ? "+" : ""}{delta.toFixed(0)}% vs original</div>
+    </div>
+  );
+}
+
+function PromotedComparison({ enabled }: { enabled: boolean }) {
+  const [hours, setHours] = useState(168);
+  const [country, setCountry] = useState("");
+  const [protocol, setProtocol] = useState("");
+  const [provider, setProvider] = useState("");
+  const { data, isLoading, error } = usePromotedComparison(enabled, hours);
+
+  const allPoints = useMemo(() => data?.points ?? [], [data]);
+
+  // Filter option lists come from all points (before country/protocol/provider
+  // filtering) so choosing one filter never empties the others' dropdowns.
+  const countries = useMemo(() => [...new Set(allPoints.map((p) => p.targetCountry).filter(Boolean))].sort(), [allPoints]);
+  const protocols = useMemo(() => [...new Set(allPoints.map((p) => p.protocolName).filter(Boolean))].sort(), [allPoints]);
+  const providers = useMemo(() => [...new Set(allPoints.map((p) => p.providerName).filter(Boolean))].sort(), [allPoints]);
+
+  const matchesFilters = useCallback((p: PromotedComparisonPoint) =>
+    (!country || p.targetCountry === country) &&
+    (!protocol || p.protocolName === protocol) &&
+    (!provider || p.providerName === provider),
+  [country, protocol, provider]);
+
+  // A point is plottable only when both arms have live samples in the window;
+  // otherwise its goodput is a meaningless 0 that would pile on the origin.
+  const hasSamples = (p: PromotedComparisonPoint) => p.promotedSamples > 0 && p.originalSamples > 0;
+  const filtered = useMemo(() => allPoints.filter((p) => hasSamples(p) && matchesFilters(p)), [allPoints, matchesFilters]);
+
+  // Hidden = points matching the active filters but lacking live samples, so the
+  // "N hidden" note stays consistent with the filtered track count above it.
+  const hidden = useMemo(() => allPoints.filter((p) => matchesFilters(p) && !hasSamples(p)).length, [allPoints, matchesFilters]);
+
+  const wins = filtered.filter((p) => p.promotedGoodput >= p.originalGoodput);
+  const losses = filtered.filter((p) => p.promotedGoodput < p.originalGoodput);
+  const axisMax = useMemo(() => {
+    // reduce (not Math.max(...spread)) so any number of points is safe.
+    const m = filtered.reduce((acc, p) => Math.max(acc, p.promotedGoodput, p.originalGoodput), 0);
+    return m > 0 ? m * 1.08 : 1; // headroom so parity line + points aren't clipped
+  }, [filtered]);
+
+  const hasFilters = Boolean(country || protocol || provider);
+
+  return (
+    <div style={card}>
+      <div style={{ display: "flex", justifyContent: "space-between", alignItems: "flex-start", gap: "1rem", flexWrap: "wrap", marginBottom: "0.6rem" }}>
+        <div>
+          <div style={{ ...sectionLabel, marginBottom: "0.15rem" }}>Promoted vs original — median goodput</div>
+          <div style={{ ...mono, fontSize: "0.55rem", color: "var(--text-muted)" }}>
+            Each point is a promoted track vs the original it beat, over the last {COMPARISON_WINDOWS.find((w) => w.hours === hours)?.label ?? `${hours}h`}, in its target market. Above the parity line = promotion still winning.
+          </div>
+        </div>
+        <div style={{ display: "flex", gap: "0.3rem", alignItems: "center", flexWrap: "wrap" }}>
+          {COMPARISON_WINDOWS.map((w) => (
+            <button type="button" key={w.hours} onClick={() => setHours(w.hours)} style={chip(hours === w.hours)} aria-pressed={hours === w.hours}>{w.label}</button>
+          ))}
+        </div>
+      </div>
+
+      <div style={{ display: "flex", gap: "0.7rem", flexWrap: "wrap", alignItems: "flex-end", marginBottom: "0.6rem" }}>
+        <div style={{ display: "flex", flexDirection: "column", minWidth: 110 }}>
+          <span style={filterLabel}>Country</span>
+          <select style={filterSelect} value={country} onChange={(e) => setCountry(e.target.value)}>
+            <option value="">All</option>
+            {countries.map((c) => <option key={c} value={c}>{c}</option>)}
+          </select>
+        </div>
+        <div style={{ display: "flex", flexDirection: "column", minWidth: 130 }}>
+          <span style={filterLabel}>Protocol</span>
+          <select style={filterSelect} value={protocol} onChange={(e) => setProtocol(e.target.value)}>
+            <option value="">All</option>
+            {protocols.map((p) => <option key={p} value={p}>{p}</option>)}
+          </select>
+        </div>
+        <div style={{ display: "flex", flexDirection: "column", minWidth: 130 }}>
+          <span style={filterLabel}>Provider</span>
+          <select style={filterSelect} value={provider} onChange={(e) => setProvider(e.target.value)}>
+            <option value="">All</option>
+            {providers.map((p) => <option key={p} value={p}>{p}</option>)}
+          </select>
+        </div>
+        {hasFilters && (
+          <button type="button" onClick={() => { setCountry(""); setProtocol(""); setProvider(""); }} style={chip(false)}>Clear</button>
+        )}
+      </div>
+
+      {error ? (
+        <div style={{ ...mono, fontSize: "0.65rem", color: "var(--accent-danger, #ff4060)" }}>{error}</div>
+      ) : data?.statsError ? (
+        <div style={{ ...mono, fontSize: "0.62rem", color: "#e0a060", background: "#f0a03012", border: "1px solid #f0a03030", borderRadius: "var(--radius-sm)", padding: "0.5rem 0.7rem" }}>{data.statsError}</div>
+      ) : !data || (isLoading && filtered.length === 0) ? (
+        <div style={{ ...mono, fontSize: "0.65rem", color: "var(--text-muted)" }}>Loading comparison…</div>
+      ) : filtered.length === 0 ? (
+        <div style={{ ...mono, fontSize: "0.65rem", color: "var(--text-muted)" }}>
+          No promoted tracks with live samples in this window{hasFilters ? " for the selected filters" : ""}.
+        </div>
+      ) : (
+        <>
+          <ResponsiveContainer width="100%" height={340}>
+            <ScatterChart margin={{ top: 8, right: 16, bottom: 24, left: 16 }}>
+              <CartesianGrid strokeDasharray="3 3" stroke="#ffffff10" />
+              <XAxis
+                type="number" dataKey="originalGoodput" name="Original goodput"
+                domain={[0, axisMax]} tickFormatter={formatBytesPerSec}
+                tick={{ fontSize: 10, fill: "#8890a0" }}
+                label={{ value: "Original (control) goodput", position: "insideBottom", offset: -14, fontSize: 10, fill: "#8890a0" }}
+              />
+              <YAxis
+                type="number" dataKey="promotedGoodput" name="Promoted goodput"
+                domain={[0, axisMax]} tickFormatter={formatBytesPerSec} width={64}
+                tick={{ fontSize: 10, fill: "#8890a0" }}
+                label={{ value: "Promoted goodput", angle: -90, position: "insideLeft", fontSize: 10, fill: "#8890a0" }}
+              />
+              <ZAxis range={[60, 60]} />
+              <ReferenceLine
+                segment={[{ x: 0, y: 0 }, { x: axisMax, y: axisMax }]}
+                stroke="#8890a0" strokeDasharray="4 4" ifOverflow="hidden"
+              />
+              <Tooltip content={<ComparisonTooltip />} cursor={{ strokeDasharray: "3 3" }} />
+              <Legend wrapperStyle={{ fontSize: 11 }} />
+              <Scatter name="Promotion winning" data={wins} fill={WIN_COLOR} />
+              <Scatter name="Promotion losing" data={losses} fill={LOSS_COLOR} />
+            </ScatterChart>
+          </ResponsiveContainer>
+          <div style={{ ...mono, fontSize: "0.55rem", color: "var(--text-muted)", marginTop: "0.4rem" }}>
+            {filtered.length} promoted {filtered.length === 1 ? "track" : "tracks"} · {wins.length} winning · {losses.length} losing
+            {hidden > 0 && ` · ${hidden} hidden (no live samples in window)`}
+          </div>
+        </>
+      )}
+    </div>
+  );
+}
+
 // ── Top-level tab content ──
 
 export default function ExperimentsOverview({ enabled }: { enabled: boolean }) {
   const [view, setView] = useState<"experiments" | "settings">("experiments");
   const [selectedId, setSelectedId] = useState<number | null>(null);
+  // Status filter driven by the pipeline strip. Retired experiments are hidden by
+  // default — they're the bulk of terminal history and rarely what you're after.
+  const [hiddenStatuses, setHiddenStatuses] = useState<Set<string>>(() => new Set(["retired"]));
   const { experiments, pipeline, isLoading, hasLoaded, error, refresh } = useExperiments(enabled);
   const settings = useExperimentSettings(enabled);
 
+  const toggleStatus = useCallback((status: string) => {
+    setHiddenStatuses((prev) => {
+      const next = new Set(prev);
+      if (next.has(status)) next.delete(status); else next.add(status);
+      return next;
+    });
+  }, []);
+
+  const visibleExperiments = useMemo(
+    () => experiments.filter((e) => !hiddenStatuses.has(e.status)),
+    [experiments, hiddenStatuses],
+  );
+
   // Surface a banner when the core automation workers are paused.
   const automationOff = useMemo(() => {
     const ed = settings.settings?.editable ?? [];
@@ -559,11 +786,12 @@ export default function ExperimentsOverview({ enabled }: { enabled: boolean }) {
           {error && (
             <div style={{ ...mono, fontSize: "0.65rem", color: "var(--accent-danger, #ff4060)", background: "#ff406012", border: "1px solid #ff406030", borderRadius: "var(--radius-sm)", padding: "0.5rem 0.75rem" }}>{error}</div>
           )}
-          <PipelineStrip pipeline={pipeline} />
+          <PipelineStrip pipeline={pipeline} hiddenStatuses={hiddenStatuses} onToggle={toggleStatus} />
+          <PromotedComparison enabled={enabled && view === "experiments"} />
           {isLoading && !hasLoaded ? (
             <div style={{ ...mono, color: "var(--text-muted)", padding: "1rem" }}>Loading experiments…</div>
           ) : (
-            <ExperimentsTable experiments={experiments} selectedId={selectedId} onSelect={setSelectedId} onChanged={refresh} />
+            <ExperimentsTable experiments={visibleExperiments} selectedId={selectedId} onSelect={setSelectedId} onChanged={refresh} />
           )}
         </>
       )}
diff --git a/src/hooks/useExperiments.ts b/src/hooks/useExperiments.ts
