feat: dashboard vertical change markets, config detection (#481)

* feat: dashboard vertical change markets, config detection

* fix: no-shell git markers, bounded dashboard size, non-throwing config detection

* fix: widen deploy marker window to 30 commits

10 markers spanned ~10 days, but the dashboard opens at -P3M, so nearly
the whole default window had no deploy attribution — the point of the
markers. 30 markers now covers 2026-06-11 onward, roughly two months.

Each marker costs ~1.1KB across the 11 attribution widgets it is copied
into; the synthesized prod body goes 50,674 -> 71,827 bytes against the
stack's 90KB synth guard and PutDashboard's 100KB hard limit.

Export DEPLOY_MARKER_COUNT and MAX_LABEL_LENGTH so the tests assert
against the constants instead of drifting copies (the truncation test had
already drifted to 80 vs an actual cap of 50), and derive the size-budget
test's worst case from them so raising either has to re-clear the budget.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* fix: previous endpoints

* feat: put deploy markers on the invocations-by-version widget

This widget dates each Lambda version's first traffic, and its own docstring
says you resolve version -> commit by lining that up against the nearest
deploy marker — but it sat in the unmarked ops group, so the comparison meant
eyeballing across two widgets with different y-axes. Marking it makes the
mapping readable in one place. At width 24 it is also the roomiest home for
31 markers on the dashboard.

Position is unchanged (fan-out-by-chain -> this -> quote latency); only the
annotations differ. Prod body goes 71,827 -> 74,762 bytes against the stack's
90KB synth guard.

Export the widget builder and fold it into the size-budget test's subset, so
that budget keeps counting every widget the stack actually annotates.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>

Author: alanhwu

bin/app.ts

@@ -49,6 +49,9 @@ export class APIPipeline extends Stack {
     const code = CodePipelineSource.connection('Uniswap/uniswapx-parameterization-api', 'main', {
       connectionArn:
         'arn:aws:codestar-connections:us-east-2:644039819003:connection/4806faf1-c31e-4ea2-a5bf-c6fc1fa79487',
+      // Full git clone (not a source zip) so the synth step can read git history:
+      // the dashboard derives its vertical deploy markers from `git log` at synth.
+      codeBuildCloneOutput: true,
     });
 
     const synthStep = new CodeBuildStep('Synth', {

bin/stacks/api-stack.ts

@@ -263,6 +263,11 @@ export class APIStack extends cdk.Stack {
         ANALYTICS_STREAM_ARN: firehoseStack.analyticsStreamArn,
       },
       timeout: Duration.seconds(30),
+      // NOTE: deliberately no currentVersionOptions.description commit stamping —
+      // a per-build description forces a new Version + provisioned-concurrency
+      // re-warm on every merge (incl. dashboard-only ones), causing cold-start
+      // blips. ExecutedVersion → commit resolves via the dashboard's deploy
+      // markers (version publish time vs merge time) instead.
     });
 
     const quoteLambdaAlias = new aws_lambda.Alias(this, `GetOrdersLiveAlias`, {

bin/stacks/deploy-markers.ts

@@ -0,0 +1,71 @@
+import { execFileSync } from 'child_process';
+
+export type VerticalAnnotation = {
+  label: string;
+  value: string; // ISO 8601 timestamp
+};
+
+// Markers are copied into every widget they annotate, so each one costs ~1.1KB of
+// dashboard body across the attribution widgets, against a hard 100KB PutDashboard
+// limit (the size guard in param-dashboard-stack.ts trips at 90KB). 30 markers puts
+// the body around 72KB. Raise these only with a fresh measurement — see the size
+// budget test in test/dashboards/deploy-markers.test.ts.
+export const MAX_LABEL_LENGTH = 50;
+// Sized to span the dashboard's -P3M default window rather than the last few days:
+// at this repo's merge cadence 30 markers is roughly a month of history.
+export const DEPLOY_MARKER_COUNT = 30;
+
+/**
+ * Hand-maintained markers for changes that move the latency graphs but leave no
+ * trace in this repo's git history (config-repo changes, partner-side events).
+ * Automatic coverage exists for the two common cases — this-repo merges (git-derived
+ * markers below) and RFQ config edits (the RFQ_CONFIG_CHANGED metric strip) — so
+ * this list is only for rare milestones worth labeling forever.
+ */
+export const MILESTONES: VerticalAnnotation[] = [
+  { value: '2026-08-07T01:30:00Z', label: 'bulk dead quoter removal (RFQ config)' },
+];
+
+/** Parses `git log --format=%H|%cI|%s` output into vertical annotations. */
+export function parseGitLog(raw: string): VerticalAnnotation[] {
+  return raw
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0)
+    .map((line) => {
+      const [sha, committedAt, ...subjectParts] = line.split('|');
+      return {
+        value: committedAt,
+        label: `${sha.slice(0, 7)} ${subjectParts.join('|')}`.slice(0, MAX_LABEL_LENGTH),
+      };
+    })
+    .filter((a) => !Number.isNaN(Date.parse(a.value)));
+}
+
+/**
+ * Vertical markers for the last merges to main, read from git at synth time. The
+ * pipeline's synth step has a full clone (codeBuildCloneOutput in bin/app.ts).
+ * Marker time is the merge-commit time; the new code serves ~15-30 min later when
+ * the pipeline finishes — invisible at the dashboard's 3-month default zoom, and
+ * the invocations-by-version widget gives the exact traffic-shift moment.
+ *
+ * execFileSync (not execSync): the format string contains `|`, which a shell would
+ * parse as a pipeline; execFileSync passes argv directly with no shell involved.
+ *
+ * Synth must never fail because history is unavailable (shallow checkout, artifact
+ * edge cases), so any error degrades to no markers — but LOUDLY, so a regression
+ * here is visible in the synth log instead of silently shipping a bare dashboard.
+ */
+export function deployMarkers(): VerticalAnnotation[] {
+  try {
+    const raw = execFileSync('git', ['log', '--first-parent', '-n', `${DEPLOY_MARKER_COUNT}`, '--format=%H|%cI|%s'], {
+      encoding: 'utf8',
+      stdio: ['ignore', 'pipe', 'pipe'],
+    });
+    return parseGitLog(raw);
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    console.warn(`deploy-markers: git history unavailable, dashboard will have no deploy markers: ${e}`);
+    return [];
+  }
+}

bin/stacks/param-dashboard-stack.ts

@@ -12,6 +12,7 @@ import {
   SoftQuoteMetricDimension,
 } from '../../lib/entities';
 import { ChainId, SUPPORTED_CHAINS } from '../../lib/util/chains';
+import { deployMarkers, MILESTONES, VerticalAnnotation } from './deploy-markers';
 
 export type MetricPath =
   | string
@@ -49,10 +50,14 @@ export type LambdaWidget = {
       };
     };
     annotations?: {
-      horizontal: {
+      horizontal?: {
         label?: string;
         value: number;
       }[];
+      vertical?: {
+        label?: string;
+        value: string; // ISO 8601 timestamp
+      }[];
     };
   };
 };
@@ -106,6 +111,94 @@ const WASTED_WAIT_COLORS = { p50: '#e34948', p90: '#d03b3b' };
 // rollup streams (soft+hard combined) that the shared quote injector emits.
 const combined = (metricName: string, opts: Exclude<MetricPath, string>): MetricPath[] => ['Uniswap', metricName, opts];
 
+/**
+ * Stamps event markers (this-repo deploys from git, plus hand-kept milestones) as
+ * vertical annotations on every time-series widget, so latency steps line up with
+ * the change that caused them. Non-graph widgets (tiles, logs) pass through.
+ */
+export const withEventMarkers = (widgets: LambdaWidget[], markers: VerticalAnnotation[]): LambdaWidget[] =>
+  markers.length === 0
+    ? widgets
+    : widgets.map((w) =>
+        w.properties.view === 'timeSeries'
+          ? {
+              ...w,
+              properties: {
+                ...w.properties,
+                annotations: {
+                  ...w.properties.annotations,
+                  vertical: [...(w.properties.annotations?.vertical ?? []), ...markers],
+                },
+              },
+            }
+          : w
+      );
+
+/**
+ * Config-repo changes leave no trace in this repo's git history, so they get their
+ * own marker source: the RFQ_CONFIG_CHANGED metric, emitted by the webhook config
+ * provider whenever a refresh observes a different filler config. Rendered as a
+ * thin strip on the same time axis as the story graphs above it — presence of a
+ * spike means "the config changed here"; its height is just warm-instance count.
+ */
+const ConfigChangeStripWidget = (region: string): LambdaWidget => ({
+  height: 3,
+  width: 24,
+  type: 'metric',
+  properties: {
+    // Two emission streams: quote lambdas publish through the request logger
+    // (dimensionless rollup), the fade-rate cron through its CircuitBreaker-
+    // dimensioned logger. Chart both so a change observed only by the cron
+    // still paints a spike.
+    metrics: [
+      combined(Metric.RFQ_CONFIG_CHANGED, { stat: 'Sum', label: 'observed by quote lambdas' }),
+      [
+        'Uniswap',
+        Metric.RFQ_CONFIG_CHANGED,
+        'Service',
+        CircuitBreakerMetricDimension.Service,
+        { stat: 'Sum', label: 'observed by circuit-breaker cron' },
+      ],
+    ],
+    view: 'timeSeries',
+    stacked: false,
+    region,
+    period: 300,
+    title: 'RFQ config changes observed (spike = filler config changed; height is not meaningful)',
+  },
+});
+
+/**
+ * The exact moment traffic shifted to each Lambda version — the precise complement
+ * to the git-derived markers, whose timestamps are merge time (~15-30 min before
+ * serving). Version numbers map to commits by lining a version's first-traffic
+ * time up with the nearest deploy marker (deliberately no per-version commit
+ * stamping: that would force a provisioned-concurrency re-warm on every merge),
+ * which is why this widget carries the markers itself rather than making you read
+ * the timestamp off a different graph.
+ */
+export const InvocationsByVersionWidget = (region: string, quoteLambdaFunctionName: string): LambdaWidget => ({
+  height: 6,
+  width: 24,
+  type: 'metric',
+  properties: {
+    metrics: [
+      [
+        {
+          expression: `SEARCH('{AWS/Lambda,FunctionName,Resource,ExecutedVersion} FunctionName="${quoteLambdaFunctionName}" MetricName="Invocations"', 'Sum', 300)`,
+          id: 'invocationsByVersion',
+          region,
+        },
+      ],
+    ],
+    view: 'timeSeries',
+    stacked: true,
+    region,
+    period: 300,
+    title: 'Soft quote invocations by Lambda version (exact deploy traffic-shift moments)',
+  },
+});
+
 // Per-service metric path builders. Soft and hard are charted separately: hard-quote
 // latency includes cosigning and the order post, a structurally different pipeline.
 const serviceMetric =
@@ -892,21 +985,30 @@ export class ParamDashboardStack extends cdk.NestedStack {
 
     const region = cdk.Stack.of(this).region;
 
-    new aws_cloudwatch.CfnDashboard(this, 'UniswapXParamDashboard', {
-      dashboardName: `UniswapXParamDashboard`,
-      dashboardBody: JSON.stringify({
-        periodOverride: 'inherit',
-        // Default to a 3-month window: the point of the top rows is the long-run
-        // downward staircase, not the last hour.
-        start: '-P3M',
-        // Widgets auto-flow in array order (no explicit x/y): story row first,
-        // attribution rows next, the pre-existing ops widgets after.
-        widgets: [
-          LatencyStoryRows(region),
-          PhaseDecompositionWidgets(region),
-          WastedWaitWidgets(region),
-          E2EByChainWidgets(region),
-          FanoutByChainWidgets(region),
+    // Markers are copied into every widget they annotate; the dashboard body has a
+    // hard 100KB PutDashboard limit (an unbounded regime measured ~122KB). They
+    // therefore go ONLY on the attribution graphs — story rows, phase
+    // decomposition, straggler tax, invocations-by-version — never the ops widgets,
+    // and both the marker count and label length are capped in deploy-markers.ts.
+    const eventMarkers = [...MILESTONES, ...deployMarkers()];
+
+    const dashboardBody = JSON.stringify({
+      periodOverride: 'inherit',
+      // Default to a 3-month window: the point of the top rows is the long-run
+      // downward staircase, not the last hour.
+      start: '-P3M',
+      // Widgets auto-flow in array order (no explicit x/y): story rows first,
+      // attribution rows next, the pre-existing ops widgets after.
+      widgets: [
+        withEventMarkers(LatencyStoryRows(region), eventMarkers),
+        [ConfigChangeStripWidget(region)],
+        withEventMarkers([PhaseDecompositionWidgets(region), WastedWaitWidgets(region)].flat(), eventMarkers),
+        [E2EByChainWidgets(region), FanoutByChainWidgets(region)].flat(),
+        // Marked deliberately, unlike the ops widgets below it: this widget's whole
+        // job is to date each version's first traffic, and that timestamp only means
+        // something read against the merge that produced the version.
+        withEventMarkers([InvocationsByVersionWidget(region, props.quoteLambda.functionName)], eventMarkers),
+        [
           LatencyWidget(region),
           RFQLatencyWidget(region),
           QuotesRequestedWidget(region),
@@ -919,7 +1021,22 @@ export class ParamDashboardStack extends cdk.NestedStack {
           FailingRFQLogsWidget(region, props.quoteLambda.logGroup.logGroupName),
           CircuitBreakerWidgets(region),
         ].flat(),
-      }),
+      ].flat(),
+    });
+
+    // Fail at synth, not at deploy: PutDashboard rejects bodies over 100KB with a
+    // stack-update failure. The margin absorbs CDK token expansion (function/log
+    // group names serialize as short placeholders here but resolve longer).
+    if (dashboardBody.length > 90_000) {
+      throw new Error(
+        `UniswapXParamDashboard body is ${dashboardBody.length} bytes; PutDashboard rejects >100KB. ` +
+          'Trim event markers or widgets (see deploy-markers.ts caps).'
+      );
+    }
+
+    new aws_cloudwatch.CfnDashboard(this, 'UniswapXParamDashboard', {
+      dashboardName: `UniswapXParamDashboard`,
+      dashboardBody,
     });
   }
 }

lib/cron/fade-rate-v2.ts

@@ -1,13 +1,14 @@
 import { DynamoDBClient } from '@aws-sdk/client-dynamodb';
 import { DynamoDBDocumentClient } from '@aws-sdk/lib-dynamodb';
+import { setGlobalMetric } from '@uniswap/smart-order-router';
 import { metricScope, MetricsLogger, Unit } from 'aws-embedded-metrics';
 import { ScheduledHandler } from 'aws-lambda/trigger/cloudwatch-events';
 import { EventBridgeEvent } from 'aws-lambda/trigger/eventbridge';
 import Logger from 'bunyan';
 
 import { ethers } from 'ethers';
 import { BETA_S3_KEY, PRODUCTION_S3_KEY, WEBHOOK_CONFIG_BUCKET } from '../constants';
-import { CircuitBreakerMetricDimension, Metric, metricContext } from '../entities';
+import { AWSMetricsLogger, CircuitBreakerMetricDimension, Metric, metricContext } from '../entities';
 import { checkDefined } from '../preconditions/preconditions';
 import { S3WebhookConfigurationProvider } from '../providers';
 import {
@@ -85,6 +86,14 @@ export const handler: ScheduledHandler = metricScope((metrics) => async (_event:
 async function main(metrics: MetricsLogger) {
   metrics.setNamespace('Uniswap');
   metrics.setDimensions(CircuitBreakerMetricDimension);
+  // The webhook config provider emits RFQ_CONFIG_CHANGED through the
+  // smart-order-router module-global metric. The quote lambdas bind it per
+  // request in their injector; without this binding here, the cron's
+  // fetchEndpoints() below would observe config changes but publish no
+  // datapoint. Cron emissions land under Service=CircuitBreaker (this logger's
+  // dimensions) — the dashboard's config-change strip charts that stream
+  // alongside the quote lambdas' dimensionless one.
+  setGlobalMetric(new AWSMetricsLogger(metrics));
 
   const sharedConfig: SharedConfigs = {
     Database: checkDefined(process.env.REDSHIFT_DATABASE),

lib/entities/aws-metrics-logger.ts

@@ -82,6 +82,12 @@ export enum Metric {
   // End-to-end handler latency on every response path (200, 404, thrown errors), unlike
   // QUOTE_LATENCY which fires only on 200s and is blind to slow 404s.
   QUOTE_E2E_LATENCY = 'QUOTE_E2E_LATENCY',
+  // Emitted when a config refresh observes a different RFQ filler config than the
+  // previous fetch on the same instance. Marks config-repo changes (filler adds/
+  // removals) on the latency dashboard, which no git-based deploy marker can see.
+  // Value is meaningless as a magnitude (each warm instance emits once): presence
+  // in a period = the config changed in that period.
+  RFQ_CONFIG_CHANGED = 'RFQ_CONFIG_CHANGED',
 
   // Metrics for circuit breaker
   CIRCUIT_BREAKER_V2_CONSECUTIVE_BLOCKS = 'CIRCUIT_BREAKER_V2_CONSECUTIVE_BLOCKS',