Simplification pass over the DO-reset resilience work

Architectural: the idempotent-read retry moves from 19 per-call-site
withDoResetRetry wraps (which had already drifted — providers.tsx left
getAiConfig unwrapped while OnboardingWizard wrapped it) to a single
withReadRetries proxy installed where useAuth creates the stub, keyed by
a method-level allowlist — idempotency is a property of the method, not
the call site. Future reads get the policy for free; writes and unlisted
methods pass through untouched.

Mechanical: the backend Proxy's RAW_USER_STUB symbol escape hatch is
replaced by caching the raw stub in a second field — no symbol, no
double casts, and the thrash guard compares against the trap's own
target. logRpcFailure now owns do-reset telemetry via a reportSite
option, collapsing the classify-report-log ritual at three action sites.
The workerd dead-capability message now lives once, in workshop-shared,
referenced by the backend matcher and the frontend classifier, so the
frontend canary guards both.

Block/line: the two coded-error families in api.ts share one
codedErrorFamily factory (membership derived from the message record
instead of hand-enumerated); the composer's send-hiccup hint is one
state value scoped by its render condition instead of a boolean, a ref
mirrored during render, and a comparison at set time; fake-timer
try/finally scaffolding in rpcErrors.test.ts becomes afterEach, twin
retry tests merge into it.each, and error fixtures share an rpcError
helper. New tests pin the chokepoint: listed reads retry once, writes
never do.

Author: ndisidore

packages/workshop-backend/src/server.ts

@@ -1,7 +1,7 @@
 import { RpcStub, RpcTarget, newWorkersRpcResponse } from "capnweb";
 import { validateRpc } from "capnweb-validate";
 import type { JWTPayload } from "jose";
-import { PublicApi, AuthenticatedApi, Overseer, GadgetMetadataWithTimestamps, AiChatAuthorInfo, AiModelConfig, AiGatewayInfo, AiModelProvider, ConnectedAccountsSubscriber, ConnectedAccountsFilter, GatekeeperVendorFilter, ObserverConfigCallback, BlueprintLibrarySummary, BlueprintPublicInfo, BlueprintUserSummary, BlueprintBindingAssignment, AgentSpawnerConfig, WorkpieceId, BLUEPRINT_SCREENSHOT_PATH_PREFIX, BLUEPRINT_SCREENSHOT_R2_PREFIX, blueprintScreenshotUrl, ServerConfig, CloudflareUsageInfo, CloudflareAccountOption, LoginAttempt, GatekeeperAppInfo, AdminApi, GatekeeperVendorInfo, OutputFormatOffer, ListOutputsResult, createOpenGadgetError, getOpenGadgetErrorCode, OPEN_GADGET_ERROR_CODES, AUTH_ERROR_CODES, createAuthError } from '@gadgets/workshop-shared/api';
+import { PublicApi, AuthenticatedApi, Overseer, GadgetMetadataWithTimestamps, AiChatAuthorInfo, AiModelConfig, AiGatewayInfo, AiModelProvider, ConnectedAccountsSubscriber, ConnectedAccountsFilter, GatekeeperVendorFilter, ObserverConfigCallback, BlueprintLibrarySummary, BlueprintPublicInfo, BlueprintUserSummary, BlueprintBindingAssignment, AgentSpawnerConfig, WorkpieceId, BLUEPRINT_SCREENSHOT_PATH_PREFIX, BLUEPRINT_SCREENSHOT_R2_PREFIX, blueprintScreenshotUrl, ServerConfig, CloudflareUsageInfo, CloudflareAccountOption, LoginAttempt, GatekeeperAppInfo, AdminApi, GatekeeperVendorInfo, OutputFormatOffer, ListOutputsResult, createOpenGadgetError, getOpenGadgetErrorCode, OPEN_GADGET_ERROR_CODES, AUTH_ERROR_CODES, createAuthError, WORKERD_DEAD_CAPABILITY_MESSAGE } from '@gadgets/workshop-shared/api';
 import type { UiFeatureFlags } from "@gadgets/workshop-shared/feature-flags";
 import { getServerConfig } from "./deployment-config.js";
 import { isPasswordAuthEnabled, getAuthGatekeeperAllowlist } from "./auth/config.js";
@@ -71,10 +71,6 @@ type Env = Cloudflare.Env & {
 
 // =======================================================================================
 
-// Escape hatch on the wrapped user-DO stub (see #wrapUserStub) exposing its raw target, so the
-// never-sent retry path can re-issue a call without re-entering the retry wrapper.
-const RAW_USER_STUB = Symbol("rawUserStub");
-
 @validateRpc()
 class AuthenticatedApiImpl extends RpcTarget implements AuthenticatedApi {
   constructor(private ctx: ExecutionContext, private env: Env,
@@ -91,25 +87,26 @@ class AuthenticatedApiImpl extends RpcTarget implements AuthenticatedApi {
   private adminSettings: DurableObjectNamespace<AdminSettings>;
   private users: DurableObjectNamespace<UserDurableObject>;
 
+  // One stub per session, because e-order (in-order delivery to the DO) is guaranteed per stub.
+  // A stub is permanently broken once its incarnation of the object resets, so the wrapper below
+  // drops both fields on a stub-breaking rejection and the next call re-resolves fresh ones.
+  // `rawUserStub` is the Proxy's target, kept so the never-sent retry can re-issue a call
+  // without re-entering the wrapper.
   private userStub?: DurableObjectStub<UserDurableObject>;
+  private rawUserStub?: DurableObjectStub<UserDurableObject>;
 
-  // E-order (in-order delivery to the DO) is guaranteed per stub, so the session shares one stub
-  // while it's healthy. But a stub is bound to one incarnation of the object and is permanently
-  // broken once that incarnation resets, so the wrapper below drops the cached stub the moment a
-  // call through it fails with a stub-breaking error, and the next call re-resolves a fresh one.
   private get user(): DurableObjectStub<UserDurableObject> {
-    return this.userStub ??= this.#wrapUserStub(this.users.get(this.userId));
+    return this.userStub ??=
+        this.#wrapUserStub(this.rawUserStub = this.users.get(this.userId));
   }
 
   // Calls on a stub whose DO already reset while idle reject flagless with the brokenness
   // reason; the flagged (`durableObjectReset`) error only reaches calls in flight at reset time.
-  // Verified in a workerd probe — see DO_RESET_MESSAGES in workshop-frontend/src/rpcErrors.ts,
-  // which pins the first string. The distinction matters: a flagless rejection matching one of
-  // these proves the call never reached the DO, so re-issuing it on a fresh stub cannot
-  // double-execute — safe even for writes.
+  // The distinction matters: a flagless rejection matching one of these proves the call never
+  // reached the DO, so re-issuing it on a fresh stub cannot double-execute — safe even for writes.
   static readonly #DEAD_CAPABILITY_MESSAGES = [
-    // What production resets leave as the brokenness reason.
-    "The execution context which hosts this callback is no longer running",
+    // What production resets leave as the brokenness reason; the frontend canary test pins it.
+    WORKERD_DEAD_CAPABILITY_MESSAGE,
     // What vitest-pool-workers' abortAllDurableObjects() leaves — never occurs in production;
     // listed so integration tests exercise the real never-sent retry path.
     "Application called abortAllDurableObjects().",
@@ -120,9 +117,8 @@ class AuthenticatedApiImpl extends RpcTarget implements AuthenticatedApi {
   // (see #DEAD_CAPABILITY_MESSAGES), re-issues it once on the fresh stub. Flagged errors are
   // rethrown untouched — the call may have executed, and the frontend owns that recovery.
   #wrapUserStub(stub: DurableObjectStub<UserDurableObject>): DurableObjectStub<UserDurableObject> {
-    const wrapped: DurableObjectStub<UserDurableObject> = new Proxy(stub, {
+    return new Proxy(stub, {
       get: (target, prop) => {
-        if (prop === RAW_USER_STUB) return target;
         const value = Reflect.get(target, prop);
         if (typeof value !== "function") return value;
         return (...args: unknown[]) => {
@@ -139,22 +135,21 @@ class AuthenticatedApiImpl extends RpcTarget implements AuthenticatedApi {
             if (flagged || neverSent) {
               // Guard against thrashing: concurrent failures on the same dead stub must not
               // each discard the replacement the first one already resolved.
-              if (this.userStub === wrapped) this.userStub = undefined;
+              if (this.rawUserStub === target) this.userStub = this.rawUserStub = undefined;
               if (neverSent) {
-                // Retry exactly once, against the raw target of the re-armed stub — going
-                // through the proxy again would retry unboundedly under repeated resets.
-                const raw = (this.user as unknown as Record<symbol, unknown>)[RAW_USER_STUB] as
-                    DurableObjectStub<UserDurableObject>;
-                return Reflect.apply(Reflect.get(raw, prop) as (...a: unknown[]) => unknown,
-                    raw, args);
+                // Retry exactly once, on the raw target of the re-armed cache — retrying through
+                // the proxy would retry unboundedly under repeated resets.
+                void this.user;
+                const fresh = this.rawUserStub!;
+                return Reflect.apply(Reflect.get(fresh, prop) as (...a: unknown[]) => unknown,
+                    fresh, args);
               }
             }
             throw err;
           });
         };
       },
     });
-    return wrapped;
   }
 
   #isAdmin(): boolean {

packages/workshop-frontend/src/BlueprintLandingPage.tsx

@@ -1,4 +1,4 @@
-import { logRpcFailure, withDoResetRetry } from './rpcErrors'
+import { logRpcFailure } from './rpcErrors'
 import { useState, useEffect, useCallback, useMemo, useRef, type ReactNode } from 'react'
 import { useNavigate, useParams, useRouter } from '@tanstack/react-router'
 import { RpcStub, RpcTarget } from 'capnweb'
@@ -121,7 +121,7 @@ export default function BlueprintLandingPage({ rpcStub }: Props) {
   // When authenticated, fetch models for binding assignment.
   useEffect(() => {
     if (isAuthenticated && authenticatedApi) {
-      withDoResetRetry(() => authenticatedApi.listModels())
+      authenticatedApi.listModels()
         .then(setModels)
         .catch(err => logRpcFailure('Failed to load models:', err))
     } else {
@@ -184,7 +184,7 @@ export default function BlueprintLandingPage({ rpcStub }: Props) {
       ready() {}
     }
 
-    withDoResetRetry(() => authenticatedApi.subscribeConnectedAccounts(new AccountsSubscriber()))
+    authenticatedApi.subscribeConnectedAccounts(new AccountsSubscriber())
       .then(stub => {
         if (cancelled) {
           stub[Symbol.dispose]()

packages/workshop-frontend/src/ChatInterface.tsx

@@ -1,4 +1,4 @@
-import { isDurableObjectResetError, isTransientRpcError, logRpcFailure, reportDoResetError } from "./rpcErrors";
+import { isTransientRpcError, logRpcFailure } from "./rpcErrors";
 import {
   Fragment,
   memo,
@@ -1842,11 +1842,10 @@ export const ChatInput = ({
   const [capsules, setCapsules] = useState<InputCapsule[]>([]);
   const [pendingAttachments, setPendingAttachments] = useState<PendingAttachment[]>([]);
   const [isSending, setIsSending] = useState(false);
-  const [sendHiccup, setSendHiccup] = useState(false);
-  const chatKeyRef = useRef(chatKey);
-  chatKeyRef.current = chatKey;
-
-  useEffect(() => setSendHiccup(false), [chatKey]);
+  // The chat the "may not have been sent" hint belongs to; the render condition scopes it, and
+  // leaving the chat dismisses it.
+  const [sendHiccup, setSendHiccup] = useState<{ chatKey?: number | null } | null>(null);
+  useEffect(() => setSendHiccup(null), [chatKey]);
   const [isAttachmentDragActive, setIsAttachmentDragActive] = useState(false);
   const [selectedSlashCommand, setSelectedSlashCommand] = useState<SelectedSlashCommand | null>(null);
   // The caret the slash command picker parses at. Deliberately updated only when it moves to a
@@ -2285,7 +2284,7 @@ export const ChatInput = ({
 
   const handleSend = async () => {
     if (sendInFlightRef.current || isSending || isBlocked) return;
-    setSendHiccup(false);
+    setSendHiccup(null);
     const attachmentsSnapshot = pendingAttachments;
     const readyAttachments = attachmentsSnapshot
       .filter((attachment) => attachment.uploadState === "ready" && attachment.ref)
@@ -2467,9 +2466,7 @@ export const ChatInput = ({
     const submittedChatKey = chatKey;
     void handleSend().catch((err) => {
       // The onSend handlers already log; the composer only needs the hint state.
-      if (isTransientRpcError(err) && chatKeyRef.current === submittedChatKey) {
-        setSendHiccup(true);
-      }
+      if (isTransientRpcError(err)) setSendHiccup({ chatKey: submittedChatKey });
     });
   };
 
@@ -3064,7 +3061,7 @@ export const ChatInput = ({
           </div>
         )}
         {draftUpdateBanner}
-        {sendHiccup && (
+        {sendHiccup && sendHiccup.chatKey === chatKey && (
           <div className="px-4 pt-2 text-xs text-kumo-warning">
             Connection hiccup — your message may not have been sent. Check the thread, then try again.
           </div>
@@ -5386,8 +5383,7 @@ function ChatInterface({
         );
       }
     } catch (err) {
-      if (isDurableObjectResetError(err)) reportDoResetError("chat.send", err);
-      if (!logRpcFailure("Failed to send message:", err)) {
+      if (!logRpcFailure("Failed to send message:", err, { reportSite: "chat.send" })) {
         toasts.add({ title: "Failed to send message", variant: "error" });
       }
       throw err;
@@ -5410,8 +5406,7 @@ function ChatInterface({
           message, model, capsules, attachments, formats);
       onNavigateToChatRef.current(newChatId);
     } catch (err) {
-      if (isDurableObjectResetError(err)) reportDoResetError("chat.new", err);
-      if (!logRpcFailure("Failed to create new chat:", err)) {
+      if (!logRpcFailure("Failed to create new chat:", err, { reportSite: "chat.new" })) {
         toasts.add({ title: "Failed to start conversation", variant: "error" });
       }
       throw err;

packages/workshop-frontend/src/ConnectAccountModal.tsx

@@ -4,7 +4,6 @@ import { RpcStub } from 'capnweb'
 import { AuthenticatedApi, GatekeeperVendorFilter } from '@gadgets/workshop-shared/api'
 import { VendorDescription } from '@gadgets/workshop-shared/gatekeeper'
 import VendorCard from './VendorCard'
-import { withDoResetRetry } from './rpcErrors'
 
 interface ConnectAccountModalProps {
   visible: boolean
@@ -42,7 +41,7 @@ export default function ConnectAccountModal({
     const fetchVendors = async () => {
       setVendorsLoading(true)
       try {
-        const vendorList = await withDoResetRetry(() => authenticatedApi.listGatekeeperVendors(filter))
+        const vendorList = await authenticatedApi.listGatekeeperVendors(filter)
         const unavailable = vendorList.filter(v => v.unavailable)
         if (unavailable.length > 0) {
           toasts.add({

packages/workshop-frontend/src/GatekeeperModal.tsx

@@ -1,4 +1,4 @@
-import { logRpcFailure, withDoResetRetry } from './rpcErrors'
+import { logRpcFailure } from './rpcErrors'
 import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
 import { Dialog, useKumoToastManager, type PortalContainer } from '@cloudflare/kumo'
 import {
@@ -369,7 +369,7 @@ export default function GatekeeperModal({
     setSpawnerEnv(
       (spawnerEnvCandidatesRef.current ?? []).map(entry => ({ ...entry, enabled: true })))
 
-    withDoResetRetry(() => authenticatedApi.listModels()).then(models => {
+    authenticatedApi.listModels().then(models => {
       if (cancelled) return
       setAvailableModels(models)
       if (models.length > 0) {
@@ -388,7 +388,7 @@ export default function GatekeeperModal({
       toasts.add({ title: "Couldn't load AI models", variant: 'error' })
     })
 
-    withDoResetRetry(() => authenticatedApi.listGatekeeperVendors()).then(vendors => {
+    authenticatedApi.listGatekeeperVendors().then(vendors => {
       if (cancelled) return
       setVendors(vendors)
     }).catch(err => {
@@ -432,7 +432,7 @@ export default function GatekeeperModal({
     }
 
     const subscriber = new AccountsSubscriber()
-    withDoResetRetry(() => authenticatedApi.subscribeConnectedAccounts(subscriber))
+    authenticatedApi.subscribeConnectedAccounts(subscriber)
       .then(stub => {
         if (cancelled) {
           stub[Symbol.dispose]()

packages/workshop-frontend/src/ObserverConfigModal.tsx

@@ -1,4 +1,3 @@
-import { withDoResetRetry } from './rpcErrors'
 import { useState, useEffect, useRef } from 'react'
 import { Dialog, Select, Loader, Text, useKumoToastManager } from '@cloudflare/kumo'
 import { Warning, Plus, ArrowClockwise, CheckCircle } from '@phosphor-icons/react'
@@ -144,8 +143,8 @@ export default function ObserverConfigModal({
       }
     }
 
-    withDoResetRetry(() => authenticatedApi
-      .subscribeConnectedAccounts(new Subscriber(), { includeForcedAutoProvisionedAccounts: true }))
+    authenticatedApi
+      .subscribeConnectedAccounts(new Subscriber(), { includeForcedAutoProvisionedAccounts: true })
       .then(stub => {
         if (cancelled) { stub[Symbol.dispose](); return }
         subStub = stub
@@ -166,10 +165,10 @@ export default function ObserverConfigModal({
   // ── load vendor metadata for display and resource-scope resolution ─────────────
   useEffect(() => {
     let cancelled = false
-    withDoResetRetry(() => Promise.all([
+    Promise.all([
       authenticatedApi.listGatekeeperVendors(),
       authenticatedApi.listAddableGatekeepers(),
-    ]))
+    ])
       .then(([vendors, addable]) => {
         if (cancelled) return
         const map = new Map<string, GatekeeperVendorInfo>()

packages/workshop-frontend/src/OnboardingWizard.tsx

@@ -1,4 +1,4 @@
-import { logRpcFailure, withDoResetRetry } from './rpcErrors'
+import { logRpcFailure } from './rpcErrors'
 import { useState, useEffect, useRef, useCallback } from 'react'
 import { useKumoToastManager } from '@cloudflare/kumo'
 import { RpcTarget } from 'capnweb'
@@ -120,10 +120,10 @@ export default function OnboardingWizard({
   // Load models + AI config
   const fetchModels = useCallback(async () => {
     try {
-      const [modelList, cfg] = await withDoResetRetry(() => Promise.all([
+      const [modelList, cfg] = await Promise.all([
         authenticatedApi.listModels(),
         authenticatedApi.getAiConfig(),
-      ]))
+      ])
       setModels(modelList)
       setAiConfig(cfg)
       // Default to the first model in the list
@@ -221,7 +221,7 @@ export default function OnboardingWizard({
     const subscriber = new AccountsSubscriber()
     let subscriptionStub: { [Symbol.dispose](): void } | null = null
 
-    withDoResetRetry(() => authenticatedApi.subscribeConnectedAccounts(subscriber))
+    authenticatedApi.subscribeConnectedAccounts(subscriber)
       .then((stub) => {
         if (cancelled) {
           stub[Symbol.dispose]()

packages/workshop-frontend/src/ResourcePicker.tsx

@@ -1,4 +1,4 @@
-import { logRpcFailure, withDoResetRetry } from './rpcErrors'
+import { logRpcFailure } from './rpcErrors'
 import { useState, useEffect, useRef, useMemo, useCallback, type MutableRefObject } from 'react'
 import { Tooltip, useKumoToastManager } from '@cloudflare/kumo'
 import { Plus, CaretRight, Warning } from '@phosphor-icons/react'
@@ -159,8 +159,7 @@ export default function ResourcePicker({
     const subscriber = new AccountsSubscriber()
     const subscribe = async () => {
       try {
-        const stub = await withDoResetRetry(
-            () => authenticatedApi.subscribeConnectedAccounts(subscriber))
+        const stub = await authenticatedApi.subscribeConnectedAccounts(subscriber)
         if (cancelled) stub[Symbol.dispose]()
         else subscriptionRef.current = { stub }
       } catch (error) {

packages/workshop-frontend/src/components/AppShell/SidebarWorkspaces.tsx

@@ -1,4 +1,4 @@
-import { logRpcFailure, withDoResetRetry } from '../../rpcErrors'
+import { logRpcFailure } from '../../rpcErrors'
 import {
   createContext,
   useCallback,
@@ -90,7 +90,7 @@ export function SidebarWorkspacesProvider({ children }: { children: ReactNode })
   useEffect(() => {
     let cancelled = false
     setGadgetsLoading(true)
-    withDoResetRetry(() => authenticatedApi.listGadgets())
+    authenticatedApi.listGadgets()
       .then((list) => {
         if (cancelled) return
         setGadgets(list)

packages/workshop-frontend/src/components/ConnectionChips.tsx

@@ -1,4 +1,3 @@
-import { withDoResetRetry } from '../rpcErrors'
 import { Plus } from '@phosphor-icons/react'
 import { Link } from '@tanstack/react-router'
 import { useAuthenticatedApi } from '../AuthContext'
@@ -46,7 +45,7 @@ export default function ConnectionChips() {
     }
 
     const subscriber = new ChipsSubscriber()
-    const subPromise = withDoResetRetry(() => authenticatedApi.subscribeConnectedAccounts(subscriber))
+    const subPromise = authenticatedApi.subscribeConnectedAccounts(subscriber)
     subPromise.then((stub) => {
       if (cancelled) {
         stub[Symbol.dispose]()