feat(agent): runtime operations manager + widget host refresh

apps/app-steward/package.json

@@ -21,6 +21,7 @@
   "dependencies": {
     "@noble/curves": "^2.0.1",
     "@simplewebauthn/browser": "^13.0.0",
+    "@solana/web3.js": "^1.98.4",
     "@stwd/sdk": "^0.8.0",
     "@elizaos/core": "workspace:*",
     "@elizaos/shared": "workspace:*",

apps/app-steward/src/browser-workspace-wallet.ts

@@ -32,6 +32,7 @@ export interface BrowserWorkspaceWalletState {
   solanaAddress: string | null;
   solanaConnected: boolean;
   solanaMessageSigningAvailable: boolean;
+  solanaTransactionSigningAvailable: boolean;
 }
 
 export interface BrowserWorkspaceWalletTransactionResult
@@ -53,6 +54,20 @@ export interface BrowserWorkspaceSolanaMessageSignatureResult {
   signatureBase64: string;
 }
 
+export interface BrowserWorkspaceSolanaTransactionResult {
+  address: string;
+  mode: "local-key" | "steward";
+  /** Base64-encoded fully-signed transaction (always present on success). */
+  signedTransactionBase64: string;
+  /**
+   * Optional broadcast signature (base58) when the steward broadcast the
+   * transaction. Omitted when the caller asked for signing only.
+   */
+  signature?: string;
+  /** Cluster the steward signed/broadcast against. */
+  cluster: "mainnet" | "devnet" | "testnet";
+}
+
 export type BrowserWorkspaceWalletRpcMethod =
   | "eth_accounts"
   | "eth_requestAccounts"
@@ -64,7 +79,9 @@ export type BrowserWorkspaceWalletRpcMethod =
 
 export type BrowserWorkspaceSolanaMethod =
   | "solana_connect"
-  | "solana_signMessage";
+  | "solana_signMessage"
+  | "solana_signTransaction"
+  | "solana_signAndSendTransaction";
 
 export type BrowserWorkspaceWalletMethod =
   | "getState"
@@ -109,6 +126,7 @@ export const EMPTY_BROWSER_WORKSPACE_WALLET_STATE: BrowserWorkspaceWalletState =
     solanaAddress: null,
     solanaConnected: false,
     solanaMessageSigningAvailable: false,
+    solanaTransactionSigningAvailable: false,
   };
 
 export function getBrowserWorkspaceWalletAddress(
@@ -206,10 +224,14 @@ export function buildBrowserWorkspaceWalletState(params: {
       solanaAddress,
       solanaConnected,
       solanaMessageSigningAvailable: false,
+      solanaTransactionSigningAvailable: solanaConnected,
     };
   }
 
   if (mode === "local") {
+    const solanaTransactionSigningAvailable = Boolean(
+      solanaAddress && walletConfig?.solanaSigningAvailable,
+    );
     return {
       address,
       connected: evmConnected || solanaConnected,
@@ -229,10 +251,12 @@ export function buildBrowserWorkspaceWalletState(params: {
       ),
       signingAvailable:
         Boolean(evmAddress && walletConfig?.executionReady) ||
-        solanaMessageSigningAvailable,
+        solanaMessageSigningAvailable ||
+        solanaTransactionSigningAvailable,
       solanaAddress,
       solanaConnected,
       solanaMessageSigningAvailable,
+      solanaTransactionSigningAvailable,
     };
   }
 
@@ -256,6 +280,7 @@ export function buildBrowserWorkspaceWalletState(params: {
       solanaAddress,
       solanaConnected,
       solanaMessageSigningAvailable: false,
+      solanaTransactionSigningAvailable: false,
     };
   }
 

apps/app-steward/src/routes/wallet-browser-compat-routes.ts

@@ -267,6 +267,85 @@ async function signLocalBrowserSolanaMessage(
   };
 }
 
+type SolanaCluster = "mainnet" | "devnet" | "testnet";
+
+function normalizeSolanaCluster(value: unknown): SolanaCluster {
+  if (value === "devnet" || value === "testnet" || value === "mainnet") {
+    return value;
+  }
+  return "mainnet";
+}
+
+function clusterRpcUrl(cluster: SolanaCluster): string {
+  switch (cluster) {
+    case "devnet":
+      return "https://api.devnet.solana.com";
+    case "testnet":
+      return "https://api.testnet.solana.com";
+    default:
+      return "https://api.mainnet-beta.solana.com";
+  }
+}
+
+async function signLocalBrowserSolanaTransaction(
+  body: Record<string, unknown>,
+): Promise<{
+  address: string;
+  mode: "local-key";
+  signedTransactionBase64: string;
+  signature?: string;
+  cluster: SolanaCluster;
+}> {
+  const transactionBase64 = normalizeString(body.transactionBase64);
+  if (!transactionBase64) {
+    throw new Error("transactionBase64 is required.");
+  }
+  const broadcast = normalizeBoolean(body.broadcast, false);
+  const cluster = normalizeSolanaCluster(body.cluster);
+
+  const { address, seed } = resolveLocalSolanaSeed();
+
+  // Lazy import — @solana/web3.js is a transitive dep through
+  // @elizaos/agent and only needed when an actual Solana transaction
+  // signing request lands.
+  const web3 = await import("@solana/web3.js");
+  const { Keypair, VersionedTransaction, Transaction, Connection } = web3;
+
+  const keypair = Keypair.fromSeed(new Uint8Array(seed));
+  const txBytes = Buffer.from(transactionBase64, "base64");
+
+  // Solana transactions have a single-byte version prefix on v0/versioned
+  // transactions (high bit set). Try the versioned path first; fall back to
+  // legacy on parse failure to support both shapes uniformly.
+  let signedBytes: Uint8Array;
+  let broadcastSignature: string | undefined;
+  try {
+    const versioned = VersionedTransaction.deserialize(txBytes);
+    versioned.sign([keypair]);
+    signedBytes = versioned.serialize();
+    if (broadcast) {
+      const conn = new Connection(clusterRpcUrl(cluster), "confirmed");
+      broadcastSignature = await conn.sendRawTransaction(signedBytes);
+    }
+  } catch (_err) {
+    const legacy = Transaction.from(txBytes);
+    legacy.partialSign(keypair);
+    signedBytes = legacy.serialize();
+    if (broadcast) {
+      const conn = new Connection(clusterRpcUrl(cluster), "confirmed");
+      broadcastSignature = await conn.sendRawTransaction(signedBytes);
+    }
+  }
+
+  return {
+    address,
+    mode: "local-key",
+    signedTransactionBase64: Buffer.from(signedBytes).toString("base64"),
+    ...(broadcastSignature ? { signature: broadcastSignature } : {}),
+    cluster,
+  };
+}
+
 export async function handleWalletBrowserCompatRoutes(
   req: http.IncomingMessage,
   res: http.ServerResponse,
@@ -279,7 +358,8 @@ export async function handleWalletBrowserCompatRoutes(
     method !== "POST" ||
     (url.pathname !== "/api/wallet/browser-transaction" &&
       url.pathname !== "/api/wallet/browser-sign-message" &&
-      url.pathname !== "/api/wallet/browser-solana-sign-message")
+      url.pathname !== "/api/wallet/browser-solana-sign-message" &&
+      url.pathname !== "/api/wallet/browser-solana-transaction")
   ) {
     return false;
   }
@@ -349,6 +429,31 @@ export async function handleWalletBrowserCompatRoutes(
     return true;
   }
 
+  if (url.pathname === "/api/wallet/browser-solana-transaction") {
+    if (hasLocalSolanaKey) {
+      try {
+        sendJsonResponse(
+          res,
+          200,
+          await signLocalBrowserSolanaTransaction(body),
+        );
+        return true;
+      } catch (error) {
+        const failureMessage =
+          error instanceof Error ? error.message : String(error);
+        sendJsonErrorResponse(res, 503, failureMessage);
+        return true;
+      }
+    }
+
+    sendJsonErrorResponse(
+      res,
+      503,
+      "No browser Solana transaction signer is available.",
+    );
+    return true;
+  }
+
   const request: StewardSignRequest = {
     broadcast: normalizeBoolean(body.broadcast, true),
     chainId:

apps/app/tsconfig.json

@@ -16,7 +16,7 @@
     "noUnusedParameters": false,
     "noFallthroughCasesInSwitch": true,
     "forceConsistentCasingInFileNames": true,
-    "ignoreDeprecations": "6.0",
+    "ignoreDeprecations": "5.0",
     "experimentalDecorators": true,
     "useUnknownInCatchVariables": true,
     "types": ["vite/client", "bun-types"],

packages/agent/src/actions/browser-session.ts

@@ -37,11 +37,29 @@ type BrowserSessionParameters = {
     | "state"
     | "tab"
     | "type"
-    | "wait";
+    | "wait"
+    | "realistic-click"
+    | "realistic-fill"
+    | "realistic-type"
+    | "realistic-press"
+    | "cursor-move"
+    | "cursor-hide";
   tabAction?: "close" | "list" | "new" | "switch";
   text?: string;
   timeoutMs?: number;
   url?: string;
+  /** Cursor animation duration (ms) for realistic-* + cursor-* subactions. */
+  cursorDurationMs?: number;
+  /** Per-character delay for realistic-type / realistic-fill (ms). */
+  perCharDelayMs?: number;
+  /** Replace existing input value when filling (vs append). */
+  replace?: boolean;
+  /** Cursor target X (CSS pixels) for cursor-move. */
+  x?: number;
+  /** Cursor target Y (CSS pixels) for cursor-move. */
+  y?: number;
+  /** Hint that the agent is operating in a watch-mode (page-browser) scope. */
+  watchMode?: boolean;
 };
 
 function getMessageText(message: Memory | undefined): string {
@@ -69,12 +87,18 @@ function inferBrowserSubaction(
     return "tab";
   }
 
+  // In watch mode the user is observing the agent drive the browser; prefer
+  // the realistic-* subactions so the cursor moves and pointer events fire
+  // faithfully. Default-mode (no watcher) keeps the leaner click()/value=
+  // path for speed.
+  const watchMode = params?.watchMode === true;
+
   if (params?.selector && params?.text) {
-    return "type";
+    return watchMode ? "realistic-fill" : "type";
   }
 
   if (params?.selector) {
-    return "click";
+    return watchMode ? "realistic-click" : "click";
   }
 
   if (params?.url?.trim() || extractFirstUrl(messageText)) {
@@ -106,6 +130,16 @@ function formatBrowserSessionResult(
   }
 
   if (result.value !== undefined) {
+    if (
+      command.subaction === "cursor-move" &&
+      result.value !== null &&
+      typeof result.value === "object" &&
+      "x" in result.value &&
+      "y" in result.value
+    ) {
+      const cursor = result.value as { x: number; y: number };
+      return `Cursor moved to (${Math.round(cursor.x)}, ${Math.round(cursor.y)}) in ${result.mode} mode.`;
+    }
     const serialized =
       typeof result.value === "string"
         ? result.value
@@ -158,6 +192,11 @@ export const browserSessionAction: Action = {
       text: params?.text,
       timeoutMs: params?.timeoutMs,
       url,
+      cursorDurationMs: params?.cursorDurationMs,
+      perCharDelayMs: params?.perCharDelayMs,
+      replace: params?.replace,
+      x: params?.x,
+      y: params?.y,
     };
 
     try {
@@ -220,6 +259,12 @@ export const browserSessionAction: Action = {
           "tab",
           "type",
           "wait",
+          "realistic-click",
+          "realistic-fill",
+          "realistic-type",
+          "realistic-press",
+          "cursor-move",
+          "cursor-hide",
         ],
       },
     },
@@ -280,6 +325,44 @@ export const browserSessionAction: Action = {
       required: false,
       schema: { type: "string" as const },
     },
+    {
+      name: "watchMode",
+      description:
+        "Hint that the user is watching; prefers realistic-* subactions for click/fill so the cursor moves visibly and pointer events fire faithfully.",
+      required: false,
+      schema: { type: "boolean" as const },
+    },
+    {
+      name: "cursorDurationMs",
+      description: "Cursor animation duration (ms) for realistic-* subactions",
+      required: false,
+      schema: { type: "number" as const },
+    },
+    {
+      name: "perCharDelayMs",
+      description: "Per-character delay for realistic-type/realistic-fill (ms)",
+      required: false,
+      schema: { type: "number" as const },
+    },
+    {
+      name: "replace",
+      description:
+        "Replace existing input value when filling (vs append) — applies to realistic-fill",
+      required: false,
+      schema: { type: "boolean" as const },
+    },
+    {
+      name: "x",
+      description: "Cursor target X (CSS pixels) for cursor-move",
+      required: false,
+      schema: { type: "number" as const },
+    },
+    {
+      name: "y",
+      description: "Cursor target Y (CSS pixels) for cursor-move",
+      required: false,
+      schema: { type: "number" as const },
+    },
   ],
   examples: [
     [