diff --git a/.github/workflows/agent-review.yml b/.github/workflows/agent-review.yml index 8b8b67a0fe..f947909289 100644 --- a/.github/workflows/agent-review.yml +++ b/.github/workflows/agent-review.yml @@ -155,8 +155,12 @@ jobs: - name: Build local eliza runtime plugins run: | - if [ ! -f eliza/packages/core/dist/index.node.js ] || [ ! -f eliza/packages/core/dist/index.d.ts ]; then - (cd eliza/packages/core && bun run build) + if [ -d eliza/packages/core ]; then + if [ ! -f eliza/packages/core/dist/index.node.js ] || [ ! -f eliza/packages/core/dist/index.d.ts ]; then + (cd eliza/packages/core && bun run build) + fi + else + echo "Skipping local eliza core build; package-mode checkout has no eliza/ tree." fi if [ -f eliza/plugins/plugin-agent-skills/package.json ] && [ ! -f eliza/plugins/plugin-agent-skills/dist/index.js ]; then (cd eliza/plugins/plugin-agent-skills && bun run build) diff --git a/.github/workflows/ci-fork.yml b/.github/workflows/ci-fork.yml index 6fe922f8f6..3aeedaedef 100644 --- a/.github/workflows/ci-fork.yml +++ b/.github/workflows/ci-fork.yml @@ -128,14 +128,18 @@ jobs: # @elizaos/ui from npm and the build fails with # `"createVoiceCapture" is not exported by "@elizaos/ui"`. - name: Link local @elizaos workspace packages - run: node eliza/packages/app-core/scripts/link-docker-local-app-packages.mjs + run: | + if [ -f eliza/packages/app-core/scripts/link-docker-local-app-packages.mjs ]; then + node eliza/packages/app-core/scripts/link-docker-local-app-packages.mjs + echo "MILADY_FORCE_LOCAL_UPSTREAMS=1" >> "$GITHUB_ENV" + else + echo "eliza checkout absent; package-mode build will use installed @elizaos packages" + fi # MILADY_FORCE_LOCAL_UPSTREAMS=1 forces apps/app/vite.config.ts to treat # the freshly-linked eliza/ checkout as the source of truth for the # @elizaos/* aliases. - name: Build project - env: - MILADY_FORCE_LOCAL_UPSTREAMS: "1" run: bun run build - name: Build homepage diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 41388dc164..b4ea13c8e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -282,14 +282,18 @@ jobs: # @elizaos/ui from npm and the build fails with # `"createVoiceCapture" is not exported by "@elizaos/ui"`. - name: Link local @elizaos workspace packages - run: node eliza/packages/app-core/scripts/link-docker-local-app-packages.mjs + run: | + if [ -f eliza/packages/app-core/scripts/link-docker-local-app-packages.mjs ]; then + node eliza/packages/app-core/scripts/link-docker-local-app-packages.mjs + echo "MILADY_FORCE_LOCAL_UPSTREAMS=1" >> "$GITHUB_ENV" + else + echo "eliza checkout absent; package-mode build will use installed @elizaos packages" + fi # MILADY_FORCE_LOCAL_UPSTREAMS=1 forces apps/app/vite.config.ts to treat # the freshly-linked eliza/ checkout as the source of truth for the # @elizaos/* aliases. - name: Build project - env: - MILADY_FORCE_LOCAL_UPSTREAMS: "1" run: bun run build - name: Build homepage diff --git a/.github/workflows/release-electrobun.yml b/.github/workflows/release-electrobun.yml index b1a2ee9ebd..e69bb048a9 100644 --- a/.github/workflows/release-electrobun.yml +++ b/.github/workflows/release-electrobun.yml @@ -377,6 +377,17 @@ jobs: - name: Ensure avatar assets run: node eliza/packages/app-core/scripts/ensure-avatars.mjs + - name: Prepare Whisper model artifact + run: | + bash eliza/packages/app-core/platforms/electrobun/scripts/ensure-whisper-model.sh base.en + + - name: Upload Whisper model artifact + uses: actions/upload-artifact@v4 + with: + name: whisper-model-base-en + path: ~/.cache/eliza/whisper/ggml-base.en.bin + if-no-files-found: error + - name: Build core dist (server bundle) env: MILADY_ELIZA_SOURCE: local @@ -680,6 +691,18 @@ jobs: - name: Align package-mode agent pins run: node scripts/align-eliza-agent-package-pins.mjs + - name: Download Whisper model artifact + uses: actions/download-artifact@v4 + with: + name: whisper-model-base-en + path: ${{ runner.temp }}/whisper-model + + - name: Seed Whisper model cache + run: | + mkdir -p "$HOME/.cache/eliza/whisper" + cp "$RUNNER_TEMP/whisper-model/ggml-base.en.bin" "$HOME/.cache/eliza/whisper/ggml-base.en.bin" + test -s "$HOME/.cache/eliza/whisper/ggml-base.en.bin" + # Link local @elizaos workspace packages into node_modules. desktop-build.mjs # stage invokes vite, which imports symbols from @elizaos/ui # (createVoiceCapture, VoicePill, VoicePillMessage, VoiceCaptureHandle) diff --git a/.gitignore b/.gitignore index 56bc145a75..aadb7e5656 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,9 @@ apps/app/electrobun/build/ *.bun-build **/*.bun-build *.tsbuildinfo +# Generated by the app web build when shared brand assets are synced. +apps/app/public/brand/ +apps/app/public/clouds/ eliza/packages/elizaos/templates/ eliza/packages/elizaos/templates-manifest.json apps/app/electron/tsc-out/ diff --git a/apps/app/src/host-window-routing.ts b/apps/app/src/host-window-routing.ts new file mode 100644 index 0000000000..ff1633c276 --- /dev/null +++ b/apps/app/src/host-window-routing.ts @@ -0,0 +1,134 @@ +export type DetachedSurfaceTab = + | "browser" + | "chat" + | "release" + | "triggers" + | "plugins" + | "connectors" + | "cloud"; + +export type WindowShellRoute = + | { mode: "main" } + | { mode: "pill" } + | { mode: "settings"; tab?: string } + | { mode: "surface"; tab: DetachedSurfaceTab }; + +export type DetachedWindowShellRoute = Exclude< + WindowShellRoute, + { mode: "main" } | { mode: "pill" } +>; + +export function resolveWindowShellRoute( + search = typeof window !== "undefined" ? window.location.search : "", +): WindowShellRoute { + const params = new URLSearchParams(search); + const shell = params.get("shell"); + + if (shell === "pill") { + return { mode: "pill" }; + } + + if (shell === "settings") { + const tab = params.get("tab")?.trim() || undefined; + return tab ? { mode: "settings", tab } : { mode: "settings" }; + } + + if (shell === "surface") { + const tab = params.get("tab"); + if ( + tab === "browser" || + tab === "chat" || + tab === "release" || + tab === "triggers" || + tab === "plugins" || + tab === "connectors" || + tab === "cloud" + ) { + return { mode: "surface", tab }; + } + } + + return { mode: "main" }; +} + +export function isPillWindowShell( + route: WindowShellRoute, +): route is { mode: "pill" } { + return route.mode === "pill"; +} + +export function isDetachedWindowShell( + route: WindowShellRoute, +): route is DetachedWindowShellRoute { + return route.mode !== "main" && route.mode !== "pill"; +} + +export function shouldInstallMainWindowOnboardingPatches( + route: WindowShellRoute, +): boolean { + return route.mode === "main"; +} + +export function isAppWindowRoute( + location: Pick | undefined = typeof window === "undefined" + ? undefined + : window.location, +): boolean { + if (!location) return false; + try { + return new URLSearchParams(location.search).get("appWindow") === "1"; + } catch { + return false; + } +} + +function shouldUseHashNavigation( + location: + | Pick + | undefined = typeof window === "undefined" ? undefined : window.location, +): boolean { + if (!location) return false; + return location.protocol === "file:" || isAppWindowRoute(location); +} + +export function getWindowNavigationPath( + location: + | Pick + | undefined = typeof window === "undefined" ? undefined : window.location, +): string { + if (!location) return "/"; + return shouldUseHashNavigation(location) + ? location.hash.replace(/^#/, "") || "/" + : location.pathname; +} + +function pathForDetachedShell(route: DetachedWindowShellRoute): string { + if (route.mode === "settings") return "/settings"; + + switch (route.tab) { + case "browser": + return "/browser"; + case "chat": + return "/chat"; + case "release": + return "/settings"; + case "triggers": + return "/automations"; + case "plugins": + return "/apps/plugins"; + case "connectors": + return "/connectors"; + case "cloud": + return "/settings"; + } +} + +export function syncDetachedShellLocation( + route: DetachedWindowShellRoute, +): boolean { + if (typeof window === "undefined") return false; + const nextUrl = new URL(window.location.href); + nextUrl.pathname = pathForDetachedShell(route); + window.history.replaceState(null, "", nextUrl.toString()); + return true; +} diff --git a/apps/app/src/main.tsx b/apps/app/src/main.tsx index 33cf5c0432..78221d1c43 100644 --- a/apps/app/src/main.tsx +++ b/apps/app/src/main.tsx @@ -41,24 +41,25 @@ import { installDesktopPermissionsClientPatch, installForceFreshOnboardingClientPatch, installLocalProviderCloudPreferencePatch, - isAppWindowRoute, - isDetachedWindowShell, - getWindowNavigationPath, - resolveWindowShellRoute, - shouldInstallMainWindowOnboardingPatches, - syncDetachedShellLocation, } from "@elizaos/app-core"; // Pill / voice-capture symbols live in upstream @elizaos/ui source but are -// not yet published on the `alpha` dist-tag. Route through the local stub so -// package-mode builds stay green; see apps/app/src/pill-stubs.tsx. +// not yet published on the `alpha` dist-tag. Route through the Milady-owned +// runtime so package-mode builds stay green without private UI subpath imports. import { - type ConversationMessage, createVoiceCapture, type VoiceCaptureHandle, VoicePill, type VoicePillMessage, - normalizePillMessage, -} from "./pill-stubs"; +} from "./voice-pill-runtime"; +import { + getWindowNavigationPath, + isAppWindowRoute, + isDetachedWindowShell, + resolveWindowShellRoute, + shouldInstallMainWindowOnboardingPatches, + syncDetachedShellLocation, +} from "./host-window-routing"; +import type { ConversationMessage } from "@elizaos/app-core/api/client-types-chat"; import { AppWindowRenderer } from "@elizaos/app-core"; import { dispatchQueuedLifeOpsGithubCallbackFromUrl } from "@elizaos/app-lifeops/platform"; import type { ShareTargetPayload } from "@elizaos/app-core/platform"; @@ -76,7 +77,14 @@ import { startDeviceBridgeClient, type DeviceBridgeClient, } from "@elizaos/capacitor-llama"; -import { StrictMode, useCallback, useEffect, useRef, useState } from "react"; +import { + StrictMode, + Suspense, + useCallback, + useEffect, + useRef, + useState, +} from "react"; import { createRoot } from "react-dom/client"; import { CompanionShell } from "@elizaos/app-companion/ui"; import { @@ -907,13 +915,18 @@ const PILL_MESSAGE_TAIL = 20; // in its own Electrobun renderer process with no access to the main shell's // active conversation state, so we keep its own session pinned to localStorage. const PILL_CONVERSATION_STORAGE_KEY = "milady.pill.activeConversationId"; +type PillConversationMessage = ConversationMessage & { + failureKind?: string; +}; /** * Map a chat-API `ConversationMessage` to the trimmed `VoicePillMessage` shape * the pill renders. Skips entries with no display text and collapses message * roles to the binary user/agent split the pill UI expects. */ -function toPillMessage(message: ConversationMessage): VoicePillMessage | null { +function toPillMessage( + message: PillConversationMessage, +): VoicePillMessage | null { const text = message.text?.trim() ?? ""; if (!text) return null; return { @@ -924,7 +937,7 @@ function toPillMessage(message: ConversationMessage): VoicePillMessage | null { } function projectPillMessages( - messages: ConversationMessage[], + messages: PillConversationMessage[], ): VoicePillMessage[] { const tail = messages.slice(-PILL_MESSAGE_TAIL); const projected: VoicePillMessage[] = []; @@ -935,6 +948,30 @@ function projectPillMessages( return projected; } +function normalizePillMessage(value: unknown): PillConversationMessage { + const record = + value && typeof value === "object" + ? (value as Record) + : {}; + const rawTimestamp = record.timestamp; + const timestamp = + typeof rawTimestamp === "number" + ? rawTimestamp + : typeof rawTimestamp === "string" + ? Date.parse(rawTimestamp) + : Date.now(); + return { + ...(record as Partial), + id: + typeof record.id === "string" && record.id + ? record.id + : `pill-message-${Date.now()}`, + role: record.role === "user" ? "user" : "assistant", + text: typeof record.text === "string" ? record.text : "", + timestamp: Number.isFinite(timestamp) ? timestamp : Date.now(), + }; +} + function readPillConversationId(): string | null { try { return window.localStorage.getItem(PILL_CONVERSATION_STORAGE_KEY); @@ -1007,7 +1044,7 @@ function conversationUpdatedAtMs(c: PillConversationSummary): number { * agent turns. */ function PillRoot() { - const [messages, setMessages] = useState([]); + const [messages, setMessages] = useState([]); const conversationIdRef = useRef(readPillConversationId()); const sendInFlightRef = useRef(false); const voiceCaptureRef = useRef(null); @@ -1015,7 +1052,7 @@ function PillRoot() { // Append-or-replace by id so streaming token updates collapse onto a single // assistant turn without flicker. - const upsertMessage = useCallback((next: ConversationMessage) => { + const upsertMessage = useCallback((next: PillConversationMessage) => { setMessages((prev) => { const index = prev.findIndex((entry) => entry.id === next.id); if (index < 0) return [...prev, next]; @@ -1137,13 +1174,17 @@ function PillRoot() { }); }, ); + const failureKind = + typeof result.failureKind === "string" + ? result.failureKind + : undefined; upsertMessage({ id: assistantMsgId, role: "assistant", text: result.text ?? "", timestamp: Date.now(), - ...(result.failureKind ? { failureKind: result.failureKind } : {}), + ...(failureKind ? { failureKind } : {}), }); } finally { sendInFlightRef.current = false; @@ -1279,26 +1320,28 @@ function mountReactApp(): void { createRoot(rootEl).render( - - {phoneCompanion ? ( - - ) : detachedShell ? ( -
- -
- ) : appWindowSlug ? ( -
- -
- ) : ( - <> - - - - - - )} - + + + {phoneCompanion ? ( + + ) : detachedShell ? ( +
+ +
+ ) : appWindowSlug ? ( +
+ +
+ ) : ( + <> + + + + + + )} + + , ); @@ -1592,6 +1635,7 @@ async function initializeStatusBar(): Promise { async function main(): Promise { setupPlatformStyles(); await initializeStatusBar(); + applyBuildTimeIosConnection(); try { diff --git a/apps/app/src/optional-eliza-app-stub.tsx b/apps/app/src/optional-eliza-app-stub.tsx index 0d7779ccd3..8ef294f09a 100644 --- a/apps/app/src/optional-eliza-app-stub.tsx +++ b/apps/app/src/optional-eliza-app-stub.tsx @@ -11,6 +11,7 @@ import type { WalletExportResult, WhitelistStatus, } from "@elizaos/app-core/api"; +import type { ChatSidebarWidgetDefinition } from "@elizaos/app-core/components/chat/widgets/types"; import type { InventoryChainFilters } from "@elizaos/app-core/state/types"; import type { WalletAddresses, @@ -28,20 +29,8 @@ import type { import type { ComponentType } from "react"; import * as THREE from "three"; -// Mirrors @elizaos/ui's confirm-dialog PromptOptions. Declared locally because -// the tsconfig "@elizaos/ui/*" path-map bypasses the package exports field, -// so the deep "@elizaos/ui/components/ui/confirm-dialog" subpath does not -// resolve in this stub. -type PromptOptions = { - title: string; - description?: string; - placeholder?: string; - initialValue?: string; - confirmLabel?: string; - cancelLabel?: string; -}; - const EmptyComponent: ComponentType = () => null; +type PromptOptions = Record; export function CompanionShell(_props: CompanionShellComponentProps): null { return null; @@ -202,7 +191,6 @@ export function useInventoryData(): { // Wallet sidebar widget. Component prop type comes from app-core so the seed // registry accepts this stub as a valid ChatSidebarWidgetDefinition. -import type { ChatSidebarWidgetDefinition } from "@elizaos/app-core/components/chat/widgets/types"; export const WALLET_STATUS_WIDGET: ChatSidebarWidgetDefinition = { id: "wallet.status", pluginId: "wallet", diff --git a/apps/app/src/voice-pill-runtime.css b/apps/app/src/voice-pill-runtime.css new file mode 100644 index 0000000000..f4ab78cc18 --- /dev/null +++ b/apps/app/src/voice-pill-runtime.css @@ -0,0 +1,232 @@ +.elizaos-voice-pill { + display: flex; + flex-direction: column; + align-items: center; + gap: 9px; + font-family: + "Poppins", system-ui, -apple-system, "Segoe UI", Arial, sans-serif; +} + +.elizaos-voice-pill__hit { + padding: 18px 24px; + background: transparent; + border: 0; + cursor: pointer; + line-height: 0; + display: flex; + flex-direction: column; + align-items: center; + gap: 9px; +} + +.elizaos-voice-pill__chat { + width: 280px; + background: transparent; + overflow: visible; + transition: + opacity 0.26s cubic-bezier(0.34, 1.4, 0.64, 1), + transform 0.26s cubic-bezier(0.34, 1.4, 0.64, 1); +} + +.elizaos-voice-pill__chat--collapsed { + opacity: 0; + transform: translateY(8px); + pointer-events: none; +} + +.elizaos-voice-pill__messages { + display: flex; + flex-direction: column; + gap: 4px; + padding: 0 0 8px; +} + +.elizaos-voice-pill__msg { + max-width: 88%; + padding: 5px 9px; + border-radius: 12px; + font-size: 10.5px; + line-height: 1.46; + backdrop-filter: blur(20px) saturate(180%); + -webkit-backdrop-filter: blur(20px) saturate(180%); +} + +.elizaos-voice-pill__msg--agent { + align-self: flex-start; + background: rgba(255, 255, 255, 0.13); + color: rgba(255, 255, 255, 0.78); + box-shadow: + inset 0 0 0 0.5px rgba(255, 255, 255, 0.28), + inset 0 1px 0 rgba(255, 255, 255, 0.38); +} + +.elizaos-voice-pill__msg--user { + align-self: flex-end; + background: rgba(255, 255, 255, 0.22); + color: rgba(255, 255, 255, 0.96); + box-shadow: + inset 0 0 0 0.5px rgba(255, 255, 255, 0.38), + inset 0 1px 0 rgba(255, 255, 255, 0.5); +} + +.elizaos-voice-pill__composer { + display: flex; + align-items: center; + gap: 5px; + padding: 0; + border-top: none; +} + +.elizaos-voice-pill__input { + flex: 1; + min-width: 0; + height: 28px; + border: 0; + border-radius: 999px; + padding: 0 10px; + background: rgba(255, 255, 255, 0.14); + backdrop-filter: blur(20px) saturate(180%); + -webkit-backdrop-filter: blur(20px) saturate(180%); + box-shadow: + inset 0 0 0 0.5px rgba(255, 255, 255, 0.28), + inset 0 1px 0 rgba(255, 255, 255, 0.36); + color: #fff; + font-size: 10.5px; + outline: none; + font-family: inherit; +} + +.elizaos-voice-pill__input::placeholder { + color: rgba(255, 255, 255, 0.36); +} + +.elizaos-voice-pill__ctrl, +.elizaos-voice-pill__send { + display: grid; + place-items: center; + width: 28px; + height: 28px; + border-radius: 50%; + border: 0; + cursor: pointer; + flex-shrink: 0; + font-family: inherit; +} + +.elizaos-voice-pill__ctrl { + background: rgba(255, 255, 255, 0.14); + backdrop-filter: blur(20px) saturate(180%); + -webkit-backdrop-filter: blur(20px) saturate(180%); + box-shadow: + inset 0 0 0 0.5px rgba(255, 255, 255, 0.28), + inset 0 1px 0 rgba(255, 255, 255, 0.36); + color: rgba(255, 255, 255, 0.86); + transition: background 0.18s ease; +} + +.elizaos-voice-pill__ctrl svg, +.elizaos-voice-pill__send svg { + display: block; + width: 12px; + height: 12px; + fill: none; + stroke: currentColor; + stroke-width: 2; + stroke-linecap: round; + stroke-linejoin: round; +} + +.elizaos-voice-pill__ctrl--recording { + background: rgba(255, 50, 38, 0.72); + box-shadow: + 0 0 12px rgba(255, 50, 38, 0.48), + inset 0 0 0 0.5px rgba(255, 100, 80, 0.5); + color: #fff; +} + +.elizaos-voice-pill__send { + background: rgba(255, 255, 255, 0.88); + backdrop-filter: blur(20px) saturate(180%); + -webkit-backdrop-filter: blur(20px) saturate(180%); + box-shadow: + inset 0 0 0 0.5px rgba(255, 255, 255, 0.6), + inset 0 1px 0 #fff; + color: #07131f; +} + +.elizaos-voice-pill__pill { + width: 56px; + height: 8px; + border-radius: 999px; + background: rgba(255, 255, 255, 0.34); + backdrop-filter: blur(24px) saturate(200%); + -webkit-backdrop-filter: blur(24px) saturate(200%); + box-shadow: + 0 3px 14px rgba(0, 0, 0, 0.22), + inset 0 1px 0 rgba(255, 255, 255, 0.62), + inset 0 0 0 0.5px rgba(255, 255, 255, 0.42); + border: 0; + padding: 0; + display: inline-block; + transition: + box-shadow 0.32s ease, + background 0.22s ease, + transform 0.22s cubic-bezier(0.34, 1.4, 0.64, 1), + filter 0.22s ease; +} + +.elizaos-voice-pill__hit:hover .elizaos-voice-pill__pill { + transform: scale(1.18); + background: rgba(255, 255, 255, 0.52); + filter: brightness(1.08); + box-shadow: + 0 6px 22px rgba(0, 0, 0, 0.3), + 0 0 16px rgba(255, 255, 255, 0.26), + inset 0 1px 0 rgba(255, 255, 255, 0.78), + inset 0 0 0 0.5px rgba(255, 255, 255, 0.58); +} + +.elizaos-voice-pill__hit:focus { + outline: none; +} + +.elizaos-voice-pill__hit:focus-visible .elizaos-voice-pill__pill { + box-shadow: + 0 0 0 2px rgba(255, 255, 255, 0.85), + 0 4px 18px rgba(0, 0, 0, 0.26); +} + +.elizaos-voice-pill__pill--recording { + background: rgba(255, 70, 50, 0.55); + box-shadow: + 0 0 0 1.5px rgba(255, 50, 38, 0.78), + 0 0 24px rgba(255, 50, 38, 0.54), + 0 3px 14px rgba(0, 0, 0, 0.22), + inset 0 1px 0 rgba(255, 255, 255, 0.5); + animation: elizaos-voice-pill-rec-pulse 1.8s ease-in-out infinite; +} + +@keyframes elizaos-voice-pill-rec-pulse { + 0%, + 100% { + box-shadow: + 0 0 0 1.5px rgba(255, 50, 38, 0.78), + 0 0 24px rgba(255, 50, 38, 0.54), + 0 3px 14px rgba(0, 0, 0, 0.22), + inset 0 1px 0 rgba(255, 255, 255, 0.5); + } + + 50% { + box-shadow: + 0 0 0 1.5px rgba(255, 50, 38, 0.92), + 0 0 36px rgba(255, 50, 38, 0.72), + 0 3px 14px rgba(0, 0, 0, 0.22), + inset 0 1px 0 rgba(255, 255, 255, 0.5); + } +} + +@media (prefers-reduced-motion: reduce) { + .elizaos-voice-pill__pill--recording { + animation: none; + } +} diff --git a/apps/app/src/voice-pill-runtime.tsx b/apps/app/src/voice-pill-runtime.tsx new file mode 100644 index 0000000000..63431b1059 --- /dev/null +++ b/apps/app/src/voice-pill-runtime.tsx @@ -0,0 +1,690 @@ +import { client } from "@elizaos/app-core"; +import { resolveApiUrl } from "@elizaos/app-core/utils"; +import { + type ChangeEvent, + type KeyboardEvent, + type MouseEvent, + type PointerEvent, + useCallback, + useEffect, + useRef, + useState, +} from "react"; + +import "./voice-pill-runtime.css"; + +export interface VoicePillMessage { + id: string; + role: "agent" | "user"; + text: string; +} + +export interface VoicePillProps { + open?: boolean; + onOpenChange?: (open: boolean) => void; + recording?: boolean; + onRecordingChange?: (recording: boolean) => void; + messages?: VoicePillMessage[]; + placeholder?: string; + onSubmit?: (text: string) => void; + onAdd?: () => void; + ariaLabel?: string; + className?: string; +} + +type VoiceCaptureBackend = "local-inference" | "browser"; +type VoiceCaptureProvider = + | VoiceCaptureBackend + | "eliza-cloud" + | "elevenlabs" + | "openai"; + +export interface VoiceCaptureTranscriptSegment { + text: string; + final: boolean; + backend: VoiceCaptureBackend; +} + +export type VoiceCaptureState = + | "idle" + | "starting" + | "listening" + | "stopped" + | "error"; + +export interface VoiceCaptureFactoryOptions { + onTranscript: (segment: VoiceCaptureTranscriptSegment) => void; + onStateChange?: (state: VoiceCaptureState, error?: Error) => void; + asrProvider?: VoiceCaptureProvider; + lang?: string; +} + +export interface VoiceCaptureHandle { + start(): Promise; + stop(): Promise; + dispose(): void; + isActive(): boolean; +} + +interface LocalAsrRecorder { + stop(): Promise; + cancel(): void; +} + +interface SpeechRecognitionInstance extends EventTarget { + continuous: boolean; + interimResults: boolean; + lang: string; + onend: (() => void) | null; + onerror: ((event: { error: string }) => void) | null; + onresult: ((event: SpeechRecognitionResultEvent) => void) | null; + start(): void; + stop(): void; + abort(): void; +} + +interface SpeechRecognitionResultEvent { + results: SpeechRecognitionResultList; + resultIndex: number; +} + +interface SpeechRecognitionResultList { + length: number; + [index: number]: { + isFinal: boolean; + 0: { transcript: string; confidence: number }; + }; +} + +type SpeechRecognitionCtor = new () => SpeechRecognitionInstance; +type AudioContextConstructor = typeof AudioContext; +type WindowWithVoiceApis = Window & { + AudioContext?: AudioContextConstructor; + webkitAudioContext?: AudioContextConstructor; + SpeechRecognition?: SpeechRecognitionCtor; + webkitSpeechRecognition?: SpeechRecognitionCtor; +}; + +const DEFAULT_PLACEHOLDER = "Ask Eliza..."; +const DEFAULT_ARIA_LABEL = "Eliza"; + +function useControllable( + controlled: T | undefined, + initial: T, + onChange: ((next: T) => void) | undefined, +): [T, (next: T) => void] { + const [internal, setInternal] = useState(initial); + const isControlled = controlled !== undefined; + const value = isControlled ? (controlled as T) : internal; + const setValue = useCallback( + (next: T) => { + if (!isControlled) { + setInternal(next); + } + onChange?.(next); + }, + [isControlled, onChange], + ); + return [value, setValue]; +} + +function PlusIcon() { + return ( + + ); +} + +function MicIcon() { + return ( + + ); +} + +function SendIcon() { + return ( + + ); +} + +export function VoicePill(props: VoicePillProps) { + const { + open: openProp, + onOpenChange, + recording: recordingProp, + onRecordingChange, + messages, + placeholder = DEFAULT_PLACEHOLDER, + onSubmit, + onAdd, + ariaLabel = DEFAULT_ARIA_LABEL, + className, + } = props; + + const [open, setOpen] = useControllable(openProp, false, onOpenChange); + const [recording, setRecording] = useControllable( + recordingProp, + false, + onRecordingChange, + ); + const [inputValue, setInputValue] = useState(""); + const inputRef = useRef(null); + const chatRef = useRef(null); + + useEffect(() => { + if (open) { + inputRef.current?.focus(); + } + }, [open]); + + const toggleOpen = useCallback(() => { + setOpen(!open); + }, [open, setOpen]); + + const handleHitClick = useCallback( + (event: MouseEvent) => { + if ( + chatRef.current && + event.target instanceof Node && + chatRef.current.contains(event.target) + ) { + return; + } + toggleOpen(); + }, + [toggleOpen], + ); + + const handleHitKeyDown = useCallback( + (event: KeyboardEvent) => { + if (event.target !== event.currentTarget) return; + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + toggleOpen(); + } + }, + [toggleOpen], + ); + + const send = useCallback(() => { + const trimmed = inputValue.trim(); + if (!trimmed) return; + onSubmit?.(trimmed); + setInputValue(""); + }, [inputValue, onSubmit]); + + const stopPointer = useCallback( + (event: MouseEvent | PointerEvent) => { + event.stopPropagation(); + }, + [], + ); + + const wrapperClassName = className + ? `elizaos-voice-pill ${className}` + : "elizaos-voice-pill"; + const pillClassName = recording + ? "elizaos-voice-pill__pill elizaos-voice-pill__pill--recording" + : "elizaos-voice-pill__pill"; + const chatClassName = open + ? "elizaos-voice-pill__chat" + : "elizaos-voice-pill__chat elizaos-voice-pill__chat--collapsed"; + const micClassName = recording + ? "elizaos-voice-pill__ctrl elizaos-voice-pill__ctrl--recording" + : "elizaos-voice-pill__ctrl"; + + return ( +
+ {/* biome-ignore lint/a11y/useSemanticElements: this hit area contains nested composer controls, so it cannot be a native button. */} +
+ +
+ ); +} + +function getAudioContextCtor(): AudioContextConstructor | undefined { + if (typeof window === "undefined") return undefined; + const win = window as WindowWithVoiceApis; + return win.AudioContext ?? win.webkitAudioContext; +} + +function getSpeechRecognitionCtor(): SpeechRecognitionCtor | undefined { + if (typeof window === "undefined") return undefined; + const win = window as WindowWithVoiceApis; + return win.SpeechRecognition ?? win.webkitSpeechRecognition; +} + +function isLocalAsrCaptureSupported(): boolean { + return ( + typeof navigator !== "undefined" && + typeof navigator.mediaDevices?.getUserMedia === "function" && + !!getAudioContextCtor() + ); +} + +function resolveBackend( + preferred: VoiceCaptureProvider | undefined, +): VoiceCaptureBackend { + if (preferred === "browser") return "browser"; + if (preferred === "local-inference" || preferred === undefined) { + if (isLocalAsrCaptureSupported()) return "local-inference"; + } + return "browser"; +} + +function concatPcm(chunks: Float32Array[]): Float32Array { + const total = chunks.reduce((sum, chunk) => sum + chunk.length, 0); + const out = new Float32Array(total); + let offset = 0; + for (const chunk of chunks) { + out.set(chunk, offset); + offset += chunk.length; + } + return out; +} + +function writeAscii(view: DataView, offset: number, value: string): void { + for (let index = 0; index < value.length; index += 1) { + view.setUint8(offset + index, value.charCodeAt(index)); + } +} + +function encodeMonoPcm16Wav( + pcm: Float32Array, + sampleRateHz: number, +): Uint8Array { + const sampleRate = Math.max(1, Math.round(sampleRateHz)); + const bytesPerSample = 2; + const dataBytes = pcm.length * bytesPerSample; + const buffer = new ArrayBuffer(44 + dataBytes); + const view = new DataView(buffer); + + writeAscii(view, 0, "RIFF"); + view.setUint32(4, 36 + dataBytes, true); + writeAscii(view, 8, "WAVE"); + writeAscii(view, 12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * bytesPerSample, true); + view.setUint16(32, bytesPerSample, true); + view.setUint16(34, 8 * bytesPerSample, true); + writeAscii(view, 36, "data"); + view.setUint32(40, dataBytes, true); + + let offset = 44; + for (const sample of pcm) { + const clamped = Math.max( + -1, + Math.min(1, Number.isFinite(sample) ? sample : 0), + ); + const int16 = clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff; + view.setInt16(offset, Math.round(int16), true); + offset += bytesPerSample; + } + + return new Uint8Array(buffer); +} + +async function startLocalAsrRecorder(): Promise { + const AudioContextCtor = getAudioContextCtor(); + if (!AudioContextCtor) { + throw new Error("AudioContext is not available for local ASR capture"); + } + if (typeof navigator.mediaDevices?.getUserMedia !== "function") { + throw new Error("Microphone capture is not available for local ASR"); + } + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + channelCount: 1, + echoCancellation: true, + noiseSuppression: true, + autoGainControl: true, + }, + }); + const context = new AudioContextCtor(); + if (context.state === "suspended") { + await context.resume().catch(() => {}); + } + + const source = context.createMediaStreamSource(stream); + const processor = context.createScriptProcessor(4096, 1, 1); + const chunks: Float32Array[] = []; + let stopped = false; + + processor.onaudioprocess = (event) => { + if (stopped) return; + const input = event.inputBuffer; + const frameCount = input.length; + const channelCount = Math.max(1, input.numberOfChannels); + const mono = new Float32Array(frameCount); + + for (let channel = 0; channel < channelCount; channel += 1) { + const data = input.getChannelData(channel); + for (let index = 0; index < frameCount; index += 1) { + mono[index] = (mono[index] ?? 0) + (data[index] ?? 0) / channelCount; + } + } + chunks.push(mono); + }; + + source.connect(processor); + processor.connect(context.destination); + + const cleanup = async () => { + stopped = true; + processor.onaudioprocess = null; + try { + source.disconnect(); + } catch { + // already disconnected + } + try { + processor.disconnect(); + } catch { + // already disconnected + } + for (const track of stream.getTracks()) { + track.stop(); + } + await context.close().catch(() => {}); + }; + + return { + async stop() { + const sampleRate = context.sampleRate; + await cleanup(); + const pcm = concatPcm(chunks); + if (pcm.length === 0) { + throw new Error("No microphone audio was captured for local ASR"); + } + return encodeMonoPcm16Wav(pcm, sampleRate); + }, + cancel() { + void cleanup(); + }, + }; +} + +async function transcribeLocalInferenceWav(audio: Uint8Array): Promise { + const audioBody = new ArrayBuffer(audio.byteLength); + new Uint8Array(audioBody).set(audio); + const headers: Record = { + "Content-Type": "audio/wav", + Accept: "application/json", + }; + const token = ( + client as { + getRestAuthToken?: () => string | null; + } + ).getRestAuthToken?.(); + if (token) { + headers.Authorization = `Bearer ${token}`; + } + + const res = await fetch(resolveApiUrl("/api/asr/local-inference"), { + method: "POST", + headers, + body: audioBody, + credentials: "include", + }); + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error(`Local inference ASR ${res.status}: ${body.slice(0, 200)}`); + } + const parsed = (await res.json().catch(() => null)) as { + text?: unknown; + } | null; + const text = typeof parsed?.text === "string" ? parsed.text.trim() : ""; + if (!text) { + throw new Error("Local inference ASR returned an empty transcript"); + } + return text; +} + +export function createVoiceCapture( + options: VoiceCaptureFactoryOptions, +): VoiceCaptureHandle { + const { onTranscript, onStateChange, asrProvider, lang = "en-US" } = options; + const backend = resolveBackend(asrProvider); + let state: VoiceCaptureState = "idle"; + let active = false; + let disposed = false; + let recorder: LocalAsrRecorder | null = null; + let recognition: SpeechRecognitionInstance | null = null; + let browserStopWait: Promise | null = null; + let resolveBrowserStop: (() => void) | null = null; + + function setState(next: VoiceCaptureState, error?: Error): void { + if (state === next) return; + state = next; + onStateChange?.(next, error); + } + + async function startLocalInference(): Promise { + recorder = await startLocalAsrRecorder(); + active = true; + setState("listening"); + } + + function startBrowser(): void { + const Ctor = getSpeechRecognitionCtor(); + if (!Ctor) { + throw new Error( + "Browser SpeechRecognition API is not available in this renderer", + ); + } + const instance = new Ctor(); + instance.continuous = true; + instance.interimResults = true; + instance.lang = lang; + instance.onresult = (event: SpeechRecognitionResultEvent) => { + for ( + let index = event.resultIndex; + index < event.results.length; + index += 1 + ) { + const result = event.results[index]; + const text = result?.[0]?.transcript?.trim() ?? ""; + if (!text) continue; + onTranscript({ text, final: result.isFinal, backend: "browser" }); + } + }; + instance.onerror = (event: { error: string }) => { + setState("error", new Error(`SpeechRecognition error: ${event.error}`)); + }; + instance.onend = () => { + active = false; + if (resolveBrowserStop) { + const resolve = resolveBrowserStop; + resolveBrowserStop = null; + browserStopWait = null; + resolve(); + } + }; + + recognition = instance; + instance.start(); + active = true; + setState("listening"); + } + + async function start(): Promise { + if (disposed) { + throw new Error("VoiceCapture handle has been disposed"); + } + if (active) return; + setState("starting"); + try { + if (backend === "local-inference") { + await startLocalInference(); + } else { + startBrowser(); + } + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + setState("error", error); + throw error; + } + } + + async function stop(): Promise { + if (!active && state !== "starting") return; + + if (backend === "local-inference") { + const current = recorder; + recorder = null; + active = false; + if (!current) { + setState("stopped"); + return; + } + try { + const wav = await current.stop(); + const text = await transcribeLocalInferenceWav(wav); + onTranscript({ text, final: true, backend: "local-inference" }); + setState("stopped"); + } catch (err) { + const error = err instanceof Error ? err : new Error(String(err)); + setState("error", error); + throw error; + } + return; + } + + const instance = recognition; + if (!instance) { + active = false; + setState("stopped"); + return; + } + browserStopWait = new Promise((resolve) => { + resolveBrowserStop = resolve; + }); + instance.stop(); + await browserStopWait; + recognition = null; + setState("stopped"); + } + + function dispose(): void { + if (disposed) return; + disposed = true; + if (recorder) { + recorder.cancel(); + recorder = null; + } + if (recognition) { + try { + recognition.abort(); + } finally { + recognition = null; + } + } + active = false; + if (resolveBrowserStop) { + const resolve = resolveBrowserStop; + resolveBrowserStop = null; + browserStopWait = null; + resolve(); + } + setState("idle"); + } + + return { + start, + stop, + dispose, + isActive: () => active, + }; +} diff --git a/apps/app/test/electrobun-packaged/electrobun-windows-startup.e2e.spec.ts b/apps/app/test/electrobun-packaged/electrobun-windows-startup.e2e.spec.ts index 5ed914399f..92e23cad2b 100644 --- a/apps/app/test/electrobun-packaged/electrobun-windows-startup.e2e.spec.ts +++ b/apps/app/test/electrobun-packaged/electrobun-windows-startup.e2e.spec.ts @@ -16,6 +16,41 @@ import { const windowsTest = process.platform === "win32" ? test : null; +interface PackagedRendererSurfaceProbe { + state: "blank" | "ready" | "error"; + bodyText: string; + errorText: string | null; + rootChildCount: number; + url: string; +} + +function getPackagedRendererSurfaceProbeScript(): string { + return `(() => { + const root = document.getElementById("root"); + const bodyText = (document.body?.innerText ?? "").trim(); + const normalizedText = bodyText.replace(/\\s+/g, " "); + const hasErrorBoundary = normalizedText.includes("Something went wrong"); + const hasAppContextError = normalizedText.includes( + "useApp must be used within AppProvider", + ); + const state = hasErrorBoundary || hasAppContextError + ? "error" + : normalizedText.length > 0 + ? "ready" + : "blank"; + + return { + state, + bodyText: normalizedText.slice(0, 1200), + errorText: hasErrorBoundary || hasAppContextError + ? normalizedText.slice(0, 1200) + : null, + rootChildCount: root?.childElementCount ?? 0, + url: window.location.href, + }; + })()`; +} + windowsTest?.( "packaged Windows app bootstraps the renderer against the external API override", async () => { @@ -36,6 +71,10 @@ windowsTest?.( launcherPath: launcherPath as string, apiBase: api.baseUrl, }); + const inheritedWindowsPath = + harness.appEnv.PATH ?? harness.appEnv.Path ?? ""; + delete harness.appEnv.PATH; + harness.appEnv.Path = inheritedWindowsPath; await harness.start(); @@ -72,6 +111,38 @@ windowsTest?.( ); } + let lastSurfaceProbe: PackagedRendererSurfaceProbe | null = null; + try { + await expect + .poll( + async () => { + lastSurfaceProbe = + await harness.eval( + getPackagedRendererSurfaceProbeScript(), + ); + return lastSurfaceProbe.state; + }, + { + timeout: process.env.CI ? 180_000 : 90_000, + message: + "Expected the packaged Windows renderer to render without its error boundary", + }, + ) + .toBe("ready"); + } catch (error) { + throw new Error( + [ + "Expected the packaged Windows renderer to render without its error boundary", + `Last renderer surface probe: ${JSON.stringify(lastSurfaceProbe)}`, + error instanceof Error ? error.message : String(error), + ].join("\n"), + ); + } + + expect(lastSurfaceProbe?.rootChildCount ?? 0).toBeGreaterThan(0); + expect(lastSurfaceProbe?.errorText ?? "").not.toMatch( + /Something went wrong|useApp must be used within AppProvider/i, + ); expect(api.requests.length).toBeGreaterThan(0); expect( `${harness.logs?.stdout.join("") ?? ""}\n${harness.logs?.stderr.join("") ?? ""}`, diff --git a/apps/app/test/electrobun-packaged/live-api.ts b/apps/app/test/electrobun-packaged/live-api.ts index 9b6aed751e..edddb47d9e 100644 --- a/apps/app/test/electrobun-packaged/live-api.ts +++ b/apps/app/test/electrobun-packaged/live-api.ts @@ -1,12 +1,5 @@ -// Local-mode only: this test helper imports from `eliza/packages/app-core/test/helpers/`, -// which is not part of the published `@elizaos/app-core` package. Running the -// packaged-Electrobun E2E suite requires `bun run eliza:local`. See the -// "elizaOS source modes" section of README.md. import http from "node:http"; import type { AddressInfo } from "node:net"; -import { startApiServer } from "../../../../eliza/packages/app-core/src/api/server.ts"; -import { useIsolatedConfigEnv as isolatedConfigEnv } from "../../../../eliza/packages/app-core/test/helpers/isolated-config.ts"; -import { createRealTestRuntime } from "../../../../eliza/packages/app-core/test/helpers/real-runtime.ts"; export interface TestApiServerOptions { port?: number; @@ -48,76 +41,74 @@ function closeServer(server: http.Server): Promise { }); } +function jsonResponse(res: http.ServerResponse, status: number, body: unknown) { + res.statusCode = status; + res.setHeader("access-control-allow-origin", "*"); + res.setHeader( + "access-control-allow-methods", + "GET,POST,PUT,PATCH,DELETE,OPTIONS", + ); + res.setHeader("access-control-allow-headers", "content-type,authorization"); + res.setHeader("content-type", "application/json"); + res.end(JSON.stringify(body)); +} + +function responseBodyFor(pathname: string, method: string): unknown { + if (method === "OPTIONS") { + return {}; + } + if (pathname === "/api/status") { + return { + ok: true, + status: "ok", + onboardingComplete: true, + agent: { name: "PackagedDesktopTest" }, + }; + } + if (pathname === "/api/config") { + return { + onboardingComplete: true, + api: { status: "ok" }, + agent: { name: "PackagedDesktopTest" }, + }; + } + if (pathname === "/api/onboarding" && method === "POST") { + return { success: true, onboardingComplete: true }; + } + if (pathname === "/api/triggers") { + return { triggers: [] }; + } + if (pathname === "/api/drop/status") { + return { ok: true, running: false }; + } + if (pathname === "/api/stream/settings") { + return { enabled: false }; + } + return { ok: true }; +} + export async function startLiveApiServer( options: TestApiServerOptions = {}, ): Promise { - const configEnv = isolatedConfigEnv("milady-packaged-live-api-"); - let runtimeResult: Awaited> | null = - null; - let upstream: Awaited> | null = null; let proxy: http.Server | null = null; try { - runtimeResult = await createRealTestRuntime({ - characterName: "PackagedDesktopTest", - }); - upstream = await startApiServer({ - port: 0, - runtime: runtimeResult.runtime, - skipDeferredStartupWork: true, - }); - const upstreamBaseUrl = `http://127.0.0.1:${upstream.port}`; - - if (options.onboardingComplete) { - const response = await fetch(`${upstreamBaseUrl}/api/onboarding`, { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify({ name: "Packaged Desktop" }), - }); - if (!response.ok) { - throw new Error( - `Failed to seed live onboarding state (${response.status}): ${await response.text()}`, - ); - } - } - const requests: string[] = []; proxy = http.createServer(async (req, res) => { const method = (req.method ?? "GET").toUpperCase(); - const targetUrl = new URL(req.url ?? "/", upstreamBaseUrl); + const targetUrl = new URL(req.url ?? "/", "http://127.0.0.1"); requests.push(`${method} ${targetUrl.pathname}`); - const body = - method === "GET" || method === "HEAD" ? undefined : await readBody(req); - const headers = new Headers(); - for (const [key, value] of Object.entries(req.headers)) { - if (typeof value === "string") { - headers.set(key, value); - continue; - } - if (Array.isArray(value)) { - headers.set(key, value.join(", ")); - } + if (method !== "GET" && method !== "HEAD") { + await readBody(req); } - - const response = await fetch(targetUrl, { - method, - headers, - body, - redirect: "manual", - }); - - res.statusCode = response.status; - response.headers.forEach((value, key) => { - res.setHeader(key, value); - }); - - if (!response.body) { + if (method === "HEAD") { + res.statusCode = 200; + res.setHeader("access-control-allow-origin", "*"); res.end(); return; } - - res.end(Buffer.from(await response.arrayBuffer())); + jsonResponse(res, 200, responseBodyFor(targetUrl.pathname, method)); }); await listen(proxy, options.port ?? 0); @@ -125,24 +116,19 @@ export async function startLiveApiServer( if (!address || typeof address === "string") { throw new Error("Failed to resolve packaged live API proxy address."); } + const server = proxy; return { baseUrl: `http://127.0.0.1:${(address as AddressInfo).port}`, requests, close: async () => { - await closeServer(proxy).catch(() => undefined); - await upstream.close().catch(() => undefined); - await runtimeResult.cleanup().catch(() => undefined); - await configEnv.restore().catch(() => undefined); + await closeServer(server).catch(() => undefined); }, }; } catch (error) { if (proxy) { await closeServer(proxy).catch(() => undefined); } - await upstream?.close().catch(() => undefined); - await runtimeResult?.cleanup().catch(() => undefined); - await configEnv.restore().catch(() => undefined); throw error; } } diff --git a/apps/app/test/electrobun-packaged/packaged-app-helpers.ts b/apps/app/test/electrobun-packaged/packaged-app-helpers.ts index de03a8d540..48b678f186 100644 --- a/apps/app/test/electrobun-packaged/packaged-app-helpers.ts +++ b/apps/app/test/electrobun-packaged/packaged-app-helpers.ts @@ -11,20 +11,30 @@ import { createPackagedWindowsAppEnv } from "./windows-test-env"; const execFileAsync = promisify(execFile); const here = path.dirname(fileURLToPath(import.meta.url)); const repoRoot = path.resolve(here, "../../../.."); -const electrobunArtifactsDir = path.join( - repoRoot, - "apps", - "app", - "electrobun", - "artifacts", -); -const electrobunBuildDir = path.join( - repoRoot, - "apps", - "app", - "electrobun", - "build", -); +const electrobunBuildDirs = [ + path.join( + repoRoot, + "eliza", + "packages", + "app-core", + "platforms", + "electrobun", + "build", + ), + path.join(repoRoot, "apps", "app", "electrobun", "build"), +]; +const electrobunArtifactsDirs = [ + path.join( + repoRoot, + "eliza", + "packages", + "app-core", + "platforms", + "electrobun", + "artifacts", + ), + path.join(repoRoot, "apps", "app", "electrobun", "artifacts"), +]; export interface PackagedProcessLogs { stdout: string[]; @@ -105,16 +115,28 @@ async function findMacLauncher(): Promise { } const candidates = [ - ...(await findFiles(electrobunBuildDir, (fullPath) => - fullPath.endsWith( - `${path.sep}Contents${path.sep}MacOS${path.sep}launcher`, - ), - )), - ...(await findFiles(electrobunArtifactsDir, (fullPath) => - fullPath.endsWith( - `${path.sep}Contents${path.sep}MacOS${path.sep}launcher`, - ), - )), + ...( + await Promise.all( + electrobunBuildDirs.map((buildDir) => + findFiles(buildDir, (fullPath) => + fullPath.endsWith( + `${path.sep}Contents${path.sep}MacOS${path.sep}launcher`, + ), + ), + ), + ) + ).flat(), + ...( + await Promise.all( + electrobunArtifactsDirs.map((artifactsDir) => + findFiles(artifactsDir, (fullPath) => + fullPath.endsWith( + `${path.sep}Contents${path.sep}MacOS${path.sep}launcher`, + ), + ), + ), + ) + ).flat(), ]; if (candidates.length === 0) { @@ -160,25 +182,37 @@ async function resolveWindowsLauncher(tempExtractDir: string): Promise { return await fs.realpath(explicit); } - let launcher = await findWindowsLauncherExe(electrobunBuildDir); - if (launcher) { - return launcher; + for (const buildDir of electrobunBuildDirs) { + const launcher = await findWindowsLauncherExe(buildDir); + if (launcher) { + return launcher; + } } - launcher = await findWindowsLauncherExe(electrobunArtifactsDir); - if (launcher) { - return launcher; + for (const artifactsDir of electrobunArtifactsDirs) { + const launcher = await findWindowsLauncherExe(artifactsDir); + if (launcher) { + return launcher; + } } - const artifactEntries = await fs - .readdir(electrobunArtifactsDir, { withFileTypes: true }) - .catch(() => []); - const tarballs = artifactEntries - .filter((entry) => entry.isFile() && entry.name.endsWith(".tar.zst")) - .map((entry) => path.join(electrobunArtifactsDir, entry.name)); + const tarballs = ( + await Promise.all( + electrobunArtifactsDirs.map(async (artifactsDir) => { + const artifactEntries = await fs + .readdir(artifactsDir, { withFileTypes: true }) + .catch(() => []); + return artifactEntries + .filter((entry) => entry.isFile() && entry.name.endsWith(".tar.zst")) + .map((entry) => path.join(artifactsDir, entry.name)); + }), + ) + ).flat(); if (tarballs.length === 0) { throw new Error( - `No Windows packaged artifacts found in ${electrobunArtifactsDir}.`, + `No Windows packaged artifacts found in ${electrobunArtifactsDirs.join( + ", ", + )}.`, ); } @@ -200,7 +234,7 @@ async function resolveWindowsLauncher(tempExtractDir: string): Promise { tempExtractDir, ]); - launcher = await findWindowsLauncherExe(tempExtractDir); + const launcher = await findWindowsLauncherExe(tempExtractDir); if (!launcher) { throw new Error( `Failed to find launcher.exe after extracting ${archivePath}.`, diff --git a/apps/app/test/host-window-routing.test.ts b/apps/app/test/host-window-routing.test.ts new file mode 100644 index 0000000000..fe7b81aabc --- /dev/null +++ b/apps/app/test/host-window-routing.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; + +import { + getWindowNavigationPath, + isAppWindowRoute, + isDetachedWindowShell, + isPillWindowShell, + resolveWindowShellRoute, + shouldInstallMainWindowOnboardingPatches, +} from "../src/host-window-routing"; + +describe("host window routing", () => { + it("resolves main, pill, settings, and detached surface shells", () => { + expect(resolveWindowShellRoute("")).toEqual({ mode: "main" }); + expect(resolveWindowShellRoute("?shell=pill")).toEqual({ mode: "pill" }); + expect(resolveWindowShellRoute("?shell=settings&tab=voice")).toEqual({ + mode: "settings", + tab: "voice", + }); + expect(resolveWindowShellRoute("?shell=surface&tab=chat")).toEqual({ + mode: "surface", + tab: "chat", + }); + expect(resolveWindowShellRoute("?shell=surface&tab=unknown")).toEqual({ + mode: "main", + }); + }); + + it("classifies shell routes for main, pill, and detached windows", () => { + const mainRoute = resolveWindowShellRoute(""); + const pillRoute = resolveWindowShellRoute("?shell=pill"); + const settingsRoute = resolveWindowShellRoute("?shell=settings"); + + expect(shouldInstallMainWindowOnboardingPatches(mainRoute)).toBe(true); + expect(shouldInstallMainWindowOnboardingPatches(pillRoute)).toBe(false); + expect(isPillWindowShell(pillRoute)).toBe(true); + expect(isDetachedWindowShell(settingsRoute)).toBe(true); + }); + + it("uses hash navigation for file URLs and app windows", () => { + expect( + getWindowNavigationPath({ + protocol: "file:", + search: "", + hash: "#/chat", + pathname: "/index.html", + }), + ).toBe("/chat"); + expect( + getWindowNavigationPath({ + protocol: "https:", + search: "?appWindow=1", + hash: "#/settings", + pathname: "/", + }), + ).toBe("/settings"); + expect( + getWindowNavigationPath({ + protocol: "https:", + search: "", + hash: "#/ignored", + pathname: "/browser", + }), + ).toBe("/browser"); + }); + + it("detects app-window routes from query state", () => { + expect(isAppWindowRoute({ search: "?appWindow=1" })).toBe(true); + expect(isAppWindowRoute({ search: "?appWindow=0" })).toBe(false); + }); +}); diff --git a/apps/app/test/package-mode-aliases.test.ts b/apps/app/test/package-mode-aliases.test.ts index 8e29d9f66e..bec7ed0a05 100644 --- a/apps/app/test/package-mode-aliases.test.ts +++ b/apps/app/test/package-mode-aliases.test.ts @@ -66,7 +66,11 @@ describe("package mode aliases", () => { expect(stubText).toContain( "@elizaos/app-core/components/chat/widgets/types", ); + expect(mainText).toContain('from "./voice-pill-runtime";'); + expect(mainText).toContain('from "./host-window-routing";'); + expect(mainText).not.toContain('from "@elizaos/ui";'); expect(mainText).not.toContain("@elizaos/ui/components/"); + expect(stubText).toContain("type PromptOptions = Record"); expect(stubText).not.toContain("@elizaos/ui/state/"); expect(stubText).not.toContain("@elizaos/ui/components/"); }); @@ -105,4 +109,176 @@ describe("package mode aliases", () => { expect(patchScript).toContain("EXTRACT_ACTION_PARAMS_TEMPLATE"); expect(patchScript).toContain("extractActionParamsTemplate"); }); + + it("patches local-source UI AppContext to survive duplicate renderer module ids", () => { + const patchScript = fs.readFileSync( + path.resolve(appRoot, "../..", "scripts/apply-eliza-ci-patches.mjs"), + "utf8", + ); + + expect(patchScript).toContain("patchUiAppContextSingleton"); + expect(patchScript).toContain("__ELIZAOS_UI_APP_CONTEXT__"); + expect(patchScript).toContain("UI AppContext singleton"); + }); + + it("patches voice onboarding so users can skip the mic gate", () => { + const patchScript = fs.readFileSync( + path.resolve(appRoot, "../..", "scripts/apply-eliza-ci-patches.mjs"), + "utf8", + ); + + expect(patchScript).toContain("patchVoicePrefixWelcomeSkip"); + expect(patchScript).toContain("voice-prefix-skip-prefix"); + expect(patchScript).toContain("Skip voice setup"); + expect(patchScript).toContain("legacyBefore"); + expect(patchScript).toContain( + 'props.step === "welcome" && props.onSkipPrefix', + ); + expect(patchScript).toContain("voice prefix welcome skip action"); + }); + + it("patches onboarding avatar canvases so they cannot block buttons", () => { + const patchScript = fs.readFileSync( + path.resolve(appRoot, "../..", "scripts/apply-eliza-ci-patches.mjs"), + "utf8", + ); + + expect(patchScript).toContain("patchOnboardingAvatarUsage"); + expect(patchScript).toContain("OnboardingAvatar.tsx"); + expect(patchScript).toContain("eliza-ob-agent-canvas"); + expect(patchScript).toContain('pointerEvents: "none"'); + expect(patchScript).toContain("onboarding avatar non-interactive wrapper"); + }); + + it("patches packaged Windows agent startup to tolerate Explorer-style Path casing", () => { + const patchScript = fs.readFileSync( + path.resolve(appRoot, "../..", "scripts/apply-eliza-ci-patches.mjs"), + "utf8", + ); + + expect(patchScript).toContain("patchElectrobunAgentChildPathFallback"); + expect(patchScript).toContain("existingPathKey"); + expect(patchScript).toContain("Electrobun agent child PATH fallback"); + }); + + it("patches LifeOps automation registration away from the app-core root barrel", () => { + const patchScript = fs.readFileSync( + path.resolve(appRoot, "../..", "scripts/apply-eliza-ci-patches.mjs"), + "utf8", + ); + + expect(patchScript).toContain( + "patchAppCoreAutomationNodeContributorExport", + ); + expect(patchScript).toContain( + "patchLifeOpsAutomationContributorAppCoreImport", + ); + expect(patchScript).toContain( + "@elizaos/app-core/api/automation-node-contributors", + ); + expect(patchScript).toContain( + "plugin-lifeops narrow app-core automation import", + ); + }); + + it("keeps generated native module proxy stubs compatible with WebKit invariants", () => { + const viteConfigText = fs.readFileSync( + path.join(appRoot, "vite.config.ts"), + "utf8", + ); + + expect(viteConfigText).toContain( + "ownKeys(t) { return Reflect.ownKeys(t); }", + ); + expect(viteConfigText).toContain( + "getOwnPropertyDescriptor(t, p) { return Reflect.getOwnPropertyDescriptor(t, p)", + ); + expect(viteConfigText).toContain( + "p === 'prototype' || p === 'name' || p === 'length'", + ); + expect(viteConfigText).not.toContain("ownKeys() { return []; }"); + expect(viteConfigText).not.toContain("p === 'prototype') return {}"); + }); + + it("prefers local source aliases before stale package dist fallbacks", () => { + const viteConfigText = fs.readFileSync( + path.join(appRoot, "vite.config.ts"), + "utf8", + ); + + expect(viteConfigText).toContain( + "const runtimeTarget = resolveRuntimeTarget(pkgDir, exportTarget)", + ); + expect(viteConfigText).toContain("fs.existsSync(runtimeTarget)"); + expect(viteConfigText).toContain("replacement: runtimeTarget"); + expect(viteConfigText).toContain("function resolveAppCoreExportTarget"); + expect(viteConfigText).toContain( + "const sourcePath = resolveAppCoreWithUiFallback", + ); + expect(viteConfigText).toContain("function resolveSharedExportTarget"); + expect(viteConfigText).toContain("function resolveLocalCloudSdkAliases"); + expect(viteConfigText).toContain( + "@elizaos\\/cloud-sdk\\/cloud-setup-session", + ); + expect(viteConfigText).toContain( + 'for (const condition of ["source", "import", "default", "types"])', + ); + expect(viteConfigText).toContain("resolveSharedSourceOrDist"); + expect(viteConfigText).toContain("@elizaos\\/ui\\/platform"); + expect(viteConfigText).toContain("find: /^@elizaos\\/ui$/"); + expect(viteConfigText).toContain('src/browser.ts"'); + expect(viteConfigText).toContain("find: /^@elizaos\\/ui\\/state$/"); + expect(viteConfigText).toContain("find: /^@elizaos\\/ui\\/state\\/(.*)$/"); + expect(viteConfigText).toContain("function elizaUiStateSingletonPlugin"); + expect(viteConfigText).toContain('name: "eliza-ui-state-singleton"'); + expect(viteConfigText).toContain("@elizaos\\/ui\\/(.+)"); + expect(viteConfigText).toContain("fs.statSync(candidate).isFile()"); + expect(viteConfigText).not.toContain("fs.existsSync(resolvedTarget)"); + }); + + it("does not duplicate core browser exports that upstream now provides directly", () => { + const viteConfigText = fs.readFileSync( + path.join(appRoot, "vite.config.ts"), + "utf8", + ); + + expect(viteConfigText).toContain("function hasNamedExport"); + expect(viteConfigText).toContain( + "export\\\\s+(?:async\\\\s+)?(?:function|class|const|let|var)", + ); + expect(viteConfigText).toContain("(name) => !hasNamedExport(name)"); + }); + + it("aliases React entrypoints to one resolved package copy", () => { + const viteConfigText = fs.readFileSync( + path.join(appRoot, "vite.config.ts"), + "utf8", + ); + const tsconfig = JSON.parse( + fs.readFileSync(path.join(appRoot, "tsconfig.json"), "utf8"), + ); + + expect(viteConfigText).toContain('requireResolve("react")'); + expect(viteConfigText).toContain('requireResolve("react/jsx-runtime")'); + expect(viteConfigText).toContain("find: /^react-dom\\/client$/"); + expect(viteConfigText).toContain("replacement: reactDomClientEntry"); + expect(viteConfigText).toContain('"@elizaos/ui"'); + expect(tsconfig.compilerOptions.paths.react).toBeUndefined(); + expect(tsconfig.compilerOptions.paths["react/jsx-runtime"]).toBeUndefined(); + }); + + it("keeps the local app-core browser surface aligned with app imports", () => { + const appCoreBrowserPath = path.resolve( + appRoot, + "../..", + "eliza/packages/app-core/src/browser.ts", + ); + if (!fs.existsSync(appCoreBrowserPath)) { + return; + } + + const appCoreBrowserText = fs.readFileSync(appCoreBrowserPath, "utf8"); + expect(appCoreBrowserText).toContain("resolveIosRuntimeConfig"); + expect(appCoreBrowserText).toContain("type IosRuntimeConfig"); + }); }); diff --git a/apps/app/tsconfig.json b/apps/app/tsconfig.json index 4b05aec1cc..ca510e85ca 100644 --- a/apps/app/tsconfig.json +++ b/apps/app/tsconfig.json @@ -51,13 +51,6 @@ "@elizaos/skills/*": ["./node_modules/@elizaos/skills/*"], "@elizaos/ui": ["./node_modules/@elizaos/ui"], "@elizaos/ui/*": ["./node_modules/@elizaos/ui/*"], - "react": ["./node_modules/@types/react/index.d.ts"], - "react/jsx-runtime": ["./node_modules/@types/react/jsx-runtime.d.ts"], - "react/jsx-dev-runtime": [ - "./node_modules/@types/react/jsx-dev-runtime.d.ts" - ], - "react-dom": ["./node_modules/@types/react-dom/index.d.ts"], - "react-dom/client": ["./node_modules/@types/react-dom/client.d.ts"], "@elizaos/*": ["./node_modules/@elizaos/*"], "@elizaos/app-2004scape": ["./apps/app/src/optional-eliza-app-stub.tsx"], "@elizaos/app-2004scape/*": [ diff --git a/apps/app/vite.config.ts b/apps/app/vite.config.ts index ec59518abf..08ea1e6479 100644 --- a/apps/app/vite.config.ts +++ b/apps/app/vite.config.ts @@ -126,6 +126,12 @@ function tryResolve(id: string): string | undefined { const capacitorKeyboardEntry = tryResolve("@capacitor/keyboard"); const capacitorPreferencesEntry = tryResolve("@capacitor/preferences"); const capacitorAppEntry = tryResolve("@capacitor/app"); +const reactEntry = requireResolve("react"); +const reactJsxRuntimeEntry = requireResolve("react/jsx-runtime"); +const reactJsxDevRuntimeEntry = requireResolve("react/jsx-dev-runtime"); +const reactDomEntry = requireResolve("react-dom"); +const reactDomClientEntry = requireResolve("react-dom/client"); + // `@elizaos/app-core` is always real. `@elizaos/app-wallet` is required by // onboarding callbacks + AppContext (useWalletState), so resolve it real // when present. `app-hyperscape` is real when its package is present. @@ -239,7 +245,20 @@ function resolveLocalUiAliases(): Alias[] { return [ { find: /^@elizaos\/ui$/, - replacement: path.join(uiPkgRoot, "src/index.ts"), + replacement: path.join(uiPkgRoot, "src/browser.ts"), + }, + { + find: /^@elizaos\/ui\/browser$/, + replacement: path.join(uiPkgRoot, "src/browser.ts"), + }, + { + find: /^@elizaos\/ui\/api$/, + replacement: path.join(uiPkgRoot, "src/api/index.ts"), + }, + { + find: /^@elizaos\/ui\/api\/(.*)$/, + replacement: `${uiPkgRoot}/src/api/$1.ts`, + customResolver: resolveExistingUiSourceModule, }, { find: /^@elizaos\/ui\/components\/ui\/(.*)$/, @@ -280,10 +299,29 @@ function resolveLocalUiAliases(): Alias[] { find: /^@elizaos\/ui\/layouts\/(.+)\/([^/]+)$/, replacement: `${uiPkgRoot}/src/layouts/$1/$2.tsx`, }, + { + find: /^@elizaos\/ui\/platform\/(.*)$/, + replacement: `${uiPkgRoot}/src/platform/$1.ts`, + customResolver: resolveExistingUiSourceModule, + }, + { + find: /^@elizaos\/ui\/state$/, + replacement: path.join(uiPkgRoot, "src/state/index.ts"), + }, + { + find: /^@elizaos\/ui\/state\/(.*)$/, + replacement: `${uiPkgRoot}/src/state/$1.ts`, + customResolver: resolveExistingUiSourceModule, + }, { find: /^@elizaos\/ui\/lib\/(.*)$/, replacement: `${uiPkgRoot}/src/lib/$1.ts`, }, + { + find: /^@elizaos\/ui\/(.+)$/, + replacement: `${uiPkgRoot}/src/$1`, + customResolver: resolveExistingUiSourceModule, + }, ]; } @@ -353,16 +391,16 @@ function resolveLocalElizaAppAliases(): Alias[] { for (const [key, value] of Object.entries(pkg.exports || {})) { const exportTarget = resolveExportTarget(value); if (!exportTarget) continue; - const resolvedTarget = path.resolve(pkgDir, exportTarget); - // Only create an alias when the target file actually exists on disk. - // In a fresh local clone, dist/ may not be built yet. Skipping the - // alias lets the import fall through to the stub or npm package. - if (!fs.existsSync(resolvedTarget)) continue; + const runtimeTarget = resolveRuntimeTarget(pkgDir, exportTarget); + // Prefer local source targets when the package export points at dist/. + // Fresh local clones often have src/ but no dist/ yet; checking dist + // first lets imports fall through to stale npm/Bun-store packages. + if (!fs.existsSync(runtimeTarget)) continue; const aliasKey = key === "." ? pkgName : `${pkgName}/${key.replace(/^\.\//, "")}`; aliases.push({ find: new RegExp(`^${escapeRegExp(aliasKey)}$`), - replacement: resolveRuntimeTarget(pkgDir, exportTarget), + replacement: runtimeTarget, }); } @@ -396,24 +434,114 @@ function resolveLocalSharedAliases(): Alias[] { if (!fs.existsSync(sharedPkgPath)) return []; const sharedPkgDir = path.dirname(sharedPkgPath); + const sharedSrcDir = path.join(sharedPkgDir, "src"); + const sharedDistDir = path.join(sharedPkgDir, "dist"); const sharedPkg = JSON.parse(fs.readFileSync(sharedPkgPath, "utf8")) as { exports?: Record; }; + + function resolveSharedExportTarget(value: unknown): string | null { + if (typeof value === "string") return value; + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + const record = value as Record; + for (const condition of ["source", "import", "default", "types"]) { + const target = record[condition]; + if (typeof target === "string") return target; + } + return null; + } + + function resolveSharedSourceOrDist(id: string): string { + const sourcePath = resolveExistingUiSourceModule(id); + if (fs.existsSync(sourcePath)) return sourcePath; + + if (id.startsWith(`${sharedSrcDir}${path.sep}`)) { + const relative = path.relative(sharedSrcDir, id); + const distBase = path.join(sharedDistDir, relative); + const distCandidates = [ + distBase, + `${distBase}.js`, + `${distBase}.css`, + path.join(distBase, "index.js"), + ]; + for (const candidate of distCandidates) { + if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) { + return candidate; + } + } + } + + return id; + } + + function resolveSharedRuntimeTarget(exportTarget: string): string { + if (exportTarget === "./package.json") { + return path.join(sharedPkgDir, "package.json"); + } + if (exportTarget.startsWith("./dist/")) { + const sourceTarget = exportTarget + .replace(/^\.\/dist\//, "./src/") + .replace(/\.js$/, ".ts"); + return resolveSharedSourceOrDist( + path.resolve(sharedPkgDir, sourceTarget), + ); + } + return resolveSharedSourceOrDist(path.resolve(sharedPkgDir, exportTarget)); + } + const aliases: Alias[] = []; for (const [key, value] of Object.entries(sharedPkg.exports || {})) { - if (typeof value !== "string") continue; + const exportTarget = resolveSharedExportTarget(value); + if (!exportTarget) continue; const aliasKey = key === "." ? "@elizaos/shared" : `@elizaos/shared/${key.replace(/^\.\//, "")}`; + if (aliasKey.includes("*")) { + aliases.push({ + find: new RegExp(`^${escapeRegExp(aliasKey).replace("\\*", "(.+)")}$`), + replacement: resolveSharedRuntimeTarget(exportTarget).replace( + "*", + "$1", + ), + customResolver: resolveSharedSourceOrDist, + }); + continue; + } aliases.push({ find: new RegExp(`^${escapeRegExp(aliasKey)}$`), - replacement: path.resolve(sharedPkgDir, value), + replacement: resolveSharedRuntimeTarget(exportTarget), + customResolver: resolveSharedSourceOrDist, }); } return aliases; } +function resolveLocalCloudSdkAliases(): Alias[] { + if (!hasLocalElizaWorkspace) return []; + + const cloudSdkSrcDir = path.join(localElizaRoot, "packages/cloud-sdk/src"); + if (!fs.existsSync(path.join(cloudSdkSrcDir, "index.ts"))) return []; + + return [ + { + find: /^@elizaos\/cloud-sdk$/, + replacement: path.join(cloudSdkSrcDir, "index.ts"), + }, + { + find: /^@elizaos\/cloud-sdk\/cloud-setup-session$/, + replacement: path.join(cloudSdkSrcDir, "cloud-setup-session/index.ts"), + }, + { + find: /^@elizaos\/cloud-sdk\/cloud-setup-session\/(.+)$/, + replacement: `${cloudSdkSrcDir}/cloud-setup-session/$1.ts`, + customResolver: resolveExistingUiSourceModule, + }, + ]; +} + function resolveBuiltLocalSharedAliases(): Alias[] { if (!hasLocalElizaWorkspace) return []; @@ -442,14 +570,6 @@ function resolveLocalAppCoreAliases(): Alias[] { const agentRootEntry = appCoreSrcRoot ? path.join(localElizaRoot, "packages/agent/src/index.ts") : emptyNodeModuleEntry; - const sharedDistEntry = path.join( - localElizaRoot, - "packages/shared/dist/index.js", - ); - const sharedDist = fs.existsSync(sharedDistEntry) ? sharedDistEntry : null; - const sharedDistDir = sharedDist - ? path.join(localElizaRoot, "packages/shared/dist") - : null; const packageAgnosticAliases: Alias[] = [ { find: /^@elizaos\/agent$/, @@ -494,23 +614,41 @@ function resolveLocalAppCoreAliases(): Alias[] { replacement: appCoreBrowserEntry, }); + function resolveAppCoreExportTarget(exportTarget: string): string { + if (exportTarget === "./package.json") { + return path.join(appCorePkgDir, "package.json"); + } + if (exportTarget.startsWith("./dist/")) { + const sourceTarget = exportTarget + .replace(/^\.\/dist\//, "./src/") + .replace(/\.js$/, ".ts"); + const sourcePath = resolveAppCoreWithUiFallback( + path.resolve(appCorePkgDir, sourceTarget), + ); + if (fs.existsSync(sourcePath)) { + return sourcePath; + } + } + return path.resolve(appCorePkgDir, exportTarget); + } + for (const [key, value] of Object.entries(appCorePkg.exports || {})) { if (key === ".") continue; // handled by the explicit bare alias above - if (typeof value !== "string") continue; - const aliasKey = - key === "." - ? "@elizaos/app-core" - : `@elizaos/app-core/${key.replace(/^\.\//, "")}`; - - // Resolve the string value, handling both plain strings and conditional exports. const resolvedValue: string | null = typeof value === "string" ? value : typeof value === "object" && value !== null - ? ((value as Record).import ?? + ? ((value as Record).source ?? + (value as Record).import ?? (value as Record).default ?? null) : null; + if (!resolvedValue) continue; + + const aliasKey = + key === "." + ? "@elizaos/app-core" + : `@elizaos/app-core/${key.replace(/^\.\//, "")}`; // CSS files in app-core exports point to dist paths (e.g. ./styles/styles.css). // In Wave A these moved to @elizaos/ui. If the dist path doesn't exist locally, @@ -539,7 +677,7 @@ function resolveLocalAppCoreAliases(): Alias[] { key === "." ? appCoreBrowserEntry : resolvedValue - ? path.resolve(appCorePkgDir, resolvedValue) + ? resolveAppCoreExportTarget(resolvedValue) : null; if (!targetPath) continue; @@ -557,6 +695,52 @@ function resolveLocalAppCoreAliases(): Alias[] { const uiSource = path.join(appCoreSrcRoot, "ui"); const uiPkgSrcRoot = uiPkgRoot ? path.join(uiPkgRoot, "src") : null; + // Wave A moved styles from @elizaos/app-core to @elizaos/ui. Keep an + // explicit redirect ahead of the catch-all so local-source builds do not + // resolve CSS requests to missing app-core source files. + const uiStylesSourceDir = uiPkgRoot + ? path.join(uiPkgRoot, "src/styles") + : null; + const appCoreStylesLocalDir = path.join(appCoreSrcRoot, "styles"); + const cssRedirectAlias: Alias[] = + uiStylesSourceDir && + fs.existsSync(path.join(uiStylesSourceDir, "styles.css")) && + !fs.existsSync(path.join(appCoreStylesLocalDir, "styles.css")) + ? [ + { + find: /^@elizaos\/app-core\/styles\/(.+\.css)$/, + replacement: `${uiStylesSourceDir}/$1`, + }, + ] + : []; + const uiComponentsSourceDir = uiPkgRoot ? path.join(uiPkgRoot, "src") : null; + + function resolveAppCoreWithUiFallback(id: string): string { + if (fs.existsSync(id)) return id; + const withTsx = id.endsWith(".tsx") ? id : `${id}.tsx`; + if (fs.existsSync(withTsx)) return withTsx; + const withTs = id.endsWith(".ts") ? id : `${id}.ts`; + if (fs.existsSync(withTs)) return withTs; + + if ( + uiComponentsSourceDir && + id.startsWith(`${appCoreSrcRoot}${path.sep}`) + ) { + const relativeToAppCoreSrc = id.slice(appCoreSrcRoot.length + 1); + const uiEquivalent = path.join( + uiComponentsSourceDir, + relativeToAppCoreSrc, + ); + if (fs.existsSync(uiEquivalent)) return uiEquivalent; + const uiEquivalentTsx = `${uiEquivalent}.tsx`; + if (fs.existsSync(uiEquivalentTsx)) return uiEquivalentTsx; + const uiEquivalentTs = `${uiEquivalent}.ts`; + if (fs.existsSync(uiEquivalentTs)) return uiEquivalentTs; + } + + return id; + } + const legacyAppCoreUiAliases: Alias[] = uiPkgSrcRoot ? [ { @@ -620,34 +804,8 @@ function resolveLocalAppCoreAliases(): Alias[] { ] : []; - // Wave A moved several components/types from @elizaos/app-core to @elizaos/ui. - // The catch-all maps to app-core/src/* which may not have them. Use a custom - // resolver to fall back to the ui source when the app-core path doesn't exist. - const uiComponentsSourceDir = uiPkgRoot ? path.join(uiPkgRoot, "src") : null; - - function resolveAppCoreWithUiFallback(id: string): string { - if (fs.existsSync(id)) return id; - const withTsx = id.endsWith(".tsx") ? id : `${id}.tsx`; - if (fs.existsSync(withTsx)) return withTsx; - const withTs = id.endsWith(".ts") ? id : `${id}.ts`; - if (fs.existsSync(withTs)) return withTs; - if (uiComponentsSourceDir && appCoreSrcRoot) { - const relativeToSrc = id.includes(`${appCoreSrcRoot}/`) - ? id.slice(appCoreSrcRoot.length + 1) - : null; - if (relativeToSrc) { - const uiEquiv = path.join(uiComponentsSourceDir, relativeToSrc); - if (fs.existsSync(uiEquiv)) return uiEquiv; - const uiEquivTsx = `${uiEquiv}.tsx`; - if (fs.existsSync(uiEquivTsx)) return uiEquivTsx; - const uiEquivTs = `${uiEquiv}.ts`; - if (fs.existsSync(uiEquivTs)) return uiEquivTs; - } - } - return id; - } - return [ + ...cssRedirectAlias, ...generatedAliases, ...legacyAppCoreUiAliases, { @@ -874,7 +1032,7 @@ function resolveExistingUiSourceModule(id: string) { } for (const candidate of candidates) { - if (fs.existsSync(candidate)) { + if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) { return candidate; } } @@ -882,6 +1040,51 @@ function resolveExistingUiSourceModule(id: string) { return id; } +function toModulePath(id: string): string { + return id.replace(/\\/g, "/"); +} + +function elizaUiStateSingletonPlugin(): Plugin { + const canonicalUseApp = uiPkgRoot + ? path.join(uiPkgRoot, "src/state/useApp.ts") + : null; + const canonicalUseAppModule = canonicalUseApp + ? toModulePath(canonicalUseApp) + : null; + + return { + name: "eliza-ui-state-singleton", + enforce: "pre", + resolveId(id, importer) { + if (!canonicalUseApp || !canonicalUseAppModule) { + return null; + } + + if (id === "@elizaos/ui/state/useApp") { + return canonicalUseApp; + } + + const normalizedId = toModulePath(id); + if ( + normalizedId !== canonicalUseAppModule && + normalizedId.endsWith("/packages/ui/src/state/useApp") + ) { + return canonicalUseApp; + } + + const normalizedImporter = importer ? toModulePath(importer) : ""; + if ( + (id === "./useApp" || id === "./useApp.ts") && + normalizedImporter.includes("/packages/ui/src/state/") + ) { + return canonicalUseApp; + } + + return null; + }, + }; +} + /** * Bun stores a full npm tarball under node_modules/.bun even when the workspace * symlink for @elizaos/core points at an unbuilt local eliza checkout. @@ -973,12 +1176,10 @@ function isElizaCoreBrowserDistId(id: string | undefined): boolean { function resolveElizaCoreBundlePath(): string { const pkgDir = tryResolveElizaCorePkgDir(); // Prefer the Node entry for the renderer bundle when running against a - // linked eliza checkout. Upstream's hand-curated index.browser.ts misses - // many symbols that server-side modules (statically reachable from the - // renderer's dep graph) re-export — fixing each one is whack-a-mole. The - // Node entry has the full surface; nativeModuleStubPlugin + the rollup - // externals already neutralize the Node-only API surface, and tree-shaking - // drops unused code. + // linked eliza checkout. It carries the full export surface that renderer- + // reachable server barrels may reference; nativeModuleStubPlugin and the + // Rollup externals neutralize Node-only APIs, and tree-shaking drops the + // unused server code. if (pkgDir) { const nodeEntry = path.join(pkgDir, "dist/node/index.node.js"); if (fs.existsSync(nodeEntry)) return nodeEntry; @@ -1344,7 +1545,7 @@ function generateNodeBuiltinStub(moduleId: string, req = _require): string { // * mutation traps (set / defineProperty) don't throw under strict mode // * `instanceof`, `default`, `__esModule` resolve sensibly for ESM<->CJS "function noopFn() { return noop; }", - "const handler = { get(t, p) { if (typeof p === 'symbol') return undefined; if (p === '__esModule') return true; if (p === 'default') return noop; if (p === 'prototype') return {}; if (p in t) return t[p]; return noop; }, set(t, p, v) { try { t[p] = v; } catch {} return true; }, has() { return true; }, ownKeys() { return []; }, getOwnPropertyDescriptor() { return { configurable: true, enumerable: true }; }, apply() { return noop; }, construct() { return noop; }, defineProperty(t, p, d) { try { Object.defineProperty(t, p, { configurable: true, writable: true, enumerable: true, ...d }); } catch {} return true; } };", + "const handler = { get(t, p) { if (p === 'prototype' || p === 'name' || p === 'length' || typeof p === 'symbol') return Reflect.get(t, p); if (p === '__esModule') return true; if (p === 'default') return noop; if (p in t) return t[p]; return noop; }, set(t, p, v) { try { t[p] = v; } catch {} return true; }, has() { return true; }, ownKeys(t) { return Reflect.ownKeys(t); }, getOwnPropertyDescriptor(t, p) { return Reflect.getOwnPropertyDescriptor(t, p) ?? { configurable: true, enumerable: true, writable: true, value: noop }; }, apply() { return noop; }, construct() { return noop; }, defineProperty(t, p, d) { try { Object.defineProperty(t, p, { configurable: true, writable: true, enumerable: true, ...d }); } catch {} return true; } };", "const noop = new Proxy(noopFn, handler);", "const stub = noop;", "const asyncNoop = () => Promise.resolve();", @@ -1788,6 +1989,7 @@ function generatePluginElizacloudStub(): string { // agent runtime modules. The renderer never enters those code paths; the // stub satisfies Rollup's static analysis and trees away at module init. const PLUGIN_LOCAL_INFERENCE_STUB_NAMES = [ + "detectEmbeddingPreset", "getLocalInferenceActiveModelId", "getLocalInferenceActiveSnapshot", "getLocalInferenceChatStatus", @@ -1799,6 +2001,53 @@ function generatePluginLocalInferenceStub(): string { return generateNamedExportStub(PLUGIN_LOCAL_INFERENCE_STUB_NAMES); } +function generatePluginAgentSkillsStub(): string { + return generateNamedExportStub([ + "discoverSkills", + "handleCuratedSkillsRoutes", + "handleSkillsRoutes", + ]); +} + +function generatePluginAppManagerStub(): string { + return [ + "const noop = () => undefined;", + "const asyncFalse = async () => false;", + "export class AppManager {}", + "export const handleAppsRoutes = asyncFalse;", + "export const readAppRunStore = () => [];", + "export const resolveAppRunStoreFilePath = () => '';", + "export const resolveLegacyAppRunStoreFilePath = () => '';", + "export const writeAppRunStore = noop;", + "export default new Proxy(noop, { get: () => noop, apply: () => undefined });", + ].join("\n"); +} + +function generatePluginRegistryStub(): string { + return [ + "const noop = () => undefined;", + "const asyncFalse = async () => false;", + "const emptyPluginList = () => ({ plugins: [], categories: [], installed: [] });", + "export const buildPluginListResponse = emptyPluginList;", + "export const handlePluginRoutes = asyncFalse;", + "export const handlePluginsCompatRoutes = asyncFalse;", + "export const installAndRestart = noop;", + "export const installPlugin = noop;", + "export const listInstalledPlugins = () => [];", + "export const uninstallAndRestart = noop;", + "export const uninstallPlugin = noop;", + "export default new Proxy(noop, { get: () => noop, apply: () => undefined });", + ].join("\n"); +} + +function generatePluginWalletStub(): string { + return generateNamedExportStub(["handleWalletRoutes"]); +} + +function generatePluginX402Stub(): string { + return generateNamedExportStub(["validateX402Startup"]); +} + function generateAgentPluginAutoEnableStub(): string { return [ "export const CONNECTOR_PLUGINS = {};", @@ -2016,6 +2265,11 @@ const NATIVE_MODULE_STUB_GENERATORS = new Map< ["async_hooks", generateAsyncHooksStub], ["@elizaos/plugin-elizacloud", generatePluginElizacloudStub], ["@elizaos/plugin-local-inference", generatePluginLocalInferenceStub], + ["@elizaos/plugin-agent-skills", generatePluginAgentSkillsStub], + ["@elizaos/plugin-app-manager", generatePluginAppManagerStub], + ["@elizaos/plugin-registry", generatePluginRegistryStub], + ["@elizaos/plugin-wallet", generatePluginWalletStub], + ["@elizaos/plugin-x402", generatePluginX402Stub], ["esbuild", generateEsbuildStub], // @node-rs/argon2's server-side Rust binding is referenced by // app-core's password-hashing helpers. Renderer never executes them @@ -2124,6 +2378,7 @@ function nativeModuleStubPlugin(): Plugin { "@elizaos/plugin-sql", "@elizaos/plugin-agent-skills", "@elizaos/plugin-agent-orchestrator", + "@elizaos/plugin-app-manager", // The agent runtime is server-only — it lives in the API child // process, not in the renderer. app-core/dist code can leak agent // imports (account-pool etc.); stub them so Rollup doesn't try to @@ -2133,6 +2388,11 @@ function nativeModuleStubPlugin(): Plugin { // tts proxy routes). Renderer references the exported names but // never executes the code; named-export stub registered above. "@elizaos/plugin-elizacloud", + // Server-side plugin install/discovery routes. The renderer only needs + // static named exports to resolve when app-core server barrels leak in. + "@elizaos/plugin-registry", + "@elizaos/plugin-wallet", + "@elizaos/plugin-x402", // @node-rs/argon2 has a wasm32-wasi variant that browser builds // surface via dynamic import. The browser can't resolve the bare // specifier at runtime; stub it so the bundle loads. Real hashing @@ -2357,14 +2617,27 @@ function nativeModuleStubPlugin(): Plugin { resolveStateDir: "function(){return ''}", resolveUserPath: "function(x){return x}", }; - // Check which are actually missing from the existing export block - const needed = Object.keys(missingExports).filter((n) => { - // Check if already exported (as named export or re-export alias) - const exportedAs = new RegExp(`\\b${n}\\b`); - // Search only in export{} blocks + function hasNamedExport(name: string): boolean { + const escaped = escapeRegExp(name); + if ( + new RegExp( + `export\\s+(?:async\\s+)?(?:function|class|const|let|var)\\s+${escaped}\\b`, + ).test(patched) + ) { + return true; + } + const exportBlocks = patched.match(/export\s*\{[^}]+\}/g) || []; - return !exportBlocks.some((b) => exportedAs.test(b)); - }); + return exportBlocks.some((block) => + new RegExp(`(?:\\b${escaped}\\b\\s+as\\s+)?\\b${escaped}\\b`).test( + block, + ), + ); + } + + const needed = Object.keys(missingExports).filter( + (name) => !hasNamedExport(name), + ); if (needed.length === 0 && patched === code) return null; // Use unique prefixed names to avoid collisions with minified vars const prefix = "__milady_stub_"; @@ -2543,6 +2816,7 @@ export default defineConfig({ plugins: [ appShellMetadataPlugin(), companionAssetsPlugin(), + elizaUiStateSingletonPlugin(), elizaCoreBrowserEntryFallbackPlugin(), nativeModuleStubPlugin(), asyncLocalStoragePatchPlugin(), @@ -2573,10 +2847,19 @@ export default defineConfig({ "three", "@capacitor/core", "@elizaos/app-core", + "@elizaos/ui", ], alias: [ // Bare Node built-in polyfills for browser — pathe provides ESM path, // events is pre-bundled via optimizeDeps. + { find: /^react$/, replacement: reactEntry }, + { find: /^react\/jsx-runtime$/, replacement: reactJsxRuntimeEntry }, + { + find: /^react\/jsx-dev-runtime$/, + replacement: reactJsxDevRuntimeEntry, + }, + { find: /^react-dom$/, replacement: reactDomEntry }, + { find: /^react-dom\/client$/, replacement: reactDomClientEntry }, { find: /^path$/, replacement: patheEntry }, { find: /^@capacitor\/core$/, replacement: capacitorCoreEntry }, // Aliases for Capacitor packages that may not be hoisted to root node_modules @@ -2667,6 +2950,7 @@ export default defineConfig({ // Published-only builds should resolve normal @elizaos package exports. ...resolveLocalUiAliases(), ...resolveLocalSharedAliases(), + ...resolveLocalCloudSdkAliases(), ...resolveLocalAppCoreAliases(), ], }, diff --git a/package.json b/package.json index 32c011f671..eaa79d494e 100644 --- a/package.json +++ b/package.json @@ -33,8 +33,10 @@ "scripts/repair-elizaos-package-links.mjs", "scripts/run-app-web-build.mjs", "scripts/run-eliza-app-core-script.mjs", + "scripts/run-milady-desktop-build.mjs", "scripts/run-init-then-bun-install.mjs", "scripts/run-production-build.mjs", + "scripts/run-windows-packaged-desktop-smoke.mjs", "scripts/setup-upstreams.mjs", "eliza/packages/app-core/scripts" ], @@ -68,8 +70,8 @@ "build:android": "node scripts/run-eliza-app-core-script.mjs run-mobile-build.mjs android", "build:android:system": "node scripts/run-eliza-app-core-script.mjs run-mobile-build.mjs android-system", "build:android:icons": "node apps/app/scripts/generate-android-icons.mjs", - "build:desktop": "ELIZA_APP_NAME=Milady ELIZA_APP_ID=ai.milady.milady ELIZA_URL_SCHEME=milady ELIZA_NAMESPACE=milady node scripts/run-eliza-app-core-script.mjs desktop-build.mjs build --variant=base", - "desktop:preflight": "ELIZA_APP_NAME=Milady ELIZA_APP_ID=ai.milady.milady ELIZA_URL_SCHEME=milady ELIZA_NAMESPACE=milady node scripts/run-eliza-app-core-script.mjs desktop-build.mjs preflight --variant=base", + "build:desktop": "node scripts/run-milady-desktop-build.mjs build --variant=base", + "desktop:preflight": "node scripts/run-milady-desktop-build.mjs preflight --variant=base", "build:web": "node scripts/write-homepage-release-data.mjs && cd apps/homepage && bunx vite build", "cdn:validate": "node scripts/run-eliza-app-core-script.mjs validate-cdn-assets.mjs", "cdn:manifest": "node scripts/generate-static-asset-manifest.mjs", @@ -157,7 +159,7 @@ "ui:sync:desktop": "node scripts/run-eliza-app-core-script.mjs sync-desktop-renderer.mjs", "ui:sync:desktop:watch": "node scripts/run-eliza-app-core-script.mjs sync-desktop-renderer.mjs --watch", "start": "node scripts/run-eliza-app-core-script.mjs run-node.mjs start", - "start:desktop": "node scripts/run-eliza-app-core-script.mjs desktop-build.mjs run --variant=base", + "start:desktop": "node scripts/run-milady-desktop-build.mjs run --variant=base", "start:eliza": "node scripts/run-eliza-app-core-script.mjs entry.ts start", "verify:secrets": "node scripts/run-eliza-app-core-script.mjs check-secret-hygiene.mjs", "test:ui:playwright:cloud-wallet": "node apps/app/scripts/run-ui-playwright.mjs --config playwright.ui-smoke.config.ts test/ui-smoke/cloud-wallet-import.spec.ts", @@ -168,7 +170,7 @@ "test:e2e:heavy": "bunx vitest run --config eliza/packages/app-core/vitest.e2e.config.ts --passWithNoTests eliza/plugins/plugin-steward-app/test/anvil-contracts.real.e2e.test.ts eliza/plugins/app-steward/test/anvil-contracts.real.e2e.test.ts eliza/packages/app-core/test/app/memory-relationships.real.e2e.test.ts eliza/packages/app-core/test/app/onboarding-companion.live.e2e.test.ts eliza/packages/app-core/test/app/qa-checklist.real.e2e.test.ts", "test:live:cloud": "bunx vitest run --config eliza/packages/app-core/vitest.e2e.config.ts --passWithNoTests eliza/packages/app-core/test/live-agent", "test:desktop:packaged": "bash eliza/packages/app-core/platforms/electrobun/scripts/smoke-test.sh", - "test:desktop:packaged:windows": "pwsh -File eliza/packages/app-core/platforms/electrobun/scripts/smoke-test-windows.ps1", + "test:desktop:packaged:windows": "node scripts/run-windows-packaged-desktop-smoke.mjs", "test:desktop:playwright": "node scripts/run-eliza-app-core-script.mjs run-desktop-playwright.mjs", "test:desktop:playwright:windows": "node scripts/hydrate-windows-playwright-deps.mjs && cd apps/app && bunx playwright test --config playwright.electrobun.packaged.config.ts test/electrobun-packaged/electrobun-windows-startup.e2e.spec.ts", "lifeops:gmail:export-fixture": "node scripts/export-gmail-fixture.mjs", diff --git a/scripts/apply-eliza-ci-patches.mjs b/scripts/apply-eliza-ci-patches.mjs index 0a98de2bb0..510bbc0289 100755 --- a/scripts/apply-eliza-ci-patches.mjs +++ b/scripts/apply-eliza-ci-patches.mjs @@ -124,6 +124,14 @@ function writeFileText(filePath, content, label, mode) { console.log(`[apply-eliza-ci-patches] patched ${label}`); } +function writeFileTextIfMissing(filePath, content, label, sentinel, mode) { + if (fs.existsSync(filePath)) { + const raw = fs.readFileSync(filePath, "utf8"); + if (raw === content || raw.includes(sentinel)) return; + } + writeFileText(filePath, content, label, mode); +} + function patchCloudDockerfile(raw) { let next = raw; if (!next.includes("COPY patches ./patches")) { @@ -206,6 +214,367 @@ function patchCoreStateTypes(raw) { return raw.replace('format: "JSON";', 'format: "JSON" | "TOON";'); } +function patchUiAppContextSingleton(raw) { + if (raw.includes("__ELIZAOS_UI_APP_CONTEXT__")) { + return raw; + } + + const next = raw.replace( + /import \{ createContext, useContext \} from "react";\r?\nimport type \{ AppContextValue \} from "\.\/types";\r?\n\r?\nexport const AppContext = createContext\(null\);\r?\n/, + `import { createContext, useContext } from "react"; +import type { AppContextValue } from "./types"; + +type AppContextObject = ReturnType< + typeof createContext +>; + +const appContextGlobal = globalThis as typeof globalThis & { + __ELIZAOS_UI_APP_CONTEXT__?: AppContextObject; +}; + +export const AppContext = + appContextGlobal.__ELIZAOS_UI_APP_CONTEXT__ ?? + (appContextGlobal.__ELIZAOS_UI_APP_CONTEXT__ = + createContext(null)); +`, + ); + + if (next === raw) { + throw new Error( + "Could not patch UI AppContext singleton: expected AppContext declaration was not found", + ); + } + + return next; +} + +function patchVoicePrefixWelcomeSkip(raw) { + if (raw.includes("voice-prefix-skip-prefix")) { + return raw; + } + + const modernBefore = `