From 76591b9d2356f22c82e33c48ba2f8a9f0275cf41 Mon Sep 17 00:00:00 2001 From: abhi1992002 Date: Thu, 6 Aug 2026 10:51:22 +0530 Subject: [PATCH 01/11] feat(frontend): refine onboarding audio orb --- autogpt_platform/frontend/package.json | 1 + autogpt_platform/frontend/pnpm-lock.yaml | 18 + .../onboarding/__tests__/brain-dump.test.tsx | 171 +++++- .../steps/BrainDumpStep/BrainDumpStep.tsx | 408 +++++++++++--- .../BrainDumpStep/__tests__/helpers.test.ts | 50 +- .../BrainDumpStep/components/MicButton.tsx | 102 +--- .../BrainDumpStep/components/OrbFrame.tsx | 96 ++-- .../BrainDumpStep/components/OrbSelector.tsx | 42 ++ .../BrainDumpStep/components/OrbUiOrb.tsx | 32 ++ .../BrainDumpStep/components/OrbVisual.tsx | 79 +++ .../components/RecordingStatus.tsx | 36 +- .../BrainDumpStep/components/TapHint.tsx | 11 - .../BrainDumpStep/components/WavyOrb.tsx | 504 ++++++++++++++++++ .../components/__tests__/WavyOrb.test.tsx | 219 ++++++++ .../BrainDumpStep/components/useAudioLevel.ts | 57 ++ .../onboarding/steps/BrainDumpStep/helpers.ts | 39 +- .../steps/BrainDumpStep/useBrainDumpStep.ts | 16 +- .../src/components/atoms/Reveal/Reveal.tsx | 15 +- .../components/atoms/SwapFade/SwapFade.tsx | 10 +- .../molecules/GlassOrb/GlassOrb.tsx | 53 +- .../molecules/GlassOrb/GlassSurface.tsx | 16 +- 21 files changed, 1664 insertions(+), 311 deletions(-) create mode 100644 autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbSelector.tsx create mode 100644 autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbUiOrb.tsx create mode 100644 autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx delete mode 100644 autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/TapHint.tsx create mode 100644 autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/WavyOrb.tsx create mode 100644 autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/WavyOrb.test.tsx create mode 100644 autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/useAudioLevel.ts diff --git a/autogpt_platform/frontend/package.json b/autogpt_platform/frontend/package.json index b01da44f621b..056601bf3302 100644 --- a/autogpt_platform/frontend/package.json +++ b/autogpt_platform/frontend/package.json @@ -104,6 +104,7 @@ "next": "15.5.18", "next-themes": "0.4.6", "nuqs": "2.7.2", + "orb-ui": "0.7.0", "pg": "8.21.0", "posthog-js": "1.334.1", "react": "18.3.1", diff --git a/autogpt_platform/frontend/pnpm-lock.yaml b/autogpt_platform/frontend/pnpm-lock.yaml index cf7a3f5074fa..cca4ee45bed0 100644 --- a/autogpt_platform/frontend/pnpm-lock.yaml +++ b/autogpt_platform/frontend/pnpm-lock.yaml @@ -229,6 +229,9 @@ importers: nuqs: specifier: 2.7.2 version: 2.7.2(next@15.5.18(@babel/core@7.28.5)(@opentelemetry/api@1.9.0)(@playwright/test@1.56.1)(react-dom@18.3.1(react@18.3.1))(react@18.3.1))(react@18.3.1) + orb-ui: + specifier: 0.7.0 + version: 0.7.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) pg: specifier: 8.21.0 version: 8.21.0 @@ -7359,6 +7362,16 @@ packages: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} + orb-ui@0.7.0: + resolution: {integrity: sha512-ipwMdpGO406RqHHky5O1gTz94gAUIapE+bYc50cALJrWejtbi7/HYK3TbY5YGbpQn9Re2lSiIuloi2Dm6Tucrg==} + peerDependencies: + livekit-client: '>=2.20.0 <3' + react: '>=18.0.0' + react-dom: '>=18.0.0' + peerDependenciesMeta: + livekit-client: + optional: true + orval@7.13.0: resolution: {integrity: sha512-8Q8BviorGpY2c252CxeeE8eFs7iBJX4KaTGxKmaardvRXjO0oWnEnaeAx9H5cB0FRYZPPKC0n5YHyo1GLs//CQ==} hasBin: true @@ -17260,6 +17273,11 @@ snapshots: type-check: 0.4.0 word-wrap: 1.2.5 + orb-ui@0.7.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1): + dependencies: + react: 18.3.1 + react-dom: 18.3.1(react@18.3.1) + orval@7.13.0(openapi-types@12.1.3)(typescript@5.9.3): dependencies: '@apidevtools/swagger-parser': 12.1.0(openapi-types@12.1.3) diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx index 11f6450fb0c1..d0bcb7845d45 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx @@ -195,6 +195,36 @@ class FakeMediaRecorder { } } +class FakeAudioContext { + state: AudioContextState = "running"; + + createAnalyser() { + return { + fftSize: 256, + frequencyBinCount: 128, + smoothingTimeConstant: 0, + getByteTimeDomainData(samples: Uint8Array) { + samples.fill(128); + }, + } as unknown as AnalyserNode; + } + + createMediaStreamSource() { + return { + connect() {}, + disconnect() {}, + } as unknown as MediaStreamAudioSourceNode; + } + + resume() { + return Promise.resolve(); + } + + close() { + return Promise.resolve(); + } +} + const getUserMedia = vi.fn(); function installBrowserRecordingAPIs() { @@ -203,6 +233,11 @@ function installBrowserRecordingAPIs() { writable: true, value: FakeMediaRecorder, }); + Object.defineProperty(globalThis, "AudioContext", { + configurable: true, + writable: true, + value: FakeAudioContext, + }); Object.defineProperty(navigator, "mediaDevices", { configurable: true, writable: true, @@ -308,12 +343,47 @@ describe("onboarding brain dump — flag gating", () => { render(); expect(await screen.findByText(DUMP_HEADLINE)).toBeDefined(); - expect( - screen.getByRole("button", { name: "Start recording" }), - ).toBeDefined(); + expect(screen.getByRole("button", { name: "Start talking" })).toBeDefined(); + expect(screen.queryByTestId("orb-progress-ring")).toBeNull(); expect(screen.queryByText(PILLBOX_HEADING)).toBeNull(); }); + it("offers each orb style", async () => { + mockFlags = { "onboarding-brain-dump": true }; + landOnPainPointsStep(); + + render(); + + const selector = await screen.findByRole("combobox", { + name: "Orb style", + }); + expect(selector.textContent).toContain("Current orb"); + expect(screen.getByTestId("orb-frame").style.width).toBe("184px"); + expect(screen.getByTestId("orb-decorative-ring")).toBeDefined(); + expect(screen.getByTestId("orb-voice-wave")).toBeDefined(); + + selector.focus(); + await userEvent.keyboard("{Enter}"); + + expect(screen.getByRole("option", { name: "Current orb" })).toBeDefined(); + expect(screen.getByRole("option", { name: "Wavy orb" })).toBeDefined(); + expect(screen.getByRole("option", { name: "Orb UI" })).toBeDefined(); + + await userEvent.keyboard("{ArrowDown}{Enter}"); + + expect(screen.getByTestId("orb-wavy")).toBeDefined(); + expect(screen.queryByTestId("orb-decorative-ring")).toBeNull(); + expect(screen.queryByTestId("orb-voice-wave")).toBeNull(); + expect(screen.getByRole("button", { name: "Start talking" })).toBeDefined(); + + selector.focus(); + await userEvent.keyboard("{Enter}{ArrowDown}{Enter}"); + + expect(screen.getByTestId("orb-ui")).toBeDefined(); + expect(screen.queryByTestId("orb-wavy")).toBeNull(); + expect(screen.getByRole("button", { name: "Start talking" })).toBeDefined(); + }); + it("leaves the pillboxes untouched and makes no brain-dump request when the flag is off", async () => { const calls = recordBrainDumpTraffic(); mockFlags = {}; @@ -326,9 +396,7 @@ describe("onboarding brain dump — flag gating", () => { screen.getByText("Pick the tasks you'd love to hand off to AutoPilot"), ).toBeDefined(); expect(screen.queryByText(DUMP_HEADLINE)).toBeNull(); - expect( - screen.queryByRole("button", { name: "Start recording" }), - ).toBeNull(); + expect(screen.queryByRole("button", { name: "Start talking" })).toBeNull(); expect(screen.queryByText("Skip for now")).toBeNull(); // Give any stray effect a chance to fire before declaring silence. @@ -366,7 +434,7 @@ describe("onboarding brain dump — typed fallback", () => { await screen.findByText(DUMP_HEADLINE); await userEvent.click( - screen.getByRole("button", { name: "Start recording" }), + screen.getByRole("button", { name: "Start talking" }), ); expect( @@ -376,9 +444,7 @@ describe("onboarding brain dump — typed fallback", () => { ).toBeDefined(); // Same headline, not a dead end. expect(screen.getByText(DUMP_HEADLINE)).toBeDefined(); - expect( - screen.queryByRole("button", { name: "Start recording" }), - ).toBeNull(); + expect(screen.queryByRole("button", { name: "Start talking" })).toBeNull(); // Offering a way back to the orb would be a dead end here: the browser // has already refused the microphone. expect(screen.queryByRole("button", { name: "record instead" })).toBeNull(); @@ -454,13 +520,13 @@ describe("onboarding brain dump — finishing a take", () => { await screen.findByText(DUMP_HEADLINE); await userEvent.click( - screen.getByRole("button", { name: "Start recording" }), + screen.getByRole("button", { name: "Start talking" }), ); + expect(await screen.findByTestId("orb-progress-ring")).toBeDefined(); await waitFor(() => expect(partUploads).toHaveLength(1)); - const doneButtons = await screen.findAllByRole("button", { - name: "I'm done", - }); - await userEvent.click(doneButtons[doneButtons.length - 1]); + await userEvent.click( + await screen.findByRole("button", { name: "Send recording" }), + ); expect(await screen.findByTestId("step-preparing")).toBeDefined(); expect(bodies).toHaveLength(1); @@ -501,17 +567,67 @@ describe("onboarding brain dump — finishing a take", () => { expect(screen.getByRole("button", { name: "Skip for now" })).toBeDefined(); await userEvent.click( - screen.getByRole("button", { name: "Start recording" }), + screen.getByRole("button", { name: "Start talking" }), ); await waitFor(() => expect(partUploads).toHaveLength(1)); - const doneButtons = await screen.findAllByRole("button", { - name: "I'm done", - }); - await userEvent.click(doneButtons[doneButtons.length - 1]); + await userEvent.click( + await screen.findByRole("button", { name: "Send recording" }), + ); expect(await screen.findByText("Got it. One second…")).toBeDefined(); expect(screen.queryByRole("button", { name: "Skip for now" })).toBeNull(); }); + + it("shows immediate progress while canceling a recording", async () => { + let finishDiscard: (() => void) | undefined; + recordBrainDumpTraffic(); + server.use( + getDiscardBrainDumpMockHandler200(async () => { + await new Promise((resolve) => { + finishDiscard = resolve; + }); + return { status: null }; + }), + ); + mockFlags = { "onboarding-brain-dump": true }; + landOnPainPointsStep(); + + render(); + await screen.findByText(DUMP_HEADLINE); + await userEvent.click( + screen.getByRole("button", { name: "Start talking" }), + ); + await userEvent.click( + await screen.findByRole("button", { name: "Cancel recording" }), + ); + + expect(screen.getByTestId("recording-feedback-slot")).toBeDefined(); + const cancelingButton = (await screen.findByRole("button", { + name: "Canceling recording", + })) as HTMLButtonElement; + expect(cancelingButton.getAttribute("aria-busy")).toBe("true"); + expect(await screen.findByTestId("recording-control-loader")).toBeDefined(); + expect(await screen.findByText("Discarding this take…")).toBeDefined(); + expect( + ( + screen.getByRole("button", { + name: "Send recording", + }) as HTMLButtonElement + ).disabled, + ).toBe(true); + expect( + ( + screen.getByRole("button", { + name: "Retry recording", + }) as HTMLButtonElement + ).disabled, + ).toBe(true); + + finishDiscard?.(); + expect( + await screen.findByRole("button", { name: "Start talking" }), + ).toBeDefined(); + }); }); describe("onboarding brain dump — recovery", () => { @@ -576,9 +692,7 @@ describe("onboarding brain dump — recovery", () => { await waitFor(() => expect(screen.queryByText("Pick up where you left off?")).toBeNull(), ); - expect( - screen.getByRole("button", { name: "Start recording" }), - ).toBeDefined(); + expect(screen.getByRole("button", { name: "Start talking" })).toBeDefined(); }); }); @@ -607,16 +721,15 @@ describe("onboarding brain dump — failure", () => { await screen.findByText(DUMP_HEADLINE); await userEvent.click( - screen.getByRole("button", { name: "Start recording" }), + screen.getByRole("button", { name: "Start talking" }), ); - // Wait for the first chunk to reach the server so "I'm done" is not + // Wait for the first chunk to reach the server so sending is not // racing the upload queue. await waitFor(() => expect(partUploads).toHaveLength(1)); - const doneButtons = await screen.findAllByRole("button", { - name: "I'm done", - }); - await userEvent.click(doneButtons[doneButtons.length - 1]); + await userEvent.click( + await screen.findByRole("button", { name: "Send recording" }), + ); expect(await screen.findByText("That didn't go through.")).toBeDefined(); // The failure has to come from finalize reporting `failed`, not from the diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx index 145967ce2ad5..3855295ab9bc 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx @@ -1,29 +1,42 @@ "use client"; +import { useState } from "react"; +import { AnimatePresence, motion, useReducedMotion } from "framer-motion"; import { Button } from "@/components/atoms/Button/Button"; +import { Icon } from "@/components/atoms/Icon/Icon"; import { Text } from "@/components/atoms/Text/Text"; +import { + ArrowReloadHorizontalIcon, + Cancel01Icon, + Loading03Icon, + Mic01Icon, + SentIcon, +} from "@hugeicons/core-free-icons"; +import { type IconSvgElement } from "@hugeicons/react"; import { cn } from "@/lib/utils"; -import { ElapsedTime } from "./components/ElapsedTime"; import { FailureState } from "./components/FailureState"; import { DEFAULT_GLASS_PARAMS } from "@/components/molecules/GlassOrb/GlassSurface"; +import { ElapsedTime } from "./components/ElapsedTime"; import { MicButton, OrbScreen } from "./components/MicButton"; import { PrivacyNote } from "./components/PrivacyNote"; import { RecordingStatus } from "./components/RecordingStatus"; import { RecoveryPrompt } from "./components/RecoveryPrompt"; import { RevealGroup, RevealItem } from "@/components/atoms/Reveal/Reveal"; import { SwapFade } from "@/components/atoms/SwapFade/SwapFade"; -import { TapHint } from "./components/TapHint"; import { TypedFallback } from "./components/TypedFallback"; +import { OrbSelector, OrbVariant } from "./components/OrbSelector"; +import { DEFAULT_WAVY_ORB_SETTINGS } from "./components/WavyOrb"; import { ringProgress } from "./helpers"; import { ScreenState, useBrainDumpStep } from "./useBrainDumpStep"; -const MIC_CAPTION = "Tap and talk. Most people go 2 to 3 minutes."; const FAILURE_HEADLINE = "That didn't go through."; const TIME_LIMIT_CAPTION = "That's 30 minutes — the most we record in one go. Saving all of it…"; export function BrainDumpStep() { const dump = useBrainDumpStep(); + const prefersReducedMotion = useReducedMotion(); + const [orbVariant, setOrbVariant] = useState("glass"); const isRecording = dump.screen === "recording"; const isProcessing = dump.screen === "processing"; const isMicScreen = dump.screen === "rest" || isRecording; @@ -36,19 +49,49 @@ export function BrainDumpStep() { function orbClick(screen: OrbScreen) { if (screen === "processing") return undefined; if (screen === "failed") return dump.handleRetry; - return screen === "recording" ? dump.handleDone : dump.handleStart; + return screen === "rest" ? dump.handleStart : undefined; } return ( <> - + {isRecording && ( + )} + + + -
+ {isRecording && ( +
+ +
+ )} + +
+ {orbScreen && ( + + )} {isRecording && ( -
- )} - {(dump.screen === "rest" || dump.screen === "failed") && ( - - )} - {isTyping && !dump.isMicBlocked && ( - - )} - {dump.screen === "recovery" && ( - - )} - -
- )} + )} + {isTyping && !dump.isMicBlocked && ( + + )} + {dump.screen === "recovery" && ( + + )} + + + )} - {showSubline && } + {showSubline && !isRecording && } ); } @@ -243,11 +303,191 @@ function OrbCaption({ ); } - // The failure copy sits with its buttons in FailureState, so it is not - // held back by the swap's exit animation. - if (screen === "failed") return null; + return null; +} - if (screen === "recording") return null; +function OrbControlButton({ + screen, + onClick, +}: { + screen: "rest" | "failed"; + onClick?: () => void; +}) { + const ariaLabel = screen === "failed" ? "Try again" : "Start talking"; + + return ( + + ); +} + +function RecordingControls({ + onStop, + onSend, + onRetry, + elapsedSeconds, + showSilenceNudge, + isOffline, +}: { + onStop: () => Promise; + onSend: () => Promise; + onRetry: () => Promise; + elapsedSeconds: number; + showSilenceNudge: boolean; + isOffline: boolean; +}) { + const [pendingAction, setPendingAction] = useState( + null, + ); + + async function runAction( + action: RecordingAction, + callback: () => Promise, + ) { + if (pendingAction) return; + setPendingAction(action); + try { + await callback(); + } finally { + setPendingAction(null); + } + } - return ; + const pendingStatus = + pendingAction === "cancel" + ? "Discarding this take…" + : pendingAction === "send" + ? "Sending your recording…" + : pendingAction === "retry" + ? "Starting a fresh take…" + : null; + + return ( +
+
+ void runAction("cancel", onStop)} + /> + void runAction("send", onSend)} + primary + /> + void runAction("retry", onRetry)} + /> +
+
+
+ {pendingStatus ? ( +
+ +

{pendingStatus}

+ +
+ ) : ( + + )} +
+
+
+ ); +} + +type RecordingAction = "cancel" | "send" | "retry"; + +function RecordingControlButton({ + label, + pendingLabel, + icon, + action, + pendingAction, + onClick, + primary = false, +}: { + label: string; + pendingLabel: string; + icon: IconSvgElement; + action: RecordingAction; + pendingAction: RecordingAction | null; + onClick: () => void; + primary?: boolean; +}) { + const isLoading = pendingAction === action; + const isInactive = pendingAction !== null && !isLoading; + + return ( + + ); } diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts index c71501288937..4ef78b2d0960 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts @@ -5,6 +5,7 @@ import { headline, isPermissionDenied, pickMimeType, + recordingFeedbackAt, RING_TARGET_SECONDS, ringProgress, } from "../helpers"; @@ -22,26 +23,53 @@ describe("headline", () => { describe("encouragementAt", () => { it("shows nothing before the first line is due", () => { expect(encouragementAt(0)).toBeNull(); - expect(encouragementAt(9.9)).toBeNull(); + expect(encouragementAt(19.9)).toBeNull(); }); it("shows a line for six seconds and then goes quiet", () => { - expect(encouragementAt(10)).toBe("Keep going, this is gold"); - expect(encouragementAt(15.9)).toBe("Keep going, this is gold"); - expect(encouragementAt(16)).toBeNull(); + expect(encouragementAt(20)).toBe("Keep going, this is gold"); + expect(encouragementAt(25.9)).toBe("Keep going, this is gold"); + expect(encouragementAt(26)).toBeNull(); }); - // After the last line the screen stays quiet — a nag every 20s would - // turn encouragement into pressure. - it("stops encouraging after the last line", () => { - expect(encouragementAt(45)).toBe( - "You're building AutoPilot's memory right now", - ); - expect(encouragementAt(51)).toBeNull(); + it("uses twenty-second milestones through the second minute", () => { + expect(encouragementAt(40)).not.toBeNull(); + expect(encouragementAt(60)).not.toBeNull(); + expect(encouragementAt(80)).not.toBeNull(); + expect(encouragementAt(100)).not.toBeNull(); + expect(encouragementAt(120)).not.toBeNull(); + expect(encouragementAt(126)).toBeNull(); + }); + + it("uses thirty-second milestones after the second minute", () => { + expect(encouragementAt(149.9)).toBeNull(); + expect(encouragementAt(150)).not.toBeNull(); + expect(encouragementAt(180)).not.toBeNull(); + expect(encouragementAt(240)).not.toBeNull(); + expect(encouragementAt(360)).not.toBeNull(); + }); + + it("stays quiet after the six-minute message", () => { + expect(encouragementAt(366)).toBeNull(); expect(encouragementAt(600)).toBeNull(); }); }); +describe("recordingFeedbackAt", () => { + it("shows duration guidance only after recording has settled in", () => { + expect(recordingFeedbackAt(3.9)).toBeNull(); + expect(recordingFeedbackAt(4)).toBe("Most people talk for 2 to 3 minutes."); + expect(recordingFeedbackAt(9.9)).toBe( + "Most people talk for 2 to 3 minutes.", + ); + }); + + it("shows the first encouragement at twenty seconds", () => { + expect(recordingFeedbackAt(10)).toBeNull(); + expect(recordingFeedbackAt(20)).toBe("Keep going, this is gold"); + }); +}); + describe("formatElapsed", () => { it("pads the seconds and floors the fraction", () => { expect(formatElapsed(0)).toBe("0:00"); diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/MicButton.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/MicButton.tsx index 80ea74ccdfd7..2915b995e4af 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/MicButton.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/MicButton.tsx @@ -1,34 +1,17 @@ -"use client"; - -import { Icon } from "@/components/atoms/Icon/Icon"; -import { - ArrowReloadHorizontalIcon, - HandIcon, - Mic01Icon, -} from "@hugeicons/core-free-icons"; -import { AnimatePresence, motion, useReducedMotion } from "framer-motion"; -import { AudioWaveform } from "@/app/(platform)/copilot/components/ChatInput/components/AudioWaveform"; import { GlassParams } from "@/components/molecules/GlassOrb/GlassSurface"; import { OrbFrame } from "./OrbFrame"; +import { OrbVariant } from "./OrbSelector"; +import { type WavyOrbSettings } from "./WavyOrb"; export type OrbScreen = "rest" | "recording" | "processing" | "failed"; -const GLYPH_CLASS = - "text-white/95 drop-shadow-[0_2px_12px_rgba(90,40,180,0.5)]"; - -const ARIA_LABEL: Record = { - rest: "Start recording", - recording: "I'm done", - processing: undefined, - failed: "Try again", -}; - interface Props { screen: OrbScreen; progress: number; audioStream: MediaStream | null; glassParams: GlassParams; - onClick?: () => void; + variant: OrbVariant; + wavySettings: WavyOrbSettings; } export function MicButton({ @@ -36,82 +19,17 @@ export function MicButton({ progress, audioStream, glassParams, - onClick, + variant, + wavySettings, }: Props) { - const prefersReducedMotion = useReducedMotion(); - return ( - {/* Only the glyph inside the orb changes as the step advances — the - orb, the ring and the button itself stay mounted throughout. */} - - - - - - + /> ); } - -function OrbGlyph({ - screen, - audioStream, -}: { - screen: OrbScreen; - audioStream: MediaStream | null; -}) { - if (screen === "processing") { - return ; - } - - if (screen === "failed") { - return ( - - ); - } - - if (screen === "recording") { - return ( - - ); - } - - return ; -} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbFrame.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbFrame.tsx index 4c28bc019e97..655561632377 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbFrame.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbFrame.tsx @@ -1,10 +1,12 @@ "use client"; -import { ReactNode } from "react"; -import { GlassOrb } from "@/components/molecules/GlassOrb/GlassOrb"; import { GlassParams } from "@/components/molecules/GlassOrb/GlassSurface"; +import { OrbVariant } from "./OrbSelector"; +import { OrbVisual } from "./OrbVisual"; +import { useAudioLevel } from "./useAudioLevel"; +import { type WavyOrbSettings } from "./WavyOrb"; -export const ORB_SIZE = 160; +export const ORB_SIZE = 184; const STROKE = 6; const RADIUS = (ORB_SIZE - STROKE) / 2; const CIRCUMFERENCE = 2 * Math.PI * RADIUS; @@ -14,46 +16,55 @@ const LOADER_ARC = CIRCUMFERENCE * 0.25; interface Props { glassParams: GlassParams; + variant: OrbVariant; + audioStream: MediaStream | null; + wavySettings: WavyOrbSettings; // Omitted when there is nothing to meter — the arc is not rendered at all. progress?: number; // Indeterminate: a single arc chasing the ring while work is in flight. isLoading?: boolean; - onClick?: () => void; - ariaLabel?: string; - children?: ReactNode; } // The orb as it appears everywhere in this step: the same glass ball seated // in the same neumorphic ring, whether or not it is interactive. export function OrbFrame({ glassParams, + variant, + audioStream, + wavySettings, progress, isLoading, - onClick, - ariaLabel, - children, }: Props) { - const orb = {children}; + const isRecording = progress !== undefined; + const audioLevel = useAudioLevel(isRecording ? audioStream : null); return ( -
-
+
+ {variant === "glass" && ( +
+ )} {/* A depth meter, not a limit — the ring fills toward three minutes and then simply holds, so passing it reads as an achievement rather than a warning. */} - - {progress !== undefined && ( + {variant === "glass" && progress !== undefined && ( + - )} - - - {isLoading && ( + + )} + {variant === "glass" && isLoading && ( )} - {/* Always a button, even when inert: swapping the element type would - tear down the orb and replay its animation on every state change. */} - + +
); } diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbSelector.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbSelector.tsx new file mode 100644 index 000000000000..52588f54b13e --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbSelector.tsx @@ -0,0 +1,42 @@ +"use client"; + +import { Select } from "@/components/atoms/Select/Select"; + +export type OrbVariant = "glass" | "wavy" | "orb-ui"; + +const ORB_OPTIONS = [ + { value: "glass", label: "Current orb" }, + { value: "wavy", label: "Wavy orb" }, + { value: "orb-ui", label: "Orb UI" }, +]; + +interface Props { + value: OrbVariant; + onChange: (value: OrbVariant) => void; +} + +export function OrbSelector({ value, onChange }: Props) { + function handleChange(nextValue: string) { + if ( + nextValue === "glass" || + nextValue === "wavy" || + nextValue === "orb-ui" + ) { + onChange(nextValue); + } + } + + return ( + - ); -} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbUiOrb.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbUiOrb.tsx deleted file mode 100644 index 1427c30f5d36..000000000000 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbUiOrb.tsx +++ /dev/null @@ -1,35 +0,0 @@ -"use client"; - -import { useState } from "react"; -import { type MotionValue, useMotionValueEvent } from "framer-motion"; -import { Orb, type OrbState } from "orb-ui"; - -export const ORB_UI_SIZE = 280; - -interface Props { - audioLevel: MotionValue; - state: OrbState; -} - -export function OrbUiOrb({ audioLevel, state }: Props) { - const [volume, setVolume] = useState(audioLevel.get()); - const isIdlePreview = state === "idle"; - - useMotionValueEvent(audioLevel, "change", (latest) => { - const nextVolume = Math.round(latest * 20) / 20; - setVolume((currentVolume) => - currentVolume === nextVolume ? currentVolume : nextVolume, - ); - }); - - return ( - - ); -} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx index 5eda61334a4d..fa10275cdade 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx @@ -1,80 +1,77 @@ "use client"; -import { Icon } from "@/components/atoms/Icon/Icon"; import { GlassOrb } from "@/components/molecules/GlassOrb/GlassOrb"; import { GlassParams } from "@/components/molecules/GlassOrb/GlassSurface"; -import { AudioWave01Icon } from "@hugeicons/core-free-icons"; import { motion, type MotionValue, useTransform } from "framer-motion"; -import { ORB_UI_SIZE, OrbUiOrb } from "./OrbUiOrb"; -import { OrbVariant } from "./OrbSelector"; -import { type WavyOrbSettings } from "./WavyOrb/helpers"; -import { WavyOrb } from "./WavyOrb/WavyOrb"; +import { type AudioBarLevels } from "./useAudioBars"; interface Props { - variant: OrbVariant; glassParams: GlassParams; - audioStream: MediaStream | null; - audioLevel: MotionValue; - wavySettings: WavyOrbSettings; + audioBars: AudioBarLevels; isRecording: boolean; - isLoading: boolean; } -export function OrbVisual({ - variant, - glassParams, - audioStream, - audioLevel, - wavySettings, - isRecording, - isLoading, -}: Props) { - if (variant === "glass") { - return ( -
- - - -
- ); - } +export function OrbVisual({ glassParams, audioBars, isRecording }: Props) { + return ( +
+ + + +
+ ); +} - if (variant === "orb-ui") { - return ( -
-
- -
-
- ); - } +const AUDIO_BARS = [ + { id: "low", height: 22, idleScale: 0.48 }, + { id: "low-mid", height: 34, idleScale: 0.58 }, + { id: "mid", height: 46, idleScale: 0.72 }, + { id: "high-mid", height: 34, idleScale: 0.58 }, + { id: "high", height: 22, idleScale: 0.48 }, +]; +function AudioBars({ + levels, + isRecording, +}: { + levels: AudioBarLevels; + isRecording: boolean; +}) { return ( -
-
- -
+
+ {AUDIO_BARS.map((bar, index) => ( + + ))}
); } -function ReactiveWaveIcon({ audioLevel }: { audioLevel: MotionValue }) { - const scale = useTransform(audioLevel, [0, 1], [1, 1.2]); +function AudioBar({ + level, + height, + idleScale, + isRecording, +}: { + level: MotionValue; + height: number; + idleScale: number; + isRecording: boolean; +}) { + const reactiveScaleY = useTransform(level, [0, 1], [idleScale * 0.45, 1]); return ( - - - + ); } diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/WavyOrb/WavyOrb.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/WavyOrb/WavyOrb.tsx deleted file mode 100644 index 191c0c1180e3..000000000000 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/WavyOrb/WavyOrb.tsx +++ /dev/null @@ -1,34 +0,0 @@ -"use client"; - -import { type WavyOrbSettings } from "./helpers"; -import { useWavyOrb } from "./useWavyOrb"; - -interface Props { - audioStream: MediaStream | null; - settings: WavyOrbSettings; -} - -export function WavyOrb({ audioStream, settings }: Props) { - const { canvasRef, isSupported } = useWavyOrb(audioStream, settings); - - if (!isSupported) { - return ( -
-
-
- ); - } - - return ( - - ); -} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/WavyOrb/helpers.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/WavyOrb/helpers.ts deleted file mode 100644 index 8ea11e9a861b..000000000000 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/WavyOrb/helpers.ts +++ /dev/null @@ -1,433 +0,0 @@ -export interface WavyOrbSettings { - opacity: number; - haze: number; - brightness: number; - amplitude: number; - spread: number; - speed: number; - edgeFade: number; - tint: string; - tintMix: number; -} - -export const DEFAULT_WAVY_ORB_SETTINGS: WavyOrbSettings = { - opacity: 1, - haze: 0.25, - brightness: 1, - amplitude: 1, - spread: 1, - speed: 0.48, - edgeFade: 0.82, - tint: "#8b5cf6", - tintMix: 0.83, -}; - -export const VERTEX_SHADER = `#version 300 es -void main() { - vec2 position = vec2((gl_VertexID << 1) & 2, gl_VertexID & 2); - gl_Position = vec4(position * 2.0 - 1.0, 0.0, 1.0); -}`; - -export const FRAGMENT_SHADER = `#version 300 es -precision highp float; - -uniform vec2 uResolution; -uniform float uTime; -uniform float uLow; -uniform float uMid; -uniform float uHigh; -uniform float uLevel; -uniform float uWake; -uniform float uWakeLag; -uniform float uOpacity; -uniform float uHaze; -uniform float uBrightness; -uniform float uAmplitude; -uniform float uSpread; -uniform float uSpeed; -uniform float uEdgeFade; -uniform vec3 uTint; -uniform float uTintMix; - -out vec4 outColor; - -const float PI = 3.14159265359; - -vec3 spectral4(int sampleIndex) { - vec3 c0 = vec3(0.26, 0.18, 1.0); - vec3 c1 = vec3(0.74, 0.17, 0.96); - vec3 c2 = vec3(1.0, 0.22, 0.52); - vec3 c3 = vec3(1.0, 0.66, 0.22); - return sampleIndex == 0 ? c0 : sampleIndex == 1 ? c1 : sampleIndex == 2 ? c2 : c3; -} - -float waveY(float x, float amplitude, float envelope, float drift, float harmonic) { - float fundamental = sin(x * 1.1 + drift); - float partial = sin(x * 2.53 + drift * 1.6 + 1.7); - float tilt = 1.0 + 0.14 * sin(x * 0.42 - drift * 0.6); - return amplitude * envelope * tilt * (fundamental + harmonic * partial); -} - -float thicknessAt(float xNormal, float mid) { - float taper = 1.0 - 0.55 * clamp(abs(xNormal) * 0.75, 0.0, 1.0); - return (0.012 + 0.009 * taper) * (1.0 + 0.2 * mid); -} - -vec3 ribbon( - vec2 p, - float aspect, - float amplitude, - float spread, - float drift, - float harmonic, - float mid, - float level, - float soften -) { - float xNormal = p.x / max(aspect, 1.0); - float envelope = cos(PI * 0.5 * min(abs(0.68 * xNormal), 1.0)); - envelope *= envelope; - float thickness = thicknessAt(xNormal, mid) * soften; - float hazeScale = mix(0.45, 2.4, uHaze); - float soft = (0.009 + 0.006 * mid) * soften * hazeScale; - float intensity = 0.019 * (1.0 + 0.45 * level); - float mainWave = waveY(p.x, amplitude, envelope, drift, harmonic); - vec3 numerator = vec3(0.0); - vec3 denominator = vec3(0.0); - - for (int sampleIndex = 0; sampleIndex < 4; sampleIndex++) { - vec3 hue = spectral4(sampleIndex); - denominator += hue; - float phase = mix(-spread, spread, float(sampleIndex) / 3.0); - float shiftedWave = waveY( - p.x, - amplitude + 0.03 * mid, - envelope, - drift + phase, - harmonic - ); - float distanceToLine = abs(p.y - shiftedWave); - float line = intensity / - (sqrt(distanceToLine * distanceToLine + soft * soft) + thickness); - line *= exp(-distanceToLine * distanceToLine); - float low = min(mainWave, shiftedWave); - float high = max(mainWave, shiftedWave); - float outsideBand = max(0.0, max(p.y - high, low - p.y)); - float hazeFalloff = mix(0.026, 0.11, uHaze); - float band = 4.2 * intensity * exp(-outsideBand / (hazeFalloff * soften)); - numerator += hue * (line + band); - } - - float denominatorSum = (denominator.r + denominator.g + denominator.b) / 3.0; - vec3 color = numerator / max(denominatorSum, 0.00001); - float distanceToMain = abs(p.y - mainWave); - color += 0.42 * intensity / - (sqrt(distanceToMain * distanceToMain + soft * soft) + thickness); - return color; -} - -float hash21(vec2 p) { - p = fract(p * vec2(123.34, 456.21)); - p += dot(p, p + 45.32); - return fract(p.x * p.y); -} - -void main() { - float aspect = uResolution.x / uResolution.y; - vec2 p = (gl_FragCoord.xy + 0.5) * 2.0 / uResolution - 1.0; - float screenX = abs(p.x); - float screenY = p.y; - p.x *= aspect; - p /= 0.82; - - float wake = clamp(uWake, 0.0, 1.0); - float amplitude = mix(0.06, 0.27 + 0.3 * uLow, wake) * uAmplitude; - float lag = clamp(uWakeLag, 0.0, 1.0); - float spread = mix( - 0.55, - 1.55 + 0.9 * uHigh + 0.35 * uMid + 0.7 * uLevel, - lag - ) * uSpread; - float harmonic = mix(0.10, 0.22 + 0.12 * uHigh, wake); - float xNormal = p.x / max(aspect, 1.0); - float drift = uTime * uSpeed * mix(0.7, 1.35, wake); - float ends = exp(-pow(xNormal * 1.05, 2.0)); - - vec3 color = ribbon( - p, - aspect, - amplitude, - spread, - drift, - harmonic, - uMid, - uLevel, - 1.0 - ); - vec3 softRibbon = ribbon( - p, - aspect, - amplitude * 0.94, - spread * 1.05, - drift - 0.18, - harmonic * 0.8, - uMid, - uLevel, - 3.2 + 1.3 * uLevel - ); - color += softRibbon * (mix(0.12, 0.34, uHaze) + 0.16 * uLevel); - - const float surface = 0.50; - vec2 reflectedPoint = vec2(p.x, 2.0 * surface - p.y); - vec3 reflection = ribbon( - reflectedPoint, - aspect, - amplitude * 0.86, - spread, - drift, - harmonic, - uMid, - uLevel, - 1.45 - ); - float underSurface = smoothstep(0.0, 0.16, p.y - surface); - float depth = clamp((p.y - surface) / 0.95, 0.0, 1.0); - color += reflection * 0.52 * underSurface * (1.0 - depth) * (1.0 - depth); - color *= uBrightness; - color = pow(max(color, 0.0), vec3(1.45)); - - float above = smoothstep(1.0, 0.34, -screenY); - float below = smoothstep(1.06, 0.52, screenY); - float edge = screenY < 0.0 ? above : below; - float sideFade = 1.0 - smoothstep(uEdgeFade, 1.0, screenX); - color *= edge * ends * sideFade; - - float density = clamp(max(max(color.r, color.g), color.b) * 1.9, 0.0, 1.0); - density = smoothstep(0.06, 0.92, density); - vec3 hue = color / max(max(max(color.r, color.g), color.b), 0.000001); - hue = mix(hue, uTint, uTintMix); - vec3 paper = vec3(0.965, 0.969, 0.973); - vec3 outputColor = mix(paper, paper * hue, 0.72); - float grain = hash21(gl_FragCoord.xy * 0.75 + fract(uTime) * 91.7); - outputColor += (grain - 0.5) / 255.0; - outColor = vec4(clamp(outputColor, 0.0, 1.0), density * uOpacity); -}`; - -export interface Bands { - low: number; - mid: number; - high: number; - level: number; -} - -export function hexToRgb(hex: string) { - const value = Number.parseInt(hex.slice(1), 16); - return [ - ((value >> 16) & 255) / 255, - ((value >> 8) & 255) / 255, - (value & 255) / 255, - ] as const; -} - -export function compileShader( - gl: WebGL2RenderingContext, - type: number, - source: string, -) { - const shader = gl.createShader(type); - if (!shader) return null; - gl.shaderSource(shader, source); - gl.compileShader(shader); - if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { - gl.deleteShader(shader); - return null; - } - return shader; -} - -export function follow(current: number, target: number, delta: number) { - const rate = target > current ? 3.8 : 1.35; - return current + (target - current) * Math.min(1, delta * rate); -} - -export function bandAverage( - frequencies: Uint8Array, - binHz: number, - lowHz: number, - highHz: number, -) { - const start = Math.max(0, Math.floor(lowHz / binHz)); - const end = Math.min(frequencies.length, Math.ceil(highHz / binHz)); - if (end <= start) return 0; - let sum = 0; - for (let index = start; index < end; index++) { - sum += frequencies[index]; - } - return sum / (end - start) / 255; -} - -interface RendererOptions { - canvas: HTMLCanvasElement; - audioStream: MediaStream | null; - getSettings: () => WavyOrbSettings; - prefersReducedMotion: boolean; -} - -export interface WavyOrbRenderer { - draw: (now: number) => void; - dispose: () => void; -} - -export function createWavyOrbRenderer({ - canvas, - audioStream, - getSettings, - prefersReducedMotion, -}: RendererOptions): WavyOrbRenderer | null { - const gl = canvas.getContext("webgl2", { - alpha: true, - antialias: false, - premultipliedAlpha: false, - powerPreference: "low-power", - }); - if (!gl) return null; - const context = gl; - - const vertexShader = compileShader(gl, gl.VERTEX_SHADER, VERTEX_SHADER); - const fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, FRAGMENT_SHADER); - if (!vertexShader || !fragmentShader) { - if (vertexShader) gl.deleteShader(vertexShader); - if (fragmentShader) gl.deleteShader(fragmentShader); - return null; - } - - const program = gl.createProgram(); - if (!program) { - gl.deleteShader(vertexShader); - gl.deleteShader(fragmentShader); - return null; - } - gl.attachShader(program, vertexShader); - gl.attachShader(program, fragmentShader); - gl.linkProgram(program); - gl.deleteShader(vertexShader); - gl.deleteShader(fragmentShader); - if (!gl.getProgramParameter(program, gl.LINK_STATUS)) { - gl.deleteProgram(program); - return null; - } - - const vertexArray = gl.createVertexArray(); - gl.bindVertexArray(vertexArray); - gl.useProgram(program); - const uniforms = { - resolution: gl.getUniformLocation(program, "uResolution"), - time: gl.getUniformLocation(program, "uTime"), - low: gl.getUniformLocation(program, "uLow"), - mid: gl.getUniformLocation(program, "uMid"), - high: gl.getUniformLocation(program, "uHigh"), - level: gl.getUniformLocation(program, "uLevel"), - wake: gl.getUniformLocation(program, "uWake"), - wakeLag: gl.getUniformLocation(program, "uWakeLag"), - opacity: gl.getUniformLocation(program, "uOpacity"), - haze: gl.getUniformLocation(program, "uHaze"), - brightness: gl.getUniformLocation(program, "uBrightness"), - amplitude: gl.getUniformLocation(program, "uAmplitude"), - spread: gl.getUniformLocation(program, "uSpread"), - speed: gl.getUniformLocation(program, "uSpeed"), - edgeFade: gl.getUniformLocation(program, "uEdgeFade"), - tint: gl.getUniformLocation(program, "uTint"), - tintMix: gl.getUniformLocation(program, "uTintMix"), - }; - - let audioContext: AudioContext | null = null; - let analyser: AnalyserNode | null = null; - let frequencies: Uint8Array | null = null; - if (audioStream && typeof AudioContext !== "undefined") { - audioContext = new AudioContext(); - if (audioContext.state === "suspended") { - void audioContext.resume().catch(() => undefined); - } - analyser = audioContext.createAnalyser(); - analyser.fftSize = 1024; - analyser.smoothingTimeConstant = 0.93; - audioContext.createMediaStreamSource(audioStream).connect(analyser); - frequencies = new Uint8Array(analyser.frequencyBinCount); - } - - const bands: Bands = { low: 0, mid: 0, high: 0, level: 0 }; - let lastFrame = performance.now(); - let wake = 0; - let wakeLag = 0; - - function resize() { - const dpr = Math.min(window.devicePixelRatio || 1, 2); - const width = Math.max(1, Math.round(canvas.clientWidth * dpr)); - const height = Math.max(1, Math.round(canvas.clientHeight * dpr)); - if (canvas.width !== width || canvas.height !== height) { - canvas.width = width; - canvas.height = height; - } - context.viewport(0, 0, width, height); - } - - function readBands(now: number, delta: number) { - if (analyser && frequencies && audioContext) { - analyser.getByteFrequencyData(frequencies); - const binHz = audioContext.sampleRate / analyser.fftSize; - const low = bandAverage(frequencies, binHz, 60, 320); - const mid = bandAverage(frequencies, binHz, 320, 1600); - const high = bandAverage(frequencies, binHz, 1600, 6000); - bands.low = follow(bands.low, low * 1.35, delta); - bands.mid = follow(bands.mid, mid * 1.35, delta); - bands.high = follow(bands.high, high * 1.45, delta); - bands.level = follow(bands.level, (low + mid + high) * 0.58, delta); - return; - } - - const seconds = now / 1000; - bands.low = 0.38 + Math.sin(seconds * 0.83) * 0.16; - bands.mid = 0.32 + Math.sin(seconds * 1.37 + 1.2) * 0.13; - bands.high = 0.26 + Math.sin(seconds * 2.11 + 2.4) * 0.11; - bands.level = 0.38 + Math.sin(seconds * 0.61) * 0.12; - } - - function draw(now: number) { - resize(); - const delta = Math.min((now - lastFrame) / 1000, 0.1); - lastFrame = now; - readBands(now, delta); - const wakeTarget = Math.min(1, bands.level * 1.35); - const wakeRate = wakeTarget > wake ? 1.8 : 0.65; - wake += (wakeTarget - wake) * Math.min(1, delta * wakeRate); - wakeLag += (wake - wakeLag) * Math.min(1, delta * 2.8); - context.uniform2f(uniforms.resolution, canvas.width, canvas.height); - context.uniform1f(uniforms.time, prefersReducedMotion ? 3.2 : now / 1000); - context.uniform1f(uniforms.low, bands.low); - context.uniform1f(uniforms.mid, bands.mid); - context.uniform1f(uniforms.high, bands.high); - context.uniform1f(uniforms.level, bands.level); - context.uniform1f(uniforms.wake, wake); - context.uniform1f(uniforms.wakeLag, wakeLag); - const settings = getSettings(); - const tint = hexToRgb(settings.tint); - context.uniform1f(uniforms.opacity, settings.opacity); - context.uniform1f(uniforms.haze, settings.haze); - context.uniform1f(uniforms.brightness, settings.brightness); - context.uniform1f(uniforms.amplitude, settings.amplitude); - context.uniform1f(uniforms.spread, settings.spread); - context.uniform1f(uniforms.speed, settings.speed); - context.uniform1f(uniforms.edgeFade, settings.edgeFade); - context.uniform3f(uniforms.tint, tint[0], tint[1], tint[2]); - context.uniform1f(uniforms.tintMix, settings.tintMix); - context.drawArrays(context.TRIANGLES, 0, 3); - } - - function dispose() { - void audioContext?.close().catch(() => undefined); - if (vertexArray) context.deleteVertexArray(vertexArray); - context.deleteProgram(program); - } - - return { draw, dispose }; -} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/WavyOrb/useWavyOrb.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/WavyOrb/useWavyOrb.ts deleted file mode 100644 index c764feb0eef6..000000000000 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/WavyOrb/useWavyOrb.ts +++ /dev/null @@ -1,78 +0,0 @@ -"use client"; - -import { useEffect, useRef, useState } from "react"; -import { useReducedMotion } from "framer-motion"; -import { createWavyOrbRenderer, type WavyOrbSettings } from "./helpers"; - -export function useWavyOrb( - audioStream: MediaStream | null, - settings: WavyOrbSettings, -) { - const canvasRef = useRef(null); - const settingsRef = useRef(settings); - const prefersReducedMotion = useReducedMotion(); - const [isSupported, setIsSupported] = useState(true); - - useEffect(() => { - settingsRef.current = settings; - }, [settings]); - - useEffect(() => { - const canvas = canvasRef.current; - if (!canvas) return; - const renderer = createWavyOrbRenderer({ - canvas, - audioStream, - getSettings: () => settingsRef.current, - prefersReducedMotion: Boolean(prefersReducedMotion), - }); - if (!renderer) { - setIsSupported(false); - return; - } - const activeRenderer = renderer; - - let animationFrame = 0; - let isVisible = true; - let isIntersecting = true; - - function render(now: number) { - if (!isVisible || !isIntersecting) return; - activeRenderer.draw(now); - animationFrame = requestAnimationFrame(render); - } - - function syncAnimation() { - isVisible = document.visibilityState !== "hidden"; - cancelAnimationFrame(animationFrame); - if (!isVisible || !isIntersecting) return; - if (prefersReducedMotion) { - activeRenderer.draw(performance.now()); - return; - } - animationFrame = requestAnimationFrame(render); - } - - const resizeObserver = new ResizeObserver(() => - activeRenderer.draw(performance.now()), - ); - const intersectionObserver = new IntersectionObserver(([entry]) => { - isIntersecting = entry.isIntersecting; - syncAnimation(); - }); - resizeObserver.observe(canvas); - intersectionObserver.observe(canvas); - document.addEventListener("visibilitychange", syncAnimation); - syncAnimation(); - - return () => { - cancelAnimationFrame(animationFrame); - resizeObserver.disconnect(); - intersectionObserver.disconnect(); - document.removeEventListener("visibilitychange", syncAnimation); - activeRenderer.dispose(); - }; - }, [audioStream, prefersReducedMotion]); - - return { canvasRef, isSupported }; -} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/OrbFrame.test.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/OrbFrame.test.tsx index 8d30c3d762ce..c6912bd08bf5 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/OrbFrame.test.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/OrbFrame.test.tsx @@ -2,14 +2,15 @@ import { render } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_GLASS_PARAMS } from "@/components/molecules/GlassOrb/GlassSurface"; import { OrbFrame } from "../OrbFrame"; -import { DEFAULT_WAVY_ORB_SETTINGS } from "../WavyOrb/helpers"; -const { useAudioLevelMock } = vi.hoisted(() => ({ - useAudioLevelMock: vi.fn(() => ({ get: () => 0 })), +const { useAudioBarsMock } = vi.hoisted(() => ({ + useAudioBarsMock: vi.fn(() => + Array.from({ length: 5 }, () => ({ get: () => 0 })), + ), })); -vi.mock("../useAudioLevel", () => ({ - useAudioLevel: useAudioLevelMock, +vi.mock("../useAudioBars", () => ({ + useAudioBars: useAudioBarsMock, })); vi.mock("../OrbVisual", () => ({ @@ -19,34 +20,26 @@ vi.mock("../OrbVisual", () => ({ })); beforeEach(() => { - useAudioLevelMock.mockClear(); + useAudioBarsMock.mockClear(); }); describe("OrbFrame", () => { - it("does not start the shared audio analyzer for the wavy variant", () => { + it("meters the audio stream only while recording", () => { const stream = {} as MediaStream; const { rerender } = render( - , + , ); - expect(useAudioLevelMock).toHaveBeenLastCalledWith(null); + expect(useAudioBarsMock).toHaveBeenLastCalledWith(null); rerender( , ); - expect(useAudioLevelMock).toHaveBeenLastCalledWith(stream); + expect(useAudioBarsMock).toHaveBeenLastCalledWith(stream); }); }); diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/WavyOrb.test.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/WavyOrb.test.tsx deleted file mode 100644 index 133879b26210..000000000000 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/WavyOrb.test.tsx +++ /dev/null @@ -1,258 +0,0 @@ -import { act, render, screen } from "@testing-library/react"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { DEFAULT_WAVY_ORB_SETTINGS } from "../WavyOrb/helpers"; -import { WavyOrb } from "../WavyOrb/WavyOrb"; - -const { reducedMotion } = vi.hoisted(() => ({ - reducedMotion: { value: false }, -})); - -vi.mock("framer-motion", async (importOriginal) => { - const original = await importOriginal(); - return { - ...original, - useReducedMotion: () => reducedMotion.value, - }; -}); - -type ObserverCallback = (entries: Array<{ isIntersecting: boolean }>) => void; - -const animationFrames = new Map(); -const intersectionCallbacks: ObserverCallback[] = []; -const resizeCallbacks: Array<() => void> = []; -const closeAudioContext = vi.fn(() => Promise.resolve()); -const resumeAudioContext = vi.fn(() => Promise.resolve()); -let animationFrameId = 0; - -function createWebGlContext() { - const shader = {} as WebGLShader; - const program = {} as WebGLProgram; - const vertexArray = {} as WebGLVertexArrayObject; - - return { - VERTEX_SHADER: 35633, - FRAGMENT_SHADER: 35632, - COMPILE_STATUS: 35713, - LINK_STATUS: 35714, - TRIANGLES: 4, - createShader: vi.fn(() => shader), - shaderSource: vi.fn(), - compileShader: vi.fn(), - getShaderParameter: vi.fn(() => true), - deleteShader: vi.fn(), - createProgram: vi.fn(() => program), - attachShader: vi.fn(), - linkProgram: vi.fn(), - getProgramParameter: vi.fn(() => true), - deleteProgram: vi.fn(), - createVertexArray: vi.fn(() => vertexArray), - bindVertexArray: vi.fn(), - deleteVertexArray: vi.fn(), - useProgram: vi.fn(), - getUniformLocation: vi.fn( - (_program: WebGLProgram, name: string) => - ({ name }) as unknown as WebGLUniformLocation, - ), - viewport: vi.fn(), - uniform2f: vi.fn(), - uniform1f: vi.fn(), - uniform3f: vi.fn(), - drawArrays: vi.fn(), - }; -} - -function flushAnimationFrames(count = 1) { - for (let index = 0; index < count; index++) { - const pending = Array.from(animationFrames.values()); - animationFrames.clear(); - for (const callback of pending) callback(performance.now()); - } -} - -function setReducedMotion(matches: boolean) { - reducedMotion.value = matches; - vi.stubGlobal( - "matchMedia", - vi.fn().mockReturnValue({ - matches, - addEventListener: vi.fn(), - removeEventListener: vi.fn(), - addListener: vi.fn(), - removeListener: vi.fn(), - }), - ); -} - -beforeEach(() => { - animationFrames.clear(); - intersectionCallbacks.length = 0; - resizeCallbacks.length = 0; - animationFrameId = 0; - closeAudioContext.mockClear(); - resumeAudioContext.mockClear(); - setReducedMotion(false); - - vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { - animationFrames.set(++animationFrameId, callback); - return animationFrameId; - }); - vi.stubGlobal("cancelAnimationFrame", (id: number) => { - animationFrames.delete(id); - }); - vi.stubGlobal( - "IntersectionObserver", - class { - constructor(callback: ObserverCallback) { - intersectionCallbacks.push(callback); - } - observe() {} - disconnect() {} - }, - ); - vi.stubGlobal( - "ResizeObserver", - class { - constructor(callback: () => void) { - resizeCallbacks.push(callback); - } - observe() {} - disconnect() {} - }, - ); - Object.defineProperty(HTMLElement.prototype, "clientWidth", { - configurable: true, - get: () => 620, - }); - Object.defineProperty(HTMLElement.prototype, "clientHeight", { - configurable: true, - get: () => 240, - }); - Object.defineProperty(document, "visibilityState", { - configurable: true, - value: "visible", - }); -}); - -afterEach(() => { - vi.unstubAllGlobals(); - vi.restoreAllMocks(); - Reflect.deleteProperty(HTMLElement.prototype, "clientWidth"); - Reflect.deleteProperty(HTMLElement.prototype, "clientHeight"); -}); - -describe("WavyOrb", () => { - it("draws while visible and releases WebGL resources on unmount", () => { - const gl = createWebGlContext(); - vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue( - gl as unknown as WebGL2RenderingContext, - ); - - const { unmount } = render( - , - ); - - expect(screen.getByTestId("orb-wavy")).toBeDefined(); - expect(gl.createShader).toHaveBeenCalledTimes(2); - - act(() => { - flushAnimationFrames(2); - resizeCallbacks[0](); - intersectionCallbacks[0]([{ isIntersecting: false }]); - }); - expect(gl.drawArrays).toHaveBeenCalled(); - expect(animationFrames.size).toBe(0); - - act(() => { - intersectionCallbacks[0]([{ isIntersecting: true }]); - flushAnimationFrames(); - }); - expect(gl.viewport).toHaveBeenCalledWith(0, 0, 620, 240); - - unmount(); - expect(gl.deleteVertexArray).toHaveBeenCalled(); - expect(gl.deleteProgram).toHaveBeenCalled(); - }); - - it("uses microphone frequency bands when a stream is available", () => { - const gl = createWebGlContext(); - const getByteFrequencyData = vi.fn((values: Uint8Array) => values.fill(64)); - const connect = vi.fn(); - vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue( - gl as unknown as WebGL2RenderingContext, - ); - vi.stubGlobal( - "AudioContext", - class { - state: AudioContextState = "suspended"; - sampleRate = 48000; - createAnalyser() { - return { - fftSize: 1024, - frequencyBinCount: 512, - smoothingTimeConstant: 0, - getByteFrequencyData, - }; - } - createMediaStreamSource() { - return { connect }; - } - close = closeAudioContext; - resume = resumeAudioContext; - }, - ); - - const { unmount } = render( - , - ); - - act(() => flushAnimationFrames()); - expect(connect).toHaveBeenCalled(); - expect(resumeAudioContext).toHaveBeenCalled(); - expect(getByteFrequencyData).toHaveBeenCalled(); - expect(gl.uniform1f).toHaveBeenCalled(); - - unmount(); - expect(closeAudioContext).toHaveBeenCalled(); - }); - - it("draws one static frame when reduced motion is requested", () => { - setReducedMotion(true); - const gl = createWebGlContext(); - vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue( - gl as unknown as WebGL2RenderingContext, - ); - - render(); - - expect(gl.drawArrays).toHaveBeenCalledTimes(1); - expect(animationFrames.size).toBe(0); - expect(gl.uniform1f).toHaveBeenCalledWith( - expect.objectContaining({ name: "uTime" }), - 3.2, - ); - }); - - it("renders a fallback when WebGL2 is unavailable", () => { - vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue(null); - - render(); - - expect(screen.getByTestId("orb-wavy").tagName).toBe("DIV"); - }); - - it("renders a fallback when shader compilation fails", () => { - const gl = createWebGlContext(); - gl.getShaderParameter.mockReturnValue(false); - vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue( - gl as unknown as WebGL2RenderingContext, - ); - - render(); - - expect(screen.getByTestId("orb-wavy").tagName).toBe("DIV"); - expect(gl.deleteShader).toHaveBeenCalled(); - }); -}); diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/useAudioBars.test.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/useAudioBars.test.tsx new file mode 100644 index 000000000000..037a57899b0c --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/useAudioBars.test.tsx @@ -0,0 +1,98 @@ +import { act, renderHook } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAudioBars } from "../useAudioBars"; + +let activeBand = 0; +let nextFrame: FrameRequestCallback | undefined; +const disconnect = vi.fn(); +const closeAudioContext = vi.fn(() => Promise.resolve()); + +class FakeAudioContext { + state: AudioContextState = "running"; + sampleRate = 48_000; + + createAnalyser() { + return { + fftSize: 512, + frequencyBinCount: 256, + smoothingTimeConstant: 0, + getByteFrequencyData(samples: Uint8Array) { + samples.fill(0); + const binWidth = 48_000 / 512; + const ranges = [ + [80, 250], + [250, 500], + [500, 900], + [900, 1600], + [1600, 3000], + ]; + const [minimum, maximum] = ranges[activeBand]; + const start = Math.max(1, Math.ceil(minimum / binWidth)); + const end = Math.min(samples.length, Math.ceil(maximum / binWidth)); + + for (let index = start; index < end; index += 1) { + samples[index] = 255; + } + }, + } as unknown as AnalyserNode; + } + + createMediaStreamSource() { + return { + connect() {}, + disconnect, + } as unknown as MediaStreamAudioSourceNode; + } + + resume() { + return Promise.resolve(); + } + + close() { + return closeAudioContext(); + } +} + +beforeEach(() => { + activeBand = 0; + nextFrame = undefined; + disconnect.mockClear(); + closeAudioContext.mockClear(); + vi.stubGlobal("AudioContext", FakeAudioContext); + vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { + nextFrame = callback; + return 1; + }); + vi.stubGlobal("cancelAnimationFrame", vi.fn()); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("useAudioBars", () => { + it("updates each bar from its own voice-frequency band", () => { + const { result, unmount } = renderHook(() => + useAudioBars({} as MediaStream), + ); + + act(() => nextFrame?.(performance.now() + 100)); + + expect(result.current[0].get()).toBeGreaterThan(0); + expect(result.current.slice(1).every((level) => level.get() === 0)).toBe( + true, + ); + + activeBand = 4; + act(() => nextFrame?.(performance.now() + 200)); + + expect(result.current[4].get()).toBeGreaterThan(0); + expect(result.current[1].get()).toBe(0); + expect(result.current[2].get()).toBe(0); + expect(result.current[3].get()).toBe(0); + + unmount(); + expect(disconnect).toHaveBeenCalledOnce(); + expect(closeAudioContext).toHaveBeenCalledOnce(); + }); +}); diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/useAudioBars.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/useAudioBars.ts new file mode 100644 index 000000000000..326593de8353 --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/useAudioBars.ts @@ -0,0 +1,83 @@ +"use client"; + +import { useEffect } from "react"; +import { type MotionValue, useMotionValue } from "framer-motion"; + +export type AudioBarLevels = MotionValue[]; + +const VOICE_BANDS = [ + [80, 250], + [250, 500], + [500, 900], + [900, 1600], + [1600, 3000], +] as const; + +export function useAudioBars(audioStream: MediaStream | null) { + const first = useMotionValue(0); + const second = useMotionValue(0); + const third = useMotionValue(0); + const fourth = useMotionValue(0); + const fifth = useMotionValue(0); + + useEffect(() => { + const levels = [first, second, third, fourth, fifth]; + + if (!audioStream) { + levels.forEach((level) => level.set(0)); + return; + } + + const audioContext = new AudioContext(); + if (audioContext.state === "suspended") { + void audioContext.resume(); + } + const analyser = audioContext.createAnalyser(); + analyser.fftSize = 512; + analyser.smoothingTimeConstant = 0.45; + const source = audioContext.createMediaStreamSource(audioStream); + source.connect(analyser); + const samples = new Uint8Array(analyser.frequencyBinCount); + const binWidth = audioContext.sampleRate / analyser.fftSize; + let animationFrame = 0; + const currentLevels = [0, 0, 0, 0, 0]; + let lastFrame = performance.now(); + + function update(now: number) { + const delta = Math.min((now - lastFrame) / 1000, 0.1); + lastFrame = now; + analyser.getByteFrequencyData(samples); + + VOICE_BANDS.forEach(([minimum, maximum], index) => { + const start = Math.max(1, Math.ceil(minimum / binWidth)); + const end = Math.min(samples.length, Math.ceil(maximum / binWidth)); + let energy = 0; + + for (let sampleIndex = start; sampleIndex < end; sampleIndex += 1) { + const normalized = samples[sampleIndex] / 255; + energy += normalized * normalized; + } + + const rms = Math.sqrt(energy / Math.max(1, end - start)); + const target = Math.min(1, Math.max(0, (rms - 0.025) * 2.2)); + const rate = target > currentLevels[index] ? 18 : 7; + currentLevels[index] += + (target - currentLevels[index]) * Math.min(1, delta * rate); + levels[index].set(currentLevels[index]); + }); + + animationFrame = requestAnimationFrame(update); + } + + animationFrame = requestAnimationFrame(update); + + return () => { + cancelAnimationFrame(animationFrame); + source.disconnect(); + levels.forEach((level) => level.set(0)); + void audioContext.close().catch(() => undefined); + }; + }, [audioStream, fifth, first, fourth, second, third]); + + return [first, second, third, fourth, fifth]; +} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/useAudioLevel.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/useAudioLevel.ts deleted file mode 100644 index c1ef525c3118..000000000000 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/useAudioLevel.ts +++ /dev/null @@ -1,57 +0,0 @@ -"use client"; - -import { useEffect } from "react"; -import { useMotionValue } from "framer-motion"; - -export function useAudioLevel(audioStream: MediaStream | null) { - const level = useMotionValue(0); - - useEffect(() => { - if (!audioStream) { - level.set(0); - return; - } - - const audioContext = new AudioContext(); - if (audioContext.state === "suspended") { - void audioContext.resume(); - } - const analyser = audioContext.createAnalyser(); - analyser.fftSize = 512; - analyser.smoothingTimeConstant = 0.55; - audioContext.createMediaStreamSource(audioStream).connect(analyser); - const samples = new Uint8Array(analyser.fftSize); - let animationFrame = 0; - let currentLevel = 0; - let lastFrame = performance.now(); - - function update(now: number) { - const delta = Math.min((now - lastFrame) / 1000, 0.1); - lastFrame = now; - analyser.getByteTimeDomainData(samples); - - let sum = 0; - for (const sample of samples) { - const normalized = (sample - 128) / 128; - sum += normalized * normalized; - } - - const rms = Math.sqrt(sum / samples.length); - const target = Math.min(1, Math.max(0, (rms - 0.01) * 12)); - const rate = target > currentLevel ? 16 : 4.5; - currentLevel += (target - currentLevel) * Math.min(1, delta * rate); - level.set(currentLevel); - animationFrame = requestAnimationFrame(update); - } - - animationFrame = requestAnimationFrame(update); - - return () => { - cancelAnimationFrame(animationFrame); - level.set(0); - void audioContext.close().catch(() => undefined); - }; - }, [audioStream, level]); - - return level; -} From e0660df4cf53065fcf3f93c1608dcd0238b2f9c5 Mon Sep 17 00:00:00 2001 From: abhi1992002 Date: Thu, 6 Aug 2026 16:08:12 +0530 Subject: [PATCH 11/11] fix(frontend): address audio orb review feedback --- autogpt_platform/frontend/package.json | 1 - autogpt_platform/frontend/pnpm-lock.yaml | 15 --- .../onboarding/__tests__/brain-dump.test.tsx | 37 +++++++- .../__tests__/useBrainDumpRecorder.test.ts | 52 +++++++++++ .../components/OrbControlButton.tsx | 11 ++- .../BrainDumpStep/components/OrbVisual.tsx | 18 +++- .../CancelRecordingDialog.tsx | 40 ++++++++ .../RecordingControls/RecordingControls.tsx | 86 +++++++++++------- .../components/RecordingStatus.tsx | 2 +- .../__tests__/useAudioBars.test.tsx | 91 +++++++++++++++++-- .../BrainDumpStep/components/useAudioBars.ts | 43 ++++++--- .../BrainDumpStep/useBrainDumpRecorder.ts | 50 +++++----- .../molecules/GlassOrb/GlassOrb.tsx | 53 ++--------- 13 files changed, 353 insertions(+), 146 deletions(-) create mode 100644 autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/CancelRecordingDialog.tsx diff --git a/autogpt_platform/frontend/package.json b/autogpt_platform/frontend/package.json index 51719b6eca50..e8d8288075bb 100644 --- a/autogpt_platform/frontend/package.json +++ b/autogpt_platform/frontend/package.json @@ -40,7 +40,6 @@ "@hugeicons/react": "^1.1.9", "@next/third-parties": "15.4.6", "@number-flow/react": "0.6.0", - "@phosphor-icons/react": "2.1.10", "@posthog/react": "1.7.0", "@radix-ui/react-accordion": "1.2.12", "@radix-ui/react-alert-dialog": "1.1.15", diff --git a/autogpt_platform/frontend/pnpm-lock.yaml b/autogpt_platform/frontend/pnpm-lock.yaml index c8203dca1b9b..6448839594a8 100644 --- a/autogpt_platform/frontend/pnpm-lock.yaml +++ b/autogpt_platform/frontend/pnpm-lock.yaml @@ -38,9 +38,6 @@ importers: '@number-flow/react': specifier: 0.6.0 version: 0.6.0(react-dom@18.3.1(react@18.3.1))(react@18.3.1) - '@phosphor-icons/react': - specifier: 2.1.10 - version: 2.1.10(react-dom@18.3.1(react@18.3.1))(react@18.3.1) '@posthog/react': specifier: 1.7.0 version: 1.7.0(@types/react@18.3.17)(posthog-js@1.334.1)(react@18.3.1) @@ -2337,13 +2334,6 @@ packages: '@orval/zod@7.13.0': resolution: {integrity: sha512-jEEj0uRO5D5D1CHKQdth5Atl5Ap4/P21SMiOFmVpiArlXr4LQtMpbkiPVM2tsQXIhtC38c9oMt8+rOx1rYSjcw==} - '@phosphor-icons/react@2.1.10': - resolution: {integrity: sha512-vt8Tvq8GLjheAZZYa+YG/pW7HDbov8El/MANW8pOAz4eGxrwhnbfrQZq0Cp4q8zBEu8NIhHdnr+r8thnfRSNYA==} - engines: {node: '>=10'} - peerDependencies: - react: '>= 16.8' - react-dom: '>= 16.8' - '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -11353,11 +11343,6 @@ snapshots: - supports-color - typescript - '@phosphor-icons/react@2.1.10(react-dom@18.3.1(react@18.3.1))(react@18.3.1)': - dependencies: - react: 18.3.1 - react-dom: 18.3.1(react@18.3.1) - '@pkgjs/parseargs@0.11.0': optional: true diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx index bc36112d8ca5..d80dbde05c30 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx @@ -599,6 +599,24 @@ describe("onboarding brain dump — finishing a take", () => { await screen.findByRole("button", { name: "Cancel recording" }), ); + expect(await screen.findByText("Discard recording?")).toBeDefined(); + expect( + screen.getByText(/This permanently deletes your current take/), + ).toBeDefined(); + + await userEvent.click( + screen.getByRole("button", { name: "Keep recording" }), + ); + expect(screen.queryByText("Discard recording?")).toBeNull(); + expect(finishDiscard).toBeUndefined(); + + await userEvent.click( + screen.getByRole("button", { name: "Cancel recording" }), + ); + await userEvent.click( + await screen.findByRole("button", { name: "Discard recording" }), + ); + expect(screen.getByTestId("recording-feedback-slot")).toBeDefined(); const cancelingButton = (await screen.findByRole("button", { name: "Canceling recording", @@ -612,14 +630,29 @@ describe("onboarding brain dump — finishing a take", () => { name: "Send recording", }) as HTMLButtonElement ).disabled, - ).toBe(true); + ).toBe(false); + expect( + screen + .getByRole("button", { name: "Send recording" }) + .getAttribute("aria-disabled"), + ).toBe("true"); expect( ( screen.getByRole("button", { name: "Retry recording", }) as HTMLButtonElement ).disabled, - ).toBe(true); + ).toBe(false); + expect( + screen + .getByRole("button", { name: "Retry recording" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + expect(cancelingButton.disabled).toBe(false); + expect(cancelingButton.getAttribute("aria-disabled")).toBe("true"); + await waitFor(() => expect(document.activeElement).toBe(cancelingButton)); + await userEvent.click(cancelingButton); + expect(screen.queryByText("Discard recording?")).toBeNull(); await waitFor(() => expect(finishDiscard).toBeDefined()); finishDiscard!(); diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpRecorder.test.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpRecorder.test.ts index 9858f9a08f11..3e2c5e94f23d 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpRecorder.test.ts +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpRecorder.test.ts @@ -75,6 +75,12 @@ class SlowStoppingMediaRecorder { } } +class ThrowingStopMediaRecorder extends SlowStoppingMediaRecorder { + stop() { + throw new Error("encoder failed to stop"); + } +} + // Hands a chunk over on demand, so the chunk → IndexedDB → queue // ordering can be observed a step at a time. class ChunkingMediaRecorder { @@ -417,6 +423,52 @@ describe("useBrainDumpRecorder", () => { expect(result.current.hitTimeLimit).toBe(false); }); + it("releases recording resources when stopping fails", async () => { + vi.stubGlobal("MediaRecorder", ThrowingStopMediaRecorder); + const track = { stop: vi.fn() }; + const closeAudioContext = vi.fn().mockResolvedValue(undefined); + stubGetUserMedia(vi.fn().mockResolvedValue({ getTracks: () => [track] })); + vi.stubGlobal( + "AudioContext", + class { + close = closeAudioContext; + createAnalyser() { + return { + fftSize: 0, + frequencyBinCount: 8, + getByteTimeDomainData: vi.fn(), + }; + } + createMediaStreamSource() { + return { connect: vi.fn() }; + } + }, + ); + const { result } = renderHook(() => useBrainDumpRecorder()); + + await act(async () => { + await result.current.start(); + }); + + let stopError: unknown; + await act(async () => { + try { + await result.current.stop(); + } catch (error) { + stopError = error; + } + }); + + const elapsedAfterFailure = result.current.elapsedSeconds; + act(() => vi.advanceTimersByTime(1_000)); + + expect(stopError).toEqual(new Error("encoder failed to stop")); + expect(track.stop).toHaveBeenCalledOnce(); + expect(closeAudioContext).toHaveBeenCalledOnce(); + expect(result.current.phase).toBe("stopped"); + expect(result.current.elapsedSeconds).toBe(elapsedAfterFailure); + }); + // The regression: the meta row was written with `durationSecs: 0` at // start and only refreshed by `stop()`. A crash or a refresh — the exact // case recovery exists for — never gets there, so the prompt offered diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbControlButton.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbControlButton.tsx index 6ded13cf8582..1db9296f52c3 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbControlButton.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbControlButton.tsx @@ -1,11 +1,12 @@ "use client"; import { Button } from "@/components/atoms/Button/Button"; +import { Icon } from "@/components/atoms/Icon/Icon"; import { SwapFade } from "@/components/atoms/SwapFade/SwapFade"; import { - ArrowCounterClockwiseIcon, - MicrophoneIcon, -} from "@phosphor-icons/react"; + ArrowReloadHorizontalIcon, + Mic01Icon, +} from "@hugeicons/core-free-icons"; interface Props { screen: "rest" | "failed"; @@ -25,9 +26,9 @@ export function OrbControlButton({ screen, onClick }: Props) { > {screen === "failed" ? ( - + ) : ( - + )} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx index fa10275cdade..3e212b0ae7fd 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx @@ -2,7 +2,12 @@ import { GlassOrb } from "@/components/molecules/GlassOrb/GlassOrb"; import { GlassParams } from "@/components/molecules/GlassOrb/GlassSurface"; -import { motion, type MotionValue, useTransform } from "framer-motion"; +import { + motion, + type MotionValue, + useReducedMotion, + useTransform, +} from "framer-motion"; import { type AudioBarLevels } from "./useAudioBars"; interface Props { @@ -36,6 +41,8 @@ function AudioBars({ levels: AudioBarLevels; isRecording: boolean; }) { + const prefersReducedMotion = useReducedMotion(); + return (
))}
@@ -59,11 +67,13 @@ function AudioBar({ height, idleScale, isRecording, + prefersReducedMotion, }: { level: MotionValue; height: number; idleScale: number; isRecording: boolean; + prefersReducedMotion: boolean; }) { const reactiveScaleY = useTransform(level, [0, 1], [idleScale * 0.45, 1]); @@ -71,7 +81,11 @@ function AudioBar({ ); } diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/CancelRecordingDialog.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/CancelRecordingDialog.tsx new file mode 100644 index 000000000000..698ebbda9256 --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/CancelRecordingDialog.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { Button } from "@/components/atoms/Button/Button"; +import { Text } from "@/components/atoms/Text/Text"; +import { Dialog } from "@/components/molecules/Dialog/Dialog"; + +interface Props { + isOpen: boolean; + onOpenChange: (open: boolean) => void; + onConfirm: () => void; +} + +export function CancelRecordingDialog({ + isOpen, + onOpenChange, + onConfirm, +}: Props) { + return ( + + + + This permanently deletes your current take. You can keep recording or + discard it and start again. + + + + + + + + ); +} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/RecordingControls.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/RecordingControls.tsx index 47e6027e2175..2f4bc6abe10f 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/RecordingControls.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/RecordingControls.tsx @@ -1,18 +1,20 @@ "use client"; -import { useRef, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import * as Sentry from "@sentry/nextjs"; import { Button } from "@/components/atoms/Button/Button"; +import { Icon } from "@/components/atoms/Icon/Icon"; import { SwapFade } from "@/components/atoms/SwapFade/SwapFade"; import { - ArrowCounterClockwiseIcon, - PaperPlaneTiltIcon, - SpinnerGapIcon, - XIcon, - type Icon as PhosphorIcon, -} from "@phosphor-icons/react"; + ArrowReloadHorizontalIcon, + Cancel01Icon, + Loading03Icon, + SentIcon, +} from "@hugeicons/core-free-icons"; +import type { IconSvgElement } from "@hugeicons/react"; import { cn } from "@/lib/utils"; import { RecordingStatus } from "../RecordingStatus"; +import { CancelRecordingDialog } from "./CancelRecordingDialog"; interface Props { onStop: () => Promise; @@ -36,7 +38,14 @@ export function RecordingControls({ const [pendingAction, setPendingAction] = useState( null, ); + const [isCancelDialogOpen, setIsCancelDialogOpen] = useState(false); const isActionPendingRef = useRef(false); + const cancelControlRef = useRef(null); + + useEffect(() => { + if (pendingAction !== "cancel") return; + cancelControlRef.current?.querySelector("button")?.focus(); + }, [pendingAction]); async function runAction( action: RecordingAction, @@ -69,18 +78,20 @@ export function RecordingControls({ return (
- void runAction("cancel", onStop)} - /> +
+ setIsCancelDialogOpen(true)} + /> +
void runAction("send", onSend)} @@ -89,7 +100,7 @@ export function RecordingControls({ void runAction("retry", onRetry)} @@ -100,21 +111,30 @@ export function RecordingControls({ className="relative h-8 w-80 max-w-[calc(100vw-2rem)]" >
- {pendingStatus ? ( + {pendingStatus && (

{pendingStatus}
- ) : ( - )}
+ {!pendingStatus && ( + + )}
+ { + setIsCancelDialogOpen(false); + void runAction("cancel", onStop); + }} + />
); } @@ -122,7 +142,7 @@ export function RecordingControls({ interface RecordingControlButtonProps { label: string; pendingLabel: string; - icon: PhosphorIcon; + icon: IconSvgElement; action: RecordingAction; pendingAction: RecordingAction | null; onClick: () => void; @@ -140,14 +160,18 @@ function RecordingControlButton({ }: RecordingControlButtonProps) { const isLoading = pendingAction === action; const isInactive = pendingAction !== null && !isLoading; - const ControlIcon = icon; + + function handleClick() { + if (pendingAction !== null) return; + onClick(); + } return (