diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx index 11f6450fb0c1..d80dbde05c30 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/__tests__/brain-dump.test.tsx @@ -195,6 +195,40 @@ class FakeMediaRecorder { } } +class FakeAudioContext { + state: AudioContextState = "running"; + sampleRate = 48_000; + + createAnalyser() { + return { + fftSize: 256, + frequencyBinCount: 128, + smoothingTimeConstant: 0, + getByteTimeDomainData(samples: Uint8Array) { + samples.fill(128); + }, + getByteFrequencyData(samples: Uint8Array) { + samples.fill(0); + }, + } as unknown as AnalyserNode; + } + + createMediaStreamSource() { + return { + connect() {}, + disconnect() {}, + } as unknown as MediaStreamAudioSourceNode; + } + + resume() { + return Promise.resolve(); + } + + close() { + return Promise.resolve(); + } +} + const getUserMedia = vi.fn(); function installBrowserRecordingAPIs() { @@ -203,6 +237,11 @@ function installBrowserRecordingAPIs() { writable: true, value: FakeMediaRecorder, }); + Object.defineProperty(globalThis, "AudioContext", { + configurable: true, + writable: true, + value: FakeAudioContext, + }); Object.defineProperty(navigator, "mediaDevices", { configurable: true, writable: true, @@ -308,12 +347,41 @@ describe("onboarding brain dump — flag gating", () => { render(); expect(await screen.findByText(DUMP_HEADLINE)).toBeDefined(); - expect( - screen.getByRole("button", { name: "Start recording" }), - ).toBeDefined(); + expect(screen.getByRole("button", { name: "Start talking" })).toBeDefined(); + expect(screen.queryByTestId("orb-progress-ring")).toBeNull(); expect(screen.queryByText(PILLBOX_HEADING)).toBeNull(); }); + it("renders the original orb with reactive audio bars", async () => { + mockFlags = { "onboarding-brain-dump": true }; + landOnPainPointsStep(); + + render(); + + expect(await screen.findByTestId("orb-current")).toBeDefined(); + expect(screen.getByTestId("orb-frame").style.width).toBe("184px"); + expect(screen.getByTestId("orb-decorative-ring")).toBeDefined(); + expect(screen.getByTestId("orb-audio-bars")).toBeDefined(); + const audioBars = screen.getAllByTestId("orb-audio-bar"); + expect(audioBars).toHaveLength(5); + expect(audioBars.map((bar) => bar.style.height)).toEqual([ + "22px", + "34px", + "46px", + "34px", + "22px", + ]); + expect(audioBars.map((bar) => bar.style.transform)).toEqual([ + "scaleY(0.48)", + "scaleY(0.58)", + "scaleY(0.72)", + "scaleY(0.58)", + "scaleY(0.48)", + ]); + expect(screen.queryByRole("combobox", { name: "Orb style" })).toBeNull(); + expect(screen.getByRole("button", { name: "Start talking" })).toBeDefined(); + }); + it("leaves the pillboxes untouched and makes no brain-dump request when the flag is off", async () => { const calls = recordBrainDumpTraffic(); mockFlags = {}; @@ -326,9 +394,7 @@ describe("onboarding brain dump — flag gating", () => { screen.getByText("Pick the tasks you'd love to hand off to AutoPilot"), ).toBeDefined(); expect(screen.queryByText(DUMP_HEADLINE)).toBeNull(); - expect( - screen.queryByRole("button", { name: "Start recording" }), - ).toBeNull(); + expect(screen.queryByRole("button", { name: "Start talking" })).toBeNull(); expect(screen.queryByText("Skip for now")).toBeNull(); // Give any stray effect a chance to fire before declaring silence. @@ -366,7 +432,7 @@ describe("onboarding brain dump — typed fallback", () => { await screen.findByText(DUMP_HEADLINE); await userEvent.click( - screen.getByRole("button", { name: "Start recording" }), + screen.getByRole("button", { name: "Start talking" }), ); expect( @@ -376,9 +442,7 @@ describe("onboarding brain dump — typed fallback", () => { ).toBeDefined(); // Same headline, not a dead end. expect(screen.getByText(DUMP_HEADLINE)).toBeDefined(); - expect( - screen.queryByRole("button", { name: "Start recording" }), - ).toBeNull(); + expect(screen.queryByRole("button", { name: "Start talking" })).toBeNull(); // Offering a way back to the orb would be a dead end here: the browser // has already refused the microphone. expect(screen.queryByRole("button", { name: "record instead" })).toBeNull(); @@ -454,13 +518,13 @@ describe("onboarding brain dump — finishing a take", () => { await screen.findByText(DUMP_HEADLINE); await userEvent.click( - screen.getByRole("button", { name: "Start recording" }), + screen.getByRole("button", { name: "Start talking" }), ); + expect(await screen.findByTestId("orb-progress-ring")).toBeDefined(); await waitFor(() => expect(partUploads).toHaveLength(1)); - const doneButtons = await screen.findAllByRole("button", { - name: "I'm done", - }); - await userEvent.click(doneButtons[doneButtons.length - 1]); + await userEvent.click( + await screen.findByRole("button", { name: "Send recording" }), + ); expect(await screen.findByTestId("step-preparing")).toBeDefined(); expect(bodies).toHaveLength(1); @@ -501,17 +565,101 @@ describe("onboarding brain dump — finishing a take", () => { expect(screen.getByRole("button", { name: "Skip for now" })).toBeDefined(); await userEvent.click( - screen.getByRole("button", { name: "Start recording" }), + screen.getByRole("button", { name: "Start talking" }), ); await waitFor(() => expect(partUploads).toHaveLength(1)); - const doneButtons = await screen.findAllByRole("button", { - name: "I'm done", - }); - await userEvent.click(doneButtons[doneButtons.length - 1]); + await userEvent.click( + await screen.findByRole("button", { name: "Send recording" }), + ); expect(await screen.findByText("Got it. One second…")).toBeDefined(); expect(screen.queryByRole("button", { name: "Skip for now" })).toBeNull(); }); + + it("shows immediate progress while canceling a recording", async () => { + let finishDiscard: (() => void) | undefined; + recordBrainDumpTraffic(); + server.use( + getDiscardBrainDumpMockHandler200(async () => { + await new Promise((resolve) => { + finishDiscard = resolve; + }); + return { status: null }; + }), + ); + mockFlags = { "onboarding-brain-dump": true }; + landOnPainPointsStep(); + + render(); + await screen.findByText(DUMP_HEADLINE); + await userEvent.click( + screen.getByRole("button", { name: "Start talking" }), + ); + await userEvent.click( + await screen.findByRole("button", { name: "Cancel recording" }), + ); + + expect(await screen.findByText("Discard recording?")).toBeDefined(); + expect( + screen.getByText(/This permanently deletes your current take/), + ).toBeDefined(); + + await userEvent.click( + screen.getByRole("button", { name: "Keep recording" }), + ); + expect(screen.queryByText("Discard recording?")).toBeNull(); + expect(finishDiscard).toBeUndefined(); + + await userEvent.click( + screen.getByRole("button", { name: "Cancel recording" }), + ); + await userEvent.click( + await screen.findByRole("button", { name: "Discard recording" }), + ); + + expect(screen.getByTestId("recording-feedback-slot")).toBeDefined(); + const cancelingButton = (await screen.findByRole("button", { + name: "Canceling recording", + })) as HTMLButtonElement; + expect(cancelingButton.getAttribute("aria-busy")).toBe("true"); + expect(await screen.findByTestId("recording-control-loader")).toBeDefined(); + expect(await screen.findByText("Discarding this take…")).toBeDefined(); + expect( + ( + screen.getByRole("button", { + name: "Send recording", + }) as HTMLButtonElement + ).disabled, + ).toBe(false); + expect( + screen + .getByRole("button", { name: "Send recording" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + expect( + ( + screen.getByRole("button", { + name: "Retry recording", + }) as HTMLButtonElement + ).disabled, + ).toBe(false); + expect( + screen + .getByRole("button", { name: "Retry recording" }) + .getAttribute("aria-disabled"), + ).toBe("true"); + expect(cancelingButton.disabled).toBe(false); + expect(cancelingButton.getAttribute("aria-disabled")).toBe("true"); + await waitFor(() => expect(document.activeElement).toBe(cancelingButton)); + await userEvent.click(cancelingButton); + expect(screen.queryByText("Discard recording?")).toBeNull(); + + await waitFor(() => expect(finishDiscard).toBeDefined()); + finishDiscard!(); + expect( + await screen.findByRole("button", { name: "Start talking" }), + ).toBeDefined(); + }); }); describe("onboarding brain dump — recovery", () => { @@ -576,9 +724,7 @@ describe("onboarding brain dump — recovery", () => { await waitFor(() => expect(screen.queryByText("Pick up where you left off?")).toBeNull(), ); - expect( - screen.getByRole("button", { name: "Start recording" }), - ).toBeDefined(); + expect(screen.getByRole("button", { name: "Start talking" })).toBeDefined(); }); }); @@ -607,16 +753,15 @@ describe("onboarding brain dump — failure", () => { await screen.findByText(DUMP_HEADLINE); await userEvent.click( - screen.getByRole("button", { name: "Start recording" }), + screen.getByRole("button", { name: "Start talking" }), ); - // Wait for the first chunk to reach the server so "I'm done" is not + // Wait for the first chunk to reach the server so sending is not // racing the upload queue. await waitFor(() => expect(partUploads).toHaveLength(1)); - const doneButtons = await screen.findAllByRole("button", { - name: "I'm done", - }); - await userEvent.click(doneButtons[doneButtons.length - 1]); + await userEvent.click( + await screen.findByRole("button", { name: "Send recording" }), + ); expect(await screen.findByText("That didn't go through.")).toBeDefined(); // The failure has to come from finalize reporting `failed`, not from the diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx index 145967ce2ad5..dc904ca3a362 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/BrainDumpStep.tsx @@ -1,29 +1,30 @@ "use client"; +import { AnimatePresence, motion, useReducedMotion } from "framer-motion"; import { Button } from "@/components/atoms/Button/Button"; import { Text } from "@/components/atoms/Text/Text"; import { cn } from "@/lib/utils"; -import { ElapsedTime } from "./components/ElapsedTime"; import { FailureState } from "./components/FailureState"; import { DEFAULT_GLASS_PARAMS } from "@/components/molecules/GlassOrb/GlassSurface"; +import { ElapsedTime } from "./components/ElapsedTime"; import { MicButton, OrbScreen } from "./components/MicButton"; import { PrivacyNote } from "./components/PrivacyNote"; -import { RecordingStatus } from "./components/RecordingStatus"; +import { RecordingControls } from "./components/RecordingControls/RecordingControls"; import { RecoveryPrompt } from "./components/RecoveryPrompt"; import { RevealGroup, RevealItem } from "@/components/atoms/Reveal/Reveal"; import { SwapFade } from "@/components/atoms/SwapFade/SwapFade"; -import { TapHint } from "./components/TapHint"; import { TypedFallback } from "./components/TypedFallback"; +import { OrbControlButton } from "./components/OrbControlButton"; import { ringProgress } from "./helpers"; import { ScreenState, useBrainDumpStep } from "./useBrainDumpStep"; -const MIC_CAPTION = "Tap and talk. Most people go 2 to 3 minutes."; const FAILURE_HEADLINE = "That didn't go through."; const TIME_LIMIT_CAPTION = "That's 30 minutes — the most we record in one go. Saving all of it…"; export function BrainDumpStep() { const dump = useBrainDumpStep(); + const prefersReducedMotion = useReducedMotion(); const isRecording = dump.screen === "recording"; const isProcessing = dump.screen === "processing"; const isMicScreen = dump.screen === "rest" || isRecording; @@ -36,28 +37,46 @@ export function BrainDumpStep() { function orbClick(screen: OrbScreen) { if (screen === "processing") return undefined; if (screen === "failed") return dump.handleRetry; - return screen === "recording" ? dump.handleDone : dump.handleStart; + return screen === "rest" ? dump.handleStart : undefined; } return ( <> - + {isRecording && ( + )} + + + -
- {isRecording && ( - + {isRecording && ( +
+ +
+ )} + +
{/* Skipping mid-submit would advance the wizard a second time behind the finalize that is already in flight, landing past the last step on a blank screen. */} @@ -72,16 +91,21 @@ export function BrainDumpStep() { )}
-
+
- + {dump.screen === "failed" ? FAILURE_HEADLINE : dump.headline} {showSubline && ( Just talk.{" "} @@ -95,19 +119,43 @@ export function BrainDumpStep() {
{orbScreen && ( - - + + + + + {orbScreen === "recording" ? ( + + ) : orbScreen !== "processing" ? ( + + ) : null} {/* Both slots keep their height across rest → recording → processing, so advancing a screen swaps their contents without nudging the orb or the headline. Failure has its own layout below the orb and needs neither. */} - {orbScreen !== "failed" && ( + {orbScreen !== "failed" && !isRecording && ( <>
-
- {isRecording && } -
+
)} @@ -161,64 +207,48 @@ export function BrainDumpStep() { {/* Viewport-anchored, and kept outside the reveal group: an ancestor that animates `filter` or `transform` would turn these into absolutely positioned elements. */} - {(isMicScreen || - isTyping || - dump.screen === "failed" || - dump.screen === "recovery") && ( -
- - {isRecording && ( -
- + {!isRecording && + (isMicScreen || + isTyping || + dump.screen === "failed" || + dump.screen === "recovery") && ( +
+ + {(dump.screen === "rest" || dump.screen === "failed") && ( -
- )} - {(dump.screen === "rest" || dump.screen === "failed") && ( - - )} - {isTyping && !dump.isMicBlocked && ( - - )} - {dump.screen === "recovery" && ( - - )} - -
- )} + )} + {isTyping && !dump.isMicBlocked && ( + + )} + {dump.screen === "recovery" && ( + + )} +
+
+ )} - {showSubline && } + {showSubline && !isRecording && } ); } @@ -243,11 +273,5 @@ function OrbCaption({ ); } - // The failure copy sits with its buttons in FailureState, so it is not - // held back by the swap's exit animation. - if (screen === "failed") return null; - - if (screen === "recording") return null; - - return ; + return null; } diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts index c71501288937..4ef78b2d0960 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/helpers.test.ts @@ -5,6 +5,7 @@ import { headline, isPermissionDenied, pickMimeType, + recordingFeedbackAt, RING_TARGET_SECONDS, ringProgress, } from "../helpers"; @@ -22,26 +23,53 @@ describe("headline", () => { describe("encouragementAt", () => { it("shows nothing before the first line is due", () => { expect(encouragementAt(0)).toBeNull(); - expect(encouragementAt(9.9)).toBeNull(); + expect(encouragementAt(19.9)).toBeNull(); }); it("shows a line for six seconds and then goes quiet", () => { - expect(encouragementAt(10)).toBe("Keep going, this is gold"); - expect(encouragementAt(15.9)).toBe("Keep going, this is gold"); - expect(encouragementAt(16)).toBeNull(); + expect(encouragementAt(20)).toBe("Keep going, this is gold"); + expect(encouragementAt(25.9)).toBe("Keep going, this is gold"); + expect(encouragementAt(26)).toBeNull(); }); - // After the last line the screen stays quiet — a nag every 20s would - // turn encouragement into pressure. - it("stops encouraging after the last line", () => { - expect(encouragementAt(45)).toBe( - "You're building AutoPilot's memory right now", - ); - expect(encouragementAt(51)).toBeNull(); + it("uses twenty-second milestones through the second minute", () => { + expect(encouragementAt(40)).not.toBeNull(); + expect(encouragementAt(60)).not.toBeNull(); + expect(encouragementAt(80)).not.toBeNull(); + expect(encouragementAt(100)).not.toBeNull(); + expect(encouragementAt(120)).not.toBeNull(); + expect(encouragementAt(126)).toBeNull(); + }); + + it("uses thirty-second milestones after the second minute", () => { + expect(encouragementAt(149.9)).toBeNull(); + expect(encouragementAt(150)).not.toBeNull(); + expect(encouragementAt(180)).not.toBeNull(); + expect(encouragementAt(240)).not.toBeNull(); + expect(encouragementAt(360)).not.toBeNull(); + }); + + it("stays quiet after the six-minute message", () => { + expect(encouragementAt(366)).toBeNull(); expect(encouragementAt(600)).toBeNull(); }); }); +describe("recordingFeedbackAt", () => { + it("shows duration guidance only after recording has settled in", () => { + expect(recordingFeedbackAt(3.9)).toBeNull(); + expect(recordingFeedbackAt(4)).toBe("Most people talk for 2 to 3 minutes."); + expect(recordingFeedbackAt(9.9)).toBe( + "Most people talk for 2 to 3 minutes.", + ); + }); + + it("shows the first encouragement at twenty seconds", () => { + expect(recordingFeedbackAt(10)).toBeNull(); + expect(recordingFeedbackAt(20)).toBe("Keep going, this is gold"); + }); +}); + describe("formatElapsed", () => { it("pads the seconds and floors the fraction", () => { expect(formatElapsed(0)).toBe("0:00"); diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpRecorder.test.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpRecorder.test.ts index 9858f9a08f11..3e2c5e94f23d 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpRecorder.test.ts +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpRecorder.test.ts @@ -75,6 +75,12 @@ class SlowStoppingMediaRecorder { } } +class ThrowingStopMediaRecorder extends SlowStoppingMediaRecorder { + stop() { + throw new Error("encoder failed to stop"); + } +} + // Hands a chunk over on demand, so the chunk → IndexedDB → queue // ordering can be observed a step at a time. class ChunkingMediaRecorder { @@ -417,6 +423,52 @@ describe("useBrainDumpRecorder", () => { expect(result.current.hitTimeLimit).toBe(false); }); + it("releases recording resources when stopping fails", async () => { + vi.stubGlobal("MediaRecorder", ThrowingStopMediaRecorder); + const track = { stop: vi.fn() }; + const closeAudioContext = vi.fn().mockResolvedValue(undefined); + stubGetUserMedia(vi.fn().mockResolvedValue({ getTracks: () => [track] })); + vi.stubGlobal( + "AudioContext", + class { + close = closeAudioContext; + createAnalyser() { + return { + fftSize: 0, + frequencyBinCount: 8, + getByteTimeDomainData: vi.fn(), + }; + } + createMediaStreamSource() { + return { connect: vi.fn() }; + } + }, + ); + const { result } = renderHook(() => useBrainDumpRecorder()); + + await act(async () => { + await result.current.start(); + }); + + let stopError: unknown; + await act(async () => { + try { + await result.current.stop(); + } catch (error) { + stopError = error; + } + }); + + const elapsedAfterFailure = result.current.elapsedSeconds; + act(() => vi.advanceTimersByTime(1_000)); + + expect(stopError).toEqual(new Error("encoder failed to stop")); + expect(track.stop).toHaveBeenCalledOnce(); + expect(closeAudioContext).toHaveBeenCalledOnce(); + expect(result.current.phase).toBe("stopped"); + expect(result.current.elapsedSeconds).toBe(elapsedAfterFailure); + }); + // The regression: the meta row was written with `durationSecs: 0` at // start and only refreshed by `stop()`. A crash or a refresh — the exact // case recovery exists for — never gets there, so the prompt offered diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpStep.test.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpStep.test.ts index 5cc4cd7ddf68..9f1157a8772e 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpStep.test.ts +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/__tests__/useBrainDumpStep.test.ts @@ -6,7 +6,8 @@ import type { RecordingPart } from "../recordingStore"; // `recorderState` after a render does NOT reach the handlers already // closed over — which is why the hook passes ids and durations around by // argument instead of reading them off the recorder. -const { recorderState } = vi.hoisted(() => ({ +const { captureException, recorderState } = vi.hoisted(() => ({ + captureException: vi.fn(), recorderState: { phase: "idle" as "idle" | "recording" | "stopping" | "stopped", hitTimeLimit: false, @@ -56,6 +57,8 @@ vi.mock("@/services/onboarding/brain-dump-analytics", () => ({ trackBrainDump: (...args: unknown[]) => trackBrainDump(...args), })); +vi.mock("@sentry/nextjs", () => ({ captureException })); + import { useOnboardingWizardStore } from "../../../store"; import { useBrainDumpStep } from "../useBrainDumpStep"; @@ -168,6 +171,99 @@ describe("useBrainDumpStep — recording", () => { expect(result.current.screen).toBe("recording"); }); + it("tracks cancel and discards the active take", async () => { + recorderState.recordingId = "rec-1"; + const { result } = await renderStep(); + + await act(async () => { + await result.current.handleStop(); + }); + + expect(events()).toContain("brain_dump_canceled"); + expect(clearRecording).toHaveBeenCalledWith("rec-1"); + expect(discardBrainDump).toHaveBeenCalledWith({ recording_id: "rec-1" }); + expect(result.current.screen).toBe("rest"); + }); + + it("returns to rest and discards the take when stopping fails", async () => { + recorderState.recordingId = "rec-1"; + recorderState.stop.mockRejectedValueOnce(new Error("stop failed")); + const { result } = await renderStep(); + + await act(async () => { + await result.current.handleStop(); + }); + + expect(result.current.screen).toBe("rest"); + expect(discardBrainDump).toHaveBeenCalledWith({ recording_id: "rec-1" }); + expect(captureException).toHaveBeenCalled(); + }); + + it("does not submit a take while cancellation owns it", async () => { + const stopping = deferred(); + recorderState.recordingId = "rec-1"; + recorderState.stop.mockReturnValue(stopping.promise); + const { result, rerender } = await renderStep(); + + await act(async () => { + await result.current.handleStart(); + }); + + let cancelPromise!: Promise; + act(() => { + cancelPromise = result.current.handleStop(); + }); + act(() => { + recorderState.hitTimeLimit = true; + rerender(); + }); + await act(async () => { + await Promise.resolve(); + }); + + expect(finalizeBrainDump).not.toHaveBeenCalled(); + expect(result.current.screen).toBe("rest"); + + await act(async () => { + stopping.resolve(30 * 60); + await cancelPromise; + }); + + expect(discardBrainDump).toHaveBeenCalledWith({ recording_id: "rec-1" }); + expect(result.current.screen).toBe("rest"); + }); + + it("does not discard a take while time-limit submission owns it", async () => { + const finalizing = deferred>(); + recorderState.recordingId = "rec-1"; + finalizeBrainDump.mockReturnValue(finalizing.promise); + const { result, rerender } = await renderStep(); + + await act(async () => { + await result.current.handleStart(); + }); + act(() => { + recorderState.hitTimeLimit = true; + rerender(); + }); + await waitFor(() => expect(finalizeBrainDump).toHaveBeenCalled()); + + await act(async () => { + await result.current.handleStop(); + }); + + expect(discardBrainDump).not.toHaveBeenCalled(); + expect(recorderState.stop).not.toHaveBeenCalled(); + + await act(async () => { + finalizing.resolve(completed()); + await Promise.resolve(); + }); + await waitFor(() => + expect(useOnboardingWizardStore.getState().currentStep).toBe(2), + ); + }); + // The nudge is for someone who has not started talking yet, so it keys // on whether the mic has heard anything — not on elapsed time alone. it("nudges only while recording, past the threshold, and still silent", async () => { @@ -284,6 +380,20 @@ describe("useBrainDumpStep — the 30-minute cap", () => { }); describe("useBrainDumpStep — finishing a take", () => { + it("shows the failure state when stopping the recorder fails", async () => { + recorderState.recordingId = "rec-1"; + recorderState.stop.mockRejectedValueOnce(new Error("stop failed")); + const { result } = await renderStep(); + + await act(async () => { + await result.current.handleDone(); + }); + + expect(result.current.screen).toBe("failed"); + expect(finalizeBrainDump).not.toHaveBeenCalled(); + expect(captureException).toHaveBeenCalled(); + }); + it("finalizes with the duration reported by stop(), not the last render", async () => { // The render's value is stale by however long stopping took, and the // backend splits long recordings on this number. @@ -461,6 +571,23 @@ describe("useBrainDumpStep — restart", () => { expect(result.current.screen).not.toBe("failed"); }); + it("starts a fresh take when stopping the previous recorder fails", async () => { + recorderState.recordingId = "rec-old"; + recorderState.stop.mockRejectedValueOnce(new Error("stop failed")); + const { result } = await renderStep(); + + await act(async () => { + await result.current.handleRestart(); + }); + + expect(discardBrainDump).toHaveBeenCalledWith({ + recording_id: "rec-old", + }); + expect(recorderState.start).toHaveBeenCalled(); + expect(result.current.screen).toBe("recording"); + expect(captureException).toHaveBeenCalled(); + }); + it("drops back to rest when the mic does not reopen", async () => { recorderState.recordingId = "rec-old"; recorderState.start.mockResolvedValue(false); @@ -824,6 +951,35 @@ describe("useBrainDumpStep — recovery", () => { expect(events()).toContain("brain_dump_recovery_used"); }); + it("does not leave recovery while its submission owns the take", async () => { + const finalizing = deferred>(); + recorderState.findRecoverable.mockResolvedValue(recovered); + getParts.mockResolvedValue([part(0, "rec-crashed")]); + finalizeBrainDump.mockReturnValue(finalizing.promise); + const { result } = await renderStep(); + await waitFor(() => expect(result.current.screen).toBe("recovery")); + + act(() => { + void result.current.handleResumeRecovered(); + }); + await waitFor(() => expect(finalizeBrainDump).toHaveBeenCalled()); + + await act(async () => { + await result.current.handleTypeInsteadOfRecovered(); + }); + + expect(result.current.screen).toBe("processing"); + expect(result.current.recoverable).toEqual(recovered); + + await act(async () => { + finalizing.resolve(completed()); + await Promise.resolve(); + }); + await waitFor(() => + expect(useOnboardingWizardStore.getState().currentStep).toBe(2), + ); + }); + it("releases the server buffer when the take is abandoned", async () => { recorderState.findRecoverable.mockResolvedValue(recovered); getParts.mockResolvedValue([part(0, "rec-crashed")]); diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/MicButton.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/MicButton.tsx index 80ea74ccdfd7..8af6975bb677 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/MicButton.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/MicButton.tsx @@ -1,34 +1,13 @@ -"use client"; - -import { Icon } from "@/components/atoms/Icon/Icon"; -import { - ArrowReloadHorizontalIcon, - HandIcon, - Mic01Icon, -} from "@hugeicons/core-free-icons"; -import { AnimatePresence, motion, useReducedMotion } from "framer-motion"; -import { AudioWaveform } from "@/app/(platform)/copilot/components/ChatInput/components/AudioWaveform"; import { GlassParams } from "@/components/molecules/GlassOrb/GlassSurface"; import { OrbFrame } from "./OrbFrame"; export type OrbScreen = "rest" | "recording" | "processing" | "failed"; -const GLYPH_CLASS = - "text-white/95 drop-shadow-[0_2px_12px_rgba(90,40,180,0.5)]"; - -const ARIA_LABEL: Record = { - rest: "Start recording", - recording: "I'm done", - processing: undefined, - failed: "Try again", -}; - interface Props { screen: OrbScreen; progress: number; audioStream: MediaStream | null; glassParams: GlassParams; - onClick?: () => void; } export function MicButton({ @@ -36,82 +15,13 @@ export function MicButton({ progress, audioStream, glassParams, - onClick, }: Props) { - const prefersReducedMotion = useReducedMotion(); - return ( - {/* Only the glyph inside the orb changes as the step advances — the - orb, the ring and the button itself stay mounted throughout. */} - - - - - - + /> ); } - -function OrbGlyph({ - screen, - audioStream, -}: { - screen: OrbScreen; - audioStream: MediaStream | null; -}) { - if (screen === "processing") { - return ; - } - - if (screen === "failed") { - return ( - - ); - } - - if (screen === "recording") { - return ( - - ); - } - - return ; -} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbControlButton.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbControlButton.tsx new file mode 100644 index 000000000000..1db9296f52c3 --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbControlButton.tsx @@ -0,0 +1,36 @@ +"use client"; + +import { Button } from "@/components/atoms/Button/Button"; +import { Icon } from "@/components/atoms/Icon/Icon"; +import { SwapFade } from "@/components/atoms/SwapFade/SwapFade"; +import { + ArrowReloadHorizontalIcon, + Mic01Icon, +} from "@hugeicons/core-free-icons"; + +interface Props { + screen: "rest" | "failed"; + onClick?: () => void; +} + +export function OrbControlButton({ screen, onClick }: Props) { + const ariaLabel = screen === "failed" ? "Try again" : "Start talking"; + + return ( + + ); +} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbFrame.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbFrame.tsx index 4c28bc019e97..745d5723ac37 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbFrame.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbFrame.tsx @@ -1,10 +1,10 @@ "use client"; -import { ReactNode } from "react"; -import { GlassOrb } from "@/components/molecules/GlassOrb/GlassOrb"; import { GlassParams } from "@/components/molecules/GlassOrb/GlassSurface"; +import { OrbVisual } from "./OrbVisual"; +import { useAudioBars } from "./useAudioBars"; -export const ORB_SIZE = 160; +export const ORB_SIZE = 184; const STROKE = 6; const RADIUS = (ORB_SIZE - STROKE) / 2; const CIRCUMFERENCE = 2 * Math.PI * RADIUS; @@ -14,30 +14,32 @@ const LOADER_ARC = CIRCUMFERENCE * 0.25; interface Props { glassParams: GlassParams; + audioStream: MediaStream | null; // Omitted when there is nothing to meter — the arc is not rendered at all. progress?: number; // Indeterminate: a single arc chasing the ring while work is in flight. isLoading?: boolean; - onClick?: () => void; - ariaLabel?: string; - children?: ReactNode; } // The orb as it appears everywhere in this step: the same glass ball seated // in the same neumorphic ring, whether or not it is interactive. export function OrbFrame({ glassParams, + audioStream, progress, isLoading, - onClick, - ariaLabel, - children, }: Props) { - const orb = {children}; + const isRecording = progress !== undefined; + const audioBars = useAudioBars(isRecording ? audioStream : null); return ( -
+
- {progress !== undefined && ( + {progress !== undefined && ( + - )} - - + + )} {isLoading && ( )} - {/* Always a button, even when inert: swapping the element type would - tear down the orb and replay its animation on every state change. */} - +
+ +
); } diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx new file mode 100644 index 000000000000..3e212b0ae7fd --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/OrbVisual.tsx @@ -0,0 +1,91 @@ +"use client"; + +import { GlassOrb } from "@/components/molecules/GlassOrb/GlassOrb"; +import { GlassParams } from "@/components/molecules/GlassOrb/GlassSurface"; +import { + motion, + type MotionValue, + useReducedMotion, + useTransform, +} from "framer-motion"; +import { type AudioBarLevels } from "./useAudioBars"; + +interface Props { + glassParams: GlassParams; + audioBars: AudioBarLevels; + isRecording: boolean; +} + +export function OrbVisual({ glassParams, audioBars, isRecording }: Props) { + return ( +
+ + + +
+ ); +} + +const AUDIO_BARS = [ + { id: "low", height: 22, idleScale: 0.48 }, + { id: "low-mid", height: 34, idleScale: 0.58 }, + { id: "mid", height: 46, idleScale: 0.72 }, + { id: "high-mid", height: 34, idleScale: 0.58 }, + { id: "high", height: 22, idleScale: 0.48 }, +]; + +function AudioBars({ + levels, + isRecording, +}: { + levels: AudioBarLevels; + isRecording: boolean; +}) { + const prefersReducedMotion = useReducedMotion(); + + return ( +
+ {AUDIO_BARS.map((bar, index) => ( + + ))} +
+ ); +} + +function AudioBar({ + level, + height, + idleScale, + isRecording, + prefersReducedMotion, +}: { + level: MotionValue; + height: number; + idleScale: number; + isRecording: boolean; + prefersReducedMotion: boolean; +}) { + const reactiveScaleY = useTransform(level, [0, 1], [idleScale * 0.45, 1]); + + return ( + + ); +} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/CancelRecordingDialog.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/CancelRecordingDialog.tsx new file mode 100644 index 000000000000..698ebbda9256 --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/CancelRecordingDialog.tsx @@ -0,0 +1,40 @@ +"use client"; + +import { Button } from "@/components/atoms/Button/Button"; +import { Text } from "@/components/atoms/Text/Text"; +import { Dialog } from "@/components/molecules/Dialog/Dialog"; + +interface Props { + isOpen: boolean; + onOpenChange: (open: boolean) => void; + onConfirm: () => void; +} + +export function CancelRecordingDialog({ + isOpen, + onOpenChange, + onConfirm, +}: Props) { + return ( + + + + This permanently deletes your current take. You can keep recording or + discard it and start again. + + + + + + + + ); +} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/RecordingControls.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/RecordingControls.tsx new file mode 100644 index 000000000000..2f4bc6abe10f --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingControls/RecordingControls.tsx @@ -0,0 +1,205 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; +import * as Sentry from "@sentry/nextjs"; +import { Button } from "@/components/atoms/Button/Button"; +import { Icon } from "@/components/atoms/Icon/Icon"; +import { SwapFade } from "@/components/atoms/SwapFade/SwapFade"; +import { + ArrowReloadHorizontalIcon, + Cancel01Icon, + Loading03Icon, + SentIcon, +} from "@hugeicons/core-free-icons"; +import type { IconSvgElement } from "@hugeicons/react"; +import { cn } from "@/lib/utils"; +import { RecordingStatus } from "../RecordingStatus"; +import { CancelRecordingDialog } from "./CancelRecordingDialog"; + +interface Props { + onStop: () => Promise; + onSend: () => Promise; + onRetry: () => Promise; + elapsedSeconds: number; + showSilenceNudge: boolean; + isOffline: boolean; +} + +type RecordingAction = "cancel" | "send" | "retry"; + +export function RecordingControls({ + onStop, + onSend, + onRetry, + elapsedSeconds, + showSilenceNudge, + isOffline, +}: Props) { + const [pendingAction, setPendingAction] = useState( + null, + ); + const [isCancelDialogOpen, setIsCancelDialogOpen] = useState(false); + const isActionPendingRef = useRef(false); + const cancelControlRef = useRef(null); + + useEffect(() => { + if (pendingAction !== "cancel") return; + cancelControlRef.current?.querySelector("button")?.focus(); + }, [pendingAction]); + + async function runAction( + action: RecordingAction, + callback: () => Promise, + ) { + if (isActionPendingRef.current) return; + isActionPendingRef.current = true; + setPendingAction(action); + try { + await callback(); + } catch (error) { + Sentry.captureException(error, { + tags: { component: "RecordingControls", action }, + }); + } finally { + isActionPendingRef.current = false; + setPendingAction(null); + } + } + + const pendingStatus = + pendingAction === "cancel" + ? "Discarding this take…" + : pendingAction === "send" + ? "Sending your recording…" + : pendingAction === "retry" + ? "Starting a fresh take…" + : null; + + return ( +
+
+
+ setIsCancelDialogOpen(true)} + /> +
+ void runAction("send", onSend)} + primary + /> + void runAction("retry", onRetry)} + /> +
+
+
+ {pendingStatus && ( +
+ +

{pendingStatus}

+
+
+ )} +
+ {!pendingStatus && ( + + )} +
+ { + setIsCancelDialogOpen(false); + void runAction("cancel", onStop); + }} + /> +
+ ); +} + +interface RecordingControlButtonProps { + label: string; + pendingLabel: string; + icon: IconSvgElement; + action: RecordingAction; + pendingAction: RecordingAction | null; + onClick: () => void; + primary?: boolean; +} + +function RecordingControlButton({ + label, + pendingLabel, + icon, + action, + pendingAction, + onClick, + primary = false, +}: RecordingControlButtonProps) { + const isLoading = pendingAction === action; + const isInactive = pendingAction !== null && !isLoading; + + function handleClick() { + if (pendingAction !== null) return; + onClick(); + } + + return ( + + ); +} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingStatus.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingStatus.tsx index e27154ebf327..e7b7aa4afc9b 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingStatus.tsx +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/RecordingStatus.tsx @@ -1,40 +1,40 @@ "use client"; import { Icon } from "@/components/atoms/Icon/Icon"; +import { SwapFade } from "@/components/atoms/SwapFade/SwapFade"; import { Text } from "@/components/atoms/Text/Text"; -import { cn } from "@/lib/utils"; import { CloudOffIcon } from "@hugeicons/core-free-icons"; -import { encouragementAt, SILENCE_NUDGE_COPY } from "../helpers"; +import { recordingFeedbackAt, SILENCE_NUDGE_COPY } from "../helpers"; interface Props { elapsedSeconds: number; showSilenceNudge: boolean; isOffline: boolean; - isSavedLocally: boolean; } const OFFLINE_COPY = "You're offline — we'll send this when you're back."; -const SAVED_LOCALLY_COPY = "Saved on this device as you talk"; export function RecordingStatus({ elapsedSeconds, showSilenceNudge, isOffline, - isSavedLocally, }: Props) { - const encouragement = encouragementAt(elapsedSeconds); + const feedback = showSilenceNudge + ? null + : recordingFeedbackAt(elapsedSeconds); return (
- - {encouragement ?? ""} - +
+ + + {feedback ?? ""} + + +
{showSilenceNudge && ( @@ -56,12 +56,6 @@ export function RecordingStatus({
)} - - {isSavedLocally && ( - - {SAVED_LOCALLY_COPY} - - )}
); } diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/TapHint.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/TapHint.tsx deleted file mode 100644 index bfceca95aba4..000000000000 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/TapHint.tsx +++ /dev/null @@ -1,11 +0,0 @@ -"use client"; - -import { Text } from "@/components/atoms/Text/Text"; - -export function TapHint({ caption }: { caption: string }) { - return ( - - {caption} - - ); -} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/OrbFrame.test.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/OrbFrame.test.tsx new file mode 100644 index 000000000000..c6912bd08bf5 --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/OrbFrame.test.tsx @@ -0,0 +1,45 @@ +import { render } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DEFAULT_GLASS_PARAMS } from "@/components/molecules/GlassOrb/GlassSurface"; +import { OrbFrame } from "../OrbFrame"; + +const { useAudioBarsMock } = vi.hoisted(() => ({ + useAudioBarsMock: vi.fn(() => + Array.from({ length: 5 }, () => ({ get: () => 0 })), + ), +})); + +vi.mock("../useAudioBars", () => ({ + useAudioBars: useAudioBarsMock, +})); + +vi.mock("../OrbVisual", () => ({ + OrbVisual: function OrbVisual() { + return
; + }, +})); + +beforeEach(() => { + useAudioBarsMock.mockClear(); +}); + +describe("OrbFrame", () => { + it("meters the audio stream only while recording", () => { + const stream = {} as MediaStream; + const { rerender } = render( + , + ); + + expect(useAudioBarsMock).toHaveBeenLastCalledWith(null); + + rerender( + , + ); + + expect(useAudioBarsMock).toHaveBeenLastCalledWith(stream); + }); +}); diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/useAudioBars.test.tsx b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/useAudioBars.test.tsx new file mode 100644 index 000000000000..3ea5d1e55801 --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/__tests__/useAudioBars.test.tsx @@ -0,0 +1,175 @@ +import { act, renderHook } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { useAudioBars } from "../useAudioBars"; + +const { reducedMotion } = vi.hoisted(() => ({ + reducedMotion: { value: false }, +})); + +vi.mock("framer-motion", async () => { + const actual = + await vi.importActual("framer-motion"); + return { + ...actual, + useReducedMotion: () => reducedMotion.value, + }; +}); + +let activeBand = 0; +let nextFrameId = 1; +const animationFrames = new Map(); +const disconnect = vi.fn(); +const closeAudioContext = vi.fn(() => Promise.resolve()); +const cancelAnimationFrame = vi.fn((frameId: number) => { + animationFrames.delete(frameId); +}); + +class FakeAudioContext { + state: AudioContextState = "running"; + sampleRate = 48_000; + + createAnalyser() { + return { + fftSize: 512, + frequencyBinCount: 256, + smoothingTimeConstant: 0, + getByteFrequencyData(samples: Uint8Array) { + samples.fill(0); + const binWidth = 48_000 / 512; + const ranges = [ + [80, 250], + [250, 500], + [500, 900], + [900, 1600], + [1600, 3000], + ]; + const [minimum, maximum] = ranges[activeBand]; + const start = Math.max(1, Math.ceil(minimum / binWidth)); + const end = Math.min(samples.length, Math.ceil(maximum / binWidth)); + + for (let index = start; index < end; index += 1) { + samples[index] = 255; + } + }, + } as unknown as AnalyserNode; + } + + createMediaStreamSource() { + return { + connect() {}, + disconnect, + } as unknown as MediaStreamAudioSourceNode; + } + + resume() { + return Promise.resolve(); + } + + close() { + return closeAudioContext(); + } +} + +beforeEach(() => { + activeBand = 0; + nextFrameId = 1; + animationFrames.clear(); + reducedMotion.value = false; + disconnect.mockClear(); + closeAudioContext.mockClear(); + cancelAnimationFrame.mockClear(); + vi.stubGlobal("AudioContext", FakeAudioContext); + vi.stubGlobal("requestAnimationFrame", (callback: FrameRequestCallback) => { + const frameId = nextFrameId; + nextFrameId += 1; + animationFrames.set(frameId, callback); + return frameId; + }); + vi.stubGlobal("cancelAnimationFrame", cancelAnimationFrame); +}); + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe("useAudioBars", () => { + it("updates each bar from its own voice-frequency band", () => { + const { result, unmount } = renderHook(() => + useAudioBars({} as MediaStream), + ); + + flushAnimationFrame(performance.now() + 100); + + expect(result.current[0].get()).toBeGreaterThan(0); + expect(result.current.slice(1).every((level) => level.get() === 0)).toBe( + true, + ); + + activeBand = 4; + flushAnimationFrame(performance.now() + 200); + + expect(result.current[4].get()).toBeGreaterThan(0); + expect(result.current[1].get()).toBe(0); + expect(result.current[2].get()).toBe(0); + expect(result.current[3].get()).toBe(0); + + unmount(); + expect(cancelAnimationFrame).toHaveBeenCalledWith(3); + expect(animationFrames.size).toBe(0); + expect(disconnect).toHaveBeenCalledOnce(); + expect(closeAudioContext).toHaveBeenCalledOnce(); + }); + + it("does not start an analyser when reduced motion is preferred", () => { + reducedMotion.value = true; + + const { result } = renderHook(() => useAudioBars({} as MediaStream)); + + expect(animationFrames.size).toBe(0); + expect(result.current.every((level) => level.get() === 0)).toBe(true); + }); + + it("falls back to static bars when audio analysis cannot start", () => { + vi.stubGlobal( + "AudioContext", + class { + constructor() { + throw new Error("audio unavailable"); + } + }, + ); + + const { result } = renderHook(() => useAudioBars({} as MediaStream)); + + expect(animationFrames.size).toBe(0); + expect(result.current.every((level) => level.get() === 0)).toBe(true); + }); + + it("closes a context when analyser initialization fails", () => { + const closePartiallyStartedContext = vi.fn().mockResolvedValue(undefined); + vi.stubGlobal( + "AudioContext", + class { + state: AudioContextState = "running"; + close = closePartiallyStartedContext; + createAnalyser() { + throw new Error("analyser unavailable"); + } + }, + ); + + const { result } = renderHook(() => useAudioBars({} as MediaStream)); + + expect(closePartiallyStartedContext).toHaveBeenCalledOnce(); + expect(animationFrames.size).toBe(0); + expect(result.current.every((level) => level.get() === 0)).toBe(true); + }); +}); + +function flushAnimationFrame(now: number) { + const frame = animationFrames.entries().next().value; + if (!frame) return; + const [frameId, callback] = frame; + animationFrames.delete(frameId); + act(() => callback(now)); +} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/useAudioBars.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/useAudioBars.ts new file mode 100644 index 000000000000..524f713bb4e5 --- /dev/null +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/components/useAudioBars.ts @@ -0,0 +1,100 @@ +"use client"; + +import { useEffect } from "react"; +import { + type MotionValue, + useMotionValue, + useReducedMotion, +} from "framer-motion"; + +export type AudioBarLevels = MotionValue[]; + +const VOICE_BANDS = [ + [80, 250], + [250, 500], + [500, 900], + [900, 1600], + [1600, 3000], +] as const; + +export function useAudioBars(audioStream: MediaStream | null) { + const prefersReducedMotion = useReducedMotion(); + const first = useMotionValue(0); + const second = useMotionValue(0); + const third = useMotionValue(0); + const fourth = useMotionValue(0); + const fifth = useMotionValue(0); + + useEffect(() => { + const levels = [first, second, third, fourth, fifth]; + + if (!audioStream || prefersReducedMotion) { + levels.forEach((level) => level.set(0)); + return; + } + + let audioContext: AudioContext | null = null; + let analyser: AnalyserNode | null = null; + let source: MediaStreamAudioSourceNode | null = null; + + try { + audioContext = new AudioContext(); + if (audioContext.state === "suspended") { + void audioContext.resume().catch(() => undefined); + } + analyser = audioContext.createAnalyser(); + analyser.fftSize = 512; + analyser.smoothingTimeConstant = 0.45; + source = audioContext.createMediaStreamSource(audioStream); + source.connect(analyser); + } catch { + levels.forEach((level) => level.set(0)); + void audioContext?.close().catch(() => undefined); + return; + } + + const activeAnalyser = analyser; + const samples = new Uint8Array(activeAnalyser.frequencyBinCount); + const binWidth = audioContext.sampleRate / analyser.fftSize; + let animationFrame = 0; + const currentLevels = [0, 0, 0, 0, 0]; + let lastFrame = performance.now(); + + function update(now: number) { + const delta = Math.min((now - lastFrame) / 1000, 0.1); + lastFrame = now; + activeAnalyser.getByteFrequencyData(samples); + + VOICE_BANDS.forEach(([minimum, maximum], index) => { + const start = Math.max(1, Math.ceil(minimum / binWidth)); + const end = Math.min(samples.length, Math.ceil(maximum / binWidth)); + let energy = 0; + + for (let sampleIndex = start; sampleIndex < end; sampleIndex += 1) { + const normalized = samples[sampleIndex] / 255; + energy += normalized * normalized; + } + + const rms = Math.sqrt(energy / Math.max(1, end - start)); + const target = Math.min(1, Math.max(0, (rms - 0.025) * 2.2)); + const rate = target > currentLevels[index] ? 18 : 7; + currentLevels[index] += + (target - currentLevels[index]) * Math.min(1, delta * rate); + levels[index].set(currentLevels[index]); + }); + + animationFrame = requestAnimationFrame(update); + } + + animationFrame = requestAnimationFrame(update); + + return () => { + cancelAnimationFrame(animationFrame); + source.disconnect(); + levels.forEach((level) => level.set(0)); + void audioContext.close().catch(() => undefined); + }; + }, [audioStream, fifth, first, fourth, prefersReducedMotion, second, third]); + + return [first, second, third, fourth, fifth]; +} diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/helpers.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/helpers.ts index ca3e9e5d68a1..bece5a79e124 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/helpers.ts +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/helpers.ts @@ -26,17 +26,28 @@ export const SPEECH_PEAK_THRESHOLD = 12; export const SILENCE_NUDGE_COPY = "Start anywhere. What did you do yesterday that bored you?"; -// Never any copy that caps effort — "30 seconds is plenty" is the exact -// message this screen exists to avoid sending. -const ENCOURAGEMENTS = [ - { atSeconds: 10, text: "Keep going, this is gold" }, - { atSeconds: 25, text: "The more you share, the sharper AutoPilot gets" }, - { atSeconds: 45, text: "You're building AutoPilot's memory right now" }, +const ENCOURAGEMENT_COPY = [ + "Keep going, this is gold", + "The more you share, the sharper AutoPilot gets", + "You're building AutoPilot's memory right now", + "You're doing great — keep going", + "Every detail makes AutoPilot more useful", + "Share whatever comes to mind next", ] as const; -// After the last line the screen goes quiet — a nag every 20s would turn -// encouragement into pressure. +const ENCOURAGEMENT_MILESTONES = [ + 20, 40, 60, 80, 100, 120, 150, 180, 210, 240, 270, 300, 330, 360, +] as const; + +const ENCOURAGEMENTS = ENCOURAGEMENT_MILESTONES.map((atSeconds, index) => ({ + atSeconds, + text: ENCOURAGEMENT_COPY[index % ENCOURAGEMENT_COPY.length], +})); + const ENCOURAGEMENT_VISIBLE_SECONDS = 6; +const DURATION_GUIDANCE_START_SECONDS = 4; +const DURATION_GUIDANCE_END_SECONDS = 10; +export const DURATION_GUIDANCE_COPY = "Most people talk for 2 to 3 minutes."; export function encouragementAt(elapsedSeconds: number): string | null { const active = ENCOURAGEMENTS.find( @@ -47,6 +58,18 @@ export function encouragementAt(elapsedSeconds: number): string | null { return active?.text ?? null; } +export function recordingFeedbackAt(elapsedSeconds: number) { + const encouragement = encouragementAt(elapsedSeconds); + if (encouragement) return encouragement; + if ( + elapsedSeconds >= DURATION_GUIDANCE_START_SECONDS && + elapsedSeconds < DURATION_GUIDANCE_END_SECONDS + ) { + return DURATION_GUIDANCE_COPY; + } + return null; +} + export function formatElapsed(totalSeconds: number) { const minutes = Math.floor(totalSeconds / 60); const seconds = Math.floor(totalSeconds % 60); diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpRecorder.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpRecorder.ts index eb679e3c19fe..f3a4395dd6cf 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpRecorder.ts +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpRecorder.ts @@ -222,31 +222,37 @@ export function useBrainDumpRecorder() { async function stop(): Promise { const recorder = recorderRef.current; if (!recorder || recorder.state === "inactive") { + if (recorder) { + stopTracks(); + recorderRef.current = null; + setPhase("stopped"); + } return elapsedSecondsRef.current; } setPhase("stopping"); - await new Promise((resolve) => { - recorder.onstop = () => resolve(); - recorder.stop(); - }); - await Promise.all(pendingWritesRef.current); - pendingWritesRef.current = []; - stopTracks(); - recorderRef.current = null; - // Measured here rather than read off state: the awaits above mean the - // last tick is already behind, and the tail of the take counts. - const durationSecs = (Date.now() - startedAtRef.current) / 1000; - elapsedSecondsRef.current = durationSecs; - setElapsedSeconds(durationSecs); - await rememberMeta({ - recordingId: recordingIdRef.current ?? "", - mimeType: mimeTypeRef.current, - startedAt: startedAtRef.current, - durationSecs, - finalized: false, - }); - setPhase("stopped"); - return durationSecs; + try { + await new Promise((resolve) => { + recorder.onstop = () => resolve(); + recorder.stop(); + }); + await Promise.all(pendingWritesRef.current); + const durationSecs = (Date.now() - startedAtRef.current) / 1000; + elapsedSecondsRef.current = durationSecs; + setElapsedSeconds(durationSecs); + await rememberMeta({ + recordingId: recordingIdRef.current ?? "", + mimeType: mimeTypeRef.current, + startedAt: startedAtRef.current, + durationSecs, + finalized: false, + }); + return durationSecs; + } finally { + pendingWritesRef.current = []; + stopTracks(); + recorderRef.current = null; + setPhase("stopped"); + } } // On mount, an unfinalized recording in IndexedDB means the last diff --git a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpStep.ts b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpStep.ts index d010be5301f4..e941a6fa54e7 100644 --- a/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpStep.ts +++ b/autogpt_platform/frontend/src/app/(no-navbar)/onboarding/steps/BrainDumpStep/useBrainDumpStep.ts @@ -4,6 +4,7 @@ import { } from "@/app/api/__generated__/endpoints/brain-dump/brain-dump"; import { setIntroPath } from "@/services/onboarding/brain-dump-handoff"; import { useEffect, useRef, useState } from "react"; +import * as Sentry from "@sentry/nextjs"; import { useOnboardingWizardStore } from "../../store"; import { trackBrainDump } from "@/services/onboarding/brain-dump-analytics"; import { headline, SILENCE_NUDGE_SECONDS } from "./helpers"; @@ -49,6 +50,10 @@ export function useBrainDumpStep() { // `completeAndAdvance` to `nextStep()`, or the wizard advances twice and // lands past the last step with nothing to render. const isSubmittingRef = useRef(false); + const activeTakeActionRef = useRef<{ + action: "submit" | "discard"; + token: symbol; + } | null>(null); useEffect(() => { async function checkForRecovery() { @@ -108,8 +113,38 @@ export function useBrainDumpStep() { // Duration comes back from `stop()` for the same reason the id is // passed in below — `recorder.elapsedSeconds` here is this render's // value, and it is short by however long stopping took. - const durationSecs = await recorder.stop(); - await submitRecording(recorder.recordingId, durationSecs); + try { + const durationSecs = await recorder.stop(); + await submitRecording(recorder.recordingId, durationSecs); + } catch (error) { + Sentry.captureException(error, { + tags: { component: "useBrainDumpStep", action: "send" }, + }); + setScreen("failed"); + } + } + + async function handleStop() { + const recordingId = recorder.recordingId; + const actionToken = recordingId + ? claimTakeAction(recordingId, "discard") + : null; + if (recordingId && !actionToken) return; + trackBrainDump("brain_dump_canceled"); + try { + try { + await recorder.stop(); + } catch (error) { + Sentry.captureException(error, { + tags: { component: "useBrainDumpStep", action: "cancel" }, + }); + } + recorder.resetQueue(); + if (recordingId) await discardTake(recordingId); + } finally { + setScreen("rest"); + if (actionToken) releaseTakeAction(actionToken); + } } // The id is passed in rather than read off the recorder: `adoptRecovered` @@ -119,11 +154,22 @@ export function useBrainDumpStep() { recordingId: string | null, durationSecs: number, ) { + const actionToken = recordingId + ? claimTakeAction(recordingId, "submit") + : null; + if (recordingId && !actionToken) { + if (activeTakeActionRef.current?.action === "discard") { + setScreen("rest"); + } + return false; + } isSubmittingRef.current = true; try { await finalizeRecording(recordingId, durationSecs); + return true; } finally { isSubmittingRef.current = false; + if (actionToken) releaseTakeAction(actionToken); } } @@ -179,21 +225,28 @@ export function useBrainDumpStep() { // local parts and the server's half-uploaded buffer both — before the // orb starts listening again under a new recording id. async function handleRestart() { - trackBrainDump("brain_dump_restarted"); const previousId = recorder.recordingId; - await recorder.stop(); - recorder.resetQueue(); - if (previousId) await clearRecording(previousId).catch(() => undefined); - // Say which take: without an id the server drops whatever the row - // currently points at, which in a second tab is somebody else's - // buffer still being filled. - if (previousId) { - await discardBrainDump({ recording_id: previousId }).catch( - () => undefined, - ); + const actionToken = previousId + ? claimTakeAction(previousId, "discard") + : null; + if (previousId && !actionToken) return; + trackBrainDump("brain_dump_restarted"); + let started = false; + try { + try { + await recorder.stop(); + } catch (error) { + Sentry.captureException(error, { + tags: { component: "useBrainDumpStep", action: "restart" }, + }); + } + recorder.resetQueue(); + if (previousId) await discardTake(previousId); + started = await recorder.start(); + } finally { + setScreen(started ? "recording" : "rest"); + if (actionToken) releaseTakeAction(actionToken); } - const started = await recorder.start(); - if (!started) setScreen("rest"); } async function handleRetry() { @@ -323,25 +376,47 @@ export function useBrainDumpStep() { } async function handleDiscardRecovered() { - await dropRecoverable(); - setScreen("rest"); + if (await dropRecoverable()) setScreen("rest"); } async function handleTypeInsteadOfRecovered() { - await dropRecoverable(); - handleShowTyping(); + if (await dropRecoverable()) handleShowTyping(); } // Abandoning a take also releases the server's half-uploaded buffer — // otherwise those chunks sit in Redis until their TTL for a recording // nobody will ever finalize. async function dropRecoverable() { - if (!recoverable) return; - await clearRecording(recoverable.recordingId).catch(() => undefined); - await discardBrainDump({ recording_id: recoverable.recordingId }).catch( + if (!recoverable) return false; + const actionToken = claimTakeAction(recoverable.recordingId, "discard"); + if (!actionToken) return false; + try { + await discardTake(recoverable.recordingId); + setRecoverable(null); + return true; + } finally { + releaseTakeAction(actionToken); + } + } + + async function discardTake(recordingId: string) { + await clearRecording(recordingId).catch(() => undefined); + await discardBrainDump({ recording_id: recordingId }).catch( () => undefined, ); - setRecoverable(null); + } + + function claimTakeAction(recordingId: string, action: "submit" | "discard") { + if (activeTakeActionRef.current) return null; + const token = Symbol(recordingId); + activeTakeActionRef.current = { action, token }; + return token; + } + + function releaseTakeAction(token: symbol) { + if (activeTakeActionRef.current?.token === token) { + activeTakeActionRef.current = null; + } } async function handleDownloadRecording() { @@ -375,6 +450,7 @@ export function useBrainDumpStep() { reachedTimeLimit, recoverable, handleStart, + handleStop, handleDone, handleRestart, handleRetry, diff --git a/autogpt_platform/frontend/src/components/atoms/Reveal/Reveal.tsx b/autogpt_platform/frontend/src/components/atoms/Reveal/Reveal.tsx index dd19feae67d3..5a7b1d8e56f7 100644 --- a/autogpt_platform/frontend/src/components/atoms/Reveal/Reveal.tsx +++ b/autogpt_platform/frontend/src/components/atoms/Reveal/Reveal.tsx @@ -13,6 +13,15 @@ const ITEM: Variants = { }, }; +const ITEM_UNBLURRED: Variants = { + hidden: { opacity: 0, y: 14 }, + show: { + opacity: 1, + y: 0, + transition: { duration: 0.45, ease: [0.16, 1, 0.3, 1] }, + }, +}; + const ITEM_REDUCED: Variants = { hidden: { opacity: 0 }, show: { opacity: 1, transition: { duration: 0.2 } }, @@ -36,15 +45,19 @@ export function RevealGroup({ export function RevealItem({ children, className, + blur = true, }: { children: ReactNode; className?: string; + blur?: boolean; }) { const prefersReducedMotion = useReducedMotion(); return ( + @@ -58,7 +59,7 @@ export function GlassOrb({ params, children }: Props) {
- + {children && (
diff --git a/autogpt_platform/frontend/src/components/molecules/GlassOrb/GlassSurface.tsx b/autogpt_platform/frontend/src/components/molecules/GlassOrb/GlassSurface.tsx index e63ce5c7097f..dced5637b58c 100644 --- a/autogpt_platform/frontend/src/components/molecules/GlassOrb/GlassSurface.tsx +++ b/autogpt_platform/frontend/src/components/molecules/GlassOrb/GlassSurface.tsx @@ -25,7 +25,12 @@ export const DEFAULT_GLASS_PARAMS: GlassParams = { ringDark: 0.45, }; -export function GlassSurface({ params }: { params: GlassParams }) { +interface Props { + params: GlassParams; + showRim?: boolean; +} + +export function GlassSurface({ params, showRim = true }: Props) { const { frost, saturation, tint, edge } = params; const backdropFilter = `blur(${frost}px) saturate(${saturation})`; @@ -37,8 +42,12 @@ export function GlassSurface({ params }: { params: GlassParams }) { backdropFilter, WebkitBackdropFilter: backdropFilter, backgroundImage: `linear-gradient(155deg, rgba(255,255,255,${tint}), rgba(255,255,255,${tint * 0.2}) 48%, rgba(255,255,255,${tint * 0.6}))`, - border: `1px solid rgba(255,255,255,${Math.min(edge, 1)})`, - boxShadow: `inset 0 1px 3px rgba(255,255,255,${Math.min(edge * 1.2, 1)}), inset 0 -12px 28px rgba(255,255,255,${edge * 0.5}), 0 12px 40px rgba(96,64,224,0.18)`, + border: showRim + ? `1px solid rgba(255,255,255,${Math.min(edge, 1)})` + : undefined, + boxShadow: showRim + ? `inset 0 1px 3px rgba(255,255,255,${Math.min(edge * 1.2, 1)}), inset 0 -12px 28px rgba(255,255,255,${edge * 0.5}), 0 12px 40px rgba(96,64,224,0.18)` + : "0 12px 40px rgba(96,64,224,0.18)", }} />