Skip to content

Commit a0834da

Browse files
dhasilvadhasilva
authored
Voice to Content: Add states and refactor duration calculation (#35717)
* rename ContextualRow to AudioStatusPanel * add processing and error states * update duration calculation * update audio status duration to hook data * changelog * fix typo * disable unused variable check temporarily
1 parent 85b43a2 commit a0834da

File tree

7 files changed

+179
-119
lines changed

7 files changed

+179
-119
lines changed

‎projects/js-packages/ai-client/changelog/update-voice-to-content-states

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Significance: patch
2+
Type: changed
3+
4+
Voice to Content: Add states and refactor duration calculation

‎projects/js-packages/ai-client/src/components/audio-duration-display/index.tsx

+3-16
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,14 @@
1-
/*
2-
* External dependencies
3-
*/
4-
import { useState, useEffect } from '@wordpress/element';
51
/*
62
* Internal dependencies
73
*/
8-
import { formatTime, getDuration } from './lib/media.js';
4+
import { formatTime } from './lib/media.js';
95
/*
106
* Types
117
*/
128
import type React from 'react';
139

1410
type AudioDurationDisplayProps = {
15-
url: string;
11+
duration: number;
1612
className?: string | null;
1713
};
1814

@@ -23,17 +19,8 @@ type AudioDurationDisplayProps = {
2319
* @returns {React.ReactElement} Rendered component.
2420
*/
2521
export default function AudioDurationDisplay( {
26-
url,
22+
duration,
2723
className,
2824
}: AudioDurationDisplayProps ): React.ReactElement {
29-
const [ duration, setDuration ] = useState( 0 );
30-
useEffect( () => {
31-
if ( ! url ) {
32-
return;
33-
}
34-
35-
getDuration( url ).then( setDuration );
36-
}, [ url ] );
37-
3825
return <span className={ className }>{ formatTime( duration, { addDecimalPart: false } ) }</span>;
3926
}

‎projects/js-packages/ai-client/src/components/audio-duration-display/lib/media.ts

+5-37
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,3 @@
1-
/**
2-
* Function to get duration of audio file
3-
*
4-
* @param {string} url - The url of the audio file
5-
* @returns {Promise<number>} The duration of the audio file
6-
* @see https://stackoverflow.com/questions/21522036/html-audio-tag-duration-always-infinity
7-
*/
8-
export function getDuration( url: string ): Promise< number > {
9-
return new Promise( next => {
10-
const tmpAudioInstance = new Audio( url );
11-
tmpAudioInstance.addEventListener(
12-
'durationchange',
13-
function () {
14-
if ( this.duration === Infinity ) {
15-
return;
16-
}
17-
18-
const duration = this.duration;
19-
tmpAudioInstance.remove(); // remove instance from memory
20-
next( duration );
21-
},
22-
false
23-
);
24-
25-
tmpAudioInstance.load();
26-
tmpAudioInstance.currentTime = 24 * 60 * 60; // Fake big time
27-
tmpAudioInstance.volume = 0;
28-
tmpAudioInstance.play(); // This will call `durationchange` event
29-
} );
30-
}
31-
321
type FormatTimeOptions = {
332
/**
343
* Whether to add the decimal part to the formatted time.
@@ -50,19 +19,18 @@ type FormatTimeOptions = {
5019
* Formats the given time in milliseconds into a string with the format HH:MM:SS.DD,
5120
* adding hours and minutes only when needed.
5221
*
53-
* @param {number} time - The time in seconds to format.
22+
* @param {number} time - The time in milliseconds to format.
5423
* @param {FormatTimeOptions} options - The arguments.
5524
* @returns {string} The formatted time string.
5625
* @example
57-
* const formattedTime1 = formatTime( 1234567 ); // Returns "20:34.56"
58-
* const formattedTime2 = formatTime( 45123 ); // Returns "45.12"
59-
* const formattedTime3 = formatTime( 64, { addDecimalPart: false } ); // Returns "01:04"
26+
* const formattedTime1 = formatTime( 1234567, { addDecimalPart: true } ); // Returns "20:34.56"
27+
* const formattedTime2 = formatTime( 45123 ); // Returns "00.45"
28+
* const formattedTime3 = formatTime( 1200, { showHours: true } ); // Returns "00:00:01"
6029
*/
6130
export function formatTime(
6231
time: number,
63-
{ addDecimalPart = true, showMinutes = true, showHours = false }: FormatTimeOptions = {}
32+
{ addDecimalPart = false, showMinutes = true, showHours = false }: FormatTimeOptions = {}
6433
): string {
65-
time = time * 1000;
6634
const hours = Math.floor( time / 3600000 );
6735
const minutes = Math.floor( time / 60000 ) % 60;
6836
const seconds = Math.floor( time / 1000 ) % 60;

‎projects/js-packages/ai-client/src/hooks/use-media-recording/index.ts

+141-43
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,12 @@
22
* External dependencies
33
*/
44
import { useRef, useState, useEffect, useCallback } from '@wordpress/element';
5-
65
/*
76
* Types
87
*/
9-
type RecordingStateProp = 'inactive' | 'recording' | 'paused';
8+
type RecordingStateProp = 'inactive' | 'recording' | 'paused' | 'processing' | 'error';
109
type UseMediaRecordingProps = {
11-
onDone?: ( blob: Blob ) => void;
10+
onDone?: ( blob: Blob, url: string ) => void;
1211
};
1312

1413
type UseMediaRecordingReturn = {
@@ -27,6 +26,21 @@ type UseMediaRecordingReturn = {
2726
*/
2827
url: string | null;
2928

29+
/**
30+
* The error message
31+
*/
32+
error: string | null;
33+
34+
/**
35+
* The duration of the recorded audio
36+
*/
37+
duration: number;
38+
39+
/**
40+
* The error handler
41+
*/
42+
onError: ( err: string | Error ) => void;
43+
3044
controls: {
3145
/**
3246
* `start` recording handler
@@ -47,6 +61,11 @@ type UseMediaRecordingReturn = {
4761
* `stop` recording handler
4862
*/
4963
stop: () => void;
64+
65+
/**
66+
* `reset` recording handler
67+
*/
68+
reset: () => void;
5069
};
5170
};
5271

@@ -66,15 +85,25 @@ export default function useMediaRecording( {
6685
// Reference to the media recorder instance
6786
const mediaRecordRef = useRef( null );
6887

69-
// Recording state: `inactive`, `recording`, `paused`
88+
// Recording state: `inactive`, `recording`, `paused`, `processing`, `error`
7089
const [ state, setState ] = useState< RecordingStateProp >( 'inactive' );
7190

91+
// reference to the paused state to be used in the `onDataAvailable` event listener,
92+
// as the `mediaRecordRef.current.state` is already `inactive` when the recorder is stopped,
93+
// and the event listener does not react to state changes
94+
const isPaused = useRef< boolean >( false );
95+
96+
const recordStartTimestamp = useRef< number >( 0 );
97+
const [ duration, setDuration ] = useState< number >( 0 );
98+
7299
// The recorded blob
73100
const [ blob, setBlob ] = useState< Blob | null >( null );
74101

75102
// Store the recorded chunks
76103
const recordedChunks = useRef< Array< Blob > >( [] ).current;
77104

105+
const [ error, setError ] = useState< string | null >( null );
106+
78107
/**
79108
* Get the recorded blob.
80109
*
@@ -88,29 +117,114 @@ export default function useMediaRecording( {
88117

89118
// `start` recording handler
90119
const start = useCallback( ( timeslice: number ) => {
120+
clearData();
121+
91122
if ( ! timeslice ) {
92123
return mediaRecordRef?.current?.start();
93124
}
94125

95126
if ( timeslice < 100 ) {
96127
timeslice = 100; // set minimum timeslice to 100ms
97128
}
129+
130+
// Record the start time
131+
recordStartTimestamp.current = Date.now();
132+
98133
mediaRecordRef?.current?.start( timeslice );
99134
}, [] );
100135

101136
// `pause` recording handler
102137
const pause = useCallback( () => {
138+
isPaused.current = true;
103139
mediaRecordRef?.current?.pause();
140+
141+
// Calculate the duration of the recorded audio from the start time
142+
setDuration( currentDuration => currentDuration + Date.now() - recordStartTimestamp.current );
104143
}, [] );
105144

106145
// `resume` recording handler
107146
const resume = useCallback( () => {
147+
isPaused.current = false;
108148
mediaRecordRef?.current?.resume();
149+
150+
// Record the start time
151+
recordStartTimestamp.current = Date.now();
109152
}, [] );
110153

111154
// `stop` recording handler
112155
const stop = useCallback( () => {
113156
mediaRecordRef?.current?.stop();
157+
158+
if ( state === 'recording' ) {
159+
// Calculate the duration of the recorded audio from the start time
160+
setDuration( currentDuration => currentDuration + Date.now() - recordStartTimestamp.current );
161+
}
162+
}, [] );
163+
164+
// clears the recording state
165+
const clearData = useCallback( () => {
166+
recordedChunks.length = 0;
167+
setBlob( null );
168+
setError( null );
169+
setDuration( 0 );
170+
isPaused.current = false;
171+
recordStartTimestamp.current = 0;
172+
}, [] );
173+
174+
// removes the event listeners
175+
const clearListeners = useCallback( () => {
176+
/*
177+
* mediaRecordRef is not defined when
178+
* the getUserMedia API is not supported,
179+
* or when the user has not granted access
180+
*/
181+
if ( ! mediaRecordRef?.current ) {
182+
return;
183+
}
184+
185+
mediaRecordRef.current.removeEventListener( 'start', onStartListener );
186+
mediaRecordRef.current.removeEventListener( 'stop', onStopListener );
187+
mediaRecordRef.current.removeEventListener( 'pause', onPauseListener );
188+
mediaRecordRef.current.removeEventListener( 'resume', onResumeListener );
189+
mediaRecordRef.current.removeEventListener( 'dataavailable', onDataAvailableListener );
190+
mediaRecordRef.current = null;
191+
}, [] );
192+
193+
// resets the recording state, initializing the media recorder instance
194+
const reset = useCallback( () => {
195+
setState( 'inactive' );
196+
clearData();
197+
clearListeners();
198+
199+
// Check if the getUserMedia API is supported
200+
if ( ! navigator.mediaDevices?.getUserMedia ) {
201+
return;
202+
}
203+
204+
const constraints = { audio: true };
205+
206+
navigator.mediaDevices
207+
.getUserMedia( constraints )
208+
.then( stream => {
209+
mediaRecordRef.current = new MediaRecorder( stream );
210+
211+
mediaRecordRef.current.addEventListener( 'start', onStartListener );
212+
mediaRecordRef.current.addEventListener( 'stop', onStopListener );
213+
mediaRecordRef.current.addEventListener( 'pause', onPauseListener );
214+
mediaRecordRef.current.addEventListener( 'resume', onResumeListener );
215+
mediaRecordRef.current.addEventListener( 'dataavailable', onDataAvailableListener );
216+
} )
217+
.catch( err => {
218+
// @todo: handle error
219+
throw err;
220+
} );
221+
}, [] );
222+
223+
// stops the recording and sets the error state
224+
const onError = useCallback( ( err: string | Error ) => {
225+
stop();
226+
setError( typeof err === 'string' ? err : err.message );
227+
setState( 'error' );
114228
}, [] );
115229

116230
/**
@@ -122,12 +236,15 @@ export default function useMediaRecording( {
122236

123237
/**
124238
* `stop` event listener for the media recorder instance.
239+
* Happens after the last `dataavailable` event.
125240
*
126241
* @returns {void}
127242
*/
128243
function onStopListener(): void {
129-
setState( 'inactive' );
130-
onDone?.( getBlob() );
244+
setState( 'processing' );
245+
const lastBlob = getBlob();
246+
const url = URL.createObjectURL( lastBlob );
247+
onDone?.( lastBlob, url );
131248

132249
// Clear the recorded chunks
133250
recordedChunks.length = 0;
@@ -164,61 +281,42 @@ export default function useMediaRecording( {
164281

165282
// Create and store the Blob for the recorded chunks
166283
setBlob( getBlob() );
167-
}
168284

169-
// Create media recorder instance
170-
useEffect( () => {
171-
// Check if the getUserMedia API is supported
172-
if ( ! navigator.mediaDevices?.getUserMedia ) {
173-
return;
285+
// If the recorder was paused, it is the last data available event, so we do not update the duration
286+
if ( ! isPaused.current ) {
287+
setDuration( currentDuration => {
288+
const now = Date.now();
289+
const difference = now - recordStartTimestamp.current;
290+
// Update the start time
291+
recordStartTimestamp.current = now;
292+
return currentDuration + difference;
293+
} );
174294
}
295+
}
175296

176-
const constraints = { audio: true };
177-
178-
navigator.mediaDevices
179-
.getUserMedia( constraints )
180-
.then( stream => {
181-
mediaRecordRef.current = new MediaRecorder( stream );
182-
183-
mediaRecordRef.current.addEventListener( 'start', onStartListener );
184-
mediaRecordRef.current.addEventListener( 'stop', onStopListener );
185-
mediaRecordRef.current.addEventListener( 'pause', onPauseListener );
186-
mediaRecordRef.current.addEventListener( 'resume', onResumeListener );
187-
mediaRecordRef.current.addEventListener( 'dataavailable', onDataAvailableListener );
188-
} )
189-
.catch( err => {
190-
// @todo: handle error
191-
throw err;
192-
} );
297+
// Remove listeners and clear the recorded chunks
298+
useEffect( () => {
299+
reset();
193300

194301
return () => {
195-
/*
196-
* mediaRecordRef is not defined when
197-
* the getUserMedia API is not supported,
198-
* or when the user has not granted access
199-
*/
200-
if ( ! mediaRecordRef?.current ) {
201-
return;
202-
}
203-
204-
mediaRecordRef.current.removeEventListener( 'start', onStartListener );
205-
mediaRecordRef.current.removeEventListener( 'stop', onStopListener );
206-
mediaRecordRef.current.removeEventListener( 'pause', onPauseListener );
207-
mediaRecordRef.current.removeEventListener( 'resume', onResumeListener );
208-
mediaRecordRef.current.removeEventListener( 'dataavailable', onDataAvailableListener );
302+
clearListeners();
209303
};
210304
}, [] );
211305

212306
return {
213307
state,
214308
blob,
215309
url: blob ? URL.createObjectURL( blob ) : null,
310+
error,
311+
duration,
312+
onError,
216313

217314
controls: {
218315
start,
219316
pause,
220317
resume,
221318
stop,
319+
reset,
222320
},
223321
};
224322
}

0 commit comments

Comments
 (0)