Skip to content

Commit 7bb8b6b

Browse files
lhkowalskidhasilva
lhkowalski
and
dhasilva
authored
Jetpack AI: Add transcription post-processing machinery (#35734)
* Add first version of the hook * Export hook, types and constants * Rename const to refer to the functionality it's part of * Add example of useTranscriptionPostProcessing usage for testing purposes * Bump AI Client version * Add changelog file * Fix the version * Down to patch instead of minor change, since the hook is not yet in use * remove types import from ai-client component stories --------- Co-authored-by: Douglas <[email protected]>
1 parent a0834da commit 7bb8b6b

File tree

8 files changed

+174
-4
lines changed

8 files changed

+174
-4
lines changed

‎projects/js-packages/ai-client/changelog/update-voice-to-content-add-transcription-post-processing-machinery

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Significance: patch
2+
Type: added
3+
4+
AI Client: add support for transcription post-processing.

‎projects/js-packages/ai-client/src/components/ai-control/stories/index.stories.tsx

+1-2
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,6 @@ import React from 'react';
77
/**
88
* Internal dependencies
99
*/
10-
import { REQUESTING_STATES } from '../../../types.js';
1110
import AIControl from '../index.js';
1211
/**
1312
* Types
@@ -34,7 +33,7 @@ const meta: AIControlStoryMeta = {
3433
control: {
3534
type: 'select',
3635
},
37-
options: REQUESTING_STATES,
36+
options: [ 'init', 'requesting', 'suggesting', 'done', 'error' ],
3837
},
3938
},
4039
parameters: {

‎projects/js-packages/ai-client/src/components/ai-status-indicator/stories/index.stories.tsx

+1-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ import React from 'react';
55
/*
66
* Internal Dependencies
77
*/
8-
import { REQUESTING_STATES } from '../../../types.js';
98
import AiStatusIndicator, { AiStatusIndicatorProps } from '../index.js';
109

1110
type AiStatusIndicatoryStoryProps = AiStatusIndicatorProps & {
@@ -21,7 +20,7 @@ export default {
2120
control: {
2221
type: 'select',
2322
},
24-
options: REQUESTING_STATES,
23+
options: [ 'init', 'requesting', 'suggesting', 'done', 'error' ],
2524
},
2625
size: {
2726
control: {

‎projects/js-packages/ai-client/src/hooks/use-transcription-post-processing/index.ts

+135
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
/**
2+
* External dependencies
3+
*/
4+
import { useCallback, useState } from '@wordpress/element';
5+
import debugFactory from 'debug';
6+
/**
7+
* Internal dependencies
8+
*/
9+
import useAiSuggestions, { RequestingErrorProps } from '../use-ai-suggestions/index.js';
10+
import type { PromptProp } from '../../types.js';
11+
12+
const debug = debugFactory( 'jetpack-ai-client:use-transcription-post-processing' );
13+
14+
/**
15+
* Post-processing types.
16+
*/
17+
export const TRANSCRIPTION_POST_PROCESSING_ACTION_SIMPLE_DRAFT = 'voice-to-content-simple-draft';
18+
export type PostProcessingAction = typeof TRANSCRIPTION_POST_PROCESSING_ACTION_SIMPLE_DRAFT;
19+
20+
/**
21+
* The return value for the transcription post-processing hook.
22+
*/
23+
export type UseTranscriptionPostProcessingReturn = {
24+
postProcessingResult: string;
25+
isProcessingTranscription: boolean;
26+
postProcessingError: string;
27+
processTranscription: ( action: PostProcessingAction, transcription: string ) => void;
28+
};
29+
30+
/**
31+
* The props for the transcription post-processing hook.
32+
*/
33+
export type UseTranscriptionPostProcessingProps = {
34+
feature: string;
35+
onReady?: ( postProcessingResult: string ) => void;
36+
onError?: ( error: string ) => void;
37+
onUpdate?: ( currentPostProcessingResult: string ) => void;
38+
};
39+
40+
/**
41+
* A hook to handle transcription post-processing.
42+
*
43+
* @param {string} feature - The feature name that is calling the post-processing actions.
44+
* @returns {UseTranscriptionPostProcessingReturn} - Object with properties to get the post-processing results.
45+
*/
46+
export default function useTranscriptionPostProcessing( {
47+
feature,
48+
onReady,
49+
onError,
50+
onUpdate,
51+
}: UseTranscriptionPostProcessingProps ): UseTranscriptionPostProcessingReturn {
52+
const [ postProcessingResult, setPostProcessingResult ] = useState< string >( '' );
53+
const [ postProcessingError, setPostProcessingError ] = useState< string >( '' );
54+
const [ isProcessingTranscription, setIsProcessingTranscription ] = useState( false );
55+
56+
/**
57+
* Set-up the useAiSuggestions hook.
58+
*/
59+
const handleOnSuggestion = useCallback(
60+
( suggestion: string ) => {
61+
setPostProcessingResult( suggestion );
62+
onUpdate?.( suggestion );
63+
},
64+
[ setPostProcessingResult, onUpdate ]
65+
);
66+
67+
const handleOnDone = useCallback(
68+
( result: string ) => {
69+
setPostProcessingResult( result );
70+
onUpdate?.( result );
71+
onReady?.( result );
72+
},
73+
[ setPostProcessingResult, onUpdate, onReady ]
74+
);
75+
76+
const handleOnError = useCallback(
77+
( errorData: RequestingErrorProps ) => {
78+
setPostProcessingError( errorData.message );
79+
onError?.( errorData.message );
80+
},
81+
[ setPostProcessingError, onError ]
82+
);
83+
84+
const { request } = useAiSuggestions( {
85+
autoRequest: false,
86+
onSuggestion: handleOnSuggestion,
87+
onDone: handleOnDone,
88+
onError: handleOnError,
89+
} );
90+
91+
const handleTranscriptionPostProcessing = useCallback(
92+
( action: PostProcessingAction, transcription: string ) => {
93+
debug( 'Post-processing transcription' );
94+
95+
/**
96+
* Reset the transcription result and error.
97+
*/
98+
setPostProcessingResult( '' );
99+
setPostProcessingError( '' );
100+
setIsProcessingTranscription( true );
101+
102+
/**
103+
* Build the prompt to call the suggestion hook.
104+
*/
105+
const messages: PromptProp = [
106+
{
107+
role: 'jetpack-ai',
108+
context: {
109+
type: action,
110+
content: transcription,
111+
},
112+
},
113+
];
114+
115+
/**
116+
* Call the suggestion hook using the message.
117+
*/
118+
request( messages, { feature } );
119+
},
120+
[
121+
setPostProcessingResult,
122+
setPostProcessingError,
123+
setIsProcessingTranscription,
124+
request,
125+
feature,
126+
]
127+
);
128+
129+
return {
130+
postProcessingResult,
131+
isProcessingTranscription,
132+
postProcessingError,
133+
processTranscription: handleTranscriptionPostProcessing,
134+
};
135+
}

‎projects/js-packages/ai-client/src/index.ts

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ export { default as transcribeAudio } from './audio-transcription/index.js';
1212
export { default as useAiSuggestions } from './hooks/use-ai-suggestions/index.js';
1313
export { default as useMediaRecording } from './hooks/use-media-recording/index.js';
1414
export { default as useAudioTranscription } from './hooks/use-audio-transcription/index.js';
15+
export { default as useTranscriptionPostProcessing } from './hooks/use-transcription-post-processing/index.js';
1516

1617
/*
1718
* Components: Icons

‎projects/js-packages/ai-client/src/types.ts

+9
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,15 @@ export type {
4141
UseAudioTranscriptionProps,
4242
UseAudioTranscriptionReturn,
4343
} from './hooks/use-audio-transcription/index.js';
44+
export type {
45+
UseTranscriptionPostProcessingProps,
46+
UseTranscriptionPostProcessingReturn,
47+
PostProcessingAction,
48+
} from './hooks/use-transcription-post-processing/index.js';
49+
/*
50+
* Hook constants
51+
*/
52+
export { TRANSCRIPTION_POST_PROCESSING_ACTION_SIMPLE_DRAFT } from './hooks/use-transcription-post-processing/index.js';
4453

4554
/*
4655
* Requests types

‎projects/plugins/jetpack/changelog/update-voice-to-content-add-transcription-post-processing-machinery

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Significance: patch
2+
Type: other
3+
4+
Jetpack AI: Add transcription post-processing example to Voice-to-Content block.

‎projects/plugins/jetpack/extensions/blocks/voice-to-content/edit.tsx

+19
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import {
88
useMediaRecording,
99
useAudioTranscription,
1010
UseAudioTranscriptionReturn,
11+
useTranscriptionPostProcessing,
12+
TRANSCRIPTION_POST_PROCESSING_ACTION_SIMPLE_DRAFT,
1113
} from '@automattic/jetpack-ai-client';
1214
import { ThemeProvider } from '@automattic/jetpack-components';
1315
import { Button, Modal, Icon, FormFileUpload } from '@wordpress/components';
@@ -83,9 +85,26 @@ function AudioStatusPanel( { state, error = null, audioURL = null, duration = 0
8385
function ActionButtons( { state, mediaControls, onError } ) {
8486
const { start, pause, resume, stop, reset } = mediaControls ?? {};
8587

88+
const { processTranscription } = useTranscriptionPostProcessing( {
89+
feature: 'voice-to-content',
90+
onReady: result => {
91+
// eslint-disable-next-line no-console
92+
console.log( 'Post-processing ready: ', result );
93+
},
94+
onError: error => {
95+
// eslint-disable-next-line no-console
96+
console.log( 'Post-processing error: ', error );
97+
},
98+
onUpdate: currentPostProcessingResult => {
99+
// eslint-disable-next-line no-console
100+
console.log( 'Post-processing update: ', currentPostProcessingResult );
101+
},
102+
} );
103+
86104
const onTranscriptionReady = ( transcription: string ) => {
87105
// eslint-disable-next-line no-console
88106
console.log( 'Transcription ready: ', transcription );
107+
processTranscription( TRANSCRIPTION_POST_PROCESSING_ACTION_SIMPLE_DRAFT, transcription );
89108
};
90109

91110
const onTranscriptionError = ( error: string ) => {

0 commit comments

Comments
 (0)