Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/converters/_live_converters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2063,6 +2063,18 @@ export function replicatedVoiceConfigToVertex(
common.setValueByPath(toObject, ['voiceSampleAudio'], fromVoiceSampleAudio);
}

if (common.getValueByPath(fromObject, ['consentAudio']) !== undefined) {
throw new Error('consentAudio parameter is not supported in Vertex AI.');
}

if (
common.getValueByPath(fromObject, ['voiceConsentSignature']) !== undefined
) {
throw new Error(
'voiceConsentSignature parameter is not supported in Vertex AI.',
);
}

return toObject;
}

Expand Down
12 changes: 12 additions & 0 deletions src/converters/_models_converters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4725,6 +4725,18 @@ export function replicatedVoiceConfigToVertex(
common.setValueByPath(toObject, ['voiceSampleAudio'], fromVoiceSampleAudio);
}

if (common.getValueByPath(fromObject, ['consentAudio']) !== undefined) {
throw new Error('consentAudio parameter is not supported in Vertex AI.');
}

if (
common.getValueByPath(fromObject, ['voiceConsentSignature']) !== undefined
) {
throw new Error(
'voiceConsentSignature parameter is not supported in Vertex AI.',
);
}

return toObject;
}

Expand Down
12 changes: 12 additions & 0 deletions src/converters/_tunings_converters.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1186,6 +1186,18 @@ export function replicatedVoiceConfigToVertex(
common.setValueByPath(toObject, ['voiceSampleAudio'], fromVoiceSampleAudio);
}

if (common.getValueByPath(fromObject, ['consentAudio']) !== undefined) {
throw new Error('consentAudio parameter is not supported in Vertex AI.');
}

if (
common.getValueByPath(fromObject, ['voiceConsentSignature']) !== undefined
) {
throw new Error(
'voiceConsentSignature parameter is not supported in Vertex AI.',
);
}

return toObject;
}

Expand Down
12 changes: 10 additions & 2 deletions src/live.ts
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,12 @@ export class Live {
const websocketCallbacks: WebSocketCallbacks = {
onopen: onopenAwaitedCallback,
onmessage: (event: MessageEvent) => {
void handleWebSocketMessage(apiClient, callbacks.onmessage, event);
void handleWebSocketMessage(apiClient, (msg: types.LiveServerMessage) => {
if (msg.setupComplete && !session.setupComplete) {
session.setupComplete = msg.setupComplete;
}
callbacks.onmessage(msg);
}, event);
},
onerror:
callbacks?.onerror ??
Expand Down Expand Up @@ -288,7 +293,8 @@ export class Live {
}
delete clientMessage['config'];
conn.send(JSON.stringify(clientMessage));
return new Session(conn, this.apiClient);
const session = new Session(conn, this.apiClient);
return session;
}

// TODO: b/416041229 - Abstract this method to a common place.
Expand All @@ -308,6 +314,8 @@ const defaultLiveSendClientContentParamerters: types.LiveSendClientContentParame
@experimental
*/
export class Session {
setupComplete?: types.LiveServerSetupComplete;

constructor(
readonly conn: WebSocket,
private readonly apiClient: ApiClient,
Expand Down
23 changes: 23 additions & 0 deletions src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2376,6 +2376,13 @@ export declare interface ToolConfig {
includeServerSideToolInvocations?: boolean;
}

/** The signature of the voice consent check. */
export declare interface VoiceConsentSignature {
/** The signature string.
*/
signature?: string;
}

/** The configuration for the replicated voice to use. */
export declare interface ReplicatedVoiceConfig {
/** The mimetype of the voice sample. The only currently supported
Expand All @@ -2387,6 +2394,17 @@ export declare interface ReplicatedVoiceConfig {

* @remarks Encoded as base64 string. */
voiceSampleAudio?: string;
/** Recorded consent verifying ownership of the voice. This
represents 16-bit signed little-endian wav data, with a 24kHz sampling
rate.
* @remarks Encoded as base64 string. */
consentAudio?: string;
/** Signature of a previously verified consent audio. This should be
populated with a signature generated by the server for a previous
request containing the consent_audio field. When provided, the
signature is verified instead of the consent_audio field to reduce
latency. Requests will fail if the signature is invalid or expired. */
voiceConsentSignature?: VoiceConsentSignature;
}

/** Configuration for a prebuilt voice. */
Expand Down Expand Up @@ -6687,6 +6705,11 @@ export class ContentReferenceImage {
export declare interface LiveServerSetupComplete {
/** The session id of the live session. */
sessionId?: string;
/** Signature of the verified consent audio. This is populated when the
request has a ReplicatedVoiceConfig with consent_audio set, if the consent
verification was successful. This may be used in a subsequent request
instead of the consent_audio to verify the same consent. */
voiceConsentSignature?: VoiceConsentSignature;
}

/** Audio transcription in Server Conent. */
Expand Down
50 changes: 50 additions & 0 deletions test/unit/live_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -288,6 +288,56 @@ describe('live', () => {
expect(receivedMessage.setupComplete).toBeDefined();
});

it('connect should populate setupComplete with voiceConsentSignature', async () => {
const apiClient = new ApiClient({
auth: new FakeAuth(),
apiKey: 'test-api-key',
uploader: new CrossUploader(),
downloader: new CrossDownloader(),
});
const websocketFactory = new FakeWebSocketFactory();
const live = new Live(apiClient, new FakeAuth(), websocketFactory);

let capturedCallbacks: WebSocketCallbacks | undefined;
const websocketFactorySpy = spyOn(websocketFactory, 'create').and.callFake(
(url, headers, callbacks) => {
capturedCallbacks = callbacks;
return new FakeWebSocket(url, headers, callbacks);
},
);

const session = await live.connect({
model: 'models/gemini-live-2.5-flash-preview',
callbacks: {
onmessage: () => {},
},
});

expect(websocketFactorySpy).toHaveBeenCalled();
if (!capturedCallbacks) {
throw new Error('WebSocket callbacks were not captured');
}

const testMessage = {
setupComplete: {
voiceConsentSignature: {
signature: 'test_sig',
},
},
};
const jsonString = JSON.stringify(testMessage);

capturedCallbacks.onmessage({
data: jsonString,
} as MessageEvent);

// Allow the async handleWebSocketMessage to complete
await new Promise(process.nextTick);

expect(session.setupComplete).toBeDefined();
expect(session.setupComplete?.voiceConsentSignature?.signature).toBe('test_sig');
});

it('connect Gemini should fail with setup message using transparent', async () => {
const apiClient = new ApiClient({
auth: new FakeAuth(),
Expand Down
Loading