feat: enhance agent and vendor documentation with new features and configurations

- Updated `withTools` method to clarify support for both MCP and inline REST tool invocation.
- Added `GeminiSTT` vendor with detailed configuration options for Google Gemini ASR.
- Introduced `OpenAIGptLive` vendor for OpenAI GPT Live with specific parameters.
- Enhanced documentation for MLLM vendors, including support for new global and CN vendors.
- Improved examples for filler words and inline tools in agent configuration.
- Updated tests to cover new features and ensure proper serialization of configurations.

Author: seymourtang

docs/concepts/agent.md

@@ -74,7 +74,7 @@ Each method returns a new `Agent` instance with the updated configuration. Vendo
 | `withGreetingConfigs` | `withGreetingConfigs(configs: LlmGreetingConfigs): Agent<TTSSampleRate, TArea>` | Deprecated. Configure greeting playback on the LLM vendor instead. |
 | `withSal` | `withSal(config: SalConfig): Agent<TTSSampleRate, TArea>` | Set SAL configuration |
 | `withAdvancedFeatures` | `withAdvancedFeatures(features: AdvancedFeatures): Agent<TTSSampleRate, TArea>` | Set advanced features |
-| `withTools` | `withTools(enabled?: boolean): Agent<TTSSampleRate, TArea>` | Enable or disable MCP tool invocation |
+| `withTools` | `withTools(enabled?: boolean): Agent<TTSSampleRate, TArea>` | Enable or disable MCP and inline REST tool invocation |
 | `withParameters` | `withParameters(parameters: SessionParamsInput): Agent<TTSSampleRate, TArea>` | Set session parameters |
 | `withAudioScenario` | `withAudioScenario(audioScenario: ParametersAudioScenario): Agent<TTSSampleRate, TArea>` | Set `parameters.audio_scenario` |
 | `withFailureMessage` | `withFailureMessage(message: string): Agent<TTSSampleRate, TArea>` | Deprecated. Use LLM/MLLM vendor `failureMessage` instead. |

docs/concepts/vendors.md

@@ -86,6 +86,7 @@ The `sampleRate` is critical when using avatars. See [Avatar Integration](../gui
 | `MicrosoftSTT`    | Azure Speech      | `key`, `region`, `language`                      |
 | `OpenAISTT`       | OpenAI Whisper    | `apiKey`, `model?`, `language?`, `prompt?`       |
 | `GoogleSTT`       | Google Speech     | `projectId`, `location`, `adcCredentialsString`, `language` |
+| `GeminiSTT`       | Google Gemini ASR | `apiKey`, `model`, `sampleRate?`, `language?`, `wordTimestamp?` |
 | `AmazonSTT`       | Amazon Transcribe | `accessKey`, `secretKey`, `region`, `language`   |
 | `AssemblyAISTT`   | AssemblyAI        | `apiKey`, `language`, `ws_url?`                  |
 | `AresSTT`         | Agora ARES        | `keywords?`, `additionalParams?`                 |
@@ -104,13 +105,31 @@ const stt = new DeepgramSTT({
 });
 ```
 
+`GeminiSTT` serializes its API key, model, and optional transcription settings
+into the generated Gemini ASR configuration:
+
+```typescript
+import { GeminiSTT } from 'agora-agents';
+
+const stt = new GeminiSTT({
+  apiKey: 'your-gemini-key',
+  model: 'gemini-transcribe',
+  language: 'en-US',
+  wordTimestamp: true,
+});
+```
+
+ARES and Fengming hotwords are sent as the top-level `asr.keywords` field;
+additional provider-specific fields remain under `asr.params`.
+
 ## MLLM vendors
 
 MLLM (Multimodal LLM) vendors handle audio end-to-end — no separate STT or TTS step. Call `agent.withMllm(vendor)` and MLLM mode is enabled automatically; no separate `advancedFeatures` flag is needed.
 
 | Class            | Provider                        | Key constructor params                                                                                                                                                                    |
 | ---------------- | ------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `OpenAIRealtime` | OpenAI Realtime API             | `apiKey`, `model?`, `url?`, `greetingMessage?`, `failureMessage?`, `inputModalities?`, `outputModalities?`, `messages?`, `turnDetection?`                                                 |
+| `OpenAIRealtime` | OpenAI Realtime API (`mllm.vendor: openai`) | `apiKey`, `model?`, `url?`, `greetingMessage?`, `failureMessage?`, `inputModalities?`, `outputModalities?`, `messages?`, `turnDetection?` |
+| `OpenAIGptLive` | OpenAI GPT Live (`mllm.vendor: openai_gpt_live`) | `apiKey`, `model?`, `url?`, `greetingMessage?`, `failureMessage?`, `inputModalities?`, `outputModalities?`, `messages?`, `turnDetection?` |
 | `AzureOpenAIRealtime` | Azure OpenAI Realtime (global) | `apiKey`, `url`, `turnDetection`, `model?`, `voice?`, `instructions?`, `params?`, `messages?`, `outputModalities?`, `maxHistory?`, `greetingMessage?` |
 | `GeminiLive`     | Google Gemini Live API          | `apiKey`, `model`, `url?`, `voice?`, `greetingMessage?`, `failureMessage?`, `inputModalities?`, `outputModalities?`, `messages?`, `turnDetection?`                                        |
 | `VertexAI`       | Vertex AI Gemini Live           | `model`, `url?`, `projectId`, `location`, `adcCredentialsString`, `voice?`, `greetingMessage?`, `failureMessage?`, `inputModalities?`, `outputModalities?`, `messages?`, `turnDetection?` |

docs/guides/agent-builder-features.md

@@ -16,7 +16,8 @@ Every example assumes an `AgoraClient` is created first and passed to `new Agent
 |---|---|---|
 | `sal` | `withSal(config)` | Selective Attention Locking — speaker recognition and noise suppression |
 | `advancedFeatures` | `withAdvancedFeatures(features)` | Enable MLLM, RTM, SAL, tools |
-| `tools` | `withTools(enabled)` | Enable MCP tool invocation |
+| `tools` | `withTools(enabled)` | Enable MCP and inline REST tool invocation |
+| `llm.tools` | LLM vendor `tools` option | Define inline synchronous REST function-calling tools |
 | `parameters` | `withParameters(params)` | Silence config, farewell config, data channel |
 | `failureMessage` | LLM/MLLM vendor option | Message spoken when LLM fails |
 | `maxHistory` | LLM vendor option | Max conversation turns in LLM context |
@@ -84,7 +85,8 @@ const mllmAgent = new Agent({ client })
 const rtmAgent = new Agent({ client })
   .withAdvancedFeatures({ enable_rtm: true });
 
-// Enable tool invocation via MCP
+// Enable tool invocation for both MCP servers and inline LLM REST tools.
+// This must be enabled when either tool definition is configured.
 const toolsAgent = new Agent({ client })
   .withTools();
 ```
@@ -257,6 +259,59 @@ const agent = new Agent({ client })
   .withStt(/* ... */);
 ```
 
+Generated filler words can use an OpenAI-compatible provider. Static phrases
+are still required because they are used as the fallback when generation is
+not ready, fails, or returns an empty response:
+
+```typescript
+.withFillerWords({
+  enable: true,
+  trigger: { mode: 'fixed_time', fixed_time_config: { response_wait_ms: 2000 } },
+  content: {
+    mode: 'generated',
+    static_config: {
+      phrases: ['One moment please.', 'Let me check that.'],
+      selection_rule: 'shuffle',
+    },
+    generated_config: {
+      llm_provider: {
+        base_url: 'https://api.openai.com/v1/chat/completions',
+        api_key: 'your-filler-llm-key',
+        params: { model: 'gpt-4o-mini' },
+      },
+      prompt: 'Generate a short conversational filler phrase.',
+      fallback_strategy: 'static',
+    },
+  },
+})
+```
+
+The `tools` option on standard text LLM vendors is separate from MCP. It
+defines synchronous REST tools using `type: 'function'`, a JSON Schema under
+`function.parameters`, and a `server` with `GET` or `POST` plus a URL template:
+
+Call `.withTools(true)` on the agent as well; `llm.tools` is ignored unless
+`advanced_features.enable_tools` is enabled.
+
+```typescript
+new OpenAI({
+  apiKey: 'your-openai-key',
+  model: 'gpt-4o-mini',
+  url: 'https://api.openai.com/v1/chat/completions',
+  tools: [{
+    type: 'function',
+    function: {
+      name: 'lookup_order',
+      parameters: { type: 'object', properties: { orderId: { type: 'string' } } },
+    },
+    server: {
+      method: 'GET',
+      url: 'https://example.com/orders/{{args.orderId}}',
+    },
+  }],
+})
+```
+
 ## Getters
 
 Read back configuration via getter properties:

docs/guides/mllm-flow.md

@@ -6,7 +6,7 @@ description: Use global or Chinese mainland MLLM providers for end-to-end audio
 
 # MLLM Flow (Multimodal)
 
-In MLLM mode, a single multimodal model handles audio input and output end-to-end — no separate STT or TTS step. AgentKit supports global OpenAI Realtime, Azure OpenAI Realtime, Gemini Live, Vertex AI, and xAI Grok providers, plus Qwen Omni for Chinese mainland deployments.
+In MLLM mode, a single multimodal model handles audio input and output end-to-end — no separate STT or TTS step. AgentKit supports global OpenAI Realtime (`mllm.vendor = 'openai'`), OpenAI GPT Live (`mllm.vendor = 'openai_gpt_live'`), Azure OpenAI Realtime, Gemini Live, Vertex AI, and xAI Grok providers, plus Qwen Omni for Chinese mainland deployments.
 
 ## When to use MLLM
 

docs/reference/agent.md

@@ -80,7 +80,7 @@ If you omit `withStt()`, AgentKit still sends an ASR config automatically based
 
 ### `withMllm(vendor: GlobalMllmVendor | CNMllmVendor): Agent<TTSSampleRate, TArea>`
 
-Set the MLLM vendor for multimodal mode. Pass a global vendor (`OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok`) or the Chinese mainland vendor `QwenOmni`. Calling `withMllm()` automatically sets `mllm.enable = true`. MLLM mode does not require `withTts()` / `withLlm()` / `withStt()`.
+Set the MLLM vendor for multimodal mode. Pass a global vendor (`OpenAIRealtime`, `OpenAIGptLive`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok`) or the Chinese mainland vendor `QwenOmni`. Calling `withMllm()` automatically sets `mllm.enable = true`. MLLM mode does not require `withTts()` / `withLlm()` / `withStt()`.
 
 > Avatars are only supported with the cascading ASR + LLM + TTS pipeline. Combining `withMllm()` with `withAvatar()` throws at `toProperties()` and `session.start()`.
 
@@ -118,7 +118,9 @@ Set advanced features (e.g. `enable_rtm`).
 
 ### `withTools(enabled = true): Agent<TTSSampleRate, TArea>`
 
-Enable or disable MCP tool invocation by setting `advanced_features.enable_tools`.
+Enable or disable tool invocation by setting `advanced_features.enable_tools`.
+This shared switch is required for both `llm.mcp_servers` and the inline
+`llm.tools` REST tool definitions.
 
 ### `withParameters(parameters: SessionParamsInput): Agent<TTSSampleRate, TArea>`
 
@@ -251,8 +253,12 @@ In the cascading pipeline, `toProperties()` also fills in a default `asr` block
 
 Public aliases over Fern-generated types include `LlmConfig`, `SttConfig`, `AsrConfig` (= `SttConfig`), `MllmConfig`, `AvatarConfig`, session/conversation types, and think types (`ThinkOnListeningAction`, etc.).
 
+LLM configs also expose `LlmTool`, `LlmToolFunction`, `LlmToolServer`, and
+`LlmToolExecution` for inline synchronous REST tools. These are distinct from
+MCP servers configured through `mcp_servers`.
+
 Builder vendor unions: `LlmVendor`, `TtsVendor<SR>`, `SttVendor`, `AvatarVendor<SR>` (class instances accepted by `.withLlm()` / `.withTts()` / `.withStt()` / `.withAvatar()`).
 
 Wire vendor name aliases: `AsrVendorName` (ASR vendor string), `AvatarWireVendor` (avatar vendor string on `AvatarConfig`).
 
-Think value constants: `ThinkOnListeningActionInject`, `ThinkOnListeningActionInterrupt`, `ThinkOnListeningActionIgnore`, `ThinkOnThinkingActionInterrupt`, `ThinkOnThinkingActionIgnore`, `ThinkOnSpeakingActionInterrupt`, `ThinkOnSpeakingActionIgnore`.
+Think value constants include `ThinkOnListeningActionInject`, `ThinkOnListeningActionInterrupt`, `ThinkOnListeningActionIgnore`, and `ThinkOnListeningActionAppend`; corresponding `Append` constants are also available for thinking and speaking states. `append` queues the instruction until the current state finishes without interrupting the active interaction.

docs/reference/vendors.md

@@ -105,6 +105,7 @@ new OpenAI(options: OpenAIOptions)
 | `greetingAudioUrl` | `string` | No | Publicly accessible greeting audio URL |
 | `greetingConfigs` | `LlmGreetingConfigs` | No | Greeting playback configuration |
 | `templateVariables` | `Record<string, string>` | No | Template variables for messages |
+| `tools` | `LlmTool[]` | No | Inline synchronous REST function-calling tools; separate from MCP. Requires `agent.withTools(true)`. |
 
 For supported Agora-managed models, `apiKey` is optional:
 
@@ -376,18 +377,24 @@ When `.withStt()` is omitted, AgentKit now selects the wire ASR vendor from `cli
 | `MicrosoftSTT` | `key`, `region`, `language` |
 | `OpenAISTT` | `apiKey`, `model?`, `language?`, `prompt?`, `inputAudioTranscription?` |
 | `GoogleSTT` | `projectId`, `location`, `adcCredentialsString`, `language`, `model?` |
+| `GeminiSTT` | `apiKey`, `model`, `sampleRate?`, `language?`, `wordTimestamp?`, `additionalParams?` |
 | `AmazonSTT` | `accessKey`, `secretKey`, `region`, `language` |
 | `AssemblyAISTT` | `apiKey`, `language`, `ws_url?` |
 | `AresSTT` | `keywords?`, `additionalParams?` |
 | `SarvamSTT` | `apiKey`, `language` |
 | `XAiSTT` | `apiKey`, `language?`, `baseUrl?`, `sampleRate?`, `additionalParams?` |
 
+`AresSTT` and `FengmingSTT` serialize `keywords` to the top-level ASR field
+(`asr.keywords`). Additional provider fields remain under `asr.params`.
+
 ---
 
 ## MLLM vendors
 
 ### OpenAIRealtime
 
+The legacy OpenAI Realtime wrapper emits `mllm.vendor = 'openai'`.
+
 <!-- snippet: fragment -->
 ```typescript
 new OpenAIRealtime(options: OpenAIRealtimeOptions)
@@ -406,6 +413,30 @@ new OpenAIRealtime(options: OpenAIRealtimeOptions)
 | `params` | `Record<string, unknown>` | No | Additional MLLM parameters |
 | `turnDetection` | `MllmTurnDetectionConfig` | No | MLLM turn detection configuration; overrides top-level `turn_detection` |
 
+### OpenAIGptLive
+
+`OpenAIGptLive` is a separate vendor for `mllm.vendor = 'openai_gpt_live'`.
+Its `greetingMessage` option is serialized to `mllm.greeting_message`.
+This vendor does not use the generated `greeting` field.
+
+<!-- snippet: fragment -->
+```typescript
+new OpenAIGptLive(options: OpenAIGptLiveOptions)
+```
+
+| Option | Type | Required | Description |
+|---|---|---|---|
+| `apiKey` | `string` | Yes | OpenAI API key |
+| `model` | `string` | No | Model name |
+| `url` | `string` | No | WebSocket URL (defaults to `wss://api.openai.com/v1/live`) |
+| `greetingMessage` | `string` | No | MLLM greeting sent as `mllm.greeting_message` |
+| `failureMessage` | `string` | No | Message played when the model call fails |
+| `inputModalities` | `string[]` | No | Input modalities |
+| `outputModalities` | `string[]` | No | Output modalities |
+| `messages` | `Record<string, unknown>[]` | No | Conversation messages |
+| `params` | `Record<string, unknown>` | No | Additional MLLM parameters |
+| `turnDetection` | `MllmTurnDetectionConfig` | No | MLLM turn detection configuration |
+
 ### AzureOpenAIRealtime
 
 <!-- snippet: fragment -->
@@ -431,7 +462,6 @@ Global Azure OpenAI Realtime wrapper. It emits `mllm.vendor = 'azure'`; `maxHist
 
 ### GeminiLive
 
-<!-- snippet: fragment -->
 ```typescript
 new GeminiLive(options: GeminiLiveOptions)
 ```

src/agentkit/Agent.ts

@@ -441,7 +441,8 @@ export class Agent<TTSSampleRate extends number = number, TArea extends AgoraAre
     }
 
     /**
-     * Returns a new Agent with MCP tool invocation enabled or disabled.
+     * Returns a new Agent with MCP and inline LLM tool invocation enabled or disabled.
+     * Both `llm.mcp_servers` and `llm.tools` require `advanced_features.enable_tools`.
      */
     withTools(enabled = true): Agent<TTSSampleRate, TArea> {
         const newAgent = this._clone();

src/agentkit/constants.ts

@@ -40,6 +40,10 @@ export const Geofence: typeof StartAgentsRequestNS.Properties.Geofence = StartAg
 export const FillerWordsSelectionRule: typeof StartAgentsRequestNS.Properties.FillerWords.Content.StaticConfig.SelectionRule =
     StartAgentsRequestNS.Properties.FillerWords.Content.StaticConfig.SelectionRule;
 
+/** Filler word content mode: `static` | `generated`. */
+export const FillerWordsContentModeValues: typeof StartAgentsRequestNS.Properties.FillerWords.Content.Mode =
+    StartAgentsRequestNS.Properties.FillerWords.Content.Mode;
+
 /**
  * Turn detection type (deprecated; use `TurnDetectionNestedConfig.EndOfSpeech` instead).
  * - `AgoraVad`: Agora VAD
@@ -109,6 +113,10 @@ export const ThinkOnListeningActionInterrupt: AgentThinkRequestNS.OnListeningAct
 export const ThinkOnListeningActionIgnore: AgentThinkRequestNS.OnListeningAction =
     AgentThinkRequestNS.OnListeningAction.Ignore;
 
+/** Queue the think instruction until the current listening turn completes. */
+export const ThinkOnListeningActionAppend: AgentThinkRequestNS.OnListeningAction =
+    AgentThinkRequestNS.OnListeningAction.Append;
+
 /** Think action when the agent is thinking: interrupt and start a new turn. */
 export const ThinkOnThinkingActionInterrupt: AgentThinkRequestNS.OnThinkingAction =
     AgentThinkRequestNS.OnThinkingAction.Interrupt;
@@ -117,10 +125,18 @@ export const ThinkOnThinkingActionInterrupt: AgentThinkRequestNS.OnThinkingActio
 export const ThinkOnThinkingActionIgnore: AgentThinkRequestNS.OnThinkingAction =
     AgentThinkRequestNS.OnThinkingAction.Ignore;
 
+/** Queue the think instruction until the current LLM turn completes. */
+export const ThinkOnThinkingActionAppend: AgentThinkRequestNS.OnThinkingAction =
+    AgentThinkRequestNS.OnThinkingAction.Append;
+
 /** Think action when the agent is speaking: interrupt and start a new turn. */
 export const ThinkOnSpeakingActionInterrupt: AgentThinkRequestNS.OnSpeakingAction =
     AgentThinkRequestNS.OnSpeakingAction.Interrupt;
 
 /** Think action when the agent is speaking: ignore the request. */
 export const ThinkOnSpeakingActionIgnore: AgentThinkRequestNS.OnSpeakingAction =
     AgentThinkRequestNS.OnSpeakingAction.Ignore;
+
+/** Queue the think instruction until the current TTS playback completes. */
+export const ThinkOnSpeakingActionAppend: AgentThinkRequestNS.OnSpeakingAction =
+    AgentThinkRequestNS.OnSpeakingAction.Append;

src/agentkit/index.ts

@@ -60,6 +60,7 @@ export {
     CredentialMode,
     DataChannel,
     FillerWordsSelectionRule,
+    FillerWordsContentModeValues,
     Geofence,
     InterruptionDisabledStrategy,
     InterruptionDisabledStrategyAppend,
@@ -78,11 +79,14 @@ export {
     SpeakPriorityInterrupt,
     SpeakPriorityValues,
     ThinkOnListeningActionIgnore,
+    ThinkOnListeningActionAppend,
     ThinkOnListeningActionInject,
     ThinkOnListeningActionInterrupt,
     ThinkOnSpeakingActionIgnore,
+    ThinkOnSpeakingActionAppend,
     ThinkOnSpeakingActionInterrupt,
     ThinkOnThinkingActionIgnore,
+    ThinkOnThinkingActionAppend,
     ThinkOnThinkingActionInterrupt,
     TurnDetectionTypeValues,
 } from "./constants.js";
@@ -158,6 +162,9 @@ export type {
     FillerWordsContent,
     FillerWordsContentSelectionRule,
     FillerWordsContentStaticConfig,
+    FillerWordsContentGeneratedConfig,
+    FillerWordsContentGeneratedLlmProvider,
+    FillerWordsContentMode,
     FillerWordsTrigger,
     FillerWordsTriggerFixedTimeConfig,
     FishAudioTts,
@@ -180,6 +187,10 @@ export type {
     Labels,
     // Core configuration types
     LlmConfig,
+    LlmTool,
+    LlmToolExecution,
+    LlmToolFunction,
+    LlmToolServer,
     LlmGreetingConfigs,
     LlmGreetingConfigsMode,
     LlmStyle,
@@ -229,6 +240,7 @@ export type {
     StartOfSpeechMode,
     StartOfSpeechVadConfig,
     SttConfig,
+    GeminiAsrParams,
     ThinkOnListeningAction,
     ThinkOnSpeakingAction,
     ThinkOnThinkingAction,
@@ -357,6 +369,7 @@ export type {
     AzureOpenAIRealtimeParams,
     GeminiLiveOptions,
     OpenAIRealtimeOptions,
+    OpenAIGptLiveOptions,
     QwenOmniOptions,
     VertexAIOptions,
     XaiGrokOptions,
@@ -366,6 +379,7 @@ export {
     AzureOpenAIRealtime,
     GeminiLive,
     OpenAIRealtime,
+    OpenAIGptLive,
     QwenOmni,
     VertexAI,
     XaiGrok,
@@ -375,6 +389,7 @@ export type {
     AresSTTOptions,
     AssemblyAISTTOptions,
     DeepgramSTTOptions,
+    GeminiSTTOptions,
     GoogleSTTOptions,
     MicrosoftSTTOptions,
     OpenAISTTOptions,
@@ -388,6 +403,7 @@ export {
     AresSTT,
     AssemblyAISTT,
     DeepgramSTT,
+    GeminiSTT,
     GoogleSTT,
     MicrosoftSTT,
     OpenAISTT,