From d6053ce5d3e0a3f7261a75a2a4242372ef8ac0fc Mon Sep 17 00:00:00 2001 From: jinhaosong-source <287574038+jinhaosong-source@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:19:59 +0800 Subject: [PATCH] feat: add orcarouter llm2 extension OrcaRouter is an OpenAI-compatible meta-router, so this extension reuses the OpenAI SDK and only changes defaults: base_url is https://api.orcarouter.ai/v1 and model is orcarouter/auto. It supports streaming, tool calling, reasoning output, and custom attribution headers, and passes routing controls (models + route: fallback) through request parameters. Register it in the voice-assistant example dependency list and document ORCAROUTER_* in .env.example. --- ai_agents/.env.example | 6 + .../voice-assistant/tenapp/manifest.json | 3 + .../orcarouter_llm2_python/README.md | 67 +++ .../orcarouter_llm2_python/__init__.py | 8 + .../extension/orcarouter_llm2_python/addon.py | 22 + .../orcarouter_llm2_python/extension.py | 80 +++ .../orcarouter_llm2_python/manifest.json | 57 +++ .../orcarouter_llm2_python/orcarouter.py | 464 ++++++++++++++++++ .../orcarouter_llm2_python/property.json | 11 + .../orcarouter_llm2_python/pyproject.toml | 10 + .../orcarouter_llm2_python/requirements.txt | 4 + .../orcarouter_llm2_python/tests/__init__.py | 5 + .../orcarouter_llm2_python/tests/bin/start | 9 + .../tests/configs/property_basic.json | 7 + .../tests/configs/property_miss_required.json | 5 + .../orcarouter_llm2_python/tests/conftest.py | 99 ++++ .../tests/test_config.py | 42 ++ .../tests/test_think_parser.py | 78 +++ .../orcarouter_llm2_python/think_parser.py | 137 ++++++ 19 files changed, 1114 insertions(+) create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/README.md create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/__init__.py create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/addon.py create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/extension.py create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/manifest.json create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/orcarouter.py create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/property.json create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/pyproject.toml create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/requirements.txt create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/__init__.py create mode 100755 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/bin/start create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/configs/property_basic.json create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/configs/property_miss_required.json create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/conftest.py create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/test_config.py create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/test_think_parser.py create mode 100644 ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/think_parser.py diff --git a/ai_agents/.env.example b/ai_agents/.env.example index 3fcf728744..432e9a2a68 100644 --- a/ai_agents/.env.example +++ b/ai_agents/.env.example @@ -45,6 +45,12 @@ OPENAI_API_KEY= OPENAI_MODEL=gpt-4o OPENAI_PROXY_URL= +# Extension: orcarouter_llm2_python +# OrcaRouter API key (keys start with sk-orca-) +ORCAROUTER_API_KEY= +# Router or model ID, e.g. orcarouter/auto, orcarouter/fusion, openai/gpt-4o +ORCAROUTER_MODEL=orcarouter/auto + # Extension: grok_python GROK_API_BASE=https://api.x.ai/v1 GROK_API_KEY= diff --git a/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json b/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json index c8d994c78b..dc563eb65b 100644 --- a/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json +++ b/ai_agents/agents/examples/voice-assistant/tenapp/manifest.json @@ -87,6 +87,9 @@ { "path": "../../../ten_packages/extension/openai_llm2_python" }, + { + "path": "../../../ten_packages/extension/orcarouter_llm2_python" + }, { "path": "../../../ten_packages/extension/azure_tts_python" }, diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/README.md b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/README.md new file mode 100644 index 0000000000..1e2c263f7b --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/README.md @@ -0,0 +1,67 @@ +# orcarouter_llm2_python + +An extension for integrating [OrcaRouter](https://www.orcarouter.ai) into your +application. OrcaRouter is an OpenAI-compatible meta-router: a single endpoint +that fronts many upstream models and adds server-side routing plus a +per-request model-fallback chain. Because it speaks the OpenAI Chat Completions +protocol, this extension reuses the OpenAI SDK and only changes the defaults. + +> Disclosure: I'm an engineer on the OrcaRouter team. + +## Features + +- OpenAI-compatible: chat completions, streaming, tool/function calling, and + reasoning (`` blocks and `reasoning_content`) all work unchanged. +- Automatic routing: point `model` at `orcarouter/auto` and OrcaRouter picks a + live model per request (the routing strategy — cheapest / quality / balanced + / adaptive — is configured in the OrcaRouter console). +- Model-fallback chain: pass `models` + `route: "fallback"` through request + parameters to try several models in order until one succeeds. +- Attribution: `custom_headers` forwards `HTTP-Referer` / `X-Title` so the + OrcaRouter console can report which client is calling. + +## Configuration + +Set your OrcaRouter API key (keys start with `sk-orca-`): + +```bash +export ORCAROUTER_API_KEY="sk-orca-..." +# optional; defaults to orcarouter/auto +export ORCAROUTER_MODEL="orcarouter/auto" +``` + +Example model IDs: `orcarouter/auto` (per-request routing), `orcarouter/fusion` +(a Fusion panel), or any provider-prefixed model such as `openai/gpt-4o`, +`anthropic/claude-haiku-4.5`, `google/gemini-2.5-pro`. See the full catalog at +https://www.orcarouter.ai/models. + +### Model-fallback chain + +OrcaRouter tries each model in order until one succeeds (up to 5). Pass the +controls through the request `parameters`: + +```json +{ + "models": ["openai/gpt-4o", "anthropic/claude-haiku-4.5"], + "route": "fallback" +} +``` + +## API + +Refer to the `api` definition in [manifest.json] and default values in +[property.json](property.json). + +| **Property** | **Type** | **Description** | +|------------------|------------|------------------------------------------------------------------------| +| `api_key` | `string` | OrcaRouter API key (`sk-orca-...`), sent as a Bearer token | +| `base_url` | `string` | API base URL (default `https://api.orcarouter.ai/v1`) | +| `model` | `string` | Model / router ID (default `orcarouter/auto`) | +| `prompt` | `string` | Default system prompt | +| `proxy_url` | `string` | Optional HTTP(S) proxy URL | +| `custom_headers` | `object` | Extra request headers (e.g. `HTTP-Referer`, `X-Title` for attribution) | + +### Data In / Data Out / Command In / Command Out + +Same as the other LLM2 extensions (`text_data` in/out, `flush` command), +provided by the shared `llm-interface.json` contract. diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/__init__.py b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/__init__.py new file mode 100644 index 0000000000..0104112782 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/__init__.py @@ -0,0 +1,8 @@ +# +# +# Agora Real Time Engagement +# OrcaRouter LLM2 Integration +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +from . import addon diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/addon.py b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/addon.py new file mode 100644 index 0000000000..c77c9f8e99 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/addon.py @@ -0,0 +1,22 @@ +# +# +# Agora Real Time Engagement +# OrcaRouter LLM2 Integration +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +from ten_runtime import ( + Addon, + register_addon_as_extension, + TenEnv, +) + + +@register_addon_as_extension("orcarouter_llm2_python") +class OrcaRouterLLM2ExtensionAddon(Addon): + + def on_create_instance(self, ten_env: TenEnv, name: str, context) -> None: + from .extension import OrcaRouterLLM2Extension + + ten_env.log_info("OrcaRouterLLM2ExtensionAddon on_create_instance") + ten_env.on_create_instance_done(OrcaRouterLLM2Extension(name), context) diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/extension.py b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/extension.py new file mode 100644 index 0000000000..457ed886cd --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/extension.py @@ -0,0 +1,80 @@ +# +# +# Agora Real Time Engagement +# OrcaRouter LLM2 Integration +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +import asyncio +from typing import AsyncGenerator + +from ten_ai_base.llm2 import AsyncLLM2BaseExtension +from ten_ai_base.struct import ( + LLMRequest, + LLMRequestRetrievePrompt, + LLMResponse, + LLMResponseRetrievePrompt, +) +from ten_runtime.async_ten_env import AsyncTenEnv + +from .orcarouter import OrcaRouterLLM, OrcaRouterLLM2Config + + +class OrcaRouterLLM2Extension(AsyncLLM2BaseExtension): + def __init__(self, name: str): + super().__init__(name) + self.memory = [] + self.memory_cache = [] + self.config = None + self.client = None + self.sentence_fragment = "" + self.tool_task_future: asyncio.Future | None = None + self.users_count = 0 + self.last_reasoning_ts = 0 + + async def on_init(self, ten_env: AsyncTenEnv) -> None: + ten_env.log_info("on_init") + await super().on_init(ten_env) + + async def on_start(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.log_info("on_start") + await super().on_start(async_ten_env) + config_json, _ = await self.ten_env.get_property_to_json("") + self.config = OrcaRouterLLM2Config.model_validate_json(config_json) + + # Mandatory properties + if not self.config.api_key: + async_ten_env.log_info("API key is missing, exiting on_start") + return + + # Create instance + try: + self.client = OrcaRouterLLM(async_ten_env, self.config) + async_ten_env.log_info( + f"initialized with max_tokens: {self.config.max_tokens}, model: {self.config.model}" + ) + except Exception as err: + async_ten_env.log_info(f"Failed to initialize OrcaRouterLLM: {err}") + + async def on_stop(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.log_info("on_stop") + await super().on_stop(async_ten_env) + + async def on_deinit(self, async_ten_env: AsyncTenEnv) -> None: + async_ten_env.log_info("on_deinit") + await super().on_deinit(async_ten_env) + + async def on_retrieve_prompt( + self, async_ten_env: AsyncTenEnv, request: LLMRequestRetrievePrompt + ) -> LLMResponseRetrievePrompt: + """Retrieve the current prompt from config.""" + prompt = self.config.prompt if self.config else "" + async_ten_env.log_info( + f"Retrieved prompt for request_id: {request.request_id}" + ) + return LLMResponseRetrievePrompt(prompt=prompt) + + def on_call_chat_completion( + self, async_ten_env: AsyncTenEnv, request_input: LLMRequest + ) -> AsyncGenerator[LLMResponse, None]: + return self.client.get_chat_completions(request_input) diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/manifest.json b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/manifest.json new file mode 100644 index 0000000000..7c53174e6d --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/manifest.json @@ -0,0 +1,57 @@ +{ + "type": "extension", + "name": "orcarouter_llm2_python", + "version": "0.1.0", + "dependencies": [ + { + "type": "system", + "name": "ten_runtime_python", + "version": "0.11" + }, + { + "type": "system", + "name": "ten_ai_base", + "version": "0.7" + } + ], + "package": { + "include": [ + "manifest.json", + "property.json", + "**.py", + "README.md", + "pyproject.toml", + "requirements.txt" + ] + }, + "api": { + "interface": [ + { + "import_uri": "../../system/ten_ai_base/api/llm-interface.json" + } + ], + "property": { + "properties": { + "api_key": { + "type": "string" + }, + "model": { + "type": "string" + }, + "base_url": { + "type": "string" + }, + "proxy_url": { + "type": "string" + }, + "prompt": { + "type": "string" + }, + "custom_headers": { + "type": "object", + "properties": {} + } + } + } + } +} diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/orcarouter.py b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/orcarouter.py new file mode 100644 index 0000000000..fcd5b21f6d --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/orcarouter.py @@ -0,0 +1,464 @@ +# +# +# Agora Real Time Engagement +# OrcaRouter LLM2 Integration +# Copyright (c) 2024 Agora IO. All rights reserved. +# +# +from collections import defaultdict +from dataclasses import dataclass, field +from enum import Enum +import json +import random +from typing import Any, AsyncGenerator, List +from pydantic import BaseModel +import httpx +from openai import AsyncOpenAI, AsyncStream +from openai.types.chat import ChatCompletionChunk + +from ten_ai_base.struct import ( + ImageContent, + LLMMessageContent, + LLMMessageFunctionCall, + LLMMessageFunctionCallOutput, + LLMRequest, + LLMResponse, + LLMResponseMessageDelta, + LLMResponseMessageDone, + LLMResponseReasoningDelta, + LLMResponseReasoningDone, + LLMResponseToolCall, + TextContent, +) +from ten_ai_base.types import LLMToolMetadata +from ten_runtime.async_ten_env import AsyncTenEnv + +from .think_parser import ThinkParser + + +@dataclass +class OrcaRouterLLM2Config(BaseModel): + api_key: str = "" + base_url: str = "https://api.orcarouter.ai/v1" + # `orcarouter/auto` is OrcaRouter's per-account router: it picks a live + # model for each request (strategy is configured in the OrcaRouter + # console). Any provider-prefixed model ID also works, e.g. + # "openai/gpt-4o" or "orcarouter/fusion". + model: str = "orcarouter/auto" + proxy_url: str = "" + temperature: float = 0.7 + top_p: float = 1.0 + presence_penalty: float = 0.0 + frequency_penalty: float = 0.0 + max_tokens: int = 4096 + seed: int = random.randint(0, 1000000) + prompt: str = "You are a helpful assistant." + custom_headers: dict[str, Any] = field(default_factory=dict) + black_list_params: List[str] = field( + default_factory=lambda: ["messages", "tools", "stream", "n", "model"] + ) + + def is_black_list_params(self, key: str) -> bool: + return key in self.black_list_params + + +class ReasoningMode(str, Enum): + ModeV1 = "v1" + + +class OrcaRouterLLM: + client = None + + def __init__(self, ten_env: AsyncTenEnv, config: OrcaRouterLLM2Config): + self.config = config + self.ten_env = ten_env + ten_env.log_info( + f"OrcaRouterLLM initialized with model: {config.model} " + f"(base_url={config.base_url})" + ) + self.http_client = None + if config.proxy_url: + ten_env.log_info(f"Setting httpx proxy: {config.proxy_url}") + self.http_client = httpx.AsyncClient(proxy=config.proxy_url) + + # OrcaRouter is OpenAI-compatible and authenticates with a Bearer + # token. `custom_headers` lets callers add attribution headers + # (HTTP-Referer / X-Title) so the OrcaRouter console can report which + # client is calling; a caller-supplied value always wins. + default_headers = { + "Authorization": f"Bearer {config.api_key}", + } + for key, value in config.custom_headers.items(): + if isinstance(value, (dict, list)): + ten_env.log_warn( + f"Skipping custom header '{key}':" + f" value must be a scalar, got {type(value).__name__}" + ) + continue + default_headers[str(key)] = str(value) + + self.client = AsyncOpenAI( + api_key=config.api_key, + base_url=config.base_url, + default_headers=default_headers, + http_client=self.http_client, + ) + + def _convert_tools_to_dict(self, tool: LLMToolMetadata): + json_dict = { + "type": "function", + "function": { + "name": tool.name, + "description": tool.description, + "parameters": { + "type": "object", + "properties": {}, + "required": [], + "additionalProperties": False, + }, + }, + "strict": True, + } + + for param in tool.parameters: + json_dict["function"]["parameters"]["properties"][param.name] = { + "type": param.type, + "description": param.description, + } + if param.required: + json_dict["function"]["parameters"]["required"].append( + param.name + ) + if param.type == "array": + json_dict["function"]["parameters"]["properties"][param.name][ + "items" + ] = param.items + + return json_dict + + async def get_chat_completions( + self, request_input: LLMRequest + ) -> AsyncGenerator[LLMResponse, None]: + messages = request_input.messages + tools = None + parsed_messages = [] + system_prompt = request_input.prompt or self.config.prompt + + self.ten_env.log_info( + f"get_chat_completions: {len(messages)} messages, streaming: {request_input.streaming}" + ) + + for message in messages: + match message: + case LLMMessageContent(): + role = message.role + content = message.content + if isinstance(content, str): + parsed_messages.append( + {"role": role, "content": content} + ) + elif isinstance(content, list): + # Assuming content is a list of objects + content_items = [] + for item in content: + match item: + case TextContent(): + content_items.append( + {"type": "text", "text": item.text} + ) + case ImageContent(): + content_items.append( + { + "type": "image_url", + "image_url": { + "url": item.image_url.url + }, + } + ) + parsed_messages.append( + {"role": role, "content": content_items} + ) + case LLMMessageFunctionCall(): + # Handle function call messages + parsed_messages.append( + { + "role": "assistant", + "tool_calls": [ + { + "id": message.call_id, + "type": "function", + "function": { + "name": message.name, + "arguments": message.arguments, + }, + } + ], + } + ) + case LLMMessageFunctionCallOutput(): + # Handle function call output messages + parsed_messages.append( + { + "role": "tool", + "tool_call_id": message.call_id, + "content": message.output, + } + ) + + for tool in request_input.tools or []: + if tools is None: + tools = [] + tools.append(self._convert_tools_to_dict(tool)) + + # Build request. OrcaRouter normalizes provider-specific parameter + # constraints server-side (e.g. sampling params such as `temperature` + # are accepted even when a request is routed to a reasoning model), so + # we always send the standard OpenAI chat-completion parameters. + req = { + "model": self.config.model, + "messages": [ + {"role": "system", "content": system_prompt}, + *parsed_messages, + ], + "tools": tools, + "stream": request_input.streaming, + "n": 1, # Assuming single response for now + "max_tokens": self.config.max_tokens, + "temperature": self.config.temperature, + "top_p": self.config.top_p, + "presence_penalty": self.config.presence_penalty, + "frequency_penalty": self.config.frequency_penalty, + "seed": self.config.seed, + } + + # Add additional parameters if they are not in the black list. + # This is also how OrcaRouter routing controls are passed, e.g. + # parameters={"models": ["openai/gpt-4o", "anthropic/claude-haiku-4.5"], + # "route": "fallback"} enables a model-fallback chain. + for key, value in (request_input.parameters or {}).items(): + # Check if it's a valid option and not in black list + if not self.config.is_black_list_params(key): + self.ten_env.log_debug(f"set orcarouter param: {key} = {value}") + req[key] = value + + self.ten_env.log_info(f"Requesting chat completions with: {req}") + + try: + response: AsyncStream[ChatCompletionChunk] = ( + await self.client.chat.completions.create(**req) + ) + + full_content = "" + # Check for tool calls + tool_calls_dict = defaultdict( + lambda: { + "id": None, + "function": {"arguments": "", "name": None}, + "type": None, + } + ) + + parser = ThinkParser() + reasoning_mode = None + reasoning_full_content = "" + + last_chat_completion: ChatCompletionChunk | None = None + + async for chat_completion in response: + self.ten_env.log_debug(f"Chat completion: {chat_completion}") + if chat_completion is None or len(chat_completion.choices) == 0: + continue + last_chat_completion = chat_completion + choice = chat_completion.choices[0] + delta = choice.delta + + self.ten_env.log_debug(f"Processing choice: {choice}") + + content = delta.content if delta and delta.content else "" + raw_reasoning_content = ( + delta.reasoning_content + if delta and hasattr(delta, "reasoning_content") + else None + ) + reasoning_content = raw_reasoning_content or "" + + if reasoning_mode is None and raw_reasoning_content is not None: + reasoning_mode = ReasoningMode.ModeV1 + + if reasoning_mode == ReasoningMode.ModeV1: + if reasoning_content: + for ( + event_type, + event_value, + ) in parser.process_reasoning_content( + reasoning_content + ): + if event_type == "reasoning_delta": + # Use a local accumulator instead of parser state + # because parser.think_content can be reset when + # reasoning_done is emitted in the same cycle. + reasoning_full_content += event_value + yield LLMResponseReasoningDelta( + response_id=chat_completion.id, + role="assistant", + content=reasoning_full_content, + delta=event_value, + created=chat_completion.created, + ) + elif parser.state == "THINK": + for ( + event_type, + event_value, + ) in parser.process_reasoning_content(""): + if event_type == "reasoning_done": + yield LLMResponseReasoningDone( + response_id=chat_completion.id, + role="assistant", + content=event_value, + created=chat_completion.created, + ) + reasoning_full_content = "" + + if content: + full_content += content + yield LLMResponseMessageDelta( + response_id=chat_completion.id, + role="assistant", + content=full_content, + delta=content, + created=chat_completion.created, + ) + elif content: + for event_type, event_value in parser.process_content( + content + ): + if event_type == "message_delta": + full_content += event_value + yield LLMResponseMessageDelta( + response_id=chat_completion.id, + role="assistant", + content=full_content, + delta=event_value, + created=chat_completion.created, + ) + elif event_type == "reasoning_delta": + # Keep reasoning delta content cumulative and stable. + reasoning_full_content += event_value + yield LLMResponseReasoningDelta( + response_id=chat_completion.id, + role="assistant", + content=reasoning_full_content, + delta=event_value, + created=chat_completion.created, + ) + elif event_type == "reasoning_done": + yield LLMResponseReasoningDone( + response_id=chat_completion.id, + role="assistant", + content=event_value, + created=chat_completion.created, + ) + reasoning_full_content = "" + + if delta.tool_calls: + try: + for tool_call in delta.tool_calls: + self.ten_env.log_info(f"Tool call: {tool_call}") + if tool_call.index not in tool_calls_dict: + tool_calls_dict[tool_call.index] = { + "id": None, + "function": {"arguments": "", "name": None}, + "type": None, + } + + if tool_call.id: + tool_calls_dict[tool_call.index][ + "id" + ] = tool_call.id + + # If the function name is not None, set it + if tool_call.function.name: + tool_calls_dict[tool_call.index]["function"][ + "name" + ] = tool_call.function.name + + # Append the arguments if not None + if tool_call.function.arguments: + tool_calls_dict[tool_call.index]["function"][ + "arguments" + ] += tool_call.function.arguments + + # If the type is not None, set it + if tool_call.type: + tool_calls_dict[tool_call.index][ + "type" + ] = tool_call.type + except Exception as e: + import traceback + + traceback.print_exc() + self.ten_env.log_error( + f"Error processing tool call: {e} {tool_calls_dict}" + ) + + if last_chat_completion is None: + self.ten_env.log_info("No chat completion choices found.") + return + + for event_type, event_value in parser.finalize(): + if event_type == "message_delta": + full_content += event_value + yield LLMResponseMessageDelta( + response_id=last_chat_completion.id, + role="assistant", + content=full_content, + delta=event_value, + created=last_chat_completion.created, + ) + elif event_type == "reasoning_delta": + # Keep reasoning delta content cumulative and stable. + reasoning_full_content += event_value + yield LLMResponseReasoningDelta( + response_id=last_chat_completion.id, + role="assistant", + content=reasoning_full_content, + delta=event_value, + created=last_chat_completion.created, + ) + elif event_type == "reasoning_done": + yield LLMResponseReasoningDone( + response_id=last_chat_completion.id, + role="assistant", + content=event_value, + created=last_chat_completion.created, + ) + reasoning_full_content = "" + + # Convert the dictionary to a list + tool_calls_list = list(tool_calls_dict.values()) + + # Emit tool calls event (fire-and-forget) + if tool_calls_list: + for tool_call in tool_calls_list: + arguements = json.loads(tool_call["function"]["arguments"]) + self.ten_env.log_info( + f"Tool call: {choice.delta.model_dump_json()}" + ) + yield LLMResponseToolCall( + response_id=last_chat_completion.id, + id=last_chat_completion.id, + tool_call_id=tool_call["id"], + name=tool_call["function"]["name"], + arguments=arguements, + created=last_chat_completion.created, + ) + + # Emit content finished event after the loop completes + yield LLMResponseMessageDone( + response_id=last_chat_completion.id, + role="assistant", + content=full_content, + created=last_chat_completion.created, + ) + except Exception as e: + raise RuntimeError(f"CreateChatCompletion failed, err: {e}") from e diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/property.json b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/property.json new file mode 100644 index 0000000000..4860128c55 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/property.json @@ -0,0 +1,11 @@ +{ + "base_url": "https://api.orcarouter.ai/v1", + "api_key": "${env:ORCAROUTER_API_KEY}", + "model": "${env:ORCAROUTER_MODEL|orcarouter/auto}", + "max_tokens": 512, + "prompt": "", + "custom_headers": { + "HTTP-Referer": "https://www.orcarouter.ai", + "X-Title": "TEN Framework" + } +} diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/pyproject.toml b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/pyproject.toml new file mode 100644 index 0000000000..02f07c6665 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/pyproject.toml @@ -0,0 +1,10 @@ +[project] +name = "orcarouter-llm2-python" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = [ + "numpy>=2.2.6", + "openai>=2.44.0", + "pillow>=12.2.0", + "requests[socks]>=2.34.2", +] diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/requirements.txt b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/requirements.txt new file mode 100644 index 0000000000..8ff31f9a9b --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/requirements.txt @@ -0,0 +1,4 @@ +openai +numpy +requests[socks] +pillow diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/__init__.py b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/__init__.py new file mode 100644 index 0000000000..da402faf43 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/__init__.py @@ -0,0 +1,5 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/bin/start b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/bin/start new file mode 100755 index 0000000000..8e78210572 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/bin/start @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." + +export PYTHONPATH=.ten/app:.ten/app/ten_packages/system/ten_runtime_python/lib:.ten/app/ten_packages/system/ten_runtime_python/interface:.ten/app/ten_packages/system/ten_ai_base/interface:$PYTHONPATH + +pytest -s tests/ "$@" diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/configs/property_basic.json b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/configs/property_basic.json new file mode 100644 index 0000000000..8feab7fbdd --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/configs/property_basic.json @@ -0,0 +1,7 @@ +{ + "base_url": "https://api.orcarouter.ai/v1", + "api_key": "${env:ORCAROUTER_API_KEY|sk-orca-test}", + "model": "orcarouter/auto", + "max_tokens": 512, + "prompt": "You are a helpful assistant." +} diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/configs/property_miss_required.json b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/configs/property_miss_required.json new file mode 100644 index 0000000000..fdb057a779 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/configs/property_miss_required.json @@ -0,0 +1,5 @@ +{ + "base_url": "https://api.orcarouter.ai/v1", + "api_key": "", + "model": "orcarouter/auto" +} diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/conftest.py b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/conftest.py new file mode 100644 index 0000000000..001977148c --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/conftest.py @@ -0,0 +1,99 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +import json +import threading +from typing_extensions import override +import pytest +from ten_runtime import ( + App, + TenEnv, +) + + +class FakeApp(App): + def __init__(self): + super().__init__() + self.event: threading.Event | None = None + + # In the case of a fake app, we use `on_init` to allow the blocked testing + # fixture to continue execution, rather than using `on_configure`. The + # reason is that in the TEN runtime C core, the relationship between the + # addon manager and the (fake) app is bound after `on_configure_done` is + # called. So we only need to let the testing fixture continue execution + # after this action in the TEN runtime C core, and at the upper layer + # timing, the earliest point is within the `on_init()` function of the upper + # TEN app. Therefore, we release the testing fixture lock within the user + # layer's `on_init()` of the TEN app. + @override + def on_init(self, ten_env: TenEnv) -> None: + assert self.event + self.event.set() + + ten_env.on_init_done() + + @override + def on_configure(self, ten_env: TenEnv) -> None: + ten_env.init_property_from_json( + json.dumps( + { + "ten": { + "log": { + "handlers": [ + { + "matchers": [{"level": "debug"}], + "formatter": { + "type": "plain", + "colored": True, + }, + "emitter": { + "type": "console", + "config": {"stream": "stdout"}, + }, + } + ] + } + } + } + ), + ) + + ten_env.on_configure_done() + + +class FakeAppCtx: + def __init__(self, event: threading.Event): + self.fake_app: FakeApp | None = None + self.event = event + + +def run_fake_app(fake_app_ctx: FakeAppCtx): + app = FakeApp() + app.event = fake_app_ctx.event + fake_app_ctx.fake_app = app + app.run(False) + + +@pytest.fixture(scope="session", autouse=True) +def global_setup_and_teardown(): + event = threading.Event() + fake_app_ctx = FakeAppCtx(event) + + fake_app_thread = threading.Thread( + target=run_fake_app, args=(fake_app_ctx,) + ) + fake_app_thread.start() + + event.wait() + + assert fake_app_ctx.fake_app is not None + + # Yield control to the test; after the test execution is complete, continue + # with the teardown process. + yield + + # Teardown part. + fake_app_ctx.fake_app.close() + fake_app_thread.join() diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/test_config.py b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/test_config.py new file mode 100644 index 0000000000..1709117289 --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/test_config.py @@ -0,0 +1,42 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +"""Config tests for the OrcaRouter LLM2 extension. + +Importing the extension module requires the TEN AI base package, so these +tests are skipped where it is unavailable (e.g. outside the built framework). +""" +import sys +from pathlib import Path + +import pytest + +_EXT_DIR = str(Path(__file__).resolve().parents[1]) +if _EXT_DIR not in sys.path: + sys.path.insert(0, _EXT_DIR) + +pytest.importorskip("ten_ai_base") + +from orcarouter import OrcaRouterLLM2Config # noqa: E402 + + +def test_default_base_url_and_model(): + cfg = OrcaRouterLLM2Config() + assert cfg.base_url == "https://api.orcarouter.ai/v1" + assert cfg.model == "orcarouter/auto" + + +def test_black_list_params(): + cfg = OrcaRouterLLM2Config() + for key in ("messages", "tools", "stream", "n", "model"): + assert cfg.is_black_list_params(key) + # Routing controls are NOT black-listed: they must pass through to the API. + assert not cfg.is_black_list_params("models") + assert not cfg.is_black_list_params("route") + + +def test_custom_headers_defaults_empty(): + cfg = OrcaRouterLLM2Config() + assert cfg.custom_headers == {} diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/test_think_parser.py b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/test_think_parser.py new file mode 100644 index 0000000000..45eea59d4c --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/tests/test_think_parser.py @@ -0,0 +1,78 @@ +# +# This file is part of TEN Framework, an open source project. +# Licensed under the Apache License, Version 2.0. +# See the LICENSE file for more information. +# +"""Unit tests for the streaming ... reasoning parser. + +These are pure-logic tests with no TEN runtime dependency, so they run in +any environment. +""" +import sys +from pathlib import Path + +# The extension package dir (parent of tests/) holds think_parser.py. +_EXT_DIR = str(Path(__file__).resolve().parents[1]) +if _EXT_DIR not in sys.path: + sys.path.insert(0, _EXT_DIR) + +from think_parser import ThinkParser # noqa: E402 + + +def _collect(parser: ThinkParser, chunks): + """Feed content chunks through the parser and return the flat event list + plus finalize() events.""" + events = [] + for chunk in chunks: + events.extend(parser.process_content(chunk)) + events.extend(parser.finalize()) + return events + + +def _join(events, event_type): + return "".join(v for t, v in events if t == event_type) + + +def test_plain_content_no_think(): + events = _collect(ThinkParser(), ["Hello ", "world"]) + assert _join(events, "message_delta") == "Hello world" + assert not any(t.startswith("reasoning") for t, _ in events) + + +def test_single_think_block(): + events = _collect( + ThinkParser(), ["before reasoning here after"] + ) + assert _join(events, "message_delta") == "before after" + assert _join(events, "reasoning_delta") == "reasoning here" + assert any(t == "reasoning_done" for t, _ in events) + + +def test_think_split_across_chunks(): + # The open/close tags are split across delta boundaries. + events = _collect( + ThinkParser(), + ["ans:hidden ", "thoughtdone"], + ) + assert _join(events, "message_delta") == "ans:done" + assert _join(events, "reasoning_delta") == "hidden thought" + assert any(t == "reasoning_done" for t, _ in events) + + +def test_unclosed_think_finalized(): + # An open with no closing tag should still flush on finalize(). + events = _collect(ThinkParser(), ["text still thinking"]) + assert _join(events, "message_delta") == "text " + assert _join(events, "reasoning_delta") == "still thinking" + assert any(t == "reasoning_done" for t, _ in events) + + +def test_reasoning_content_channel(): + # Providers that stream a dedicated reasoning_content field. + parser = ThinkParser() + events = [] + events.extend(parser.process_reasoning_content("step 1 ")) + events.extend(parser.process_reasoning_content("step 2")) + events.extend(parser.process_reasoning_content("")) # signals done + assert _join(events, "reasoning_delta") == "step 1 step 2" + assert any(t == "reasoning_done" for t, _ in events) diff --git a/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/think_parser.py b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/think_parser.py new file mode 100644 index 0000000000..d8728a4e9c --- /dev/null +++ b/ai_agents/agents/ten_packages/extension/orcarouter_llm2_python/think_parser.py @@ -0,0 +1,137 @@ +# +# Stream parser for extracting ... reasoning blocks from +# tokenized content deltas. +# +from __future__ import annotations + +from typing import List, Tuple + + +class ThinkParser: + OPEN_TAG = "" + CLOSE_TAG = "" + + def __init__(self): + self.state = "NORMAL" # States: 'NORMAL', 'THINK' + self.think_content = "" + self.think_delta = "" + self._pending = "" + + def _partial_suffix_len(self, text: str, tag: str) -> int: + max_len = min(len(text), len(tag) - 1) + for size in range(max_len, 0, -1): + if text.endswith(tag[:size]): + return size + return 0 + + def process_content(self, new_chars: str) -> List[Tuple[str, str]]: + events: List[Tuple[str, str]] = [] + if not new_chars: + return events + + data = self._pending + new_chars + self._pending = "" + idx = 0 + + while idx < len(data): + if self.state == "NORMAL": + open_pos = data.find(self.OPEN_TAG, idx) + if open_pos < 0: + partial = self._partial_suffix_len( + data[idx:], self.OPEN_TAG + ) + visible = ( + data[idx : len(data) - partial] + if partial + else data[idx:] + ) + if visible: + events.append(("message_delta", visible)) + if partial: + self._pending = data[len(data) - partial :] + break + + if open_pos > idx: + events.append(("message_delta", data[idx:open_pos])) + + self.state = "THINK" + self.think_delta = "" + idx = open_pos + len(self.OPEN_TAG) + continue + + close_pos = data.find(self.CLOSE_TAG, idx) + if close_pos < 0: + partial = self._partial_suffix_len(data[idx:], self.CLOSE_TAG) + think_text = ( + data[idx : len(data) - partial] if partial else data[idx:] + ) + if think_text: + self.think_content += think_text + self.think_delta = think_text + events.append(("reasoning_delta", think_text)) + if partial: + self._pending = data[len(data) - partial :] + break + + think_text = data[idx:close_pos] + if think_text: + self.think_content += think_text + self.think_delta = think_text + events.append(("reasoning_delta", think_text)) + + events.append(("reasoning_done", self.think_content)) + self.think_content = "" + self.think_delta = "" + self.state = "NORMAL" + idx = close_pos + len(self.CLOSE_TAG) + + return events + + def process_reasoning_content( + self, reasoning_content: str + ) -> List[Tuple[str, str]]: + events: List[Tuple[str, str]] = [] + if reasoning_content: + if self.state == "NORMAL": + self.state = "THINK" + self.think_content += reasoning_content + self.think_delta = reasoning_content + events.append(("reasoning_delta", reasoning_content)) + elif self.state == "THINK": + events.append(("reasoning_done", self.think_content)) + self.state = "NORMAL" + self.think_delta = "" + self.think_content = "" + return events + + def process(self, new_chars): + prev_state = self.state + events = self.process_content(new_chars) + return prev_state != self.state or any( + event_type == "reasoning_done" for event_type, _ in events + ) + + def process_by_reasoning_content(self, reasoning_content): + prev_state = self.state + events = self.process_reasoning_content(reasoning_content) + return prev_state != self.state or any( + event_type == "reasoning_done" for event_type, _ in events + ) + + def finalize(self) -> List[Tuple[str, str]]: + events: List[Tuple[str, str]] = [] + if self._pending: + if self.state == "NORMAL": + events.append(("message_delta", self._pending)) + else: + self.think_content += self._pending + self.think_delta = self._pending + events.append(("reasoning_delta", self._pending)) + self._pending = "" + + if self.state == "THINK": + events.append(("reasoning_done", self.think_content)) + self.state = "NORMAL" + self.think_delta = "" + self.think_content = "" + return events