Add strict LLM provider benchmarking

Author: WW-shan

.env.example

@@ -11,6 +11,13 @@ OPENAI_CHAT_STREAM=true
 OPENAI_CHAT_RESPONSE_FORMAT=json_object
 OPENAI_TEMPERATURE=0
 
+# Secondary provider, tried after primary and before backup.
+OPENAI_SECONDARY_API_KEY=
+OPENAI_SECONDARY_BASE_URL=
+OPENAI_SECONDARY_API_MODE=chat
+OPENAI_SECONDARY_MODEL=
+OPENAI_SECONDARY_TIMEOUT=60
+
 # Backup provider.
 OPENAI_BACKUP_API_KEY=
 OPENAI_BACKUP_BASE_URL=
@@ -50,4 +57,3 @@ POLYMARKET_CHAIN_ID=137
 POLYMARKET_CLOB_API_KEY=
 POLYMARKET_CLOB_API_SECRET=
 POLYMARKET_CLOB_PASSPHRASE=
-

docs/security.md

@@ -9,6 +9,7 @@
 ## Variables that deserve attention
 
 - `OPENAI_API_KEY`
+- `OPENAI_SECONDARY_API_KEY`
 - `OPENAI_BACKUP_API_KEY`
 - `OPENAI_FALLBACK_API_KEY`
 - `ODDPOOL_API_KEY`
@@ -26,4 +27,3 @@ rg -n "sk-[A-Za-z0-9_-]{12,}|PRIVATE_KEY|API_KEY|PASS_PHRASE|passphrase|secret"
 ```
 
 If you add a new secret-bearing setting later, update `.gitignore`, `.env.example`, and the docs together.
-

poly_strategy/cli.py

@@ -1761,7 +1761,7 @@ def _build_parser() -> argparse.ArgumentParser:
     discover.add_argument("--model", help="OpenAI model name; defaults to OPENAI_MODEL")
     discover.add_argument("--fallback-model", help="OpenAI model name for retrying remaining failed batches")
     discover.add_argument("--base-url", help="OpenAI-compatible base URL; defaults to OPENAI_BASE_URL or OpenAI")
-    discover.add_argument("--api-mode", choices=["responses", "chat"], help="OpenAI-compatible API mode; defaults to OPENAI_API_MODE or responses")
+    discover.add_argument("--api-mode", choices=["responses", "chat", "messages"], help="OpenAI-compatible API mode; defaults to OPENAI_API_MODE or responses")
     discover.add_argument("--batch-size", type=int, default=10, help="markets per LLM discovery batch")
     discover.add_argument("--min-confidence", type=float, default=0.95, help="minimum candidate confidence")
     discover.add_argument("--max-markets", type=int, help="limit input markets for a small run")
@@ -1824,7 +1824,7 @@ def _build_parser() -> argparse.ArgumentParser:
     verify_groups.add_argument("--report-out", help="optional JSON verification report path")
     verify_groups.add_argument("--model", help="OpenAI model name; defaults to OPENAI_MODEL")
     verify_groups.add_argument("--base-url", help="OpenAI-compatible base URL; defaults to OPENAI_BASE_URL or OpenAI")
-    verify_groups.add_argument("--api-mode", choices=["responses", "chat"], help="OpenAI-compatible API mode; defaults to OPENAI_API_MODE or responses")
+    verify_groups.add_argument("--api-mode", choices=["responses", "chat", "messages"], help="OpenAI-compatible API mode; defaults to OPENAI_API_MODE or responses")
     verify_groups.add_argument("--min-net-edge", type=float, default=0.002, help="minimum diagnostic net edge to verify")
     verify_groups.add_argument("--top", type=int, default=10, help="maximum diagnostic groups to verify")
     verify_groups.add_argument("--min-confidence", type=float, default=0.95, help="minimum verification confidence")
@@ -1921,13 +1921,13 @@ def _build_parser() -> argparse.ArgumentParser:
     verify_cross.add_argument("--verified-only", action="store_true", help="write only verified same-binary signals")
     verify_cross.add_argument("--model", help="OpenAI model name; defaults to OPENAI_MODEL")
     verify_cross.add_argument("--base-url", help="OpenAI-compatible base URL; defaults to OPENAI_BASE_URL or OpenAI")
-    verify_cross.add_argument("--api-mode", choices=["responses", "chat"], help="OpenAI-compatible API mode; defaults to OPENAI_API_MODE or responses")
+    verify_cross.add_argument("--api-mode", choices=["responses", "chat", "messages"], help="OpenAI-compatible API mode; defaults to OPENAI_API_MODE or responses")
     verify_cross.add_argument("--backup-model", help="backup model; defaults to OPENAI_BACKUP_MODEL")
     verify_cross.add_argument("--backup-base-url", help="backup OpenAI-compatible base URL; defaults to OPENAI_BACKUP_BASE_URL")
-    verify_cross.add_argument("--backup-api-mode", choices=["responses", "chat"], help="backup API mode; defaults to OPENAI_BACKUP_API_MODE")
+    verify_cross.add_argument("--backup-api-mode", choices=["responses", "chat", "messages"], help="backup API mode; defaults to OPENAI_BACKUP_API_MODE")
     verify_cross.add_argument("--fallback-model", help="fallback model; defaults to OPENAI_FALLBACK_MODEL")
     verify_cross.add_argument("--fallback-base-url", help="fallback OpenAI-compatible base URL; defaults to OPENAI_FALLBACK_BASE_URL")
-    verify_cross.add_argument("--fallback-api-mode", choices=["responses", "chat"], help="fallback API mode; defaults to OPENAI_FALLBACK_API_MODE")
+    verify_cross.add_argument("--fallback-api-mode", choices=["responses", "chat", "messages"], help="fallback API mode; defaults to OPENAI_FALLBACK_API_MODE")
     verify_cross.add_argument("--timeout", type=float, default=60.0, help="HTTP timeout in seconds")
     verify_cross.add_argument("--backup-timeout", type=float, help="backup provider HTTP timeout; defaults to OPENAI_BACKUP_TIMEOUT or --timeout")
     verify_cross.add_argument("--fallback-timeout", type=float, help="fallback provider HTTP timeout; defaults to OPENAI_FALLBACK_TIMEOUT or --timeout")

poly_strategy/openai_rules.py

@@ -24,6 +24,8 @@ def _normalize_api_mode(api_mode: Optional[str]) -> str:
         return "responses"
     if value in {"chat", "chat_completions", "chat-completions", "chatcompletions"}:
         return "chat"
+    if value in {"messages", "message", "anthropic", "anthropic_messages", "anthropic-messages"}:
+        return "messages"
     raise OpenAIConfigError(f"unsupported OPENAI_API_MODE: {api_mode!r}")
 
 
@@ -204,7 +206,14 @@ def __init__(
             if self.proxy
             else None
         )
-        self._transport = transport or (self._post_chat_completions if self.api_mode == "chat" else self._post_responses)
+        if transport is not None:
+            self._transport = transport
+        elif self.api_mode == "chat":
+            self._transport = self._post_chat_completions
+        elif self.api_mode == "messages":
+            self._transport = self._post_messages
+        else:
+            self._transport = self._post_responses
 
     def build_payload(self, markets: Iterable[MarketText]) -> dict:
         return self._build_payload(
@@ -217,7 +226,7 @@ def build_payload(self, markets: Iterable[MarketText]) -> dict:
     def _build_payload(self, markets: Iterable[MarketText], system_prompt: str, schema_name: str, schema: dict) -> dict:
         market_rows = market_texts_to_prompt_rows(list(markets))
         prompt_text = json.dumps({"markets": market_rows}, ensure_ascii=True, sort_keys=True)
-        if self.api_mode == "chat":
+        if self.api_mode in {"chat", "messages"}:
             if schema_name == "polymarket_relation_discovery":
                 chat_system_prompt, chat_user_prompt = _relation_chat_prompts(prompt_text)
             else:
@@ -244,6 +253,17 @@ def _build_payload(self, markets: Iterable[MarketText], system_prompt: str, sche
                     "Never return verification sources, market summaries, safe_items, or a top-level markets key.\n\n"
                     f"Input markets JSON:\n{prompt_text}"
                 )
+            if self.api_mode == "messages":
+                payload = {
+                    "model": self.model,
+                    "system": chat_system_prompt,
+                    "messages": [{"role": "user", "content": chat_user_prompt}],
+                }
+                if self.max_output_tokens is not None:
+                    payload["max_tokens"] = self.max_output_tokens
+                if self.temperature is not None:
+                    payload["temperature"] = self.temperature
+                return payload
             payload = {
                 "model": self.model,
                 "messages": [
@@ -336,6 +356,23 @@ def _post_chat_completions(self, payload: dict, timeout: float) -> dict:
                 return _parse_chat_stream_response(response)
             return json.loads(response.read().decode("utf-8"))
 
+    def _post_messages(self, payload: dict, timeout: float) -> dict:
+        request = Request(
+            _messages_url(self.base_url),
+            data=json.dumps(payload).encode("utf-8"),
+            headers={
+                "authorization": f"Bearer {self.api_key}",
+                "x-api-key": self.api_key,
+                "anthropic-version": "2023-06-01",
+                "content-type": "application/json",
+                "accept": "application/json",
+                "user-agent": "poly-strategy/0.1",
+            },
+            method="POST",
+        )
+        with self._open_request(request, timeout) as response:
+            return json.loads(response.read().decode("utf-8"))
+
     def _open_request(self, request: Request, timeout: float):
         if self._opener is not None:
             return self._opener.open(request, timeout=timeout)
@@ -389,7 +426,7 @@ class OpenAICrossPlatformVerifierClient(OpenAIRuleDiscoveryClient):
     def build_payload(self, matches: Iterable[dict]) -> dict:
         rows = [_cross_platform_prompt_row(match) for match in matches]
         prompt_text = json.dumps({"matches": rows}, ensure_ascii=True, sort_keys=True)
-        if self.api_mode == "chat":
+        if self.api_mode in {"chat", "messages"}:
             required_keys = ", ".join(_CROSS_PLATFORM_RESPONSE_SCHEMA.get("required", []))
             output_contract = _chat_output_contract("polymarket_kalshi_cross_platform_verification")
             output_instruction = _chat_output_instruction("polymarket_kalshi_cross_platform_verification")
@@ -414,6 +451,17 @@ def build_payload(self, matches: Iterable[dict]) -> dict:
                 "Return only one JSON object matching the schema; no markdown, no prose.\n\n"
                 f"Input matches JSON:\n{prompt_text}"
             )
+            if self.api_mode == "messages":
+                payload = {
+                    "model": self.model,
+                    "system": chat_system_prompt,
+                    "messages": [{"role": "user", "content": chat_user_prompt}],
+                }
+                if self.max_output_tokens is not None:
+                    payload["max_tokens"] = self.max_output_tokens
+                if self.temperature is not None:
+                    payload["temperature"] = self.temperature
+                return payload
             payload = {
                 "model": self.model,
                 "messages": [
@@ -644,6 +692,15 @@ def _chat_completions_url(base_url: str) -> str:
     return f"{normalized}/v1/chat/completions"
 
 
+def _messages_url(base_url: str) -> str:
+    normalized = base_url.rstrip("/")
+    if normalized.endswith("/messages"):
+        return normalized
+    if normalized.endswith("/v1"):
+        return f"{normalized}/messages"
+    return f"{normalized}/v1/messages"
+
+
 def _parse_chat_stream_response(response) -> dict:
     content_parts = []
     for data in _iter_sse_data(response):
@@ -730,6 +787,10 @@ def _extract_output_text(response: dict) -> str:
     if isinstance(output_text, str) and output_text:
         return output_text
 
+    content_text = _content_value_to_text(response.get("content"))
+    if content_text:
+        return content_text
+
     output = response.get("output")
     if isinstance(output, list):
         for item in output:

reports/experiment-llm-complex-recognition-consolidated-summary-2026-05-13.md

@@ -0,0 +1,76 @@
+# LLM 复杂场景识别汇总结论（2026-05-13）
+
+## 结论
+
+按严格复杂场景测试结果看，三家 provider 的可用性差异很大。
+
+## 最强候选
+
+1. `windhub / doubao-seed-1-8-251228 / messages`
+   - pass recall: `7/8`
+   - perfect: `6/8`
+   - avg recall: `0.95`
+   - median latency: `76.47s`
+   - 语义最强，但太慢，不适合高频主路径。
+
+2. `windhub / deepseek-v3-2-251201 / messages`
+   - pass recall: `8/8`
+   - perfect: `3/8`
+   - avg recall: `0.91`
+   - median latency: `44.28s`
+   - 这是更均衡的主力候选。
+
+3. `secondary / gemini-2.5-flash-nothinking / messages`
+   - pass recall: `7/8`
+   - perfect: `4/8`
+   - avg recall: `0.91`
+   - median latency: `9.20s`
+   - 速度最好，但正式链路 smoke 曾出现 `HTTP 554`，不适合作为当前默认自动备份。
+
+4. `elysiver / longcat-flash-chat / chat`
+   - pass recall: `8/8`
+   - perfect: `4/8`
+   - avg recall: `0.94`
+   - median latency: `20.08s`
+   - 在 elysiver 里最稳，兼顾速度和语义。
+
+5. `elysiver / qwen3-max / messages`
+   - pass recall: `7/8`
+   - perfect: `3/8`
+   - avg recall: `0.87`
+   - median latency: `41.96s`
+   - 语义强，但慢。
+
+6. `secondary / gemini-3.1-pro-preview / chat_stream`
+   - pass recall: `6/8`
+   - perfect: `1/8`
+   - avg recall: `0.81`
+   - median latency: `9.71s`
+   - 语义弱于 `gemini-2.5-flash-nothinking/messages`，但正式 CLI smoke 通过，适合作为当前 secondary 默认备份。
+
+## 不推荐路径
+
+- `gpt-5.5-web-auto/messages` 在 elysiver 上被 moderation 直接拦截。
+- `gemini-2.5-pro`、`gemini-3-flash-preview`、`glm-5` 在 secondary 上大量 554/不稳定。
+- `42-mini`、`42-pro` 在 elysiver 上复杂语义召回偏低。
+- `deepseek-v4-flash*` 在 elysiver 上基本 504，不适合继续投入。
+
+## 实际建议
+
+- 主路径：`windhub/deepseek-v3-2-251201/messages`
+- 高语义模式：`windhub/doubao-seed-1-8-251228/messages`
+- 低延迟语义候选：`secondary/gemini-2.5-flash-nothinking/messages`
+- 当前 secondary 默认备份：`secondary/gemini-3.1-pro-preview/chat`
+- 第三备份：`elysiver/longcat-flash-chat/chat`
+
+## 正式链路 smoke
+
+- `windhub/deepseek-v3-2-251201/messages`: 通过，2-market threshold 样本发现 `1` 个 implication。
+- `secondary/gemini-2.5-flash-nothinking/messages`: 未通过，返回 `HTTP 554`。
+- `secondary/gemini-3.1-pro-preview/chat`: 通过，2-market threshold 样本发现 `1` 个 implication。
+- `elysiver/longcat-flash-chat/chat`: 通过，2-market threshold 样本发现 `1` 个 implication。
+
+## 说明
+
+- 这里的排序优先看 `perfect` 和 `pass recall`，其次看 `avg recall`，最后才看延迟。
+- 若实际部署要偏高频，应优先用主路径 + 备份路径组合，而不是单一追求最高 recall。