diff --git a/.github/workflows/live-canary.yml b/.github/workflows/live-canary.yml index dad070f36d..730be50bac 100644 --- a/.github/workflows/live-canary.yml +++ b/.github/workflows/live-canary.yml @@ -178,6 +178,22 @@ jobs: AUTH_LIVE_NOTION_ACCESS_TOKEN: ${{ secrets.AUTH_LIVE_NOTION_ACCESS_TOKEN }} AUTH_LIVE_NOTION_REFRESH_TOKEN: ${{ secrets.AUTH_LIVE_NOTION_REFRESH_TOKEN }} AUTH_LIVE_NOTION_QUERY: ${{ secrets.AUTH_LIVE_NOTION_QUERY }} + AUTH_LIVE_LINEAR_ACCESS_TOKEN: ${{ secrets.AUTH_LIVE_LINEAR_ACCESS_TOKEN }} + AUTH_LIVE_LINEAR_REFRESH_TOKEN: ${{ secrets.AUTH_LIVE_LINEAR_REFRESH_TOKEN }} + AUTH_LIVE_LINEAR_QUERY: ${{ secrets.AUTH_LIVE_LINEAR_QUERY }} + AUTH_LIVE_LINEAR_TOOL_NAME: ${{ vars.AUTH_LIVE_LINEAR_TOOL_NAME || 'linear_search_issues' }} + AUTH_LIVE_LINEAR_TOOL_ARGS_JSON: ${{ secrets.AUTH_LIVE_LINEAR_TOOL_ARGS_JSON }} + AUTH_LIVE_BRAVE_API_KEY: ${{ secrets.AUTH_LIVE_BRAVE_API_KEY }} + AUTH_LIVE_SLACK_BOT_TOKEN: ${{ secrets.AUTH_LIVE_SLACK_BOT_TOKEN }} + AUTH_LIVE_COMPOSIO_API_KEY: ${{ secrets.AUTH_LIVE_COMPOSIO_API_KEY }} + AUTH_LIVE_TELEGRAM_API_ID: ${{ secrets.AUTH_LIVE_TELEGRAM_API_ID }} + AUTH_LIVE_TELEGRAM_API_HASH: ${{ secrets.AUTH_LIVE_TELEGRAM_API_HASH }} + AUTH_LIVE_TELEGRAM_SESSION_JSON: ${{ secrets.AUTH_LIVE_TELEGRAM_SESSION_JSON }} + AUTH_LIVE_GOOGLE_DRIVE_QUERY: ${{ vars.AUTH_LIVE_GOOGLE_DRIVE_QUERY || 'trashed = false' }} + AUTH_LIVE_GOOGLE_DOC_ID: ${{ secrets.AUTH_LIVE_GOOGLE_DOC_ID }} + AUTH_LIVE_GOOGLE_SHEET_ID: ${{ secrets.AUTH_LIVE_GOOGLE_SHEET_ID }} + AUTH_LIVE_GOOGLE_SHEET_RANGE: ${{ vars.AUTH_LIVE_GOOGLE_SHEET_RANGE || 'A1:Z10' }} + AUTH_LIVE_GOOGLE_SLIDES_ID: ${{ secrets.AUTH_LIVE_GOOGLE_SLIDES_ID }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 with: diff --git a/scripts/auth_live_canary/README.md b/scripts/auth_live_canary/README.md index 6078b55fbe..fd70a60162 100644 --- a/scripts/auth_live_canary/README.md +++ b/scripts/auth_live_canary/README.md @@ -37,6 +37,13 @@ not model behavior. - `notion` Uses `mcp_notion_access_token` Runs through Responses API +- `linear` + Uses `mcp_linear_access_token` + Runs through Responses API +- `ops_workflow` + Installs Gmail, Calendar, Drive, Docs, Sheets, Slides, GitHub, Web Search, + LLM Context, Slack, Telegram, Composio, Notion, and Linear. It dispatches one + deterministic multi-tool ops brief probe through `/v1/responses`. ## Setup @@ -76,6 +83,7 @@ Run only selected providers: ```bash python3 scripts/auth_live_canary/run_live_canary.py --case gmail --case github +python3 scripts/auth_live_canary/run_live_canary.py --case ops_workflow ``` CI-style fresh-machine install: diff --git a/scripts/auth_live_canary/config.example.env b/scripts/auth_live_canary/config.example.env index 4bfdce3a2a..030d2dc962 100644 --- a/scripts/auth_live_canary/config.example.env +++ b/scripts/auth_live_canary/config.example.env @@ -1,11 +1,16 @@ -# Google / Gmail / Calendar +# Google / Gmail / Calendar / Drive / Docs / Sheets / Slides GOOGLE_OAUTH_CLIENT_ID= GOOGLE_OAUTH_CLIENT_SECRET= AUTH_LIVE_GOOGLE_ACCESS_TOKEN= AUTH_LIVE_GOOGLE_REFRESH_TOKEN= -AUTH_LIVE_GOOGLE_SCOPES=gmail.modify gmail.compose calendar.events +AUTH_LIVE_GOOGLE_SCOPES="https://www.googleapis.com/auth/gmail.modify https://www.googleapis.com/auth/gmail.compose https://www.googleapis.com/auth/calendar.events https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/documents https://www.googleapis.com/auth/spreadsheets https://www.googleapis.com/auth/presentations" # Set to 0 to skip forced refresh on first probe. AUTH_LIVE_FORCE_GOOGLE_REFRESH=1 +AUTH_LIVE_GOOGLE_DRIVE_QUERY="trashed = false" +AUTH_LIVE_GOOGLE_DOC_ID= +AUTH_LIVE_GOOGLE_SHEET_ID= +AUTH_LIVE_GOOGLE_SHEET_RANGE=A1:Z10 +AUTH_LIVE_GOOGLE_SLIDES_ID= # GitHub AUTH_LIVE_GITHUB_TOKEN= @@ -17,3 +22,24 @@ AUTH_LIVE_GITHUB_ISSUE_NUMBER= AUTH_LIVE_NOTION_ACCESS_TOKEN= AUTH_LIVE_NOTION_REFRESH_TOKEN= AUTH_LIVE_NOTION_QUERY=canary + +# Linear MCP +AUTH_LIVE_LINEAR_ACCESS_TOKEN= +AUTH_LIVE_LINEAR_REFRESH_TOKEN= +AUTH_LIVE_LINEAR_QUERY=canary +AUTH_LIVE_LINEAR_TOOL_NAME=linear_search_issues +AUTH_LIVE_LINEAR_TOOL_ARGS_JSON= + +# Brave-backed tools +AUTH_LIVE_BRAVE_API_KEY= + +# Slack +AUTH_LIVE_SLACK_BOT_TOKEN= + +# Composio +AUTH_LIVE_COMPOSIO_API_KEY= + +# Telegram user-mode tool +AUTH_LIVE_TELEGRAM_API_ID= +AUTH_LIVE_TELEGRAM_API_HASH= +AUTH_LIVE_TELEGRAM_SESSION_JSON= diff --git a/scripts/auth_live_canary/run_live_canary.py b/scripts/auth_live_canary/run_live_canary.py index ea6e044796..874d08ec29 100644 --- a/scripts/auth_live_canary/run_live_canary.py +++ b/scripts/auth_live_canary/run_live_canary.py @@ -29,7 +29,12 @@ sys.path.insert(0, str(ROOT)) from scripts.live_canary.auth_registry import SeededProviderCase, configured_seeded_cases -from scripts.live_canary.auth_runtime import activate_extension, install_extension, put_secret +from scripts.live_canary.auth_runtime import ( + activate_extension, + install_extension, + put_secret, + write_memory, +) from scripts.live_canary.common import ( DEFAULT_SECRETS_MASTER_KEY, DEFAULT_VENV, @@ -49,7 +54,17 @@ DEFAULT_OUTPUT_DIR = ROOT / "artifacts" / "auth-live-canary" OWNER_USER_ID = "auth-live-owner" -GOOGLE_SCOPE_DEFAULT = "gmail.modify gmail.compose calendar.events" +GOOGLE_SCOPE_DEFAULT = " ".join( + [ + "https://www.googleapis.com/auth/gmail.modify", + "https://www.googleapis.com/auth/gmail.compose", + "https://www.googleapis.com/auth/calendar.events", + "https://www.googleapis.com/auth/drive", + "https://www.googleapis.com/auth/documents", + "https://www.googleapis.com/auth/spreadsheets", + "https://www.googleapis.com/auth/presentations", + ] +) def expire_secret_in_db(db_path: Path, user_id: str, secret_name: str) -> None: @@ -95,6 +110,7 @@ async def create_response_probe( response_id = body.get("id") output = body.get("output", []) tool_names = [item.get("name") for item in output if item.get("type") == "function_call"] + expected_tool_names = probe.required_tool_names tool_outputs = [ item.get("output", "") for item in output @@ -120,14 +136,14 @@ async def create_response_probe( success = ( body.get("status") == "completed" - and probe.expected_tool_name in tool_names + and all(tool_name in tool_names for tool_name in expected_tool_names) and bool(tool_outputs) and not any( marker in output_text.lower() for output_text in tool_outputs for marker in ("error", "authentication required", "unauthorized", "forbidden") ) - and probe.expected_text in response_text + and (not probe.expected_text or probe.expected_text in response_text) and fetched_status == 200 ) @@ -140,6 +156,7 @@ async def create_response_probe( "response_id": response_id, "status": body.get("status"), "tool_names": tool_names, + "expected_tool_names": expected_tool_names, "tool_outputs": tool_outputs, "response_text": response_text, "get_status_code": fetched_status, @@ -291,6 +308,64 @@ async def seed_live_credentials(base_url: str, token: str, db_path: Path) -> Non provider="mcp:notion", ) + linear_access = env_str("AUTH_LIVE_LINEAR_ACCESS_TOKEN") + linear_refresh = env_str("AUTH_LIVE_LINEAR_REFRESH_TOKEN") + if linear_refresh and not linear_access: + raise CanaryError( + "AUTH_LIVE_LINEAR_ACCESS_TOKEN is required when AUTH_LIVE_LINEAR_REFRESH_TOKEN is set" + ) + if linear_access: + await put_secret( + base_url, + token, + user_id=OWNER_USER_ID, + name="mcp_linear_access_token", + value=linear_access, + provider="mcp:linear", + ) + if linear_refresh: + await put_secret( + base_url, + token, + user_id=OWNER_USER_ID, + name="mcp_linear_access_token_refresh_token", + value=linear_refresh, + provider="mcp:linear", + ) + + for env_name, secret_name, provider in ( + ("AUTH_LIVE_BRAVE_API_KEY", "brave_api_key", "brave"), + ("AUTH_LIVE_SLACK_BOT_TOKEN", "slack_bot_token", "slack"), + ("AUTH_LIVE_COMPOSIO_API_KEY", "composio_api_key", "composio"), + ("AUTH_LIVE_TELEGRAM_API_ID", "telegram_api_id", "telegram"), + ("AUTH_LIVE_TELEGRAM_API_HASH", "telegram_api_hash", "telegram"), + ): + value = env_str(env_name) + if value: + await put_secret( + base_url, + token, + user_id=OWNER_USER_ID, + name=secret_name, + value=value, + provider=provider, + ) + + telegram_api_id = env_str("AUTH_LIVE_TELEGRAM_API_ID") + telegram_api_hash = env_str("AUTH_LIVE_TELEGRAM_API_HASH") + telegram_session = env_str("AUTH_LIVE_TELEGRAM_SESSION_JSON") + if telegram_api_id: + await write_memory(base_url, token, path="telegram/api_id", content=telegram_api_id) + if telegram_api_hash: + await write_memory(base_url, token, path="telegram/api_hash", content=telegram_api_hash) + if telegram_session: + await write_memory( + base_url, + token, + path="telegram/session.json", + content=telegram_session, + ) + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) @@ -325,7 +400,14 @@ def parse_args() -> argparse.Namespace: parser.add_argument( "--case", action="append", - choices=("gmail", "google_calendar", "github", "notion"), + choices=( + "gmail", + "google_calendar", + "github", + "notion", + "linear", + "ops_workflow", + ), help="Limit the run to specific providers. Repeat for multiple values.", ) parser.add_argument( @@ -374,21 +456,27 @@ async def async_main(args: argparse.Namespace) -> int: try: await seed_live_credentials(stack.base_url, stack.gateway_token, stack.db_path) + installed: dict[str, dict[str, Any]] = {} for probe in probes: - ext = await install_extension( - stack.base_url, - stack.gateway_token, - name=probe.extension_install_name, - expected_display_name=probe.expected_display_name, - install_kind=probe.install_kind, - install_url=probe.install_url, - ) - await activate_extension( - stack.base_url, - stack.gateway_token, - extension_name=ext["name"], - expected_display_name=ext.get("display_name") or probe.expected_display_name, - ) + for installation in probe.installations: + if installation.name in installed: + continue + ext = await install_extension( + stack.base_url, + stack.gateway_token, + name=installation.name, + expected_display_name=installation.expected_display_name, + install_kind=installation.install_kind, + install_url=installation.install_url, + ) + await activate_extension( + stack.base_url, + stack.gateway_token, + extension_name=ext["name"], + expected_display_name=ext.get("display_name") + or installation.expected_display_name, + ) + installed[installation.name] = ext results: list[ProbeResult] = [] for probe in probes: diff --git a/scripts/live-canary/ACCOUNTS.md b/scripts/live-canary/ACCOUNTS.md index 4f232ec637..92801c0a06 100644 --- a/scripts/live-canary/ACCOUNTS.md +++ b/scripts/live-canary/ACCOUNTS.md @@ -70,8 +70,17 @@ Every provider should have one stable, low-risk probe target. - Gmail: one inbox with at least one readable message or draft - Google Calendar: one calendar with at least one upcoming event +- Google Drive: one accessible stable fixture query or file set +- Google Docs: one readable fixture document +- Google Sheets: one readable fixture spreadsheet/range +- Google Slides: one readable fixture presentation - GitHub: one dedicated repository with one stable issue +- Brave Search: one low-volume API key shared by Web Search and LLM Context +- Slack: one workspace with a bot token that can list channels +- Telegram: one logged-in user-mode MTProto session +- Composio: one API key with at least one readable connected-account state - Notion: one test workspace with one searchable page or database row +- Linear: one workspace with one searchable issue ## Seeded Lane Secrets @@ -100,6 +109,21 @@ Recommended scopes: - `https://www.googleapis.com/auth/gmail.modify` - `https://www.googleapis.com/auth/gmail.compose` - `https://www.googleapis.com/auth/calendar.events` +- `https://www.googleapis.com/auth/drive` +- `https://www.googleapis.com/auth/documents` +- `https://www.googleapis.com/auth/spreadsheets` +- `https://www.googleapis.com/auth/presentations` + +Required only for the combined `ops_workflow` case: + +- `AUTH_LIVE_GOOGLE_DOC_ID` +- `AUTH_LIVE_GOOGLE_SHEET_ID` +- `AUTH_LIVE_GOOGLE_SLIDES_ID` + +Optional: + +- `AUTH_LIVE_GOOGLE_DRIVE_QUERY` (defaults to `trashed = false`) +- `AUTH_LIVE_GOOGLE_SHEET_RANGE` (defaults to `A1:Z10`) ### GitHub @@ -125,6 +149,72 @@ Optional: The probe should match a stable test page or database entry. +### Linear + +Required: + +- `AUTH_LIVE_LINEAR_ACCESS_TOKEN` +- `AUTH_LIVE_LINEAR_QUERY` + +Optional: + +- `AUTH_LIVE_LINEAR_REFRESH_TOKEN` +- `AUTH_LIVE_LINEAR_TOOL_NAME` +- `AUTH_LIVE_LINEAR_TOOL_ARGS_JSON` + +Use `AUTH_LIVE_LINEAR_TOOL_NAME` and `AUTH_LIVE_LINEAR_TOOL_ARGS_JSON` if the +Linear MCP server's tool name or argument schema changes. The default tool name +is `linear_search_issues`, with arguments `{"query": ""}`. + +### Brave Search + +Required for Web Search and LLM Context probes: + +- `AUTH_LIVE_BRAVE_API_KEY` + +### Slack + +Required: + +- `AUTH_LIVE_SLACK_BOT_TOKEN` + +The combined workflow uses `list_channels` to avoid posting on every scheduled +run. + +### Telegram + +Required: + +- `AUTH_LIVE_TELEGRAM_API_ID` +- `AUTH_LIVE_TELEGRAM_API_HASH` +- `AUTH_LIVE_TELEGRAM_SESSION_JSON` + +The seeded runner writes these to `telegram/api_id`, `telegram/api_hash`, and +`telegram/session.json` in the fresh workspace before activating the tool. The +combined workflow uses `get_me` to avoid sending messages on every scheduled +run. + +### Composio + +Required: + +- `AUTH_LIVE_COMPOSIO_API_KEY` + +The combined workflow uses `connected_accounts`, which is read-only. + +### Combined Ops Workflow + +Run this after provisioning every fixture above: + +```bash +LANE=auth-live-seeded CASES=ops_workflow scripts/live-canary/run.sh +``` + +It installs and activates Gmail, Google Calendar, Google Drive, Google Docs, +Google Sheets, Google Slides, GitHub, Web Search, LLM Context, Slack, Telegram, +Composio, Notion, and Linear, then dispatches one deterministic `/v1/responses` +turn that calls every tool. + ## Browser-Consent Lane Secrets These are read by `scripts/auth_browser_canary/run_browser_canary.py`. diff --git a/scripts/live-canary/README.md b/scripts/live-canary/README.md index 9dcde68ecf..05e24650c5 100644 --- a/scripts/live-canary/README.md +++ b/scripts/live-canary/README.md @@ -83,6 +83,7 @@ Run selected auth provider cases: ```bash LANE=auth-live-seeded CASES=gmail,github scripts/live-canary/run.sh +LANE=auth-live-seeded CASES=ops_workflow scripts/live-canary/run.sh LANE=auth-browser-consent CASES=google,github scripts/live-canary/run.sh ``` diff --git a/scripts/live_canary/auth_registry.py b/scripts/live_canary/auth_registry.py index 3594ba8a45..2f6131988f 100644 --- a/scripts/live_canary/auth_registry.py +++ b/scripts/live_canary/auth_registry.py @@ -35,6 +35,14 @@ } +@dataclass(frozen=True) +class ExtensionInstall: + name: str + expected_display_name: str + install_kind: str | None = None + install_url: str | None = None + + @dataclass(frozen=True) class SeededProviderCase: key: str @@ -48,6 +56,24 @@ class SeededProviderCase: install_url: str | None = None shared_secret_name: str | None = None requires_refresh_seed: bool = False + extra_installations: tuple[ExtensionInstall, ...] = () + expected_tool_names: tuple[str, ...] = () + + @property + def installations(self) -> tuple[ExtensionInstall, ...]: + return ( + ExtensionInstall( + name=self.extension_install_name, + expected_display_name=self.expected_display_name, + install_kind=self.install_kind, + install_url=self.install_url, + ), + *self.extra_installations, + ) + + @property + def required_tool_names(self) -> tuple[str, ...]: + return self.expected_tool_names or (self.expected_tool_name,) @dataclass(frozen=True) @@ -103,9 +129,79 @@ class BrowserProviderCase: expected_text="Notion search completed successfully.", install_kind="mcp_server", ), + "linear": SeededProviderCase( + key="linear", + extension_install_name="linear", + expected_display_name="Linear", + response_prompt="search linear for canary", + expected_tool_name="linear_search_issues", + expected_text="Linear search completed successfully.", + install_kind="mcp_server", + ), + "ops_workflow": SeededProviderCase( + key="ops_workflow", + extension_install_name="gmail", + expected_display_name="Gmail", + response_prompt="run auth ops workflow canary", + expected_tool_name="gmail", + expected_text="", + extra_installations=( + ExtensionInstall("google_calendar", "Google Calendar"), + ExtensionInstall("google_drive", "Google Drive"), + ExtensionInstall("google_docs", "Google Docs"), + ExtensionInstall("google_sheets", "Google Sheets"), + ExtensionInstall("google_slides", "Google Slides"), + ExtensionInstall("github", "GitHub"), + ExtensionInstall("web_search", "Web Search"), + ExtensionInstall("llm_context", "LLM Context"), + ExtensionInstall("slack_tool", "Slack Tool"), + ExtensionInstall("telegram_mtproto", "Telegram Tool"), + ExtensionInstall("composio", "Composio"), + ExtensionInstall("notion", "Notion", install_kind="mcp_server"), + ExtensionInstall("linear", "Linear", install_kind="mcp_server"), + ), + expected_tool_names=( + "gmail", + "google_calendar", + "google_drive", + "google_docs", + "google_sheets", + "google_slides", + "github", + "web_search", + "llm_context", + "slack_tool", + "telegram_mtproto", + "composio", + "notion_notion_search", + "linear_search_issues", + ), + ), } +OPS_WORKFLOW_REQUIRED_ENVS = ( + "AUTH_LIVE_GOOGLE_ACCESS_TOKEN", + "AUTH_LIVE_GOOGLE_DOC_ID", + "AUTH_LIVE_GOOGLE_SHEET_ID", + "AUTH_LIVE_GOOGLE_SLIDES_ID", + "AUTH_LIVE_GITHUB_TOKEN", + "AUTH_LIVE_GITHUB_OWNER", + "AUTH_LIVE_GITHUB_REPO", + "AUTH_LIVE_GITHUB_ISSUE_NUMBER", + "AUTH_LIVE_BRAVE_API_KEY", + "AUTH_LIVE_SLACK_BOT_TOKEN", + "AUTH_LIVE_COMPOSIO_API_KEY", + "AUTH_LIVE_TELEGRAM_API_ID", + "AUTH_LIVE_TELEGRAM_API_HASH", + "AUTH_LIVE_TELEGRAM_SESSION_JSON", + "AUTH_LIVE_NOTION_ACCESS_TOKEN", + "AUTH_LIVE_NOTION_QUERY", + "AUTH_LIVE_LINEAR_ACCESS_TOKEN", + "AUTH_LIVE_LINEAR_QUERY", +) + + BROWSER_CASES: dict[str, BrowserProviderCase] = { "google": BrowserProviderCase( key="google", @@ -184,6 +280,37 @@ def configured_seeded_cases(selected: list[str] | None) -> list[SeededProviderCa message="AUTH_LIVE_NOTION_QUERY is required for the selected live-provider case", ) case = replace(case, response_prompt=f"search notion for {query}") + elif name == "linear": + if not env_str("AUTH_LIVE_LINEAR_ACCESS_TOKEN"): + continue + query = required_env( + "AUTH_LIVE_LINEAR_QUERY", + message="AUTH_LIVE_LINEAR_QUERY is required for the selected live-provider case", + ) + tool_name = env_str("AUTH_LIVE_LINEAR_TOOL_NAME", "linear_search_issues") + case = replace( + case, + response_prompt=f"search linear for {query}", + expected_tool_name=tool_name, + expected_tool_names=(tool_name,), + ) + elif name == "ops_workflow": + missing = [env_name for env_name in OPS_WORKFLOW_REQUIRED_ENVS if not env_str(env_name)] + if missing: + if selected is not None: + raise CanaryError( + "ops_workflow requires all fixture env vars; missing: " + + ", ".join(missing) + ) + continue + linear_tool = env_str("AUTH_LIVE_LINEAR_TOOL_NAME", "linear_search_issues") + case = replace( + case, + expected_tool_names=tuple( + linear_tool if tool == "linear_search_issues" else tool + for tool in case.expected_tool_names + ), + ) cases.append(case) return cases @@ -207,4 +334,3 @@ def configured_browser_cases(selected: list[str] | None) -> list[BrowserProvider ): cases.append(case) return cases - diff --git a/scripts/live_canary/auth_runtime.py b/scripts/live_canary/auth_runtime.py index 3297a6bd3c..e75797066e 100644 --- a/scripts/live_canary/auth_runtime.py +++ b/scripts/live_canary/auth_runtime.py @@ -29,6 +29,26 @@ async def put_secret( raise CanaryError(f"Failed to seed secret {name}: {response.status_code} {response.text}") +async def write_memory( + base_url: str, + token: str, + *, + path: str, + content: str, +) -> None: + response = await api_request( + "POST", + base_url, + "/api/memory/write", + token=token, + json_body={"path": path, "content": content, "append": False, "force": True}, + ) + if response.status_code != 200: + raise CanaryError( + f"Failed to seed workspace file {path}: {response.status_code} {response.text}" + ) + + async def list_extensions(base_url: str, token: str) -> list[dict[str, Any]]: response = await api_request("GET", base_url, "/api/extensions", token=token, timeout=30) response.raise_for_status() diff --git a/tests/e2e/mock_llm.py b/tests/e2e/mock_llm.py index 1b6949013f..31f0dd1b77 100644 --- a/tests/e2e/mock_llm.py +++ b/tests/e2e/mock_llm.py @@ -8,6 +8,7 @@ import argparse import asyncio import json +import os import re import time import uuid @@ -41,6 +42,10 @@ re.compile(r"Tool `notion_notion_search` returned:", re.IGNORECASE | re.DOTALL), "Notion search completed successfully.", ), + ( + re.compile(r"Tool `linear_.*` returned:", re.IGNORECASE | re.DOTALL), + "Linear search completed successfully.", + ), (re.compile(r"skill|install", re.IGNORECASE), "I can help you with skills management."), (re.compile(r"html.?test|injection.?test", re.IGNORECASE), 'Here is some content: and ' @@ -134,6 +139,19 @@ "notion_notion_search", lambda m: {"query": m.group("query").strip()}, ), + ( + re.compile(r"search linear for (?P.+)", re.IGNORECASE), + "linear_search_issues", + lambda m: [{ + "tool_name": os.getenv("AUTH_LIVE_LINEAR_TOOL_NAME", "linear_search_issues"), + "arguments": _linear_tool_args(m.group("query").strip()), + }], + ), + ( + re.compile(r"run auth ops workflow canary", re.IGNORECASE), + "gmail", + lambda _: _auth_ops_workflow_calls(), + ), (re.compile(r"what time|current time", re.IGNORECASE), "time", lambda _: {"operation": "now"}), ( re.compile( @@ -448,6 +466,115 @@ ] +def _env_json_object(name: str, default: dict) -> dict: + raw = os.getenv(name) + if not raw: + return dict(default) + parsed = json.loads(raw) + if not isinstance(parsed, dict): + raise TypeError(f"{name} must be a JSON object") + return parsed + + +def _linear_tool_args(query: str) -> dict: + return _env_json_object("AUTH_LIVE_LINEAR_TOOL_ARGS_JSON", {"query": query}) + + +def _auth_ops_workflow_calls() -> list[dict]: + github_owner = os.environ["AUTH_LIVE_GITHUB_OWNER"] + github_repo = os.environ["AUTH_LIVE_GITHUB_REPO"] + github_issue_number = int(os.environ["AUTH_LIVE_GITHUB_ISSUE_NUMBER"]) + notion_query = os.environ["AUTH_LIVE_NOTION_QUERY"] + linear_query = os.environ["AUTH_LIVE_LINEAR_QUERY"] + linear_tool = os.getenv("AUTH_LIVE_LINEAR_TOOL_NAME", "linear_search_issues") + + return [ + { + "tool_name": "gmail", + "arguments": { + "action": "list_messages", + "query": "newer_than:30d", + "max_results": 1, + }, + }, + { + "tool_name": "google_calendar", + "arguments": { + "action": "list_events", + "calendar_id": "primary", + "max_results": 1, + }, + }, + { + "tool_name": "google_drive", + "arguments": { + "action": "list_files", + "query": os.getenv("AUTH_LIVE_GOOGLE_DRIVE_QUERY", "trashed = false"), + "page_size": 1, + }, + }, + { + "tool_name": "google_docs", + "arguments": { + "action": "read_content", + "document_id": os.environ["AUTH_LIVE_GOOGLE_DOC_ID"], + }, + }, + { + "tool_name": "google_sheets", + "arguments": { + "action": "read_values", + "spreadsheet_id": os.environ["AUTH_LIVE_GOOGLE_SHEET_ID"], + "range": os.getenv("AUTH_LIVE_GOOGLE_SHEET_RANGE", "A1:Z10"), + }, + }, + { + "tool_name": "google_slides", + "arguments": { + "action": "get_presentation", + "presentation_id": os.environ["AUTH_LIVE_GOOGLE_SLIDES_ID"], + }, + }, + { + "tool_name": "github", + "arguments": { + "action": "get_issue", + "owner": github_owner, + "repo": github_repo, + "issue_number": github_issue_number, + }, + }, + { + "tool_name": "web_search", + "arguments": {"query": "IronClaw canary", "count": 1}, + }, + { + "tool_name": "llm_context", + "arguments": {"query": "IronClaw canary", "count": 1}, + }, + { + "tool_name": "slack_tool", + "arguments": {"action": "list_channels", "limit": 10}, + }, + { + "tool_name": "telegram_mtproto", + "arguments": {"action": "get_me"}, + }, + { + "tool_name": "composio", + "arguments": {"action": "connected_accounts"}, + }, + { + "tool_name": "notion_notion_search", + "arguments": {"query": notion_query}, + }, + { + "tool_name": linear_tool, + "arguments": _linear_tool_args(linear_query), + }, + ] + + # Runtime-configurable mock API URL for github tool call tests. # Set via POST /__mock/set_github_api_url with {"url": "http://..."} _github_api_url: str = "https://api.github.com"