From 1161726487ec9c64f8ca558dfec5b16265ba90ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Musil?= Date: Thu, 15 May 2025 18:06:36 +0200 Subject: [PATCH 1/5] feat: add mcp server support --- pyproject.toml | 3 +- run.py | 46 +++++++++++++++++------ src/crm_agent.py | 5 +-- src/dump_hubspot.py | 10 ++++- src/mcp_toolset.py | 90 +++++++++++++++++++++++++++++++++++++++++++++ src/shared.py | 4 +- 6 files changed, 139 insertions(+), 19 deletions(-) create mode 100644 src/mcp_toolset.py diff --git a/pyproject.toml b/pyproject.toml index 91b36c2..bb374e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,5 +12,6 @@ dependencies = [ "litellm", "superface", "requests", - "composio-openai" + "composio-openai", + "mcp" ] diff --git a/run.py b/run.py index 5843dfc..5c48f9c 100644 --- a/run.py +++ b/run.py @@ -1,5 +1,6 @@ import os import json +import asyncio from dotenv import load_dotenv from superface import Superface from superface.client.superface import SuperfaceAPI @@ -11,6 +12,7 @@ from src.dump_hubspot import dump_hubspot from src.evaluator import Evaluator from src.vibecode_toolset import create_vibecode_toolset +from src.mcp_toolset import MCPToolset import argparse load_dotenv() @@ -101,6 +103,16 @@ def create_composio_toolset() -> Toolset: ] ) +async def create_hubspot_mcp_toolset() -> Toolset: + toolset = MCPToolset( + npm_package="@hubspot/mcp-server", + name="HubSpot MCP Toolset", + env={ + "PRIVATE_APP_ACCESS_TOKEN": os.getenv("HUBSPOT_API_KEY"), + } + ) + await toolset.connect() + return toolset def load_tasks(slice: Optional[slice] = None) -> List[Task]: base_dir = os.path.dirname(os.path.abspath(__file__)) @@ -114,7 +126,7 @@ def load_tasks(slice: Optional[slice] = None) -> List[Task]: tasks = tasks[slice] return tasks -def solve_task(*, file: TextIO, task: Task, toolset: Toolset, model: Model, trials_count: int, seed: Optional[int] = None): +async def solve_task(*, file: TextIO, task: Task, toolset: Toolset, model: Model, trials_count: int, seed: Optional[int] = None): agent = CRMAgent( model=model, tools=toolset @@ -127,7 +139,7 @@ def solve_task(*, file: TextIO, task: Task, toolset: Toolset, model: Model, tria print("๐Ÿงน Resetting CRM...") reset_hubspot() - result = agent.solve(task=task, seed=seed) + result = await agent.solve(task=task, seed=seed) result.trial_idx = i result.trials_count = trials_count @@ -205,13 +217,13 @@ def dump_hubspot_state(): hubspot_state = dump_hubspot() print(f"HubSpot State: {hubspot_state}") -def run(*, toolsets: List[Toolset], trials_count: int, model = Model.GPT_4o, seed: Optional[int] = None): +async def run(*, toolsets: List[Toolset], trials_count: int, model = Model.GPT_4o, seed: Optional[int] = None): tasks = load_tasks() for toolset in toolsets: print(f"Running tasks for toolset: {toolset.name}") with open_results_file(toolset) as file: for task in tasks: - solve_task(task=task, toolset=toolset, model=model, trials_count=trials_count, seed=seed, file=file) + await solve_task(task=task, toolset=toolset, model=model, trials_count=trials_count, seed=seed, file=file) toolset_creators = { "superface": create_superface_toolset, @@ -219,11 +231,12 @@ def run(*, toolsets: List[Toolset], trials_count: int, model = Model.GPT_4o, see "superface_dynamic_specialist": create_superface_dynamic_specialists_toolset, "composio": create_composio_toolset, 'vibecode': create_vibecode_toolset, + 'hubspot_mcp': create_hubspot_mcp_toolset, } toolset_options = list(toolset_creators.keys()) -if __name__ == "__main__": +async def main(): parser = argparse.ArgumentParser(description="Run CRM Tools Benchmark") parser.add_argument( "--toolsets", @@ -244,12 +257,21 @@ def run(*, toolsets: List[Toolset], trials_count: int, model = Model.GPT_4o, see default=None, help="Specify the seed (default: None)" ) - args = parser.parse_args() - selected_toolsets = [toolset_creators[toolset]() for toolset in args.toolsets] + args = parser.parse_args() + selected_toolsets = [await toolset_creators[toolset]() for toolset in args.toolsets] - run( - toolsets=selected_toolsets, - trials_count=args.trials, - seed=args.seed - ) + try: + await run( + toolsets=selected_toolsets, + trials_count=args.trials, + seed=args.seed + ) + finally: + for toolset in selected_toolsets: + if hasattr(toolset, 'close'): + print(f"Closing toolset: {toolset.name}") + await toolset.close() + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/crm_agent.py b/src/crm_agent.py index 150860d..2116cf7 100644 --- a/src/crm_agent.py +++ b/src/crm_agent.py @@ -13,7 +13,7 @@ def __init__(self, *, model: Model, tools: list[Tool]): self.model = model self.tools = tools - def solve(self, task, *, max_num_steps = 30, seed: Optional[int] = None) -> SolveResult: + async def solve(self, task, *, max_num_steps = 30, seed: Optional[int] = None) -> SolveResult: messages: List[Dict[str, Any]] = [ { "role": "system", "content": CRMAgent.instructions }, { "role": "user", "content": task.prompt } @@ -44,8 +44,7 @@ def solve(self, task, *, max_num_steps = 30, seed: Optional[int] = None) -> Solv (t for t in self.tools if t.name == tool_name), None ) if tool: - tool_response = tool.run(tool_args) - + tool_response = await tool.run(tool_args) messages.append({ "role": "tool", "tool_call_id": tool_call["id"], diff --git a/src/dump_hubspot.py b/src/dump_hubspot.py index 0829d17..cf3d15f 100644 --- a/src/dump_hubspot.py +++ b/src/dump_hubspot.py @@ -1,7 +1,10 @@ +from dotenv import load_dotenv import os import requests from .shared import CrmState, CrmStateEngagements +load_dotenv() + # ๐Ÿ”ง CONFIGURATION HUBSPOT_API_KEY = os.getenv("HUBSPOT_API_KEY") HEADERS = { @@ -120,4 +123,9 @@ def get_associations(object_type, object_id, to_object_type): "limit": 500, } data = get(endpoint, params) - return data \ No newline at end of file + return data + +if __name__ == "__main__": + print("Dumping HubSpot state...") + state = dump_hubspot() + print(state) \ No newline at end of file diff --git a/src/mcp_toolset.py b/src/mcp_toolset.py new file mode 100644 index 0000000..9cb8e7f --- /dev/null +++ b/src/mcp_toolset.py @@ -0,0 +1,90 @@ +import asyncio +import json +import os +from contextlib import AsyncExitStack +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client +from typing import Optional +from .shared import Tool, Toolset + + +class MCPToolset(Toolset): + def __init__(self, *, name: str, npm_package: str, env: dict[str, str] | None): + self.name = name + self.env = env + self.npm_package = npm_package + + self.session: Optional[ClientSession] = None + self.exit_stack = AsyncExitStack() + + super().__init__(name=name, tools=[]) + + async def connect(self): + server_params = StdioServerParameters( + command="npx", + args=[self.npm_package], + env=self.env + ) + + stdio_transport = await self.exit_stack.enter_async_context(stdio_client(server_params)) + self.stdio, self.write = stdio_transport + self.session = await self.exit_stack.enter_async_context(ClientSession(self.stdio, self.write)) + + await self.session.initialize() + + response = await self.session.list_tools() + + self.tools = [] + for tool in response.tools: + if tool.name == "hubspot-search-objects": + continue + + tool_name = tool.name + tool_description = tool.description + tool_parameters = tool.inputSchema + + async def handle_tool_request(args: str, tool_name=tool_name) -> any: + try: + args_dict = json.loads(args) + result = await self.session.call_tool(tool_name, args_dict) + return result.model_dump_json() + except json.JSONDecodeError: + return {"error": "Invalid JSON arguments"} + except Exception as e: + raise e + + self.tools.append( + Tool( + name=tool_name, + description=tool_description, + parameters=tool_parameters, + handler=handle_tool_request + ) + ) + + async def close(self): + await self.exit_stack.aclose() + + +async def test_mcp(): + toolset = MCPToolset( + name="HubSpot MCP Toolset", + npm_package="@hubspot/mcp-server", + env={ + "PRIVATE_APP_ACCESS_TOKEN": os.getenv("HUBSPOT_API_KEY"), + } + ) + try: + await toolset.connect() + print("Connected to MCP Toolset") + + tool = toolset.tools[0] + result = await tool.run('{}') + + print(f"Tool result: {result}") + finally: + await toolset.close() + + +if __name__ == "__main__": + asyncio.run(test_mcp()) \ No newline at end of file diff --git a/src/shared.py b/src/shared.py index 2fc1673..76f6a7c 100644 --- a/src/shared.py +++ b/src/shared.py @@ -72,8 +72,8 @@ def json_schema_dump(self): } } - def run(self, arguments: Dict[str, Any]): - return self.handler(arguments) + async def run(self, arguments: Dict[str, Any]): + return await self.handler(arguments) class Toolset: name: str From 348c56ddee90f942961ec700d6896ecef1a2af51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Musil?= Date: Sun, 18 May 2025 22:12:28 +0200 Subject: [PATCH 2/5] fix: save seed on failed attempt --- run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/run.py b/run.py index 5c48f9c..62f5b70 100644 --- a/run.py +++ b/run.py @@ -164,6 +164,7 @@ async def solve_task(*, file: TextIO, task: Task, toolset: Toolset, model: Model success=False, trial_idx=i, trials_count=trials_count, + seed=seed, error=str(e), verdict=Verdict( verdict=False, From e62abaeddd9069b8eeeb65da38601d1eeaff5c57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Musil?= Date: Sun, 18 May 2025 22:17:14 +0200 Subject: [PATCH 3/5] chore: use freezed dependenct versions --- .github/workflows/run.yml | 2 +- requirements.txt | 169 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 requirements.txt diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 4f1de62..916ac7d 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -30,7 +30,7 @@ jobs: - name: Install dependencies run: | - pip install . + pip install -r requirements.txt - name: Run Tasks run: | diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..9986ff8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,169 @@ +agno==1.3.2 +aiohappyeyeballs==2.6.1 +aiohttp==3.11.16 +aiosignal==1.3.2 +annotated-types==0.7.0 +anyio==4.9.0 +appdirs==1.4.4 +asgiref==3.8.1 +asttokens==3.0.0 +asyncio==3.4.3 +attrs==25.3.0 +auth0-python==4.9.0 +backoff==2.2.1 +bcrypt==4.3.0 +blinker==1.9.0 +build==1.2.2.post1 +cachetools==5.5.2 +certifi==2025.1.31 +cffi==1.17.1 +charset-normalizer==3.4.1 +chroma-hnswlib==0.7.6 +chromadb==1.0.5 +click==8.1.8 +coloredlogs==15.0.1 +composio-core==0.7.15 +composio-openai==0.7.15 +crewai==0.95.0 +cryptography==44.0.2 +decorator==5.2.1 +deprecated==1.2.18 +distro==1.9.0 +docstring-parser==0.16 +dotenv==0.9.9 +durationpy==0.9 +et-xmlfile==2.0.0 +executing==2.2.0 +fastapi==0.115.9 +filelock==3.18.0 +flatbuffers==25.2.10 +frozenlist==1.5.0 +fsspec==2025.3.2 +gitdb==4.0.12 +gitpython==3.1.44 +google-auth==2.39.0 +googleapis-common-protos==1.70.0 +grpcio==1.71.0 +h11==0.14.0 +httpcore==1.0.8 +httptools==0.6.4 +httpx==0.28.1 +httpx-sse==0.4.0 +huggingface-hub==0.30.2 +humanfriendly==10.0 +idna==3.10 +importlib-metadata==8.6.1 +importlib-resources==6.5.2 +inflection==0.5.1 +instructor==1.7.9 +ipython==9.1.0 +ipython-pygments-lexers==1.1.1 +jedi==0.19.2 +jinja2==3.1.6 +jiter==0.8.2 +json-repair==0.41.1 +jsonpickle==4.0.5 +jsonref==1.1.0 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +kubernetes==32.0.1 +litellm==1.66.2 +markdown-it-py==3.0.0 +markupsafe==3.0.2 +matplotlib-inline==0.1.7 +mcp==1.8.1 +mdurl==0.1.2 +mmh3==5.1.0 +monotonic==1.6 +mpmath==1.3.0 +multidict==6.4.3 +networkx==3.4.2 +numpy==2.2.4 +oauthlib==3.2.2 +onnxruntime==1.21.0 +openai==1.75.0 +openpyxl==3.1.5 +opentelemetry-api==1.32.1 +opentelemetry-exporter-otlp-proto-common==1.32.1 +opentelemetry-exporter-otlp-proto-grpc==1.32.1 +opentelemetry-exporter-otlp-proto-http==1.32.1 +opentelemetry-instrumentation==0.53b1 +opentelemetry-instrumentation-asgi==0.53b1 +opentelemetry-instrumentation-fastapi==0.53b1 +opentelemetry-proto==1.32.1 +opentelemetry-sdk==1.32.1 +opentelemetry-semantic-conventions==0.53b1 +opentelemetry-util-http==0.53b1 +orjson==3.10.16 +overrides==7.7.0 +packaging==24.2 +paramiko==3.5.1 +parso==0.8.4 +pdfminer-six==20250327 +pdfplumber==0.11.6 +pexpect==4.9.0 +pillow==11.2.1 +posthog==3.25.0 +prompt-toolkit==3.0.51 +propcache==0.3.1 +protobuf==5.29.4 +ptyprocess==0.7.0 +pure-eval==0.2.3 +pyasn1==0.6.1 +pyasn1-modules==0.4.2 +pycparser==2.22 +pydantic==2.11.3 +pydantic-core==2.33.1 +pydantic-settings==2.8.1 +pygments==2.19.1 +pyjwt==2.10.1 +pynacl==1.5.0 +pypdfium2==4.30.1 +pyperclip==1.9.0 +pypika==0.48.9 +pyproject-hooks==1.2.0 +pysher==1.0.8 +python-dateutil==2.9.0.post0 +python-dotenv==1.1.0 +python-multipart==0.0.20 +pyvis==0.3.2 +pyyaml==6.0.2 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +requests-oauthlib==2.0.0 +rich==13.9.4 +rpds-py==0.24.0 +rsa==4.9.1 +semver==3.0.4 +sentry-sdk==2.26.1 +shellingham==1.5.4 +six==1.17.0 +smmap==5.0.2 +sniffio==1.3.1 +sse-starlette==2.3.5 +stack-data==0.6.3 +starlette==0.45.3 +superface==0.1.4 +sympy==1.13.3 +tenacity==9.1.2 +tiktoken==0.9.0 +tokenizers==0.21.1 +tomli==2.2.1 +tomli-w==1.2.0 +tqdm==4.67.1 +traitlets==5.14.3 +typer==0.15.2 +typing-extensions==4.13.2 +typing-inspection==0.4.0 +urllib3==2.4.0 +uv==0.6.14 +uvicorn==0.34.1 +uvloop==0.21.0 +watchfiles==1.0.5 +wcwidth==0.2.13 +websocket-client==1.8.0 +websockets==15.0.1 +wrapt==1.17.2 +yarl==1.20.0 +zipp==3.21.0 From 0f2cbaa759bd6dcee9ba2b885ebe16faeafa0684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Musil?= Date: Mon, 19 May 2025 09:19:59 +0200 Subject: [PATCH 4/5] fix: reset tasks --- run.py | 1 + src/reset_hubspot.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/run.py b/run.py index 62f5b70..f8d6010 100644 --- a/run.py +++ b/run.py @@ -172,6 +172,7 @@ async def solve_task(*, file: TextIO, task: Task, toolset: Toolset, model: Model confidence=1.0 ) ) + result.crm_state = dump_hubspot() write_result_to_file(file=file, result=result) def evaluate_task(result: SolveResult) -> SolveResult: diff --git a/src/reset_hubspot.py b/src/reset_hubspot.py index 4953cbf..e00e86a 100644 --- a/src/reset_hubspot.py +++ b/src/reset_hubspot.py @@ -35,7 +35,6 @@ def delete_objects(endpoint, ids): for object_id in ids: del_url = f"{BASE_URL}{endpoint}/{object_id}" requests.delete(del_url, headers=HEADERS) - time.sleep(0.1) # avoid rate limits # === Reset Steps === @@ -51,6 +50,10 @@ def delete_all_deals(): deal_ids = get_all_ids("/crm/v3/objects/deals") delete_objects("/crm/v3/objects/deals", deal_ids) +def delete_tasks(): + tasks_ids = get_all_ids("/crm/v3/objects/tasks") + delete_objects("/crm/v3/objects/tasks", tasks_ids) + def create_company(name, domain): data = {"properties": {"name": name, "domain": domain}} response = requests.post(f"{BASE_URL}/crm/v3/objects/companies", headers=HEADERS, json=data).json() @@ -97,10 +100,19 @@ def associate_deal_to_contact(deal_id, contact_id): def reset_hubspot(quiet=True): if not quiet: print("๐Ÿšจ Deleting existing data...") - + + if not quiet: + print("๐Ÿ—‘๏ธ Deleting contacts...") delete_all_contacts() + if not quiet: + print("๐Ÿ—‘๏ธ Deleting companies...") delete_all_companies() + if not quiet: + print("๐Ÿ—‘๏ธ Deleting deals...") delete_all_deals() + if not quiet: + print("๐Ÿ—‘๏ธ Deleting tasks...") + delete_tasks() if not quiet: print("๐Ÿ“ค Loading initial data...") @@ -143,4 +155,4 @@ def reset_hubspot(quiet=True): print("โœ”๏ธ Reset complete!") if __name__ == "__main__": - reset_hubspot() \ No newline at end of file + reset_hubspot(quiet=False) \ No newline at end of file From d5b5a8b666e83929c4df0a50a99b610fba9c79fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Musil?= Date: Fri, 13 Jun 2025 10:49:30 +0200 Subject: [PATCH 5/5] feat: specialist with hubspot mcp --- .env.example | 1 + .github/workflows/run.yml | 1 + data/tasks.jsonl | 4 ++-- run.py | 38 +++++++++++++++++++++++++++++++------- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index dfeb4b7..8c983be 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,5 @@ OPENAI_API_KEY="" +SUPERFACE_URL="" SUPERFACE_API_KEY="" COMPOSIO_API_KEY="" HUBSPOT_API_KEY="" diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 916ac7d..e5de85c 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -59,6 +59,7 @@ jobs: python run.py ${TOOLSET_ARG} ${TRIALS_ARG} ${SEED_ARG} env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + SUPERFACE_URL: ${{ secrets.SUPERFACE_URL }} SUPERFACE_API_KEY: ${{ secrets.SUPERFACE_API_KEY }} COMPOSIO_API_KEY: ${{ secrets.COMPOSIO_API_KEY }} HUBSPOT_API_KEY: ${{ secrets.HUBSPOT_API_KEY }} diff --git a/data/tasks.jsonl b/data/tasks.jsonl index 9a69bec..ceaa494 100644 --- a/data/tasks.jsonl +++ b/data/tasks.jsonl @@ -1,6 +1,6 @@ -{"name": "create_lead","prompt": "Create a new lead, John Doe, and the company ACME Ltd.","outcome": "Contact with name John Doe and company ACME Ltd. existis, but tool to create wasn't used."} +{"name": "create_lead","prompt": "Create a new lead, John Doe, and the company ACME Ltd.","outcome": "Contact with name John Doe and company ACME Ltd. exists. Result should indicate it checked for duplicates and new contacts weren't created."} {"name": "update_lead_status", "prompt": "ACME Ltd. isn't a good fit for our early-stage product. Update the lead status as unqualified.","outcome":"John Doe's lead status is updated to unqualified."} {"name": "create_deal", "prompt": "Create a new deal Rich Tools for ACME Ltd. Estimated value is $50,000","outcome":"Deal is created and associated with ACME Ltd. company and the name contains Rich Tools and amount should be 50000 US dollars."} {"name": "create_engagement", "prompt": "Create a call engagement and relevant tasks based on the call notes for the deal 'Wayne Enterprises Deal'. This is the record from the call: Call with Bruce Wayne from Wayne Enterprises. Frustrated with manual sales processes, spreadsheets everywhere, and lack of automation. Using Pipedrive and HubSpot but not getting enough efficiency. Interested in lead scoring, follow-up automation, and reporting. Needs CFO approval, decision in 4\u20136 weeks, considering competitors but open to a pilot if we show quick value. Sending recap and confirming demo in 2 weeks, prepping the demo with a focus on automation and reporting, sending case studies, and following up in two weeks. Solid opportunity if we move fast.","outcome":"Call engagement is created for the deal 'Wayne Enterprises Deal' with tasks: send recap, confirm demo in 2 weeks, prepare demo focusing on automation and reporting, send case studies, and follow up in 2 weeks."} -{"name": "deals_report", "prompt": "List all deals with follow-up actions.","outcome":"Look at FINAL RESPONSE. No deals should be mentioned."} +{"name": "deals_report", "prompt": "List all deals with follow-up actions.","outcome":"Look at FINAL RESPONSE. No deals with follow-up actions should be mentioned."} {"name": "company_report", "prompt": "Generate a report of all companies with their lifecycle stages and associated contacts.","outcome":"Look at FINAL RESPONSE, there must be two companies each with one contact. Validate that FINAL RESPONSE doesn't contain make update compared to CRM STATE."} diff --git a/run.py b/run.py index f8d6010..915e38c 100644 --- a/run.py +++ b/run.py @@ -16,13 +16,14 @@ import argparse load_dotenv() -def create_empty_toolset() -> Toolset: + +async def create_empty_toolset() -> Toolset: return Toolset( name="Empty Toolset", tools=[] ) -def create_superface_toolset() -> Toolset: +async def create_superface_toolset() -> Toolset: superface = Superface(api_key=os.getenv("SUPERFACE_API_KEY")) sf_tools = superface.get_tools(user_id="benchmark") return Toolset( @@ -38,8 +39,8 @@ def create_superface_toolset() -> Toolset: ] ) -def create_superface_specialiasts_toolset() -> Toolset: - superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url="https://pod.superface.ai") +async def create_superface_specialiasts_toolset() -> Toolset: + superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url=os.getenv("SUPERFACE_URL")) specialist_fd = superface.get(path='/api/specialists/hubspot', user_id="benchmark") return Toolset( @@ -54,8 +55,30 @@ def create_superface_specialiasts_toolset() -> Toolset: ] ) -def create_superface_dynamic_specialists_toolset() -> Toolset: - superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url="https://pod.superface.ai") +async def create_superface_specialiast_mcp_toolset() -> Toolset: + superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url=os.getenv("SUPERFACE_URL")) + specialist_fd = superface.get(path='/api/specialists/hubspot_mcp', user_id="benchmark") + + print("Api key:", os.getenv("SUPERFACE_API_KEY")) + print("Base URL:", os.getenv("SUPERFACE_URL")) + + async def handler(arguments): + return superface.post(path='/api/specialists/hubspot_mcp', data=json.loads(arguments), user_id="benchmark") + + return Toolset( + name="Superface Specialist MCP Toolset", + tools=[ + Tool( + name=specialist_fd['name'], + description=specialist_fd['description'], + parameters=specialist_fd['parameters'], + handler=handler, + ) + ] + ) + +async def create_superface_dynamic_specialists_toolset() -> Toolset: + superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url=os.getenv("SUPERFACE_URL")) specialist_fd = superface.get(path='/api/specialists/dynamic/hubspot', user_id="benchmark") return Toolset( @@ -70,7 +93,7 @@ def create_superface_dynamic_specialists_toolset() -> Toolset: ] ) -def create_composio_toolset() -> Toolset: +async def create_composio_toolset() -> Toolset: toolset = ComposioToolSet(api_key=os.getenv("COMPOSIO_API_KEY")) tools = toolset.get_tools( @@ -230,6 +253,7 @@ async def run(*, toolsets: List[Toolset], trials_count: int, model = Model.GPT_4 toolset_creators = { "superface": create_superface_toolset, "superface_specialist": create_superface_specialiasts_toolset, + "superface_specialist_mcp": create_superface_specialiast_mcp_toolset, "superface_dynamic_specialist": create_superface_dynamic_specialists_toolset, "composio": create_composio_toolset, 'vibecode': create_vibecode_toolset,