diff --git a/.env.example b/.env.example index dfeb4b7..ed65193 100644 --- a/.env.example +++ b/.env.example @@ -4,4 +4,10 @@ COMPOSIO_API_KEY="" HUBSPOT_API_KEY="" OPENAI_STORE_COMPLETIONS="" -COMPOSIO_LOGGING_LEVEL="" \ No newline at end of file +COMPOSIO_LOGGING_LEVEL="" + +SALESFORCE_USERNAME="" +SALESFORCE_PASSWORD="" +SALESFORCE_SECURITY_TOKEN="" +# use the domain name including the .my suffix, for example: superface-dev-ed.develop.my +SALESFORCE_DOMAIN="" \ No newline at end of file diff --git a/README.md b/README.md index 2cde922..8ec0775 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ Run the benchmark for specified toolsets: - `--toolsets`: List of toolsets you want to run the benchmark for - `--seed` *(optional)*: Specify a seed that is passed to LLMs (Default: none) - `--trials` *(optional)*: Specify how many times each toolset<>task pair should run (Default: 5) +- `--crm`: hubspot or salesforce (Default: hubspot) ```bash python run.py --toolsets superface superface_specialist superface_dynamic_specialist composio vibecode --seed 42 --trials 10 diff --git a/data/leads.jsonl b/data/leads.jsonl new file mode 100644 index 0000000..e8c581b --- /dev/null +++ b/data/leads.jsonl @@ -0,0 +1 @@ +{"lead_id": "lead_001", "name": "Peter Pan", "email": "peter_pan@acme.com", "company": "ACME Inc.", "lead_status": "New"} diff --git a/pyproject.toml b/pyproject.toml index 91b36c2..f124cef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,5 +12,6 @@ dependencies = [ "litellm", "superface", "requests", - "composio-openai" + "composio-openai", + "simple-salesforce" ] diff --git a/run.py b/run.py index 5843dfc..474d5f1 100644 --- a/run.py +++ b/run.py @@ -6,9 +6,11 @@ from composio_openai import ComposioToolSet, Action from typing import List, Optional, TextIO from src.reset_hubspot import reset_hubspot +from src.reset_salesforce import reset_salesforce from src.shared import Model, Task, Tool, Toolset, SolveResult, Verdict from src.crm_agent import CRMAgent from src.dump_hubspot import dump_hubspot +from src.dump_salesforce import dump_salesforce from src.evaluator import Evaluator from src.vibecode_toolset import create_vibecode_toolset import argparse @@ -36,46 +38,54 @@ def create_superface_toolset() -> Toolset: ] ) -def create_superface_specialiasts_toolset() -> Toolset: +def create_superface_specialiasts_toolset(crm: str = "hubspot") -> Toolset: superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url="https://pod.superface.ai") - specialist_fd = superface.get(path='/api/specialists/hubspot', user_id="benchmark") - + user_id = "benchmark" + if crm == "hubspot": + path = '/api/specialists/hubspot' + elif crm == "salesforce": + path = '/api/specialists/salesforce' + else: + raise ValueError(f"Unsupported CRM: {crm}") + specialist_fd = superface.get(path=path, user_id=user_id) return Toolset( - name="Superface Specialist Toolset", + name=f"Superface Specialist Toolset ({crm})", tools=[ Tool( name=specialist_fd['name'], description=specialist_fd['description'], parameters=specialist_fd['parameters'], - handler=lambda arguments: superface.post(path='/api/specialists/hubspot', data=json.loads(arguments), user_id="benchmark"), + handler=lambda arguments: superface.post(path=path, data=json.loads(arguments), user_id=user_id), ) ] ) -def create_superface_dynamic_specialists_toolset() -> Toolset: +def create_superface_dynamic_specialists_toolset(crm: str = "hubspot") -> Toolset: superface = SuperfaceAPI(api_key=os.getenv("SUPERFACE_API_KEY"), base_url="https://pod.superface.ai") - specialist_fd = superface.get(path='/api/specialists/dynamic/hubspot', user_id="benchmark") - + user_id = "benchmark" + if crm == "hubspot": + path = '/api/specialists/dynamic/hubspot' + elif crm == "salesforce": + path = '/api/specialists/dynamic/salesforce' + else: + raise ValueError(f"Unsupported CRM: {crm}") + specialist_fd = superface.get(path=path, user_id=user_id) return Toolset( - name="Superface Dynamic Specialist Toolset", + name=f"Superface Dynamic Specialist Toolset ({crm})", tools=[ Tool( name=specialist_fd['name'], description=specialist_fd['description'], parameters=specialist_fd['parameters'], - handler=lambda arguments: superface.post(path='/api/specialists/dynamic/hubspot', data=json.loads(arguments), user_id="benchmark"), + handler=lambda arguments: superface.post(path=path, data=json.loads(arguments), user_id=user_id), ) ] ) -def create_composio_toolset() -> Toolset: +def create_composio_toolset(crm: str = "hubspot") -> Toolset: toolset = ComposioToolSet(api_key=os.getenv("COMPOSIO_API_KEY")) - - tools = toolset.get_tools( - # filtering by tags doesn't work: https://github.com/ComposioHQ/composio/issues/1548 - # apps=[App.HUBSPOT], - # tags=[Tag.HUBSPOT_CORE, Tag.HUBSPOT_BASIC], - actions=[ + if crm == "hubspot": + actions = [ Action.HUBSPOT_CREATE_CONTACT_OBJECT_WITH_PROPERTIES, Action.HUBSPOT_CREATE_COMPANY_OBJECT, Action.HUBSPOT_SEARCH_CONTACTS_BY_CRITERIA, @@ -85,11 +95,37 @@ def create_composio_toolset() -> Toolset: Action.HUBSPOT_READ_PROPERTY_GROUPS_FOR_OBJECT_TYPE, Action.HUBSPOT_LIST_ASSOCIATION_TYPES, Action.HUBSPOT_CREATE_BATCH_OF_OBJECTS, - ], - ) - + ] + elif crm == "salesforce": + actions = [ + Action.SALESFORCE_CREATE_LEAD_WITH_SPECIFIED_CONTENT_TYPE, + Action.SALESFORCE_CREATE_NEW_CONTACT_WITH_JSON_HEADER, + Action.SALESFORCE_CREATE_NOTE_RECORD_WITH_CONTENT_TYPE_HEADER, + Action.SALESFORCE_CREATE_OPPORTUNITY_RECORD, + Action.SALESFORCE_DELETE_A_LEAD_OBJECT_BY_ITS_ID, + Action.SALESFORCE_FETCH_ACCOUNT_DETAILS_BY_ID_WITH_CONDITIONAL_QUERIES, + Action.SALESFORCE_FETCH_MODIFIED_OR_UNMODIFIED_SOBJECTS, + Action.SALESFORCE_QUERY_REPORT, + Action.SALESFORCE_RETRIEVE_ACCOUNT_DATA_AND_ERROR_RESPONSES, + Action.SALESFORCE_RETRIEVE_CONTACT_INFO_WITH_STANDARD_RESPONSES, + Action.SALESFORCE_RETRIEVE_LEAD_DATA_WITH_VARIOUS_RESPONSES, + Action.SALESFORCE_RETRIEVE_LEAD_DETAILS_BY_ID_WITH_CONDITIONAL_SUPPORT, + Action.SALESFORCE_RETRIEVE_NOTE_BY_ID_WITH_OPTIONAL_FIELDS_AND_TIME_CONDITIONS, + Action.SALESFORCE_RETRIEVE_NOTE_OBJECT_INFORMATION, + Action.SALESFORCE_RETRIEVE_OPPORTUNITIES_DATA, + Action.SALESFORCE_RETRIEVE_OPPORTUNITY_BY_ID_WITH_OPTIONAL_FIELDS, + Action.SALESFORCE_RETRIEVE_SPECIFIC_CONTACT_BY_ID, + Action.SALESFORCE_UPDATE_ACCOUNT_OBJECT_BY_ID, + Action.SALESFORCE_UPDATE_CONTACT_BY_ID, + Action.SALESFORCE_UPDATE_LEAD_BY_ID_WITH_JSON_PAYLOAD, + Action.SALESFORCE_UPDATE_OPPORTUNITY_BY_ID, + Action.SALESFORCE_UPDATE_SPECIFIC_NOTE_BY_ID + ] + else: + raise ValueError(f"Unsupported CRM: {crm}") + tools = toolset.get_tools(actions=actions) return Toolset( - name="Composio Toolset", + name=f"Composio Toolset ({crm})", tools=[ Tool( name=tool['function']['name'], @@ -114,7 +150,7 @@ def load_tasks(slice: Optional[slice] = None) -> List[Task]: tasks = tasks[slice] return tasks -def solve_task(*, file: TextIO, task: Task, toolset: Toolset, model: Model, trials_count: int, seed: Optional[int] = None): +def solve_task(*, file: TextIO, task: Task, toolset: Toolset, model: Model, trials_count: int, seed: Optional[int] = None, crm: str = "hubspot"): agent = CRMAgent( model=model, tools=toolset @@ -125,14 +161,24 @@ def solve_task(*, file: TextIO, task: Task, toolset: Toolset, model: Model, tria print(f"๐Ÿ› ๏ธ Task {task.name} {i}/{trials_count}") print("๐Ÿงน Resetting CRM...") - reset_hubspot() + if crm == "hubspot": + reset_hubspot() + elif crm == "salesforce": + reset_salesforce() + else: + raise ValueError(f"Unsupported CRM: {crm}") result = agent.solve(task=task, seed=seed) result.trial_idx = i result.trials_count = trials_count print("๐Ÿ—‚๏ธ Dumping CRM state...") - result.crm_state = dump_hubspot() + if crm == "hubspot": + result.crm_state = dump_hubspot() + elif crm == "salesforce": + result.crm_state = dump_salesforce() + else: + raise ValueError(f"Unsupported CRM: {crm}") print("๐Ÿงช Evaluating task...") result = evaluate_task(result=result) @@ -205,13 +251,13 @@ def dump_hubspot_state(): hubspot_state = dump_hubspot() print(f"HubSpot State: {hubspot_state}") -def run(*, toolsets: List[Toolset], trials_count: int, model = Model.GPT_4o, seed: Optional[int] = None): +def run(*, toolsets: List[Toolset], trials_count: int, model = Model.GPT_4o, seed: Optional[int] = None, crm: str = "hubspot"): tasks = load_tasks() for toolset in toolsets: print(f"Running tasks for toolset: {toolset.name}") with open_results_file(toolset) as file: for task in tasks: - solve_task(task=task, toolset=toolset, model=model, trials_count=trials_count, seed=seed, file=file) + solve_task(task=task, toolset=toolset, model=model, trials_count=trials_count, seed=seed, file=file, crm=crm) toolset_creators = { "superface": create_superface_toolset, @@ -232,6 +278,13 @@ def run(*, toolsets: List[Toolset], trials_count: int, model = Model.GPT_4o, see required=True, help=f"Specify one or more toolsets to run: {', '.join(toolset_options)}" ) + parser.add_argument( + "--crm", + type=str, + default="hubspot", + choices=["hubspot", "salesforce"], + help="Specify the CRM to use (default: hubspot)" + ) parser.add_argument( "--trials", type=int, @@ -246,10 +299,15 @@ def run(*, toolsets: List[Toolset], trials_count: int, model = Model.GPT_4o, see ) args = parser.parse_args() - selected_toolsets = [toolset_creators[toolset]() for toolset in args.toolsets] + selected_toolsets = [ + toolset_creators[toolset](crm=args.crm) if toolset in ["superface", "superface_specialist", "superface_dynamic_specialist", "composio"] + else toolset_creators[toolset]() + for toolset in args.toolsets + ] run( toolsets=selected_toolsets, trials_count=args.trials, - seed=args.seed + seed=args.seed, + crm=args.crm ) diff --git a/src/dump_salesforce.py b/src/dump_salesforce.py new file mode 100644 index 0000000..8cb5203 --- /dev/null +++ b/src/dump_salesforce.py @@ -0,0 +1,109 @@ +import os +from dotenv import load_dotenv +from simple_salesforce import Salesforce +from .shared import CrmState, CrmStateEngagements + +load_dotenv(override=True) + +# ๐Ÿ”ง CONFIGURATION +SF_USERNAME = os.getenv("SALESFORCE_USERNAME") +SF_PASSWORD = os.getenv("SALESFORCE_PASSWORD") +SF_SECURITY_TOKEN = os.getenv("SALESFORCE_SECURITY_TOKEN") +SF_DOMAIN = os.getenv("SALESFORCE_DOMAIN", "login") + +sf = Salesforce( + username=SF_USERNAME, + password=SF_PASSWORD, + security_token=SF_SECURITY_TOKEN, + domain=SF_DOMAIN +) + +# Property mappings for Salesforce objects +properties_map = { + "leads": ["Id", "FirstName", "LastName", "Email", "Phone", "Company", "Status", "LeadSource"], + "contacts": ["Id", "FirstName", "LastName", "Email", "Phone", "AccountId", "Title", "LeadSource"], + "accounts": ["Id", "Name", "Industry", "NumberOfEmployees", "AnnualRevenue", "Website"], + "opportunities": ["Id", "Name", "Amount", "StageName", "CloseDate", "AccountId", "LeadSource", "Probability"], + "tasks": ["Id", "Subject", "Status", "Priority", "ActivityDate", "OwnerId", "WhatId", "WhoId"], + "calls": ["Id", "Subject", "Status", "Priority", "ActivityDate", "CallType", "CallDurationInSeconds", "Description", "OwnerId", "WhatId", "WhoId"], + "notes": ["Id", "Title", "Body", "OwnerId", "ParentId", "CreatedDate", "LastModifiedDate"], + "meetings": ["Id", "Subject", "StartDateTime", "EndDateTime", "Location", "OwnerId", "WhatId", "WhoId"], +} + +def dump_salesforce(): + """ + Dumps the current state of Salesforce data into list of CrmState class + """ + leads = get_all_objects("leads") + contacts = get_all_objects("contacts") + accounts = get_all_objects("accounts") + opportunities = get_all_objects("opportunities") + tasks = get_all_objects("tasks") + calls = get_all_objects("calls") + notes = get_all_objects("notes") + meetings = get_all_objects("meetings") + + engagements = CrmStateEngagements( + emails=[], # Salesforce does not have a direct Email object in standard REST API + notes=notes, + calls=calls, + meetings=meetings, + tasks=tasks, + ) + + salesforce_state = CrmState( + leads=leads, + contacts=contacts, + companies=accounts, + deals=opportunities, + engagements=engagements, + ) + + return salesforce_state + +def get_all_objects(object_type): + """ + Get all objects for the given Salesforce object type + """ + if object_type == "leads": + soql = f"SELECT {', '.join(properties_map['leads'])} FROM Lead" + records = sf.query_all(soql)["records"] + elif object_type == "contacts": + soql = f"SELECT {', '.join(properties_map['contacts'])} FROM Contact" + records = sf.query_all(soql)["records"] + elif object_type == "accounts": + soql = f"SELECT {', '.join(properties_map['accounts'])} FROM Account" + records = sf.query_all(soql)["records"] + elif object_type == "opportunities": + soql = f"SELECT {', '.join(properties_map['opportunities'])} FROM Opportunity" + records = sf.query_all(soql)["records"] + elif object_type == "tasks": + # Print all unique TaskSubtype values + subtype_query = "SELECT TaskSubtype FROM Task GROUP BY TaskSubtype" + subtype_records = sf.query_all(subtype_query)["records"] + unique_subtypes = [rec["TaskSubtype"] for rec in subtype_records] + soql = f"SELECT {', '.join(properties_map['tasks'])} FROM Task" + records = sf.query_all(soql)["records"] + elif object_type == "calls": + soql = f"SELECT {', '.join(properties_map['calls'])} FROM Task WHERE TaskSubtype = 'Call'" + records = sf.query_all(soql)["records"] + elif object_type == "notes": + # Notes are stored in ContentNote or Note, depending on Salesforce org + try: + soql = f"SELECT {', '.join(properties_map['notes'])} FROM Note" + records = sf.query_all(soql)["records"] + except Exception: + records = [] + elif object_type == "meetings": + # Meetings are Event objects with Subject contains 'Meeting' + soql = f"SELECT {', '.join(properties_map['meetings'])} FROM Event WHERE Subject LIKE '%Meeting%'" + records = sf.query_all(soql)["records"] + else: + records = [] + + print(f"Found {len(records)} {object_type} in Salesforce") + + return records + +if __name__ == "__main__": + dump_salesforce() \ No newline at end of file diff --git a/src/reset_salesforce.py b/src/reset_salesforce.py new file mode 100644 index 0000000..e765ef4 --- /dev/null +++ b/src/reset_salesforce.py @@ -0,0 +1,177 @@ +from dotenv import load_dotenv +import os +import json +import time +from simple_salesforce import Salesforce +import base64 + +load_dotenv(override=True) + +# ๐Ÿ”ง CONFIGURATION +SF_USERNAME = os.getenv("SALESFORCE_USERNAME") +SF_PASSWORD = os.getenv("SALESFORCE_PASSWORD") +SF_SECURITY_TOKEN = os.getenv("SALESFORCE_SECURITY_TOKEN") +SF_DOMAIN = os.getenv("SALESFORCE_DOMAIN", "login") + +sf = Salesforce( + username=SF_USERNAME, + password=SF_PASSWORD, + security_token=SF_SECURITY_TOKEN, + domain=SF_DOMAIN +) + +# === Utility Functions === +def get_all_ids(object_name): + soql = f"SELECT Id FROM {object_name}" + return [item['Id'] for item in sf.query_all(soql)['records']] + +def delete_objects(object_name): + ids = get_all_ids(object_name) + for object_id in ids: + getattr(sf, object_name).delete(object_id) + time.sleep(0.1) # avoid rate limits + +# === Reset Steps === +def delete_all_cases(): + delete_objects('Case') + +def delete_all_leads(): + delete_objects('Lead') + +def delete_all_contacts(): + delete_objects('Contact') + +def delete_all_accounts(): + delete_objects('Account') + +def delete_all_opportunities(): + delete_objects('Opportunity') + +def delete_all_entitlements(): + delete_objects('Entitlement') + +def create_account(name, domain): + data = {"Name": name, "Website": domain} + result = sf.Account.create(data) + return result.get("id") + +def create_contact(name, email, lead_source, account_id=None, title=None): + first, last = name.split(" ", 1) if " " in name else (name, "") + data = { + "FirstName": first, + "LastName": last, + "Email": email, + "LeadSource": lead_source, + } + if account_id: + data["AccountId"] = account_id + if title: + data["Title"] = title + result = sf.Contact.create(data) + return result.get("id") + +def create_lead(name, email, lead_status, company): + first, last = name.split(" ", 1) if " " in name else (name, "") + data = { + "FirstName": first, + "LastName": last, + "Email": email, + "Status": lead_status, + "Company": company + } + result = sf.Lead.create(data) + return result.get("id") + +def create_opportunity(name, amount, stage, account_id=None): + # Build data dictionary with all required fields and fixed picklist values + data = { + "Name": name, + "Amount": amount, + "StageName": "Prospecting", + "Type": "CSPSellOut", + "LeadSource": "Web", + "DeliveryInstallationStatus__c": "In progress", + "Primary_Competitor_Line_of_Business__c": "Commercial Client", + "Primary_Competitor__c": "Alibaba", + "Implementation_Type__c": "Distribuce", + "Pre_sales_partner_activities__c": "EligibleBusinessCase", + "Solution_Domain__c": "BasicHardwareSoftwareSale", + "Solution_Type__c": "APOS", + "Solution_Name__c": "ExpansionOrGrowth", + "APEX_Opportunity__c": "No", + "ISG_Tech_Refresh_Oppty__c": "No", + "Solution_Scope__c": "ProfessionalServicesOnly", + "Sales_Service_Delivery_Type__c": "DellServicesResell", + "CloseDate": "2025-01-01", + "AccountId": account_id + } + + if account_id: + data["AccountId"] = account_id + + try: + result = sf.Opportunity.create(data) + return result.get("id") + except Exception as e: + print(f"Error creating opportunity {name}: {e}") + print(f"Data used: {data}") + raise + +# === Main === +def reset_salesforce(quiet=True): + if not quiet: + print("๐Ÿšจ Deleting existing Salesforce data...") + delete_all_cases() + delete_all_leads() + delete_all_contacts() + delete_all_opportunities() + delete_all_entitlements() + delete_all_accounts() + + if not quiet: + print("๐Ÿ“ค Loading initial data...") + + base_dir = os.path.dirname(os.path.abspath(__file__)) + companies_file = os.path.join(base_dir, "../data/companies.jsonl") + contacts_file = os.path.join(base_dir, "../data/contacts.jsonl") + leads_file = os.path.join(base_dir, "../data/leads.jsonl") + deals_file = os.path.join(base_dir, "../data/deals.jsonl") + + with open(companies_file, "r") as f: + companies_data = [json.loads(line) for line in f.readlines()] + with open(contacts_file, "r") as f: + contacts_data = [json.loads(line) for line in f.readlines()] + with open(leads_file, "r") as f: + leads_data = [json.loads(line) for line in f.readlines()] + with open(deals_file, "r") as f: + deals_data = [json.loads(line) for line in f.readlines()] + + if not quiet: + print("๐Ÿข Creating accounts...") + account_map = {} + for c in companies_data: + account_id = create_account(c["name"], c.get("domain", "")) + account_map[c["company_id"]] = account_id + + if not quiet: + print("๐Ÿ‘ค Creating contacts and linking...") + for contact in contacts_data: + account_id = account_map.get(contact.get("company_id")) + create_contact(contact["name"], contact["email"], contact.get("lead_source", ""), account_id, contact.get("title")) + + if not quiet: + print("๐Ÿง‘โ€๐Ÿ’ผ Creating leads...") + for lead in leads_data: + create_lead(lead["name"], lead["email"], lead["lead_status"], lead["company"]) + + if not quiet: + print("๐Ÿ’ผ Creating opportunities and linking...") + for deal in deals_data: + account_id = account_map.get(deal.get("company_id")) + create_opportunity(deal["name"], deal["amount"], deal["stage"], account_id) + + if not quiet: + print("โœ”๏ธ Salesforce reset complete!") + +if __name__ == "__main__": + reset_salesforce() \ No newline at end of file