diff --git a/sdk/ai/azure-ai-projects/CHANGELOG.md b/sdk/ai/azure-ai-projects/CHANGELOG.md index 6ae8e13f480e..e7ef38b4e29e 100644 --- a/sdk/ai/azure-ai-projects/CHANGELOG.md +++ b/sdk/ai/azure-ai-projects/CHANGELOG.md @@ -4,6 +4,9 @@ ### Features added +* Added agent tool call tracing support for all tool call types when streaming with AgentEventHandler based event handler +* Added tracing for listing agent run steps + ### Sample updates ### Bugs Fixed diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/telemetry/agents/_ai_agents_instrumentor.py b/sdk/ai/azure-ai-projects/azure/ai/projects/telemetry/agents/_ai_agents_instrumentor.py index 2d4dcb1d1b54..95f7c7cf185c 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/telemetry/agents/_ai_agents_instrumentor.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/telemetry/agents/_ai_agents_instrumentor.py @@ -9,6 +9,7 @@ import json import logging import os +from datetime import datetime from enum import Enum from typing import Any, Callable, Dict, List, Optional, Tuple, Union, cast from urllib.parse import urlparse @@ -20,12 +21,13 @@ MessageAttachment, MessageDeltaChunk, MessageIncompleteDetails, - RequiredFunctionToolCall, RunStep, RunStepDeltaChunk, + RunStepError, RunStepFunctionToolCall, RunStepToolCallDetails, - SubmitToolOutputsAction, + RunStepCodeInterpreterToolCall, + RunStepBingGroundingToolCall, ThreadMessage, ThreadRun, ToolDefinition, @@ -50,6 +52,10 @@ GEN_AI_THREAD_RUN_STATUS, GEN_AI_USAGE_INPUT_TOKENS, GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_RUN_STEP_START_TIMESTAMP, + GEN_AI_RUN_STEP_END_TIMESTAMP, + GEN_AI_RUN_STEP_STATUS, + ERROR_MESSAGE, OperationName, start_span, ) @@ -254,6 +260,12 @@ def _create_event_attributes( thread_run_id: Optional[str] = None, message_id: Optional[str] = None, message_status: Optional[str] = None, + run_step_status: Optional[str] = None, + created_at: Optional[datetime] = None, + completed_at: Optional[datetime] = None, + cancelled_at: Optional[datetime] = None, + failed_at: Optional[datetime] = None, + run_step_last_error: Optional[RunStepError] = None, usage: Optional[_models.RunStepCompletionUsage] = None, ) -> Dict[str, Any]: attrs: Dict[str, Any] = {GEN_AI_SYSTEM: AZ_AI_AGENT_SYSTEM} @@ -272,6 +284,34 @@ def _create_event_attributes( if message_status: attrs[GEN_AI_MESSAGE_STATUS] = self._status_to_string(message_status) + if run_step_status: + attrs[GEN_AI_RUN_STEP_STATUS] = self._status_to_string(run_step_status) + + if created_at: + if isinstance(created_at, datetime): + attrs[GEN_AI_RUN_STEP_START_TIMESTAMP] = created_at.isoformat() + else: + # fallback in case int or string gets passed + attrs[GEN_AI_RUN_STEP_START_TIMESTAMP] = str(created_at) + + end_timestamp = None + if completed_at: + end_timestamp = completed_at + elif cancelled_at: + end_timestamp = cancelled_at + elif failed_at: + end_timestamp = failed_at + + if isinstance(end_timestamp, datetime): + attrs[GEN_AI_RUN_STEP_END_TIMESTAMP] = end_timestamp.isoformat() + elif end_timestamp: + # fallback in case int or string gets passed + attrs[GEN_AI_RUN_STEP_END_TIMESTAMP] = str(end_timestamp) + + if run_step_last_error: + attrs[ERROR_MESSAGE] = run_step_last_error.message + attrs[ERROR_TYPE] = run_step_last_error.code + if usage: attrs[GEN_AI_USAGE_INPUT_TOKENS] = usage.prompt_tokens attrs[GEN_AI_USAGE_OUTPUT_TOKENS] = usage.completion_tokens @@ -306,6 +346,152 @@ def add_thread_message_event( usage=usage, ) + def _process_tool_calls(self, step: RunStep) -> List[Dict[str, Any]]: + """ + Helper method to process tool calls and return a list of tool call dictionaries. + + :param step: The run step containing tool call details to be processed. + :type step: RunStep + :return: A list of dictionaries, each representing a processed tool call. + :rtype: List[Dict[str, Any]] + """ + tool_calls = [] + tool_call: Dict[str, Any] = {} + for t in cast(RunStepToolCallDetails, step.step_details).tool_calls: + if not _trace_agents_content: + tool_call = { + "id": t.id, + "type": t.type, + } + elif isinstance(t, RunStepFunctionToolCall): + try: + parsed_arguments = json.loads(t.function.arguments) + except json.JSONDecodeError: + parsed_arguments = {} + + tool_call = { + "id": t.id, + "type": t.type, + "function": { + "name": t.function.name, + "arguments": parsed_arguments, + }, + } + elif isinstance(t, RunStepCodeInterpreterToolCall): + tool_call = { + "id": t.id, + "type": t.type, + "code_interpreter": { + "input": t.code_interpreter.input, + "outputs": [output.as_dict() for output in t.code_interpreter.outputs], + }, + } + elif isinstance(t, RunStepBingGroundingToolCall): + tool_call = { + "id": t.id, + "type": t.type, + t.type: {"bing_grounding": t.bing_grounding}, + } + else: + tool_details = t.as_dict()[t.type] + + tool_call = { + "id": t.id, + "type": t.type, + t.type: tool_details, + } + tool_calls.append(tool_call) + return tool_calls + + def _add_tool_call_event( + self, + span, + step: RunStep, + event_name: str, + is_run_step_listing: bool = False, + ) -> None: + """ + Adds a tool call event to a span. + + This method processes tool calls from a given run step and adds them as an event + to the provided span. It includes relevant attributes such as the run step status, + timestamps, tool call details, and optionally the message status. + + :param span: The span instance where the tool call event will be recorded. + :type span: AbstractSpan + :param step: The run step containing details about the tool calls to be processed. + :type step: RunStep + :param event_name: The name of the event to be added to the span (e.g., "gen_ai.run_step.tool_calls"). + :type event_name: str + :param is_run_step_listing: A flag indicating whether the event is part of a run step listing. + If True, the run step status is included in the attributes; otherwise, the message status is included. + :type is_run_step_listing: bool + :return: None + """ + tool_calls = self._process_tool_calls(step) + + run_step_status = None + message_status = None + if is_run_step_listing: + run_step_status = step.status + else: + message_status = step.status + + attributes = self._create_event_attributes( + thread_id=step.thread_id, + agent_id=step.agent_id, + thread_run_id=step.run_id, + message_status=message_status, + run_step_status=run_step_status, + created_at=step.created_at, + completed_at=step.completed_at, + cancelled_at=step.cancelled_at, + failed_at=step.failed_at, + run_step_last_error=step.last_error, + usage=step.usage, + ) + + if tool_calls: + attributes[GEN_AI_EVENT_CONTENT] = json.dumps({"tool_calls": tool_calls}, ensure_ascii=False) + + span.span_instance.add_event(name=event_name, attributes=attributes) + + def add_run_step_event(self, span, step: RunStep) -> None: + """ + Adds a run step event to the span. + + This method determines the type of the run step and adds the appropriate event + to the provided span. It processes either a "message_creation" or "tool_calls" + run step and includes relevant attributes such as the run step status, timestamps, + and tool call or message details. + + :param span: The span instance where the run step event will be recorded. + :type span: AbstractSpan + :param step: The run step containing details about the event to be added. + :type step: RunStep + :return: None + """ + if step["type"] == "message_creation": + self._add_message_creation_run_step_event(span, step) + elif step["type"] == "tool_calls": + self._add_tool_call_event(span, step, "gen_ai.run_step.tool_calls", is_run_step_listing=True) + + def _add_message_creation_run_step_event(self, span, step: RunStep) -> None: + attributes = self._create_event_attributes( + thread_id=step.thread_id, + agent_id=step.agent_id, + thread_run_id=step.run_id, + message_id=step["step_details"]["message_creation"]["message_id"], + run_step_status=step.status, + created_at=step.created_at, + completed_at=step.completed_at, + cancelled_at=step.cancelled_at, + failed_at=step.failed_at, + run_step_last_error=step.last_error, + usage=step.usage, + ) + span.span_instance.add_event(name="gen_ai.run_step.message_creation", attributes=attributes) + def _add_message_event( self, span, @@ -392,67 +578,7 @@ def _status_to_string(self, status: Any) -> str: return status.value if hasattr(status, "value") else status def _add_tool_assistant_message_event(self, span, step: RunStep) -> None: - tool_calls = [ - { - "id": t.id, - "type": t.type, - "function": ( - {"name": t.function.name, "arguments": json.loads(t.function.arguments)} - if isinstance(t, RunStepFunctionToolCall) - else None - ), - } - for t in cast(RunStepToolCallDetails, step.step_details).tool_calls - ] - - attributes = self._create_event_attributes( - thread_id=step.thread_id, - agent_id=step.agent_id, - thread_run_id=step.run_id, - message_status=step.status, - usage=step.usage, - ) - - if _trace_agents_content: - attributes[GEN_AI_EVENT_CONTENT] = json.dumps({"tool_calls": tool_calls}, ensure_ascii=False) - else: - tool_calls_non_recording = self._remove_function_call_names_and_arguments(tool_calls=tool_calls) - attributes[GEN_AI_EVENT_CONTENT] = json.dumps({"tool_calls": tool_calls_non_recording}, ensure_ascii=False) - span.span_instance.add_event(name="gen_ai.assistant.message", attributes=attributes) - - def _add_tool_event_from_thread_run(self, span, run: ThreadRun) -> None: - tool_calls = [] - - for t in run.required_action.submit_tool_outputs.tool_calls: # type: ignore - try: - parsed_arguments = json.loads(t.function.arguments) - except json.JSONDecodeError: - parsed_arguments = {} - - tool_call = { - "id": t.id, - "type": t.type, - "function": ( - {"name": t.function.name, "arguments": parsed_arguments} - if isinstance(t, RequiredFunctionToolCall) - else None - ), - } - tool_calls.append(tool_call) - - attributes = self._create_event_attributes( - thread_id=run.thread_id, - agent_id=run.agent_id, - thread_run_id=run.id, - message_status=run.status, - ) - - if _trace_agents_content: - attributes[GEN_AI_EVENT_CONTENT] = json.dumps({"tool_calls": tool_calls}) - else: - tool_calls_non_recording = self._remove_function_call_names_and_arguments(tool_calls=tool_calls) - attributes[GEN_AI_EVENT_CONTENT] = json.dumps({"tool_calls": tool_calls_non_recording}) - span.span_instance.add_event(name="gen_ai.assistant.message", attributes=attributes) + self._add_tool_call_event(span, step, "gen_ai.assistant.message", is_run_step_listing=False) def set_end_run(self, span: "AbstractSpan", run: Optional[ThreadRun]) -> None: if run and span and span.span_instance.is_recording: @@ -608,6 +734,11 @@ def start_create_thread_span( def start_list_messages_span(self, project_name: str, thread_id: Optional[str] = None) -> "Optional[AbstractSpan]": return start_span(OperationName.LIST_MESSAGES, project_name, thread_id=thread_id) + def start_list_run_steps_span( + self, project_name: str, run_id: Optional[str] = None, thread_id: Optional[str] = None + ) -> "Optional[AbstractSpan]": + return start_span(OperationName.LIST_RUN_STEPS, project_name, run_id=run_id, thread_id=thread_id) + def trace_create_agent(self, function, *args, **kwargs): project_name = args[ # pylint: disable=protected-access # pyright: ignore [reportFunctionMemberAccess] 0 @@ -1248,6 +1379,74 @@ def trace_list_messages(self, function, *args, **kwargs): return result + def trace_list_run_steps(self, function, *args, **kwargs): + project_name = args[ # pylint: disable=protected-access # pyright: ignore [reportFunctionMemberAccess] + 0 + ]._config.project_name + run_id = kwargs.get("run_id") + thread_id = kwargs.get("thread_id") + + span = self.start_list_run_steps_span(project_name=project_name, run_id=run_id, thread_id=thread_id) + + if span is None: + return function(*args, **kwargs) + + with span: + try: + result = function(*args, **kwargs) + if hasattr(result, "data") and result.data is not None: + for step in result.data: + self.add_run_step_event(span, step) + + except Exception as exc: + # Set the span status to error + if isinstance(span.span_instance, Span): # pyright: ignore [reportPossiblyUnboundVariable] + span.span_instance.set_status( + StatusCode.ERROR, # pyright: ignore [reportPossiblyUnboundVariable] + description=str(exc), + ) + module = getattr(exc, "__module__", "") + module = module if module != "builtins" else "" + error_type = f"{module}.{type(exc).__name__}" if module else type(exc).__name__ + self._set_attributes(span, ("error.type", error_type)) + raise + + return result + + async def trace_list_run_steps_async(self, function, *args, **kwargs): + project_name = args[ # pylint: disable=protected-access # pyright: ignore [reportFunctionMemberAccess] + 0 + ]._config.project_name + run_id = kwargs.get("run_id") + thread_id = kwargs.get("thread_id") + + span = self.start_list_run_steps_span(project_name=project_name, run_id=run_id, thread_id=thread_id) + + if span is None: + return function(*args, **kwargs) + + with span: + try: + result = await function(*args, **kwargs) + if hasattr(result, "data") and result.data is not None: + for step in result.data: + self.add_run_step_event(span, step) + + except Exception as exc: + # Set the span status to error + if isinstance(span.span_instance, Span): # pyright: ignore [reportPossiblyUnboundVariable] + span.span_instance.set_status( + StatusCode.ERROR, # pyright: ignore [reportPossiblyUnboundVariable] + description=str(exc), + ) + module = getattr(exc, "__module__", "") + module = module if module != "builtins" else "" + error_type = f"{module}.{type(exc).__name__}" if module else type(exc).__name__ + self._set_attributes(span, ("error.type", error_type)) + raise + + return result + async def trace_list_messages_async(self, function, *args, **kwargs): project_name = args[ # pylint: disable=protected-access # pyright: ignore [reportFunctionMemberAccess] 0 @@ -1403,6 +1602,8 @@ def inner(*args, **kwargs): # pylint: disable=R0911 if class_function_name.startswith("AgentsOperations.list_messages"): kwargs.setdefault("merge_span", True) return self.trace_list_messages(function, *args, **kwargs) + if class_function_name.startswith("AgentsOperations.list_run_steps"): + return self.trace_list_run_steps(function, *args, **kwargs) if class_function_name.startswith("AgentRunStream.__exit__"): return self.handle_run_stream_exit(function, *args, **kwargs) # Handle the default case (if the function name does not match) @@ -1471,6 +1672,9 @@ async def inner(*args, **kwargs): # pylint: disable=R0911 if class_function_name.startswith("AgentsOperations.list_messages"): kwargs.setdefault("merge_span", True) return await self.trace_list_messages_async(function, *args, **kwargs) + if class_function_name.startswith("AgentsOperations.list_run_steps"): + kwargs.setdefault("merge_span", True) + return await self.trace_list_run_steps_async(function, *args, **kwargs) if class_function_name.startswith("AsyncAgentRunStream.__aexit__"): return self.handle_run_stream_exit(function, *args, **kwargs) # Handle the default case (if the function name does not match) @@ -1524,6 +1728,7 @@ def _agents_apis(self): ), ("azure.ai.projects.operations", "AgentsOperations", "create_stream", TraceType.AGENTS, "create_stream"), ("azure.ai.projects.operations", "AgentsOperations", "list_messages", TraceType.AGENTS, "list_messages"), + ("azure.ai.projects.operations", "AgentsOperations", "list_run_steps", TraceType.AGENTS, "list_run_steps"), ("azure.ai.projects.models", "AgentRunStream", "__exit__", TraceType.AGENTS, "__exit__"), ) async_apis = ( @@ -1585,6 +1790,13 @@ def _agents_apis(self): TraceType.AGENTS, "list_messages", ), + ( + "azure.ai.projects.aio.operations", + "AgentsOperations", + "list_run_steps", + TraceType.AGENTS, + "list_run_steps", + ), ("azure.ai.projects.models", "AsyncAgentRunStream", "__aexit__", TraceType.AGENTS, "__aexit__"), ) return sync_apis, async_apis @@ -1742,11 +1954,6 @@ def on_thread_message(self, message: "ThreadMessage") -> None: # type: ignore[f def on_thread_run(self, run: "ThreadRun") -> None: # type: ignore[func-returns-value] retval = None - if run.status == "requires_action" and isinstance(run.required_action, SubmitToolOutputsAction): - self.instrumentor._add_tool_event_from_thread_run( # pylint: disable=protected-access # pyright: ignore [reportFunctionMemberAccess] - self.span, run - ) - if self.inner_handler: retval = self.inner_handler.on_thread_run(run) # type: ignore self.last_run = run @@ -1758,8 +1965,15 @@ def on_run_step(self, step: "RunStep") -> None: # type: ignore[func-returns-val if self.inner_handler: retval = self.inner_handler.on_run_step(step) # type: ignore - # todo - report errors for failure statuses here and in run ? - if step.type == "message_creation" and step.status == RunStepStatus.COMPLETED: + if ( + step.type == "tool_calls" + and isinstance(step.step_details, RunStepToolCallDetails) + and step.status == RunStepStatus.COMPLETED + ): + self.instrumentor._add_tool_assistant_message_event( # pylint: disable=protected-access # pyright: ignore [reportFunctionMemberAccess] + self.span, step + ) + elif step.type == "message_creation" and step.status == RunStepStatus.COMPLETED: self.instrumentor.add_thread_message_event(self.span, cast(ThreadMessage, self.last_message), step.usage) self.last_message = None @@ -1849,11 +2063,6 @@ async def on_thread_message(self, message: "ThreadMessage") -> None: # type: ig async def on_thread_run(self, run: "ThreadRun") -> None: # type: ignore[func-returns-value] retval = None - if run.status == "requires_action" and isinstance(run.required_action, SubmitToolOutputsAction): - self.instrumentor._add_tool_event_from_thread_run( # pylint: disable=protected-access # pyright: ignore [reportFunctionMemberAccess] - self.span, run - ) - if self.inner_handler: retval = await self.inner_handler.on_thread_run(run) # type: ignore self.last_run = run @@ -1865,8 +2074,15 @@ async def on_run_step(self, step: "RunStep") -> None: # type: ignore[func-retur if self.inner_handler: retval = await self.inner_handler.on_run_step(step) # type: ignore - # todo - report errors for failure statuses here and in run ? - if step.type == "message_creation" and step.status == RunStepStatus.COMPLETED: + if ( + step.type == "tool_calls" + and isinstance(step.step_details, RunStepToolCallDetails) + and step.status == RunStepStatus.COMPLETED + ): + self.instrumentor._add_tool_assistant_message_event( # pylint: disable=protected-access # pyright: ignore [reportFunctionMemberAccess] + self.span, step + ) + elif step.type == "message_creation" and step.status == RunStepStatus.COMPLETED: self.instrumentor.add_thread_message_event(self.span, cast(ThreadMessage, self.last_message), step.usage) self.last_message = None diff --git a/sdk/ai/azure-ai-projects/azure/ai/projects/telemetry/agents/_utils.py b/sdk/ai/azure-ai-projects/azure/ai/projects/telemetry/agents/_utils.py index bdc18e1381e8..04d0e3b36801 100644 --- a/sdk/ai/azure-ai-projects/azure/ai/projects/telemetry/agents/_utils.py +++ b/sdk/ai/azure-ai-projects/azure/ai/projects/telemetry/agents/_utils.py @@ -42,7 +42,11 @@ GEN_AI_USAGE_OUTPUT_TOKENS = "gen_ai.usage.output_tokens" GEN_AI_SYSTEM_MESSAGE = "gen_ai.system.message" GEN_AI_EVENT_CONTENT = "gen_ai.event.content" +GEN_AI_RUN_STEP_START_TIMESTAMP = "gen_ai.run_step.start.timestamp" +GEN_AI_RUN_STEP_END_TIMESTAMP = "gen_ai.run_step.end.timestamp" +GEN_AI_RUN_STEP_STATUS = "gen_ai.run_step.status" ERROR_TYPE = "error.type" +ERROR_MESSAGE = "error.message" class OperationName(Enum): @@ -52,6 +56,7 @@ class OperationName(Enum): START_THREAD_RUN = "start_thread_run" EXECUTE_TOOL = "execute_tool" LIST_MESSAGES = "list_messages" + LIST_RUN_STEPS = "list_run_steps" SUBMIT_TOOL_OUTPUTS = "submit_tool_outputs" PROCESS_THREAD_RUN = "process_thread_run" diff --git a/sdk/ai/azure-ai-projects/tests/telemetry/test_ai_agents_instrumentor.py b/sdk/ai/azure-ai-projects/tests/telemetry/test_ai_agents_instrumentor.py index db43e4536d71..86673784a8ad 100644 --- a/sdk/ai/azure-ai-projects/tests/telemetry/test_ai_agents_instrumentor.py +++ b/sdk/ai/azure-ai-projects/tests/telemetry/test_ai_agents_instrumentor.py @@ -616,9 +616,11 @@ def fetch_weather(location: str) -> str: "gen_ai.thread.id": "*", "gen_ai.agent.id": "*", "gen_ai.thread.run.id": "*", - "gen_ai.message.status": "requires_action", - # "gen_ai.usage.input_tokens": "+", # not available at the moment - # "gen_ai.usage.output_tokens": "+", # not available at the moment + "gen_ai.message.status": "completed", + "gen_ai.run_step.start.timestamp": "*", + "gen_ai.run_step.end.timestamp": "*", + "gen_ai.usage.input_tokens": "+", + "gen_ai.usage.output_tokens": "+", "gen_ai.event.content": '{"tool_calls": [{"id": "*", "type": "function", "function": {"name": "fetch_weather", "arguments": {"location": "New York"}}}]}', }, }, @@ -764,9 +766,11 @@ def fetch_weather(location: str) -> str: "gen_ai.thread.id": "*", "gen_ai.agent.id": "*", "gen_ai.thread.run.id": "*", - "gen_ai.message.status": "requires_action", - # "gen_ai.usage.input_tokens": "+", # not available at the moment - # "gen_ai.usage.output_tokens": "+", # not available at the moment + "gen_ai.message.status": "completed", + "gen_ai.run_step.start.timestamp": "*", + "gen_ai.run_step.end.timestamp": "*", + "gen_ai.usage.input_tokens": "+", + "gen_ai.usage.output_tokens": "+", "gen_ai.event.content": '{"tool_calls": [{"id": "*", "type": "function", "function": {"name": "fetch_weather", "arguments": {"location": "Sofia"}}}]}', }, }, @@ -947,9 +951,11 @@ def fetch_weather(location: str) -> str: "gen_ai.thread.id": "*", "gen_ai.agent.id": "*", "gen_ai.thread.run.id": "*", - "gen_ai.message.status": "requires_action", - # "gen_ai.usage.input_tokens": "+", # not available at the moment - # "gen_ai.usage.output_tokens": "+", # not available at the moment + "gen_ai.message.status": "completed", + "gen_ai.run_step.start.timestamp": "*", + "gen_ai.run_step.end.timestamp": "*", + "gen_ai.usage.input_tokens": "+", + "gen_ai.usage.output_tokens": "+", "gen_ai.event.content": '{"tool_calls": [{"id": "*", "type": "function"}]}', }, }, @@ -1010,6 +1016,109 @@ def fetch_weather(location: str) -> str: events_match = GenAiTraceVerifier().check_span_events(span, expected_events) assert events_match == True + # Need a recordig for this test + # @pytest.mark.usefixtures("instrument_with_content") + # @agentClientPreparer() + # #@recorded_by_proxy + # def test_agent_streaming_run_steps_with_toolset_with_tracing_content_recording_enabled(self, **kwargs): + # def fetch_weather(location: str) -> str: + # """ + # Fetches the weather information for the specified location. + + # :param location (str): The location to fetch weather for. + # :return: Weather information as a JSON string. + # :rtype: str + # """ + # # In a real-world scenario, you'd integrate with a weather API. + # # Here, we'll mock the response. + # mock_weather_data = {"New York": "Sunny", "London": "Cloudy", "Tokyo": "Rainy"} + # weather = mock_weather_data.get(location, "Weather data not available for this location.") + # weather_json = json.dumps({"weather": weather}) + # return weather_json + + # user_functions: Set[Callable[..., Any]] = { + # fetch_weather, + # } + + # functions = FunctionTool(user_functions) + # toolset = ToolSet() + # toolset.add(functions) + + # client = self.create_client(**kwargs) + # agent = client.agents.create_agent( + # model="gpt-4o", name="my-agent", instructions="You are helpful agent", toolset=toolset + # ) + + # # workaround for https://github.com/Azure/azure-sdk-for-python/issues/40086 + # client.agents.enable_auto_function_calls(toolset=toolset) + + # thread = client.agents.create_thread() + # message = client.agents.create_message( + # thread_id=thread.id, role="user", content="What is the weather in New York?" + # ) + + # event_handler=MyEventHandler() + # with client.agents.create_stream( + # thread_id=thread.id, agent_id=agent.id, event_handler=event_handler + # ) as stream: + # stream.until_done() + + # # delete agent and close client + # client.agents.delete_agent(agent.id) + # print("Deleted agent") + # messages = client.agents.list_messages(thread_id=thread.id) + # client.agents.list_run_steps(thread_id=thread.id, run_id=event_handler.run_id) + # client.close() + + # self.exporter.force_flush() + # spans = self.exporter.get_spans_by_name("list_run_steps") + # assert len(spans) == 1 + # span = spans[0] + # expected_attributes = [ + # ("gen_ai.system", "az.ai.agents"), + # ("gen_ai.operation.name", "list_run_steps"), + # ("server.address", ""), + # ("gen_ai.thread.id", ""), + # ("gen_ai.thread.run.id", ""), + # ] + # attributes_match = GenAiTraceVerifier().check_span_attributes(span, expected_attributes) + # assert attributes_match == True + + # expected_events = [ + # { + # "name": "gen_ai.run_step.message_creation", + # "attributes": { + # "gen_ai.system": "az.ai.agents", + # "gen_ai.thread.id": "*", + # "gen_ai.agent.id": "*", + # "gen_ai.thread.run.id": "*", + # "gen_ai.message.id": "*", + # "gen_ai.run_step.status": "completed", + # "gen_ai.run_step.start.timestamp": "*", + # "gen_ai.run_step.end.timestamp": "*", + # "gen_ai.usage.input_tokens": "+", + # "gen_ai.usage.output_tokens": "+", + # }, + # }, + # { + # "name": "gen_ai.run_step.tool_calls", + # "attributes": { + # "gen_ai.system": "az.ai.agents", + # "gen_ai.thread.id": "*", + # "gen_ai.agent.id": "*", + # "gen_ai.thread.run.id": "*", + # "gen_ai.run_step.status": "completed", + # "gen_ai.run_step.start.timestamp": "*", + # "gen_ai.run_step.end.timestamp": "*", + # "gen_ai.usage.input_tokens": "+", + # "gen_ai.usage.output_tokens": "+", + # "gen_ai.event.content": '{"tool_calls": [{"id": "*", "type": "function", "function": {"name": "fetch_weather", "arguments": {"location": "New York"}}}]}', + # }, + # }, + # ] + # events_match = GenAiTraceVerifier().check_span_events(span, expected_events) + # assert events_match == True + class MyEventHandler(AgentEventHandler): @@ -1024,7 +1133,7 @@ def on_thread_message(self, message: "ThreadMessage") -> None: def on_thread_run(self, run: "ThreadRun") -> None: print(f"ThreadRun status: {run.status}") - + self.run_id = run.id if run.status == "failed": print(f"Run failed. Error: {run.last_error}") diff --git a/sdk/ai/azure-ai-projects/tests/telemetry/test_ai_agents_instrumentor_async.py b/sdk/ai/azure-ai-projects/tests/telemetry/test_ai_agents_instrumentor_async.py index 4e89c66ade13..1902d60f75d8 100644 --- a/sdk/ai/azure-ai-projects/tests/telemetry/test_ai_agents_instrumentor_async.py +++ b/sdk/ai/azure-ai-projects/tests/telemetry/test_ai_agents_instrumentor_async.py @@ -559,9 +559,11 @@ def fetch_weather(location: str) -> str: "gen_ai.thread.id": "*", "gen_ai.agent.id": "*", "gen_ai.thread.run.id": "*", - "gen_ai.message.status": "requires_action", - # "gen_ai.usage.input_tokens": "+", # not available at the moment - # "gen_ai.usage.output_tokens": "+", # not available at the moment + "gen_ai.message.status": "completed", + "gen_ai.run_step.start.timestamp": "*", + "gen_ai.run_step.end.timestamp": "*", + "gen_ai.usage.input_tokens": "+", + "gen_ai.usage.output_tokens": "+", "gen_ai.event.content": '{"tool_calls": [{"id": "*", "type": "function", "function": {"name": "fetch_weather", "arguments": {"location": "New York"}}}]}', }, }, @@ -779,9 +781,9 @@ def fetch_weather(location: str) -> str: "gen_ai.thread.id": "*", "gen_ai.agent.id": "*", "gen_ai.thread.run.id": "*", - "gen_ai.message.status": "requires_action", - # "gen_ai.usage.input_tokens": "+", # not available at the moment - # "gen_ai.usage.output_tokens": "+", # not available at the moment + "gen_ai.message.status": "completed", + "gen_ai.usage.input_tokens": "+", + "gen_ai.usage.output_tokens": "+", "gen_ai.event.content": '{"tool_calls": [{"id": "*", "type": "function"}]}', }, },