From fc8459de6cdcbfd091d9ff233b29dbd6eeee147d Mon Sep 17 00:00:00 2001 From: PriyanJindal Date: Mon, 21 Apr 2025 14:27:07 -0700 Subject: [PATCH 1/9] check author From e4de4652ee3831b379ba4b33656b85123717be97 Mon Sep 17 00:00:00 2001 From: PriyanJindal Date: Thu, 1 May 2025 16:57:54 -0700 Subject: [PATCH 2/9] adding demo project traces (datasets and evals not yet) --- .../code_based_agent/main.py | 7 ++++- .../code_based_agent/tool_calling_evals.py | 0 .../utils/instrument.py | 5 ++- src/phoenix/trace/fixtures.py | 31 +++++++++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 examples/agent_framework_comparison/code_based_agent/tool_calling_evals.py diff --git a/examples/agent_framework_comparison/code_based_agent/main.py b/examples/agent_framework_comparison/code_based_agent/main.py index 2714b6d1de..80160273e5 100644 --- a/examples/agent_framework_comparison/code_based_agent/main.py +++ b/examples/agent_framework_comparison/code_based_agent/main.py @@ -9,17 +9,20 @@ sys.path.insert(1, os.path.join(sys.path[0], "..")) from router import router from utils.instrument import Framework, instrument +from dotenv import load_dotenv +load_dotenv() def gradio_interface(message, history): tracer = trace.get_tracer(__name__) with tracer.start_as_current_span("code_based_agent") as span: span.set_attribute(SpanAttributes.INPUT_VALUE, message) span.set_attribute(SpanAttributes.OPENINFERENCE_SPAN_KIND, "AGENT") - + message = [{"role": "user", "content": message}] context = {} TraceContextTextMapPropagator().inject(context) + agent_response = router(message, context) span.set_attribute(SpanAttributes.OUTPUT_VALUE, agent_response) span.set_status(trace.Status(trace.StatusCode.OK)) @@ -32,5 +35,7 @@ def launch_app(): if __name__ == "__main__": + print(os.environ.get("PHOENIX_COLLECTOR_ENDPOINT")) + print(os.environ.get("PHOENIX_API_KEY")) instrument(project_name="agent-demo", framework=Framework.CODE_BASED) launch_app() diff --git a/examples/agent_framework_comparison/code_based_agent/tool_calling_evals.py b/examples/agent_framework_comparison/code_based_agent/tool_calling_evals.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/examples/agent_framework_comparison/utils/instrument.py b/examples/agent_framework_comparison/utils/instrument.py index 07670c79a7..1af96284cc 100644 --- a/examples/agent_framework_comparison/utils/instrument.py +++ b/examples/agent_framework_comparison/utils/instrument.py @@ -23,7 +23,10 @@ def instrument(project_name="code-based-agent", framework=Framework.CODE_BASED): "PHOENIX_COLLECTOR_ENDPOINT environment variable is not set. " "Please set it before running the agent." ) - tracer_provider = register(project_name=project_name) + print(os.environ.get("PHOENIX_COLLECTOR_ENDPOINT")) + print(os.environ.get("PHOENIX_API_KEY")) + tracer_provider = register(project_name=project_name, endpoint=os.environ.get("PHOENIX_COLLECTOR_ENDPOINT")) + if framework == Framework.LLAMA_INDEX: LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider) diff --git a/src/phoenix/trace/fixtures.py b/src/phoenix/trace/fixtures.py index ab437df9e9..0196ec5032 100644 --- a/src/phoenix/trace/fixtures.py +++ b/src/phoenix/trace/fixtures.py @@ -139,6 +139,35 @@ class TracesFixture: ), ) +demo_toolcalling_fixture = TracesFixture( + name="demo_toolcalling", + project_name="demo_agents_1", + description="Tool calling traces", + file_name="agents-toolcalling-tracesv2.parquet", + # evaluation_fixtures=( + # EvaluationFixture( + # evaluation_name="Router Tool Calling", + # file_name="agents-toolcalling-evalsv2.parquet", + # ) + # ) + dataset_fixtures=( + DatasetFixture( + file_name="questions.csv.gz", + input_keys=("query",), + output_keys=("responses",), + name="Valid Queries", + description="Valid queries for the demo agent", + ), + DatasetFixture( + file_name="invalid_questions.csv.gz", + input_keys=("query",), + output_keys=("responses",), + name="Invalid Queries", + description="Invalid queries for the demo agent", + ) + ) +) + demo_code_based_agent_fixture = TracesFixture( name="demo_code_based_agent", project_name="demo_agents", @@ -298,6 +327,7 @@ class TracesFixture: vision_fixture, anthropic_tools_fixture, project_sessions_llama_index_rag_arize_docs_fixture, + demo_toolcalling_fixture, ] NAME_TO_TRACES_FIXTURE: dict[str, TracesFixture] = { @@ -372,6 +402,7 @@ def send_dataset_fixtures( endpoint: str, fixtures: Iterable[DatasetFixture], ) -> None: + print([fixture.name for fixture in fixtures]) expiration = time() + 5 while time() < expiration: try: From 56a0db96329a06df46be63a2665871e1ac9a9e8c Mon Sep 17 00:00:00 2001 From: PriyanJindal Date: Thu, 1 May 2025 17:02:21 -0700 Subject: [PATCH 3/9] changing name to demo_agent --- src/phoenix/trace/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/phoenix/trace/fixtures.py b/src/phoenix/trace/fixtures.py index 0196ec5032..a8584cfe76 100644 --- a/src/phoenix/trace/fixtures.py +++ b/src/phoenix/trace/fixtures.py @@ -141,7 +141,7 @@ class TracesFixture: demo_toolcalling_fixture = TracesFixture( name="demo_toolcalling", - project_name="demo_agents_1", + project_name="demo_agent", description="Tool calling traces", file_name="agents-toolcalling-tracesv2.parquet", # evaluation_fixtures=( From ec77b36317416c505c8fac33ae445744e5289424 Mon Sep 17 00:00:00 2001 From: PriyanJindal Date: Thu, 1 May 2025 17:09:58 -0700 Subject: [PATCH 4/9] cleanup --- .../agent_framework_comparison/code_based_agent/main.py | 7 ------- examples/agent_framework_comparison/utils/instrument.py | 5 +---- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/examples/agent_framework_comparison/code_based_agent/main.py b/examples/agent_framework_comparison/code_based_agent/main.py index 80160273e5..47aba600c6 100644 --- a/examples/agent_framework_comparison/code_based_agent/main.py +++ b/examples/agent_framework_comparison/code_based_agent/main.py @@ -9,20 +9,15 @@ sys.path.insert(1, os.path.join(sys.path[0], "..")) from router import router from utils.instrument import Framework, instrument -from dotenv import load_dotenv - -load_dotenv() def gradio_interface(message, history): tracer = trace.get_tracer(__name__) with tracer.start_as_current_span("code_based_agent") as span: span.set_attribute(SpanAttributes.INPUT_VALUE, message) span.set_attribute(SpanAttributes.OPENINFERENCE_SPAN_KIND, "AGENT") - message = [{"role": "user", "content": message}] context = {} TraceContextTextMapPropagator().inject(context) - agent_response = router(message, context) span.set_attribute(SpanAttributes.OUTPUT_VALUE, agent_response) span.set_status(trace.Status(trace.StatusCode.OK)) @@ -35,7 +30,5 @@ def launch_app(): if __name__ == "__main__": - print(os.environ.get("PHOENIX_COLLECTOR_ENDPOINT")) - print(os.environ.get("PHOENIX_API_KEY")) instrument(project_name="agent-demo", framework=Framework.CODE_BASED) launch_app() diff --git a/examples/agent_framework_comparison/utils/instrument.py b/examples/agent_framework_comparison/utils/instrument.py index 1af96284cc..84defd4a97 100644 --- a/examples/agent_framework_comparison/utils/instrument.py +++ b/examples/agent_framework_comparison/utils/instrument.py @@ -23,11 +23,8 @@ def instrument(project_name="code-based-agent", framework=Framework.CODE_BASED): "PHOENIX_COLLECTOR_ENDPOINT environment variable is not set. " "Please set it before running the agent." ) - print(os.environ.get("PHOENIX_COLLECTOR_ENDPOINT")) - print(os.environ.get("PHOENIX_API_KEY")) - tracer_provider = register(project_name=project_name, endpoint=os.environ.get("PHOENIX_COLLECTOR_ENDPOINT")) + tracer_provider = register(project_name=project_name) - if framework == Framework.LLAMA_INDEX: LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider) elif framework == Framework.LANGGRAPH: From a0ebcc75e132b1988733b3d4c2880b13f39b6bf5 Mon Sep 17 00:00:00 2001 From: PriyanJindal Date: Thu, 1 May 2025 17:29:49 -0700 Subject: [PATCH 5/9] cleanup --- examples/agent_framework_comparison/code_based_agent/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/agent_framework_comparison/code_based_agent/main.py b/examples/agent_framework_comparison/code_based_agent/main.py index 47aba600c6..4b05fdf3fc 100644 --- a/examples/agent_framework_comparison/code_based_agent/main.py +++ b/examples/agent_framework_comparison/code_based_agent/main.py @@ -10,11 +10,13 @@ from router import router from utils.instrument import Framework, instrument + def gradio_interface(message, history): tracer = trace.get_tracer(__name__) with tracer.start_as_current_span("code_based_agent") as span: span.set_attribute(SpanAttributes.INPUT_VALUE, message) span.set_attribute(SpanAttributes.OPENINFERENCE_SPAN_KIND, "AGENT") + message = [{"role": "user", "content": message}] context = {} TraceContextTextMapPropagator().inject(context) From a5975dcfd2bc3c08f142017094782a1dff02b0d7 Mon Sep 17 00:00:00 2001 From: PriyanJindal Date: Thu, 1 May 2025 17:40:59 -0700 Subject: [PATCH 6/9] style: ruff auto-format --- examples/agent_framework_comparison/code_based_agent/main.py | 2 +- examples/agent_framework_comparison/utils/instrument.py | 2 +- src/phoenix/trace/fixtures.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/agent_framework_comparison/code_based_agent/main.py b/examples/agent_framework_comparison/code_based_agent/main.py index 4b05fdf3fc..2714b6d1de 100644 --- a/examples/agent_framework_comparison/code_based_agent/main.py +++ b/examples/agent_framework_comparison/code_based_agent/main.py @@ -16,7 +16,7 @@ def gradio_interface(message, history): with tracer.start_as_current_span("code_based_agent") as span: span.set_attribute(SpanAttributes.INPUT_VALUE, message) span.set_attribute(SpanAttributes.OPENINFERENCE_SPAN_KIND, "AGENT") - + message = [{"role": "user", "content": message}] context = {} TraceContextTextMapPropagator().inject(context) diff --git a/examples/agent_framework_comparison/utils/instrument.py b/examples/agent_framework_comparison/utils/instrument.py index 84defd4a97..07670c79a7 100644 --- a/examples/agent_framework_comparison/utils/instrument.py +++ b/examples/agent_framework_comparison/utils/instrument.py @@ -24,7 +24,7 @@ def instrument(project_name="code-based-agent", framework=Framework.CODE_BASED): "Please set it before running the agent." ) tracer_provider = register(project_name=project_name) - + if framework == Framework.LLAMA_INDEX: LlamaIndexInstrumentor().instrument(tracer_provider=tracer_provider) elif framework == Framework.LANGGRAPH: diff --git a/src/phoenix/trace/fixtures.py b/src/phoenix/trace/fixtures.py index a8584cfe76..b9d7cf0bfb 100644 --- a/src/phoenix/trace/fixtures.py +++ b/src/phoenix/trace/fixtures.py @@ -164,8 +164,8 @@ class TracesFixture: output_keys=("responses",), name="Invalid Queries", description="Invalid queries for the demo agent", - ) - ) + ), + ), ) demo_code_based_agent_fixture = TracesFixture( From 99ef32afd82de5bf4c3d75e320a60da6507da8ab Mon Sep 17 00:00:00 2001 From: PriyanJindal Date: Thu, 1 May 2025 17:43:01 -0700 Subject: [PATCH 7/9] removing prints --- src/phoenix/trace/fixtures.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/phoenix/trace/fixtures.py b/src/phoenix/trace/fixtures.py index b9d7cf0bfb..3f8ab16100 100644 --- a/src/phoenix/trace/fixtures.py +++ b/src/phoenix/trace/fixtures.py @@ -402,7 +402,6 @@ def send_dataset_fixtures( endpoint: str, fixtures: Iterable[DatasetFixture], ) -> None: - print([fixture.name for fixture in fixtures]) expiration = time() + 5 while time() < expiration: try: From 24d5cc342902cd521f1472c01e071af39a5a3feb Mon Sep 17 00:00:00 2001 From: PriyanJindal Date: Fri, 2 May 2025 12:16:58 -0700 Subject: [PATCH 8/9] removing empty file --- .../code_based_agent/tool_calling_evals.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 examples/agent_framework_comparison/code_based_agent/tool_calling_evals.py diff --git a/examples/agent_framework_comparison/code_based_agent/tool_calling_evals.py b/examples/agent_framework_comparison/code_based_agent/tool_calling_evals.py deleted file mode 100644 index e69de29bb2..0000000000 From c4e47e68cd65fe99ce674ebc0794b0f5a1944551 Mon Sep 17 00:00:00 2001 From: PriyanJindal <52796578+PriyanJindal@users.noreply.github.com> Date: Fri, 2 May 2025 12:33:34 -0700 Subject: [PATCH 9/9] remove evalsfixture comment Co-authored-by: Xander Song --- src/phoenix/trace/fixtures.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/phoenix/trace/fixtures.py b/src/phoenix/trace/fixtures.py index 3f8ab16100..5569e8d219 100644 --- a/src/phoenix/trace/fixtures.py +++ b/src/phoenix/trace/fixtures.py @@ -144,12 +144,6 @@ class TracesFixture: project_name="demo_agent", description="Tool calling traces", file_name="agents-toolcalling-tracesv2.parquet", - # evaluation_fixtures=( - # EvaluationFixture( - # evaluation_name="Router Tool Calling", - # file_name="agents-toolcalling-evalsv2.parquet", - # ) - # ) dataset_fixtures=( DatasetFixture( file_name="questions.csv.gz",