Add tool call instrumentation

Creates spans for RubyLLM::Chat#execute_tool with tool name, arguments, and
result. Those spans appear as children of the chat span for waterfall
visualization.

Author: clarissalimab

Gemfile

@@ -6,6 +6,7 @@ gemspec
 
 gem "ruby_llm"
 gem "opentelemetry-sdk"
+gem "opentelemetry-exporter-otlp"
 
 group :test do
   gem "minitest"

Gemfile.lock

@@ -26,6 +26,14 @@ GEM
       net-http (~> 0.5)
     faraday-retry (2.4.0)
       faraday (~> 2.0)
+    google-protobuf (4.33.4)
+      bigdecimal
+      rake (>= 13)
+    google-protobuf (4.33.4-arm64-darwin)
+      bigdecimal
+      rake (>= 13)
+    googleapis-common-protos-types (1.22.0)
+      google-protobuf (~> 4.26)
     hashdiff (1.2.1)
     json (2.18.0)
     logger (1.7.0)
@@ -38,6 +46,13 @@ GEM
     opentelemetry-api (1.7.0)
     opentelemetry-common (0.23.0)
       opentelemetry-api (~> 1.0)
+    opentelemetry-exporter-otlp (0.31.1)
+      google-protobuf (>= 3.18)
+      googleapis-common-protos-types (~> 1.3)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-sdk (~> 1.10)
+      opentelemetry-semantic_conventions
     opentelemetry-instrumentation-base (0.25.0)
       opentelemetry-api (~> 1.7)
       opentelemetry-common (~> 0.21)
@@ -79,6 +94,7 @@ PLATFORMS
 
 DEPENDENCIES
   minitest
+  opentelemetry-exporter-otlp
   opentelemetry-instrumentation-ruby_llm!
   opentelemetry-sdk
   rake

lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb

@@ -37,6 +37,22 @@ def ask(message, &block)
             end
           end
 
+          def execute_tool(tool_call)
+            attributes = {
+              "gen_ai.tool.name" => tool_call.name,
+              "gen_ai.tool.call.id" => tool_call.id,
+              "gen_ai.tool.call.arguments" => tool_call.arguments.to_json,
+              "gen_ai.tool.type" => "function"
+            }
+
+            tracer.in_span("execute_tool #{tool_call.name}", attributes: attributes, kind: OpenTelemetry::Trace::SpanKind::INTERNAL) do |span|
+              result = super
+              result_str = result.is_a?(::RubyLLM::Tool::Halt) ? result.content.to_s : result.to_s
+              span.set_attribute("gen_ai.tool.call.result", result_str[0..500])
+              result
+            end
+          end
+
           private
 
           def tracer

test/instrumentation_test.rb

@@ -66,4 +66,79 @@ def test_records_error_on_api_failure
     assert span.attributes["error.type"]
     assert_equal OpenTelemetry::Trace::Status::ERROR, span.status.code
   end
+
+  def test_creates_span_for_tool_call
+    calculator = Class.new(RubyLLM::Tool) do
+      def self.name = "calculator"
+      description "Performs math"
+      param :expression, type: "string", desc: "Math expression"
+
+      def execute(expression:)
+        eval(expression).to_s
+      end
+    end
+
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        {
+          status: 200,
+          headers: { "Content-Type" => "application/json" },
+          body: {
+            id: "chatcmpl-123",
+            object: "chat.completion",
+            model: "gpt-4o-mini",
+            choices: [{
+              index: 0,
+              message: {
+                role: "assistant",
+                content: nil,
+                tool_calls: [{
+                  id: "call_abc123",
+                  type: "function",
+                  function: { name: "calculator", arguments: '{"expression":"2+2"}' }
+                }]
+              },
+              finish_reason: "tool_calls"
+            }],
+            usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }
+          }.to_json
+        },
+        {
+          status: 200,
+          headers: { "Content-Type" => "application/json" },
+          body: {
+            id: "chatcmpl-456",
+            object: "chat.completion",
+            model: "gpt-4o-mini",
+            choices: [{
+              index: 0,
+              message: { role: "assistant", content: "The answer is 4" },
+              finish_reason: "stop"
+            }],
+            usage: { prompt_tokens: 20, completion_tokens: 5, total_tokens: 25 }
+          }.to_json
+        }
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.with_tool(calculator)
+    chat.ask("What is 2+2?")
+
+    spans = EXPORTER.finished_spans
+
+    tool_spans = spans.select { |s| s.name.start_with?("execute_tool ") }
+    chat_spans = spans.select { |s| s.name.include?("chat ") }
+
+    assert_equal 1, tool_spans.length
+    assert_equal 1, chat_spans.length
+
+    tool_span = tool_spans.first
+    assert_equal OpenTelemetry::Trace::SpanKind::INTERNAL, tool_span.kind
+    assert_equal "execute_tool calculator", tool_span.name
+    assert_equal "calculator", tool_span.attributes["gen_ai.tool.name"]
+    assert_equal '{"expression":"2+2"}', tool_span.attributes["gen_ai.tool.call.arguments"]
+    assert_equal "4", tool_span.attributes["gen_ai.tool.call.result"]
+    assert_equal "call_abc123", tool_span.attributes["gen_ai.tool.call.id"]
+    assert_equal "function", tool_span.attributes["gen_ai.tool.type"]
+  end
 end