Add initial OpenTelemetry instrumentation for RubyLLM

Provides automatic tracing for RubyLLM::Chat#ask calls with provider and
model identification and token usage tracking using [OpenTelemetry's
semantic conventions for generative AI
systems](https://opentelemetry.io/docs/specs/semconv/gen-ai/).

Author: clarissalimab

.gitignore

@@ -0,0 +1 @@
+*.gem

Gemfile

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gemspec
+
+gem "ruby_llm"
+gem "opentelemetry-sdk"
+
+group :test do
+  gem "minitest"
+  gem "webmock"
+  gem "rake"
+end

Gemfile.lock

@@ -0,0 +1,89 @@
+PATH
+  remote: .
+  specs:
+    opentelemetry-instrumentation-ruby_llm (0.1.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-instrumentation-base (~> 0.23)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.8.8)
+      public_suffix (>= 2.0.2, < 8.0)
+    base64 (0.3.0)
+    bigdecimal (4.0.1)
+    crack (1.0.1)
+      bigdecimal
+      rexml
+    event_stream_parser (1.0.0)
+    faraday (2.14.0)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-multipart (1.2.0)
+      multipart-post (~> 2.0)
+    faraday-net_http (3.4.2)
+      net-http (~> 0.5)
+    faraday-retry (2.4.0)
+      faraday (~> 2.0)
+    hashdiff (1.2.1)
+    json (2.18.0)
+    logger (1.7.0)
+    marcel (1.1.0)
+    minitest (6.0.1)
+      prism (~> 1.5)
+    multipart-post (2.4.1)
+    net-http (0.9.1)
+      uri (>= 0.11.1)
+    opentelemetry-api (1.7.0)
+    opentelemetry-common (0.23.0)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-instrumentation-base (0.25.0)
+      opentelemetry-api (~> 1.7)
+      opentelemetry-common (~> 0.21)
+      opentelemetry-registry (~> 0.1)
+    opentelemetry-registry (0.4.0)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-sdk (1.10.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-registry (~> 0.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-semantic_conventions (1.36.0)
+      opentelemetry-api (~> 1.0)
+    prism (1.9.0)
+    public_suffix (7.0.2)
+    rake (13.3.1)
+    rexml (3.4.4)
+    ruby_llm (1.11.0)
+      base64
+      event_stream_parser (~> 1)
+      faraday (>= 1.10.0)
+      faraday-multipart (>= 1)
+      faraday-net_http (>= 1)
+      faraday-retry (>= 1)
+      marcel (~> 1.0)
+      ruby_llm-schema (~> 0.2.1)
+      zeitwerk (~> 2)
+    ruby_llm-schema (0.2.5)
+    uri (1.1.1)
+    webmock (3.26.1)
+      addressable (>= 2.8.0)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+    zeitwerk (2.7.4)
+
+PLATFORMS
+  arm64-darwin-24
+  ruby
+
+DEPENDENCIES
+  minitest
+  opentelemetry-instrumentation-ruby_llm!
+  opentelemetry-sdk
+  rake
+  ruby_llm
+  webmock
+
+BUNDLED WITH
+   2.7.1

Rakefile

@@ -0,0 +1,8 @@
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.pattern = "test/**/*_test.rb"
+end
+
+task default: :test

lib/opentelemetry-instrumentation-ruby_llm.rb

@@ -0,0 +1,5 @@
+require "opentelemetry-api"
+require "opentelemetry-instrumentation-base"
+
+require_relative "opentelemetry/instrumentation/ruby_llm/version"
+require_relative "opentelemetry/instrumentation/ruby_llm/instrumentation"

lib/opentelemetry/instrumentation/ruby_llm/instrumentation.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      class Instrumentation < OpenTelemetry::Instrumentation::Base
+        instrumentation_name "OpenTelemetry::Instrumentation::RubyLLM"
+        instrumentation_version VERSION
+
+        present do
+          defined?(::RubyLLM)
+        end
+
+        install do |_config|
+          require_relative "patches/chat"
+          ::RubyLLM::Chat.prepend(Patches::Chat)
+        end
+      end
+    end
+  end
+end

lib/opentelemetry/instrumentation/ruby_llm/patches/chat.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      module Patches
+        module Chat
+          def ask(message, &block)
+            provider = @model&.provider || "unknown"
+            model_id = @model&.id || "unknown"
+
+            attributes = {
+              "gen_ai.operation.name" => "chat",
+              "gen_ai.provider.name" => provider,
+              "gen_ai.request.model" => model_id,
+            }
+
+            tracer.in_span("chat #{model_id}", attributes: attributes, kind: OpenTelemetry::Trace::SpanKind::CLIENT) do |span|
+              begin
+                result = super
+
+                if @messages.last
+                  response = @messages.last
+                  span.set_attribute("gen_ai.response.model", response.model_id) if response.model_id
+                  span.set_attribute("gen_ai.usage.input_tokens", response.input_tokens) if response.input_tokens
+                  span.set_attribute("gen_ai.usage.output_tokens", response.output_tokens) if response.output_tokens
+                  span.set_attribute("gen_ai.request.temperature", @temperature) if @temperature
+                end
+
+                result
+              rescue => e
+                span.record_exception(e)
+                span.status = OpenTelemetry::Trace::Status.error(e.message)
+                span.set_attribute("error.type", e.class.name)
+                raise
+              end
+            end
+          end
+
+          private
+
+          def tracer
+            RubyLLM::Instrumentation.instance.tracer
+          end
+        end
+      end
+    end
+  end
+end

lib/opentelemetry/instrumentation/ruby_llm/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module OpenTelemetry
+  module Instrumentation
+    module RubyLLM
+      VERSION = "0.1.0"
+    end
+  end
+end

opentelemetry-instrumentation-ruby_llm.gemspec

@@ -0,0 +1,22 @@
+require_relative "lib/opentelemetry/instrumentation/ruby_llm/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "opentelemetry-instrumentation-ruby_llm"
+  spec.version       = OpenTelemetry::Instrumentation::RubyLLM::VERSION
+  spec.authors       = ["Clarissa Borges"]
+  spec.email         = ["cborges@thoughtbot.com"]
+
+  spec.summary     = "OpenTelemetry instrumentation for RubyLLM"
+  spec.description = "Adds OpenTelemetry tracing to RubyLLM chat operations"
+  spec.homepage    = "https://github.com/thoughtbot/opentelemetry-instrumentation-ruby_llm"
+
+  spec.required_ruby_version = ">= 3.2.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+
+  spec.files = `git ls-files`.split("\n")
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "opentelemetry-api", "~> 1.0"
+  spec.add_dependency "opentelemetry-instrumentation-base", "~> 0.23"
+end

test/instrumentation_test.rb

@@ -0,0 +1,69 @@
+require "test_helper"
+
+class InstrumentationTest < Minitest::Test
+  def setup
+    EXPORTER.reset
+
+    RubyLLM.configure do |c|
+      c.openai_api_key = "fake-key-for-testing"
+    end
+  end
+
+  def test_creates_span_with_attributes
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(
+        status: 200,
+        headers: { "Content-Type" => "application/json" },
+        body: {
+          id: "chatcmpl-123",
+          object: "chat.completion",
+          model: "gpt-4o-mini",
+          choices: [
+            {
+              index: 0,
+              message: { role: "assistant", content: "Hello, world!" },
+              finish_reason: "stop"
+            }
+          ],
+          usage: {
+            prompt_tokens: 10,
+            completion_tokens: 5,
+            total_tokens: 15
+          }
+        }.to_json
+      )
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+    chat.ask("Hi")
+
+    spans = EXPORTER.finished_spans
+    assert_equal 1, spans.length
+
+    span = spans.first
+    assert_equal OpenTelemetry::Trace::SpanKind::CLIENT, span.kind
+    assert_equal "chat gpt-4o-mini", span.name
+    assert_equal "openai", span.attributes["gen_ai.provider.name"]
+    assert_equal "gpt-4o-mini", span.attributes["gen_ai.request.model"]
+    assert_equal "chat", span.attributes["gen_ai.operation.name"]
+    assert_equal 10, span.attributes["gen_ai.usage.input_tokens"]
+    assert_equal 5, span.attributes["gen_ai.usage.output_tokens"]
+  end
+
+  def test_records_error_on_api_failure
+    stub_request(:post, "https://api.openai.com/v1/chat/completions")
+      .to_return(status: 500, body: "Internal Server Error")
+
+    chat = RubyLLM.chat(model: "gpt-4o-mini")
+
+    assert_raises do
+      chat.ask("Hi")
+    end
+
+    spans = EXPORTER.finished_spans
+    span = spans.last
+
+    assert_equal "chat gpt-4o-mini", span.name
+    assert span.attributes["error.type"]
+    assert_equal OpenTelemetry::Trace::Status::ERROR, span.status.code
+  end
+end