diff --git a/helpers/metrics-test-helpers/.gitignore b/helpers/metrics-test-helpers/.gitignore new file mode 100644 index 0000000000..c3fc7426ad --- /dev/null +++ b/helpers/metrics-test-helpers/.gitignore @@ -0,0 +1 @@ +gemfiles diff --git a/helpers/metrics-test-helpers/Appraisals b/helpers/metrics-test-helpers/Appraisals new file mode 100644 index 0000000000..5e8e426ac7 --- /dev/null +++ b/helpers/metrics-test-helpers/Appraisals @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +appraise 'metrics-sdk' do + gem 'opentelemetry-metrics-sdk' +end + +appraise 'metrics-api' do + gem 'opentelemetry-metrics-api' +end + +appraise 'base' do # rubocop: disable Lint/EmptyBlock +end diff --git a/helpers/metrics-test-helpers/Gemfile b/helpers/metrics-test-helpers/Gemfile new file mode 100644 index 0000000000..10af422753 --- /dev/null +++ b/helpers/metrics-test-helpers/Gemfile @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +source 'https://rubygems.org' + +# Specify your gem's dependencies in opentelemetry-metrics-test-helpers.gemspec +gemspec + +gem 'rake', '~> 13.0' +gem 'minitest', '~> 5.16' +gem 'opentelemetry-sdk' +gem 'pry-byebug' diff --git a/helpers/metrics-test-helpers/README.md b/helpers/metrics-test-helpers/README.md new file mode 100644 index 0000000000..dc12a07619 --- /dev/null +++ b/helpers/metrics-test-helpers/README.md @@ -0,0 +1,76 @@ +# OpenTelemetry Instrumentation Test Helpers: Metrics + +This Ruby gem facilitates testing instrumentation libraries with respect to the OpenTelemetry Metrics API and SDK. + +## Usage + +Add the gem to your instrumentation's Gemfile: + +```ruby +# Gemfile + +group :test, :development do + gem 'opentelemetry-metrics-test-helpers', path: '../../helpers/metrics-test-helpers', require: false +end +``` + +It's not necessary to add this gem as a development dependency in the gemspec. +`opentelemetry-metrics-test-helpers` is not currently published to RubyGems, +and it is expected that it will always be bundled from the source in this +repository. + +Note that metrics-test-helpers makes no attempt to require +the metrics API or SDK. It is designed to work with or without the metrics API and SDK defined, but you may experience issues if the API or SDK gem is in the gemfile but not yet loaded when the test helpers are initialized. + +## Examples + +In a test_helper.rb, after the `configure` block, +require this library: + +```ruby +OpenTelemetry::SDK.configure do |c| + c.error_handler = ->(exception:, message:) { raise(exception || message) } + c.add_span_processor span_processor +end +require 'opentelemetry-metrics-test-helpers' +``` + +If the library uses Appraisals, it is recommended to appraise with and without the metrics api and sdk gems. Note that any metrics implementation in instrumentation libraries should be written against the API only, but for testing the SDK is required to collect metrics data - testing under all three scenarios (no metrics at all, api only, and with the sdk) helps ensure compliance with this requirement. + +In a test: + +```ruby +with_metrics_sdk do + let(:metric_snapshots) do + metrics_exporter.tap(&:pull) + .metric_snapshots.select { |snapshot| snapshot.data_points.any? } + .group_by(&:name) + end + + it "uses metrics", with_metrics_sdk: true do + # do something here ... + _(metric_snapshots).count.must_equal(4) + end +end +``` + +- `metrics_exporter` is automatically reset before each test. +- `#with_metrics_sdk` will only yield if the SDK is present. +- `#with_metrics_api` will only yield if the API is present + +## How can I get involved? + +The `opentelemetry-metrics-test-helpers` gem source is [on github][repo-github], along with related gems including `opentelemetry-instrumentation-pg` and `opentelemetry-instrumentation-trilogy`. + +The OpenTelemetry Ruby gems are maintained by the OpenTelemetry Ruby special interest group (SIG). You can get involved by joining us on our [GitHub Discussions][discussions-url], [Slack Channel][slack-channel] or attending our weekly meeting. See the [meeting calendar][community-meetings] for dates and times. For more information on this and other language SIGs, see the OpenTelemetry [community page][ruby-sig]. + +## License + +The `opentelemetry-helpers-sql-obfuscation` gem is distributed under the Apache 2.0 license. See [LICENSE][license-github] for more information. + +[repo-github]: https://github.com/open-telemetry/opentelemetry-ruby +[license-github]: https://github.com/open-telemetry/opentelemetry-ruby-contrib/blob/main/LICENSE +[ruby-sig]: https://github.com/open-telemetry/community#ruby-sig +[community-meetings]: https://github.com/open-telemetry/community#community-meetings +[slack-channel]: https://cloud-native.slack.com/archives/C01NWKKMKMY +[discussions-url]: https://github.com/open-telemetry/opentelemetry-ruby/discussions diff --git a/helpers/metrics-test-helpers/Rakefile b/helpers/metrics-test-helpers/Rakefile new file mode 100644 index 0000000000..816ad799bc --- /dev/null +++ b/helpers/metrics-test-helpers/Rakefile @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'bundler/gem_tasks' +require 'rake/testtask' +require 'rubocop/rake_task' + +RuboCop::RakeTask.new + +Rake::TestTask.new do |t| + t.libs << 'test' + t.libs << 'lib' + t.test_files = FileList['test/**/*_test.rb'] + t.warning = false +end + +if RUBY_ENGINE == 'truffleruby' + task default: %i[test] +else + task default: %i[test rubocop] +end diff --git a/helpers/metrics-test-helpers/bin/console b/helpers/metrics-test-helpers/bin/console new file mode 100755 index 0000000000..ec41548855 --- /dev/null +++ b/helpers/metrics-test-helpers/bin/console @@ -0,0 +1,11 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'bundler/setup' +require 'opentelemetry/metrics/test/helpers' + +# You can add fixtures and/or initialization code here to make experimenting +# with your gem easier. You can also use a different console, if you like. + +require 'irb' +IRB.start(__FILE__) diff --git a/helpers/metrics-test-helpers/bin/setup b/helpers/metrics-test-helpers/bin/setup new file mode 100755 index 0000000000..dce67d860a --- /dev/null +++ b/helpers/metrics-test-helpers/bin/setup @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' +set -vx + +bundle install + +# Do any other automated setup that you need to do here diff --git a/helpers/metrics-test-helpers/lib/opentelemetry-metrics-test-helpers.rb b/helpers/metrics-test-helpers/lib/opentelemetry-metrics-test-helpers.rb new file mode 100644 index 0000000000..f8faca4ef9 --- /dev/null +++ b/helpers/metrics-test-helpers/lib/opentelemetry-metrics-test-helpers.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require_relative 'opentelemetry/metrics_test_helpers' diff --git a/helpers/metrics-test-helpers/lib/opentelemetry/metrics_test_helpers.rb b/helpers/metrics-test-helpers/lib/opentelemetry/metrics_test_helpers.rb new file mode 100644 index 0000000000..9c4ee8b932 --- /dev/null +++ b/helpers/metrics-test-helpers/lib/opentelemetry/metrics_test_helpers.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'minitest/spec' + +module OpenTelemetry + module MetricsTestHelpers + module LoadedMetricsFeatures + OTEL_METRICS_API_LOADED = !Gem.loaded_specs['opentelemetry-metrics-api'].nil? + OTEL_METRICS_SDK_LOADED = !Gem.loaded_specs['opentelemetry-metrics-sdk'].nil? + + extend self + + def api_loaded? + OTEL_METRICS_API_LOADED + end + + def sdk_loaded? + OTEL_METRICS_SDK_LOADED + end + end + + module MinitestExtensions + def self.prepended(base) + base.extend(self) + end + + def self.included(base) + base.extend(self) + end + + def before_setup + super + reset_metrics_exporter + end + + def with_metrics_sdk + yield if LoadedMetricsFeatures.sdk_loaded? + end + + def without_metrics_sdk + yield unless LoadedMetricsFeatures.sdk_loaded? + end + + def metrics_exporter + with_metrics_sdk { METRICS_EXPORTER } + end + + def reset_meter_provider + with_metrics_sdk do + resource = OpenTelemetry.meter_provider.resource + OpenTelemetry.meter_provider = OpenTelemetry::SDK::Metrics::MeterProvider.new(resource: resource) + OpenTelemetry.meter_provider.add_metric_reader(METRICS_EXPORTER) + end + end + + def reset_metrics_exporter + with_metrics_sdk do + METRICS_EXPORTER.pull + METRICS_EXPORTER.reset + end + end + + def it(desc = 'anonymous', with_metrics_sdk: false, without_metrics_sdk: false, &block) + return super(desc, &block) unless with_metrics_sdk || without_metrics_sdk + + raise ArgumentError, 'without_metrics_sdk and with_metrics_sdk must be mutually exclusive' if without_metrics_sdk && with_metrics_sdk + + return if with_metrics_sdk && !LoadedMetricsFeatures.sdk_loaded? + return if without_metrics_sdk && LoadedMetricsFeatures.sdk_loaded? + + super(desc, &block) + end + end + + if LoadedMetricsFeatures.sdk_loaded? + METRICS_EXPORTER = OpenTelemetry::SDK::Metrics::Export::InMemoryMetricPullExporter.new + OpenTelemetry.meter_provider.add_metric_reader(METRICS_EXPORTER) + end + + Minitest::Spec.prepend(MinitestExtensions) + end +end diff --git a/helpers/metrics-test-helpers/lib/opentelemetry/metrics_test_helpers/version.rb b/helpers/metrics-test-helpers/lib/opentelemetry/metrics_test_helpers/version.rb new file mode 100644 index 0000000000..63835ee6a1 --- /dev/null +++ b/helpers/metrics-test-helpers/lib/opentelemetry/metrics_test_helpers/version.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module MetricsTestHelpers + VERSION = '0.0.1' + end +end diff --git a/helpers/metrics-test-helpers/opentelemetry-metrics-test-helpers.gemspec b/helpers/metrics-test-helpers/opentelemetry-metrics-test-helpers.gemspec new file mode 100644 index 0000000000..e65a21334f --- /dev/null +++ b/helpers/metrics-test-helpers/opentelemetry-metrics-test-helpers.gemspec @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require_relative 'lib/opentelemetry/metrics_test_helpers/version' + +Gem::Specification.new do |spec| + spec.name = 'opentelemetry-metrics-test-helpers' + spec.version = OpenTelemetry::MetricsTestHelpers::VERSION + spec.authors = ['OpenTelemetry Authors'] + spec.email = ['cncf-opentelemetry-contributors@lists.cncf.io'] + + spec.summary = 'Test helpers for adding metrics to instrumentation libraries' + spec.homepage = 'https://github.com/open-telemetry/opentelemetry-ruby-contrib' + spec.required_ruby_version = '>= 2.7.0' + spec.license = 'Apache-2.0' + + spec.metadata['homepage_uri'] = spec.homepage + spec.metadata['source_code_uri'] = spec.homepage + + # Specify which files should be added to the gem when it is released. + # The `git ls-files -z` loads the files in the RubyGem that have been added into git. + gemspec = File.basename(__FILE__) + spec.files = IO.popen(%w[git ls-files -z], chdir: __dir__, err: IO::NULL) do |ls| + ls.readlines("\x0", chomp: true).reject do |f| + (f == gemspec) || + f.start_with?(*%w[bin/ test/ spec/ features/ .git appveyor Gemfile]) + end + end + spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } + spec.require_paths = ['lib'] + + # Uncomment to register a new dependency of your gem + # spec.add_dependency "example-gem", "~> 1.0" + spec.add_dependency 'minitest' + spec.add_development_dependency 'appraisal' + spec.add_development_dependency 'rubocop' + spec.add_development_dependency 'rubocop-performance' + + # For more information and examples about making a new gem, check out our + # guide at: https://bundler.io/guides/creating_gem.html +end diff --git a/helpers/metrics-test-helpers/test/opentelemetry/metrics_test_helpers_test.rb b/helpers/metrics-test-helpers/test/opentelemetry/metrics_test_helpers_test.rb new file mode 100644 index 0000000000..e989d6f0fa --- /dev/null +++ b/helpers/metrics-test-helpers/test/opentelemetry/metrics_test_helpers_test.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'test_helper' + +describe OpenTelemetry::MetricsTestHelpers do + with_metrics_sdk do + it 'must be defined' do + _(!defined?(OpenTelemetry::SDK::Metrics).nil?).must_equal(true) + end + end + + without_metrics_sdk do + it 'must not be defined' do + _(!defined?(OpenTelemetry::SDK::Metrics).nil?).must_equal(false) + end + end +end diff --git a/helpers/metrics-test-helpers/test/test_helper.rb b/helpers/metrics-test-helpers/test/test_helper.rb new file mode 100644 index 0000000000..ea6e151863 --- /dev/null +++ b/helpers/metrics-test-helpers/test/test_helper.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +require 'opentelemetry-sdk' + +begin + require 'opentelemetry-metrics-sdk' +rescue LoadError # rubocop: disable Lint/SuppressedException +end + +begin + require 'opentelemetry-metrics-api' +rescue LoadError # rubocop: disable Lint/SuppressedException +end + +OpenTelemetry::SDK.configure + +$LOAD_PATH.unshift File.expand_path('../lib', __dir__) +require 'opentelemetry-metrics-test-helpers' + +require 'minitest/autorun' diff --git a/instrumentation/base/Appraisals b/instrumentation/base/Appraisals new file mode 100644 index 0000000000..f99256b2a2 --- /dev/null +++ b/instrumentation/base/Appraisals @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +appraise 'base' do + remove_gem 'opentelemetry-metrics-api' + remove_gem 'opentelemetry-metrics-sdk' +end + +appraise 'metrics-api' do + remove_gem 'opentelemetry-metrics-sdk' +end + +appraise 'metrics-sdk' do # rubocop: disable Lint/EmptyBlock +end diff --git a/instrumentation/base/Gemfile b/instrumentation/base/Gemfile index f649e2f64a..c2a88bb22a 100644 --- a/instrumentation/base/Gemfile +++ b/instrumentation/base/Gemfile @@ -6,4 +6,7 @@ source 'https://rubygems.org' +gem 'opentelemetry-metrics-api', '~> 0.2' +gem 'opentelemetry-metrics-sdk' + gemspec diff --git a/instrumentation/base/lib/opentelemetry/instrumentation/base.rb b/instrumentation/base/lib/opentelemetry/instrumentation/base.rb index ddf8289551..3c0a226c97 100644 --- a/instrumentation/base/lib/opentelemetry/instrumentation/base.rb +++ b/instrumentation/base/lib/opentelemetry/instrumentation/base.rb @@ -69,8 +69,9 @@ class << self integer: ->(v) { v.is_a?(Integer) }, string: ->(v) { v.is_a?(String) } }.freeze + SINGLETON_MUTEX = Thread::Mutex.new - private_constant :NAME_REGEX, :VALIDATORS + private_constant :NAME_REGEX, :VALIDATORS, :SINGLETON_MUTEX private :new @@ -163,8 +164,10 @@ def option(name, default:, validate:) end def instance - @instance ||= new(instrumentation_name, instrumentation_version, install_blk, - present_blk, compatible_blk, options) + @instance || SINGLETON_MUTEX.synchronize do + @instance ||= new(instrumentation_name, instrumentation_version, install_blk, + present_blk, compatible_blk, options) + end end private @@ -189,13 +192,15 @@ def infer_version end end - attr_reader :name, :version, :config, :installed, :tracer + attr_reader :name, :version, :config, :installed, :tracer, :meter alias installed? installed + require_relative 'metrics' + prepend(OpenTelemetry::Instrumentation::Metrics) + # rubocop:disable Metrics/ParameterLists - def initialize(name, version, install_blk, present_blk, - compatible_blk, options) + def initialize(name, version, install_blk, present_blk, compatible_blk, options) @name = name @version = version @install_blk = install_blk @@ -204,7 +209,8 @@ def initialize(name, version, install_blk, present_blk, @config = {} @installed = false @options = options - @tracer = OpenTelemetry::Trace::Tracer.new + @tracer = OpenTelemetry::Trace::Tracer.new # default no-op tracer + @meter = OpenTelemetry::Metrics::Meter.new if defined?(OpenTelemetry::Metrics::Meter) # default no-op meter end # rubocop:enable Metrics/ParameterLists @@ -217,10 +223,12 @@ def install(config = {}) return true if installed? @config = config_options(config) + return false unless installable?(config) + prepare_install instance_exec(@config, &@install_blk) - @tracer = OpenTelemetry.tracer_provider.tracer(name, version) + @installed = true end @@ -263,6 +271,10 @@ def enabled?(config = nil) private + def prepare_install + @tracer = OpenTelemetry.tracer_provider.tracer(name, version) + end + # The config_options method is responsible for validating that the user supplied # config hash is valid. # Unknown configuration keys are not included in the final config hash. @@ -317,13 +329,17 @@ def config_options(user_config) # will be OTEL_RUBY_INSTRUMENTATION_SINATRA_ENABLED. A value of 'false' will disable # the instrumentation, all other values will enable it. def enabled_by_env_var? + !disabled_by_env_var? + end + + def disabled_by_env_var? var_name = name.dup.tap do |n| n.upcase! n.gsub!('::', '_') n.gsub!('OPENTELEMETRY_', 'OTEL_RUBY_') n << '_ENABLED' end - ENV[var_name] != 'false' + ENV[var_name] == 'false' end # Checks to see if the user has passed any environment variables that set options diff --git a/instrumentation/base/lib/opentelemetry/instrumentation/metrics.rb b/instrumentation/base/lib/opentelemetry/instrumentation/metrics.rb new file mode 100644 index 0000000000..1714037b31 --- /dev/null +++ b/instrumentation/base/lib/opentelemetry/instrumentation/metrics.rb @@ -0,0 +1,197 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +begin + require 'opentelemetry-metrics-api' +rescue LoadError +end + +module OpenTelemetry + module Instrumentation + # Extensions to Instrumentation::Base that handle metrics instruments. + # The goal here is to allow metrics to be added gradually to instrumentation libraries, + # without requiring that the metrics-sdk or metrics-api gems are present in the bundle + # (if they are not, or if the metrics-api gem does not meet the minimum version requirement, + # the no-op edition is installed.) + module Metrics + METER_TYPES = %i[ + counter + observable_counter + histogram + gauge + observable_gauge + up_down_counter + observable_up_down_counter + ].freeze + + def self.prepended(base) + base.prepend(Compatibility) + base.extend(Compatibility) + base.extend(Registration) + + if base.metrics_compatible? + base.prepend(Extensions) + else + base.prepend(NoopExtensions) + end + end + + # Methods to check whether the metrics API is defined + # and is a compatible version + module Compatibility + METRICS_API_MINIMUM_GEM_VERSION = Gem::Version.new('0.2.0') + + def metrics_defined? + defined?(OpenTelemetry::Metrics) + end + + def metrics_compatible? + metrics_defined? && Gem.loaded_specs['opentelemetry-metrics-api'].version >= METRICS_API_MINIMUM_GEM_VERSION + end + + extend(self) + end + + # class-level methods to declare and register metrics instruments. + # This can be extended even if metrics is not active or present. + module Registration + METER_TYPES.each do |instrument_kind| + define_method(instrument_kind) do |name, **opts, &block| + opts[:callback] ||= block if block + register_instrument(instrument_kind, name, **opts) + end + end + + def register_instrument(kind, name, **opts) + key = [kind, name] + if instrument_configs.key?(key) + warn("Duplicate instrument configured for #{self}: #{key.inspect}") + else + instrument_configs[key] = opts + end + end + + def instrument_configs + @instrument_configs ||= {} + end + end + + # No-op instance methods for metrics instruments. + module NoopExtensions + METER_TYPES.each do |kind| + define_method(kind) { |*, **| } # rubocop: disable Lint/EmptyBlock + end + + def with_meter; end + + def metrics_enabled? + false + end + end + + # Instance methods for metrics instruments. + module Extensions + %i[ + counter + observable_counter + histogram + gauge + observable_gauge + up_down_counter + observable_up_down_counter + ].each do |kind| + define_method(kind) do |name| + get_metrics_instrument(kind, name) + end + end + + # This is based on a variety of factors, and should be invalidated when @config changes. + # It should be explicitly set in `prepare_install` for now. + def metrics_enabled? + !!@metrics_enabled + end + + # @api private + # ONLY yields if the meter is enabled. + def with_meter + yield @meter if metrics_enabled? + end + + private + + def compute_metrics_enabled + return false unless metrics_compatible? + return false if metrics_disabled_by_env_var? + + !!@config[:metrics] || metrics_enabled_by_env_var? + end + + # Checks if this instrumentation's metrics are enabled by env var. + # This follows the conventions as outlined above, using `_METRICS_ENABLED` as a suffix. + # Unlike INSTRUMENTATION_*_ENABLED variables, these are explicitly opt-in (i.e. + # if the variable is unset, and `metrics: true` is not in the instrumentation's config, + # the metrics will not be enabled) + def metrics_enabled_by_env_var? + ENV.key?(metrics_env_var_name) && ENV[metrics_env_var_name] != 'false' + end + + def metrics_disabled_by_env_var? + ENV[metrics_env_var_name] == 'false' + end + + def metrics_env_var_name + @metrics_env_var_name ||= + begin + var_name = name.dup + var_name.upcase! + var_name.gsub!('::', '_') + var_name.gsub!('OPENTELEMETRY_', 'OTEL_RUBY_') + var_name << '_METRICS_ENABLED' + var_name + end + end + + def prepare_install + @metrics_enabled = compute_metrics_enabled + if metrics_defined? + @metrics_instruments = {} + @instrument_mutex = Mutex.new + end + + @meter = OpenTelemetry.meter_provider.meter(name, version: version) if metrics_enabled? + + super + end + + def get_metrics_instrument(kind, name) + # TODO: we should probably return *something* + # if metrics is not enabled, but if the api is undefined, + # it's unclear exactly what would be suitable. + # For now, there are no public methods that call this + # if metrics isn't defined. + return unless metrics_defined? + + @metrics_instruments.fetch([kind, name]) do |key| + @instrument_mutex.synchronize do + @metrics_instruments[key] ||= create_configured_instrument(kind, name) + end + end + end + + def create_configured_instrument(kind, name) + config = self.class.instrument_configs[[kind, name]] + + if config.nil? + Kernel.warn("unconfigured instrument requested: #{kind} of '#{name}'") + return + end + + meter.public_send(:"create_#{kind}", name, **config) + end + end + end + end +end diff --git a/instrumentation/base/opentelemetry-instrumentation-base.gemspec b/instrumentation/base/opentelemetry-instrumentation-base.gemspec index 90edf29d09..3d82fad02a 100644 --- a/instrumentation/base/opentelemetry-instrumentation-base.gemspec +++ b/instrumentation/base/opentelemetry-instrumentation-base.gemspec @@ -29,6 +29,7 @@ Gem::Specification.new do |spec| spec.add_dependency 'opentelemetry-common', '~> 0.21' spec.add_dependency 'opentelemetry-registry', '~> 0.1' + spec.add_development_dependency 'appraisal', '~> 2.5' spec.add_development_dependency 'bundler', '~> 2.4' spec.add_development_dependency 'minitest', '~> 5.0' spec.add_development_dependency 'opentelemetry-test-helpers', '~> 0.3' diff --git a/instrumentation/base/test/instrumentation/base_test.rb b/instrumentation/base/test/instrumentation/base_test.rb index c58bbe5003..ac0306ed3d 100644 --- a/instrumentation/base/test/instrumentation/base_test.rb +++ b/instrumentation/base/test/instrumentation/base_test.rb @@ -53,15 +53,89 @@ def initialize(*args) end end + let(:instrumentation_with_metrics) do + Class.new(OpenTelemetry::Instrumentation::Base) do + instrumentation_name 'test_instrumentation' + instrumentation_version '0.1.1' + + option :metrics, default: false, validate: :boolean + + install { true } + present { true } + + if defined?(OpenTelemetry::Metrics) + counter 'example.counter' + observable_counter 'example.observable_counter' + histogram 'example.histogram' + gauge 'example.gauge' + observable_gauge 'example.observable_gauge' + up_down_counter 'example.up_down_counter' + observable_up_down_counter 'example.observable_up_down_counter' + end + + def example_counter + counter 'example.counter' + end + + def example_observable_counter + observable_counter 'example.observable_counter' + end + + def example_histogram + histogram 'example.histogram' + end + + def example_gauge + gauge 'example.gauge' + end + + def example_observable_gauge + observable_gauge 'example.observable_gauge' + end + + def example_up_down_counter + up_down_counter 'example.up_down_counter' + end + + def example_observable_up_down_counter + observable_up_down_counter 'example.observable_up_down_counter' + end + end + end + it 'is auto-registered' do instance = instrumentation.instance _(OpenTelemetry::Instrumentation.registry.lookup('test_instrumentation')).must_equal(instance) end describe '.instance' do + let(:instrumentation) do + Class.new(OpenTelemetry::Instrumentation::Base) do + instrumentation_name 'test_instrumentation' + instrumentation_version '0.1.1' + + def initialize(*args) + # Simulate latency by hinting the VM should switch tasks + # (this can also be accomplished by something like `sleep(0.1)`). + # This replicates the worst-case scenario when using default assignment + # to obtain a singleton, i.e. that the scheduler switches threads between + # the nil check and object initialization. + Thread.pass + super + end + end + end + it 'returns an instance' do _(instrumentation.instance).must_be_instance_of(instrumentation) end + + it 'returns the same singleton instance to every thread' do + object_ids = Array.new(2).map { Thread.new { instrumentation.instance } } + .map { |thr| thr.join.value } + + _(object_ids.uniq.count).must_equal(1) + end end describe '.option' do @@ -442,6 +516,56 @@ def initialize(*args) end end + describe 'metrics' do + let(:config) { {} } + let(:instance) { instrumentation_with_metrics.instance } + + before do + instance.install(config) + end + + if defined?(OpenTelemetry::Metrics) + describe 'with the metrics api' do + it 'is disabled by default' do + _(instance.metrics_enabled?).must_equal false + end + + it 'returns a no-op counter' do + counter = instance.example_counter + _(counter).must_be_kind_of(OpenTelemetry::Metrics::Instrument::Counter) + end + + describe 'with the option enabled' do + let(:config) { { metrics: true } } + + it 'will be enabled' do + _(instance.metrics_enabled?).must_equal true + end + + it 'returns a counter' do + counter = instance.example_counter + + _(counter).must_be_kind_of(OpenTelemetry::Internal::ProxyInstrument) + end + end + end + else + describe 'without the metrics api' do + it 'will not be enabled' do + _(instance.metrics_enabled?).must_equal false + end + + describe 'with the option enabled' do + let(:config) { { metrics: true } } + + it 'will not be enabled' do + _(instance.metrics_enabled?).must_equal false + end + end + end + end + end + def define_instrumentation_subclass(name, version = nil) names = name.split('::').map(&:to_sym) names.inject(Object) do |object, const| diff --git a/instrumentation/base/test/test_helper.rb b/instrumentation/base/test/test_helper.rb index 9efdc59b31..7febed6ea0 100644 --- a/instrumentation/base/test/test_helper.rb +++ b/instrumentation/base/test/test_helper.rb @@ -4,12 +4,14 @@ # # SPDX-License-Identifier: Apache-2.0 -require 'bundler/setup' -Bundler.require(:default, :development, :test) +require 'simplecov' SimpleCov.start SimpleCov.minimum_coverage 85 +require 'bundler/setup' +Bundler.require(:default, :development, :test) + require 'opentelemetry-instrumentation-base' require 'minitest/autorun' diff --git a/instrumentation/concurrent_ruby/opentelemetry-instrumentation-concurrent_ruby.gemspec b/instrumentation/concurrent_ruby/opentelemetry-instrumentation-concurrent_ruby.gemspec index 608f17c73a..08ebb90051 100644 --- a/instrumentation/concurrent_ruby/opentelemetry-instrumentation-concurrent_ruby.gemspec +++ b/instrumentation/concurrent_ruby/opentelemetry-instrumentation-concurrent_ruby.gemspec @@ -27,6 +27,7 @@ Gem::Specification.new do |spec| spec.add_dependency 'opentelemetry-api', '~> 1.0' spec.add_dependency 'opentelemetry-instrumentation-base', '~> 0.23.0' + spec.add_dependency 'opentelemetry-metrics-api', '~> 1.0' spec.add_development_dependency 'appraisal', '~> 2.5' spec.add_development_dependency 'bundler', '~> 2.4' diff --git a/instrumentation/sidekiq/Appraisals b/instrumentation/sidekiq/Appraisals index 3b241272e4..b14ccb7043 100644 --- a/instrumentation/sidekiq/Appraisals +++ b/instrumentation/sidekiq/Appraisals @@ -1,24 +1,39 @@ # frozen_string_literal: true -appraise 'sidekiq-7.0' do - gem 'sidekiq', '~> 7.0' -end - -appraise 'sidekiq-6.5' do - gem 'sidekiq', '>= 6.5', '< 7.0' -end +{ + 'sidekiq-7.0' => [['sidekiq', '~> 7.0']], + 'sidekiq-6.5' => [['sidekiq', '>= 6.5', '< 7.0']], + 'sidekiq-6.0' => [ + ['sidekiq', '>= 6.0', '< 6.5'], + ['redis', '< 4.8'] + ], + 'sidekiq-5.2' => [ + ['sidekiq', '~> 5.2'], + ['redis', '< 4.8'] + ], + 'sidekiq-4.2' => [ + ['sidekiq', '~> 4.2'], + ['redis', '< 4.8'] + ] +}.each do |gemfile_name, specs| + appraise gemfile_name do + specs.each do |spec| + gem(*spec) + remove_gem 'opentelemetry-metrics-api' + remove_gem 'opentelemetry-metrics-sdk' + end + end -appraise 'sidekiq-6.0' do - gem 'sidekiq', '>= 6.0', '< 6.5' - gem 'redis', '< 4.8' -end - -appraise 'sidekiq-5.2' do - gem 'sidekiq', '~> 5.2' - gem 'redis', '< 4.8' -end + appraise "#{gemfile_name}-metrics-api" do + specs.each do |spec| + gem(*spec) + remove_gem 'opentelemetry-metrics-sdk' + end + end -appraise 'sidekiq-4.2' do - gem 'sidekiq', '~> 4.2' - gem 'redis', '< 4.8' + appraise "#{gemfile_name}-metrics-sdk" do + specs.each do |spec| + gem(*spec) + end + end end diff --git a/instrumentation/sidekiq/Gemfile b/instrumentation/sidekiq/Gemfile index 84efc8a188..81614a3a53 100644 --- a/instrumentation/sidekiq/Gemfile +++ b/instrumentation/sidekiq/Gemfile @@ -6,10 +6,13 @@ source 'https://rubygems.org' +gem 'opentelemetry-metrics-api', '~> 0.2.0' + gemspec group :test do gem 'opentelemetry-instrumentation-base', path: '../base' gem 'opentelemetry-instrumentation-redis', path: '../redis' - gem 'pry-byebug' + gem 'opentelemetry-metrics-sdk' + gem 'opentelemetry-metrics-test-helpers', path: '../../helpers/metrics-test-helpers', require: false end diff --git a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/instrumentation.rb b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/instrumentation.rb index 1f3fe27200..e61d56cf40 100644 --- a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/instrumentation.rb +++ b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/instrumentation.rb @@ -107,6 +107,15 @@ class Instrumentation < OpenTelemetry::Instrumentation::Base option :trace_poller_wait, default: false, validate: :boolean option :trace_processor_process_one, default: false, validate: :boolean option :peer_service, default: nil, validate: :string + option :metrics, default: false, validate: :boolean + + counter 'messaging.client.sent.messages' + histogram 'messaging.client.operation.duration', unit: 's' + counter 'messaging.client.consumed.messages' + histogram 'messaging.process.duration', unit: 's' + + # TODO: https://github.com/open-telemetry/semantic-conventions/pull/1812 + gauge 'messaging.queue.latency', unit: 's' private @@ -115,6 +124,7 @@ def gem_version end def require_dependencies + require_relative 'middlewares/common' require_relative 'middlewares/client/tracer_middleware' require_relative 'middlewares/server/tracer_middleware' diff --git a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb index 039390a8f3..438fb5c1aa 100644 --- a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb +++ b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware.rb @@ -4,6 +4,8 @@ # # SPDX-License-Identifier: Apache-2.0 +require_relative '../common' + module OpenTelemetry module Instrumentation module Sidekiq @@ -12,6 +14,7 @@ module Client # TracerMiddleware propagates context and instruments Sidekiq client # by way of its middleware system class TracerMiddleware + include Common include ::Sidekiq::ClientMiddleware if defined?(::Sidekiq::ClientMiddleware) def call(_worker_class, job, _queue, _redis_pool) @@ -33,17 +36,49 @@ def call(_worker_class, job, _queue, _redis_pool) OpenTelemetry.propagation.inject(job) span.add_event('created_at', timestamp: job['created_at']) yield + end.tap do # rubocop: disable Style/MultilineBlockChain + count_sent_message(job) end end private - def instrumentation_config - Sidekiq::Instrumentation.instance.config + def count_sent_message(job) + with_meter do |_meter| + counter_attributes = metrics_attributes(job).merge( + { + 'messaging.operation.name' => 'create' + # server.address => # FIXME: required if available + # messaging.destination.partition.id => FIXME: recommended + # server.port => # FIXME: recommended + } + ) + + counter = messaging_client_sent_messages_counter + counter.add(1, attributes: counter_attributes) + end + end + + def messaging_client_sent_messages_counter + instrumentation.counter('messaging.client.sent.messages') end def tracer - Sidekiq::Instrumentation.instance.tracer + instrumentation.tracer + end + + def with_meter(&block) + instrumentation.with_meter(&block) + end + + def metrics_attributes(job) + { + 'messaging.system' => 'sidekiq', # FIXME: metrics semconv + 'messaging.destination.name' => job['queue'] # FIXME: metrics semconv + # server.address => # FIXME: required if available + # messaging.destination.partition.id => FIXME: recommended + # server.port => # FIXME: recommended + } end end end diff --git a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/common.rb b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/common.rb new file mode 100644 index 0000000000..3076a9ee03 --- /dev/null +++ b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/common.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +# Copyright The OpenTelemetry Authors +# +# SPDX-License-Identifier: Apache-2.0 + +module OpenTelemetry + module Instrumentation + module Sidekiq + module Middlewares + # Common logic for server and client middlewares + module Common + private + + def instrumentation + Sidekiq::Instrumentation.instance + end + + def instrumentation_config + Sidekiq::Instrumentation.instance.config + end + + # Bypasses _all_ enclosed logic unless metrics are enabled + def with_meter(&block) + instrumentation.with_meter(&block) + end + + # time an inner block and yield the duration to the given callback + def timed(callback) + return yield unless metrics_enabled? + + t0 = monotonic_now + + yield.tap do + callback.call(monotonic_now - t0) + end + end + + def realtime_now + Process.clock_gettime(Process::CLOCK_REALTIME) + end + + def monotonic_now + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + def tracer + instrumentation.tracer + end + + def metrics_enabled? + instrumentation.metrics_enabled? + end + end + end + end + end +end diff --git a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb index 90da96ea3f..f8d51ca702 100644 --- a/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb +++ b/instrumentation/sidekiq/lib/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware.rb @@ -4,6 +4,8 @@ # # SPDX-License-Identifier: Apache-2.0 +require_relative '../common' + module OpenTelemetry module Instrumentation module Sidekiq @@ -12,6 +14,7 @@ module Server # TracerMiddleware propagates context and instruments Sidekiq requests # by way of its middleware system class TracerMiddleware + include Common include ::Sidekiq::ServerMiddleware if defined?(::Sidekiq::ServerMiddleware) def call(_worker, msg, _queue) @@ -32,40 +35,91 @@ def call(_worker, msg, _queue) extracted_context = OpenTelemetry.propagation.extract(msg) OpenTelemetry::Context.with_current(extracted_context) do - if instrumentation_config[:propagation_style] == :child - tracer.in_span(span_name, attributes: attributes, kind: :consumer) do |span| - span.add_event('created_at', timestamp: msg['created_at']) - span.add_event('enqueued_at', timestamp: msg['enqueued_at']) - yield - end - else - links = [] - span_context = OpenTelemetry::Trace.current_span(extracted_context).context - links << OpenTelemetry::Trace::Link.new(span_context) if instrumentation_config[:propagation_style] == :link && span_context.valid? - span = tracer.start_root_span(span_name, attributes: attributes, links: links, kind: :consumer) - OpenTelemetry::Trace.with_span(span) do - span.add_event('created_at', timestamp: msg['created_at']) - span.add_event('enqueued_at', timestamp: msg['enqueued_at']) - yield - rescue Exception => e # rubocop:disable Lint/RescueException - span.record_exception(e) - span.status = OpenTelemetry::Trace::Status.error("Unhandled exception of type: #{e.class}") - raise e - ensure - span.finish + track_queue_latency(msg) + + timed(track_process_time_callback(msg)) do + if instrumentation_config[:propagation_style] == :child + tracer.in_span(span_name, attributes: attributes, kind: :consumer) do |span| + span.add_event('created_at', timestamp: msg['created_at']) + span.add_event('enqueued_at', timestamp: msg['enqueued_at']) + yield + end + else + links = [] + span_context = OpenTelemetry::Trace.current_span(extracted_context).context + links << OpenTelemetry::Trace::Link.new(span_context) if instrumentation_config[:propagation_style] == :link && span_context.valid? + span = tracer.start_root_span(span_name, attributes: attributes, links: links, kind: :consumer) + OpenTelemetry::Trace.with_span(span) do + span.add_event('created_at', timestamp: msg['created_at']) + span.add_event('enqueued_at', timestamp: msg['enqueued_at']) + yield + rescue Exception => e # rubocop:disable Lint/RescueException + span.record_exception(e) + span.status = OpenTelemetry::Trace::Status.error("Unhandled exception of type: #{e.class}") + raise e + ensure + span.finish + end end end + + count_consumed_message(msg) end end private - def instrumentation_config - Sidekiq::Instrumentation.instance.config + def track_queue_latency(msg) + with_meter do + return unless (enqueued_at = msg['enqueued_at']) + return unless enqueued_at.is_a?(Numeric) + + latency = (realtime_now - enqueued_at).abs + + queue_latency_gauge&.record(latency, attributes: metrics_attributes(msg)) + end + end + + def track_process_time_callback(msg) + ->(duration) { track_process_time(msg, duration) } + end + + def track_process_time(msg, duration) + with_meter do + attributes = metrics_attributes(msg).merge( + { 'messaging.operation.name' => 'process' } + ) + messaging_process_duration_histogram&.record(duration, attributes: attributes) + end + end + + def messaging_process_duration_histogram + instrumentation.histogram('messaging.process.duration') + end + + def count_consumed_message(msg) + with_meter do + messaging_client_consumed_messages_counter.add(1, attributes: metrics_attributes(msg)) + end end - def tracer - Sidekiq::Instrumentation.instance.tracer + def messaging_client_consumed_messages_counter + instrumentation.counter('messaging.client.consumed.messages') + end + + def queue_latency_gauge + instrumentation.gauge('messaging.queue.latency') + end + + # FIXME: dedupe + def metrics_attributes(msg) + { + 'messaging.system' => 'sidekiq', # FIXME: metrics semconv + 'messaging.destination.name' => msg['queue'] # FIXME: metrics semconv + # server.address => # FIXME: required if available + # messaging.destination.partition.id => FIXME: recommended + # server.port => # FIXME: recommended + } end end end diff --git a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware_test.rb b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware_test.rb index a2de3d05d3..9398478432 100644 --- a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware_test.rb +++ b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/client/tracer_middleware_test.rb @@ -15,6 +15,14 @@ let(:enqueue_span) { spans.first } let(:config) { {} } + with_metrics_sdk do + let(:metric_snapshots) do + metrics_exporter.tap(&:pull) + .metric_snapshots.select { |snapshot| snapshot.data_points.any? } + .group_by(&:name) + end + end + before do instrumentation.install(config) exporter.reset @@ -81,5 +89,34 @@ _(enqueue_span.attributes['peer.service']).must_equal 'MySidekiqService' end end + + with_metrics_sdk do + it 'yields no metrics if config is not set' do + _(instrumentation.metrics_enabled?).must_equal false + SimpleJob.perform_async + SimpleJob.drain + + _(metric_snapshots).must_be_empty + end + + describe 'with metrics enabled' do + let(:config) { { metrics: true } } + + it 'metrics processing' do + _(instrumentation.metrics_enabled?).must_equal true + SimpleJob.perform_async + SimpleJob.drain + + sent_messages = metric_snapshots['messaging.client.sent.messages'] + _(sent_messages.count).must_equal 1 + _(sent_messages.first.data_points.count).must_equal 1 + _(sent_messages.first.data_points.first.value).must_equal 1 + sent_messages_attributes = sent_messages.first.data_points.first.attributes + _(sent_messages_attributes['messaging.system']).must_equal 'sidekiq' + _(sent_messages_attributes['messaging.destination.name']).must_equal 'default' # FIXME: newer semconv specifies this key + _(sent_messages_attributes['messaging.operation.name']).must_equal 'create' + end + end + end end end diff --git a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb index 145d3b7438..cdf93ecbf9 100644 --- a/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb +++ b/instrumentation/sidekiq/test/opentelemetry/instrumentation/sidekiq/middlewares/server/tracer_middleware_test.rb @@ -17,12 +17,22 @@ let(:root_span) { spans.find { |s| s.parent_span_id == OpenTelemetry::Trace::INVALID_SPAN_ID } } let(:config) { {} } + with_metrics_sdk do + let(:metric_snapshots) do + metrics_exporter.tap(&:pull) + .metric_snapshots.select { |snapshot| snapshot.data_points.any? } + .group_by(&:name) + end + end + before do instrumentation.install(config) exporter.reset end - after { instrumentation.instance_variable_set(:@installed, false) } + after do + instrumentation.instance_variable_set(:@installed, false) + end describe 'enqueue spans' do it 'before performing any jobs' do @@ -49,6 +59,54 @@ _(job_span.events[1].name).must_equal('enqueued_at') end + with_metrics_sdk do + # FIXME: still seeing order-dependent failure here + it 'yields no metrics if config is not set' do + _(OpenTelemetry::Instrumentation::Sidekiq::Instrumentation.instance.metrics_enabled?).must_equal false + SimpleJob.perform_async + SimpleJob.drain + + _(exporter.finished_spans.size).must_equal 2 + _(metric_snapshots).must_be_empty + end + + describe 'with metrics enabled' do + let(:config) { { metrics: true } } + + it 'metrics processing' do + _(instrumentation.metrics_enabled?).must_equal true + SimpleJob.perform_async + SimpleJob.drain + + queue_latency = metric_snapshots['messaging.queue.latency'] + _(queue_latency.count).must_equal 1 + _(queue_latency.first.data_points.count).must_equal 1 + queue_latency_attributes = queue_latency.first.data_points.first.attributes + _(queue_latency_attributes['messaging.system']).must_equal 'sidekiq' + _(queue_latency_attributes['messaging.destination.name']).must_equal 'default' # FIXME: newer semconv specifies this key + + process_duration = metric_snapshots['messaging.process.duration'] + _(process_duration.count).must_equal 1 + _(process_duration.first.data_points.count).must_equal 1 + process_duration_attributes = process_duration.first.data_points.first.attributes + _(process_duration_attributes['messaging.system']).must_equal 'sidekiq' + _(process_duration_attributes['messaging.operation.name']).must_equal 'process' + _(process_duration_attributes['messaging.destination.name']).must_equal 'default' + + process_duration_data_point = process_duration.first.data_points.first + _(process_duration_data_point.count).must_equal 1 + + consumed_messages = metric_snapshots['messaging.client.consumed.messages'] + _(consumed_messages.count).must_equal 1 + _(consumed_messages.first.data_points.count).must_equal 1 + consumed_messages_attributes = queue_latency.first.data_points.first.attributes + _(consumed_messages_attributes['messaging.system']).must_equal 'sidekiq' + _(consumed_messages_attributes['messaging.destination.name']).must_equal 'default' # FIXME: newer semconv specifies this key + _(consumed_messages.first.data_points.first.value).must_equal 1 + end + end + end + it 'traces when enqueued with Active Job' do SimpleJobWithActiveJob.perform_later(1, 2) Sidekiq::Worker.drain_all diff --git a/instrumentation/sidekiq/test/test_helper.rb b/instrumentation/sidekiq/test/test_helper.rb index df49e41255..987ba22b0e 100644 --- a/instrumentation/sidekiq/test/test_helper.rb +++ b/instrumentation/sidekiq/test/test_helper.rb @@ -21,6 +21,15 @@ require 'helpers/mock_loader' end +# speed up tests that rely on empty queues +Sidekiq::BasicFetch::TIMEOUT = + if Gem.loaded_specs['sidekiq'].version < Gem::Version.new('6.5.0') + # Redis 4.8 has trouble with float timeouts given as positional arguments + 1 + else + 0.1 + end + # OpenTelemetry SDK config for testing EXPORTER = OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter.new span_processor = OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(EXPORTER) @@ -29,6 +38,7 @@ c.error_handler = ->(exception:, message:) { raise(exception || message) } c.add_span_processor span_processor end +require 'opentelemetry-metrics-test-helpers' # Sidekiq redis configuration ENV['TEST_REDIS_HOST'] ||= '127.0.0.1'