add useOpenTelemetryInitializedByApplication config

Author: m-nagarajan

internal/venice-client-common/src/main/java/com/linkedin/venice/stats/VeniceMetricsConfig.java

@@ -32,11 +32,27 @@ public class VeniceMetricsConfig {
    */
   public static final String OTEL_VENICE_METRICS_ENABLED = "otel.venice.metrics.enabled";
 
+  /**
+   * Config to reuse the {@link io.opentelemetry.api.OpenTelemetry} initialized by the application
+   * or other libraries and set as {@link io.opentelemetry.api.GlobalOpenTelemetry} rather than
+   * initializing it individually. This is particularly useful for clients use cases where one
+   * application can initialize multiple venice client libraries which would otherwise initialize
+   * OpenTelemetry multiple times, which could take more resources.
+   */
+  public static final String OTEL_VENICE_USE_OPENTELEMETRY_INITIALIZED_BY_APPLICATION =
+      "otel.venice.use.opentelemetry.initialized.by.application";
+
   /**
    * Config to set the metric prefix for OpenTelemetry metrics
    */
   public static final String OTEL_VENICE_METRICS_PREFIX = "otel.venice.metrics.prefix";
 
+  /**
+   * Config to set custom description for all Histogram metrics.
+   */
+  public static final String OTEL_VENICE_METRICS_CUSTOM_DESCRIPTION_FOR_HISTOGRAM =
+      "otel.venice.metrics.custom.description.for.histogram";
+
   /**
    * Config to set the naming format for OpenTelemetry metrics
    * {@link VeniceOpenTelemetryMetricNamingFormat}
@@ -139,6 +155,17 @@ public class VeniceMetricsConfig {
   /** Feature flag to use OpenTelemetry instrumentation for metrics or not */
   private final boolean emitOTelMetrics;
 
+  /**
+   * Feature flag to use OpenTelemetry initialized by the application or not.
+   * If true, it will use the GlobalOpenTelemetry instance. It could be initialized
+   *          by the application or by some other library
+   * If false, it will initialize its own OpenTelemetry instance.
+   */
+  private final boolean useOpenTelemetryInitializedByApplication;
+
+  /** custom description for all Histogram metrics */
+  private final String otelCustomDescriptionForHistogramMetrics;
+
   /** extra configs for OpenTelemetry. Supports 2 exporter currently <br>
    * 1. {@link MetricExporter} for exporting to Http/Grpc endpoint. More details are supported via configs,
    *    check {@link Builder#extractAndSetOtelConfigs} and {@link VeniceOpenTelemetryMetricsRepository#getOtlpHttpMetricExporter}<br>
@@ -183,6 +210,8 @@ private VeniceMetricsConfig(Builder builder) {
     this.metricPrefix = builder.metricPrefix;
     this.metricEntities = builder.metricEntities;
     this.emitOTelMetrics = builder.emitOtelMetrics;
+    this.useOpenTelemetryInitializedByApplication = builder.useOpenTelemetryInitializedByApplication;
+    this.otelCustomDescriptionForHistogramMetrics = builder.otelCustomDescriptionForHistogramMetrics;
     this.exportOtelMetricsToEndpoint = builder.exportOtelMetricsToEndpoint;
     this.exportOtelMetricsIntervalInSeconds = builder.exportOtelMetricsIntervalInSeconds;
     this.otelCustomDimensionsMap = builder.otelCustomDimensionsMap;
@@ -204,6 +233,8 @@ public static class Builder {
     private String metricPrefix = null;
     private Collection<MetricEntity> metricEntities = new ArrayList<>();
     private boolean emitOtelMetrics = false;
+    private boolean useOpenTelemetryInitializedByApplication = false;
+    private String otelCustomDescriptionForHistogramMetrics = null;
     private boolean exportOtelMetricsToEndpoint = false;
     private int exportOtelMetricsIntervalInSeconds = 60;
     private Map<String, String> otelCustomDimensionsMap = new HashMap<>();
@@ -240,6 +271,16 @@ public Builder setEmitOtelMetrics(boolean emitOtelMetrics) {
       return this;
     }
 
+    public Builder setUseOpenTelemetryInitializedByApplication(boolean useOpenTelemetryInitializedByApplication) {
+      this.useOpenTelemetryInitializedByApplication = useOpenTelemetryInitializedByApplication;
+      return this;
+    }
+
+    public Builder setOtelCustomDescriptionForHistogramMetrics(String otelCustomDescriptionForHistogramMetrics) {
+      this.otelCustomDescriptionForHistogramMetrics = otelCustomDescriptionForHistogramMetrics;
+      return this;
+    }
+
     public Builder setExportOtelMetricsToEndpoint(boolean exportOtelMetricsToEndpoint) {
       this.exportOtelMetricsToEndpoint = exportOtelMetricsToEndpoint;
       return this;
@@ -325,6 +366,14 @@ public Builder extractAndSetOtelConfigs(Map<String, String> configs) {
         setMetricPrefix(configValue);
       }
 
+      if ((configValue = configs.get(OTEL_VENICE_USE_OPENTELEMETRY_INITIALIZED_BY_APPLICATION)) != null) {
+        setUseOpenTelemetryInitializedByApplication(Boolean.parseBoolean(configValue));
+      }
+
+      if ((configValue = configs.get(OTEL_VENICE_METRICS_CUSTOM_DESCRIPTION_FOR_HISTOGRAM)) != null) {
+        setOtelCustomDescriptionForHistogramMetrics(configValue);
+      }
+
       if ((configValue = configs.get(OTEL_VENICE_METRICS_EXPORT_TO_LOG)) != null) {
         setExportOtelMetricsToLog(Boolean.parseBoolean(configValue));
       }
@@ -469,6 +518,14 @@ public boolean emitOtelMetrics() {
     return emitOTelMetrics;
   }
 
+  public boolean useOpenTelemetryInitializedByApplication() {
+    return useOpenTelemetryInitializedByApplication;
+  }
+
+  public String getOtelCustomDescriptionForHistogramMetrics() {
+    return otelCustomDescriptionForHistogramMetrics;
+  }
+
   public boolean exportOtelMetricsToEndpoint() {
     return exportOtelMetricsToEndpoint;
   }

internal/venice-client-common/src/main/java/com/linkedin/venice/stats/VeniceOpenTelemetryMetricsRepository.java

@@ -3,6 +3,7 @@
 import static com.linkedin.venice.stats.VeniceOpenTelemetryMetricNamingFormat.transformMetricName;
 import static com.linkedin.venice.stats.VeniceOpenTelemetryMetricNamingFormat.validateMetricName;
 
+import com.google.common.annotations.VisibleForTesting;
 import com.linkedin.venice.exceptions.VeniceException;
 import com.linkedin.venice.stats.dimensions.VeniceDimensionInterface;
 import com.linkedin.venice.stats.dimensions.VeniceMetricsDimensions;
@@ -11,6 +12,7 @@
 import com.linkedin.venice.stats.metrics.MetricType;
 import com.linkedin.venice.stats.metrics.MetricUnit;
 import com.linkedin.venice.utils.concurrent.VeniceConcurrentHashMap;
+import io.opentelemetry.api.GlobalOpenTelemetry;
 import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.common.AttributesBuilder;
@@ -55,6 +57,7 @@ public class VeniceOpenTelemetryMetricsRepository {
   public static final String DEFAULT_METRIC_PREFIX = "venice.";
   private final VeniceMetricsConfig metricsConfig;
   private SdkMeterProvider sdkMeterProvider = null;
+  private OpenTelemetry openTelemetry = null;
   private final boolean emitOpenTelemetryMetrics;
   private final VeniceOpenTelemetryMetricNamingFormat metricFormat;
   private Meter meter;
@@ -80,62 +83,75 @@ public VeniceOpenTelemetryMetricsRepository(VeniceMetricsConfig metricsConfig) {
         metricsConfig.toString());
     this.metricPrefix = metricsConfig.getMetricPrefix();
     validateMetricName(getMetricPrefix());
-    try {
-      SdkMeterProviderBuilder builder = SdkMeterProvider.builder();
-
-      if (metricsConfig.exportOtelMetricsToEndpoint()) {
-        MetricExporter httpExporter = getOtlpHttpMetricExporter(metricsConfig);
-        builder.registerMetricReader(
-            PeriodicMetricReader.builder(httpExporter)
-                .setInterval(metricsConfig.getExportOtelMetricsIntervalInSeconds(), TimeUnit.SECONDS)
-                .build());
+    if (metricsConfig.useOpenTelemetryInitializedByApplication()) {
+      LOGGER.warn("Configured to use GlobalOpenTelemetry set by the application");
+      openTelemetry = GlobalOpenTelemetry.get();
+      if (openTelemetry == null) {
+        // this is an extra safety check as GlobalOpenTelemetry.get() will return a default noop instance
+        // if it was not originally initialized by the application
+        throw new VeniceException("OpenTelemetry is not initialized globally, but it is required for metrics.");
       }
-
-      if (metricsConfig.exportOtelMetricsToLog()) {
-        // internal to test: Disabled by default
-        builder.registerMetricReader(
-            PeriodicMetricReader.builder(new LogBasedMetricExporter(metricsConfig))
-                .setInterval(metricsConfig.getExportOtelMetricsIntervalInSeconds(), TimeUnit.SECONDS)
-                .build());
-      }
-
-      if (metricsConfig.getOtelAdditionalMetricsReader() != null) {
-        // additional metrics reader apart from the above. For instance,
-        // an in-memory metric reader can be passed in for testing purposes.
-        builder.registerMetricReader(metricsConfig.getOtelAdditionalMetricsReader());
-      }
-
-      if (metricsConfig.useOtelExponentialHistogram()) {
-        setExponentialHistogramAggregation(builder, metricsConfig);
-      }
-
-      // Set resource to empty to avoid adding any default resource attributes. The receiver
-      // pipeline can choose to add the respective resource attributes if needed.
-      builder.setResource(Resource.empty());
-
-      sdkMeterProvider = builder.build();
-
-      // Register MeterProvider with the OpenTelemetry instance
-      OpenTelemetry openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
-
-      this.meter = openTelemetry.getMeter(transformMetricName(getMetricPrefix(), metricFormat));
-
-      this.recordFailureMetric = MetricEntityStateBase.create(
-          CommonMetricsEntity.METRIC_RECORD_FAILURE.getMetricEntity(),
-          this,
-          Collections.EMPTY_MAP,
-          Attributes.empty());
-
+      LOGGER.info("Using globally initialized OpenTelemetry for {}", metricsConfig.getServiceName());
+    } else {
       LOGGER.info(
-          "OpenTelemetry initialization for {} completed with config: {}",
+          "OpenTelemetry initialization for {} started with config: {}",
           metricsConfig.getServiceName(),
-          metricsConfig);
-    } catch (Exception e) {
-      String err = "OpenTelemetry initialization for " + metricsConfig.getServiceName() + " failed with config: "
-          + metricsConfig;
-      LOGGER.error(err, e);
-      throw new VeniceException(err, e);
+          metricsConfig.toString());
+      try {
+        SdkMeterProviderBuilder builder = SdkMeterProvider.builder();
+
+        if (metricsConfig.exportOtelMetricsToEndpoint()) {
+          MetricExporter httpExporter = getOtlpHttpMetricExporter(metricsConfig);
+          builder.registerMetricReader(
+              PeriodicMetricReader.builder(httpExporter)
+                  .setInterval(metricsConfig.getExportOtelMetricsIntervalInSeconds(), TimeUnit.SECONDS)
+                  .build());
+        }
+
+        if (metricsConfig.exportOtelMetricsToLog()) {
+          // internal to test: Disabled by default
+          builder.registerMetricReader(
+              PeriodicMetricReader.builder(new LogBasedMetricExporter(metricsConfig))
+                  .setInterval(metricsConfig.getExportOtelMetricsIntervalInSeconds(), TimeUnit.SECONDS)
+                  .build());
+        }
+
+        if (metricsConfig.getOtelAdditionalMetricsReader() != null) {
+          // additional metrics reader apart from the above. For instance,
+          // an in-memory metric reader can be passed in for testing purposes.
+          builder.registerMetricReader(metricsConfig.getOtelAdditionalMetricsReader());
+        }
+
+        if (metricsConfig.useOtelExponentialHistogram()) {
+          setExponentialHistogramAggregation(builder, metricsConfig);
+        }
+
+        // Set resource to empty to avoid adding any default resource attributes. The receiver
+        // pipeline can choose to add the respective resource attributes if needed.
+        builder.setResource(Resource.empty());
+
+        sdkMeterProvider = builder.build();
+
+        // Register MeterProvider with the OpenTelemetry instance
+        openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
+        LOGGER.info(
+            "OpenTelemetry initialization for {} completed with config: {}",
+            metricsConfig.getServiceName(),
+            metricsConfig);
+      } catch (Exception e) {
+        String err = "OpenTelemetry initialization for " + metricsConfig.getServiceName() + " failed with config: "
+            + metricsConfig;
+        LOGGER.error(err, e);
+        throw new VeniceException(err, e);
+      }
     }
+
+    this.meter = openTelemetry.getMeter(transformMetricName(getMetricPrefix(), metricFormat));
+    this.recordFailureMetric = MetricEntityStateBase.create(
+        CommonMetricsEntity.METRIC_RECORD_FAILURE.getMetricEntity(),
+        this,
+        Collections.EMPTY_MAP,
+        Attributes.empty());
   }
 
   /**
@@ -162,14 +178,12 @@ MetricExporter getOtlpHttpMetricExporter(VeniceMetricsConfig metricsConfig) {
   }
 
   /**
-   * Setting Exponential Histogram aggregation for {@link MetricType#HISTOGRAM} by looping through all
-   * the metric entities set for this service to registering the view with exponential histogram aggregation for
-   * all the {@link MetricType#HISTOGRAM} metrics.
+   * Setting Exponential Histogram aggregation for each {@link MetricType#HISTOGRAM} metric by looping through all
+   * the metric entities set for this service using views.
    *
-   * There is a limitation in opentelemetry sdk to configure different histogram aggregation for different
-   * instruments, so {@link OtlpHttpMetricExporterBuilder#setDefaultAggregationSelector} to enable exponential
-   * histogram aggregation is not used here to not convert the histograms of type {@link MetricType#MIN_MAX_COUNT_SUM_AGGREGATIONS}
-   * to exponential histograms to be able to follow explict boundaries.
+   * Because the OpenTelemetry SDK cannot currently assign different histogram aggregations to different histogram
+   * instruments, we deliberately avoid {@link OtlpHttpMetricExporterBuilder#setDefaultAggregationSelector}. Using it
+   * would also convert {@link MetricType#MIN_MAX_COUNT_SUM_AGGREGATIONS} to exponential histograms.
    *
    * If the metric entities are empty, it will throw an exception. Failing fast here as
    * 1. If we configure exponential histogram aggregation for every histogram: it could lead to increased memory usage
@@ -226,6 +240,16 @@ String getMetricPrefix(MetricEntity metricEntity) {
         : createFullMetricPrefix(metricEntity.getCustomMetricPrefix()));
   }
 
+  static String getMetricDescription(MetricEntity metricEntity, VeniceMetricsConfig metricsConfig) {
+    String customDescription = metricsConfig.getOtelCustomDescriptionForHistogramMetrics();
+    if (metricEntity.getMetricType() == MetricType.HISTOGRAM && customDescription != null
+        && !customDescription.isEmpty()) {
+      return customDescription;
+    } else {
+      return metricEntity.getDescription();
+    }
+  }
+
   public DoubleHistogram createHistogram(MetricEntity metricEntity) {
     if (!emitOpenTelemetryMetrics()) {
       return null;
@@ -234,10 +258,10 @@ public DoubleHistogram createHistogram(MetricEntity metricEntity) {
       String fullMetricName = getFullMetricName(metricEntity);
       DoubleHistogramBuilder builder = meter.histogramBuilder(fullMetricName)
           .setUnit(metricEntity.getUnit().name())
-          .setDescription(metricEntity.getDescription());
+          .setDescription(getMetricDescription(metricEntity, metricsConfig));
       if (metricEntity.getMetricType() == MetricType.MIN_MAX_COUNT_SUM_AGGREGATIONS) {
         // No buckets needed to get only min/max/count/sum aggregations
-        builder.setExplicitBucketBoundariesAdvice(new ArrayList<>());
+        builder.setExplicitBucketBoundariesAdvice(new ArrayList<>()).setDescription(metricEntity.getDescription());
       }
       return builder.build();
     });
@@ -251,7 +275,7 @@ public LongCounter createCounter(MetricEntity metricEntity) {
       String fullMetricName = getFullMetricName(metricEntity);
       LongCounterBuilder builder = meter.counterBuilder(fullMetricName)
           .setUnit(metricEntity.getUnit().name())
-          .setDescription(metricEntity.getDescription());
+          .setDescription(getMetricDescription(metricEntity, metricsConfig));
       return builder.build();
     });
   }
@@ -264,7 +288,7 @@ public DoubleGauge createGuage(MetricEntity metricEntity) {
       String fullMetricName = getFullMetricName(metricEntity);
       DoubleGaugeBuilder builder = meter.gaugeBuilder(fullMetricName)
           .setUnit(metricEntity.getUnit().name())
-          .setDescription(metricEntity.getDescription());
+          .setDescription(getMetricDescription(metricEntity, metricsConfig));
       return builder.build();
     });
   }
@@ -433,17 +457,22 @@ public MetricEntity getMetricEntity() {
     }
   }
 
-  /** for testing purposes */
+  @VisibleForTesting
   SdkMeterProvider getSdkMeterProvider() {
     return sdkMeterProvider;
   }
 
-  /** for testing purposes */
+  @VisibleForTesting
+  OpenTelemetry getOpenTelemetry() {
+    return openTelemetry;
+  }
+
+  @VisibleForTesting
   Meter getMeter() {
     return meter;
   }
 
-  /** for testing purposes */
+  @VisibleForTesting
   public MetricEntityStateBase getRecordFailureMetric() {
     return this.recordFailureMetric;
   }

internal/venice-client-common/src/test/java/com/linkedin/venice/stats/VeniceMetricsConfigTest.java

@@ -15,6 +15,7 @@
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
 import static org.testng.Assert.assertTrue;
 
 import com.linkedin.venice.stats.VeniceMetricsConfig.Builder;
@@ -49,6 +50,9 @@ public void testDefaultValuesWithBasicConfig() {
     assertEquals(config.useOtelExponentialHistogram(), true);
     assertEquals(config.getOtelExponentialHistogramMaxScale(), 3);
     assertEquals(config.getOtelExponentialHistogramMaxBuckets(), 250);
+    assertTrue(config.getOtelCustomDimensionsMap().isEmpty());
+    assertFalse(config.useOpenTelemetryInitializedByApplication());
+    assertNull(config.getOtelCustomDescriptionForHistogramMetrics());
     assertNotNull(config.getTehutiMetricConfig());
   }
 
@@ -213,4 +217,23 @@ public void testSetOtelHeaders() {
         new Builder().setServiceName("TestService").setMetricPrefix("TestPrefix").setOtelHeaders(otelHeaders).build();
     assertEquals(config.getOtelHeaders().get("key1"), "value1");
   }
+
+  @Test
+  public void testSetOtelCustomDescription() {
+    String customDescription = "This is a custom description for OpenTelemetry metrics.";
+    VeniceMetricsConfig config = new Builder().setServiceName("TestService")
+        .setMetricPrefix("TestPrefix")
+        .setOtelCustomDescriptionForHistogramMetrics(customDescription)
+        .build();
+    assertEquals(config.getOtelCustomDescriptionForHistogramMetrics(), customDescription);
+  }
+
+  @Test
+  public void testUseOpenTelemetryInitializedByApplication() {
+    VeniceMetricsConfig config = new Builder().setServiceName("TestService")
+        .setMetricPrefix("TestPrefix")
+        .setUseOpenTelemetryInitializedByApplication(true)
+        .build();
+    assertTrue(config.useOpenTelemetryInitializedByApplication());
+  }
 }