Skip to content

Commit edb119b

Browse files
m-nagarajanm-nagarajan
authored
[common] fallback to initialize otel when fetching global otel fails (#2364)
Fallback to initializing a local openTelemetry if useOpenTelemetryInitializedByApplication is enabled and global otel is not initialized
1 parent 8ef18cc commit edb119b

File tree

2 files changed

+79
-74
lines changed

2 files changed

+79
-74
lines changed

‎internal/venice-client-common/src/main/java/com/linkedin/venice/stats/VeniceOpenTelemetryMetricsRepository.java‎

Lines changed: 69 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -81,86 +81,90 @@ public VeniceOpenTelemetryMetricsRepository(VeniceMetricsConfig metricsConfig) {
8181
emitOpenTelemetryMetrics = metricsConfig.emitOtelMetrics();
8282
emitTehutiMetrics = metricsConfig.emitTehutiMetrics();
8383
metricFormat = metricsConfig.getMetricNamingFormat();
84+
this.metricPrefix = metricsConfig.getMetricPrefix();
85+
validateMetricName(getMetricPrefix());
8486
if (!emitOpenTelemetryMetrics) {
8587
LOGGER.info("OpenTelemetry metrics are disabled");
8688
openTelemetry = null;
8789
return;
8890
}
89-
this.metricPrefix = metricsConfig.getMetricPrefix();
90-
validateMetricName(getMetricPrefix());
91+
this.openTelemetry = initializeOpenTelemetry(metricsConfig);
92+
this.meter = openTelemetry.getMeter(transformMetricName(getMetricPrefix(), metricFormat));
93+
this.recordFailureMetric = MetricEntityStateBase.create(
94+
CommonMetricsEntity.METRIC_RECORD_FAILURE.getMetricEntity(),
95+
this,
96+
Collections.EMPTY_MAP,
97+
Attributes.empty());
98+
}
99+
100+
private OpenTelemetry initializeOpenTelemetry(VeniceMetricsConfig metricsConfig) {
101+
OpenTelemetry otel;
91102
if (metricsConfig.useOpenTelemetryInitializedByApplication()) {
92103
LOGGER.info("Using globally initialized OpenTelemetry for {}", metricsConfig.getServiceName());
93-
openTelemetry = GlobalOpenTelemetry.get();
94-
if (openTelemetry == null || openTelemetry.getMeterProvider() == null
95-
|| openTelemetry.getMeterProvider().equals(MeterProvider.noop())) {
96-
// Fail fast if no global OpenTelemetry instance is initialized to avoid silent metric loss.
97-
// When disabled, each Venice component will initialize its own OpenTelemetry instance,
98-
// which can lead to multiple instances in the same application, especially when used as
99-
// a library (common for Venice clients) when multiple such libraries initialized.
100-
throw new VeniceException(
101-
"OpenTelemetry is not initialized globally by the application: disable the configuration or "
102-
+ "initialize OpenTelemetry in the application before initializing venice");
104+
otel = GlobalOpenTelemetry.get();
105+
if (otel == null || otel.getMeterProvider() == null || otel.getMeterProvider().equals(MeterProvider.noop())) {
106+
LOGGER.warn(
107+
"Global OpenTelemetry is not initialized properly. Falling back to local initialization for {}",
108+
metricsConfig.getServiceName());
109+
} else {
110+
LOGGER.info("Successfully obtained globally initialized OpenTelemetry for {}", metricsConfig.getServiceName());
111+
return otel;
103112
}
104-
} else {
105-
LOGGER.info(
106-
"OpenTelemetry initialization for {} started with config: {}",
107-
metricsConfig.getServiceName(),
108-
metricsConfig.toString());
109-
try {
110-
SdkMeterProviderBuilder builder = SdkMeterProvider.builder();
111-
112-
if (metricsConfig.exportOtelMetricsToEndpoint()) {
113-
MetricExporter httpExporter = getOtlpHttpMetricExporter(metricsConfig);
114-
builder.registerMetricReader(
115-
PeriodicMetricReader.builder(httpExporter)
116-
.setInterval(metricsConfig.getExportOtelMetricsIntervalInSeconds(), TimeUnit.SECONDS)
117-
.build());
118-
}
113+
}
119114

120-
if (metricsConfig.exportOtelMetricsToLog()) {
121-
// internal to test: Disabled by default
122-
builder.registerMetricReader(
123-
PeriodicMetricReader.builder(new LogBasedMetricExporter(metricsConfig))
124-
.setInterval(metricsConfig.getExportOtelMetricsIntervalInSeconds(), TimeUnit.SECONDS)
125-
.build());
126-
}
115+
LOGGER.info(
116+
"OpenTelemetry initialization for {} started with config: {}",
117+
metricsConfig.getServiceName(),
118+
metricsConfig.toString());
119+
try {
120+
SdkMeterProviderBuilder builder = SdkMeterProvider.builder();
121+
122+
if (metricsConfig.exportOtelMetricsToEndpoint()) {
123+
MetricExporter httpExporter = getOtlpHttpMetricExporter(metricsConfig);
124+
builder.registerMetricReader(
125+
PeriodicMetricReader.builder(httpExporter)
126+
.setInterval(metricsConfig.getExportOtelMetricsIntervalInSeconds(), TimeUnit.SECONDS)
127+
.build());
128+
}
127129

128-
if (metricsConfig.getOtelAdditionalMetricsReader() != null) {
129-
// additional metrics reader apart from the above. For instance,
130-
// an in-memory metric reader can be passed in for testing purposes.
131-
builder.registerMetricReader(metricsConfig.getOtelAdditionalMetricsReader());
132-
}
130+
if (metricsConfig.exportOtelMetricsToLog()) {
131+
// internal to test: Disabled by default
132+
builder.registerMetricReader(
133+
PeriodicMetricReader.builder(new LogBasedMetricExporter(metricsConfig))
134+
.setInterval(metricsConfig.getExportOtelMetricsIntervalInSeconds(), TimeUnit.SECONDS)
135+
.build());
136+
}
133137

134-
if (metricsConfig.useOtelExponentialHistogram()) {
135-
setExponentialHistogramAggregation(builder, metricsConfig);
136-
}
138+
if (metricsConfig.getOtelAdditionalMetricsReader() != null) {
139+
// additional metrics reader apart from the above. For instance,
140+
// an in-memory metric reader can be passed in for testing purposes.
141+
builder.registerMetricReader(metricsConfig.getOtelAdditionalMetricsReader());
142+
}
137143

138-
// Set resource to empty to avoid adding any default resource attributes. The receiver
139-
// pipeline can choose to add the respective resource attributes if needed.
140-
builder.setResource(Resource.empty());
141-
142-
sdkMeterProvider = builder.build();
143-
144-
// Register MeterProvider with the OpenTelemetry instance
145-
openTelemetry = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
146-
LOGGER.info(
147-
"OpenTelemetry initialization for {} completed with config: {}",
148-
metricsConfig.getServiceName(),
149-
metricsConfig);
150-
} catch (Exception e) {
151-
String err = "OpenTelemetry initialization for " + metricsConfig.getServiceName() + " failed with config: "
152-
+ metricsConfig;
153-
LOGGER.error(err, e);
154-
throw new VeniceException(err, e);
144+
if (metricsConfig.useOtelExponentialHistogram()) {
145+
setExponentialHistogramAggregation(builder, metricsConfig);
155146
}
147+
148+
// Set resource to empty to avoid adding any default resource attributes. The receiver
149+
// pipeline can choose to add the respective resource attributes if needed.
150+
builder.setResource(Resource.empty());
151+
152+
sdkMeterProvider = builder.build();
153+
154+
// Register MeterProvider with the OpenTelemetry instance
155+
otel = OpenTelemetrySdk.builder().setMeterProvider(sdkMeterProvider).build();
156+
LOGGER.info(
157+
"OpenTelemetry initialization for {} completed with config: {}",
158+
metricsConfig.getServiceName(),
159+
metricsConfig);
160+
} catch (Exception e) {
161+
String err = "OpenTelemetry initialization for " + metricsConfig.getServiceName() + " failed with config: "
162+
+ metricsConfig;
163+
LOGGER.error(err, e);
164+
throw new VeniceException(err, e);
156165
}
157166

158-
this.meter = openTelemetry.getMeter(transformMetricName(getMetricPrefix(), metricFormat));
159-
this.recordFailureMetric = MetricEntityStateBase.create(
160-
CommonMetricsEntity.METRIC_RECORD_FAILURE.getMetricEntity(),
161-
this,
162-
Collections.EMPTY_MAP,
163-
Attributes.empty());
167+
return otel;
164168
}
165169

166170
/**

‎internal/venice-client-common/src/test/java/com/linkedin/venice/stats/VeniceOpenTelemetryMetricsRepositoryTest.java‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -411,16 +411,17 @@ public void testOpenTelemetryCreation() {
411411
otelMetricsRepositoryCase2.getOpenTelemetry().getMeterProvider(),
412412
otelMetricsRepositoryCase1.getOpenTelemetry().getMeterProvider());
413413

414-
// case 3: Global is not set and useOpenTelemetryInitializedByApplication is true: exception should be thrown
414+
// case 3: Global is not set and useOpenTelemetryInitializedByApplication is true: fall back to local
415+
// initialization
415416
GlobalOpenTelemetry.resetForTest();
416-
try {
417-
new VeniceOpenTelemetryMetricsRepository(mockMetricsConfig);
418-
fail();
419-
} catch (VeniceException e) {
420-
assertTrue(
421-
e.getMessage().contains("OpenTelemetry is not initialized globally by the application"),
422-
e.getMessage());
423-
}
417+
VeniceOpenTelemetryMetricsRepository otelMetricsRepositoryCase3 =
418+
new VeniceOpenTelemetryMetricsRepository(mockMetricsConfig);
419+
assertNotNull(
420+
otelMetricsRepositoryCase3.getSdkMeterProvider(),
421+
"SdkMeterProvider should not be null when falling back to local initialization");
422+
assertNotNull(
423+
otelMetricsRepositoryCase3.getOpenTelemetry(),
424+
"OpenTelemetry should not be null when falling back to local initialization");
424425

425426
// reset
426427
when(mockMetricsConfig.useOpenTelemetryInitializedByApplication()).thenReturn(false);

0 commit comments

Comments
 (0)