Skip to content

Commit fc75656

Browse files
mergify[bot]carsonip
mergify[bot]
and
carsonip
authored
Update otel samples to send otel native data (#5826) (#5836)
(cherry picked from commit 100f433) Co-authored-by: Carson Ip <[email protected]>
1 parent f30fdc4 commit fc75656

File tree

6 files changed

+587
-40
lines changed

6 files changed

+587
-40
lines changed

‎internal/pkg/otel/samples/darwin/logs_metrics_traces.yml‎

Lines changed: 276 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,239 @@ extensions:
4444
file_storage:
4545
directory: ${env:STORAGE_DIR}
4646

47+
connectors:
48+
signaltometrics:
49+
logs:
50+
- name: service_summary
51+
include_resource_attributes:
52+
- key: service.name
53+
- key: deployment.environment # service.environment
54+
- key: telemetry.sdk.language # service.language.name
55+
- key: agent.name # set via elastictraceprocessor
56+
attributes:
57+
- key: metricset.name
58+
default_value: service_summary
59+
sum:
60+
value: "1"
61+
datapoints:
62+
- name: service_summary
63+
include_resource_attributes:
64+
- key: service.name
65+
- key: deployment.environment # service.environment
66+
- key: telemetry.sdk.language # service.language.name
67+
- key: agent.name # set via elastictraceprocessor
68+
attributes:
69+
- key: metricset.name
70+
default_value: service_summary
71+
sum:
72+
value: "1"
73+
spans:
74+
- name: service_summary
75+
include_resource_attributes:
76+
- key: service.name
77+
- key: deployment.environment # service.environment
78+
- key: telemetry.sdk.language # service.language.name
79+
- key: agent.name # set via elastictraceprocessor
80+
attributes:
81+
- key: metricset.name
82+
default_value: service_summary
83+
sum:
84+
value: Int(AdjustedCount())
85+
- name: transaction.duration.histogram
86+
description: APM service transaction aggregated metrics as histogram
87+
include_resource_attributes:
88+
- key: service.name
89+
- key: deployment.environment # service.environment
90+
- key: telemetry.sdk.language # service.language.name
91+
- key: agent.name # set via elastictraceprocessor
92+
attributes:
93+
- key: transaction.root
94+
- key: transaction.type
95+
- key: metricset.name
96+
default_value: service_transaction
97+
- key: elasticsearch.mapping.hints
98+
default_value: [_doc_count]
99+
unit: us
100+
exponential_histogram:
101+
value: Microseconds(end_time - start_time)
102+
- name: transaction.duration.summary
103+
description: APM service transaction aggregated metrics as summary
104+
include_resource_attributes:
105+
- key: service.name
106+
- key: deployment.environment # service.environment
107+
- key: telemetry.sdk.language # service.language.name
108+
- key: agent.name # set via elastictraceprocessor
109+
attributes:
110+
- key: transaction.root
111+
- key: transaction.type
112+
- key: metricset.name
113+
default_value: service_transaction
114+
- key: elasticsearch.mapping.hints
115+
default_value: [aggregate_metric_double]
116+
unit: us
117+
histogram:
118+
buckets: [1]
119+
value: Microseconds(end_time - start_time)
120+
- name: transaction.duration.histogram
121+
description: APM transaction aggregated metrics as histogram
122+
ephemeral_resource_attribute: true
123+
include_resource_attributes:
124+
- key: service.name
125+
- key: deployment.environment # service.environment
126+
- key: telemetry.sdk.language # service.language.name
127+
- key: agent.name # set via elastictraceprocessor
128+
- key: container.id
129+
- key: k8s.pod.name
130+
- key: service.version
131+
- key: service.instance.id # service.node.name
132+
- key: process.runtime.name # service.runtime.name
133+
- key: process.runtime.version # service.runtime.version
134+
- key: telemetry.sdk.version # service.language.version??
135+
- key: host.name
136+
- key: os.type # host.os.platform
137+
- key: faas.instance
138+
- key: faas.name
139+
- key: faas.version
140+
- key: cloud.provider
141+
- key: cloud.region
142+
- key: cloud.availability_zone
143+
- key: cloud.platform # cloud.servicename
144+
- key: cloud.account.id
145+
attributes:
146+
- key: transaction.root
147+
- key: transaction.name
148+
- key: transaction.type
149+
- key: transaction.result
150+
- key: event.outcome
151+
- key: metricset.name
152+
default_value: transaction
153+
- key: elasticsearch.mapping.hints
154+
default_value: [_doc_count]
155+
unit: us
156+
exponential_histogram:
157+
value: Microseconds(end_time - start_time)
158+
- name: transaction.duration.summary
159+
description: APM transaction aggregated metrics as summary
160+
ephemeral_resource_attribute: true
161+
include_resource_attributes:
162+
- key: service.name
163+
- key: deployment.environment # service.environment
164+
- key: telemetry.sdk.language # service.language.name
165+
- key: agent.name # set via elastictraceprocessor
166+
- key: container.id
167+
- key: k8s.pod.name
168+
- key: service.version
169+
- key: service.instance.id # service.node.name
170+
- key: process.runtime.name # service.runtime.name
171+
- key: process.runtime.version # service.runtime.version
172+
- key: telemetry.sdk.version # service.language.version??
173+
- key: host.name
174+
- key: os.type # host.os.platform
175+
- key: faas.instance
176+
- key: faas.name
177+
- key: faas.version
178+
- key: cloud.provider
179+
- key: cloud.region
180+
- key: cloud.availability_zone
181+
- key: cloud.platform # cloud.servicename
182+
- key: cloud.account.id
183+
attributes:
184+
- key: transaction.root
185+
- key: transaction.name
186+
- key: transaction.type
187+
- key: transaction.result
188+
- key: event.outcome
189+
- key: metricset.name
190+
default_value: transaction
191+
- key: elasticsearch.mapping.hints
192+
default_value: [aggregate_metric_double]
193+
unit: us
194+
histogram:
195+
buckets: [1]
196+
value: Microseconds(end_time - start_time)
197+
- name: span.destination.service.response_time.sum.us
198+
description: APM span destination metrics
199+
ephemeral_resource_attribute: true
200+
include_resource_attributes:
201+
- key: service.name
202+
- key: deployment.environment # service.environment
203+
- key: telemetry.sdk.language # service.language.name
204+
- key: agent.name # set via elastictraceprocessor
205+
attributes:
206+
- key: span.name
207+
- key: event.outcome
208+
- key: service.target.type
209+
- key: service.target.name
210+
- key: span.destination.service.resource
211+
- key: metricset.name
212+
default_value: service_destination
213+
unit: us
214+
sum:
215+
value: Double(Microseconds(end_time - start_time))
216+
- name: span.destination.service.response_time.count
217+
description: APM span destination metrics
218+
ephemeral_resource_attribute: true
219+
include_resource_attributes:
220+
- key: service.name
221+
- key: deployment.environment # service.environment
222+
- key: telemetry.sdk.language # service.language.name
223+
- key: agent.name # set via elastictraceprocessor
224+
attributes:
225+
- key: span.name
226+
- key: event.outcome
227+
- key: service.target.type
228+
- key: service.target.name
229+
- key: span.destination.service.resource
230+
- key: metricset.name
231+
default_value: service_destination
232+
sum:
233+
value: Int(AdjustedCount())
234+
# event.success_count is populated using 2 metric definition with different conditions
235+
# and value for the histogram bucket based on event outcome. Both metric definition
236+
# are created using same name and attribute and will result in a single histogram.
237+
# We use mapping hint of aggregate_metric_double, so, only the sum and the count
238+
# values are required and the actual histogram bucket is ignored.
239+
- name: event.success_count
240+
description: Success count as a metric for service transaction
241+
include_resource_attributes:
242+
- key: service.name
243+
- key: deployment.environment # service.environment
244+
- key: telemetry.sdk.language # service.language.name
245+
- key: agent.name # set via elastictraceprocessor
246+
attributes:
247+
- key: transaction.root
248+
- key: transaction.type
249+
- key: metricset.name
250+
default_value: service_transaction
251+
- key: elasticsearch.mapping.hints
252+
default_value: [aggregate_metric_double]
253+
conditions:
254+
- attributes["event.outcome"] != nil and attributes["event.outcome"] == "success"
255+
histogram:
256+
buckets: [1]
257+
count: Int(AdjustedCount())
258+
value: Int(AdjustedCount())
259+
- name: event.success_count
260+
description: Success count as a metric for service transaction
261+
include_resource_attributes:
262+
- key: service.name
263+
- key: deployment.environment # service.environment
264+
- key: telemetry.sdk.language # service.language.name
265+
- key: agent.name # set via elastictraceprocessor
266+
attributes:
267+
- key: transaction.root
268+
- key: transaction.type
269+
- key: metricset.name
270+
default_value: service_transaction
271+
- key: elasticsearch.mapping.hints
272+
default_value: [aggregate_metric_double]
273+
conditions:
274+
- attributes["event.outcome"] != nil and attributes["event.outcome"] != "success"
275+
histogram:
276+
buckets: [0]
277+
count: Int(AdjustedCount())
278+
value: Double(0)
279+
47280
processors:
48281
elasticinframetrics:
49282
resourcedetection:
@@ -89,14 +322,27 @@ processors:
89322
- key: process.executable.path
90323
action: delete
91324

92-
exporters:
93-
otlp/apm:
94-
endpoint: "${env:APM_ENDPOINT}"
95-
headers:
96-
# Elastic APM Server secret token or API key
97-
Authorization: "Bearer ${env:APM_SECRET_KEY}"
325+
elastictrace:
326+
lsminterval:
327+
intervals:
328+
- duration: 1m
329+
statements:
330+
- set(resource.attributes["metricset.interval"], "1m")
331+
- set(attributes["data_stream.dataset"], Concat([attributes["metricset.name"], "1m"], "."))
332+
- set(attributes["processor.event"], "metric")
333+
- duration: 10m
334+
statements:
335+
- set(resource.attributes["metricset.interval"], "10m")
336+
- set(attributes["data_stream.dataset"], Concat([attributes["metricset.name"], "10m"], "."))
337+
- set(attributes["processor.event"], "metric")
338+
- duration: 60m
339+
statements:
340+
- set(resource.attributes["metricset.interval"], "60m")
341+
- set(attributes["data_stream.dataset"], Concat([attributes["metricset.name"], "60m"], "."))
342+
- set(attributes["processor.event"], "metric")
98343

99-
elasticsearch:
344+
exporters:
345+
elasticsearch/ecs:
100346
endpoints: ["${env:ELASTIC_ENDPOINT}"]
101347
api_key: ${env:ELASTIC_API_KEY}
102348
mapping:
@@ -108,30 +354,47 @@ exporters:
108354
traces_dynamic_index:
109355
enabled: true
110356

357+
elasticsearch/otel:
358+
endpoints: [ "${env:ELASTIC_ENDPOINT}" ]
359+
api_key: ${env:ELASTIC_API_KEY}
360+
mapping:
361+
mode: otel
362+
logs_dynamic_index:
363+
enabled: true
364+
metrics_dynamic_index:
365+
enabled: true
366+
traces_dynamic_index:
367+
enabled: true
368+
111369
service:
112370
extensions: [file_storage]
113371
pipelines:
114372
traces/fromsdk:
115373
receivers: [otlp/fromsdk]
116-
processors: []
117-
exporters: [otlp/apm]
374+
processors: [elastictrace]
375+
exporters: [signaltometrics, elasticsearch/otel]
118376

119377
metrics/fromsdk:
120378
receivers: [otlp/fromsdk]
121379
processors: []
122-
exporters: [otlp/apm]
380+
exporters: [signaltometrics, elasticsearch/otel]
123381

124382
metrics/hostmetrics:
125383
receivers: [hostmetrics/system]
126384
processors: [elasticinframetrics, resourcedetection, attributes/dataset, resource/process]
127-
exporters: [elasticsearch]
385+
exporters: [elasticsearch/ecs]
386+
387+
metrics/aggregated-metrics:
388+
receivers: [signaltometrics]
389+
processors: [lsminterval]
390+
exporters: [elasticsearch/otel]
128391

129392
logs/fromsdk:
130393
receivers: [otlp/fromsdk]
131394
processors: []
132-
exporters: [otlp/apm]
395+
exporters: [signaltometrics, elasticsearch/otel]
133396

134397
logs/platformlogs:
135398
receivers: [filelog/platformlogs]
136399
processors: [resourcedetection]
137-
exporters: [elasticsearch]
400+
exporters: [elasticsearch/otel]

‎internal/pkg/otel/samples/darwin/platformlogs.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ receivers:
1111
extensions:
1212
file_storage:
1313
directory: ${env:STORAGE_DIR}
14-
14+
1515
processors:
1616
resourcedetection:
1717
detectors: ["system"]
@@ -53,11 +53,11 @@ exporters:
5353
sampling_thereafter: 1000
5454

5555
# Exporter to send logs and metrics to Elasticsearch
56-
elasticsearch:
56+
elasticsearch/otel:
5757
endpoints: ["${env:ELASTIC_ENDPOINT}"]
5858
api_key: ${env:ELASTIC_API_KEY}
5959
mapping:
60-
mode: ecs
60+
mode: otel
6161
logs_dynamic_index:
6262
enabled: true
6363
metrics_dynamic_index:
@@ -71,4 +71,4 @@ service:
7171
logs/platformlogs:
7272
receivers: [filelog/platformlogs]
7373
processors: [resourcedetection]
74-
exporters: [debug, elasticsearch]
74+
exporters: [debug, elasticsearch/otel]

‎internal/pkg/otel/samples/darwin/platformlogs_hostmetrics.yml‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,7 @@ processors:
8585

8686
exporters:
8787
# Exporter to send logs and metrics to Elasticsearch
88-
elasticsearch:
88+
elasticsearch/ecs:
8989
endpoints: ["${env:ELASTIC_ENDPOINT}"]
9090
api_key: ${env:ELASTIC_API_KEY}
9191
mapping:
@@ -96,15 +96,26 @@ exporters:
9696
enabled: true
9797
traces_dynamic_index:
9898
enabled: true
99+
elasticsearch/otel:
100+
endpoints: [ "${env:ELASTIC_ENDPOINT}" ]
101+
api_key: ${env:ELASTIC_API_KEY}
102+
mapping:
103+
mode: otel
104+
logs_dynamic_index:
105+
enabled: true
106+
metrics_dynamic_index:
107+
enabled: true
108+
traces_dynamic_index:
109+
enabled: true
99110

100111
service:
101112
extensions: [file_storage]
102113
pipelines:
103114
metrics/hostmetrics:
104115
receivers: [hostmetrics/system]
105116
processors: [elasticinframetrics, resourcedetection, attributes/dataset, resource/process]
106-
exporters: [elasticsearch]
117+
exporters: [elasticsearch/ecs]
107118
logs/platformlogs:
108119
receivers: [filelog/platformlogs]
109120
processors: [resourcedetection]
110-
exporters: [elasticsearch]
121+
exporters: [elasticsearch/otel]

0 commit comments

Comments
 (0)