Skip to content

Commit 269f55a

Browse files
crobert-1Copilotjinja2
authored
[agent] Update Istio autodetect rules (#2132)
* [agent] Update Istio autodetect rules * Update .chloggen/updateistiocreatorrules.yaml * Update .chloggen/updateistiocreatorrules.yaml Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * debug: write updated metrics and add them to CI archive * update expected testdata for istio ingress * remove debugging flag that updates expected test data * refactor test checking for expected metrics * Fix lint, add more time for test and log comparison failures * fix lint * revert expected test data change * remove debugging * make changelog note more specific * Update .chloggen/updateistiocreatorrules.yaml * reorganize updating expected test data * update changelog message * Add comment to public test helper SelectMetricSet * Update .chloggen/updateistiocreatorrules.yaml Co-authored-by: Jina Jain <jjain@splunk.com> * Update functional_tests/internal/common.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Jina Jain <jjain@splunk.com>
1 parent 6ff7f27 commit 269f55a

File tree

6 files changed

+97
-66
lines changed

6 files changed

+97
-66
lines changed

‎.chloggen/updateistiocreatorrules.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
2+
change_type: enhancement
3+
# The name of the component, or a single word describing the area of concern, (e.g. agent, clusterReceiver, gateway, operator, chart, other)
4+
component: agent
5+
# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
6+
note: Update Istio autodetect rules. The detection logic has been expanded to identify Istio components more reliably.
7+
# One or more tracking issues related to the change
8+
issues: [2132]
9+
# (Optional) One or more lines of additional information to render under the primary note.
10+
# These lines will be padded with 2 spaces and then inserted directly into the document.
11+
# Use pipe (|) for multiline entries.
12+
subtext: |
13+
This change may impact billing, as the detection rules have been updated to be less restrictive.
14+
Metrics will now be scraped from istiod, Istio gateways, and Istio sidecars (`istio-proxy` containers.)
15+
Metrics were previously only being scraped from istiod.

‎examples/autodetect-istio/rendered_manifests/configmap-agent.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -483,8 +483,9 @@ data:
483483
: 9090`'
484484
metrics_path: '`"prometheus.io/path" in annotations ? annotations["prometheus.io/path"]
485485
: "/metrics"`'
486-
rule: type == "pod" && annotations["prometheus.io/scrape"] == "true" && "istio.io/rev"
487-
in labels
486+
rule: type == "pod" && annotations["prometheus.io/scrape"] == "true" && ("istio.io/rev"
487+
in labels or "istio.io/rev" in annotations or labels["istio"] == "pilot"
488+
or name matches "istio.*")
488489
watch_observers:
489490
- k8s_observer
490491
zipkin:

‎examples/autodetect-istio/rendered_manifests/daemonset.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ spec:
3232
release: default
3333
sidecar.istio.io/inject: "false"
3434
annotations:
35-
checksum/config: 98128a3325d0df6fc05585cc520b457c60cbda496989ffa510ddf1ad79e024c1
35+
checksum/config: 493884c8e6fa7aed5bdd9aa7759d25ebde1e8c9713f45fa7acfacc019f62fad4
3636
kubectl.kubernetes.io/default-container: otel-collector
3737
spec:
3838
hostNetwork: true

‎functional_tests/internal/common.go‎

Lines changed: 34 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -378,40 +378,48 @@ func DeleteObject(t *testing.T, k8sClient *k8stest.K8sClient, objYAML string) {
378378
}
379379
}
380380

381-
// SelectMetricSetWithTimeout finds a metrics payload containing a target metric with Eventually timeout
382-
func SelectMetricSetWithTimeout(t *testing.T, expected pmetric.Metrics, targetMetric string, metricSink *consumertest.MetricsSink, ignoreLen bool, timeout time.Duration, interval time.Duration) *pmetric.Metrics {
381+
// SelectMetricSet finds a metrics payload containing a target metric without any re-checking logic.
382+
func SelectMetricSet(t *testing.T, expected pmetric.Metrics, targetMetric string, metricSink *consumertest.MetricsSink, ignoreLen bool) *pmetric.Metrics {
383383
var selectedMetrics *pmetric.Metrics
384384

385-
require.Eventuallyf(t, func() bool {
386-
for h := len(metricSink.AllMetrics()) - 1; h >= 0; h-- {
387-
m := metricSink.AllMetrics()[h]
388-
foundTargetMetric := false
389-
390-
OUTER:
391-
for i := 0; i < m.ResourceMetrics().Len(); i++ {
392-
for j := 0; j < m.ResourceMetrics().At(i).ScopeMetrics().Len(); j++ {
393-
for k := 0; k < m.ResourceMetrics().At(i).ScopeMetrics().At(j).Metrics().Len(); k++ {
394-
metric := m.ResourceMetrics().At(i).ScopeMetrics().At(j).Metrics().At(k)
395-
if metric.Name() == targetMetric {
396-
foundTargetMetric = true
397-
break OUTER
398-
}
385+
for h := len(metricSink.AllMetrics()) - 1; h >= 0; h-- {
386+
m := metricSink.AllMetrics()[h]
387+
foundTargetMetric := false
388+
389+
OUTER:
390+
for i := 0; i < m.ResourceMetrics().Len(); i++ {
391+
for j := 0; j < m.ResourceMetrics().At(i).ScopeMetrics().Len(); j++ {
392+
for k := 0; k < m.ResourceMetrics().At(i).ScopeMetrics().At(j).Metrics().Len(); k++ {
393+
metric := m.ResourceMetrics().At(i).ScopeMetrics().At(j).Metrics().At(k)
394+
if metric.Name() == targetMetric {
395+
foundTargetMetric = true
396+
break OUTER
399397
}
400398
}
401399
}
400+
}
402401

403-
if !foundTargetMetric {
404-
continue
405-
}
402+
if !foundTargetMetric {
403+
continue
404+
}
406405

407-
if ignoreLen || (m.ResourceMetrics().Len() == expected.ResourceMetrics().Len() && m.MetricCount() == expected.MetricCount()) {
408-
selectedMetrics = &m
409-
t.Logf("Found target metric '%s' in payload with %d total metrics", targetMetric, m.MetricCount())
410-
return true
411-
}
406+
if ignoreLen || (m.ResourceMetrics().Len() == expected.ResourceMetrics().Len() && m.MetricCount() == expected.MetricCount()) {
407+
selectedMetrics = &m
408+
t.Logf("Found target metric '%s' in payload with %d total metrics", targetMetric, m.MetricCount())
409+
break
412410
}
413-
t.Logf("Waiting for target metric '%s'...", targetMetric)
414-
return false
411+
}
412+
413+
return selectedMetrics
414+
}
415+
416+
// SelectMetricSetWithTimeout finds a metrics payload containing a target metric with Eventually timeout
417+
func SelectMetricSetWithTimeout(t *testing.T, expected pmetric.Metrics, targetMetric string, metricSink *consumertest.MetricsSink, ignoreLen bool, timeout time.Duration, interval time.Duration) *pmetric.Metrics {
418+
var selectedMetrics *pmetric.Metrics
419+
420+
require.Eventuallyf(t, func() bool {
421+
selectedMetrics = SelectMetricSet(t, expected, targetMetric, metricSink, ignoreLen)
422+
return selectedMetrics != nil
415423
}, timeout, interval, "Failed to find target metric %s within timeout period of %v", targetMetric, timeout)
416424

417425
return selectedMetrics

‎functional_tests/istio/istio_test.go‎

Lines changed: 43 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -313,12 +313,14 @@ func Test_IstioMetrics(t *testing.T) {
313313

314314
flakyMetrics := []string{"galley_validation_config_update_error"} // only shows up when config validation fails - removed if present when comparing
315315
t.Run("istiod metrics captured", func(t *testing.T) {
316-
testIstioMetrics(t, "testdata/expected_istiod.yaml", "pilot_xds_pushes", flakyMetrics, true, metricsSink)
316+
testIstioMetrics(t, "testdata/expected_istiod.yaml", "pilot_xds_pushes",
317+
flakyMetrics, true, metricsSink)
317318
})
318319

319320
flakyMetrics = []string{"istio_agent_pilot_xds_expired_nonce"}
320321
t.Run("istio ingress metrics captured", func(t *testing.T) {
321-
testIstioMetrics(t, "testdata/expected_istioingress.yaml", "istio_requests_total", flakyMetrics, true, metricsSink)
322+
testIstioMetrics(t, "testdata/expected_istioingress.yaml",
323+
"istio_requests_total", flakyMetrics, true, metricsSink)
322324
})
323325
}
324326

@@ -328,17 +330,6 @@ func testIstioMetrics(t *testing.T, expectedMetricsFile string, includeMetricNam
328330

329331
internal.WaitForMetrics(t, 2, metricsSink)
330332

331-
selectedMetrics := internal.SelectMetricSetWithTimeout(t, expectedMetrics, includeMetricName, metricsSink, ignoreLen, 5*time.Minute, 30*time.Second)
332-
if selectedMetrics == nil {
333-
t.Error("No metric batch identified with the right metric count, exiting")
334-
return
335-
}
336-
require.NotNil(t, selectedMetrics)
337-
338-
if flakyMetricNames != nil {
339-
internal.RemoveFlakyMetrics(selectedMetrics, flakyMetricNames)
340-
}
341-
342333
var metricNames []string
343334
for i := 0; i < expectedMetrics.ResourceMetrics().Len(); i++ {
344335
for j := 0; j < expectedMetrics.ResourceMetrics().At(i).ScopeMetrics().Len(); j++ {
@@ -349,29 +340,45 @@ func testIstioMetrics(t *testing.T, expectedMetricsFile string, includeMetricNam
349340
}
350341
}
351342

352-
internal.MaybeUpdateExpectedMetricsResults(t, expectedMetricsFile, selectedMetrics)
353-
err = pmetrictest.CompareMetrics(expectedMetrics, *selectedMetrics,
354-
pmetrictest.IgnoreTimestamp(),
355-
pmetrictest.IgnoreStartTimestamp(),
356-
pmetrictest.IgnoreScopeVersion(),
357-
pmetrictest.IgnoreMetricValues(metricNames...),
358-
pmetrictest.IgnoreMetricAttributeValue("host.name"),
359-
pmetrictest.IgnoreMetricAttributeValue("k8s.pod.name"),
360-
pmetrictest.IgnoreMetricAttributeValue("k8s.pod.uid"),
361-
pmetrictest.IgnoreMetricAttributeValue("os.type"),
362-
pmetrictest.IgnoreMetricAttributeValue("server.address"),
363-
pmetrictest.IgnoreMetricAttributeValue("service.instance.id"),
364-
pmetrictest.IgnoreMetricAttributeValue("service.name"),
365-
pmetrictest.IgnoreMetricAttributeValue("url.scheme"),
366-
pmetrictest.IgnoreMetricAttributeValue("type", "pilot_xds_expired_nonce"),
367-
pmetrictest.IgnoreResourceMetricsOrder(),
368-
pmetrictest.IgnoreMetricsOrder(),
369-
pmetrictest.IgnoreScopeMetricsOrder(),
370-
pmetrictest.IgnoreMetricDataPointsOrder(),
371-
pmetrictest.IgnoreMetricAttributeValue("event"),
372-
pmetrictest.IgnoreSubsequentDataPoints(metricNames...),
373-
)
374-
require.NoError(t, err)
343+
require.Eventually(t, func() bool {
344+
for _, receivedMetrics := range metricsSink.AllMetrics() {
345+
if flakyMetricNames != nil {
346+
internal.RemoveFlakyMetrics(&receivedMetrics, flakyMetricNames)
347+
}
348+
349+
err = pmetrictest.CompareMetrics(expectedMetrics, receivedMetrics,
350+
pmetrictest.IgnoreTimestamp(),
351+
pmetrictest.IgnoreStartTimestamp(),
352+
pmetrictest.IgnoreScopeVersion(),
353+
pmetrictest.IgnoreMetricValues(metricNames...),
354+
pmetrictest.IgnoreMetricAttributeValue("host.name"),
355+
pmetrictest.IgnoreMetricAttributeValue("k8s.pod.name"),
356+
pmetrictest.IgnoreMetricAttributeValue("k8s.pod.uid"),
357+
pmetrictest.IgnoreMetricAttributeValue("os.type"),
358+
pmetrictest.IgnoreMetricAttributeValue("server.address"),
359+
pmetrictest.IgnoreMetricAttributeValue("service.instance.id"),
360+
pmetrictest.IgnoreMetricAttributeValue("service.name"),
361+
pmetrictest.IgnoreMetricAttributeValue("url.scheme"),
362+
pmetrictest.IgnoreMetricAttributeValue("type", "pilot_xds_expired_nonce"),
363+
pmetrictest.IgnoreResourceMetricsOrder(),
364+
pmetrictest.IgnoreMetricsOrder(),
365+
pmetrictest.IgnoreScopeMetricsOrder(),
366+
pmetrictest.IgnoreMetricDataPointsOrder(),
367+
pmetrictest.IgnoreMetricAttributeValue("event"),
368+
pmetrictest.IgnoreSubsequentDataPoints(metricNames...),
369+
)
370+
if err == nil {
371+
return true
372+
}
373+
t.Logf("Comparison error: %v", err)
374+
}
375+
376+
selectedMetrics := internal.SelectMetricSet(t, expectedMetrics, includeMetricName, metricsSink, ignoreLen)
377+
if selectedMetrics != nil {
378+
internal.MaybeUpdateExpectedMetricsResults(t, expectedMetricsFile, selectedMetrics)
379+
}
380+
return false
381+
}, 5*time.Minute, 1*time.Second, "Expected metrics not found")
375382
}
376383

377384
func Test_IstioTraces(t *testing.T) {

‎helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ receivers:
9393
rule: type == "pod" && annotations["prometheus.io/scrape"] == "true"
9494
{{- else }}
9595
# Enable prometheus scraping for Istio pods only
96-
rule: type == "pod" && annotations["prometheus.io/scrape"] == "true" && "istio.io/rev" in labels
96+
rule: type == "pod" && annotations["prometheus.io/scrape"] == "true" && ("istio.io/rev" in labels or "istio.io/rev" in annotations or labels["istio"] == "pilot" or name matches "istio.*")
9797
{{- end }}
9898
config:
9999
{{- if .Values.featureGates.useLightPrometheusReceiver }}

0 commit comments

Comments
 (0)