Skip to content

Commit d1a44b2

Browse files
jinja2jinja2
authored
[chore] test to track res/log attrs for containers (#2336)
* [chore] test to track res/log attrs for containers * Apply suggestion from @jinja2 * update meta * review fix * add container.id
1 parent cd40b3a commit d1a44b2

File tree

5 files changed

+304
-93
lines changed

5 files changed

+304
-93
lines changed

‎functional_tests/functional/functional_test.go‎

Lines changed: 41 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -144,19 +144,21 @@ func deployPrometheusResources(t *testing.T, extensionsClient *clientset.Clients
144144
}, 3*time.Minute, 3*time.Second, "CRD %s stuck in Terminating", crd.Name)
145145
}
146146

147-
crd, err = apiExtensions.Create(t.Context(), crd, metav1.CreateOptions{})
148-
if err != nil {
149-
if k8serrors.IsAlreadyExists(err) {
150-
t.Logf("CRD %s already exists, skipping creation", crd.Name)
147+
crdName := crd.Name
148+
crdSpec := crd.Spec
149+
created, createErr := apiExtensions.Create(t.Context(), crd, metav1.CreateOptions{})
150+
if createErr != nil {
151+
if k8serrors.IsAlreadyExists(createErr) {
152+
t.Logf("CRD %s already exists, skipping creation", crdName)
151153
} else {
152-
require.NoError(t, err)
154+
require.NoError(t, createErr)
153155
}
154156
} else {
155-
t.Logf("Deployed CRD %s", crd.Name)
157+
t.Logf("Deployed CRD %s", created.Name)
156158
}
157159

158160
require.EventuallyWithT(t, func(tt *assert.CollectT) {
159-
latest, latestErr := apiExtensions.Get(t.Context(), crd.Name, metav1.GetOptions{})
161+
latest, latestErr := apiExtensions.Get(t.Context(), crdName, metav1.GetOptions{})
160162
assert.NoError(tt, latestErr)
161163
established := false
162164
for _, cond := range latest.Status.Conditions {
@@ -165,14 +167,14 @@ func deployPrometheusResources(t *testing.T, extensionsClient *clientset.Clients
165167
}
166168
}
167169
assert.True(tt, established)
168-
}, 3*time.Minute, 3*time.Second, "CRD %s not established", crd.Name)
170+
}, 3*time.Minute, 3*time.Second, "CRD %s not established", crdName)
169171

170-
for _, version := range crd.Spec.Versions {
172+
for _, version := range crdSpec.Versions {
171173
sch.AddKnownTypeWithName(
172174
schema.GroupVersionKind{
173-
Group: crd.Spec.Group,
175+
Group: crdSpec.Group,
174176
Version: version.Name,
175-
Kind: crd.Spec.Names.Kind,
177+
Kind: crdSpec.Names.Kind,
176178
},
177179
&unstructured.Unstructured{},
178180
)
@@ -332,70 +334,28 @@ func deployChartsAndApps(t *testing.T, testKubeConfig string) {
332334

333335
deployments := client.AppsV1().Deployments(internal.DefaultNamespace)
334336

335-
// NodeJS test app
336-
stream, err = os.ReadFile(filepath.Join(testDir, "nodejs", "deployment.yaml"))
337-
require.NoError(t, err)
338-
deployment, _, err := decode(stream, nil, nil)
339-
require.NoError(t, err)
340-
_, err = deployments.Create(t.Context(), deployment.(*appsv1.Deployment), metav1.CreateOptions{})
341-
if err != nil {
342-
_, err2 := deployments.Update(t.Context(), deployment.(*appsv1.Deployment), metav1.UpdateOptions{})
343-
assert.NoError(t, err2)
344-
if err2 != nil {
345-
require.NoError(t, err)
346-
}
347-
}
348-
// Java test app
349-
stream, err = os.ReadFile(filepath.Join(testDir, "java", "deployment.yaml"))
350-
require.NoError(t, err)
351-
deployment, _, err = decode(stream, nil, nil)
352-
require.NoError(t, err)
353-
_, err = deployments.Create(t.Context(), deployment.(*appsv1.Deployment), metav1.CreateOptions{})
354-
if err != nil {
355-
_, err2 := deployments.Update(t.Context(), deployment.(*appsv1.Deployment), metav1.UpdateOptions{})
356-
assert.NoError(t, err2)
357-
if err2 != nil {
358-
require.NoError(t, err)
359-
}
360-
}
361-
// .NET test app
362-
stream, err = os.ReadFile(filepath.Join(testDir, "dotnet", "deployment.yaml"))
363-
require.NoError(t, err)
364-
deployment, _, err = decode(stream, nil, nil)
365-
require.NoError(t, err)
366-
_, err = deployments.Create(t.Context(), deployment.(*appsv1.Deployment), metav1.CreateOptions{})
367-
if err != nil {
368-
_, err2 := deployments.Update(t.Context(), deployment.(*appsv1.Deployment), metav1.UpdateOptions{})
369-
assert.NoError(t, err2)
370-
if err2 != nil {
371-
require.NoError(t, err)
372-
}
373-
}
374-
// Python test app
375-
stream, err = os.ReadFile(filepath.Join(testDir, "python", "deployment.yaml"))
376-
require.NoError(t, err)
377-
deployment, _, err = decode(stream, nil, nil)
378-
require.NoError(t, err)
379-
_, err = deployments.Create(t.Context(), deployment.(*appsv1.Deployment), metav1.CreateOptions{})
380-
if err != nil {
381-
_, err2 := deployments.Update(t.Context(), deployment.(*appsv1.Deployment), metav1.UpdateOptions{})
382-
assert.NoError(t, err2)
383-
if err2 != nil {
384-
require.NoError(t, err)
337+
deployApp := func(filePath string) {
338+
data, readErr := os.ReadFile(filePath)
339+
require.NoError(t, readErr)
340+
dep, _, decodeErr := decode(data, nil, nil)
341+
require.NoError(t, decodeErr)
342+
_, createErr := deployments.Create(t.Context(), dep.(*appsv1.Deployment), metav1.CreateOptions{})
343+
if k8serrors.IsAlreadyExists(createErr) {
344+
_, updateErr := deployments.Update(t.Context(), dep.(*appsv1.Deployment), metav1.UpdateOptions{})
345+
require.NoError(t, updateErr)
346+
} else {
347+
require.NoError(t, createErr)
385348
}
386349
}
387-
// Prometheus annotation
388-
stream, err = os.ReadFile(filepath.Join(testDir, manifestsDir, "deployment_with_prometheus_annotations.yaml"))
389-
require.NoError(t, err)
390-
deployment, _, err = decode(stream, nil, nil)
391-
require.NoError(t, err)
392-
_, err = deployments.Create(t.Context(), deployment.(*appsv1.Deployment), metav1.CreateOptions{})
393-
if err != nil {
394-
_, err2 := deployments.Update(t.Context(), deployment.(*appsv1.Deployment), metav1.UpdateOptions{})
395-
assert.NoError(t, err2)
396-
if err2 != nil {
397-
require.NoError(t, err)
398-
}
350+
for _, f := range []string{
351+
filepath.Join(testDir, "nodejs", "deployment.yaml"),
352+
filepath.Join(testDir, "java", "deployment.yaml"),
353+
filepath.Join(testDir, "dotnet", "deployment.yaml"),
354+
filepath.Join(testDir, "python", "deployment.yaml"),
355+
filepath.Join(testDir, manifestsDir, "log_attr_test_deployment.yaml"),
356+
filepath.Join(testDir, manifestsDir, "deployment_with_prometheus_annotations.yaml"),
357+
} {
358+
deployApp(f)
399359
}
400360

401361
// Service
@@ -498,6 +458,9 @@ func teardown(ctx context.Context, t *testing.T, testKubeConfig string) {
498458
_ = deployments.Delete(ctx, "prometheus-annotation-test", metav1.DeleteOptions{
499459
GracePeriodSeconds: &waitTime,
500460
})
461+
_ = deployments.Delete(ctx, "log-attr-test", metav1.DeleteOptions{
462+
GracePeriodSeconds: &waitTime,
463+
})
501464
_ = client.CoreV1().Services(internal.DefaultNamespace).Delete(ctx, "prometheus-annotation-service",
502465
metav1.DeleteOptions{
503466
GracePeriodSeconds: &waitTime,
@@ -608,6 +571,9 @@ func runLocalClusterTests(t *testing.T) {
608571
t.Run("Python profiling captured", testPythonProfiling)
609572
t.Run("kubernetes cluster metrics", testK8sClusterReceiverMetrics)
610573
t.Run("agent logs", testAgentLogs)
574+
t.Run("container log attributes validation", func(t *testing.T) {
575+
validateLogAttributes(t, globalSinks.logsConsumer)
576+
})
611577
t.Run("test HEC metrics", testHECMetrics)
612578
t.Run("test k8s objects", testK8sObjects)
613579
t.Run("test agent metrics", testAgentMetrics)
@@ -713,7 +679,7 @@ func validateResourceAttributes(t *testing.T, clientset *kubernetes.Clientset, k
713679
actualResourceAttributes := readAndNormalizeMetrics(t, tmpFile.Name(), "k8s.cluster.name").ResourceMetrics().At(0).Resource().Attributes()
714680
expectedResourceAttributes := readAndNormalizeMetrics(t, expectedResourceAttributesFile, "k8s.cluster.name").ResourceMetrics().At(0).Resource().Attributes()
715681

716-
require.True(t, expectedResourceAttributes.Equal(actualResourceAttributes), "Resource Attributes comparison failed for %s , expected values %s , actual values %s", collectorType, formatResourceAttributesString(expectedResourceAttributes), formatResourceAttributesString(actualResourceAttributes))
682+
require.True(t, expectedResourceAttributes.Equal(actualResourceAttributes), "Resource Attributes comparison failed for %s , expected values %s , actual values %s", collectorType, internal.FormatAttributes(expectedResourceAttributes), internal.FormatAttributes(actualResourceAttributes))
717683

718684
t.Cleanup(func() {
719685
require.NoError(t, os.Remove(tmpFile.Name()))
@@ -723,16 +689,7 @@ func validateResourceAttributes(t *testing.T, clientset *kubernetes.Clientset, k
723689
func readAndNormalizeMetrics(t *testing.T, filePath string, skipKeys ...string) pmetric.Metrics {
724690
metrics, err := golden.ReadMetrics(filePath)
725691
require.NoError(t, err)
726-
attrs := metrics.ResourceMetrics().At(0).Resource().Attributes()
727-
attrs.Range(func(k string, _ pcommon.Value) bool {
728-
for _, skipKey := range skipKeys {
729-
if k == skipKey {
730-
return true
731-
}
732-
}
733-
attrs.PutStr(k, "abcd")
734-
return true
735-
})
692+
internal.NormalizeAttributes(metrics.ResourceMetrics().At(0).Resource().Attributes(), skipKeys...)
736693
return metrics
737694
}
738695

@@ -1329,12 +1286,3 @@ func metricDataPointsHaveAttrs(metric pmetric.Metric, kvPairs ...string) bool {
13291286
}
13301287
return false
13311288
}
1332-
1333-
func formatResourceAttributesString(attributesMap pcommon.Map) string {
1334-
var attrsStr strings.Builder
1335-
attributesMap.Range(func(k string, v pcommon.Value) bool {
1336-
attrsStr.WriteString(k + "=" + v.Str() + ";")
1337-
return true
1338-
})
1339-
return attrsStr.String()
1340-
}

‎functional_tests/functional/log_attributes_test.go‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
// Copyright Splunk Inc.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package functional
5+
6+
import (
7+
"os"
8+
"path/filepath"
9+
"regexp"
10+
"strings"
11+
"testing"
12+
"time"
13+
14+
"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden"
15+
"github.com/stretchr/testify/assert"
16+
"github.com/stretchr/testify/require"
17+
"go.opentelemetry.io/collector/consumer/consumertest"
18+
"go.opentelemetry.io/collector/pdata/pcommon"
19+
"go.opentelemetry.io/collector/pdata/plog"
20+
21+
"github.com/signalfx/splunk-otel-collector-chart/functional_tests/internal"
22+
)
23+
24+
// dynamicAttrPlaceholders maps attribute keys whose values vary between runs
25+
// (node names, UIDs, etc.) to deterministic placeholders of similar size/shape
26+
// so golden-file comparisons remain stable.
27+
var dynamicAttrPlaceholders = map[string]string{
28+
"k8s.pod.uid": "f47ac10b-58cc-4372-a567-0e02b2c3d479",
29+
"k8s.pod.name": "log-attr-test-7bc5f4d89b-xj8k2",
30+
"k8s.node.name": "node-001",
31+
"host.name": "node-001",
32+
"container.id": "a1b2c3d4e5f6071890abcdef12345678a1b2c3d4e5f6071890abcdef12345678",
33+
}
34+
35+
// podUIDRe matches a UUID embedded in file paths (the pod UID segment).
36+
var podUIDRe = regexp.MustCompile(`[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}`)
37+
38+
// podNameRe matches a deployment-generated pod name (name-replicaset-suffix).
39+
var podNameRe = regexp.MustCompile(`log-attr-test-[0-9a-z]+-[0-9a-z]+`)
40+
41+
func validateLogAttributes(t *testing.T, logsConsumer *consumertest.LogsSink) {
42+
internal.WaitForLogs(t, 5, logsConsumer)
43+
44+
var found bool
45+
foundLog := plog.NewLogs()
46+
require.EventuallyWithT(t, func(tt *assert.CollectT) {
47+
for _, l := range logsConsumer.AllLogs() {
48+
for j := 0; j < l.ResourceLogs().Len(); j++ {
49+
rl := l.ResourceLogs().At(j)
50+
for k := 0; k < rl.ScopeLogs().Len(); k++ {
51+
sl := rl.ScopeLogs().At(k)
52+
for m := 0; m < sl.LogRecords().Len(); m++ {
53+
lr := sl.LogRecords().At(m)
54+
v, ok := lr.Attributes().Get("k8s.container.name")
55+
if !ok || v.AsString() != "log-attr-test" {
56+
continue
57+
}
58+
if _, hasContainerID := lr.Attributes().Get("container.id"); !hasContainerID {
59+
continue
60+
}
61+
if strings.Contains(lr.Body().AsString(), "LOG_ATTR_VALIDATION_MARKER") {
62+
foundLog = plog.NewLogs()
63+
newRL := foundLog.ResourceLogs().AppendEmpty()
64+
rl.Resource().CopyTo(newRL.Resource())
65+
newSL := newRL.ScopeLogs().AppendEmpty()
66+
lr.CopyTo(newSL.LogRecords().AppendEmpty())
67+
found = true
68+
}
69+
}
70+
}
71+
}
72+
}
73+
assert.True(tt, found, "log from log-attr-test container not found")
74+
}, 3*time.Minute, 5*time.Second)
75+
76+
normalizeLogData(&foundLog)
77+
78+
t.Logf("Normalized log resource attributes: %s", internal.FormatAttributes(foundLog.ResourceLogs().At(0).Resource().Attributes()))
79+
t.Logf("Normalized log record attributes: %s", internal.FormatAttributes(foundLog.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes()))
80+
81+
expectedFile := filepath.Join(testDir, expectedValuesDir, "expected_container_log_attributes.yaml")
82+
internal.MaybeUpdateExpectedLogsResults(t, expectedFile, &foundLog)
83+
84+
if _, statErr := os.Stat(expectedFile); os.IsNotExist(statErr) {
85+
t.Skipf("Expected log attributes file not found at %s; run with UPDATE_EXPECTED_RESULTS=true to generate", expectedFile)
86+
return
87+
}
88+
89+
expected, err := golden.ReadLogs(expectedFile)
90+
require.NoError(t, err)
91+
92+
actualResAttrs := foundLog.ResourceLogs().At(0).Resource().Attributes()
93+
expectedResAttrs := expected.ResourceLogs().At(0).Resource().Attributes()
94+
require.True(t, expectedResAttrs.Equal(actualResAttrs),
95+
"Log resource attributes mismatch.\nExpected: %s\nActual: %s",
96+
internal.FormatAttributes(expectedResAttrs), internal.FormatAttributes(actualResAttrs))
97+
98+
actualLogAttrs := foundLog.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes()
99+
expectedLogAttrs := expected.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes()
100+
require.True(t, expectedLogAttrs.Equal(actualLogAttrs),
101+
"Log record attributes mismatch.\nExpected: %s\nActual: %s",
102+
internal.FormatAttributes(expectedLogAttrs), internal.FormatAttributes(actualLogAttrs))
103+
}
104+
105+
// normalizeLogData replaces dynamic attribute values with deterministic
106+
// placeholders so golden-file comparisons are stable across runs. Static/
107+
// config-driven values (sourcetype, index, os.type, etc.) are kept as-is.
108+
func normalizeLogData(logs *plog.Logs) {
109+
for i := 0; i < logs.ResourceLogs().Len(); i++ {
110+
rl := logs.ResourceLogs().At(i)
111+
normalizeDynamicAttrs(rl.Resource().Attributes())
112+
for j := 0; j < rl.ScopeLogs().Len(); j++ {
113+
sl := rl.ScopeLogs().At(j)
114+
for k := 0; k < sl.LogRecords().Len(); k++ {
115+
lr := sl.LogRecords().At(k)
116+
lr.Body().SetStr("LOG_ATTR_VALIDATION_MARKER")
117+
lr.SetTimestamp(0)
118+
lr.SetObservedTimestamp(0)
119+
normalizeDynamicAttrs(lr.Attributes())
120+
}
121+
}
122+
}
123+
}
124+
125+
// normalizeDynamicAttrs replaces values of dynamic/per-run keys with fixed
126+
// placeholders.
127+
func normalizeDynamicAttrs(m pcommon.Map) {
128+
m.Range(func(k string, v pcommon.Value) bool {
129+
if placeholder, ok := dynamicAttrPlaceholders[k]; ok {
130+
m.PutStr(k, placeholder)
131+
return true
132+
}
133+
if k == "com.splunk.source" {
134+
s := podUIDRe.ReplaceAllString(v.AsString(), "f47ac10b-58cc-4372-a567-0e02b2c3d479")
135+
s = podNameRe.ReplaceAllString(s, "log-attr-test-7bc5f4d89b-xj8k2")
136+
m.PutStr(k, s)
137+
}
138+
return true
139+
})
140+
}

0 commit comments

Comments
 (0)