[Flaky Test]: TestOtelKubeStackHelm – test timed out after 2h0m0s

[pkoutsovasilis]

Failing test case

TestOtelKubeStackHelm

Error message

test timed out after 2h0m0s

Build

https://buildkite.com/elastic/elastic-agent/builds/27005#01995a59-a247-4a20-bd73-6ff335170093/302-442

OS

Linux

Stacktrace and notes

=== Failed
=== FAIL: testing/integration/k8s  (0.00s)
-test.shuffle 1758158536928708183
panic: test timed out after 2h0m0s
	running tests:
		TestOtelKubeStackHelm (1h52m33s)
		TestOtelKubeStackHelm/managed_helm_kube-stack_operator_standalone_agent_kubernetes_privileged (1h52m28s)
goroutine 27699 [running]:
testing.(*M).startAlarm.func1()
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:2484 +0x394
created by time.goFunc
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/time/sleep.go:215 +0x2d
goroutine 1 [chan receive, 112 minutes]:
testing.(*T).Run(0xc0008141c0, {0xa66436d?, 0xc000b8fa70?}, 0xa9485a0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:1859 +0x431
testing.runTests.func1(0xc0008141c0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:2279 +0x37
testing.tRunner(0xc0008141c0, 0xc000b8fbb0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:1792 +0xf4
testing.runTests(0xc000b00f00, {0xf3b28e0, 0x5, 0x5}, {0x2?, 0xc000beb170?, 0xf7d2c60?})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:2277 +0x4b4
testing.(*M).Run(0xc000492dc0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:2142 +0x64a
github.com/elastic/elastic-agent/testing/integration/k8s.TestMain(0xc000492dc0)
	/opt/buildkite-agent/builds/bk-agent-prod-gcp-1758157757652364757/elastic/elastic-agent/testing/integration/k8s/main_test.go:32 +0x92
main.main()
	_testmain.go:55 +0xa8
goroutine 7 [select, 2 minutes]:
go.opencensus.io/stats/view.(*worker).start(0xc00041f800)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:292 +0x9f
created by go.opencensus.io/stats/view.init.0 in goroutine 1
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/go.opencensus.io@v0.24.0/stats/view/worker.go:34 +0x8d
goroutine 26136 [chan receive, 112 minutes]:
testing.(*T).Run(0xc000f54380, {0xa79f31f?, 0xc0010a9d68?}, 0xc0002a2dc0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:1859 +0x431
github.com/elastic/elastic-agent/testing/integration/k8s.TestOtelKubeStackHelm(0xc000f54380)
	/opt/buildkite-agent/builds/bk-agent-prod-gcp-1758157757652364757/elastic/elastic-agent/testing/integration/k8s/otel_helm_test.go:155 +0x1392
testing.tRunner(0xc000f54380, 0xa9485a0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:1792 +0xf4
created by testing.(*T).Run in goroutine 1
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:1851 +0x413
goroutine 26168 [chan receive, 110 minutes]:
helm.sh/helm/v3/pkg/kube.perform({0xc0000feb90, 0x1, 0x1}, 0xc00184a2d0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/helm.sh/helm/v3@v3.15.4/pkg/kube/client.go:552 +0x130
helm.sh/helm/v3/pkg/kube.(*Client).WatchUntilReady(0xc00169a1b0, {0xc0000feb90, 0x1, 0x1}, 0x0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/helm.sh/helm/v3@v3.15.4/pkg/kube/client.go:538 +0x7e
helm.sh/helm/v3/pkg/action.(*Configuration).execHook(0xc000f5d640, 0xc00017e540, {0xa6115ea, 0xa}, 0x0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/helm.sh/helm/v3@v3.15.4/pkg/action/hooks.go:84 +0x3d9
helm.sh/helm/v3/pkg/action.(*Uninstall).Run(0xc001203e10, {0xa6321e6, 0xf})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/helm.sh/helm/v3@v3.15.4/pkg/action/uninstall.go:109 +0x5ae
github.com/elastic/elastic-agent/testing/integration/k8s.k8sStepHelmDeploy.func1.2()
	/opt/buildkite-agent/builds/bk-agent-prod-gcp-1758157757652364757/elastic/elastic-agent/testing/integration/k8s/kubernetes_agent_standalone_test.go:1161 +0x168
testing.(*common).Cleanup.func1()
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:1211 +0x106
testing.(*common).runCleanup(0xc0016ad180, 0xc000ad4d10?)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:1445 +0xea
testing.tRunner.func2()
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:1786 +0x25
testing.tRunner(0xc0016ad180, 0xc0002a2dc0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:1798 +0x11e
created by testing.(*T).Run in goroutine 26136
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/testing/testing.go:1851 +0x413
goroutine 27334 [chan receive, 110 minutes]:
k8s.io/client-go/tools/cache.(*controller).Run.func1()
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/controller.go:138 +0x25
created by k8s.io/client-go/tools/cache.(*controller).Run in goroutine 27333
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/controller.go:137 +0xab
goroutine 53 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x7a9faf1a6cb0, 0x72)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/runtime/netpoll.go:351 +0x85
internal/poll.(*pollDesc).wait(0xc000310300?, 0xc000f16000?, 0x0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/internal/poll/fd_poll_runtime.go:84 +0x27
internal/poll.(*pollDesc).waitRead(...)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/internal/poll/fd_poll_runtime.go:89
internal/poll.(*FD).Read(0xc000310300, {0xc000f16000, 0xa000, 0xa000})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/internal/poll/fd_unix.go:165 +0x27a
net.(*netFD).Read(0xc000310300, {0xc000f16000?, 0x3322f56?, 0xf8105a0?})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/net/fd_posix.go:55 +0x25
net.(*conn).Read(0xc0009b02b0, {0xc000f16000?, 0x7a9fac5ccaf8?, 0x7a9ff664aa78?})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/net/net.go:194 +0x45
crypto/tls.(*atLeastReader).Read(0xc0007a6168, {0xc000f16000?, 0x9ffb?, 0x2?})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/crypto/tls/conn.go:809 +0x3b
bytes.(*Buffer).ReadFrom(0xc000aca9b8, {0xb146740, 0xc0007a6168})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/bytes/buffer.go:211 +0x98
crypto/tls.(*Conn).readFromUntil(0xc000aca708, {0xb1469e0, 0xc0009b02b0}, 0x3343b34?)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/crypto/tls/conn.go:831 +0xde
crypto/tls.(*Conn).readRecordOrCCS(0xc000aca708, 0x0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/crypto/tls/conn.go:629 +0x3cf
crypto/tls.(*Conn).readRecord(...)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/crypto/tls/conn.go:591
crypto/tls.(*Conn).Read(0xc000aca708, {0xc000d56000, 0x1000, 0x10101c0001b4380?})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/crypto/tls/conn.go:1385 +0x145
bufio.(*Reader).Read(0xc000d44840, {0xc000d062e0, 0x9, 0x331bf89?})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/bufio/bufio.go:245 +0x197
io.ReadAtLeast({0xb144da0, 0xc000d44840}, {0xc000d062e0, 0x9, 0x9}, 0x9)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/io/io.go:335 +0x91
io.ReadFull(...)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/io/io.go:354
golang.org/x/net/http2.readFrameHeader({0xc000d062e0, 0x9, 0xc000c73d58?}, {0xb144da0?, 0xc000d44840?})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/golang.org/x/net@v0.38.0/http2/frame.go:242 +0x65
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000d062a0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/golang.org/x/net@v0.38.0/http2/frame.go:506 +0x7d
golang.org/x/net/http2.(*clientConnReadLoop).run(0xc000c73fa8)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/golang.org/x/net@v0.38.0/http2/transport.go:2258 +0xda
golang.org/x/net/http2.(*ClientConn).readLoop(0xc000183a40)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/golang.org/x/net@v0.38.0/http2/transport.go:2127 +0x79
created by golang.org/x/net/http2.(*Transport).newClientConn in goroutine 52
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/golang.org/x/net@v0.38.0/http2/transport.go:912 +0xde5
goroutine 27222 [select, 110 minutes]:
k8s.io/client-go/tools/cache.(*Reflector).startResync(0xc00031d0e0, 0xc0011d6d20, 0xc000d1e2a0, 0xc0004dc1c0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/reflector.go:389 +0x106
created by k8s.io/client-go/tools/cache.(*Reflector).watchWithResync in goroutine 27335
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/reflector.go:413 +0x105
goroutine 27678 [select, 2 minutes]:
golang.org/x/net/http2.(*clientStream).writeRequest(0xc000926600, 0xc000912500, 0x0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/golang.org/x/net@v0.38.0/http2/transport.go:1570 +0xc85
golang.org/x/net/http2.(*clientStream).doRequest(0xc000926600, 0xc00070d1d0?, 0xc0000f97a8?)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/golang.org/x/net@v0.38.0/http2/transport.go:1431 +0x56
created by golang.org/x/net/http2.(*ClientConn).roundTrip in goroutine 27335
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/golang.org/x/net@v0.38.0/http2/transport.go:1336 +0x485
goroutine 27333 [sync.Cond.Wait, 110 minutes]:
sync.runtime_notifyListWait(0xc001142658, 0x2)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/runtime/sema.go:597 +0x159
sync.(*Cond).Wait(0xc001859000?)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/sync/cond.go:71 +0x85
k8s.io/client-go/tools/cache.(*DeltaFIFO).Pop(0xc001142630, 0xc000a7e4d0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/delta_fifo.go:588 +0x230
k8s.io/client-go/tools/cache.(*controller).processLoop(0xc0011426e0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/controller.go:195 +0x30
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0007c1808?)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/util/wait/backoff.go:226 +0x33
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc001ccff18, {0xb147340, 0xc00250dec0}, 0x1, 0xc0011d6d20)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/util/wait/backoff.go:227 +0xaf
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc001ccff18, 0x3b9aca00, 0x0, 0x1, 0xc0011d6d20)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/util/wait/backoff.go:204 +0x7f
k8s.io/apimachinery/pkg/util/wait.Until(...)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/util/wait/backoff.go:161
k8s.io/client-go/tools/cache.(*controller).Run(0xc0011426e0, 0xc0011d6d20)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/controller.go:166 +0x375
k8s.io/client-go/tools/watch.NewIndexerInformerWatcher.func4()
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/watch/informerwatcher.go:146 +0x7b
created by k8s.io/client-go/tools/watch.NewIndexerInformerWatcher in goroutine 27331
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/watch/informerwatcher.go:143 +0x394
goroutine 27332 [sync.Cond.Wait, 110 minutes]:
sync.runtime_notifyListWait(0xc000795490, 0x2)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/runtime/sema.go:597 +0x159
sync.(*Cond).Wait(0xc0011d6d90?)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/sync/cond.go:71 +0x85
k8s.io/client-go/tools/watch.(*eventProcessor).takeBatch(0xc00250dd40)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/watch/informerwatcher.go:64 +0x85
k8s.io/client-go/tools/watch.(*eventProcessor).run(0xc00250dd40)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/watch/informerwatcher.go:51 +0x31
created by k8s.io/client-go/tools/watch.NewIndexerInformerWatcher in goroutine 27331
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/watch/informerwatcher.go:140 +0x305
goroutine 27335 [select, 2 minutes]:
k8s.io/client-go/tools/cache.handleAnyWatch({0xc001640430?, 0x5?, 0xf7d2c60?}, {0xb1750b8, 0xc0001b44c0}, {0x7a9fac2709f0, 0xc001142630}, {0xb25a020, 0xa5a4140}, 0x0, ...)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/reflector.go:802 +0x225
k8s.io/client-go/tools/cache.handleWatch({0x5?, 0x0?, 0xf7d2c60?}, {0xb1750b8?, 0xc0001b44c0?}, {0x7a9fac2709f0?, 0xc001142630?}, {0xb25a020?, 0xa5a4140?}, 0x0, ...)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/reflector.go:768 +0xdd
k8s.io/client-go/tools/cache.(*Reflector).watch(0xc00031d0e0, {0x0?, 0x0?}, 0xc0011d6d20, 0xc0004dc1c0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/reflector.go:466 +0x565
k8s.io/client-go/tools/cache.(*Reflector).watchWithResync(0xc00031d0e0, {0x0, 0x0}, 0xc0011d6d20)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/reflector.go:414 +0x12c
k8s.io/client-go/tools/cache.(*Reflector).ListAndWatch(0xc00031d0e0, 0xc0011d6d20)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/reflector.go:377 +0x3bc
k8s.io/client-go/tools/cache.(*Reflector).Run.func1()
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/reflector.go:315 +0x25
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc0007c1808?)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/util/wait/backoff.go:226 +0x33
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc001211f50, {0xb147360, 0xc001d6e410}, 0x1, 0xc0011d6d20)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/util/wait/backoff.go:227 +0xaf
k8s.io/client-go/tools/cache.(*Reflector).Run(0xc00031d0e0, 0xc0011d6d20)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/cache/reflector.go:314 +0x1af
k8s.io/client-go/tools/cache.(*controller).Run.(*Group).StartWithChannel.func2()
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/util/wait/wait.go:55 +0x1b
k8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1()
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/util/wait/wait.go:72 +0x4c
created by k8s.io/apimachinery/pkg/util/wait.(*Group).Start in goroutine 27333
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/util/wait/wait.go:70 +0x73
goroutine 27331 [select, 110 minutes]:
k8s.io/client-go/tools/watch.UntilWithoutRetry({0xb1c2cd8, 0xc001d6e3c0}, {0xb175228, 0xc00101a760}, {0xc001709f30, 0x1, 0x10?})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/watch/until.go:73 +0x2df
k8s.io/client-go/tools/watch.UntilWithSync({0xb1c2cd8, 0xc001d6e3c0}, {0xb172d58?, 0xc00184a420?}, {0xb172d30?, 0xc0009b0170?}, 0x0, {0xc0001e0f30, 0x1, 0x1})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/tools/watch/until.go:152 +0x290
helm.sh/helm/v3/pkg/kube.(*Client).watchUntilReady(0xc00169a1b0, 0x0, 0xc0013f7b80)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/helm.sh/helm/v3@v3.15.4/pkg/kube/client.go:730 +0x58a
helm.sh/helm/v3/pkg/kube.(*Client).WatchUntilReady.(*Client).watchTimeout.func1(0xc0013f7b80?)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/helm.sh/helm/v3@v3.15.4/pkg/kube/client.go:517 +0x25
helm.sh/helm/v3/pkg/kube.batchPerform.func1(0x0?)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/helm.sh/helm/v3@v3.15.4/pkg/kube/client.go:592 +0x2c
created by helm.sh/helm/v3/pkg/kube.batchPerform in goroutine 27330
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/helm.sh/helm/v3@v3.15.4/pkg/kube/client.go:591 +0x65
goroutine 27679 [sync.Cond.Wait, 2 minutes]:
sync.runtime_notifyListWait(0xc000926648, 0x0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/runtime/sema.go:597 +0x159
sync.(*Cond).Wait(0x331a11e?)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/sync/cond.go:71 +0x85
golang.org/x/net/http2.(*pipe).Read(0xc000926630, {0xc000afc000, 0x200, 0x200})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/golang.org/x/net@v0.38.0/http2/pipe.go:76 +0xd6
golang.org/x/net/http2.transportResponseBody.Read({0x33dfaeb?}, {0xc000afc000?, 0x331aab4?, 0x200000000?})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/golang.org/x/net@v0.38.0/http2/transport.go:2560 +0x59
encoding/json.(*Decoder).refill(0xc0009128c0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/encoding/json/stream.go:165 +0x188
encoding/json.(*Decoder).readValue(0xc0009128c0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/encoding/json/stream.go:140 +0x85
encoding/json.(*Decoder).Decode(0xc0009128c0, {0x9990120, 0xc00184a120})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/go/src/encoding/json/stream.go:63 +0x75
k8s.io/apimachinery/pkg/util/framer.(*jsonFrameReader).Read(0xc000c29080, {0xc000cfac00, 0x400, 0x400})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/util/framer/framer.go:151 +0x15c
k8s.io/apimachinery/pkg/runtime/serializer/streaming.(*decoder).Decode(0xc001d6e0a0, 0x0, {0xb173910, 0xc0001b4500})
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/runtime/serializer/streaming/streaming.go:77 +0xa3
k8s.io/client-go/rest/watch.(*Decoder).Decode(0xc00159e400)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/client-go@v0.32.3/rest/watch/decoder.go:49 +0x4b
k8s.io/apimachinery/pkg/watch.(*StreamWatcher).receive(0xc0001b44c0)
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/watch/streamwatcher.go:105 +0xc4
created by k8s.io/apimachinery/pkg/watch.NewStreamWatcher in goroutine 27335
	/opt/buildkite-agent/.asdf/installs/golang/1.24.7/packages/pkg/mod/k8s.io/apimachinery@v0.32.3/pkg/watch/streamwatcher.go:76 +0x105
FAIL	github.com/elastic/elastic-agent/testing/integration/k8s	7200.078s
=== FAIL: testing/integration/k8s TestOtelKubeStackHelm (unknown)
=== FAIL: testing/integration/k8s TestOtelKubeStackHelm/managed_helm_kube-stack_operator_standalone_agent_kubernetes_privileged (unknown)

[elasticmachine]

Pinging @elastic/elastic-agent-control-plane (Team:Elastic-Agent-Control-Plane)

[pkoutsovasilis]

@rogercoll please when you have the time take a look of these failures

[rogercoll]

@pkoutsovasilis Have you observed this failure multiple times? The issue with that build was that the Helm Chart could not be installed due to leftover resources in the cluster from a previous build:

ClusterRole "kube-stack-otel-opentelemetry-operator-manager" in namespace "" exists and cannot be imported into the current release: invalid ownership metadata; annotation validation error: key "meta.helm.sh/release-namespace" must equal "olz4ee1caiskcgibr9gnjgdury2zrormb9w2fu4ac": current value is "q64acpf69hud40b4ubyqsb4qmysbqiyikyk8b0mo

We could add a cleanup job before running the tests, but ideally we should identify which test is not properly cleaning up its deployment resources. I was unable to reproduce the issue: #10145

[pkoutsovasilis]

@rogercoll take a look here

elastic-agent/testing/integration/k8s/kubernetes_agent_standalone_test.go

Lines 1158 to 1166 in 1647e39

	t.Cleanup(func() {
	if t.Failed() {
	k8sDumpPods(t, ctx, kCtx.client, t.Name(), namespace, kCtx.logsBasePath, kCtx.createdAt)
	}
	uninstallAction := action.NewUninstall(actionConfig)
	uninstallAction.Wait = true
	_, _ = uninstallAction.Run(releaseName)
	})

and here

elastic-agent/testing/integration/k8s/kubernetes_agent_standalone_test.go

Lines 923 to 928 in 1647e39

	t.Cleanup(func() {
	err := k8sDeleteObjects(ctx, kCtx.client, k8sDeleteOpts{wait: true}, k8sNamespace)
	if err != nil {
	t.Logf("failed to delete namespace: %v", err)
	}
	})

in theory we clean-up everything with a helm uninstall and we also delete the namespace but this last bit is irrelevant here

[rogercoll]

in theory we clean-up everything with a helm uninstall and we also delete the namespace but this last bit is irrelevant here

Right, but if I am not mistaken in case the test timeout or panic, Go won't run the "Cleanup" function.

[rogercoll]

Looks like the issue was not flaky, but there was an actual bug in the upstream Helm Chart where all versions were affected, see open-telemetry/opentelemetry-helm-charts#1876

PR that bumps to the fixed version #10145