[Bug] Go agent OOM during memory allocation

[youngxhui]

Search before asking

• I had searched in the issues and found no similar issues.

Apache SkyWalking Component

Go Agent (apache/skywalking-go)

What happened

Go Agent OOM error caused the application to crash.

log1

runtime stack:
runtime.throw({0x1c8ede1?, 0x20601a?})
	/usr/local/go/src/runtime/panic.go:1101 +0x48 fp=0xc000341e00 sp=0xc000341dd0 pc=0x47a2e8
runtime.sysMapOS(0xc05a800000, 0xc00f400000)
	/usr/local/go/src/runtime/mem_linux.go:168 +0x11b fp=0xc000341e40 sp=0xc000341e00 pc=0x41db9b
runtime.sysMap(0xc05a800000, 0xc00f400000, 0x2edc868?)
	/usr/local/go/src/runtime/mem.go:155 +0x34 fp=0xc000341e60 sp=0xc000341e40 pc=0x41d574
runtime.(*mheap).grow(0x2ecc660, 0x6007924?)
	/usr/local/go/src/runtime/mheap.go:1547 +0x236 fp=0xc000341ed8 sp=0xc000341e60 pc=0x430dd6
runtime.(*mheap).allocSpan(0x2ecc660, 0x6007924, 0x0, 0x1)
	/usr/local/go/src/runtime/mheap.go:1252 +0x1ad fp=0xc000341f80 sp=0xc000341ed8 pc=0x43042d
runtime.(*mheap).alloc.func1()
	/usr/local/go/src/runtime/mheap.go:970 +0x5c fp=0xc000341fc8 sp=0xc000341f80 pc=0x42fefc
runtime.systemstack(0x482401)
	/usr/local/go/src/runtime/asm_amd64.s:514 +0x4a fp=0xc000341fd8 sp=0xc000341fc8 pc=0x4805ca

goroutine 113 gp=0xc0004034a0 m=13 mp=0xc000520808 [running]:
runtime.systemstack_switch()
	/usr/local/go/src/runtime/asm_amd64.s:479 +0x8 fp=0xc02cbe49f8 sp=0xc02cbe49e8 pc=0x480568
runtime.(*mheap).alloc(0xc00f248000?, 0x6007924?, 0xb2?)
	/usr/local/go/src/runtime/mheap.go:964 +0x5b fp=0xc02cbe4a40 sp=0xc02cbe49f8 pc=0x42fe5b
runtime.(*mcache).allocLarge(0xc000602960?, 0xc00f2462d8, 0x1)
	/usr/local/go/src/runtime/mcache.go:235 +0x7d fp=0xc02cbe4a90 sp=0xc02cbe4a40 pc=0x41c69d
runtime.mallocgcLarge(0xc0003ac6f0?, 0x18b4340, 0x1)
	/usr/local/go/src/runtime/malloc.go:1540 +0x79 fp=0xc02cbe4af0 sp=0xc02cbe4a90 pc=0x419859
runtime.mallocgc(0xc00f2462d8, 0x18b4340, 0x1)
	/usr/local/go/src/runtime/malloc.go:1063 +0xc5 fp=0xc02cbe4b20 sp=0xc02cbe4af0 pc=0x477b65
runtime.makeslice(0x1f195f8?, 0xc010da3170?, 0xc010f79a40?)
	/usr/local/go/src/runtime/slice.go:116 +0x49 fp=0xc02cbe4b48 sp=0xc02cbe4b20 pc=0x47c789
google.golang.org/protobuf/proto.MarshalOptions.marshal({{}, 0x8?, 0x20?, 0x4f?}, {0x0, 0x0, 0x0}, {0x1f195f8, 0xc010da3170})
	/root/go/pkg/mod/google.golang.org/[email protected]/proto/encode.go:190 +0x396 fp=0xc02cbe4c18 sp=0xc02cbe4b48 pc=0xb9ae16
google.golang.org/protobuf/proto.Marshal({0x1ef44a0, 0xc010da3170})
	/root/go/pkg/mod/google.golang.org/[email protected]/proto/encode.go:111 +0x50 fp=0xc02cbe4c68 sp=0xc02cbe4c18 pc=0xb9a6d0
google.golang.org/grpc/encoding/proto.codec.Marshal({}, {0x1b58a60?, 0xc010da3170?})
	/root/go/pkg/mod/google.golang.org/[email protected]/encoding/proto/proto.go:47 +0x52 fp=0xc02cbe4cb0 sp=0xc02cbe4c68 pc=0xcae9f2
google.golang.org/grpc/encoding/proto.(*codec).Marshal(0xc0113ae5b0?, {0x1b58a60?, 0xc010da3170?})
	<autogenerated>:1 +0x31 fp=0xc02cbe4cd0 sp=0xc02cbe4cb0 pc=0xcaecf1
google.golang.org/grpc.encode({0x7f7bde5430a8?, 0x2ee31c0?}, {0x1b58a60?, 0xc010da3170?})
	/root/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:647 +0x3e fp=0xc02cbe4d20 sp=0xc02cbe4cd0 pc=0xd0bd9e
google.golang.org/grpc.prepareMsg({0x1b58a60?, 0xc010da3170?}, {0x7f7bde5430a8?, 0x2ee31c0?}, {0x0, 0x0}, {0x0, 0x0})
	/root/go/pkg/mod/google.golang.org/[email protected]/stream.go:1772 +0xc7 fp=0xc02cbe4d98 sp=0xc02cbe4d20 pc=0xd22dc7
google.golang.org/grpc.(*clientStream).SendMsg(0xc0109757a0, {0x1b58a60, 0xc010da3170})
	/root/go/pkg/mod/google.golang.org/[email protected]/stream.go:894 +0xd7 fp=0xc02cbe4ef8 sp=0xc02cbe4d98 pc=0xd1d457
skywalking.apache.org/repo/goapi/collect/language/agent/v3.(*traceSegmentReportServiceCollectClient).Send(0x1f04510?, 0x1f048a8?)
	/root/go/pkg/mod/skywalking.apache.org/repo/[email protected]/collect/language/agent/v3/Tracing_grpc.pb.go:61 +0x25 fp=0xc02cbe4f20 sp=0xc02cbe4ef8 pc=0x1711f85
github.com/apache/skywalking-go/agent/reporter.(*gRPCReporter).initSendPipeline.func1()
	grpc.go:360 +0x21a fp=0xc02cbe4fe0 sp=0xc02cbe4f20 pc=0x171fe1a
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc02cbe4fe8 sp=0xc02cbe4fe0 pc=0x482401
created by github.com/apache/skywalking-go/agent/reporter.(*gRPCReporter).initSendPipeline in goroutine 1
	grpc.go:334 +0x5f

log2

runtime stack:
runtime.throw({0x1b0bc5d?, 0x206001?})
	/usr/local/go/src/runtime/panic.go:1101 +0x48 fp=0x7ffece9aaf00 sp=0x7ffece9aaed0 pc=0x47a2e8
runtime.sysMapOS(0xc003c00000, 0xc000c00000)
	/usr/local/go/src/runtime/mem_linux.go:168 +0x11b fp=0x7ffece9aaf40 sp=0x7ffece9aaf00 pc=0x41db9b
runtime.sysMap(0xc003c00000, 0xc000c00000, 0x2c99368?)
	/usr/local/go/src/runtime/mem.go:155 +0x34 fp=0x7ffece9aaf60 sp=0x7ffece9aaf40 pc=0x41d574
runtime.(*mheap).grow(0x2c89160, 0x600041d?)
	/usr/local/go/src/runtime/mheap.go:1547 +0x236 fp=0x7ffece9aafd8 sp=0x7ffece9aaf60 pc=0x430dd6
runtime.(*mheap).allocSpan(0x2c89160, 0x600041d, 0x0, 0x1)
	/usr/local/go/src/runtime/mheap.go:1252 +0x1ad fp=0x7ffece9ab080 sp=0x7ffece9aafd8 pc=0x43042d
runtime.(*mheap).alloc.func1()
	/usr/local/go/src/runtime/mheap.go:970 +0x5c fp=0x7ffece9ab0c8 sp=0x7ffece9ab080 pc=0x42fefc
runtime.systemstack(0x484e9f)
	/usr/local/go/src/runtime/asm_amd64.s:514 +0x4a fp=0x7ffece9ab0d8 sp=0x7ffece9ab0c8 pc=0x4805ca

goroutine 65 gp=0xc0002a8d20 m=0 mp=0x2c7ebc0 [running]:
runtime.systemstack_switch()
	/usr/local/go/src/runtime/asm_amd64.s:479 +0x8 fp=0xc00060a9f8 sp=0xc00060a9e8 pc=0x480568
runtime.(*mheap).alloc(0xc00083a000?, 0x600041d?, 0x88?)
	/usr/local/go/src/runtime/mheap.go:964 +0x5b fp=0xc00060aa40 sp=0xc00060a9f8 pc=0x42fe5b
runtime.(*mcache).allocLarge(0x419334?, 0xc00083949b, 0x1)
	/usr/local/go/src/runtime/mcache.go:235 +0x7d fp=0xc00060aa90 sp=0xc00060aa40 pc=0x41c69d
runtime.mallocgcLarge(0x477b39?, 0x178b1a0, 0x1)
	/usr/local/go/src/runtime/malloc.go:1540 +0x79 fp=0xc00060aaf0 sp=0xc00060aa90 pc=0x419859
runtime.mallocgc(0xc00083949b, 0x178b1a0, 0x1)
	/usr/local/go/src/runtime/malloc.go:1063 +0xc5 fp=0xc00060ab20 sp=0xc00060aaf0 pc=0x477b65
runtime.makeslice(0x1d70830?, 0xc001405440?, 0xc001a90000?)
	/usr/local/go/src/runtime/slice.go:116 +0x49 fp=0xc00060ab48 sp=0xc00060ab20 pc=0x47c789
google.golang.org/protobuf/proto.MarshalOptions.marshal({{}, 0x88?, 0x24?, 0x28?}, {0x0, 0x0, 0x0}, {0x1d70830, 0xc001405440})
	/root/go/pkg/mod/google.golang.org/[email protected]/proto/encode.go:190 +0x396 fp=0xc00060ac18 sp=0xc00060ab48 pc=0xb94cd6
google.golang.org/protobuf/proto.Marshal({0x1d4f680, 0xc001405440})
	/root/go/pkg/mod/google.golang.org/[email protected]/proto/encode.go:111 +0x50 fp=0xc00060ac68 sp=0xc00060ac18 pc=0xb94590
google.golang.org/grpc/encoding/proto.codec.Marshal({}, {0x19ebb00?, 0xc001405440?})
	/root/go/pkg/mod/google.golang.org/[email protected]/encoding/proto/proto.go:47 +0x52 fp=0xc00060acb0 sp=0xc00060ac68 pc=0xca1b52
google.golang.org/grpc/encoding/proto.(*codec).Marshal(0xc0009aad00?, {0x19ebb00?, 0xc001405440?})
	<autogenerated>:1 +0x31 fp=0xc00060acd0 sp=0xc00060acb0 pc=0xca1e51
google.golang.org/grpc.encode({0x7fa8b6c2d390?, 0x2c9fcc0?}, {0x19ebb00?, 0xc001405440?})
	/root/go/pkg/mod/google.golang.org/[email protected]/rpc_util.go:647 +0x3e fp=0xc00060ad20 sp=0xc00060acd0 pc=0xcfeefe
google.golang.org/grpc.prepareMsg({0x19ebb00?, 0xc001405440?}, {0x7fa8b6c2d390?, 0x2c9fcc0?}, {0x0, 0x0}, {0x0, 0x0})
	/root/go/pkg/mod/google.golang.org/[email protected]/stream.go:1772 +0xc7 fp=0xc00060ad98 sp=0xc00060ad20 pc=0xd15f27
google.golang.org/grpc.(*clientStream).SendMsg(0xc0007346c0, {0x19ebb00, 0xc001405440})
	/root/go/pkg/mod/google.golang.org/[email protected]/stream.go:894 +0xd7 fp=0xc00060aef8 sp=0xc00060ad98 pc=0xd105b7
skywalking.apache.org/repo/goapi/collect/language/agent/v3.(*traceSegmentReportServiceCollectClient).Send(0xc000c31dd0?, 0xc00026c630?)
	/root/go/pkg/mod/skywalking.apache.org/repo/[email protected]/collect/language/agent/v3/Tracing_grpc.pb.go:61 +0x25 fp=0xc00060af20 sp=0xc00060aef8 pc=0x15c28a5
github.com/apache/skywalking-go/agent/reporter.(*gRPCReporter).initSendPipeline.func1()
	grpc.go:360 +0x21a fp=0xc00060afe0 sp=0xc00060af20 pc=0x15d073a
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1700 +0x1 fp=0xc00060afe8 sp=0xc00060afe0 pc=0x482401
created by github.com/apache/skywalking-go/agent/reporter.(*gRPCReporter).initSendPipeline in goroutine 1
	grpc.go:334 +0x5f

What you expected to happen

no carch

How to reproduce

The application is a high-concurrency service, and it experiences a panic every few hours.

I can not reproduce in my local env.

Anything else

No response

Are you willing to submit a pull request to fix on your own?

• Yes I am willing to submit a pull request on my own!

Code of Conduct

• I agree to follow this project's Code of Conduct

[wu-sheng]

To address OOM, you need to profile the application to see which part consumes so much memory.
Even without bugs, agents could OOM because more data(trace and trace context related) is in the memory.

[wu-sheng]

fyi @mrproliu