feat: add recover to goroutine to prevent unexpected panics (#226)

Notice, as we can't identify the root cause, this is just a temporary fix, which may cause further issues.

Author: youngxhui

CHANGES.md

@@ -8,6 +8,7 @@ Release Notes.
 
 * Support Windows plugin test.
 * Support Kafka reporter.
+* Add recover to goroutine to prevent unexpected panics.
 
 #### Plugins
 

plugins/core/reporter/cds_manager.go

@@ -65,6 +65,11 @@ func (r *CDSManager) InitCDS(entity *Entity, cdsWatchers []AgentConfigChangeWatc
 
 	// fetch config
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				r.logger.Errorf("CDSManager InitCDS panic err %v", err)
+			}
+		}()
 		for {
 			switch r.connManager.GetConnectionStatus(r.serverAddr) {
 			case ConnectionStatusShutdown:

plugins/core/reporter/grpc/grpc.go

@@ -175,6 +175,11 @@ func (r *gRPCReporter) initSendPipeline() {
 		return
 	}
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				r.logger.Errorf("gRPCReporter initSendPipeline trace client Collect panic err %v", err)
+			}
+		}()
 	StreamLoop:
 		for {
 			switch r.connManager.GetConnectionStatus(r.serverAddr) {
@@ -205,6 +210,11 @@ func (r *gRPCReporter) initSendPipeline() {
 		}
 	}()
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				r.logger.Errorf("gRPCReporter initSendPipeline metrics client CollectBatch panic err %v", err)
+			}
+		}()
 	StreamLoop:
 		for {
 			switch r.connManager.GetConnectionStatus(r.serverAddr) {
@@ -236,6 +246,11 @@ func (r *gRPCReporter) initSendPipeline() {
 		}
 	}()
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				r.logger.Errorf("gRPCReporter initSendPipeline log client Collect panic err %v", err)
+			}
+		}()
 	StreamLoop:
 		for {
 			switch r.connManager.GetConnectionStatus(r.serverAddr) {
@@ -303,6 +318,11 @@ func (r *gRPCReporter) check() {
 		return
 	}
 	go func() {
+		defer func() {
+			if err := recover(); err != nil {
+				r.logger.Errorf("gRPCReporter check panic err %v", err)
+			}
+		}()
 		instancePropertiesSubmitted := false
 		for {
 			switch r.connManager.GetConnectionStatus(r.serverAddr) {