diff --git a/v3/examples/server/main.go b/v3/examples/server/main.go index 856ed6cfc..d08fa56ff 100644 --- a/v3/examples/server/main.go +++ b/v3/examples/server/main.go @@ -4,18 +4,24 @@ package main import ( + "context" "errors" "fmt" "io" + "log" + "math" "math/rand" "net/http" "os" + "runtime/trace" "sync" "time" "github.com/newrelic/go-agent/v3/newrelic" ) +var wasteSomeTime chan byte + func index(w http.ResponseWriter, r *http.Request) { io.WriteString(w, "hello world") } @@ -52,6 +58,87 @@ func noticeErrorWithAttributes(w http.ResponseWriter, r *http.Request) { }) } +func CPUspinner(w http.ResponseWriter, r *http.Request) { + txn := newrelic.FromContext(r.Context()) + var i int + var hypot, gamma3, xy float64 + + sgmt := txn.StartSegment("spinner") + defer sgmt.End() + for i := 0; i < 50_000_000; i++ { + if i%1_000_000 == 0 { + io.WriteString(w, fmt.Sprintf("iteration %d\r\n", i)) + } + hypot = math.Hypot(123.56789, 23.4567889) + gamma3 = math.Gamma(3) + xy = math.Pow(20, 3.5) + } + txn.Application().RecordCustomEvent("CPUspinner", map[string]any{ + "iterations": i, + "hypot": hypot, + "gamma": gamma3, + "xy": xy, + }) +} + +var a [][]byte + +func alloc100(w http.ResponseWriter, r *http.Request) { + a = append(a, make([]byte, 1024*1024*100, 1024*1024*100)) + io.WriteString(w, "added 100MB to heap") +} + +func traceprof(w http.ResponseWriter, r *http.Request) { + ctx, task := trace.NewTask(context.Background(), "tracedTask") + trace.Log(ctx, "tracedTask", "started") + trace.WithRegion(ctx, "tracedFunction", func() { + trace.Log(ctx, "process step", "a") + trace.Log(ctx, "process step", "b") + trace.Logf(ctx, "process data", "x=%d", 42) + trace.WithRegion(ctx, "subFunction", func() { + trace.Log(ctx, "process step", "c") + }) + }) + task.End() + io.WriteString(w, fmt.Sprintf("traced some functions to the profiler (%v)", + trace.IsEnabled())) +} + +// Make a blizzard of goroutines, some of which will block for a while +func goStorm(w http.ResponseWriter, r *http.Request) { + txn := newrelic.FromContext(r.Context()) + txn.RecordLog(newrelic.LogData{ + Message: "Launched goroutine storm", + Severity: "info", + }) + + var group sync.WaitGroup + for i := range 10_000 { + group.Add(1) + go func(tx *newrelic.Transaction, goRoutineNumber, total int, wg *sync.WaitGroup) { + defer wg.Done() + <-wasteSomeTime + tx.RecordLog(newrelic.LogData{ + Message: fmt.Sprintf("Storm goroutine #%d/%d terminated", goRoutineNumber+1, total), + Severity: "info", + }) + log.Printf("Terminated goroutine %d/%d", goRoutineNumber+1, total) + }(txn, i, 10_000, &group) + log.Printf("Launched goroutine %d/%d", i+1, 10_000) + } + + go func(tx *newrelic.Transaction, wg *sync.WaitGroup) { + wg.Wait() + tx.RecordLog(newrelic.LogData{ + Message: "Goroutine storm is over", + Severity: "info", + }) + log.Print("Goroutine storm is over") + }(txn, &group) + + io.WriteString(w, "A blizzard of goroutines was released") +} + func customEvent(w http.ResponseWriter, r *http.Request) { txn := newrelic.FromContext(r.Context()) @@ -264,6 +351,14 @@ func logTxnMessage(w http.ResponseWriter, r *http.Request) { } func main() { + go func() { + wasteSomeTime = make(chan byte) + for { + wasteSomeTime <- 0 + time.Sleep(time.Millisecond * 100) + } + }() + app, err := newrelic.NewApplication( newrelic.ConfigAppName("Example App"), newrelic.ConfigFromEnvironment(), @@ -271,12 +366,44 @@ func main() { newrelic.ConfigAppLogForwardingEnabled(true), newrelic.ConfigCodeLevelMetricsEnabled(true), newrelic.ConfigCodeLevelMetricsPathPrefix("go-agent/v3"), + newrelic.ConfigProfilingEnabled(true), + newrelic.ConfigProfilingWithSegments(true), + newrelic.ConfigCustomInsightsEventsMaxSamplesStored(500000), + newrelic.ConfigProfilingInclude( + newrelic.ProfilingTypeCPU| + newrelic.ProfilingTypeGoroutine| + newrelic.ProfilingTypeHeap| + newrelic.ProfilingTypeMutex| + newrelic.ProfilingTypeThreadCreate| + newrelic.ProfilingTypeTrace| + newrelic.ProfilingTypeBlock), + newrelic.ConfigProfilingSampleInterval(time.Millisecond*500), + newrelic.ConfigProfilingCPUReportInterval(time.Minute*5), ) if err != nil { fmt.Println(err) os.Exit(1) } + //if err := app.SetProfileOutputDirectory("/tmp"); err != nil { + // fmt.Println("unable to set profiling directory: %v", err) + //} + if err := app.OpenProfileAuditLog("/tmp/profile-audit"); err != nil { + panic(err) + } + if err := app.WaitForConnection(time.Second * 120); err != nil { + log.Printf("Failed to connect in 120 seconds: %v", err) + } + + if c, ok := app.Config(); ok { + log.Printf("Starting %s", c.AppName) + if c.Profiling.Enabled { + log.Printf("Profiling: %v every %v", c.Profiling.SelectedProfiles.Strings(), c.Profiling.Interval) + } + } + + app.SetProfileOutputMELT() + http.HandleFunc(newrelic.WrapHandleFunc(app, "/", index)) http.HandleFunc(newrelic.WrapHandleFunc(app, "/version", versionHandler)) http.HandleFunc(newrelic.WrapHandleFunc(app, "/notice_error", noticeError)) @@ -296,6 +423,10 @@ func main() { http.HandleFunc(newrelic.WrapHandleFunc(app, "/async", async)) http.HandleFunc(newrelic.WrapHandleFunc(app, "/message", message)) http.HandleFunc(newrelic.WrapHandleFunc(app, "/log", logTxnMessage)) + http.HandleFunc(newrelic.WrapHandleFunc(app, "/cpuspin", CPUspinner)) + http.HandleFunc(newrelic.WrapHandleFunc(app, "/gostorm", goStorm)) + http.HandleFunc(newrelic.WrapHandleFunc(app, "/alloc100", alloc100)) + http.HandleFunc(newrelic.WrapHandleFunc(app, "/trace", traceprof)) //loc := newrelic.ThisCodeLocation() backgroundCache := newrelic.NewCachedCodeLocation() @@ -321,5 +452,30 @@ func main() { io.WriteString(w, "A background log message was recorded") }) - http.ListenAndServe(":8000", nil) + server := http.Server{ + Addr: ":8000", + } + shutdownError := make(chan error) + + http.HandleFunc("/shutdown", func(w http.ResponseWriter, req *http.Request) { + ctx, cancelServer := context.WithTimeout(context.Background(), time.Second*60) + defer cancelServer() + shutdownError <- server.Shutdown(ctx) + }) + if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("HTTP server failed to start: %v", err) + } + + log.Println("HTTP server shutdown initiated...") + if status := <-shutdownError; status != nil { + log.Printf("HTTP server shutdown error: %v", status) + } else { + log.Println("HTTP server shutdown, shutting down APM agent...") + } + app.ShutdownProfiler(true) + if err := app.CloseProfileAuditLog(); err != nil { + panic(err) + } + app.Shutdown(time.Second * 60) + log.Println("Agent shutdown.") } diff --git a/v3/go.mod b/v3/go.mod index 7b72c5861..681e7c9dc 100644 --- a/v3/go.mod +++ b/v3/go.mod @@ -1,16 +1,26 @@ module github.com/newrelic/go-agent/v3 -go 1.24 +go 1.24.0 require ( + github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 + github.com/nsf/jsondiff v0.0.0-20230430225905-43f6cf3098c1 + golang.org/x/exp v0.0.0-20251009144603-d2f985daa21b google.golang.org/grpc v1.65.0 google.golang.org/protobuf v1.34.2 ) +require ( + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.15.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect +) retract v3.22.0 // release process error corrected in v3.22.1 retract v3.25.0 // release process error corrected in v3.25.1 retract v3.34.0 // this release erronously referred to and invalid protobuf dependency -retract v3.40.0 // this release erronously had deadlocks in utilization.go and incorrectly added aws-sdk-go to the go.mod file \ No newline at end of file + +retract v3.40.0 // this release erronously had deadlocks in utilization.go and incorrectly added aws-sdk-go to the go.mod file diff --git a/v3/integrations/logcontext-v2/logWriter/go.mod b/v3/integrations/logcontext-v2/logWriter/go.mod index 048e53c6b..3b5064728 100644 --- a/v3/integrations/logcontext-v2/logWriter/go.mod +++ b/v3/integrations/logcontext-v2/logWriter/go.mod @@ -7,6 +7,6 @@ require ( github.com/newrelic/go-agent/v3/integrations/logcontext-v2/nrwriter v1.0.0 ) - replace github.com/newrelic/go-agent/v3/integrations/logcontext-v2/nrwriter => ../nrwriter + replace github.com/newrelic/go-agent/v3 => ../../.. diff --git a/v3/integrations/logcontext-v2/nrlogrus/go.mod b/v3/integrations/logcontext-v2/nrlogrus/go.mod index 640fe7e6c..eb4087b56 100644 --- a/v3/integrations/logcontext-v2/nrlogrus/go.mod +++ b/v3/integrations/logcontext-v2/nrlogrus/go.mod @@ -7,5 +7,4 @@ require ( github.com/sirupsen/logrus v1.8.1 ) - replace github.com/newrelic/go-agent/v3 => ../../.. diff --git a/v3/integrations/logcontext-v2/nrslog/go.mod b/v3/integrations/logcontext-v2/nrslog/go.mod index 185bcae59..4d7a4d39d 100644 --- a/v3/integrations/logcontext-v2/nrslog/go.mod +++ b/v3/integrations/logcontext-v2/nrslog/go.mod @@ -4,5 +4,4 @@ go 1.24 require github.com/newrelic/go-agent/v3 v3.41.0 - replace github.com/newrelic/go-agent/v3 => ../../.. diff --git a/v3/integrations/logcontext-v2/nrwriter/go.mod b/v3/integrations/logcontext-v2/nrwriter/go.mod index 747e5c145..a6ec6765d 100644 --- a/v3/integrations/logcontext-v2/nrwriter/go.mod +++ b/v3/integrations/logcontext-v2/nrwriter/go.mod @@ -4,5 +4,4 @@ go 1.24 require github.com/newrelic/go-agent/v3 v3.41.0 - replace github.com/newrelic/go-agent/v3 => ../../.. diff --git a/v3/integrations/logcontext-v2/nrzap/go.mod b/v3/integrations/logcontext-v2/nrzap/go.mod index 45a89b857..faaab6302 100644 --- a/v3/integrations/logcontext-v2/nrzap/go.mod +++ b/v3/integrations/logcontext-v2/nrzap/go.mod @@ -7,5 +7,4 @@ require ( go.uber.org/zap v1.24.0 ) - replace github.com/newrelic/go-agent/v3 => ../../.. diff --git a/v3/integrations/logcontext-v2/nrzerolog/go.mod b/v3/integrations/logcontext-v2/nrzerolog/go.mod index 0e33ca1d7..015f3538b 100644 --- a/v3/integrations/logcontext-v2/nrzerolog/go.mod +++ b/v3/integrations/logcontext-v2/nrzerolog/go.mod @@ -7,5 +7,4 @@ require ( github.com/rs/zerolog v1.26.1 ) - replace github.com/newrelic/go-agent/v3 => ../../.. diff --git a/v3/integrations/logcontext-v2/zerologWriter/go.mod b/v3/integrations/logcontext-v2/zerologWriter/go.mod index 0cc068405..0ea475767 100644 --- a/v3/integrations/logcontext-v2/zerologWriter/go.mod +++ b/v3/integrations/logcontext-v2/zerologWriter/go.mod @@ -8,6 +8,6 @@ require ( github.com/rs/zerolog v1.27.0 ) - replace github.com/newrelic/go-agent/v3/integrations/logcontext-v2/nrwriter => ../nrwriter + replace github.com/newrelic/go-agent/v3 => ../../.. diff --git a/v3/integrations/logcontext/nrlogrusplugin/go.mod b/v3/integrations/logcontext/nrlogrusplugin/go.mod index 1b025a13a..687b7bdc3 100644 --- a/v3/integrations/logcontext/nrlogrusplugin/go.mod +++ b/v3/integrations/logcontext/nrlogrusplugin/go.mod @@ -10,5 +10,4 @@ require ( github.com/sirupsen/logrus v1.4.0 ) - replace github.com/newrelic/go-agent/v3 => ../../.. diff --git a/v3/integrations/nramqp/go.mod b/v3/integrations/nramqp/go.mod index 50a59b4f9..1cd1ca24f 100644 --- a/v3/integrations/nramqp/go.mod +++ b/v3/integrations/nramqp/go.mod @@ -6,4 +6,5 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 github.com/rabbitmq/amqp091-go v1.9.0 ) + replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrawsbedrock/go.mod b/v3/integrations/nrawsbedrock/go.mod index f77b2c11a..6145e7abf 100644 --- a/v3/integrations/nrawsbedrock/go.mod +++ b/v3/integrations/nrawsbedrock/go.mod @@ -11,5 +11,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrawssdk-v1/go.mod b/v3/integrations/nrawssdk-v1/go.mod index 6327d3de6..bba758c6a 100644 --- a/v3/integrations/nrawssdk-v1/go.mod +++ b/v3/integrations/nrawssdk-v1/go.mod @@ -11,5 +11,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrawssdk-v2/go.mod b/v3/integrations/nrawssdk-v2/go.mod index 33518fdbc..42e2ddbd1 100644 --- a/v3/integrations/nrawssdk-v2/go.mod +++ b/v3/integrations/nrawssdk-v2/go.mod @@ -15,5 +15,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrb3/go.mod b/v3/integrations/nrb3/go.mod index 609d0a911..c256e5725 100644 --- a/v3/integrations/nrb3/go.mod +++ b/v3/integrations/nrb3/go.mod @@ -4,5 +4,4 @@ go 1.24 require github.com/newrelic/go-agent/v3 v3.41.0 - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrconnect/example/go.mod b/v3/integrations/nrconnect/example/go.mod index fed677be1..5fca70e07 100644 --- a/v3/integrations/nrconnect/example/go.mod +++ b/v3/integrations/nrconnect/example/go.mod @@ -10,7 +10,6 @@ require ( google.golang.org/protobuf v1.34.2 ) - replace github.com/newrelic/go-agent/v3/integrations/nrconnect => .. replace github.com/newrelic/go-agent/v3 => ../../.. diff --git a/v3/integrations/nrconnect/go.mod b/v3/integrations/nrconnect/go.mod index 4c63b078d..45f12a9e1 100644 --- a/v3/integrations/nrconnect/go.mod +++ b/v3/integrations/nrconnect/go.mod @@ -8,5 +8,4 @@ require ( google.golang.org/protobuf v1.34.2 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrecho-v3/go.mod b/v3/integrations/nrecho-v3/go.mod index 2e3c7f443..b5afe1dee 100644 --- a/v3/integrations/nrecho-v3/go.mod +++ b/v3/integrations/nrecho-v3/go.mod @@ -11,5 +11,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrecho-v4/go.mod b/v3/integrations/nrecho-v4/go.mod index d6219453a..73f7e723c 100644 --- a/v3/integrations/nrecho-v4/go.mod +++ b/v3/integrations/nrecho-v4/go.mod @@ -9,5 +9,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrelasticsearch-v7/go.mod b/v3/integrations/nrelasticsearch-v7/go.mod index b3310d4ad..19de9ace1 100644 --- a/v3/integrations/nrelasticsearch-v7/go.mod +++ b/v3/integrations/nrelasticsearch-v7/go.mod @@ -9,5 +9,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrfasthttp/go.mod b/v3/integrations/nrfasthttp/go.mod index 2599325df..de85705fc 100644 --- a/v3/integrations/nrfasthttp/go.mod +++ b/v3/integrations/nrfasthttp/go.mod @@ -7,5 +7,4 @@ require ( github.com/valyala/fasthttp v1.49.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrfiber/go.mod b/v3/integrations/nrfiber/go.mod index 11175a806..5bc8915fb 100644 --- a/v3/integrations/nrfiber/go.mod +++ b/v3/integrations/nrfiber/go.mod @@ -9,5 +9,4 @@ require ( github.com/valyala/fasthttp v1.51.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrgin/go.mod b/v3/integrations/nrgin/go.mod index c8ae5aa9d..d4a1c7683 100644 --- a/v3/integrations/nrgin/go.mod +++ b/v3/integrations/nrgin/go.mod @@ -9,5 +9,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrgorilla/go.mod b/v3/integrations/nrgorilla/go.mod index 6e5dd8005..57274363e 100644 --- a/v3/integrations/nrgorilla/go.mod +++ b/v3/integrations/nrgorilla/go.mod @@ -10,5 +10,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrgraphgophers/go.mod b/v3/integrations/nrgraphgophers/go.mod index cf5f956f4..1c7a084f3 100644 --- a/v3/integrations/nrgraphgophers/go.mod +++ b/v3/integrations/nrgraphgophers/go.mod @@ -10,5 +10,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrgraphqlgo/go.mod b/v3/integrations/nrgraphqlgo/go.mod index e7cd4e70a..d52cf0056 100644 --- a/v3/integrations/nrgraphqlgo/go.mod +++ b/v3/integrations/nrgraphqlgo/go.mod @@ -7,5 +7,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrgrpc/go.mod b/v3/integrations/nrgrpc/go.mod index 3692aded0..45b7a912b 100644 --- a/v3/integrations/nrgrpc/go.mod +++ b/v3/integrations/nrgrpc/go.mod @@ -13,7 +13,6 @@ require ( google.golang.org/protobuf v1.34.2 ) - replace github.com/newrelic/go-agent/v3/integrations/nrsecurityagent => ../../integrations/nrsecurityagent replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrhttprouter/go.mod b/v3/integrations/nrhttprouter/go.mod index 75b396473..c4566fd12 100644 --- a/v3/integrations/nrhttprouter/go.mod +++ b/v3/integrations/nrhttprouter/go.mod @@ -10,5 +10,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrlambda/go.mod b/v3/integrations/nrlambda/go.mod index 46722be9f..e4b7014c4 100644 --- a/v3/integrations/nrlambda/go.mod +++ b/v3/integrations/nrlambda/go.mod @@ -7,5 +7,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrlogrus/go.mod b/v3/integrations/nrlogrus/go.mod index 5d072040e..501c3268d 100644 --- a/v3/integrations/nrlogrus/go.mod +++ b/v3/integrations/nrlogrus/go.mod @@ -12,6 +12,6 @@ require ( github.com/sirupsen/logrus v1.8.1 ) - replace github.com/newrelic/go-agent/v3/integrations/logcontext-v2/nrlogrus => ../logcontext-v2/nrlogrus + replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrlogxi/go.mod b/v3/integrations/nrlogxi/go.mod index b1e421ef4..0c3747097 100644 --- a/v3/integrations/nrlogxi/go.mod +++ b/v3/integrations/nrlogxi/go.mod @@ -10,5 +10,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrmicro/go.mod b/v3/integrations/nrmicro/go.mod index b2f95cc42..1ea9ac6bb 100644 --- a/v3/integrations/nrmicro/go.mod +++ b/v3/integrations/nrmicro/go.mod @@ -13,5 +13,4 @@ require ( google.golang.org/protobuf v1.36.6 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrmssql/go.mod b/v3/integrations/nrmssql/go.mod index d85a1c48d..7d934eb5b 100644 --- a/v3/integrations/nrmssql/go.mod +++ b/v3/integrations/nrmssql/go.mod @@ -7,5 +7,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrmysql/go.mod b/v3/integrations/nrmysql/go.mod index 035c04d0c..4c03c6eea 100644 --- a/v3/integrations/nrmysql/go.mod +++ b/v3/integrations/nrmysql/go.mod @@ -10,5 +10,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrnats/go.mod b/v3/integrations/nrnats/go.mod index fa2d9e1e5..9457fe326 100644 --- a/v3/integrations/nrnats/go.mod +++ b/v3/integrations/nrnats/go.mod @@ -12,5 +12,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nropenai/go.mod b/v3/integrations/nropenai/go.mod index eab458893..da8a8381e 100644 --- a/v3/integrations/nropenai/go.mod +++ b/v3/integrations/nropenai/go.mod @@ -9,5 +9,4 @@ require ( github.com/sashabaranov/go-openai v1.20.2 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrpgx/example/sqlx/go.mod b/v3/integrations/nrpgx/example/sqlx/go.mod index 75109be59..6c30090b5 100644 --- a/v3/integrations/nrpgx/example/sqlx/go.mod +++ b/v3/integrations/nrpgx/example/sqlx/go.mod @@ -1,11 +1,15 @@ // This sqlx example is a separate module to avoid adding sqlx dependency to the // nrpgx go.mod file. module github.com/newrelic/go-agent/v3/integrations/nrpgx/example/sqlx + go 1.24 + require ( github.com/jmoiron/sqlx v1.2.0 github.com/newrelic/go-agent/v3 v3.41.0 github.com/newrelic/go-agent/v3/integrations/nrpgx v0.0.0 ) + replace github.com/newrelic/go-agent/v3/integrations/nrpgx => ../../ + replace github.com/newrelic/go-agent/v3 => ../../../.. diff --git a/v3/integrations/nrpgx/go.mod b/v3/integrations/nrpgx/go.mod index ae10c748f..12fbc28d6 100644 --- a/v3/integrations/nrpgx/go.mod +++ b/v3/integrations/nrpgx/go.mod @@ -8,5 +8,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrpgx5/go.mod b/v3/integrations/nrpgx5/go.mod index 398e0198a..fad3cf45a 100644 --- a/v3/integrations/nrpgx5/go.mod +++ b/v3/integrations/nrpgx5/go.mod @@ -11,5 +11,4 @@ require ( github.com/stretchr/testify v1.8.1 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrpkgerrors/go.mod b/v3/integrations/nrpkgerrors/go.mod index 56c47a2a2..1d5838e36 100644 --- a/v3/integrations/nrpkgerrors/go.mod +++ b/v3/integrations/nrpkgerrors/go.mod @@ -11,5 +11,4 @@ require ( github.com/pkg/errors v0.8.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrpq/example/sqlx/go.mod b/v3/integrations/nrpq/example/sqlx/go.mod index 32edab71e..ea643fceb 100644 --- a/v3/integrations/nrpq/example/sqlx/go.mod +++ b/v3/integrations/nrpq/example/sqlx/go.mod @@ -1,12 +1,16 @@ // This sqlx example is a separate module to avoid adding sqlx dependency to the // nrpq go.mod file. module github.com/newrelic/go-agent/v3/integrations/nrpq/example/sqlx + go 1.24 + require ( github.com/jmoiron/sqlx v1.2.0 github.com/lib/pq v1.1.0 github.com/newrelic/go-agent/v3 v3.41.0 github.com/newrelic/go-agent/v3/integrations/nrpq v0.0.0 ) + replace github.com/newrelic/go-agent/v3/integrations/nrpq => ../../ + replace github.com/newrelic/go-agent/v3 => ../../../.. diff --git a/v3/integrations/nrpq/go.mod b/v3/integrations/nrpq/go.mod index 83124e076..140a89421 100644 --- a/v3/integrations/nrpq/go.mod +++ b/v3/integrations/nrpq/go.mod @@ -9,5 +9,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrredis-v7/go.mod b/v3/integrations/nrredis-v7/go.mod index 5c545500d..46fda5c60 100644 --- a/v3/integrations/nrredis-v7/go.mod +++ b/v3/integrations/nrredis-v7/go.mod @@ -8,5 +8,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrredis-v8/go.mod b/v3/integrations/nrredis-v8/go.mod index 0de3dcb11..5d2f0b95c 100644 --- a/v3/integrations/nrredis-v8/go.mod +++ b/v3/integrations/nrredis-v8/go.mod @@ -8,5 +8,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrredis-v9/go.mod b/v3/integrations/nrredis-v9/go.mod index 323dd1a82..d28e2ff3a 100644 --- a/v3/integrations/nrredis-v9/go.mod +++ b/v3/integrations/nrredis-v9/go.mod @@ -8,5 +8,4 @@ require ( github.com/redis/go-redis/v9 v9.0.2 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrsarama/go.mod b/v3/integrations/nrsarama/go.mod index f661bbb8f..fb94f32f2 100644 --- a/v3/integrations/nrsarama/go.mod +++ b/v3/integrations/nrsarama/go.mod @@ -10,5 +10,4 @@ require ( github.com/stretchr/testify v1.8.1 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrsecurityagent/go.mod b/v3/integrations/nrsecurityagent/go.mod index 03d471346..1bb502397 100644 --- a/v3/integrations/nrsecurityagent/go.mod +++ b/v3/integrations/nrsecurityagent/go.mod @@ -9,5 +9,4 @@ require ( gopkg.in/yaml.v2 v2.4.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrsqlite3/go.mod b/v3/integrations/nrsqlite3/go.mod index dcec32288..4138d6506 100644 --- a/v3/integrations/nrsqlite3/go.mod +++ b/v3/integrations/nrsqlite3/go.mod @@ -10,5 +10,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrstan/go.mod b/v3/integrations/nrstan/go.mod index d1b8e9d6c..b67068105 100644 --- a/v3/integrations/nrstan/go.mod +++ b/v3/integrations/nrstan/go.mod @@ -11,5 +11,4 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrstan/test/go.mod b/v3/integrations/nrstan/test/go.mod index 15fb00341..1edd618c5 100644 --- a/v3/integrations/nrstan/test/go.mod +++ b/v3/integrations/nrstan/test/go.mod @@ -13,7 +13,6 @@ require ( github.com/newrelic/go-agent/v3/integrations/nrstan v0.0.0 ) - replace github.com/newrelic/go-agent/v3/integrations/nrstan => ../ replace github.com/newrelic/go-agent/v3 => ../../.. diff --git a/v3/integrations/nrzap/go.mod b/v3/integrations/nrzap/go.mod index cf9332c56..e2e302c26 100644 --- a/v3/integrations/nrzap/go.mod +++ b/v3/integrations/nrzap/go.mod @@ -10,5 +10,4 @@ require ( go.uber.org/zap v1.12.0 ) - replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/integrations/nrzerolog/go.mod b/v3/integrations/nrzerolog/go.mod index ee06374a9..09baa6c03 100644 --- a/v3/integrations/nrzerolog/go.mod +++ b/v3/integrations/nrzerolog/go.mod @@ -6,4 +6,5 @@ require ( github.com/newrelic/go-agent/v3 v3.41.0 github.com/rs/zerolog v1.28.0 ) + replace github.com/newrelic/go-agent/v3 => ../.. diff --git a/v3/internal/tools/profiling/auditprofile b/v3/internal/tools/profiling/auditprofile new file mode 100755 index 000000000..762fcc05d --- /dev/null +++ b/v3/internal/tools/profiling/auditprofile @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# +# Compare Go agent profiling audit log file with agent debugging log output to see which expected +# custom events actually were reported out to the back end. +# +import sys +import json +import re + +class AuditRecord: + def __init__(self, event_type=None, harvest_seq=None, sample_seq=None, error=None, **k): + self.event_type = event_type + self.harvest_seq = harvest_seq + self.sample_seq = sample_seq + self.error = error + self.seen = False + +class AuditLog (dict): + def __init__(self, filename): + # The audit file has a single line per expected profile sample, in JSON format. + l=0 + with open(filename) as audit_data: + for audit_record in audit_data: + l+=1 + record = AuditRecord(**json.loads(audit_record)) + if record.event_type is None or record.harvest_seq is None or record.sample_seq is None: + raise ValueError(f'line {l}: missing required audit fields <{record.event_type}, {record.harvest_seq}, {record.sample_seq}>') + if self.k(record.event_type, record.harvest_seq, record.sample_seq) in self: + raise KeyError(f'line {l}: duplicate audit record <{record.event_type}, {record.harvest_seq}, {record.sample_seq}>') + self[self.k(record.event_type, record.harvest_seq, record.sample_seq)] = record + + def k(self, event_type, harvest_seq, sample_seq): + return f'{event_type};{harvest_seq};{sample_seq}' + + def counts(self): + counters = {} + for record in self.values(): + counters[record.event_type] = counters.get(record.event_type, 0) + 1 + return counters + + def account_for(self, event_type, harvest_seq, sample_seq): + k = self.k(event_type, harvest_seq, sample_seq) + if k not in self: + raise KeyError(f"Unable to account for event <{event_type}, {harvest_seq}, {sample_seq}>: not in audit!") + if self[k].seen: + raise KeyError(f"Reported twice! <{event_type}, {harvest_seq}, {sample_seq}>") + self[k].seen = True + + def unaccounted_for(self): + counters = {} + for record in self.values(): + if record.seen: + counters[record.event_type] = counters.get(record.event_type, 0) + 1 + return counters + + + + +if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} audit-file debug-log") + exit(1) + +print("Reading audit log...") +audit = AuditLog(sys.argv[1]) +print(f'Audit log contains {len(audit):,} entries:') +tally = audit.counts() +for profile_type in sorted(tally.keys()): + print(f'{tally[profile_type]:10,d} {profile_type}') + +# Now, pick through the debug log and see if we find evidence that the agent +# actually sent all the events we asked it to. +skip_count = 0 +other_count = 0 +processed_count = 0 +dropped_count = 0 +dropped_instances = 0 +l=0 +with open(sys.argv[2]) as debug_log: + for debug_line in debug_log: + l += 1 + if m := re.search(r'(\{.*\})\s*$', debug_line): + try: + data = json.loads(m.group(1)) + except json.decoder.JSONDecodeError: + print(f"Skipping input line {l}: {debug_line[:30]}... (malformed JSON)") + skip_count += 1 + continue + + if 'msg' in data and data['msg'] == 'rpm request' and 'context' in data and 'command' in data['context'] and data['context']['command'] == 'custom_event_data': + if 'payload' not in data['context']: + print(f'Skipping input line {l}: {debug_line[:30]}... (missing payload field)') + skip_count += 1 + continue + + reservoir_size = data['context']['payload'][1]['reservoir_size'] + events_seen = data['context']['payload'][1]['events_seen'] + + if reservoir_size < events_seen : + print(f'WARNING: {events_seen} EVENTS IN RESERVIOR SIZE OF {reservoir_size}!') + dropped_count += events_seen - reservoir_size + dropped_instances += 1 + + for sample in data['context']['payload'][2]: + audit.account_for(sample[0]['type'], sample[1]['harvest_seq'], sample[1]['sample_seq']) + + processed_count += 1 + else: + other_count += 1 + else: + print(f'Skipping input line {l}: {debug_line[:30]}... (doesn\'t look like a debug log line)') + skip_count += 1 + +print(f'Processed {processed_count} lines of debug log entries for custom events') +print(f' {other_count} lines of debug log entries that were other things') +print(f'Skipped {skip_count} lines that weren\'t valid debug log data') +if dropped_instances > 0: + print(f"During the program's execution, it exceeded the reservoir size {dropped_instances:,} time{'s' if dropped_instances != 1 else ''}") + print(f"resulting in an overage of {dropped_count:,} event{'' if dropped_count==1 else 's'}!") + print('This was based on the debug log reports of event harvest reservoir values, and does not represent the') + print('actual number of dropped events due to how the harvester actually works.') +else: + print("All expected events were found in the debug log.") + +tally = audit.unaccounted_for() +total = 0 +if len(tally) > 0: + print(f"UNACCOUNTED-FOR PROFILE EVENTS OUT OF {len(audit):,} TOTAL:") + for profile_type in sorted(tally.keys()): + print(f'{tally[profile_type]:10,d} {(tally[profile_type]*100.0)/len(audit):3.0f}% {profile_type}') + total += tally[profile_type] + print(f'{total:10,d} {(total*100.0)/len(audit):3.0f}% TOTAL') diff --git a/v3/newrelic/config.go b/v3/newrelic/config.go index d8c8a52ff..bc144729a 100644 --- a/v3/newrelic/config.go +++ b/v3/newrelic/config.go @@ -19,6 +19,90 @@ import ( "github.com/newrelic/go-agent/v3/internal/utilization" ) +type ProfilingType uint32 + +const ( + ProfilingTypeBlock ProfilingType = 1 << iota + ProfilingTypeCPU + ProfilingTypeGoroutine + ProfilingTypeHeap + ProfilingTypeMutex + ProfilingTypeThreadCreate + ProfilingTypeTrace +) + +func (p *ProfilingType) FromStrings(types []string, additive bool) error { + if p == nil { + return fmt.Errorf("nil ProfilingType pointer") + } + + if !additive { + *p = 0 + } + + for _, t := range types { + switch t { + case "block": + *p |= ProfilingTypeBlock + case "cpu": + *p |= ProfilingTypeCPU + case "goroutine": + *p |= ProfilingTypeGoroutine + case "heap": + *p |= ProfilingTypeHeap + case "mutex": + *p |= ProfilingTypeMutex + case "threadcreate": + *p |= ProfilingTypeThreadCreate + case "trace": + *p |= ProfilingTypeTrace + default: + return fmt.Errorf("unknown ProfilingType \"%s\"", t) + } + } + return nil +} + +func (p ProfilingType) Strings() []string { + var typeSet []string + + if ProfilingTypeBlock&p != 0 { + typeSet = append(typeSet, "block") + } + if ProfilingTypeCPU&p != 0 { + typeSet = append(typeSet, "cpu") + } + if ProfilingTypeGoroutine&p != 0 { + typeSet = append(typeSet, "goroutine") + } + if ProfilingTypeHeap&p != 0 { + typeSet = append(typeSet, "heap") + } + if ProfilingTypeMutex&p != 0 { + typeSet = append(typeSet, "mutex") + } + if ProfilingTypeThreadCreate&p != 0 { + typeSet = append(typeSet, "threadcreate") + } + if ProfilingTypeTrace&p != 0 { + typeSet = append(typeSet, "trace") + } + return typeSet +} + +func (p ProfilingType) MarshalJSON() ([]byte, error) { + return json.Marshal(p.Strings()) +} + +func (p *ProfilingType) UnmarshalJSON(data []byte) error { + var t []string + + if err := json.Unmarshal(data, &t); err != nil { + return err + } + return p.FromStrings(t, false) +} + // Config contains Application and Transaction behavior settings. type Config struct { // AppName is used by New Relic to link data across servers. @@ -481,6 +565,37 @@ type Config struct { // This list of ignored prefixes itself is not reported outside the agent. IgnoredPrefixes []string } + + // Profiling Configuration + Profiling struct { + // Enabled controls whether the profiler is running. + Enabled bool + // WithSegments controls whether we report the actively-running segments + // along with the sample data so they can be associated with them when data are + // analyzed later. + WithSegments bool + // SelectedProfiles indicates which kinds of profiles we're collecting and reporting. + SelectedProfiles ProfilingType + // Interval is the rate at which the profiler gathers and reports non-CPU profile data. + Interval time.Duration + // CPUReportInterval is the rate at which we stop the CPU profiler to let it report + // out the data it's collected so far, after which we restart it again to collect more + // data. If this is zero, we won't interrupt the profiler to report anything + // until we shut down the whole agent profiler. + CPUReportInterval time.Duration + // CPUSampleRateHz is the internal collection speed at which the CPU profiler is running + // to collect CPU stats. + CPUSampleRateHz int + // BlockRate is the rate at which we collect block profile data. If <=0, we stop collecting + // block profiles altogether. Otherwise, we try to get 1/n. By default we set this to 1, which + // tries to collect them all. + BlockRate int + // MutexRate is the rate at which we collect Mutex profile data. If 0, we stop collecting + // mutex profiles altogether. Otherwise, we try to get 1/n. By default we set this to 1, which + // tries to collect them all. + MutexRate int + } + // Security is used to post security configuration on UI. Security interface{} `json:"Security,omitempty"` } @@ -736,6 +851,11 @@ func defaultConfig() Config { // Module Dependency Metrics c.ModuleDependencyMetrics.Enabled = true c.ModuleDependencyMetrics.RedactIgnoredPrefixes = true + + // Profiling settings + c.Profiling.CPUSampleRateHz = 100 + c.Profiling.BlockRate = 1 + c.Profiling.MutexRate = 1 return c } diff --git a/v3/newrelic/config_options.go b/v3/newrelic/config_options.go index ea79d1b03..f2518eb33 100644 --- a/v3/newrelic/config_options.go +++ b/v3/newrelic/config_options.go @@ -10,6 +10,7 @@ import ( "os" "strconv" "strings" + "time" "unicode/utf8" ) @@ -439,67 +440,145 @@ func ConfigLabels(labels map[string]string) ConfigOption { } } -// ConfigCustomInsightsCustomAttributesEnabled enables or disables sending our application -// custom attributes (which are configured via ConfigCustomInsightsCustomAttributesValues) with forwarded log events. -// Defaults: enabled=false -// This may also be set using the NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CUSTOM_ATTRIBUTES_ENABLED environment variable. -func ConfigCustomInsightsCustomAttributesEnabled(enabled bool) ConfigOption { +// ConfigProfilingEnabled turns on profiling of the runtime, which is further broken down into +// specific areas to measure by ConfigProfilingInclude. +func ConfigProfilingEnabled(enabled bool) ConfigOption { + return func(cfg *Config) { + cfg.Profiling.Enabled = enabled + } +} + +// ConfigProfilingWithSegments includes segment information in profile samples so that +// the samples may be associated with what segments were active at the time they were taken, +// and therefore deduce which code units may be responsible for the resources expended. +func ConfigProfilingWithSegments(enabled bool) ConfigOption { + return func(cfg *Config) { + cfg.Profiling.WithSegments = enabled + } +} + +// ConfigProfilingInclude enables specific profiler modules to measure aspects of the runtime. +// These are specified as a list of constant values, e.g., ProfilingTypeCPU, etc. +func ConfigProfilingInclude(ptype ...ProfilingType) ConfigOption { + return func(cfg *Config) { + for _, pt := range ptype { + cfg.Profiling.SelectedProfiles |= pt + } + } +} + +// ConfigProfilingIncludeByName is just like ConfigProfilingInclude, execpt that it takes +// a slice of human-friendly strings with the profiling type names, e.g., "cpu" for ProfilingTypeCPU. +func ConfigProfilingIncludeByName(ptype []string) ConfigOption { + return func(cfg *Config) { + cfg.Profiling.SelectedProfiles.FromStrings(ptype, false) + } +} + +// ConfigProfilingIncludeByNames is just like ConfigProfilingInclude, execpt that it takes +// a list of human-friendly strings with the profiling type names, e.g., "cpu" for ProfilingTypeCPU. +func ConfigProfilingIncludeByNames(ptype ...string) ConfigOption { + return func(cfg *Config) { + cfg.Profiling.SelectedProfiles.FromStrings(ptype, false) + } +} + +// ConfigProfilingSampleInterval controls the pace at which we sample and report the collected profile data to the destination, +// except for CPU profiles (which are buffered internally until the profiler is stopped, under normal circumstances). +func ConfigProfilingSampleInterval(interval time.Duration) ConfigOption { + return func(cfg *Config) { + cfg.Profiling.Interval = interval + } +} + +// ConfigProfilingCPUReportInterval controls the pace at which we report the collected CPU profile data. Since the +// CPU profiler internally buffers and aggregates its data during its entire run, reporting its data out only when +// it is stopped, this means that every time this interval of time elapses, we actually need to stop the CPU profiler, +// let it report out its data, then start a new CPU profiler run for a new set of profile data. Keep this in mind as you +// determine the report interval to give yourself realtime visibility, vs. having a single comprehensive set of profile +// data that represents the entire runtime performance of your application. +func ConfigProfilingCPUReportInterval(interval time.Duration) ConfigOption { return func(cfg *Config) { - cfg.CustomInsightsEvents.CustomAttributesEnabled = enabled + cfg.Profiling.CPUReportInterval = interval } } -// ConfigCustomInsightsCustomAttributesValues configures a set of custom attributes to add as attributes to all log events forwarded to New Relic. -// This may also be set using the NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CUSTOM_ATTRIBUTES environment variable. -func ConfigCustomInsightsCustomAttributesValues(customAttributes map[string]string) ConfigOption { +// ConfigProfilingCPUSampleRateHz controls the CPU profiler's internal sample rate at which it collects the system's CPU usage +// data as it works. By default this is set to 100 Hz, but you can adjust that here if you want to collect data more or less +// frequently. +func ConfigProfilingCPUSampleRateHz(rate int) ConfigOption { return func(cfg *Config) { - cfg.CustomInsightsEvents.CustomAttributesValues = make(map[string]string) - maps.Copy(cfg.CustomInsightsEvents.CustomAttributesValues, customAttributes) + cfg.Profiling.CPUSampleRateHz = rate + } +} + +// ConfigProfilingBlockRate controls the number of block profile samples we try to collect. The default value of +// 1 tries to collect all data. Increasing this to some value n reduces that to try to collect 1/n of the blocks +// seen as the profiler looks at the blocked routines. A value less than or equal to 0 means not to collect block profile +// data at all. +func ConfigProfilingBlockRate(rate int) ConfigOption { + return func(cfg *Config) { + cfg.Profiling.BlockRate = rate + } +} + +// ConfigProfilingMutexRate controls the number of mutex profile samples we try to collect. The default value of +// 1 tries to collect all data. Increasing this to some value n reduces that to try to collect 1/n of the mutex samples. +// A value of 0 means not to collect this data at all. +func ConfigProfilingMutexRate(rate int) ConfigOption { + return func(cfg *Config) { + cfg.Profiling.MutexRate = rate } } // ConfigFromEnvironment populates the config based on environment variables: // -// NEW_RELIC_APP_NAME sets AppName -// NEW_RELIC_ATTRIBUTES_EXCLUDE sets Attributes.Exclude using a comma-separated list, eg. "request.headers.host,request.method" -// NEW_RELIC_ATTRIBUTES_INCLUDE sets Attributes.Include using a comma-separated list -// NEW_RELIC_MODULE_DEPENDENCY_METRICS_ENABLED sets ModuleDependencyMetrics.Enabled -// NEW_RELIC_MODULE_DEPENDENCY_METRICS_IGNORED_PREFIXES sets ModuleDependencyMetrics.IgnoredPrefixes -// NEW_RELIC_MODULE_DEPENDENCY_METRICS_REDACT_IGNORED_PREFIXES sets ModuleDependencyMetrics.RedactIgnoredPrefixes to a boolean value -// NEW_RELIC_CODE_LEVEL_METRICS_ENABLED sets CodeLevelMetrics.Enabled -// NEW_RELIC_CODE_LEVEL_METRICS_SCOPE sets CodeLevelMetrics.Scope using a comma-separated list, e.g. "transaction" -// NEW_RELIC_CODE_LEVEL_METRICS_PATH_PREFIX sets CodeLevelMetrics.PathPrefixes using a comma-separated list -// NEW_RELIC_CODE_LEVEL_METRICS_REDACT_PATH_PREFIXES sets CodeLevelMetrics.RedactPathPrefixes to a boolean value -// NEW_RELIC_CODE_LEVEL_METRICS_REDACT_IGNORED_PREFIXES sets CodeLevelMetrics.RedactIgnoredPrefixes to a boolean value -// NEW_RELIC_CODE_LEVEL_METRICS_IGNORED_PREFIX sets CodeLevelMetrics.IgnoredPrefixes using a comma-separated list -// NEW_RELIC_DISTRIBUTED_TRACING_ENABLED sets DistributedTracer.Enabled using strconv.ParseBool -// NEW_RELIC_ENABLED sets Enabled using strconv.ParseBool -// NEW_RELIC_HIGH_SECURITY sets HighSecurity using strconv.ParseBool -// NEW_RELIC_HOST sets Host -// NEW_RELIC_INFINITE_TRACING_SPAN_EVENTS_QUEUE_SIZE sets InfiniteTracing.SpanEvents.QueueSize using strconv.Atoi -// NEW_RELIC_INFINITE_TRACING_TRACE_OBSERVER_PORT sets InfiniteTracing.TraceObserver.Port using strconv.Atoi -// NEW_RELIC_INFINITE_TRACING_TRACE_OBSERVER_HOST sets InfiniteTracing.TraceObserver.Host -// NEW_RELIC_LABELS sets Labels using a semi-colon delimited string of colon-separated pairs, eg. "Server:One;DataCenter:Primary" -// NEW_RELIC_LICENSE_KEY sets License -// NEW_RELIC_LOG sets Logger to log to either "stdout" or "stderr" (filenames are not supported) -// NEW_RELIC_LOG_LEVEL controls the NEW_RELIC_LOG level, must be "debug" for debug, or empty for info -// NEW_RELIC_PROCESS_HOST_DISPLAY_NAME sets HostDisplayName -// NEW_RELIC_SECURITY_POLICIES_TOKEN sets SecurityPoliciesToken -// NEW_RELIC_UTILIZATION_BILLING_HOSTNAME sets Utilization.BillingHostname -// NEW_RELIC_UTILIZATION_LOGICAL_PROCESSORS sets Utilization.LogicalProcessors using strconv.Atoi -// NEW_RELIC_UTILIZATION_TOTAL_RAM_MIB sets Utilization.TotalRAMMIB using strconv.Atoi -// NEW_RELIC_APPLICATION_LOGGING_ENABLED sets ApplicationLogging.Enabled. Set to false to disable all application logging features. -// NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED sets ApplicationLogging.LogForwarding.Enabled. Set to false to disable in agent log forwarding. -// NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LABELS_ENABLED sets ApplicationLogging.LogForwarding.Labels.Enabled to enable sending application labels with forwarded logs. -// NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LABELS_EXCLUDE sets ApplicationLogging.LogForwarding.Labels.Exclude to filter out a set of unwanted label types from the ones reported with logs. -// NEW_RELIC_APPLICATION_LOGGING_METRICS_ENABLED sets ApplicationLogging.Metrics.Enabled. Set to false to disable the collection of application log metrics. -// NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED sets ApplicationLogging.LocalDecoration.Enabled. Set to true to enable local log decoration. -// NEW_RELIC_APPLICATION_LOGGING_FORWARDING_MAX_SAMPLES_STORED sets ApplicationLogging.LogForwarding.Limit. Set to 0 to prevent captured logs from being forwarded. -// NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CUSTOM_ATTRIBUTES_ENABLED sets CustomInsightsEvents.CustomAttributesEnabled to enable sending application custom attributes with forwarded logs. -// NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CUSTOM_ATTRIBUTES sets CustomInsightsEvents.CustomAttributesValues A hash with key/value pairs to add as custom attributes to all log events forwarded to New Relic. -// NEW_RELIC_AI_MONITORING_ENABLED sets AIMonitoring.Enabled -// NEW_RELIC_AI_MONITORING_STREAMING_ENABLED sets AIMonitoring.Streaming.Enabled -// NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED sets AIMonitoring.RecordContent.Enabled +// NEW_RELIC_APP_NAME sets AppName +// NEW_RELIC_ATTRIBUTES_EXCLUDE sets Attributes.Exclude using a comma-separated list, eg. "request.headers.host,request.method" +// NEW_RELIC_ATTRIBUTES_INCLUDE sets Attributes.Include using a comma-separated list +// NEW_RELIC_MODULE_DEPENDENCY_METRICS_ENABLED sets ModuleDependencyMetrics.Enabled +// NEW_RELIC_MODULE_DEPENDENCY_METRICS_IGNORED_PREFIXES sets ModuleDependencyMetrics.IgnoredPrefixes +// NEW_RELIC_MODULE_DEPENDENCY_METRICS_REDACT_IGNORED_PREFIXES sets ModuleDependencyMetrics.RedactIgnoredPrefixes to a boolean value +// NEW_RELIC_CODE_LEVEL_METRICS_ENABLED sets CodeLevelMetrics.Enabled +// NEW_RELIC_CODE_LEVEL_METRICS_SCOPE sets CodeLevelMetrics.Scope using a comma-separated list, e.g. "transaction" +// NEW_RELIC_CODE_LEVEL_METRICS_PATH_PREFIX sets CodeLevelMetrics.PathPrefixes using a comma-separated list +// NEW_RELIC_CODE_LEVEL_METRICS_REDACT_PATH_PREFIXES sets CodeLevelMetrics.RedactPathPrefixes to a boolean value +// NEW_RELIC_CODE_LEVEL_METRICS_REDACT_IGNORED_PREFIXES sets CodeLevelMetrics.RedactIgnoredPrefixes to a boolean value +// NEW_RELIC_CODE_LEVEL_METRICS_IGNORED_PREFIX sets CodeLevelMetrics.IgnoredPrefixes using a comma-separated list +// NEW_RELIC_DISTRIBUTED_TRACING_ENABLED sets DistributedTracer.Enabled using strconv.ParseBool +// NEW_RELIC_ENABLED sets Enabled using strconv.ParseBool +// NEW_RELIC_HIGH_SECURITY sets HighSecurity using strconv.ParseBool +// NEW_RELIC_HOST sets Host +// NEW_RELIC_INFINITE_TRACING_SPAN_EVENTS_QUEUE_SIZE sets InfiniteTracing.SpanEvents.QueueSize using strconv.Atoi +// NEW_RELIC_INFINITE_TRACING_TRACE_OBSERVER_PORT sets InfiniteTracing.TraceObserver.Port using strconv.Atoi +// NEW_RELIC_INFINITE_TRACING_TRACE_OBSERVER_HOST sets InfiniteTracing.TraceObserver.Host +// NEW_RELIC_LABELS sets Labels using a semi-colon delimited string of colon-separated pairs, eg. "Server:One;DataCenter:Primary" +// NEW_RELIC_LICENSE_KEY sets License +// NEW_RELIC_LOG sets Logger to log to either "stdout" or "stderr" (filenames are not supported) +// NEW_RELIC_LOG_LEVEL controls the NEW_RELIC_LOG level, must be "debug" for debug, or empty for info +// NEW_RELIC_PROCESS_HOST_DISPLAY_NAME sets HostDisplayName +// NEW_RELIC_SECURITY_POLICIES_TOKEN sets SecurityPoliciesToken +// NEW_RELIC_UTILIZATION_BILLING_HOSTNAME sets Utilization.BillingHostname +// NEW_RELIC_UTILIZATION_LOGICAL_PROCESSORS sets Utilization.LogicalProcessors using strconv.Atoi +// NEW_RELIC_UTILIZATION_TOTAL_RAM_MIB sets Utilization.TotalRAMMIB using strconv.Atoi +// NEW_RELIC_APPLICATION_LOGGING_ENABLED sets ApplicationLogging.Enabled. Set to false to disable all application logging features. +// NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED sets ApplicationLogging.LogForwarding.Enabled. Set to false to disable in agent log forwarding. +// NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LABELS_ENABLED sets ApplicationLogging.LogForwarding.Labels.Enabled to enable sending application labels with forwarded logs. +// NEW_RELIC_APPLICATION_LOGGING_FORWARDING_LABELS_EXCLUDE sets ApplicationLogging.LogForwarding.Labels.Exclude to filter out a set of unwanted label types from the ones reported with logs. +// NEW_RELIC_APPLICATION_LOGGING_METRICS_ENABLED sets ApplicationLogging.Metrics.Enabled. Set to false to disable the collection of application log metrics. +// NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED sets ApplicationLogging.LocalDecoration.Enabled. Set to true to enable local log decoration. +// NEW_RELIC_APPLICATION_LOGGING_FORWARDING_MAX_SAMPLES_STORED sets ApplicationLogging.LogForwarding.Limit. Set to 0 to prevent captured logs from being forwarded. +// NEW_RELIC_AI_MONITORING_ENABLED sets AIMonitoring.Enabled +// NEW_RELIC_AI_MONITORING_STREAMING_ENABLED sets AIMonitoring.Streaming.Enabled +// NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED sets AIMonitoring.RecordContent.Enabled +// NEW_RELIC_PROFILING_ENABLED sets Profiling.Enabled +// NEW_RELIC_PROFILING_SAMPLE_INTERVAL_MS sets Profiling.Interval +// NEW_RELIC_PROFILING_CPU_REPORT_INTERVAL_MS sets Profiling.CPUReportInterval +// NEW_RELIC_PROFILING_CPU_SAMPLE_RATE_HZ sets Profiling.CPUSampleRateHz +// NEW_RELIC_PROFILING_CPU_BLOCK_RATE sets Profiling.BlockRate +// NEW_RELIC_PROFILING_CPU_MUTEX_RATE sets Profiling.MutexRate +// NEW_RELIC_PROFILING_WITH_SEGMENTS sets Profiling.WithSegments +// NEW_RELIC_PROFILING_INCLUDE="heap,cpu,..." sets Profiling.SelectedProfiles // // This function is strict and will assign Config.Error if any of the // environment variables cannot be parsed. @@ -522,14 +601,19 @@ func configFromEnvironment(getenv func(string) string) ConfigOption { } } } - assignInt := func(field *int, name string) { + assignIntOk := func(field *int, name string) bool { if env := getenv(name); env != "" { - if i, err := strconv.Atoi(env); nil != err { + if i, err := strconv.Atoi(env); err != nil { cfg.Error = fmt.Errorf("invalid %s value: %s", name, env) } else { *field = i + return true } } + return false + } + assignInt := func(field *int, name string) { + _ = assignIntOk(field, name) } assignString := func(field *string, name string) { if env := getenv(name); env != "" { @@ -639,6 +723,26 @@ func configFromEnvironment(getenv func(string) string) ConfigOption { cfg.Error = fmt.Errorf("invalid NEW_RELIC_LOG value %s", env) } } + + assignBool(&cfg.Profiling.Enabled, "NEW_RELIC_PROFILING_ENABLED") + assignBool(&cfg.Profiling.WithSegments, "NEW_RELIC_PROFILING_WITH_SEGMENTS") + + // This allows setting interval to 0 explicitly by environment variable while still + // allowing it to be defaulted by leaving it out of the environment altogether. + var intervalMS int + if assignIntOk(&intervalMS, "NEW_RELIC_PROFILING_SAMPLE_INTERVAL_MS") && intervalMS >= 0 { + cfg.Profiling.Interval = time.Duration(intervalMS) * time.Millisecond + } + var intervalCPU int + if assignIntOk(&intervalCPU, "NEW_RELIC_PROFILING_CPU_REPORT_INTERVAL_MS") && intervalCPU >= 0 { + cfg.Profiling.CPUReportInterval = time.Duration(intervalCPU) * time.Millisecond + } + if env := getenv("NEW_RELIC_PROFILING_INCLUDE"); env != "" { + cfg.Profiling.SelectedProfiles.FromStrings(strings.Split(env, ","), false) + } + assignInt(&cfg.Profiling.CPUSampleRateHz, "NEW_RELIC_PROFILING_CPU_SAMPLE_RATE_HZ") + assignInt(&cfg.Profiling.BlockRate, "NEW_RELIC_PROFILING_BLOCK_RATE") + assignInt(&cfg.Profiling.MutexRate, "NEW_RELIC_PROFILING_MUTEX_RATE") } } diff --git a/v3/newrelic/config_test.go b/v3/newrelic/config_test.go index 77b377000..638356cf0 100644 --- a/v3/newrelic/config_test.go +++ b/v3/newrelic/config_test.go @@ -17,6 +17,7 @@ import ( "github.com/newrelic/go-agent/v3/internal" "github.com/newrelic/go-agent/v3/internal/crossagent" "github.com/newrelic/go-agent/v3/internal/utilization" + "github.com/nsf/jsondiff" ) type labelsTestCase struct { @@ -213,6 +214,16 @@ func TestCopyConfigReferenceFieldsPresent(t *testing.T) { "Labels":{"zip":"zap"}, "Logger":"*logger.logFile", "ModuleDependencyMetrics":{"Enabled":true,"IgnoredPrefixes":null,"RedactIgnoredPrefixes":true}, + "Profiling": { + "BlockRate": 1, + "CPUReportInterval": 0, + "CPUSampleRateHz": 100, + "Enabled":false, + "Interval": 0, + "MutexRate": 1, + "SelectedProfiles": null, + "WithSegments": false + }, "RuntimeSampler":{"Enabled":true}, "SecurityPoliciesToken":"", "ServerlessMode":{ @@ -325,8 +336,13 @@ func TestCopyConfigReferenceFieldsPresent(t *testing.T) { } out := standardizeNumbers(string(js)) if out != expect { - t.Error(expect) - t.Error(out) + o := jsondiff.DefaultConsoleOptions() + o.SkipMatches = true + whatHappened, differences := jsondiff.Compare([]byte(expect), []byte(out), &o) + t.Errorf("Config fields not as expected: %s:", whatHappened) + t.Error(differences) + //t.Error(expect) + //t.Error(out) } } @@ -432,6 +448,16 @@ func TestCopyConfigReferenceFieldsAbsent(t *testing.T) { "Labels":null, "Logger":null, "ModuleDependencyMetrics":{"Enabled":true,"IgnoredPrefixes":null,"RedactIgnoredPrefixes":true}, + "Profiling": { + "BlockRate": 1, + "CPUReportInterval": 0, + "CPUSampleRateHz": 100, + "Enabled":false, + "Interval": 0, + "MutexRate": 1, + "SelectedProfiles": null, + "WithSegments": false + }, "RuntimeSampler":{"Enabled":true}, "SecurityPoliciesToken":"", "ServerlessMode":{ @@ -514,8 +540,13 @@ func TestCopyConfigReferenceFieldsAbsent(t *testing.T) { } out := standardizeNumbers(string(js)) if out != expect { - t.Error(expect) - t.Error(out) + o := jsondiff.DefaultConsoleOptions() + o.SkipMatches = true + whatHappened, differences := jsondiff.Compare([]byte(expect), []byte(out), &o) + t.Errorf("Config fields not as expected: %s:", whatHappened) + t.Error(differences) + //t.Error(expect) + //t.Error(out) } } diff --git a/v3/newrelic/internal_app.go b/v3/newrelic/internal_app.go index d187cd85c..ef2a53a13 100644 --- a/v3/newrelic/internal_app.go +++ b/v3/newrelic/internal_app.go @@ -69,6 +69,9 @@ type app struct { // high water mark alarms heapHighWaterMarkAlarms heapHighWaterMarkAlarmSet + // profiler + profiler profilerConfig + serverless *serverlessHarvest } @@ -470,6 +473,10 @@ func newApp(c config) *app { go runSampler(app, runtimeSamplerPeriod) } } + // for now run in it's own goroutine but we may move this up to the main process later + if app.config.Profiling.Enabled { + app.StartProfiler() + } } return app diff --git a/v3/newrelic/internal_app_test.go b/v3/newrelic/internal_app_test.go index 368bc8a4b..9b542ba2c 100644 --- a/v3/newrelic/internal_app_test.go +++ b/v3/newrelic/internal_app_test.go @@ -439,7 +439,7 @@ func TestAppProcess_ConnectChan_TraceObserverVariants(t *testing.T) { select { case <-testApp.app.shutdownComplete: - case <-time.After(2 * timeout): + case <-time.After(80 * timeout): t.Fatal("shutdown did not complete in time") } }) diff --git a/v3/newrelic/oom_monitor.go b/v3/newrelic/oom_monitor.go index 3050fa24a..c0b3fa7de 100644 --- a/v3/newrelic/oom_monitor.go +++ b/v3/newrelic/oom_monitor.go @@ -31,24 +31,24 @@ type heapHighWaterMarkAlarmSet struct { // with an interval less than or equal to 0 is equivalent to calling HeapHighWaterMarkAlarmDisable. // // If there was already a running heap monitor, this merely changes its sample interval time. -func (a *Application) HeapHighWaterMarkAlarmEnable(interval time.Duration) { - if a == nil || a.app == nil { +func (app *Application) HeapHighWaterMarkAlarmEnable(interval time.Duration) { + if app == nil || app.app == nil { return } if interval <= 0 { - a.HeapHighWaterMarkAlarmDisable() + app.HeapHighWaterMarkAlarmDisable() return } - a.app.heapHighWaterMarkAlarms.lock.Lock() - defer a.app.heapHighWaterMarkAlarms.lock.Unlock() - if a.app.heapHighWaterMarkAlarms.sampleTicker == nil { - a.app.heapHighWaterMarkAlarms.sampleTicker = time.NewTicker(interval) - a.app.heapHighWaterMarkAlarms.done = make(chan byte) - go a.app.heapHighWaterMarkAlarms.monitor() + app.app.heapHighWaterMarkAlarms.lock.Lock() + defer app.app.heapHighWaterMarkAlarms.lock.Unlock() + if app.app.heapHighWaterMarkAlarms.sampleTicker == nil { + app.app.heapHighWaterMarkAlarms.sampleTicker = time.NewTicker(interval) + app.app.heapHighWaterMarkAlarms.done = make(chan byte) + go app.app.heapHighWaterMarkAlarms.monitor() } else { - a.app.heapHighWaterMarkAlarms.sampleTicker.Reset(interval) + app.app.heapHighWaterMarkAlarms.sampleTicker.Reset(interval) } } @@ -74,40 +74,39 @@ func (as *heapHighWaterMarkAlarmSet) monitor() { } // HeapHighWaterMarkAlarmShutdown stops the monitoring goroutine and deallocates the entire -// monitoring completely. All alarms are canceled and disabled. -func (a *Application) HeapHighWaterMarkAlarmShutdown() { - if a == nil || a.app == nil { +// monitoring completely. All alarms are cancelled and disabled. +func (app *Application) HeapHighWaterMarkAlarmShutdown() { + if app == nil || app.app == nil { return } - a.app.heapHighWaterMarkAlarms.lock.Lock() - defer a.app.heapHighWaterMarkAlarms.lock.Unlock() - - if a.app.heapHighWaterMarkAlarms.sampleTicker != nil { - a.app.heapHighWaterMarkAlarms.sampleTicker.Stop() + app.app.heapHighWaterMarkAlarms.lock.Lock() + defer app.app.heapHighWaterMarkAlarms.lock.Unlock() + if app.app.heapHighWaterMarkAlarms.sampleTicker != nil { + app.app.heapHighWaterMarkAlarms.sampleTicker.Stop() } - if a.app.heapHighWaterMarkAlarms.done != nil { - a.app.heapHighWaterMarkAlarms.done <- 0 + if app.app.heapHighWaterMarkAlarms.done != nil { + app.app.heapHighWaterMarkAlarms.done <- 0 } - if a.app.heapHighWaterMarkAlarms.alarms != nil { - clear(a.app.heapHighWaterMarkAlarms.alarms) - a.app.heapHighWaterMarkAlarms.alarms = nil + if app.app.heapHighWaterMarkAlarms.alarms != nil { + clear(app.app.heapHighWaterMarkAlarms.alarms) + app.app.heapHighWaterMarkAlarms.alarms = nil } - a.app.heapHighWaterMarkAlarms.sampleTicker = nil + app.app.heapHighWaterMarkAlarms.sampleTicker = nil } // HeapHighWaterMarkAlarmDisable stops sampling the heap memory allocation started by // HeapHighWaterMarkAlarmEnable. It is safe to call even if HeapHighWaterMarkAlarmEnable was // never called or the alarms were already disabled. -func (a *Application) HeapHighWaterMarkAlarmDisable() { - if a == nil || a.app == nil { +func (app *Application) HeapHighWaterMarkAlarmDisable() { + if app == nil || app.app == nil { return } - a.app.heapHighWaterMarkAlarms.lock.Lock() - defer a.app.heapHighWaterMarkAlarms.lock.Unlock() - if a.app.heapHighWaterMarkAlarms.sampleTicker != nil { - a.app.heapHighWaterMarkAlarms.sampleTicker.Stop() + app.app.heapHighWaterMarkAlarms.lock.Lock() + defer app.app.heapHighWaterMarkAlarms.lock.Unlock() + if app.app.heapHighWaterMarkAlarms.sampleTicker != nil { + app.app.heapHighWaterMarkAlarms.sampleTicker.Stop() } } @@ -122,38 +121,38 @@ func (a *Application) HeapHighWaterMarkAlarmDisable() { // If HeapHighWaterMarkAlarmSet is called with the same memory limit as a previous call, the // supplied callback function will replace the one previously registered for that limit. If // the function is given as nil, then that memory limit alarm is removed from the list. -func (a *Application) HeapHighWaterMarkAlarmSet(limit uint64, f func(uint64, *runtime.MemStats)) { - if a == nil || a.app == nil { +func (app *Application) HeapHighWaterMarkAlarmSet(limit uint64, f func(uint64, *runtime.MemStats)) { + if app == nil || app.app == nil { return } - a.app.heapHighWaterMarkAlarms.lock.Lock() - defer a.app.heapHighWaterMarkAlarms.lock.Unlock() + app.app.heapHighWaterMarkAlarms.lock.Lock() + defer app.app.heapHighWaterMarkAlarms.lock.Unlock() - if a.app.heapHighWaterMarkAlarms.alarms == nil { - a.app.heapHighWaterMarkAlarms.alarms = make(map[uint64]func(uint64, *runtime.MemStats)) + if app.app.heapHighWaterMarkAlarms.alarms == nil { + app.app.heapHighWaterMarkAlarms.alarms = make(map[uint64]func(uint64, *runtime.MemStats)) } if f == nil { - delete(a.app.heapHighWaterMarkAlarms.alarms, limit) + delete(app.app.heapHighWaterMarkAlarms.alarms, limit) } else { - a.app.heapHighWaterMarkAlarms.alarms[limit] = f + app.app.heapHighWaterMarkAlarms.alarms[limit] = f } } // HeapHighWaterMarkAlarmClearAll removes all high water mark alarms from the memory monitor // set. -func (a *Application) HeapHighWaterMarkAlarmClearAll() { - if a == nil || a.app == nil { +func (app *Application) HeapHighWaterMarkAlarmClearAll() { + if app == nil || app.app == nil { return } - a.app.heapHighWaterMarkAlarms.lock.Lock() - defer a.app.heapHighWaterMarkAlarms.lock.Unlock() + app.app.heapHighWaterMarkAlarms.lock.Lock() + defer app.app.heapHighWaterMarkAlarms.lock.Unlock() - if a.app.heapHighWaterMarkAlarms.alarms == nil { + if app.app.heapHighWaterMarkAlarms.alarms == nil { return } - clear(a.app.heapHighWaterMarkAlarms.alarms) + clear(app.app.heapHighWaterMarkAlarms.alarms) } diff --git a/v3/newrelic/profiler.go b/v3/newrelic/profiler.go new file mode 100644 index 000000000..4343a88e9 --- /dev/null +++ b/v3/newrelic/profiler.go @@ -0,0 +1,908 @@ +// Copyright 2022 New Relic Corporation. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package newrelic + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "os" + "path" + "runtime" + "runtime/pprof" + "runtime/trace" + "strings" + "sync" + "time" + "unicode" + + xtrace "golang.org/x/exp/trace" + + "github.com/google/pprof/profile" +) + +const ( + ProfileCustomEventType = "Profile" + ProfileTypeAttributeName = "profile_type" + ProfileLanguageAttributeName = "language" + ProfileLanguageAttributeValue = "go" +) + +const ( + profileNilDest byte = iota + profileLocalFile + profileIngestOTEL + profileIngestMELT +) + +type profilerAuditRecord struct { + EventType string `json:"event_type"` + HarvestSeq int64 `json:"harvest_seq"` + SampleSeq int `json:"sample_seq"` + Reason string `json:"error,omitempty"` + Attributes int `json:"attr_count"` + RawData map[string]any `json:"raw_data,omitempty"` +} + +func auditQty(audit io.Writer, eventType string, harvestNumber int64, samples int) { + if audit != nil { + if b, jerr := json.Marshal(profilerAuditRecord{ + EventType: "INFO_QTY:" + eventType, + HarvestSeq: harvestNumber, + Attributes: samples, + }); jerr == nil { + audit.Write(b) + audit.Write([]byte{'\n'}) + } + } +} + +func auditLog(audit io.Writer, format string, data ...any) { + if audit != nil { + if b, jerr := json.Marshal(profilerAuditRecord{ + EventType: "INFO", + Reason: fmt.Sprintf(format, data...), + }); jerr == nil { + audit.Write(b) + audit.Write([]byte{'\n'}) + } + } +} + +func profilerError(a *app, audit io.Writer, eventType string, harvestSeq int64, err error, debug bool, format string, data ...any) { + if debug { + fmt.Printf("ERROR "+format, data...) + fmt.Println(err.Error()) + } else { + a.Error(fmt.Sprintf(format, data...), map[string]any{ + "event-type": ProfileCustomEventType, + ProfileTypeAttributeName: eventType, + "reason": err.Error(), + }) + if audit != nil { + auditError(audit, eventType, harvestSeq, err, format, data...) + } + } +} + +func auditError(audit io.Writer, eventType string, harvestSeq int64, e error, format string, data ...any) { + if audit != nil { + m := fmt.Sprintf(format, data...) + if b, jerr := json.Marshal(profilerAuditRecord{ + EventType: eventType, + HarvestSeq: harvestSeq, + Reason: fmt.Sprintf("%s: %v", m, e.Error()), + }); jerr == nil { + audit.Write(b) + audit.Write([]byte{'\n'}) + } + } +} + +type profilerConfig struct { + lock sync.RWMutex // protects creation of the ticker and access to map + segLock sync.RWMutex // protects access to segment list + sampleTicker *time.Ticker // once made, only read by monitor goroutine + cpuReportTicker *time.Ticker // once made, only read by monitor goroutine + isRunning bool + selected ProfilingType // which profiling types we've selected to report + auditFile *os.File // debugging audit file of profile data (nil for normal production runs) + done chan byte + outputDirectory string + ingestSwitch chan byte + outputSwitch chan string + switchResult chan error + activeSegments map[string]struct{} + blockRate int + mutexRate int + cpuSampleRateHz int +} + +func (p *profilerConfig) IsRunning() bool { + p.lock.RLock() + defer p.lock.RUnlock() + return p.isRunning +} + +func (p *profilerConfig) SetRunning(state bool) { + p.lock.Lock() + p.isRunning = state + p.lock.Unlock() +} + +func (a *app) StartProfiler() { + if a == nil { + return + } + + a.profiler.lock.Lock() + a.profiler.selected = a.config.Profiling.SelectedProfiles + a.profiler.blockRate = a.config.Profiling.BlockRate + a.profiler.mutexRate = a.config.Profiling.MutexRate + a.profiler.cpuSampleRateHz = a.config.Profiling.CPUSampleRateHz + a.profiler.lock.Unlock() + a.setProfileSampleInterval(a.config.Profiling.Interval) + a.setProfileCPUReportInterval(a.config.Profiling.CPUReportInterval) +} + +// AddSegmentToProfiler signals that a segment has started which the profiler should report as being +// in play during all subsequent samples until RemoveSegmentFromProfiler is called with the same segment +// name. If the ConfigProfilingWithSegments(true) option is set, this will automatically be called when +// txn.StartSegment is invoked, but if you start a custom segment in any other way, you'll need to +// call AddSegmentToProfiler manually yourself since otherwise the profiler won't be able to be told +// the segment name to track. +// +// Note that this assumes segment names are unique at any given point in the program's runtime. +func (app *Application) AddSegmentToProfiler(name string) { + app.app.profiler.segLock.Lock() + if app.app.profiler.activeSegments == nil { + app.app.profiler.activeSegments = make(map[string]struct{}) + } + app.app.profiler.activeSegments[name] = struct{}{} + app.app.profiler.segLock.Unlock() +} + +// The following are undocumented because they're intended for internal testing +// purposes. They open and close an audit log of the profile samples collected +// and reported for the profiler. + +func (app *Application) OpenProfileAuditLog(filename string) error { + var err error + if app == nil || app.app == nil { + return fmt.Errorf("nil application") + } + app.app.profiler.lock.Lock() + app.app.profiler.auditFile, err = os.Create(filename) + app.app.profiler.lock.Unlock() + return err +} + +func (app *Application) CloseProfileAuditLog() error { + var err error + if app == nil || app.app == nil { + return fmt.Errorf("nil application") + } + app.app.profiler.lock.Lock() + err = app.app.profiler.auditFile.Close() + app.app.profiler.auditFile = nil + app.app.profiler.lock.Unlock() + return err +} + +// RemoveSegmentFromProfiler signals that a segment has terminated and the profiler should stop +// tracking that segment name to collected samples. If the ConfigProfilingWithSegments(true) option is +// set, this will automatically be called when the segment's End method is invoked. +func (app *Application) RemoveSegmentFromProfiler(name string) { + app.app.profiler.segLock.Lock() + if app.app.profiler.activeSegments != nil { + delete(app.app.profiler.activeSegments, name) + } + app.app.profiler.segLock.Unlock() +} + +// ShutdownProfiler stops the collection and reporting of profile data and stops the +// monitor background goroutine. If the waitForShutdown parameter is true, it will block +// until the monitor goroutine has completed its final harvest of profile samples and fully +// shut down before returning. +func (app *Application) ShutdownProfiler(waitForShutdown bool) { + if app == nil || app.app == nil { + return + } + app.SetProfileSampleInterval(0) + app.app.profiler.done <- 0 + if waitForShutdown { + for app.app.profiler.IsRunning() { + time.Sleep(time.Millisecond * 100) + } + } +} + +// SetProfileCPUSampleRateHz adjusts the sample time for CPU profile data. +// Changing this value does not actually take effect until the next time the +// CPU profiler is restarted. This will be when it is explicitly started, or +// when app.ReportCPUProfileStats is called (either manually or via the periodic +// timer set in motion via the ConfigProfilingCPUReportInterval option). +func (app *Application) SetProfileCPUSampleRateHz(hz int) { + if app == nil || app.app == nil { + return + } + app.app.profiler.lock.Lock() + app.app.profiler.cpuSampleRateHz = hz + app.app.profiler.lock.Unlock() +} + +// SetProfileCPUReportInterval adjusts the pace at which we report the collected CPU profile data, just +// like the ConfigProfilingCPUReportInverval agent configuration option does, but this allows the value +// to be adjusted at runtime at will. Setting this to 0 stops the interruption of the CPU profiler, allowing +// it to run until explicitly stopped when the overall agent profiler is shut down. +func (app *Application) SetProfileCPUReportInterval(interval time.Duration) { + if app == nil || app.app == nil { + return + } + + app.app.setProfileCPUReportInterval(interval) +} + +func (app *app) setProfileCPUReportInterval(interval time.Duration) { + app.profiler.lock.Lock() + defer app.profiler.lock.Unlock() + + if interval <= 0 { + if app.profiler.cpuReportTicker != nil { + app.profiler.cpuReportTicker.Stop() + } + return + } + + if app.profiler.cpuReportTicker == nil { + app.profiler.cpuReportTicker = time.NewTicker(interval) + } else { + app.profiler.cpuReportTicker.Reset(interval) + } +} + +// SetProfileSampleInterval adjusts the sample time for profile data. +// If set to 0, the profiler is paused entirely, but its data are not deallocated +// nor are the profiles removed. Calling this method again with a positive interval +// resumes sampling again. +// +// This does not affect sample rates for CPU data. Use SetProfileCPUSampleInterval +// and/or SetProfileCPUReportingInterval for that instead. +func (app *Application) SetProfileSampleInterval(interval time.Duration) { + if app == nil || app.app == nil { + return + } + + app.app.setProfileSampleInterval(interval) +} + +// SetProfileOutputDirectory changes the destination for the profiler's output so that +// all further profile data will be written to disk files in the specified directory +// instead of being sent to an ingest backend endpoint. +// +// This can be useful when debugging locally, if you want to get local profile data, or +// if you want manual control over where profile data gets reported. +func (app *Application) SetProfileOutputDirectory(dirname string) error { + if app != nil && app.app != nil { + app.app.profiler.outputSwitch <- dirname + return <-app.app.profiler.switchResult + } + return fmt.Errorf("nil application") +} + +// SetProfileOutputOTEL changes the destination for the profiler's output so that +// all further profile data will be written to an OTEL-compatible profiling signal +// endpoint. + +// (future) + +/* func (app *Application) SetProfileOutputOTEL() error { + if app != nil && app.app != nil { + app.app.profiler.ingestSwitch <- profileIngestOTEL + return <-app.app.profiler.switchResult + } + return fmt.Errorf("nil application") +} +*/ + +// SetProfileOutputMELT changes the destination for the profiler's output so that +// all further profile data will be written to a New Relic MELT endpoint as custom +// log events +func (app *Application) SetProfileOutputMELT() error { + if app != nil && app.app != nil { + app.app.profiler.ingestSwitch <- profileIngestMELT + return <-app.app.profiler.switchResult + } + return fmt.Errorf("nil application") +} + +func (app *app) setProfileSampleInterval(interval time.Duration) { + app.profiler.lock.Lock() + defer app.profiler.lock.Unlock() + + if interval <= 0 { + if app.profiler.sampleTicker != nil { + app.profiler.sampleTicker.Stop() + } + return + } + + if app.profiler.sampleTicker == nil { + app.profiler.sampleTicker = time.NewTicker(interval) + app.profiler.done = make(chan byte) + app.profiler.ingestSwitch = make(chan byte) + app.profiler.outputSwitch = make(chan string) + app.profiler.switchResult = make(chan error) + go app.profiler.monitor(app) + } else { + app.profiler.sampleTicker.Reset(interval) + } +} + +func (pc *profilerConfig) isBlockSelected() bool { + return (pc.selected & ProfilingTypeBlock) != 0 +} + +func (pc *profilerConfig) isCPUSelected() bool { + return (pc.selected & ProfilingTypeCPU) != 0 +} + +func (pc *profilerConfig) isGoroutineSelected() bool { + return (pc.selected & ProfilingTypeGoroutine) != 0 +} + +func (pc *profilerConfig) isHeapSelected() bool { + return (pc.selected & ProfilingTypeHeap) != 0 +} + +func (pc *profilerConfig) isMutexSelected() bool { + return (pc.selected & ProfilingTypeMutex) != 0 +} + +func (pc *profilerConfig) isThreadCreateSelected() bool { + return (pc.selected & ProfilingTypeThreadCreate) != 0 +} + +func (pc *profilerConfig) isTraceSelected() bool { + return (pc.selected & ProfilingTypeTrace) != 0 +} + +func sanitizeProfileEventAttrs(attrs map[string]any) { + if len(attrs) > customEventAttributeLimit { + // too many attributes for an event; sacrifice some location names + for i := len(attrs) - 1; i >= 0 && len(attrs) > customEventAttributeLimit; i-- { + key := fmt.Sprintf("location.%d", i) + if _, exists := attrs[key]; exists { + delete(attrs, key) + } + } + } + if len(attrs) > customEventAttributeLimit { + // still too many? kill the span ids too, then, as a move of desperation... + for i := len(attrs) - 1; i >= 0 && len(attrs) > customEventAttributeLimit; i-- { + key := fmt.Sprintf("span.%d", i) + if _, exists := attrs[key]; exists { + delete(attrs, key) + } + } + } +} + +func (pc *profilerConfig) monitor(a *app) { + if pc == nil { + return + } + + pc.SetRunning(true) + defer pc.SetRunning(false) + + auditLog(pc.auditFile, "monitor started") + if pc.isBlockSelected() { + runtime.SetBlockProfileRate(pc.blockRate) + } + if pc.isMutexSelected() { + _ = runtime.SetMutexProfileFraction(pc.mutexRate) + } + + profileDestination := profileNilDest + var heap_f, goroutine_f, threadcreate_f, block_f, mutex_f, cpu_f, trace_f *os.File + var cpuData, traceData bytes.Buffer + var err error + var harvestNumber int64 + + reportBufferedTraceSamples := func(profileData *bytes.Buffer, eventType string, debug bool, audit io.Writer) { + var err error + var event xtrace.Event + + reader, err := xtrace.NewReader(profileData) + if err != nil { + profilerError(a, pc.auditFile, eventType, harvestNumber, err, debug, "cannot create trace reader") + return + } + sampleNumber := 0 + for { + event, err = reader.ReadEvent() + if err == io.EOF { + break + } + if err != nil { + profilerError(a, pc.auditFile, eventType, harvestNumber, err, debug, "error reading trace events") + return + } + attrs := map[string]any{ + "harvest_seq": harvestNumber, + "sample_seq": sampleNumber, + "kind": event.Kind().String(), + ProfileTypeAttributeName: eventType, + ProfileLanguageAttributeName: ProfileLanguageAttributeValue, + } + sampleNumber++ + + switch event.Kind() { + case xtrace.EventLog: + elog := event.Log() + attrs["task"] = uint64(elog.Task) + attrs["category"] = elog.Category + attrs["message"] = elog.Message + case xtrace.EventMetric: + em := event.Metric() + attrs["name"] = em.Name + if em.Value.Kind() == xtrace.ValueUint64 { + attrs["value"] = em.Value.Uint64() + } else if em.Value.Kind() == xtrace.ValueString { + attrs["value"] = em.Value.String() + } + + /* event.Kind().String() EventSync EventMetric EventLabel EventStackSample EventRangeBegin EventRangeActive EventRangeEnd EventTaskBegin EventTaskEnd EventRegionBegin EventRegionEnd EventLog EventStateTransition EventExperimental*/ + } + + pc.segLock.RLock() + if pc.activeSegments != nil { + segmentSeq := 0 + for segmentName, _ := range pc.activeSegments { + attrs[fmt.Sprintf("segment.%d", segmentSeq)] = segmentName + segmentSeq++ + } + } + pc.segLock.RUnlock() + sanitizeProfileEventAttrs(attrs) + if debug { + fmt.Printf("EVENT %s: %v\n", eventType, attrs) + } else { + if err = a.RecordCustomEvent(ProfileCustomEventType, attrs); err != nil { + profilerError(a, pc.auditFile, eventType, harvestNumber, err, debug, "unable to record profiling data as custom event") + } else if audit != nil { + // the custom event succeeded. add that to the audit trail too + if b, jerr := json.Marshal(profilerAuditRecord{ + EventType: eventType, + HarvestSeq: harvestNumber, + SampleSeq: sampleNumber, + }); jerr == nil { + audit.Write(b) + audit.Write([]byte{'\n'}) + } + } + } + } + } + reportBufferedProfileSamples := func(profileData *bytes.Buffer, eventType string, debug bool, audit io.Writer) { + var p *profile.Profile + if p, err = profile.ParseData(profileData.Bytes()); err == nil { + auditQty(audit, eventType, harvestNumber, len(p.Sample)) + for sampleNumber, sampleData := range p.Sample { + attrs := map[string]any{ + "harvest_seq": harvestNumber, + "sample_seq": sampleNumber, + ProfileTypeAttributeName: eventType, + ProfileLanguageAttributeName: ProfileLanguageAttributeValue, + } + for i, dataValue := range sampleData.Value { + attrs[normalizeAttrNameFromSampleValueType(p.SampleType[i].Type, p.SampleType[i].Unit)] = dataValue + } + pc.segLock.RLock() + if pc.activeSegments != nil { + segmentSeq := 0 + for segmentName, _ := range pc.activeSegments { + attrs[fmt.Sprintf("segment.%d", segmentSeq)] = segmentName + segmentSeq++ + } + } + pc.segLock.RUnlock() + for i, codeLoc := range sampleData.Location { + if codeLoc.Line != nil && len(codeLoc.Line) > 0 { + attrs[fmt.Sprintf("location.%d", i)] = fmt.Sprintf("%s:%d", codeLoc.Line[0].Function.Name, codeLoc.Line[0].Line) + } + } + attrs["time_ns"] = p.TimeNanos + attrs["duration_ns"] = p.DurationNanos + attrs[normalizeAttrNameFromSampleValueType("sample_period_"+p.PeriodType.Type, p.PeriodType.Unit)] = p.Period + sanitizeProfileEventAttrs(attrs) + if debug { + fmt.Printf("EVENT %s: %v\n", eventType, attrs) + } else { + if err = a.RecordCustomEvent(ProfileCustomEventType, attrs); err != nil { + a.Error("unable to record profiling data as custom event", map[string]any{ + "event-type": ProfileCustomEventType, + ProfileTypeAttributeName: eventType, + "reason": err.Error(), + }) + if audit != nil { + // add note in our audit record that we failed to record this sample + if b, jerr := json.Marshal(profilerAuditRecord{ + EventType: eventType, + HarvestSeq: harvestNumber, + SampleSeq: sampleNumber, + Reason: err.Error(), + }); jerr == nil { + audit.Write(b) + audit.Write([]byte{'\n'}) + } + } + } else if audit != nil { + // the custom event succeeded. add that to the audit trail too + if b, jerr := json.Marshal(profilerAuditRecord{ + EventType: eventType, + HarvestSeq: harvestNumber, + SampleSeq: sampleNumber, + }); jerr == nil { + audit.Write(b) + audit.Write([]byte{'\n'}) + } + } + } + } + } else { + if debug { + fmt.Printf("ERROR parsing %s: %v\n", eventType, err) + } else { + a.Error("unable to parse profiling data", map[string]any{ + "event-type": ProfileCustomEventType, + ProfileTypeAttributeName: eventType, + "reason": err.Error(), + }) + if audit != nil { + if b, jerr := json.Marshal(profilerAuditRecord{ + EventType: eventType, + HarvestSeq: harvestNumber, + Reason: err.Error(), + }); jerr == nil { + audit.Write(b) + audit.Write([]byte{'\n'}) + } + } + } + } + } + + closeLocalFiles := func() { + auditLog(pc.auditFile, "closeLocalFiles called") + if profileDestination == profileNilDest { + // no action needed + } else if profileDestination == profileLocalFile { + _ = heap_f.Close() + _ = goroutine_f.Close() + _ = threadcreate_f.Close() + _ = block_f.Close() + _ = mutex_f.Close() + if pc.isCPUSelected() { + pprof.StopCPUProfile() + _ = cpu_f.Close() + } + if pc.isTraceSelected() { + trace.Stop() + _ = trace_f.Close() + } + } else { + // we're sending to an ingest endpoint of some sort + if pc.isCPUSelected() { + pprof.StopCPUProfile() + reportBufferedProfileSamples(&cpuData, "ProfileCPU", false, pc.auditFile) + cpuData.Reset() + } + if pc.isTraceSelected() { + trace.Stop() + reportBufferedTraceSamples(&traceData, "ProfileTrace", false, pc.auditFile) + traceData.Reset() + } + } + profileDestination = profileNilDest + } + defer closeLocalFiles() + + for { + select { + // To prevent interthread contention without the need for mutexes, we use channels here + // to let user threads request switching profile output destinations here and only this + // monitor thread ever writes anything. + // + case newDestination := <-pc.ingestSwitch: + switch newDestination { + case profileIngestOTEL, profileIngestMELT, profileNilDest: + if profileDestination == profileLocalFile { + closeLocalFiles() + } + if pc.isCPUSelected() { + runtime.SetCPUProfileRate(pc.cpuSampleRateHz) + if err = pprof.StartCPUProfile(&cpuData); err != nil { + pc.switchResult <- err + return + } + } + if pc.isTraceSelected() { + if err = trace.Start(&traceData); err != nil { + pc.switchResult <- err + return + } + } + profileDestination = newDestination + pc.switchResult <- nil + default: + pc.switchResult <- fmt.Errorf("Invalid profile destination code %v", newDestination) + } + + case newDirectory := <-pc.outputSwitch: + var err error + + pc.outputDirectory = newDirectory + closeLocalFiles() + if pc.isHeapSelected() { + if heap_f, err = os.OpenFile(path.Join(newDirectory, "heap.pprof"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); err != nil { + pc.switchResult <- err + return + } + } + if pc.isGoroutineSelected() { + if goroutine_f, err = os.OpenFile(path.Join(newDirectory, "goroutine.pprof"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); err != nil { + pc.switchResult <- err + return + } + } + if pc.isThreadCreateSelected() { + if threadcreate_f, err = os.OpenFile(path.Join(newDirectory, "threadcreate.pprof"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); err != nil { + pc.switchResult <- err + return + } + } + if pc.isBlockSelected() { + if block_f, err = os.OpenFile(path.Join(newDirectory, "block.pprof"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); err != nil { + pc.switchResult <- err + return + } + } + if pc.isMutexSelected() { + if mutex_f, err = os.OpenFile(path.Join(newDirectory, "mutex.pprof"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); err != nil { + pc.switchResult <- err + return + } + } + if pc.isTraceSelected() { + if trace_f, err = os.OpenFile(path.Join(newDirectory, "trace.pprof"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); err != nil { + pc.switchResult <- err + return + } + if err = trace.Start(trace_f); err != nil { + pc.switchResult <- err + return + } + } + if pc.isCPUSelected() { + if cpu_f, err = os.OpenFile(path.Join(newDirectory, "cpu.pprof"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); err != nil { + pc.switchResult <- err + return + } + runtime.SetCPUProfileRate(pc.cpuSampleRateHz) + if err = pprof.StartCPUProfile(cpu_f); err != nil { + pc.switchResult <- err + return + } + } + profileDestination = profileLocalFile + pc.switchResult <- nil + + case <-pc.cpuReportTicker.C: + if pc.isCPUSelected() { + if profileDestination == profileNilDest { + // nothing to do here + } else { + // shut down the profiler, let it report out + pprof.StopCPUProfile() + runtime.SetCPUProfileRate(pc.cpuSampleRateHz) + if profileDestination == profileLocalFile { + // cycle to a new destination file + _ = cpu_f.Close() + if cpu_f, err = os.OpenFile(path.Join(pc.outputDirectory, fmt.Sprintf("cpu.pprof%v", harvestNumber)), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); err != nil { + a.Error("error restarting CPU profiler", map[string]any{ + "event-type": "ProfileCPU", + "filename": fmt.Sprintf("cpu.pprof%v", harvestNumber), + "operation": "OpenFile", + "reason": err.Error(), + }) + pc.selected &= ^ProfilingTypeCPU + } else if err = pprof.StartCPUProfile(cpu_f); err != nil { + a.Error("error restarting CPU profiler", map[string]any{ + "event-type": "ProfileCPU", + "operation": "StartCPUProfile", + "reason": err.Error(), + }) + pc.selected &= ^ProfilingTypeCPU + } + } else { + // report to ingest endpoint + reportBufferedProfileSamples(&cpuData, "ProfileCPU", false, pc.auditFile) + cpuData.Reset() + if err = pprof.StartCPUProfile(&cpuData); err != nil { + a.Error("error restarting CPU profiler", map[string]any{ + "event-type": "ProfileCPU", + "operation": "StartCPUProfile", + "reason": err.Error(), + }) + pc.selected &= ^ProfilingTypeCPU + } + } + } + harvestNumber++ + } + + case <-pc.sampleTicker.C: + if profileDestination == profileNilDest { + continue + } + + if profileDestination == profileLocalFile { + if pc.isHeapSelected() { + pprof.Lookup("heap").WriteTo(heap_f, 1) + } + if pc.isGoroutineSelected() { + pprof.Lookup("goroutine").WriteTo(goroutine_f, 1) + } + if pc.isThreadCreateSelected() { + pprof.Lookup("threadcreate").WriteTo(threadcreate_f, 1) + } + if pc.isBlockSelected() { + pprof.Lookup("block").WriteTo(block_f, 1) + } + if pc.isMutexSelected() { + pprof.Lookup("mutex").WriteTo(mutex_f, 1) + } + // The tracer writes to the file on its own, as does the cpu profiler, + // so we don't need to do anything for them here. + } else { + // Otherwise, we need to process the profile data internally and report it out somewhere. + reportProfileSample := func(profileName, eventType string, debug bool, audit io.Writer) { + var data bytes.Buffer + pprof.Lookup(profileName).WriteTo(&data, 0) + var p *profile.Profile + if p, err = profile.ParseData(data.Bytes()); err == nil { + auditQty(audit, eventType, harvestNumber, len(p.Sample)) + for sampleNumber, sampleData := range p.Sample { + attrs := map[string]any{ + "harvest_seq": harvestNumber, + "sample_seq": sampleNumber, + ProfileTypeAttributeName: eventType, + ProfileLanguageAttributeName: ProfileLanguageAttributeValue, + } + for i, dataValue := range sampleData.Value { + attrs[normalizeAttrNameFromSampleValueType(p.SampleType[i].Type, p.SampleType[i].Unit)] = dataValue + } + pc.segLock.RLock() + if pc.activeSegments != nil { + segmentSeq := 0 + for segmentName, _ := range pc.activeSegments { + attrs[fmt.Sprintf("segment.%d", segmentSeq)] = segmentName + segmentSeq++ + } + } + pc.segLock.RUnlock() + for i, codeLoc := range sampleData.Location { + if codeLoc.Line != nil && len(codeLoc.Line) > 0 { + attrs[fmt.Sprintf("location.%d", i)] = fmt.Sprintf("%s:%d", codeLoc.Line[0].Function.Name, codeLoc.Line[0].Line) + } + } + attrs["time_ns"] = p.TimeNanos + attrs["duration_ns"] = p.DurationNanos + attrs[normalizeAttrNameFromSampleValueType("sample_period_"+p.PeriodType.Type, p.PeriodType.Unit)] = p.Period + sanitizeProfileEventAttrs(attrs) + if debug { + fmt.Printf("EVENT %s: %v\n", eventType, attrs) + } else { + if err = a.RecordCustomEvent(ProfileCustomEventType, attrs); err != nil { + a.Error("unable to record "+eventType+" profiling data as custom event", map[string]any{ + "event-type": ProfileCustomEventType, + ProfileTypeAttributeName: eventType, + "reason": err.Error(), + }) + if audit != nil { + // add not in our audit record that we failed to record this sample + if b, jerr := json.Marshal(profilerAuditRecord{ + EventType: eventType, + HarvestSeq: harvestNumber, + SampleSeq: sampleNumber, + Reason: err.Error(), + Attributes: len(attrs), + RawData: attrs, + }); jerr == nil { + audit.Write(b) + audit.Write([]byte{'\n'}) + } + } + } else if audit != nil { + // the custom event succeeded. add that to the audit trail too + if b, jerr := json.Marshal(profilerAuditRecord{ + EventType: eventType, + HarvestSeq: harvestNumber, + SampleSeq: sampleNumber, + Attributes: len(attrs), + RawData: attrs, + }); jerr == nil { + audit.Write(b) + audit.Write([]byte{'\n'}) + } + } + } + } + } else { + if debug { + fmt.Printf("ERROR parsing %s: %v\n", eventType, err) + } else { + a.Error("unable to parse "+eventType+" profiling data", map[string]any{ + "event-type": ProfileCustomEventType, + ProfileTypeAttributeName: eventType, + "reason": err.Error(), + }) + if audit != nil { + if b, jerr := json.Marshal(profilerAuditRecord{ + EventType: eventType, + HarvestSeq: harvestNumber, + Reason: err.Error(), + }); jerr == nil { + audit.Write(b) + audit.Write([]byte{'\n'}) + } + } + } + } + } + + if pc.isHeapSelected() { + reportProfileSample("heap", "ProfileHeap", false, pc.auditFile) + } + if pc.isGoroutineSelected() { + reportProfileSample("goroutine", "ProfileGoroutine", false, pc.auditFile) + } + if pc.isThreadCreateSelected() { + reportProfileSample("threadcreate", "ProfileThreadCreate", false, pc.auditFile) + } + if pc.isBlockSelected() { + reportProfileSample("block", "ProfileBlock", false, pc.auditFile) + } + if pc.isMutexSelected() { + reportProfileSample("mutex", "ProfileMutex", false, pc.auditFile) + } + // We don't get trace data until we stop the profiler (but we can use the flight recorder instead + // if we need that) + harvestNumber++ + } + case <-pc.done: + // We were told to terminate our profile monitoring + auditLog(pc.auditFile, "monitor stopped") + return + } + } +} + +func normalizeAttrNameFromSampleValueType(typeName, unitName string) string { + return strings.Map(func(s rune) rune { + if unicode.IsSpace(s) { + return '_' + } + if unicode.IsLetter(s) { + return s + } + if s == '_' { + return s + } + return -1 + }, strings.ToLower(typeName+"_"+unitName)) +} diff --git a/v3/newrelic/segments.go b/v3/newrelic/segments.go index 328db4283..ed45db74e 100644 --- a/v3/newrelic/segments.go +++ b/v3/newrelic/segments.go @@ -184,6 +184,13 @@ func (s *Segment) End() { "name": s.Name, }) } + + if s.StartTime.thread != nil && s.StartTime.thread.txn != nil { + conf, ok := s.StartTime.thread.txn.Application().Config() + if ok && conf.Profiling.Enabled && conf.Profiling.WithSegments { + s.StartTime.thread.txn.Application().RemoveSegmentFromProfiler(s.Name) + } + } } // AddAttribute adds a key value pair to the current DatastoreSegment. diff --git a/v3/newrelic/transaction.go b/v3/newrelic/transaction.go index fa3b785bb..f2ce4e5c2 100644 --- a/v3/newrelic/transaction.go +++ b/v3/newrelic/transaction.go @@ -356,6 +356,11 @@ func (txn *Transaction) StartSegment(name string) *Segment { // async segment start secureAgent.SendEvent("NEW_GOROUTINE_LINKER", txn.thread.getCsecData()) } + + conf, ok := txn.Application().Config() + if ok && conf.Profiling.Enabled && conf.Profiling.WithSegments { + txn.Application().AddSegmentToProfiler(name) + } return &Segment{ StartTime: txn.StartSegmentNow(), Name: name,