From 900ef7172fabcf21c0556a04d36e39e8a970d284 Mon Sep 17 00:00:00 2001 From: "erwan.viollet" Date: Thu, 2 Jul 2026 17:37:09 +0200 Subject: [PATCH 1/6] Add JFR support and java_cpu_jfr scenario MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the ability to parse Java Flight Recorder (.jfr) files as part of the correctness analysis pipeline. analysis/jfr.go: - parseJFR(): converts raw JFR bytes to a map of metric name → *profile.Profile, using github.com/grafana/jfr-parser/parser directly (no pyroscope/api dependency). - convertJFRFiles(): called at the start of AnalyzeResults; walks the output directory for *.jfr files and writes per-metric pprof files (e.g. profile.jfr → profile_cpu.pprof, profile_lock.pprof, …). Existing pprof analysis code then picks these up via the normal filename-regex matching. Supported JFR event types: jdk.ExecutionSample (cpu/wall), jdk.ObjectAllocationInNewTLAB, jdk.ObjectAllocationOutsideTLAB, jdk.JavaMonitorEnter, and the Datadog WallClockSample extension. scenarios/java_cpu_jfr: - Simple Fibonacci CPU burner recorded with built-in JDK JFR (eclipse-temurin:21-jdk, settings=profile, dumponexit=true). - Asserts ≥80% of CPU samples contain DummyApp.fibonacci. - EXECUTION_TIME_SEC env var honoured (default 30 s). --- analysis/analysis.go | 5 + analysis/jfr.go | 231 +++++++++++++++++++ correctness_test.go | 4 + go.mod | 3 +- go.sum | 53 ++++- scenarios/java_cpu_jfr/Dockerfile | 22 ++ scenarios/java_cpu_jfr/DummyApp.java | 33 +++ scenarios/java_cpu_jfr/expected_profile.json | 17 ++ 8 files changed, 362 insertions(+), 6 deletions(-) create mode 100644 analysis/jfr.go create mode 100644 scenarios/java_cpu_jfr/Dockerfile create mode 100644 scenarios/java_cpu_jfr/DummyApp.java create mode 100644 scenarios/java_cpu_jfr/expected_profile.json diff --git a/analysis/analysis.go b/analysis/analysis.go index 6978b51..1fe9c2e 100644 --- a/analysis/analysis.go +++ b/analysis/analysis.go @@ -662,7 +662,12 @@ func AnalyzePprofFile(r Reporter, pprofFile string, typedStacks TypedStacks, tes // AnalyzeResults loads the expected_profile.json at jsonFilePath and asserts // every pprof file under pprofFolder matches it. Failures are reported via r. +// Any *.jfr files in pprofFolder are converted to per-metric pprof files +// before the analysis loop runs. func AnalyzeResults(r Reporter, jsonFilePath string, pprofFolder string) { + // Convert any JFR files to pprof before the analysis loop. + convertJFRFiles(r, pprofFolder) + stackTestData, err := ReadJSONFile(jsonFilePath) if err != nil { r.Fatalf("Error opening file %s: %v", jsonFilePath, err) diff --git a/analysis/jfr.go b/analysis/jfr.go new file mode 100644 index 0000000..10affef --- /dev/null +++ b/analysis/jfr.go @@ -0,0 +1,231 @@ +// Package analysis — JFR support. +// +// This file adds the ability to read JFR (Java Flight Recorder) files and +// convert them to pprof profiles that the rest of the analysis pipeline can +// consume. +// +// Flow: +// +// 1. At the start of AnalyzeResults, convertJFRFiles walks the output folder +// for any *.jfr files. +// 2. Each JFR file is parsed with github.com/grafana/jfr-parser/parser. +// 3. Per-metric pprof profiles are written as _.pprof alongside +// the source JFR file (e.g. profile.jfr → profile_cpu.pprof). +// 4. The normal pprof analysis loop then picks those up via the usual +// filename regex. +// +// Supported JFR metrics (= pprof profile-type names): +// - "cpu" : jdk.ExecutionSample (non-sleeping threads) +// - "wall" : jdk.ExecutionSample when event=wall, or +// Datadog WallClockSample +// - "alloc_in_tlab" : jdk.ObjectAllocationInNewTLAB +// - "alloc_outside_tlab" : jdk.ObjectAllocationOutsideTLAB +// - "lock" : jdk.JavaMonitorEnter +package analysis + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/google/pprof/profile" + "github.com/grafana/jfr-parser/parser" + "github.com/grafana/jfr-parser/parser/types" +) + +// parseJFR converts raw JFR bytes into a map of metric name → pprof profile. +func parseJFR(data []byte) (map[string]*profile.Profile, error) { + p := parser.NewParser(data, parser.Options{ + SymbolProcessor: parser.ProcessSymbols, + }) + + type builder struct { + prof *profile.Profile + mapping *profile.Mapping + funcByID map[types.MethodRef]*profile.Function + locByID map[types.MethodRef]*profile.Location + } + + builders := make(map[string]*builder) + + getBuilder := func(metric, sampleType, sampleUnit string) *builder { + b, ok := builders[metric] + if !ok { + m := &profile.Mapping{ID: 1, HasFunctions: true} + b = &builder{ + prof: &profile.Profile{ + SampleType: []*profile.ValueType{{Type: sampleType, Unit: sampleUnit}}, + PeriodType: &profile.ValueType{Type: sampleType, Unit: "nanoseconds"}, + Period: 10_000_000, // default 100 Hz + Mapping: []*profile.Mapping{m}, + }, + mapping: m, + funcByID: make(map[types.MethodRef]*profile.Function), + locByID: make(map[types.MethodRef]*profile.Location), + } + builders[metric] = b + } + return b + } + + // resolveFrameName returns "ClassName.methodName" for a JFR method reference. + // JVM internal '/' separators in class names are normalised to '.'. + resolveFrameName := func(methodRef types.MethodRef) string { + m := p.GetMethod(methodRef) + if m == nil { + return "" + } + methodName := p.GetSymbolString(m.Name) + cls := p.GetClass(m.Type) + if cls == nil { + return methodName + } + clsName := strings.ReplaceAll(p.GetSymbolString(cls.Name), "/", ".") + return clsName + "." + methodName + } + + // addSample appends one stack-trace observation to the named metric profile. + addSample := func(metric, sampleType, sampleUnit string, stackRef types.StackTraceRef, count int64) { + st := p.GetStacktrace(stackRef) + if st == nil || len(st.Frames) == 0 { + return + } + b := getBuilder(metric, sampleType, sampleUnit) + + // JFR frames[0] = leaf (top of stack), which matches the pprof convention + // that sample.Location[0] is the leaf. + locs := make([]*profile.Location, 0, len(st.Frames)) + for _, frame := range st.Frames { + loc, ok := b.locByID[frame.Method] + if !ok { + fnName := resolveFrameName(frame.Method) + if fnName == "" { + continue + } + fn, fnOK := b.funcByID[frame.Method] + if !fnOK { + fn = &profile.Function{ + ID: uint64(len(b.prof.Function) + 1), + Name: fnName, + } + b.prof.Function = append(b.prof.Function, fn) + b.funcByID[frame.Method] = fn + } + loc = &profile.Location{ + ID: uint64(len(b.prof.Location) + 1), + Mapping: b.mapping, + Line: []profile.Line{{Function: fn}}, + } + b.prof.Location = append(b.prof.Location, loc) + b.locByID[frame.Method] = loc + } + locs = append(locs, loc) + } + if len(locs) == 0 { + return + } + b.prof.Sample = append(b.prof.Sample, &profile.Sample{ + Location: locs, + Value: []int64{count}, + }) + } + + var event string + for { + typ, err := p.ParseEvent() + if err == io.EOF { + break + } + if err != nil { + // Non-fatal: a truncated JFR file (e.g. from dumponexit=true) may end + // mid-chunk. Return what we have so far plus the error description. + return nil, fmt.Errorf("jfr ParseEvent: %w", err) + } + + switch typ { + case p.TypeMap.T_EXECUTION_SAMPLE: + ts := p.GetThreadState(p.ExecutionSample.State) + if ts != nil && ts.Name != "STATE_SLEEPING" { + addSample("cpu", "cpu", "samples", p.ExecutionSample.StackTrace, 1) + } + if event == "wall" { + addSample("wall", "wall", "samples", p.ExecutionSample.StackTrace, 1) + } + case p.TypeMap.T_WALL_CLOCK_SAMPLE: + addSample("wall", "wall", "samples", + p.WallClockSample.StackTrace, int64(p.WallClockSample.Samples)) + case p.TypeMap.T_ALLOC_IN_NEW_TLAB: + addSample("alloc_in_tlab", "alloc_in_new_tlab_objects", "count", + p.ObjectAllocationInNewTLAB.StackTrace, 1) + case p.TypeMap.T_ALLOC_OUTSIDE_TLAB: + addSample("alloc_outside_tlab", "alloc_outside_tlab_objects", "count", + p.ObjectAllocationOutsideTLAB.StackTrace, 1) + case p.TypeMap.T_MONITOR_ENTER: + addSample("lock", "contentions", "count", + p.JavaMonitorEnter.StackTrace, 1) + case p.TypeMap.T_ACTIVE_SETTING: + if p.ActiveSetting.Name == "event" { + event = p.ActiveSetting.Value + } + } + } + + result := make(map[string]*profile.Profile, len(builders)) + for metric, b := range builders { + result[metric] = b.prof + } + return result, nil +} + +// convertJFRFiles walks dir for *.jfr files and converts each to a set of +// per-metric pprof files written into the same directory. +// Output files are named _.pprof (e.g. profile_cpu.pprof). +// Errors are non-fatal: they are logged through r and the function continues. +func convertJFRFiles(r Reporter, dir string) { + entries, err := os.ReadDir(dir) + if err != nil { + r.Logf("convertJFRFiles: reading dir %s: %v", dir, err) + return + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".jfr") { + continue + } + jfrPath := filepath.Join(dir, entry.Name()) + stem := strings.TrimSuffix(entry.Name(), ".jfr") + + data, err := os.ReadFile(jfrPath) + if err != nil { + r.Logf("convertJFRFiles: reading %s: %v", jfrPath, err) + continue + } + + profiles, err := parseJFR(data) + if err != nil { + r.Logf("convertJFRFiles: parsing %s: %v", jfrPath, err) + continue + } + if len(profiles) == 0 { + r.Logf("convertJFRFiles: no profiles found in %s", jfrPath) + continue + } + + for metric, prof := range profiles { + outPath := filepath.Join(dir, stem+"_"+metric+".pprof") + var buf bytes.Buffer + if err := prof.Write(&buf); err != nil { + r.Logf("convertJFRFiles: serialising %s metric %s: %v", entry.Name(), metric, err) + continue + } + if err := os.WriteFile(outPath, buf.Bytes(), 0644); err != nil { + r.Logf("convertJFRFiles: writing %s: %v", outPath, err) + continue + } + r.Logf("Converted JFR %s → %s (%d samples)", + entry.Name(), filepath.Base(outPath), len(prof.Sample)) + } + } +} diff --git a/correctness_test.go b/correctness_test.go index 6d53a08..995c504 100644 --- a/correctness_test.go +++ b/correctness_test.go @@ -75,6 +75,10 @@ func TestDDProfScenarios(t *testing.T) { testScenarios(t, ".*ddprof.*") } +func TestJFRScenarios(t *testing.T) { + testScenarios(t, ".*jfr.*") +} + func TestPHPScenarios(t *testing.T) { testScenarios(t, ".*php.*") } diff --git a/go.mod b/go.mod index 68646b7..dbf1f12 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,8 @@ module github.com/DataDog/prof-correctness go 1.25.1 require ( - github.com/google/pprof v0.0.0-20240528025155-186aa0362fba + github.com/google/pprof v0.0.0-20260507013755-92041b743c96 + github.com/grafana/jfr-parser v0.17.1 github.com/klauspost/compress v1.18.4 github.com/pierrec/lz4/v4 v4.1.25 github.com/xeipuuv/gojsonschema v1.2.0 diff --git a/go.sum b/go.sum index e75b789..aaef178 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,24 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240528025155-186aa0362fba h1:ql1qNgCyOB7iAEk8JTNM+zJrgIbnyCKX/wdlyPufP5g= -github.com/google/pprof v0.0.0-20240528025155-186aa0362fba/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20260507013755-92041b743c96 h1:YDDnaZ9afWajDboPMt9Vikqca/yWAX7KAxVzb4lJU1M= +github.com/google/pprof v0.0.0-20260507013755-92041b743c96/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grafana/jfr-parser v0.17.1 h1:dTECCXL+B9V6qu5x4msSKE9/FJPKeK0jrwLa5fkpkCA= +github.com/grafana/jfr-parser v0.17.1/go.mod h1:+4zCC+tEWot6oQWjC72bG9TDDBiiiawMprp4EM1BioU= github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -18,23 +33,47 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/collector/featuregate v1.62.0 h1:pYY7RlulSCTOS9mFWxasMLwYJCfNXHtnOkZlv3jg/V4= go.opentelemetry.io/collector/featuregate v1.62.0/go.mod h1:4ga1QBMPEejXXmpyJS8lmaRpknJ3Lb9Bvk6e420bUFU= +go.opentelemetry.io/collector/internal/testutil v0.156.0 h1:Nu02vhHA2UQ3Yjyjisk3N24HHxwvw7PQiTz9O1PuiUY= +go.opentelemetry.io/collector/internal/testutil v0.156.0/go.mod h1:Jkjs6rkqs973LqgZ0Fe3zrokQRKULYXPIf4HuqStiEE= go.opentelemetry.io/collector/pdata v1.62.0 h1:xGdwl2Cs5Rq5nKs0nYvAxm3Qq20HcySVAmUElATS8Es= go.opentelemetry.io/collector/pdata v1.62.0/go.mod h1:WFy5R6XGpz2Q4MaekeEm+qc4GY5V3+BhQIwGPkp+fj0= go.opentelemetry.io/collector/pdata/pprofile v0.156.0 h1:TnQzA2d5iMGH5//mGLqPjwdYqsFD/A7o2WgDdppxdVM= go.opentelemetry.io/collector/pdata/pprofile v0.156.0/go.mod h1:3dtjs/mliblJJCCTXUE0AkpBNfBEybPruj3ml6WCOoI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.opentelemetry.io/proto/slim/otlp v1.10.0 h1:iR97Vs/ZDR+y9TfuP9b1XBtdPWeC+OMslIBmhcLU7jM= +go.opentelemetry.io/proto/slim/otlp v1.10.0/go.mod h1:lV9250stpjYLPNA5viFabIgP2QlUGRT1GdTgAf8SIUk= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0 h1:RUF5rO0hAlgiJt1fzQVzcVs3vZVNHIcMLgOgG4rWNcQ= +go.opentelemetry.io/proto/slim/otlp/collector/profiles/v1development v0.3.0/go.mod h1:I89cynRj8y+383o7tEQVg2SVA6SRgDVIouWPUVXjx0U= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0 h1:CQvJSldHRUN6Z8jsUeYv8J0lXRvygALXIzsmAeCcZE0= +go.opentelemetry.io/proto/slim/otlp/profiles/v1development v0.3.0/go.mod h1:xSQ+mEfJe/GjK1LXEyVOoSI1N9JV9ZI923X5kup43W4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= @@ -43,9 +82,13 @@ golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scenarios/java_cpu_jfr/Dockerfile b/scenarios/java_cpu_jfr/Dockerfile new file mode 100644 index 0000000..9b21030 --- /dev/null +++ b/scenarios/java_cpu_jfr/Dockerfile @@ -0,0 +1,22 @@ +FROM eclipse-temurin:21-jdk + +WORKDIR /app + +# Copy and compile the Java source +COPY ./scenarios/java_cpu_jfr/DummyApp.java . +RUN javac DummyApp.java + +# Force re-run of the next layer when the cache date changes. +ARG CACHE_DATE=2024-01-01 + +# EXECUTION_TIME_SEC is injected by the test harness; default to 30 s. +ENV EXECUTION_TIME_SEC=30 + +# Run with built-in JFR. +# -XX:+FlightRecorder : enable JFR (no-op on JDK 11+ where it's always on) +# -XX:StartFlightRecording=... : start a continuous recording; dump on JVM exit. +# settings=profile : enable CPU/alloc/lock profiling at low overhead. +CMD java \ + -XX:+FlightRecorder \ + "-XX:StartFlightRecording=filename=/app/data/profile.jfr,dumponexit=true,settings=profile,name=correctness" \ + DummyApp diff --git a/scenarios/java_cpu_jfr/DummyApp.java b/scenarios/java_cpu_jfr/DummyApp.java new file mode 100644 index 0000000..355a339 --- /dev/null +++ b/scenarios/java_cpu_jfr/DummyApp.java @@ -0,0 +1,33 @@ +/** + * Simple CPU-intensive app for JFR correctness testing. + * + * Continuously computes Fibonacci numbers to keep the CPU busy. + * The JFR recording is started by the JVM flags passed on the command line; + * this app just runs for the requested duration and exits. + * + * The EXECUTION_TIME_SEC environment variable (default: 30) controls how long + * the app runs, matching the convention used by prof-correctness. + */ +public class DummyApp { + public static void main(String[] args) throws InterruptedException { + int seconds = 30; + String env = System.getenv("EXECUTION_TIME_SEC"); + if (env != null && !env.isEmpty()) { + seconds = Integer.parseInt(env.trim()); + } + System.out.println("Running DummyApp for " + seconds + " seconds"); + + long endMs = System.currentTimeMillis() + seconds * 1000L; + int n = 40; // fib(40) takes ~0.4 s per call, keeps CPU fully busy + while (System.currentTimeMillis() < endMs) { + fibonacci(n); + } + System.out.println("DummyApp finished: " + fibonacci(n)); + } + + /** Intentionally naive recursive Fibonacci — the hot function we assert on. */ + static long fibonacci(int n) { + if (n <= 1) return n; + return fibonacci(n - 1) + fibonacci(n - 2); + } +} diff --git a/scenarios/java_cpu_jfr/expected_profile.json b/scenarios/java_cpu_jfr/expected_profile.json new file mode 100644 index 0000000..914d809 --- /dev/null +++ b/scenarios/java_cpu_jfr/expected_profile.json @@ -0,0 +1,17 @@ +{ + "test_name": "java_cpu_jfr", + "pprof-regex": ".*_cpu\\.pprof$", + "stacks": [ + { + "profile-type": "cpu", + "stack-content": [ + { + "regular_expression": ".*DummyApp\\.fibonacci.*", + "percent": 95, + "error_margin": 15 + } + ] + } + ], + "scale_by_duration": false +} From 9bd5b68b0f90c8c480a55180509cba38f26e8dc4 Mon Sep 17 00:00:00 2001 From: "erwan.viollet" Date: Fri, 3 Jul 2026 12:02:26 +0200 Subject: [PATCH 2/6] ci: add java job for JFR scenarios --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe68c7c..0108d91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,11 @@ jobs: with: test_scenarios: python.* secrets: inherit + java: + uses: ./.github/workflows/test.yml + with: + test_scenarios: java.* + secrets: inherit full_host: uses: ./.github/workflows/test.yml with: From 1e07eb34c2c32962dd3dfebc0f65f3d75fa68dfc Mon Sep 17 00:00:00 2001 From: "erwan.viollet" Date: Sat, 25 Jul 2026 16:57:33 +0200 Subject: [PATCH 3/6] Generalize JFR pprof conversion --- analysis/jfr.go | 169 ++++++--------------------------------- analysis/jfr_mappings.go | 78 ++++++++++++++++++ go.mod | 6 ++ go.sum | 20 +++++ 4 files changed, 128 insertions(+), 145 deletions(-) create mode 100644 analysis/jfr_mappings.go diff --git a/analysis/jfr.go b/analysis/jfr.go index 10affef..de876ed 100644 --- a/analysis/jfr.go +++ b/analysis/jfr.go @@ -8,174 +8,53 @@ // // 1. At the start of AnalyzeResults, convertJFRFiles walks the output folder // for any *.jfr files. -// 2. Each JFR file is parsed with github.com/grafana/jfr-parser/parser. +// 2. Each JFR file is parsed with github.com/grafana/jfr-parser/pprof. // 3. Per-metric pprof profiles are written as _.pprof alongside // the source JFR file (e.g. profile.jfr → profile_cpu.pprof). // 4. The normal pprof analysis loop then picks those up via the usual // filename regex. -// -// Supported JFR metrics (= pprof profile-type names): -// - "cpu" : jdk.ExecutionSample (non-sleeping threads) -// - "wall" : jdk.ExecutionSample when event=wall, or -// Datadog WallClockSample -// - "alloc_in_tlab" : jdk.ObjectAllocationInNewTLAB -// - "alloc_outside_tlab" : jdk.ObjectAllocationOutsideTLAB -// - "lock" : jdk.JavaMonitorEnter package analysis import ( "bytes" "fmt" - "io" "os" "path/filepath" "strings" + "time" "github.com/google/pprof/profile" - "github.com/grafana/jfr-parser/parser" - "github.com/grafana/jfr-parser/parser/types" + jfrpprof "github.com/grafana/jfr-parser/pprof" ) -// parseJFR converts raw JFR bytes into a map of metric name → pprof profile. +// parseJFR converts raw JFR bytes into a map of profile-name → pprof profile. func parseJFR(data []byte) (map[string]*profile.Profile, error) { - p := parser.NewParser(data, parser.Options{ - SymbolProcessor: parser.ProcessSymbols, - }) - - type builder struct { - prof *profile.Profile - mapping *profile.Mapping - funcByID map[types.MethodRef]*profile.Function - locByID map[types.MethodRef]*profile.Location - } - - builders := make(map[string]*builder) - - getBuilder := func(metric, sampleType, sampleUnit string) *builder { - b, ok := builders[metric] - if !ok { - m := &profile.Mapping{ID: 1, HasFunctions: true} - b = &builder{ - prof: &profile.Profile{ - SampleType: []*profile.ValueType{{Type: sampleType, Unit: sampleUnit}}, - PeriodType: &profile.ValueType{Type: sampleType, Unit: "nanoseconds"}, - Period: 10_000_000, // default 100 Hz - Mapping: []*profile.Mapping{m}, - }, - mapping: m, - funcByID: make(map[types.MethodRef]*profile.Function), - locByID: make(map[types.MethodRef]*profile.Location), - } - builders[metric] = b - } - return b - } - - // resolveFrameName returns "ClassName.methodName" for a JFR method reference. - // JVM internal '/' separators in class names are normalised to '.'. - resolveFrameName := func(methodRef types.MethodRef) string { - m := p.GetMethod(methodRef) - if m == nil { - return "" - } - methodName := p.GetSymbolString(m.Name) - cls := p.GetClass(m.Type) - if cls == nil { - return methodName - } - clsName := strings.ReplaceAll(p.GetSymbolString(cls.Name), "/", ".") - return clsName + "." + methodName - } - - // addSample appends one stack-trace observation to the named metric profile. - addSample := func(metric, sampleType, sampleUnit string, stackRef types.StackTraceRef, count int64) { - st := p.GetStacktrace(stackRef) - if st == nil || len(st.Frames) == 0 { - return - } - b := getBuilder(metric, sampleType, sampleUnit) - - // JFR frames[0] = leaf (top of stack), which matches the pprof convention - // that sample.Location[0] is the leaf. - locs := make([]*profile.Location, 0, len(st.Frames)) - for _, frame := range st.Frames { - loc, ok := b.locByID[frame.Method] - if !ok { - fnName := resolveFrameName(frame.Method) - if fnName == "" { - continue - } - fn, fnOK := b.funcByID[frame.Method] - if !fnOK { - fn = &profile.Function{ - ID: uint64(len(b.prof.Function) + 1), - Name: fnName, - } - b.prof.Function = append(b.prof.Function, fn) - b.funcByID[frame.Method] = fn - } - loc = &profile.Location{ - ID: uint64(len(b.prof.Location) + 1), - Mapping: b.mapping, - Line: []profile.Line{{Function: fn}}, - } - b.prof.Location = append(b.prof.Location, loc) - b.locByID[frame.Method] = loc - } - locs = append(locs, loc) - } - if len(locs) == 0 { - return - } - b.prof.Sample = append(b.prof.Sample, &profile.Sample{ - Location: locs, - Value: []int64{count}, - }) + profiles, err := jfrpprof.ParseJFR(data, &jfrpprof.ParseInput{ + StartTime: time.Unix(0, 0), + EndTime: time.Unix(0, 0), + // Keep prof-correctness values sample-like (1 per CPU/wall event) while + // reusing jfr-parser's pprof conversion, which otherwise scales CPU/wall + // samples by 1e9/SampleRate. + SampleRate: 1_000_000_000, + }, nil) + if err != nil { + return nil, fmt.Errorf("jfr ParseJFR: %w", err) } - var event string - for { - typ, err := p.ParseEvent() - if err == io.EOF { - break + result := make(map[string]*profile.Profile, len(profiles.Profiles)) + for _, parsed := range profiles.Profiles { + data, err := parsed.Profile.MarshalVT() + if err != nil { + return nil, fmt.Errorf("jfr marshal profile %s: %w", parsed.Metric, err) } + prof, err := profile.ParseData(data) if err != nil { - // Non-fatal: a truncated JFR file (e.g. from dumponexit=true) may end - // mid-chunk. Return what we have so far plus the error description. - return nil, fmt.Errorf("jfr ParseEvent: %w", err) + return nil, fmt.Errorf("jfr parse pprof profile %s: %w", parsed.Metric, err) } - - switch typ { - case p.TypeMap.T_EXECUTION_SAMPLE: - ts := p.GetThreadState(p.ExecutionSample.State) - if ts != nil && ts.Name != "STATE_SLEEPING" { - addSample("cpu", "cpu", "samples", p.ExecutionSample.StackTrace, 1) - } - if event == "wall" { - addSample("wall", "wall", "samples", p.ExecutionSample.StackTrace, 1) - } - case p.TypeMap.T_WALL_CLOCK_SAMPLE: - addSample("wall", "wall", "samples", - p.WallClockSample.StackTrace, int64(p.WallClockSample.Samples)) - case p.TypeMap.T_ALLOC_IN_NEW_TLAB: - addSample("alloc_in_tlab", "alloc_in_new_tlab_objects", "count", - p.ObjectAllocationInNewTLAB.StackTrace, 1) - case p.TypeMap.T_ALLOC_OUTSIDE_TLAB: - addSample("alloc_outside_tlab", "alloc_outside_tlab_objects", "count", - p.ObjectAllocationOutsideTLAB.StackTrace, 1) - case p.TypeMap.T_MONITOR_ENTER: - addSample("lock", "contentions", "count", - p.JavaMonitorEnter.StackTrace, 1) - case p.TypeMap.T_ACTIVE_SETTING: - if p.ActiveSetting.Name == "event" { - event = p.ActiveSetting.Value - } + for _, fn := range prof.Function { + fn.Name = strings.ReplaceAll(fn.Name, "/", ".") } - } - - result := make(map[string]*profile.Profile, len(builders)) - for metric, b := range builders { - result[metric] = b.prof + result[jfrProfileName(parsed.Metric, prof)] = prof } return result, nil } diff --git a/analysis/jfr_mappings.go b/analysis/jfr_mappings.go new file mode 100644 index 0000000..f2e1c1c --- /dev/null +++ b/analysis/jfr_mappings.go @@ -0,0 +1,78 @@ +package analysis + +import ( + "strings" + "unicode" + + "github.com/google/pprof/profile" +) + +// jfrProfileMappings keeps the JFR event/profile naming policy in one place. +// +// The raw JFR event IDs are metadata-defined and differ between JDK built-ins +// and Datadog/java-profiler events. github.com/grafana/jfr-parser normalises +// the supported event variants into pprof profiles; this table maps those +// normalised pprof sample-type sets to the file suffix used by prof-correctness +// expectations. +type jfrProfileMapping struct { + metric string + sampleTypes []string + name string +} + +var jfrProfileMappings = []jfrProfileMapping{ + {metric: "process_cpu", sampleTypes: []string{"cpu"}, name: "cpu"}, + {metric: "wall", sampleTypes: []string{"wall"}, name: "wall"}, + {metric: "memory", sampleTypes: []string{"alloc_in_new_tlab_objects", "alloc_in_new_tlab_bytes"}, name: "alloc_in_tlab"}, + {metric: "memory", sampleTypes: []string{"alloc_outside_tlab_objects", "alloc_outside_tlab_bytes"}, name: "alloc_outside_tlab"}, + {metric: "memory", sampleTypes: []string{"alloc_sample_objects", "alloc_sample_bytes"}, name: "alloc_sample"}, + {metric: "memory", sampleTypes: []string{"live"}, name: "live"}, + {metric: "memory", sampleTypes: []string{"malloc_objects", "malloc_bytes"}, name: "malloc"}, + {metric: "mutex", sampleTypes: []string{"contentions", "delay"}, name: "lock"}, + {metric: "block", sampleTypes: []string{"contentions", "delay"}, name: "park"}, +} + +func jfrProfileName(metric string, prof *profile.Profile) string { + sampleTypes := make([]string, 0, len(prof.SampleType)) + for _, sampleType := range prof.SampleType { + sampleTypes = append(sampleTypes, sampleType.Type) + } + + for _, mapping := range jfrProfileMappings { + if mapping.metric == metric && sameStrings(mapping.sampleTypes, sampleTypes) { + return mapping.name + } + } + + parts := append([]string{metric}, sampleTypes...) + return sanitizeJFRProfileName(strings.Join(parts, "_")) +} + +func sameStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func sanitizeJFRProfileName(name string) string { + var b strings.Builder + lastUnderscore := false + for _, r := range name { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + lastUnderscore = false + continue + } + if !lastUnderscore { + b.WriteByte('_') + lastUnderscore = true + } + } + return strings.Trim(b.String(), "_") +} diff --git a/go.mod b/go.mod index dbf1f12..d0b85a5 100644 --- a/go.mod +++ b/go.mod @@ -13,14 +13,20 @@ require ( ) require ( + github.com/google/gnostic v0.7.1 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/grafana/pyroscope/api v1.5.0 // indirect github.com/hashicorp/go-version v1.9.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kr/text v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.opentelemetry.io/collector/featuregate v1.62.0 // indirect go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect diff --git a/go.sum b/go.sum index aaef178..3a792ad 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -10,6 +11,10 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/gnostic v0.7.1 h1:t5Kc7j/8kYr8t2u11rykRrPPovlEMG4+xdc/SpekATs= +github.com/google/gnostic v0.7.1/go.mod h1:KSw6sxnxEBFM8jLPfJd46xZP+yQcfE8XkiqfZx5zR28= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -19,12 +24,18 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grafana/jfr-parser v0.17.1 h1:dTECCXL+B9V6qu5x4msSKE9/FJPKeK0jrwLa5fkpkCA= github.com/grafana/jfr-parser v0.17.1/go.mod h1:+4zCC+tEWot6oQWjC72bG9TDDBiiiawMprp4EM1BioU= +github.com/grafana/pyroscope/api v1.5.0 h1:5vBeCOJ6uAKPN1k/M51/zLvrXYVxP0WJJT7MGbq68wA= +github.com/grafana/pyroscope/api v1.5.0/go.mod h1:JXy9oodWgLUVkUUlSk9+2Y0D4DWWwTQJo8NLXT36zg4= github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -33,9 +44,13 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 h1:S1hI5JiKP7883xBzZAr1ydcxrKNSVNm7+3+JwjxZEsg= +github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1:ZQntvDG8TkPgljxtA0R9frDoND4QORU1VXz015N5Ks4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -76,6 +91,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= @@ -90,5 +107,8 @@ google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 8a8ba1800d1c53cfb557ce22369aad181433e9bc Mon Sep 17 00:00:00 2001 From: "erwan.viollet" Date: Tue, 28 Jul 2026 12:20:42 +0200 Subject: [PATCH 4/6] Use JFR parser fork with Datadog event aliases --- go.mod | 4 +++- go.sum | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d0b85a5..0a67042 100644 --- a/go.mod +++ b/go.mod @@ -29,8 +29,10 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect - golang.org/x/text v0.37.0 // indirect + golang.org/x/text v0.39.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/grpc v1.82.0 // indirect google.golang.org/protobuf v1.36.11 // indirect ) + +replace github.com/grafana/jfr-parser => github.com/r1viollet/pyroscope-jfr-parser v0.0.0-20260728101739-fe03b6cb52f6 diff --git a/go.sum b/go.sum index 3a792ad..962c50d 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,6 @@ github.com/google/pprof v0.0.0-20260507013755-92041b743c96 h1:YDDnaZ9afWajDboPMt github.com/google/pprof v0.0.0-20260507013755-92041b743c96/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grafana/jfr-parser v0.17.1 h1:dTECCXL+B9V6qu5x4msSKE9/FJPKeK0jrwLa5fkpkCA= -github.com/grafana/jfr-parser v0.17.1/go.mod h1:+4zCC+tEWot6oQWjC72bG9TDDBiiiawMprp4EM1BioU= github.com/grafana/pyroscope/api v1.5.0 h1:5vBeCOJ6uAKPN1k/M51/zLvrXYVxP0WJJT7MGbq68wA= github.com/grafana/pyroscope/api v1.5.0/go.mod h1:JXy9oodWgLUVkUUlSk9+2Y0D4DWWwTQJo8NLXT36zg4= github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= @@ -49,6 +47,8 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/r1viollet/pyroscope-jfr-parser v0.0.0-20260728101739-fe03b6cb52f6 h1:w/xEltTLRDEJPinvxiSENVYfTSZutd0brRT1s8lTVwQ= +github.com/r1viollet/pyroscope-jfr-parser v0.0.0-20260728101739-fe03b6cb52f6/go.mod h1:jNzwTrMNDUt6p/HcWjjtkTuZEfxUqr/gn/8hw0LaBJY= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -97,8 +97,8 @@ golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= -golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/text v0.39.0 h1:UbZz4pLOvn600D6Oh6GGEI6VAmndrEBLv8/6BEXzyus= +golang.org/x/text v0.39.0/go.mod h1:3UwRclnC2g0TU9x8PZiyfOajCd1zaUNHF9cvqcQZ+ZM= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= From d8e95941e582b5b5389e66c238f9b6678564210c Mon Sep 17 00:00:00 2001 From: "erwan.viollet" Date: Tue, 28 Jul 2026 12:57:37 +0200 Subject: [PATCH 5/6] Parse JFR directly into profile model --- analysis/analysis.go | 7 +- analysis/jfr.go | 211 +++++++++++-------- analysis/jfr_mappings.go | 78 ------- analysis/model.go | 14 +- go.mod | 8 +- go.sum | 24 +-- scenarios/java_cpu_jfr/expected_profile.json | 2 +- 7 files changed, 143 insertions(+), 201 deletions(-) delete mode 100644 analysis/jfr_mappings.go diff --git a/analysis/analysis.go b/analysis/analysis.go index 1fe9c2e..87b40c0 100644 --- a/analysis/analysis.go +++ b/analysis/analysis.go @@ -661,13 +661,8 @@ func AnalyzePprofFile(r Reporter, pprofFile string, typedStacks TypedStacks, tes } // AnalyzeResults loads the expected_profile.json at jsonFilePath and asserts -// every pprof file under pprofFolder matches it. Failures are reported via r. -// Any *.jfr files in pprofFolder are converted to per-metric pprof files -// before the analysis loop runs. +// every profile file under pprofFolder matches it. Failures are reported via r. func AnalyzeResults(r Reporter, jsonFilePath string, pprofFolder string) { - // Convert any JFR files to pprof before the analysis loop. - convertJFRFiles(r, pprofFolder) - stackTestData, err := ReadJSONFile(jsonFilePath) if err != nil { r.Fatalf("Error opening file %s: %v", jsonFilePath, err) diff --git a/analysis/jfr.go b/analysis/jfr.go index de876ed..6edaf63 100644 --- a/analysis/jfr.go +++ b/analysis/jfr.go @@ -1,110 +1,153 @@ // Package analysis — JFR support. // -// This file adds the ability to read JFR (Java Flight Recorder) files and -// convert them to pprof profiles that the rest of the analysis pipeline can -// consume. -// -// Flow: -// -// 1. At the start of AnalyzeResults, convertJFRFiles walks the output folder -// for any *.jfr files. -// 2. Each JFR file is parsed with github.com/grafana/jfr-parser/pprof. -// 3. Per-metric pprof profiles are written as _.pprof alongside -// the source JFR file (e.g. profile.jfr → profile_cpu.pprof). -// 4. The normal pprof analysis loop then picks those up via the usual -// filename regex. +// This file adds the ability to read JFR (Java Flight Recorder) files and map +// JFR events directly into the analyzer's neutral ProfileSet. The mapping from +// event names to profile semantics intentionally lives here: prof-correctness is +// the consumer that knows which JDK / Datadog profiler events should satisfy a +// given expected_profile.json assertion. package analysis import ( - "bytes" "fmt" - "os" - "path/filepath" + "io" "strings" - "time" - "github.com/google/pprof/profile" - jfrpprof "github.com/grafana/jfr-parser/pprof" + "github.com/grafana/jfr-parser/parser" + "github.com/grafana/jfr-parser/parser/types" ) -// parseJFR converts raw JFR bytes into a map of profile-name → pprof profile. -func parseJFR(data []byte) (map[string]*profile.Profile, error) { - profiles, err := jfrpprof.ParseJFR(data, &jfrpprof.ParseInput{ - StartTime: time.Unix(0, 0), - EndTime: time.Unix(0, 0), - // Keep prof-correctness values sample-like (1 per CPU/wall event) while - // reusing jfr-parser's pprof conversion, which otherwise scales CPU/wall - // samples by 1e9/SampleRate. - SampleRate: 1_000_000_000, - }, nil) - if err != nil { - return nil, fmt.Errorf("jfr ParseJFR: %w", err) - } +// FromJFR builds a ProfileSet directly from Java Flight Recorder events. +func FromJFR(data []byte) (*ProfileSet, error) { + p := parser.NewParser(data, parser.Options{ + SymbolProcessor: parser.ProcessSymbols, + }) + ps := newProfileSet() - result := make(map[string]*profile.Profile, len(profiles.Profiles)) - for _, parsed := range profiles.Profiles { - data, err := parsed.Profile.MarshalVT() - if err != nil { - return nil, fmt.Errorf("jfr marshal profile %s: %w", parsed.Metric, err) + var cpuTotal int64 + var durationNanos uint64 + seenChunks := map[jfrChunkKey]bool{} + + for { + event, err := p.ParseRawEvent() + if err == io.EOF { + break } - prof, err := profile.ParseData(data) if err != nil { - return nil, fmt.Errorf("jfr parse pprof profile %s: %w", parsed.Metric, err) + return nil, fmt.Errorf("jfr ParseRawEvent: %w", err) + } + if event.Type == nil { + continue + } + + header := p.ChunkHeader() + chunk := jfrChunkKey{startNanos: header.StartNanos, durationNanos: header.DurationNanos} + if !seenChunks[chunk] { + seenChunks[chunk] = true + durationNanos += header.DurationNanos } - for _, fn := range prof.Function { - fn.Name = strings.ReplaceAll(fn.Name, "/", ".") + + switch event.Type.Name { + case "jdk.ExecutionSample", "datadog.ExecutionSample": + fields, err := p.DecodeRawEventFields(event) + if err != nil { + return nil, fmt.Errorf("jfr decode %s: %w", event.Type.Name, err) + } + cpuTotal += addJFRCPU(ps, p, fields) } - result[jfrProfileName(parsed.Metric, prof)] = prof } - return result, nil + + if cpuTotal > 0 && durationNanos > 0 { + ps.addProfileDuration("cpu", cpuTotal, float64(durationNanos)/1e9) + } + + return ps.finalize(), nil +} + +type jfrChunkKey struct { + startNanos uint64 + durationNanos uint64 } -// convertJFRFiles walks dir for *.jfr files and converts each to a set of -// per-metric pprof files written into the same directory. -// Output files are named _.pprof (e.g. profile_cpu.pprof). -// Errors are non-fatal: they are logged through r and the function continues. -func convertJFRFiles(r Reporter, dir string) { - entries, err := os.ReadDir(dir) - if err != nil { - r.Logf("convertJFRFiles: reading dir %s: %v", dir, err) - return +func addJFRCPU(ps *ProfileSet, p *parser.Parser, fields map[string]parser.RawField) int64 { + stackField, ok := jfrField(fields, "stackTrace") + if !ok { + return 0 } - for _, entry := range entries { - if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".jfr") { - continue - } - jfrPath := filepath.Join(dir, entry.Name()) - stem := strings.TrimSuffix(entry.Name(), ".jfr") - data, err := os.ReadFile(jfrPath) - if err != nil { - r.Logf("convertJFRFiles: reading %s: %v", jfrPath, err) - continue + // Match jfr-parser/pprof's CPU semantics: execution samples from sleeping + // threads do not count as CPU samples. If the state is absent, keep the + // sample rather than silently dropping producer-specific events. + if state, ok := jfrField(fields, "state"); ok { + if ts := p.GetThreadState(types.ThreadStateRef(state.Uint64)); ts != nil && ts.Name == "STATE_SLEEPING" { + return 0 } + } - profiles, err := parseJFR(data) - if err != nil { - r.Logf("convertJFRFiles: parsing %s: %v", jfrPath, err) - continue - } - if len(profiles) == 0 { - r.Logf("convertJFRFiles: no profiles found in %s", jfrPath) - continue - } + folded := foldJFRStack(p, types.StackTraceRef(stackField.Uint64)) + if folded == "" { + return 0 + } - for metric, prof := range profiles { - outPath := filepath.Join(dir, stem+"_"+metric+".pprof") - var buf bytes.Buffer - if err := prof.Write(&buf); err != nil { - r.Logf("convertJFRFiles: serialising %s metric %s: %v", entry.Name(), metric, err) - continue - } - if err := os.WriteFile(outPath, buf.Bytes(), 0644); err != nil { - r.Logf("convertJFRFiles: writing %s: %v", outPath, err) - continue - } - r.Logf("Converted JFR %s → %s (%d samples)", - entry.Name(), filepath.Base(outPath), len(prof.Sample)) + val := int64(1) + if weight, ok := jfrField(fields, "weight"); ok && weight.Uint64 > 0 { + val = int64(weight.Uint64) + } + + ps.add("cpu", StackSample{Stack: folded, Val: val, Labels: executionSampleLabels(fields)}) + return val +} + +func executionSampleLabels(fields map[string]parser.RawField) map[string][]string { + labels := map[string][]string{} + if spanID, ok := jfrField(fields, "spanId"); ok && spanID.Uint64 != 0 { + labels[LabelSpanID] = []string{fmt.Sprintf("%d", spanID.Uint64)} + } + if localRootSpanID, ok := jfrField(fields, "localRootSpanId"); ok && localRootSpanID.Uint64 != 0 { + labels[LabelLocalRootSID] = []string{fmt.Sprintf("%d", localRootSpanID.Uint64)} + } + traceHi, traceHiOK := jfrField(fields, "traceIdHi") + traceLo, traceLoOK := jfrField(fields, "traceIdLo") + if (traceHiOK || traceLoOK) && (traceHi.Uint64 != 0 || traceLo.Uint64 != 0) { + labels[LabelTraceID] = []string{fmt.Sprintf("%016x%016x", traceHi.Uint64, traceLo.Uint64)} + } + return labels +} + +func jfrField(fields map[string]parser.RawField, name string) (parser.RawValue, bool) { + field, ok := fields[name] + if !ok { + return parser.RawValue{}, false + } + return field.First() +} + +func foldJFRStack(p *parser.Parser, stackRef types.StackTraceRef) string { + st := p.GetStacktrace(stackRef) + if st == nil || len(st.Frames) == 0 { + return "" + } + // JFR frames are leaf-first. The analyzer uses root-first folded stacks, + // matching FromPprof and the historical expected_profile.json captures. + frames := make([]string, 0, len(st.Frames)) + for i := len(st.Frames) - 1; i >= 0; i-- { + name := jfrFrameName(p, st.Frames[i].Method) + if name != "" { + frames = append(frames, name) } } + return strings.Join(frames, ";") +} + +func jfrFrameName(p *parser.Parser, methodRef types.MethodRef) string { + m := p.GetMethod(methodRef) + if m == nil { + return "" + } + methodName := p.GetSymbolString(m.Name) + cls := p.GetClass(m.Type) + if cls == nil { + return methodName + } + clsName := strings.ReplaceAll(p.GetSymbolString(cls.Name), "/", ".") + return clsName + "." + methodName } diff --git a/analysis/jfr_mappings.go b/analysis/jfr_mappings.go deleted file mode 100644 index f2e1c1c..0000000 --- a/analysis/jfr_mappings.go +++ /dev/null @@ -1,78 +0,0 @@ -package analysis - -import ( - "strings" - "unicode" - - "github.com/google/pprof/profile" -) - -// jfrProfileMappings keeps the JFR event/profile naming policy in one place. -// -// The raw JFR event IDs are metadata-defined and differ between JDK built-ins -// and Datadog/java-profiler events. github.com/grafana/jfr-parser normalises -// the supported event variants into pprof profiles; this table maps those -// normalised pprof sample-type sets to the file suffix used by prof-correctness -// expectations. -type jfrProfileMapping struct { - metric string - sampleTypes []string - name string -} - -var jfrProfileMappings = []jfrProfileMapping{ - {metric: "process_cpu", sampleTypes: []string{"cpu"}, name: "cpu"}, - {metric: "wall", sampleTypes: []string{"wall"}, name: "wall"}, - {metric: "memory", sampleTypes: []string{"alloc_in_new_tlab_objects", "alloc_in_new_tlab_bytes"}, name: "alloc_in_tlab"}, - {metric: "memory", sampleTypes: []string{"alloc_outside_tlab_objects", "alloc_outside_tlab_bytes"}, name: "alloc_outside_tlab"}, - {metric: "memory", sampleTypes: []string{"alloc_sample_objects", "alloc_sample_bytes"}, name: "alloc_sample"}, - {metric: "memory", sampleTypes: []string{"live"}, name: "live"}, - {metric: "memory", sampleTypes: []string{"malloc_objects", "malloc_bytes"}, name: "malloc"}, - {metric: "mutex", sampleTypes: []string{"contentions", "delay"}, name: "lock"}, - {metric: "block", sampleTypes: []string{"contentions", "delay"}, name: "park"}, -} - -func jfrProfileName(metric string, prof *profile.Profile) string { - sampleTypes := make([]string, 0, len(prof.SampleType)) - for _, sampleType := range prof.SampleType { - sampleTypes = append(sampleTypes, sampleType.Type) - } - - for _, mapping := range jfrProfileMappings { - if mapping.metric == metric && sameStrings(mapping.sampleTypes, sampleTypes) { - return mapping.name - } - } - - parts := append([]string{metric}, sampleTypes...) - return sanitizeJFRProfileName(strings.Join(parts, "_")) -} - -func sameStrings(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - return true -} - -func sanitizeJFRProfileName(name string) string { - var b strings.Builder - lastUnderscore := false - for _, r := range name { - if unicode.IsLetter(r) || unicode.IsDigit(r) { - b.WriteRune(r) - lastUnderscore = false - continue - } - if !lastUnderscore { - b.WriteByte('_') - lastUnderscore = true - } - } - return strings.Trim(b.String(), "_") -} diff --git a/analysis/model.go b/analysis/model.go index d8dcd1b..bee2871 100644 --- a/analysis/model.go +++ b/analysis/model.go @@ -24,6 +24,7 @@ package analysis import ( "path/filepath" "sort" + "strings" "github.com/google/pprof/profile" ) @@ -143,13 +144,18 @@ func (ps *ProfileSet) finalize() *ProfileSet { return ps } -// LoadProfileSet reads a profile file (pprof or OTLP) and returns the neutral -// ProfileSet. Format is chosen by filename suffix (.otlp/.otlp.pb -> OTLP +// isJFRName reports whether name should be parsed as a Java Flight Recorder recording. +func isJFRName(name string) bool { + return strings.HasSuffix(strings.ToLower(name), ".jfr") +} + +// LoadProfileSet reads a profile file (pprof, OTLP or JFR) and returns the neutral +// ProfileSet. Format is chosen by filename suffix (.jfr -> JFR, .otlp/.otlp.pb -> OTLP // proto, .otlp.json -> OTLP JSON, else pprof) with an OTLP fallback if pprof // parsing fails. Ambiguous suffixes such as .pb (used by both pprof and OTLP) // go through the content-based fallback rather than being forced to a format. // The per-format parsing lives in the respective adapter file (pprof.go / -// otlp.go). +// otlp.go / jfr.go). func LoadProfileSet(path string) (*ProfileSet, error) { content, err := readAndDecompress(path) if err != nil { @@ -158,6 +164,8 @@ func LoadProfileSet(path string) (*ProfileSet, error) { name := filepath.Base(path) switch { + case isJFRName(name): + return FromJFR(content) case isOTLPJSONName(name): return loadOTLP(content, true) case isOTLPProtoName(name): diff --git a/go.mod b/go.mod index 0a67042..2e67d60 100644 --- a/go.mod +++ b/go.mod @@ -13,20 +13,14 @@ require ( ) require ( - github.com/google/gnostic v0.7.1 // indirect - github.com/google/gnostic-models v0.7.0 // indirect - github.com/grafana/pyroscope/api v1.5.0 // indirect github.com/hashicorp/go-version v1.9.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kr/text v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect - github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect go.opentelemetry.io/collector/featuregate v1.62.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/net v0.55.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.39.0 // indirect @@ -35,4 +29,4 @@ require ( google.golang.org/protobuf v1.36.11 // indirect ) -replace github.com/grafana/jfr-parser => github.com/r1viollet/pyroscope-jfr-parser v0.0.0-20260728101739-fe03b6cb52f6 +replace github.com/grafana/jfr-parser => github.com/r1viollet/pyroscope-jfr-parser v0.0.0-20260803080954-eddc025c5c4e diff --git a/go.sum b/go.sum index 962c50d..77b7388 100644 --- a/go.sum +++ b/go.sum @@ -1,6 +1,5 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -11,10 +10,6 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/gnostic v0.7.1 h1:t5Kc7j/8kYr8t2u11rykRrPPovlEMG4+xdc/SpekATs= -github.com/google/gnostic v0.7.1/go.mod h1:KSw6sxnxEBFM8jLPfJd46xZP+yQcfE8XkiqfZx5zR28= -github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= -github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -22,18 +17,12 @@ github.com/google/pprof v0.0.0-20260507013755-92041b743c96 h1:YDDnaZ9afWajDboPMt github.com/google/pprof v0.0.0-20260507013755-92041b743c96/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grafana/pyroscope/api v1.5.0 h1:5vBeCOJ6uAKPN1k/M51/zLvrXYVxP0WJJT7MGbq68wA= -github.com/grafana/pyroscope/api v1.5.0/go.mod h1:JXy9oodWgLUVkUUlSk9+2Y0D4DWWwTQJo8NLXT36zg4= github.com/hashicorp/go-version v1.9.0 h1:CeOIz6k+LoN3qX9Z0tyQrPtiB1DFYRPfCIBtaXPSCnA= github.com/hashicorp/go-version v1.9.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= -github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= -github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -42,15 +31,11 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0= github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= -github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25 h1:S1hI5JiKP7883xBzZAr1ydcxrKNSVNm7+3+JwjxZEsg= -github.com/planetscale/vtprotobuf v0.6.1-0.20250313105119-ba97887b0a25/go.mod h1:ZQntvDG8TkPgljxtA0R9frDoND4QORU1VXz015N5Ks4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/r1viollet/pyroscope-jfr-parser v0.0.0-20260728101739-fe03b6cb52f6 h1:w/xEltTLRDEJPinvxiSENVYfTSZutd0brRT1s8lTVwQ= -github.com/r1viollet/pyroscope-jfr-parser v0.0.0-20260728101739-fe03b6cb52f6/go.mod h1:jNzwTrMNDUt6p/HcWjjtkTuZEfxUqr/gn/8hw0LaBJY= -github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= -github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/r1viollet/pyroscope-jfr-parser v0.0.0-20260803080954-eddc025c5c4e h1:qOsPPgSArMHlR10zugkK6l5pCZp+8fK99pIcwxpYJ7M= +github.com/r1viollet/pyroscope-jfr-parser v0.0.0-20260803080954-eddc025c5c4e/go.mod h1:jNzwTrMNDUt6p/HcWjjtkTuZEfxUqr/gn/8hw0LaBJY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= @@ -91,8 +76,6 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= -go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= @@ -107,8 +90,5 @@ google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/scenarios/java_cpu_jfr/expected_profile.json b/scenarios/java_cpu_jfr/expected_profile.json index 914d809..25bb8cd 100644 --- a/scenarios/java_cpu_jfr/expected_profile.json +++ b/scenarios/java_cpu_jfr/expected_profile.json @@ -1,6 +1,6 @@ { "test_name": "java_cpu_jfr", - "pprof-regex": ".*_cpu\\.pprof$", + "pprof-regex": ".*\\.jfr$", "stacks": [ { "profile-type": "cpu", From 96c4983517e3762b13e75359a64f211c92a21a05 Mon Sep 17 00:00:00 2001 From: "erwan.viollet" Date: Mon, 3 Aug 2026 09:31:30 +0200 Subject: [PATCH 6/6] Run Java JFR scenario with Datadog profiler --- scenarios/java_cpu_jfr/Dockerfile | 50 ++++++++++++++------ scenarios/java_cpu_jfr/DummyApp.java | 2 +- scenarios/java_cpu_jfr/expected_profile.json | 2 +- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/scenarios/java_cpu_jfr/Dockerfile b/scenarios/java_cpu_jfr/Dockerfile index 9b21030..dd3367f 100644 --- a/scenarios/java_cpu_jfr/Dockerfile +++ b/scenarios/java_cpu_jfr/Dockerfile @@ -2,21 +2,43 @@ FROM eclipse-temurin:21-jdk WORKDIR /app -# Copy and compile the Java source +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Install the Datadog Java tracer, which carries the Datadog native Java +# profiler (ddprof). The profiler writes Datadog-specific JFR events such as +# datadog.ExecutionSample; this scenario validates that prof-correctness can +# consume those recordings directly. +RUN curl -fsSL -o /app/dd-java-agent.jar https://dtdg.co/latest-java-tracer + COPY ./scenarios/java_cpu_jfr/DummyApp.java . RUN javac DummyApp.java -# Force re-run of the next layer when the cache date changes. -ARG CACHE_DATE=2024-01-01 - -# EXECUTION_TIME_SEC is injected by the test harness; default to 30 s. -ENV EXECUTION_TIME_SEC=30 +# EXECUTION_TIME_SEC is injected by the test harness; default to 60 s. +ENV EXECUTION_TIME_SEC=60 -# Run with built-in JFR. -# -XX:+FlightRecorder : enable JFR (no-op on JDK 11+ where it's always on) -# -XX:StartFlightRecording=... : start a continuous recording; dump on JVM exit. -# settings=profile : enable CPU/alloc/lock profiling at low overhead. -CMD java \ - -XX:+FlightRecorder \ - "-XX:StartFlightRecording=filename=/app/data/profile.jfr,dumponexit=true,settings=profile,name=correctness" \ - DummyApp +# Run the app under the Datadog Java profiler and use the tracer's built-in +# debug dump path to persist the JFR snapshot locally. The prof-correctness +# harness finds the timestamped dd-profiler-debug-*.jfr file by regex. +CMD sh -eu -c '\ + mkdir -p /app/data/dumps /app/data/tmp; \ + java \ + -javaagent:/app/dd-java-agent.jar \ + -Ddd.service=prof-correctness-java-cpu-jfr \ + -Ddd.env=local \ + -Ddd.trace.enabled=false \ + -Ddd.profiling.enabled=true \ + -Ddd.profiling.ddprof.enabled=true \ + -Ddd.profiling.ddprof.cpu.enabled=true \ + -Ddd.profiling.ddprof.wall.enabled=false \ + -Ddd.profiling.upload.period=30 \ + -Ddd.profiling.upload.timeout=1 \ + -Ddd.profiling.start-force-first=true \ + -Ddd.profiling.tempdir=/app/data/tmp \ + -Ddd.profiling.debug.dump_path=/app/data/dumps \ + -Ddd.profiling.url=http://127.0.0.1:8126/ \ + -Ddd.telemetry.enabled=false \ + -Ddd.jmxfetch.enabled=false \ + DummyApp \ +' diff --git a/scenarios/java_cpu_jfr/DummyApp.java b/scenarios/java_cpu_jfr/DummyApp.java index 355a339..4a5370c 100644 --- a/scenarios/java_cpu_jfr/DummyApp.java +++ b/scenarios/java_cpu_jfr/DummyApp.java @@ -2,7 +2,7 @@ * Simple CPU-intensive app for JFR correctness testing. * * Continuously computes Fibonacci numbers to keep the CPU busy. - * The JFR recording is started by the JVM flags passed on the command line; + * The Datadog Java profiler is started by the JVM flags passed on the command line; * this app just runs for the requested duration and exits. * * The EXECUTION_TIME_SEC environment variable (default: 30) controls how long diff --git a/scenarios/java_cpu_jfr/expected_profile.json b/scenarios/java_cpu_jfr/expected_profile.json index 25bb8cd..6093693 100644 --- a/scenarios/java_cpu_jfr/expected_profile.json +++ b/scenarios/java_cpu_jfr/expected_profile.json @@ -1,6 +1,6 @@ { "test_name": "java_cpu_jfr", - "pprof-regex": ".*\\.jfr$", + "pprof-regex": "^dd-profiler-debug-.*\\.jfr$", "stacks": [ { "profile-type": "cpu",