Skip to content

Commit 85861b1

Browse files
korniltsevkorniltsev
committed
fix(python): handle cold interpreter func chunks
- handle cold interpreter func chunks - add coredump alpine320, alpine320-nobuildid
1 parent 0708a7f commit 85861b1

File tree

11 files changed

+324
-51
lines changed

11 files changed

+324
-51
lines changed

‎interpreter/loaderinfo.go

+4
Original file line numberDiff line numberDiff line change
@@ -68,3 +68,7 @@ func (i *LoaderInfo) FileName() string {
6868
func (i *LoaderInfo) Gaps() []util.Range {
6969
return i.gaps
7070
}
71+
72+
func (i *LoaderInfo) ElfOpener() pfelf.ELFOpener {
73+
return i.elfRef.ELFOpener
74+
}

‎interpreter/python/python.go

+57-12
Original file line numberDiff line numberDiff line change
@@ -750,19 +750,9 @@ func Loader(ebpf interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interpr
750750
autoTLSKey += 4
751751
}
752752

753-
// The Python main interpreter loop history in CPython git is:
754-
//
755-
//nolint:lll
756-
// 87af12bff33 v3.11 2022-02-15 _PyEval_EvalFrameDefault(PyThreadState*,_PyInterpreterFrame*,int)
757-
// ae0a2b75625 v3.10 2021-06-25 _PyEval_EvalFrameDefault(PyThreadState*,_interpreter_frame*,int)
758-
// 0b72b23fb0c v3.9 2020-03-12 _PyEval_EvalFrameDefault(PyThreadState*,PyFrameObject*,int)
759-
// 3cebf938727 v3.6 2016-09-05 _PyEval_EvalFrameDefault(PyFrameObject*,int)
760-
// 49fd7fa4431 v3.0 2006-04-21 PyEval_EvalFrameEx(PyFrameObject*,int)
761-
interpRanges, err := info.GetSymbolAsRanges("_PyEval_EvalFrameDefault")
753+
interpRanges, err := findInterpreterRanges(ef, info)
762754
if err != nil {
763-
if interpRanges, err = info.GetSymbolAsRanges("PyEval_EvalFrameEx"); err != nil {
764-
return nil, err
765-
}
755+
return nil, err
766756
}
767757

768758
pd := &pythonData{
@@ -837,3 +827,58 @@ func Loader(ebpf interpreter.EbpfHandler, info *interpreter.LoaderInfo) (interpr
837827

838828
return pd, nil
839829
}
830+
831+
func findInterpreterRanges(ef *pfelf.File, info *interpreter.LoaderInfo) ([]util.Range, error) {
832+
// The Python main interpreter loop history in CPython git is:
833+
//
834+
//nolint:lll
835+
// 87af12bff33 v3.11 2022-02-15 _PyEval_EvalFrameDefault(PyThreadState*,_PyInterpreterFrame*,int)
836+
// ae0a2b75625 v3.10 2021-06-25 _PyEval_EvalFrameDefault(PyThreadState*,_interpreter_frame*,int)
837+
// 0b72b23fb0c v3.9 2020-03-12 _PyEval_EvalFrameDefault(PyThreadState*,PyFrameObject*,int)
838+
// 3cebf938727 v3.6 2016-09-05 _PyEval_EvalFrameDefault(PyFrameObject*,int)
839+
// 49fd7fa4431 v3.0 2006-04-21 PyEval_EvalFrameEx(PyFrameObject*,int)
840+
interpRanges, err := info.GetSymbolAsRanges("_PyEval_EvalFrameDefault")
841+
if err != nil {
842+
if interpRanges, err = info.GetSymbolAsRanges("PyEval_EvalFrameEx"); err != nil {
843+
return nil, err
844+
}
845+
return interpRanges, nil
846+
}
847+
if len(interpRanges) == 0 {
848+
return nil, errors.New("no _PyEval_EvalFrameDefault/PyEval_EvalFrameEx symbol found")
849+
}
850+
gnuBuildID, _ := ef.GetBuildID()
851+
if coldRange, ok := interpreterColdRanges[gnuBuildID]; ok {
852+
interpRanges = append(interpRanges, coldRange)
853+
return interpRanges, nil
854+
}
855+
// find debug file, check .cold function
856+
// TODO: find cold chunk range without debug file
857+
debugELF, _ := ef.OpenDebugLink(info.FileName(), info.ElfOpener())
858+
if debugELF == nil {
859+
return interpRanges, nil
860+
}
861+
defer debugELF.Close()
862+
coldSym, err := debugELF.LookupSymbolSlow("_PyEval_EvalFrameDefault.cold")
863+
if err != nil {
864+
return interpRanges, nil
865+
}
866+
interpRanges = append(interpRanges, util.Range{
867+
Start: uint64(coldSym.Address),
868+
End: uint64(coldSym.Address) + coldSym.Size,
869+
})
870+
return interpRanges, nil
871+
}
872+
873+
var interpreterColdRanges = map[string]util.Range{
874+
// alpine:3.20
875+
"4913fe1380aebd0f4f0d69411b797d7e22d2799b": {
876+
Start: 0x88a9a,
877+
End: 0x88a9a + 0xdcdf,
878+
},
879+
// alpine 3.21
880+
"f0f26a21d40d3c089975a8b136fc2469df40a0e6": {
881+
Start: 0x89228,
882+
End: 0x89228 + 0x35d9,
883+
},
884+
}

‎libpf/pfelf/file.go

+47-11
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,9 @@ const (
4646
// parsed sections (e.g. symbol tables and string tables; libxul
4747
// has about 4MB .dynstr)
4848
maxBytesLargeSection = 16 * 1024 * 1024
49+
50+
sectionNameSymtab = ".symtab"
51+
sectionNameDynSym = ".dynsym"
4952
)
5053

5154
// ErrSymbolNotFound is returned when requested symbol was not found
@@ -885,51 +888,84 @@ func (f *File) LookupSymbolAddress(symbol libpf.SymbolName) (libpf.SymbolValue,
885888

886889
// loadSymbolTable reads given symbol table
887890
func (f *File) loadSymbolTable(name string) (*libpf.SymbolMap, error) {
891+
symMap := new(libpf.SymbolMap)
892+
err := f.eachSymbolSlow(name, func(s libpf.Symbol) bool {
893+
symMap.Add(s)
894+
return true
895+
})
896+
if err != nil {
897+
return nil, err
898+
}
899+
symMap.Finalize()
900+
return symMap, nil
901+
}
902+
903+
func (f *File) eachSymbolSlow(name string, it func(s libpf.Symbol) bool) error {
888904
symTab := f.Section(name)
889905
if symTab == nil {
890-
return nil, fmt.Errorf("failed to read %v: section not present", name)
906+
return fmt.Errorf("failed to read %v: section not present", name)
891907
}
892908
if symTab.Link >= uint32(len(f.Sections)) {
893-
return nil, fmt.Errorf("failed to read %v strtab: link %v out of range",
909+
return fmt.Errorf("failed to read %v strtab: link %v out of range",
894910
name, symTab.Link)
895911
}
896912
strTab := f.Sections[symTab.Link]
897913
strs, err := strTab.Data(maxBytesLargeSection)
898914
if err != nil {
899-
return nil, fmt.Errorf("failed to read %v: %v", strTab.Name, err)
915+
return fmt.Errorf("failed to read %v: %v", strTab.Name, err)
900916
}
901917
syms, err := symTab.Data(maxBytesLargeSection)
902918
if err != nil {
903-
return nil, fmt.Errorf("failed to read %v: %v", name, err)
919+
return fmt.Errorf("failed to read %v: %v", name, err)
904920
}
905921

906-
symMap := libpf.SymbolMap{}
907922
symSz := int(unsafe.Sizeof(elf.Sym64{}))
908923
for i := 0; i < len(syms); i += symSz {
909924
sym := (*elf.Sym64)(unsafe.Pointer(&syms[i]))
910925
name, ok := getString(strs, int(sym.Name))
911926
if !ok {
912927
continue
913928
}
914-
symMap.Add(libpf.Symbol{
929+
cont := it(libpf.Symbol{
915930
Name: libpf.SymbolName(name),
916931
Address: libpf.SymbolValue(sym.Value),
917932
Size: sym.Size,
918933
})
934+
if !cont {
935+
break
936+
}
919937
}
920-
symMap.Finalize()
921-
922-
return &symMap, nil
938+
return nil
923939
}
924940

925941
// ReadSymbols reads the full dynamic symbol table from the ELF
926942
func (f *File) ReadSymbols() (*libpf.SymbolMap, error) {
927-
return f.loadSymbolTable(".symtab")
943+
return f.loadSymbolTable(sectionNameSymtab)
928944
}
929945

930946
// ReadDynamicSymbols reads the full dynamic symbol table from the ELF
931947
func (f *File) ReadDynamicSymbols() (*libpf.SymbolMap, error) {
932-
return f.loadSymbolTable(".dynsym")
948+
return f.loadSymbolTable(sectionNameDynSym)
949+
}
950+
951+
func (f *File) LookupSymbolSlow(name string) (libpf.Symbol, error) {
952+
found := false
953+
res := libpf.Symbol{}
954+
err := f.eachSymbolSlow(sectionNameSymtab, func(s libpf.Symbol) bool {
955+
if string(s.Name) == name {
956+
found = true
957+
res = s
958+
return false
959+
}
960+
return true
961+
})
962+
if err != nil {
963+
return libpf.Symbol{}, err
964+
}
965+
if !found {
966+
return libpf.Symbol{}, ErrSymbolNotFound
967+
}
968+
return res, nil
933969
}
934970

935971
// DynString returns the strings listed for the given tag in the file's dynamic

‎processmanager/ebpf/ebpf.go

+31-18
Original file line numberDiff line numberDiff line change
@@ -253,29 +253,42 @@ func LoadMaps(ctx context.Context, maps map[string]*cebpf.Map) (EbpfHandler, err
253253
// UpdateInterpreterOffsets adds the given moduleRanges to the eBPF map interpreterOffsets.
254254
func (impl *ebpfMapsImpl) UpdateInterpreterOffsets(ebpfProgIndex uint16, fileID host.FileID,
255255
offsetRanges []util.Range) error {
256-
if offsetRanges == nil {
257-
return errors.New("offsetRanges is nil")
258-
}
259-
for _, offsetRange := range offsetRanges {
260-
// The keys of this map are executable-id-and-offset-into-text entries, and
261-
// the offset_range associated with them gives the precise area in that page
262-
// where the main interpreter loop is located. This is required to unwind
263-
// nicely from native code into interpreted code.
264-
key := uint64(fileID)
265-
value := C.OffsetRange{
266-
lower_offset: C.u64(offsetRange.Start),
267-
upper_offset: C.u64(offsetRange.End),
268-
program_index: C.u16(ebpfProgIndex),
269-
}
270-
if err := impl.interpreterOffsets.Update(unsafe.Pointer(&key), unsafe.Pointer(&value),
271-
cebpf.UpdateAny); err != nil {
272-
log.Fatalf("Failed to place interpreter range in map: %v", err)
273-
}
256+
key, value, err := InterpreterOffsetKeyValue(ebpfProgIndex, fileID, offsetRanges)
257+
if err != nil {
258+
return err
259+
}
260+
if err := impl.interpreterOffsets.Update(unsafe.Pointer(&key), unsafe.Pointer(&value),
261+
cebpf.UpdateAny); err != nil {
262+
log.Fatalf("Failed to place interpreter range in map: %v", err)
274263
}
275264

276265
return nil
277266
}
278267

268+
func InterpreterOffsetKeyValue(ebpfProgIndex uint16, fileID host.FileID,
269+
offsetRanges []util.Range) (uint64, C.OffsetRange, error) {
270+
if len(offsetRanges) != 1 && len(offsetRanges) != 2 {
271+
return 0, C.OffsetRange{}, fmt.Errorf("ivalid ranges %+v", offsetRanges)
272+
}
273+
// The keys of this map are executable-id-and-offset-into-text entries, and
274+
// the offset_range associated with them gives the precise area in that page
275+
// where the main interpreter loop is located. This is required to unwind
276+
// nicely from native code into interpreted code.
277+
key := uint64(fileID)
278+
first := offsetRanges[0]
279+
value := C.OffsetRange{
280+
lower_offset1: C.u64(first.Start),
281+
upper_offset1: C.u64(first.End),
282+
program_index: C.u16(ebpfProgIndex),
283+
}
284+
if len(offsetRanges) == 2 {
285+
second := offsetRanges[1]
286+
value.lower_offset2 = C.u64(second.Start)
287+
value.upper_offset2 = C.u64(second.End)
288+
}
289+
return key, value, nil
290+
}
291+
279292
// getInterpreterTypeMap returns the eBPF map for the given typ
280293
// or an error if typ is not supported.
281294
func (impl *ebpfMapsImpl) getInterpreterTypeMap(typ libpf.InterpreterType) (*cebpf.Map, error) {

‎support/ebpf/native_stack_trace.ebpf.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ bpf_map_def SEC("maps") interpreter_offsets = {
6060
.type = BPF_MAP_TYPE_HASH,
6161
.key_size = sizeof(u64),
6262
.value_size = sizeof(OffsetRange),
63-
.max_entries = 32,
63+
.max_entries = 256,
6464
};
6565

6666
// Maps fileID and page to information of stack deltas associated with that page.

‎support/ebpf/tracemgmt.h

+3-1
Original file line numberDiff line numberDiff line change
@@ -422,7 +422,9 @@ static inline int get_next_interpreter(PerCPURecord *record)
422422
// Check if the section id happens to be in the interpreter map.
423423
OffsetRange *range = bpf_map_lookup_elem(&interpreter_offsets, &section_id);
424424
if (range != 0) {
425-
if ((section_offset >= range->lower_offset) && (section_offset <= range->upper_offset)) {
425+
if (
426+
((section_offset >= range->lower_offset1) && (section_offset <= range->upper_offset1)) ||
427+
((section_offset >= range->lower_offset2) && (section_offset <= range->upper_offset2))) {
426428
DEBUG_PRINT("interpreter_offsets match %d", range->program_index);
427429
if (!unwinder_is_done(record, range->program_index)) {
428430
increment_metric(metricID_UnwindCallInterpreter);

‎support/ebpf/tracer.ebpf.release.amd64

4.36 KB
Binary file not shown.

‎support/ebpf/types.h

+5-2
Original file line numberDiff line numberDiff line change
@@ -773,8 +773,11 @@ typedef struct StackDeltaPageInfo {
773773
// the upper boundary of the loop, and the relevant index to call in the prog
774774
// array.
775775
typedef struct OffsetRange {
776-
u64 lower_offset;
777-
u64 upper_offset;
776+
u64 lower_offset1;
777+
u64 upper_offset1;
778+
// extra range for .cold interpreter chunk
779+
u64 lower_offset2;
780+
u64 upper_offset2;
778781
u16 program_index; // The interpreter-specific program index to call.
779782
} OffsetRange;
780783

‎tools/coredump/ebpfmaps.go

+4-6
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,11 @@ func (emc *ebpfMapsCoredump) CollectMetrics() []metrics.Metric {
4444

4545
func (emc *ebpfMapsCoredump) UpdateInterpreterOffsets(ebpfProgIndex uint16,
4646
fileID host.FileID, offsetRanges []util.Range) error {
47-
offsetRange := offsetRanges[0]
48-
value := C.OffsetRange{
49-
lower_offset: C.u64(offsetRange.Start),
50-
upper_offset: C.u64(offsetRange.End),
51-
program_index: C.u16(ebpfProgIndex),
47+
key, value, err := pmebpf.InterpreterOffsetKeyValue(ebpfProgIndex, fileID, offsetRanges)
48+
if err != nil {
49+
return err
5250
}
53-
emc.ctx.addMap(&C.interpreter_offsets, C.u64(fileID), libpf.SliceFrom(&value))
51+
emc.ctx.addMap(&C.interpreter_offsets, C.u64(key), libpf.SliceFrom(&value))
5452
return nil
5553
}
5654

‎tools/coredump/testdata/amd64/alpine320-nobuildid.json

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
{
2+
"coredump-ref": "3eb6bae4e0089983f436d6bbd4a0b7ee0d72738eac29f15495494f53bc82263d",
3+
"threads": [
4+
{
5+
"lwp": 80,
6+
"frames": [
7+
"fib+1 in /mnt/trash/qwe.py:2",
8+
"fib+3 in /mnt/trash/qwe.py:4",
9+
"fib+3 in /mnt/trash/qwe.py:4",
10+
"fib+3 in /mnt/trash/qwe.py:4",
11+
"fib+3 in /mnt/trash/qwe.py:4",
12+
"fib+3 in /mnt/trash/qwe.py:4",
13+
"fib+3 in /mnt/trash/qwe.py:4",
14+
"fib+3 in /mnt/trash/qwe.py:4",
15+
"fib+3 in /mnt/trash/qwe.py:4",
16+
"fib+3 in /mnt/trash/qwe.py:4",
17+
"fib+3 in /mnt/trash/qwe.py:4",
18+
"fib+3 in /mnt/trash/qwe.py:4",
19+
"fib+3 in /mnt/trash/qwe.py:4",
20+
"fib+3 in /mnt/trash/qwe.py:4",
21+
"fib+3 in /mnt/trash/qwe.py:4",
22+
"fib+3 in /mnt/trash/qwe.py:4",
23+
"fib+3 in /mnt/trash/qwe.py:4",
24+
"fib+3 in /mnt/trash/qwe.py:4",
25+
"fib+3 in /mnt/trash/qwe.py:4",
26+
"fib+3 in /mnt/trash/qwe.py:4",
27+
"fib+3 in /mnt/trash/qwe.py:4",
28+
"fib+3 in /mnt/trash/qwe.py:4",
29+
"fib+3 in /mnt/trash/qwe.py:4",
30+
"fib+3 in /mnt/trash/qwe.py:4",
31+
"fib+3 in /mnt/trash/qwe.py:4",
32+
"fib+3 in /mnt/trash/qwe.py:4",
33+
"fib+3 in /mnt/trash/qwe.py:4",
34+
"fib+3 in /mnt/trash/qwe.py:4",
35+
"fib+3 in /mnt/trash/qwe.py:4",
36+
"fib+3 in /mnt/trash/qwe.py:4",
37+
"fib+3 in /mnt/trash/qwe.py:4",
38+
"fib+3 in /mnt/trash/qwe.py:4",
39+
"fib+3 in /mnt/trash/qwe.py:4",
40+
"fib+3 in /mnt/trash/qwe.py:4",
41+
"fib+3 in /mnt/trash/qwe.py:4",
42+
"fib+3 in /mnt/trash/qwe.py:4",
43+
"fib+3 in /mnt/trash/qwe.py:4",
44+
"<module>+6 in /mnt/trash/qwe.py:7",
45+
"<interpreter trampoline>+0 in <shim>:1",
46+
"libpython3.12.so.1.0+0x95b58",
47+
"libpython3.12.so.1.0+0x1ecfe2",
48+
"libpython3.12.so.1.0+0x2136b6",
49+
"libpython3.12.so.1.0+0x20c91b",
50+
"libpython3.12.so.1.0+0x226f72",
51+
"libpython3.12.so.1.0+0x2264f1",
52+
"libpython3.12.so.1.0+0x2260d3",
53+
"libpython3.12.so.1.0+0x21f742",
54+
"libpython3.12.so.1.0+0x1d6ed6",
55+
"ld-musl-x86_64.so.1+0x1c709",
56+
"python3.12+0x1045",
57+
"<unwinding aborted due to error native_small_pc>"
58+
]
59+
}
60+
],
61+
"modules": [
62+
{
63+
"ref": "983f4cac5caf833fbf7d5d28ac8d6a55d3cbab6152d37a246af1a9991f72d8b1",
64+
"local-path": "/usr/lib/debug/lib/ld-musl-x86_64.so.1.debug"
65+
},
66+
{
67+
"ref": "497dd0d2b4a80bfd11339306c84aa752d811f612a398cb526a0a9ac2f426c0b8",
68+
"local-path": "/usr/lib/libpython3.12.so.1.0"
69+
},
70+
{
71+
"ref": "02a162bd137903ef2511ccb6edc32eaa63a9c9c66c8474f6ce7d9d47fc5a1e71",
72+
"local-path": "/usr/lib/debug/usr/lib/libpython3.12.so.1.0.debug"
73+
},
74+
{
75+
"ref": "9cf22fb673cad95247584fd0bc21b95aa37ae152a8bd01022a494e4f7ec3854c",
76+
"local-path": "/usr/bin/python3.12"
77+
},
78+
{
79+
"ref": "f2c83c6e5aea0e1e42b69e344a2eb0aac35b361cad7497157f727dbe30b8565e",
80+
"local-path": "/usr/lib/debug/usr/bin/python3.12.debug"
81+
},
82+
{
83+
"ref": "c9cbfe6a266c104f74629a257e2017020cba7a0da97caefa4c1d068ea6fe698c",
84+
"local-path": "/lib/ld-musl-x86_64.so.1"
85+
}
86+
]
87+
}

0 commit comments

Comments
 (0)