Skip to content

Commit a95d78e

Browse files
chrisnaschrisnas
authored
[Profiler] Improve reference chain (#8966)
## Summary of changes - Add namespace to type name in reference chain - Avoid stopping the generation of class histogram on error ## Reason for change Help comparing reference chain with class histogram ## Implementation details - change the serialization to integrate namespaces - improve error management to avoid cutting the chain and allow next computation ## Test coverage - update tests - add new tests ## Other details <!-- Fixes #{issue} --> <!-- ⚠️ Note: Where possible, please obtain 2 approvals prior to merging. Unless CODEOWNERS specifies otherwise, for external teams it is typically best to have one review from a team member, and one review from apm-dotnet. Trivial changes do not require 2 reviews. MergeQueue is NOT enabled in this repository. If you have write access to the repo, the PR has 1-2 approvals (see above), and all of the required checks have passed, you can use the Squash and Merge button to merge the PR. If you don't have write access, or you need help, reach out in the #apm-dotnet channel in Slack. -->
1 parent 6c2f702 commit a95d78e

29 files changed

Lines changed: 3308 additions & 529 deletions

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.cpp‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
#include "ClrLifetime.h"
2727
#include "Configuration.h"
2828
#include "ContentionProvider.h"
29+
#include "CoreLibModuleProvider.h"
2930
#include "CpuTimeProvider.h"
3031
#include "DebugInfoStore.h"
3132
#include "EnabledProfilers.h"
@@ -188,6 +189,9 @@ void CorProfilerCallback::InitializeServices()
188189
_pFrameStore = std::make_unique<FrameStore>(
189190
_pCorProfilerInfo, _pConfiguration.get(), _pDebugInfoStore.get(), _managedCodeCache.get());
190191

192+
// must be created before the components that resolve core library types (i.e. exceptions and heap snapshot)
193+
_pCoreLibModuleProvider = std::make_unique<CoreLibModuleProvider>(_pCorProfilerInfo);
194+
191195
// Create service instances
192196
_pThreadsCpuManager = RegisterService<ThreadsCpuManager>();
193197

@@ -305,6 +309,7 @@ void CorProfilerCallback::InitializeServices()
305309
_pCorProfilerInfo,
306310
_pManagedThreadList,
307311
_pFrameStore.get(),
312+
_pCoreLibModuleProvider.get(),
308313
_pConfiguration.get(),
309314
_rawSampleTransformer.get(),
310315
_metricsRegistry,
@@ -399,6 +404,7 @@ void CorProfilerCallback::InitializeServices()
399404
_pConfiguration.get(),
400405
_pCorProfilerInfoEvents,
401406
_pFrameStore.get(),
407+
_pCoreLibModuleProvider.get(),
402408
_pThreadsCpuManager,
403409
_metricsRegistry,
404410
_pNativeThreadList,
@@ -1598,6 +1604,15 @@ HRESULT STDMETHODCALLTYPE CorProfilerCallback::Initialize(IUnknown* corProfilerI
15981604
eventMask |= COR_PRF_MONITOR_MODULE_LOADS | COR_PRF_MONITOR_CLASS_LOADS;
15991605
}
16001606

1607+
if (_pConfiguration->IsHeapSnapshotEnabled())
1608+
{
1609+
// CoreLibModuleProvider is fed from ModuleLoadFinished and InlineVTCache needs it
1610+
// to resolve the primitive types of inline value type fields. Exception profiling
1611+
// asks for the same flag and is enabled by default, so this only matters when it
1612+
// has been turned off.
1613+
eventMask |= COR_PRF_MONITOR_MODULE_LOADS;
1614+
}
1615+
16011616
if (_pConfiguration->IsAllocationRecorderEnabled() && !_pConfiguration->GetProfilesOutputDirectory().empty())
16021617
{
16031618
// for GC for JIT
@@ -2059,6 +2074,12 @@ HRESULT STDMETHODCALLTYPE CorProfilerCallback::ModuleLoadFinished(ModuleID modul
20592074
}
20602075
#endif
20612076

2077+
// must be done first: the other consumers rely on the core library module id being set
2078+
if (_pCoreLibModuleProvider != nullptr)
2079+
{
2080+
_pCoreLibModuleProvider->OnModuleLoaded(moduleId);
2081+
}
2082+
20622083
if (_pConfiguration->IsExceptionProfilingEnabled())
20632084
{
20642085
_pExceptionsProvider->OnModuleLoaded(moduleId);
@@ -2074,6 +2095,13 @@ HRESULT STDMETHODCALLTYPE CorProfilerCallback::ModuleLoadFinished(ModuleID modul
20742095

20752096
HRESULT STDMETHODCALLTYPE CorProfilerCallback::ModuleUnloadStarted(ModuleID moduleId)
20762097
{
2098+
if (_pHeapSnapshotManager != nullptr)
2099+
{
2100+
// Notified here rather than from ModuleUnloadFinished so that the ClassIDs of the
2101+
// module stop being used before the runtime starts freeing what they point to.
2102+
_pHeapSnapshotManager->OnModuleUnloaded();
2103+
}
2104+
20772105
return S_OK;
20782106
}
20792107

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native/CorProfilerCallback.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
#include <vector>
5454

5555
class ContentionProvider;
56+
class CoreLibModuleProvider;
5657
class IService;
5758
class IThreadsCpuManager;
5859
class IManagedThreadList;
@@ -284,6 +285,8 @@ private :
284285
bool _IsManagedConfigurationSet = false; // profiler can't start before this becomes true
285286
std::unique_ptr<IAppDomainStore> _pAppDomainStore = nullptr;
286287
std::unique_ptr<IFrameStore> _pFrameStore = nullptr;
288+
// shared by the components that need to resolve types defined in the core library
289+
std::unique_ptr<CoreLibModuleProvider> _pCoreLibModuleProvider = nullptr;
287290
std::unique_ptr<IRuntimeInfo> _pRuntimeInfo = nullptr;
288291
bool _isFrameworkVersionKnown = false;
289292
std::unique_ptr<IEnabledProfilers> _pEnabledProfilers = nullptr;

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native/CoreLibModuleProvider.cpp‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
2+
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2022 Datadog, Inc.
3+
4+
#include "CoreLibModuleProvider.h"
5+
6+
#include "FrameStore.h"
7+
#include "HResultConverter.h"
8+
#include "Log.h"
9+
10+
#include <utility>
11+
12+
CoreLibModuleProvider::CoreLibModuleProvider(ICorProfilerInfo4* pCorProfilerInfo) :
13+
_pCorProfilerInfo{pCorProfilerInfo},
14+
_moduleId{0}
15+
{
16+
}
17+
18+
bool CoreLibModuleProvider::OnModuleLoaded(ModuleID moduleId)
19+
{
20+
if (_moduleId.load(std::memory_order_acquire) != 0)
21+
{
22+
return false;
23+
}
24+
25+
std::string assemblyName;
26+
if (!FrameStore::GetAssemblyName(_pCorProfilerInfo, moduleId, assemblyName))
27+
{
28+
Log::Warn("Failed to retrieve assembly name for module ", moduleId);
29+
return false;
30+
}
31+
32+
if (assemblyName != "System.Private.CoreLib" && assemblyName != "mscorlib")
33+
{
34+
return false;
35+
}
36+
37+
ModuleID noModule = 0;
38+
if (!_moduleId.compare_exchange_strong(noModule, moduleId, std::memory_order_acq_rel))
39+
{
40+
// another thread got there first
41+
return false;
42+
}
43+
44+
Log::Debug("Core library module found: ", assemblyName, " (module ", moduleId, ")");
45+
return true;
46+
}
47+
48+
ModuleID CoreLibModuleProvider::GetModuleId() const
49+
{
50+
return _moduleId.load(std::memory_order_acquire);
51+
}
52+
53+
ComPtr<IMetaDataImport2> CoreLibModuleProvider::GetMetadata()
54+
{
55+
std::lock_guard<std::mutex> lock(_lock);
56+
57+
if (GetMetadataNoLock() == nullptr)
58+
{
59+
return {};
60+
}
61+
62+
return _pMetadataImport;
63+
}
64+
65+
ClassID CoreLibModuleProvider::ResolveTypeInCoreLib(const WCHAR* fullTypeName)
66+
{
67+
if (fullTypeName == nullptr)
68+
{
69+
return 0;
70+
}
71+
72+
ModuleID moduleId = _moduleId.load(std::memory_order_acquire);
73+
if (moduleId == 0)
74+
{
75+
return 0;
76+
}
77+
78+
std::lock_guard<std::mutex> lock(_lock);
79+
80+
shared::WSTRING typeName(fullTypeName);
81+
auto entry = _resolvedTypes.find(typeName);
82+
if (entry != _resolvedTypes.end())
83+
{
84+
return entry->second;
85+
}
86+
87+
ClassID classId = ResolveTypeNoLock(moduleId, fullTypeName);
88+
89+
if (classId != 0)
90+
{
91+
_resolvedTypes.emplace(std::move(typeName), classId);
92+
}
93+
94+
return classId;
95+
}
96+
97+
IMetaDataImport2* CoreLibModuleProvider::GetMetadataNoLock()
98+
{
99+
if (_pMetadataImport.Get() != nullptr)
100+
{
101+
return _pMetadataImport.Get();
102+
}
103+
104+
ModuleID moduleId = _moduleId.load(std::memory_order_acquire);
105+
if (moduleId == 0)
106+
{
107+
return nullptr;
108+
}
109+
110+
HRESULT hr = _pCorProfilerInfo->GetModuleMetaData(
111+
moduleId, CorOpenFlags::ofRead, IID_IMetaDataImport2,
112+
reinterpret_cast<IUnknown**>(_pMetadataImport.GetAddressOf()));
113+
114+
if (FAILED(hr))
115+
{
116+
Log::Debug("GetModuleMetaData() failed for the core library with HRESULT = ", HResultConverter::ToStringWithCode(hr));
117+
_pMetadataImport.Reset();
118+
return nullptr;
119+
}
120+
121+
return _pMetadataImport.Get();
122+
}
123+
124+
ClassID CoreLibModuleProvider::ResolveTypeNoLock(ModuleID moduleId, const WCHAR* fullTypeName)
125+
{
126+
IMetaDataImport2* pMetadataImport = GetMetadataNoLock();
127+
if (pMetadataImport == nullptr)
128+
{
129+
return 0;
130+
}
131+
132+
mdTypeDef typeDef = mdTokenNil;
133+
HRESULT hr = pMetadataImport->FindTypeDefByName(fullTypeName, mdTokenNil, &typeDef);
134+
if (FAILED(hr) || TypeFromToken(typeDef) != mdtTypeDef)
135+
{
136+
return 0;
137+
}
138+
139+
ClassID classId = 0;
140+
hr = _pCorProfilerInfo->GetClassFromTokenAndTypeArgs(moduleId, typeDef, 0, nullptr, &classId);
141+
142+
return SUCCEEDED(hr) ? classId : 0;
143+
}

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native/CoreLibModuleProvider.h‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
2+
// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2022 Datadog, Inc.
3+
4+
#pragma once
5+
6+
#include "cor.h"
7+
#include "corprof.h"
8+
9+
#include "shared/src/native-src/com_ptr.h"
10+
#include "shared/src/native-src/string.h"
11+
12+
#include <atomic>
13+
#include <mutex>
14+
#include <unordered_map>
15+
16+
// Tracks the core library module (System.Private.CoreLib or mscorlib) and resolves
17+
// types defined in it.
18+
//
19+
// Several components need to look up well known types such as System.Exception or the
20+
// primitive types backing value type fields. Resolving them requires the module that
21+
// DEFINES them: ICorProfilerInfo::GetClassFromTokenAndTypeArgs only accepts an mdTypeDef
22+
// belonging to the module it is given, and feeding it a TypeRef from another module makes
23+
// the CLR attempt a type load that throws EETypeLoadException.
24+
//
25+
// This provider is fed by CorProfilerCallback::ModuleLoadFinished and is safe to query
26+
// from any thread, including during a GC callback: the core library types it resolves are
27+
// always already loaded, so no type load is triggered.
28+
class CoreLibModuleProvider
29+
{
30+
public:
31+
explicit CoreLibModuleProvider(ICorProfilerInfo4* pCorProfilerInfo);
32+
33+
CoreLibModuleProvider(const CoreLibModuleProvider&) = delete;
34+
CoreLibModuleProvider& operator=(const CoreLibModuleProvider&) = delete;
35+
36+
// Records the module id the first time the core library is seen.
37+
// Returns true only for the call that identified it.
38+
bool OnModuleLoaded(ModuleID moduleId);
39+
40+
// Returns 0 until the core library has been loaded.
41+
ModuleID GetModuleId() const;
42+
43+
// Lazily opens the core library metadata. The returned ComPtr owns an AddRef.
44+
// Returns an empty ComPtr when the core library is not loaded yet or metadata
45+
// is unavailable.
46+
ComPtr<IMetaDataImport2> GetMetadata();
47+
48+
// Resolves a type defined in the core library (e.g. WStr("System.Int32")) to its ClassID.
49+
// Returns 0 when the core library is not loaded yet or when the type cannot be resolved.
50+
ClassID ResolveTypeInCoreLib(const WCHAR* fullTypeName);
51+
52+
private:
53+
// Both helpers require _lock to be held.
54+
IMetaDataImport2* GetMetadataNoLock();
55+
ClassID ResolveTypeNoLock(ModuleID moduleId, const WCHAR* fullTypeName);
56+
57+
ICorProfilerInfo4* _pCorProfilerInfo;
58+
59+
// Set once, then never changes: read without taking the lock.
60+
std::atomic<ModuleID> _moduleId;
61+
62+
std::mutex _lock;
63+
ComPtr<IMetaDataImport2> _pMetadataImport;
64+
std::unordered_map<shared::WSTRING, ClassID> _resolvedTypes;
65+
};

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native/Datadog.Profiler.Native.vcxproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,7 @@
256256
<ClInclude Include="FrameworkThreadInfo.h" />
257257
<ClInclude Include="GCDescReader.h" />
258258
<ClInclude Include="InlineVTCache.h" />
259+
<ClInclude Include="MemoryFaultGuard.h" />
259260
<ClInclude Include="HeapSnapshotManager.h" />
260261
<ClInclude Include="SnapshotCooldown.h" />
261262
<ClInclude Include="IGCDumpListener.h" />
@@ -330,6 +331,7 @@
330331
<ClInclude Include="cgroup.h" />
331332
<ClInclude Include="ClrLifetime.h" />
332333
<ClInclude Include="Configuration.h" />
334+
<ClInclude Include="CoreLibModuleProvider.h" />
333335
<ClInclude Include="CorProfilerCallback.h" />
334336
<ClInclude Include="CorProfilerCallbackFactory.h" />
335337
<ClInclude Include="CpuTimeProvider.h" />
@@ -412,6 +414,7 @@
412414
<ClCompile Include="ClrLifetime.cpp" />
413415
<ClCompile Include="Configuration.cpp" />
414416
<ClCompile Include="ContentionProvider.cpp" />
417+
<ClCompile Include="CoreLibModuleProvider.cpp" />
415418
<ClCompile Include="CorProfilerCallback.cpp" />
416419
<ClCompile Include="CorProfilerCallbackFactory.cpp" />
417420
<ClCompile Include="CounterMetric.cpp" />

‎profiler/src/ProfilerEngine/Datadog.Profiler.Native/Datadog.Profiler.Native.vcxproj.filters‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@
5959
</Filter>
6060
</ItemGroup>
6161
<ItemGroup>
62+
<ClInclude Include="CoreLibModuleProvider.h">
63+
<Filter>CorProfiler-Infrastructure</Filter>
64+
</ClInclude>
6265
<ClInclude Include="CorProfilerCallback.h">
6366
<Filter>CorProfiler-Infrastructure</Filter>
6467
</ClInclude>
@@ -513,6 +516,9 @@
513516
<ClInclude Include="InlineVTCache.h">
514517
<Filter>HeapSnapshot</Filter>
515518
</ClInclude>
519+
<ClInclude Include="MemoryFaultGuard.h">
520+
<Filter>HeapSnapshot</Filter>
521+
</ClInclude>
516522
<ClInclude Include="ReferenceChainTraverser.h">
517523
<Filter>HeapSnapshot</Filter>
518524
</ClInclude>
@@ -545,6 +551,9 @@
545551
<ClCompile Include="HResultConverter.cpp">
546552
<Filter>Utils</Filter>
547553
</ClCompile>
554+
<ClCompile Include="CoreLibModuleProvider.cpp">
555+
<Filter>CorProfiler-Infrastructure</Filter>
556+
</ClCompile>
548557
<ClCompile Include="CorProfilerCallback.cpp">
549558
<Filter>CorProfiler-Infrastructure</Filter>
550559
</ClCompile>

0 commit comments

Comments
 (0)