Skip to content

Commit 802c42a

Browse files
asklarCopilot
andcommitted
Extract important XAML properties into tree dump
TAP DLL: capture key properties from GetPropertyValuesChain during bounds collection. Properties extracted: - Text content: Text, Content, Header, PlaceholderText, Description, Title, Glyph (filtered: skip numeric handle references >10 digits) - Accessibility: AutomationProperties.Name, AutomationProperties.AutomationId, AutomationProperties.HelpText - State: IsEnabled, Visibility, IsChecked, IsSelected, IsOn, Orientation - Other: Source, Tag Properties are serialized as a JSON object in the TAP pipe output and copied into Element.properties by graft_json_node. Text/Content/Header values also populate Element.text for display. The Win32 provider already populates properties (hwnd, style, visible, enabled). XAML elements now get similar treatment, enabling AI consumers to identify controls by their accessible names (e.g. 'Bold (Ctrl+B)'). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 3d39a10 commit 802c42a

2 files changed

Lines changed: 62 additions & 0 deletions

File tree

src/providers/xaml_diag_common.cpp

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,20 @@ static void graft_json_node(const json& j, Element& parent, const std::string& f
172172
}
173173
}
174174

175+
// Copy additional properties if present in TAP DLL output
176+
if (j.contains("properties") && j["properties"].is_object()) {
177+
for (auto& [key, val] : j["properties"].items()) {
178+
if (val.is_string()) {
179+
std::string v = sanitize(val.get<std::string>());
180+
// Use Text/Content/Header as the element's display text if not already set
181+
if (el.text.empty() && (key == "Text" || key == "Content" || key == "Header")) {
182+
el.text = v;
183+
}
184+
el.properties[key] = std::move(v);
185+
}
186+
}
187+
}
188+
175189
if (j.contains("children") && j["children"].is_array()) {
176190
for (auto& child : j["children"]) {
177191
// Pass bridge base (parentOffsetX/Y) — not accumulated — since

src/tap/lvt_tap.cpp

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -330,6 +330,44 @@ class LvtTap : public IObjectWithSite, public IVisualTreeServiceCallback2 {
330330
foundOffset = true;
331331
}
332332
}
333+
// Extract important XAML properties for the tree dump.
334+
// Only capture the first occurrence of each (most-specific in the chain).
335+
// For text-like properties, check ValueType to avoid handle references
336+
// (XAML serializes reference types as numeric handle IDs).
337+
std::wstring valueType = props[i].ValueType ? props[i].ValueType : L"";
338+
bool isTextProp = (name == L"Text" || name == L"Content" ||
339+
name == L"Header" || name == L"PlaceholderText" ||
340+
name == L"Description" || name == L"Title" ||
341+
name == L"Glyph");
342+
// Heuristic: if value looks like a numeric handle (all digits, > 10 chars),
343+
// it's a reference to a string object, not the string itself.
344+
bool looksLikeHandle = value.size() > 10;
345+
if (looksLikeHandle) {
346+
bool allDigits = true;
347+
for (wchar_t c : value) { if (c < L'0' || c > L'9') { allDigits = false; break; } }
348+
looksLikeHandle = allDigits;
349+
}
350+
bool isStateProp = (name == L"AutomationProperties.Name" ||
351+
name == L"AutomationProperties.AutomationId" ||
352+
name == L"AutomationProperties.HelpText" ||
353+
name == L"IsEnabled" || name == L"Visibility" ||
354+
name == L"IsChecked" || name == L"IsSelected" ||
355+
name == L"IsOn" || name == L"Orientation" ||
356+
name == L"Source" || name == L"Tag");
357+
bool isStringValue = valueType == L"String" || valueType == L"" ||
358+
valueType == L"Boolean" || valueType == L"Int32" ||
359+
valueType == L"Double" || valueType == L"Enum";
360+
if (!value.empty() && value != L"0" && !looksLikeHandle &&
361+
((isTextProp && isStringValue) || isStateProp)) {
362+
// Only store if not already present (first = most-specific)
363+
bool found = false;
364+
for (auto& p : node.properties) {
365+
if (p.first == name) { found = true; break; }
366+
}
367+
if (!found) {
368+
node.properties.emplace_back(name, value);
369+
}
370+
}
333371
if (props[i].Type) SysFreeString(props[i].Type);
334372
if (props[i].DeclaringType) SysFreeString(props[i].DeclaringType);
335373
if (props[i].ValueType) SysFreeString(props[i].ValueType);
@@ -490,6 +528,16 @@ class LvtTap : public IObjectWithSite, public IVisualTreeServiceCallback2 {
490528
}
491529
}
492530

531+
// Serialize extracted properties
532+
if (!n.properties.empty()) {
533+
j += L",\"properties\":{";
534+
for (size_t i = 0; i < n.properties.size(); i++) {
535+
if (i) j += L",";
536+
j += L"\"" + Escape(n.properties[i].first) + L"\":\"" + Escape(n.properties[i].second) + L"\"";
537+
}
538+
j += L"}";
539+
}
540+
493541
if (!n.childHandles.empty()) {
494542
j += L",\"children\":[";
495543
for (size_t i = 0; i < n.childHandles.size(); i++) {

0 commit comments

Comments
 (0)