Add mode, depth and boxes arguments to Get Aria Snapshot #5101 (#5163)

Add mode, depth and boxes arguments and a parsed return type to Get Aria Snapshot (#5163)

  Get Aria Snapshot now exposes the three options that locator.ariaSnapshot()
  already offers in the pinned Playwright version, and gains a return type that
  makes a snapshot usable from a test without string parsing.

  New named-only arguments:

  - mode=ai returns the snapshot optimized for AI consumption. It adds element
    references such as [ref=e2], includes snapshots of iframes inside the target,
    and does not wait for a matching element: it fails immediately instead of
    running into the usual timeout. That last difference is documented on the
    keyword and covered by its own test.
  - depth limits the snapshot to the given number of tree levels. Values of zero
    or less are rejected with a ValueError.
  - boxes appends [box=x,y,width,height] to every node.

  New return type:

  return_type=dict loads Playwright's yaml as it is, which keeps role, name and
  every annotation inside the dictionary keys. boxes= and mode=ai made that worse
  by adding more annotations. return_type=parsed splits a snapshot into a tree of
  nodes with role, name, text, props and children. Annotations without a value,
  such as [selected], become True, [level=2] becomes an integer, and box uses the
  same keys as Get BoundingBox. The /url entry of a link is an annotation in
  Playwright rather than an element, so it becomes the url property of the link.
  Playwright's own parseAriaSnapshot is bundled but not exported, so the parsing
  happens on the Python side in Browser/utils/aria_snapshot.py.

  Everything is additive. The defaults reproduce the previous output byte for
  byte, yaml and dict are untouched, and no existing return value changed.

  Tests: unit tests for the parser covering escaped quotes, valueless
  annotations, text nodes, hoisted /url and unknown future label syntax, plus
  acceptance tests for every argument and return type. One acceptance test
  compares a complete parsed tree against a full expected structure on a page
  that exercises every node key in both of its shapes; it was mutation-tested
  against dropped children, swapped name and text, and an unstripped /url.

  Closes #5101

Author: Snooz82

Browser/gen_stub.py

@@ -69,7 +69,7 @@ def parse_kw_stubs():
 from Browser.utils .data_types import (
     ClientCredential, MouseButton, KeyboardModifier, ScrollBehavior, ScrollBehavior, DialogAction, MouseButtonAction,
     NotSet, Dimensions, SizeFields, AreaFields, BoundingBoxFields, SelectionStrategy, ElementRole,
-     AriaSnapshotReturnType, KeyboardInputAction, KeyAction, TextType
+     AriaSnapshotMode, AriaSnapshotReturnType, KeyboardInputAction, KeyAction, TextType
 )
 from Browser.utils.types import Secret
 """

Browser/keywords/getters.py

@@ -37,9 +37,11 @@
 from ..base import LibraryComponent
 from ..generated.playwright_pb2 import Request
 from ..utils import keyword, logger
+from ..utils.aria_snapshot import parse_aria_snapshot
 from ..utils.data_types import (
     ROBOT_FRAMEWORK_BROWSER_NO_SET,
     AreaFields,
+    AriaSnapshotMode,
     AriaSnapshotReturnType,
     BoundingBox,
     BoundingBoxFields,
@@ -69,18 +71,35 @@ def get_aria_snapshot(
         assertion_operator: AssertionOperator | None = None,
         assertion_expected: Any | None = None,
         message: str | None = None,
-    ) -> str | dict | tuple:
-        """Returns the aria snapshot of the element found by ``selector``.
+        *,
+        mode: AriaSnapshotMode = AriaSnapshotMode.default,
+        depth: int | None = None,
+        boxes: bool = False,
+    ) -> str | dict | list | tuple:
+        """Returns the aria snapshot of the element found by ``selector``. See `AriaSnapshotReturnType` for more details and examples.
 
         | =Arguments= | =Description= |
         | ``selector`` | Selector from which the info is to be retrieved. See the `Finding elements` section for details about the selectors. |
-        | ``return_type`` | Defines the return type. Possible values are ``yaml`` (default) and ``dict``. If ``yaml`` is selected, the returned value is a string in YAML format. If ``dict`` is selected, the returned value is a dictionary. |
+        | ``return_type`` | Defines the return type. Possible values are ``yaml`` (default), ``dict`` and ``parsed``. If ``yaml`` is selected, the returned value is a string in YAML format. If ``dict`` is selected, the returned value is a dictionary. If ``parsed`` is selected, the returned value is a tree of node dictionaries. |
         | ``assertion_operator`` | See `Assertions` for further details. Defaults to None. |
         | ``assertion_expected`` | Expected value for the state |
         | ``message`` | overrides the default error message for assertion. |
+        | ``mode`` | Defines the snapshot mode. Possible values are ``default`` (default) and ``ai``. See `AriaSnapshotMode` for more details. |
+        | ``depth`` | Limits the snapshot to the given number of tree levels. Must be a positive integer. Defaults to ``None``, which does not limit the depth. |
+        | ``boxes`` | If ``True``, the bounding box of each element is appended to its line as ``[box=x,y,width,height]``. Coordinates are relative to the viewport, in CSS pixels. Defaults to ``False``. |
 
         Keyword uses strict mode, see `Finding elements` for more details about strict mode.
 
+        With ``mode=ai`` the snapshot is optimized for AI consumption: it contains element
+        references like ``[ref=e2]`` and the content of iframes inside the element. It also
+        does not wait for a matching element, but fails immediately when no element matches,
+        instead of failing with a timeout like the ``default`` mode does.
+
+        With ``return_type=dict`` the YAML returned by Playwright is loaded as is. The
+        ``[ref=...]`` and ``[box=...]`` annotations are therefore part of the dictionary
+        keys, not separate entries. Use ``return_type=parsed`` to get them as separate
+        values of each node.
+
         Optionally asserts that the snapshot matches the specified assertion. See
         `Assertions` for further details for the assertion arguments. By default assertion
         is not done.
@@ -93,16 +112,26 @@ def get_aria_snapshot(
 
         [https://forum.robotframework.org/t//4303|Comment >>]
         """
+        if depth is not None and depth <= 0:
+            raise ValueError(f"depth must be a positive integer, but got: {depth}")
         selector = self.presenter_mode(selector, self.strict_mode)
         with self.playwright.grpc_channel() as stub:
             response = stub.AriaSnapShot(
-                Request.AriaSnapShot(locator=selector, strict=self.strict_mode)
+                Request.AriaSnapShot(
+                    locator=selector,
+                    strict=self.strict_mode,
+                    mode="ai" if mode is AriaSnapshotMode.ai else "",
+                    depth=depth or 0,
+                    boxes=boxes,
+                )
             )
         logger.info(response.log)
         value = response.body
         logger.info(f"Aria Snapshot: {value}")
         if return_type is AriaSnapshotReturnType.dict:
             value = yaml.safe_load(value) if value else {}
+        elif return_type is AriaSnapshotReturnType.parsed:
+            value = parse_aria_snapshot(value)
         formatter = self.get_assertion_formatter("Get Aria Snapshot")
         return verify_assertion(
             value,

Browser/utils/aria_snapshot.py

@@ -0,0 +1,110 @@
+# Copyright 2020-     Robot Framework Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import re
+from typing import Any
+
+import yaml
+from robot.utils import DotDict
+
+_LABEL = re.compile(
+    r"""
+    ^(?P<role>\S+)
+    (?:\s+"(?P<name>(?:[^"\\]|\\.)*)")?
+    (?P<properties>(?:\s*\[[^\]]*\])*)
+    \s*$
+    """,
+    re.VERBOSE,
+)
+_PROPERTY = re.compile(r"\[([^\]]*)\]")
+_INTEGER = re.compile(r"^-?\d+$")
+_BOX_FIELDS = ("x", "y", "width", "height")
+
+
+def parse_aria_snapshot(snapshot: str) -> list[DotDict]:
+    """Turns Playwright's aria snapshot YAML into a tree of node dictionaries.
+
+    Every node has the keys ``role``, ``name``, ``text``, ``props`` and
+    ``children``. Nodes are ``DotDict``s, so their values are reachable both as
+    ``node["role"]`` and as ``node.role``.
+    """
+    if not snapshot:
+        return []
+    return _to_nodes(yaml.safe_load(snapshot))
+
+
+def _to_nodes(loaded: Any) -> list[DotDict]:
+    if not loaded:
+        return []
+    return [_to_node(entry) for entry in loaded]
+
+
+def _to_node(entry: Any) -> DotDict:
+    if isinstance(entry, dict):
+        [(label, content)] = entry.items()
+        return _node(label, content)
+    return _node(entry, None)
+
+
+def _node(label: str, content: Any) -> DotDict:
+    node = _parse_label(label)
+    if isinstance(content, list):
+        node.children = _hoist_properties(node, content)
+    elif content is not None:
+        node.text = str(content)
+    return node
+
+
+def _hoist_properties(node: DotDict, content: list) -> list[DotDict]:
+    """Playwright renders link targets as a ``/url`` child, not as an element."""
+    children = []
+    for entry in content:
+        if isinstance(entry, dict):
+            [(label, value)] = entry.items()
+            if label.startswith("/"):
+                node.props[label[1:]] = value
+                continue
+        children.append(_to_node(entry))
+    return children
+
+
+def _parse_label(label: str) -> DotDict:
+    match = _LABEL.match(label)
+    if not match:
+        return DotDict(role=label, name=None, text=None, props=DotDict(), children=[])
+    name = match["name"]
+    return DotDict(
+        role=match["role"],
+        name=None if name is None else name.replace('\\"', '"'),
+        text=None,
+        props=_parse_properties(match["properties"]),
+        children=[],
+    )
+
+
+def _parse_properties(properties: str) -> DotDict:
+    parsed = DotDict()
+    for property_ in _PROPERTY.findall(properties):
+        name, separator, value = property_.partition("=")
+        parsed[name] = _parse_property_value(name, value) if separator else True
+    return parsed
+
+
+def _parse_property_value(name: str, value: str) -> Any:
+    if name == "box":
+        coordinates = value.split(",")
+        if len(coordinates) == len(_BOX_FIELDS):
+            return DotDict(zip(_BOX_FIELDS, [int(c) for c in coordinates], strict=True))
+    if _INTEGER.match(value):
+        return int(value)
+    return value