Enhance accessibility overlay with new styles and structure for better visibility and interaction

Author: DjDeveloperr

client/src/features/accessibility/AccessibilityOverlay.tsx

@@ -1,9 +1,13 @@
+import type { AriaRole, CSSProperties } from "react";
+
 import type { AccessibilityNode } from "../../api/types";
 import {
   accessibilityKind,
+  accessibilityIdentifier,
   accessibilityRootFrame,
   buildAccessibilityTree,
   findAccessibilityItem,
+  flattenAccessibilityTree,
   primaryAccessibilityText,
   validFrame,
 } from "./accessibilityTree";
@@ -21,6 +25,11 @@ export function AccessibilityOverlay({
 }: AccessibilityOverlayProps) {
   const rootFrame = accessibilityRootFrame(roots);
   const tree = buildAccessibilityTree(roots);
+  const overlayItems = rootFrame
+    ? flattenAccessibilityTree(tree).filter((item) =>
+        validFrame(item.node.frame),
+      )
+    : [];
   const selected = selectedId
     ? framedNode(findAccessibilityItem(tree, selectedId)?.node)
     : null;
@@ -29,18 +38,37 @@ export function AccessibilityOverlay({
       ? framedNode(findAccessibilityItem(tree, hoveredId)?.node)
       : null;
 
-  if (!rootFrame || (!selected && !hovered)) {
+  if (!rootFrame) {
+    return null;
+  }
+  if (overlayItems.length === 0 && !selected && !hovered) {
     return null;
   }
 
   return (
-    <div className="accessibility-overlay" aria-hidden="true">
-      {hovered ? (
-        <NodeRect node={hovered} rootFrame={rootFrame} variant="hovered" />
-      ) : null}
-      {selected ? (
-        <NodeRect node={selected} rootFrame={rootFrame} variant="selected" />
-      ) : null}
+    <div
+      aria-label="Simulator accessibility overlay"
+      className="accessibility-overlay"
+    >
+      <div className="accessibility-dom-overlay">
+        {overlayItems.map((item) => (
+          <AccessibilityDomNode
+            depth={item.depth}
+            id={item.id}
+            key={item.id}
+            node={item.node}
+            rootFrame={rootFrame}
+          />
+        ))}
+      </div>
+      <div className="accessibility-visual-overlay" aria-hidden="true">
+        {hovered ? (
+          <NodeRect node={hovered} rootFrame={rootFrame} variant="hovered" />
+        ) : null}
+        {selected ? (
+          <NodeRect node={selected} rootFrame={rootFrame} variant="selected" />
+        ) : null}
+      </div>
     </div>
   );
 }
@@ -96,3 +124,104 @@ function NodeRect({
     </div>
   );
 }
+
+function AccessibilityDomNode({
+  depth,
+  id,
+  node,
+  rootFrame,
+}: {
+  depth: number;
+  id: string;
+  node: AccessibilityNode;
+  rootFrame: { height: number; width: number; x: number; y: number };
+}) {
+  if (!validFrame(node.frame)) {
+    return null;
+  }
+
+  const label = accessibilityDomLabel(node);
+  const kind = accessibilityKind(node);
+  const role = accessibilityDomRole(kind);
+
+  return (
+    <div
+      aria-checked={
+        role === "checkbox" || role === "switch"
+          ? (node.checked ?? undefined)
+          : undefined
+      }
+      aria-disabled={node.enabled === false ? true : undefined}
+      aria-label={label}
+      aria-level={depth + 1}
+      aria-selected={node.selected ?? undefined}
+      className="accessibility-dom-node"
+      data-simdeck-accessibility-id={id}
+      data-simdeck-accessibility-identifier={
+        accessibilityIdentifier(node) || undefined
+      }
+      data-simdeck-accessibility-kind={kind}
+      data-simdeck-accessibility-source={node.source || undefined}
+      role={role}
+      style={frameStyle(node.frame, rootFrame)}
+      title={label}
+    />
+  );
+}
+
+function frameStyle(
+  frame: { height: number; width: number; x: number; y: number },
+  rootFrame: { height: number; width: number; x: number; y: number },
+): CSSProperties {
+  return {
+    height: `${(frame.height / rootFrame.height) * 100}%`,
+    left: `${((frame.x - rootFrame.x) / rootFrame.width) * 100}%`,
+    top: `${((frame.y - rootFrame.y) / rootFrame.height) * 100}%`,
+    width: `${(frame.width / rootFrame.width) * 100}%`,
+  };
+}
+
+function accessibilityDomLabel(node: AccessibilityNode): string {
+  const text = primaryAccessibilityText(node);
+  const identifier = accessibilityIdentifier(node);
+  const kind = accessibilityKind(node);
+  if (text && identifier && text !== identifier) {
+    return `${kind}: ${text} (${identifier})`;
+  }
+  return text || identifier || kind;
+}
+
+function accessibilityDomRole(kind: string): AriaRole {
+  const normalized = kind.toLowerCase();
+  if (normalized.includes("button")) {
+    return "button";
+  }
+  if (normalized.includes("checkbox")) {
+    return "checkbox";
+  }
+  if (normalized.includes("switch")) {
+    return "switch";
+  }
+  if (
+    normalized.includes("textfield") ||
+    normalized.includes("text field") ||
+    normalized.includes("textbox") ||
+    normalized.includes("searchfield")
+  ) {
+    return "textbox";
+  }
+  if (normalized.includes("slider")) {
+    return "slider";
+  }
+  if (normalized.includes("image") || normalized.includes("icon")) {
+    return "img";
+  }
+  if (
+    normalized.includes("text") ||
+    normalized.includes("label") ||
+    normalized.includes("static")
+  ) {
+    return "text";
+  }
+  return "group";
+}

client/src/styles/components.css

@@ -1667,6 +1667,23 @@
   pointer-events: none;
 }
 
+.accessibility-dom-overlay,
+.accessibility-visual-overlay {
+  position: absolute;
+  inset: 0;
+  pointer-events: none;
+}
+
+.accessibility-dom-node {
+  position: absolute;
+  min-width: 1px;
+  min-height: 1px;
+  overflow: hidden;
+  border: 0;
+  opacity: 0;
+  pointer-events: auto;
+}
+
 .touch-interaction-overlay {
   position: absolute;
   inset: 0;