slim find_ui_elements and add sense to prompt

Author: aadya940

orbit/_tools/ui.py

@@ -126,6 +126,32 @@ def _cached_find_elements_hwnd(
     )
 
 
+def _slim_element(el: dict) -> dict:
+    """Strip an element dict to the fields the LLM needs."""
+    slim = {"oculos_id": el.get("oculos_id")}
+    if el.get("element_type"):
+        slim["element_type"] = el["element_type"]
+    for key in ("name", "title", "label"):
+        val = el.get(key)
+        if val:
+            slim[key] = val
+    for key in ("value", "text_content"):
+        val = el.get(key)
+        if val:
+            slim[key] = val
+    for key in ("checked", "toggle_state", "is_selected"):
+        val = el.get(key)
+        if val is not None:
+            slim[key] = val
+    if el.get("is_enabled") is False:
+        slim["is_enabled"] = False
+    return slim
+
+
+def _slim_elements(elements: list[dict]) -> list[dict]:
+    return [_slim_element(el) for el in elements]
+
+
 def list_active_windows() -> Dict[str, Any]:
     """
     Retrieves a list of all currently visible desktop windows.
@@ -318,7 +344,7 @@ def find_ui_elements(
             eid = el.get("oculos_id")
             if eid:
                 _element_meta[eid] = {"pid": pid, "query": query, "element_type": element_type}
-        return {"status": "success", "elements": elements}
+        return {"status": "success", "elements": _slim_elements(elements)}
     except Exception as e:
         return {"status": "error", "message": f"Failed to find elements: {str(e)}"}
 
@@ -391,7 +417,7 @@ def find_ui_elements_hwnd(
                 "message": "No elements found matching the criteria.",
                 "elements": [],
             }
-        return {"status": "success", "elements": elements}
+        return {"status": "success", "elements": _slim_elements(elements)}
     except Exception as e:
         return {
             "status": "error",

orbit/prompts.py

@@ -1,6 +1,13 @@
 SYSTEM_PROMPT = """
 You are an expert desktop automation agent. Complete tasks with the minimum number of tool calls.
 
+── SENSE ────────────────────────────────────────────────────────────
+0. Before acting, understand the context. Read what the page is asking,
+   use information from the task description and any referenced files,
+   and provide thoughtful, relevant responses. Never fill a field with
+   placeholder or meaningless values — if you don't have the information,
+   call request_human instead of guessing.
+
 ── WINDOW & PID MANAGEMENT ───────────────────────────────────────────
 1. Call list_active_windows once to get PIDs. Cache every PID immediately.
    Never repeat unless a new window has opened.