feat(tarko): switch gui agent to percentage coordinates

Remove devicePixelRatio from screenshot metadata and return percentage
coordinates instead of absolute coordinates from browser operator.
Simplify BrowserControlRenderer to use percentage coordinates directly.

- Remove devicePixelRatio metadata from environment_input events
- Calculate percentage coordinates in browser operator execute method
- Update BrowserControlRenderer to use percentage positioning
- Remove unused getDevicePixelRatio method

Author: ulivz

multimodal/agent-tars/core/src/browser/browser-gui-agent.ts

@@ -196,9 +196,8 @@ wait()                                         - Wait 5 seconds and take a scree
           // Get page title
           const title = document.title || 'Untitled Page';
 
-          // @ts-expect-error
           // Get visible text content
-          const getVisibleText = (node) => {
+          const getVisibleText = (node: any) => {
             if (node.nodeType === Node.TEXT_NODE) {
               return node.textContent || '';
             }
@@ -214,13 +213,10 @@ wait()                                         - Wait 5 seconds and take a scree
 
             let text = '';
             for (const child of Array.from(node.childNodes)) {
-              // @ts-expect-error
-              if (child.nodeType === Node.ELEMENT_NODE) {
+              if ((child as any).nodeType === Node.ELEMENT_NODE) {
                 text += getVisibleText(child);
-                // @ts-expect-error
-              } else if (child.nodeType === Node.TEXT_NODE) {
-                // @ts-expect-error
-                text += child.textContent || '';
+              } else if ((child as any).nodeType === Node.TEXT_NODE) {
+                text += (child as any).textContent || '';
               }
             }
 
@@ -247,7 +243,7 @@ wait()                                         - Wait 5 seconds and take a scree
           description: 'Page Content After Browser Action',
           metadata: {
             type: 'text',
-          },
+          } as AgentEventStream.TextMetadata,
         });
 
         // Send the event
@@ -344,7 +340,7 @@ wait()                                         - Wait 5 seconds and take a scree
         description: 'Browser Screenshot',
         metadata: {
           type: 'screenshot',
-        },
+        } as AgentEventStream.ScreenshotMetadata,
       });
 
       return eventStream.sendEvent(event);
@@ -405,8 +401,7 @@ wait()                                         - Wait 5 seconds and take a scree
         description: 'Browser Screenshot',
         metadata: {
           type: 'screenshot',
-          devicePixelRatio: await this.getDevicePixelRatio(),
-        },
+        } as AgentEventStream.ScreenshotMetadata,
       });
 
       eventStream.sendEvent(event);
@@ -603,19 +598,7 @@ wait()                                         - Wait 5 seconds and take a scree
     }
   }
 
-  /**
-   * Get the device pixel ratio from the browser page
-   */
-  private async getDevicePixelRatio(): Promise<number> {
-    try {
-      const page = await this.getPage();
-      const devicePixelRatio = await page.evaluate(() => window.devicePixelRatio);
-      return devicePixelRatio || 1;
-    } catch (error) {
-      this.logger.warn('Failed to get device pixel ratio, defaulting to 1:', error);
-      return 1;
-    }
-  }
+
 
   /**
    * Get access to the underlying Puppeteer page

multimodal/gui-agent/operator-browser/src/browser-operator.ts

@@ -297,13 +297,16 @@ export class BrowserOperator extends Operator {
       throw error;
     }
 
+    // Calculate percentage coordinates for GUI Agent
+    const startXPercent = startX ? (startX * deviceScaleFactor) / screenWidth : null;
+    const startYPercent = startY ? (startY * deviceScaleFactor) / screenHeight : null;
+
     return {
-      // Hand it over to the upper layer to avoid redundancy
-      // @ts-expect-error fix type later
-      startX,
-      startY,
+      // Return percentage coordinates instead of absolute coordinates
+      startX: startXPercent,
+      startY: startYPercent,
       action_inputs,
-    };
+    } as any;
   }
 
   private async handleClick(x: number, y: number) {

multimodal/tarko/agent-web-ui/src/standalone/workspace/renderers/BrowserControlRenderer.tsx

@@ -30,13 +30,11 @@ export const BrowserControlRenderer: React.FC<BrowserControlRendererProps> = ({
 }) => {
   const { activeSessionId, messages, toolResults, replayState } = useSession();
   const [relatedImage, setRelatedImage] = useState<string | null>(null);
-  const [imageSize, setImageSize] = useState<{ width: number; height: number } | null>(null);
   const [mousePosition, setMousePosition] = useState<{ x: number; y: number } | null>(null);
   const [previousMousePosition, setPreviousMousePosition] = useState<{
     x: number;
     y: number;
   } | null>(null);
-  const [devicePixelRatio, setDevicePixelRatio] = useState<number>(window.devicePixelRatio);
   const imageRef = useRef<HTMLImageElement>(null);
 
   // Extract the visual operation details from panelContent
@@ -64,11 +62,11 @@ export const BrowserControlRenderer: React.FC<BrowserControlRendererProps> = ({
         setPreviousMousePosition(mousePosition);
       }
 
-      // Set new position if coordinates are valid
+      // Set new position if coordinates are valid (now as percentages)
       if (typeof startX === 'number' && typeof startY === 'number') {
         setMousePosition({
-          x: startX,
-          y: startY,
+          x: startX * 100, // Convert to percentage
+          y: startY * 100, // Convert to percentage
         });
       }
     }
@@ -114,15 +112,6 @@ export const BrowserControlRenderer: React.FC<BrowserControlRendererProps> = ({
         if (imgContent && 'image_url' in imgContent && imgContent.image_url.url) {
           setRelatedImage(imgContent.image_url.url);
           foundImage = true;
-
-          // Extract devicePixelRatio from environment input metadata if available
-          if (
-            msg.metadata &&
-            AgentEventStream.isScreenshotMetadata(msg.metadata) &&
-            msg.metadata.devicePixelRatio
-          ) {
-            setDevicePixelRatio(msg.metadata.devicePixelRatio);
-          }
           break;
         }
       }
@@ -137,15 +126,7 @@ export const BrowserControlRenderer: React.FC<BrowserControlRendererProps> = ({
     }
   }, [activeSessionId, messages, toolCallId, environmentImage]);
 
-  // Handler to get image dimensions when loaded
-  const handleImageLoad = () => {
-    if (imageRef.current) {
-      setImageSize({
-        width: imageRef.current.naturalWidth,
-        height: imageRef.current.naturalHeight,
-      });
-    }
-  };
+
 
   return (
     <div className="space-y-6">
@@ -159,27 +140,26 @@ export const BrowserControlRenderer: React.FC<BrowserControlRendererProps> = ({
                 src={relatedImage}
                 alt="Browser Screenshot"
                 className="w-full h-auto object-contain max-h-[70vh]"
-                onLoad={handleImageLoad}
               />
 
               {/* Enhanced mouse cursor overlay */}
-              {mousePosition && imageSize && (
+              {mousePosition && (
                 <motion.div
                   className="absolute pointer-events-none"
                   initial={
                     previousMousePosition
                       ? {
-                          left: `${(previousMousePosition.x / imageSize.width) * 100 * devicePixelRatio}%`,
-                          top: `${(previousMousePosition.y / imageSize.height) * 100 * devicePixelRatio}%`,
+                          left: `${previousMousePosition.x}%`,
+                          top: `${previousMousePosition.y}%`,
                         }
                       : {
-                          left: `${(mousePosition.x / imageSize.width) * 100 * devicePixelRatio}%`,
-                          top: `${(mousePosition.y / imageSize.height) * 100 * devicePixelRatio}%`,
+                          left: `${mousePosition.x}%`,
+                          top: `${mousePosition.y}%`,
                         }
                   }
                   animate={{
-                    left: `${(mousePosition.x / imageSize.width) * 100 * devicePixelRatio}%`,
-                    top: `${(mousePosition.y / imageSize.height) * 100 * devicePixelRatio}%`,
+                    left: `${mousePosition.x}%`,
+                    top: `${mousePosition.y}%`,
                   }}
                   transition={{ duration: 0.5, ease: [0.16, 1, 0.3, 1] }}
                   style={{