feat: change render method

Author: imguoguo

browser/src/App.tsx

@@ -1,4 +1,4 @@
-import { useCallback, useEffect, useRef, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 import { Alert, Result, Spin } from 'antd';
 import clsx from 'clsx';
 import { useAtomValue, useSetAtom, useAtom } from 'jotai';
@@ -29,17 +29,17 @@ const App = () => {
   const isKeyboardEnable = useAtomValue(isKeyboardEnableAtom);
   const setResolution = useSetAtom(resolutionAtom);
   const canvasRef = useRef<HTMLCanvasElement>(null);
-  const requestRef = useRef<number>(0);
   const videoRef = useRef<HTMLVideoElement>(null);
 
   const [isLoading, setIsLoading] = useState(true);
   const [isCameraAvailable, setIsCameraAvailable] = useState(false);
 
-  const renderFrame = useCallback(() => {
-    const video = videoRef.current;
+  const videoStyle = clsx('block select-none origin-center max-w-full max-h-full object-scale-down', mouseStyle)
+
+  const renderFrame = (frame: VideoFrame) => {
     const canvas = canvasRef.current;
 
-    if (!video || !canvas || video.paused || video.ended) return;
+    if (!canvas) return;
 
     const ctx = canvas.getContext('2d');
     if (!ctx) return;
@@ -51,17 +51,15 @@ const App = () => {
     ctx.rotate((videoRotate * Math.PI) / 180);
 
     ctx.drawImage(
-      video,
-      -video.videoWidth / 2,
-      -video.videoHeight / 2,
-      video.videoWidth,
-      video.videoHeight
+      frame,
+      -frame.displayWidth / 2,
+      -frame.displayHeight / 2,
+      frame.displayWidth,
+      frame.displayHeight
     );
 
     ctx.restore();
-
-    requestRef.current = requestAnimationFrame(renderFrame);
-  }, [videoRotate]);
+  };
 
   const setCanvas = () => {
     const video = videoRef.current;
@@ -77,11 +75,20 @@ const App = () => {
       canvas.height = video.videoWidth;
     }
 
-    if (requestRef.current) {
-      cancelAnimationFrame(requestRef.current);
+    if (videoRotate !== 0) {
+      processVideoFrames();
     }
+  }
 
-    renderFrame();
+  const processVideoFrames = () => {
+    const video = videoRef.current;
+    if (video == null || videoRotate === 0) return;
+    video.requestVideoFrameCallback(() => {
+      const frame = new VideoFrame(video);
+      renderFrame(frame);
+      frame.close();
+      processVideoFrames();
+    })
   }
 
   useEffect(() => {
@@ -173,21 +180,23 @@ const App = () => {
 
       <video
         id="video"
-        className="hidden"
+        className={clsx(videoRotate == 0 ? videoStyle : "hidden", "min-h-[480px] min-w-[640px]")}
         ref={videoRef}
         autoPlay
         playsInline
         onLoadedMetadata={setCanvas}
       />
 
-      <canvas
-        id="video-canvas"
-        ref={canvasRef}
-        className={clsx('block min-h-[480px] min-w-[640px] select-none origin-center max-w-full max-h-full object-scale-down', mouseStyle)}
-        style={{
-          transform: `scale(${videoScale})`,
-        }}
-      />
+      {videoRotate == 0 ||
+        <canvas
+          id="video-canvas"
+          ref={canvasRef}
+          className={videoStyle}
+          style={{
+            transform: `scale(${videoScale})`,
+          }}
+        />
+      }
 
       <VirtualKeyboard isBigScreen={isBigScreen} />
     </>

browser/src/components/mouse/absolute.tsx

@@ -1,7 +1,7 @@
 import { useEffect, useRef } from 'react';
 import { useAtomValue, useSetAtom } from 'jotai';
 
-import { resolutionAtom } from '@/jotai/device.ts';
+import { resolutionAtom, videoRotateAtom } from '@/jotai/device.ts';
 import {
   mouseJigglerModeAtom,
   mouseLastMoveTimeAtom,
@@ -13,6 +13,7 @@ import { Key } from '@/libs/device/mouse.ts';
 
 export const Absolute = () => {
   const resolution = useAtomValue(resolutionAtom);
+  const videoRotate = useAtomValue(videoRotateAtom);
   const scrollDirection = useAtomValue(scrollDirectionAtom);
   const scrollInterval = useAtomValue(scrollIntervalAtom);
   const mouseJigglerMode = useAtomValue(mouseJigglerModeAtom);
@@ -23,7 +24,7 @@ export const Absolute = () => {
 
   // listen mouse events
   useEffect(() => {
-    const canvas = document.getElementById('video-canvas');
+    const canvas = document.getElementById(videoRotate == 0 ? 'video': 'video-canvas');
     if (!canvas) return;
 
     canvas.addEventListener('mousedown', handleMouseDown);
@@ -157,7 +158,7 @@ export const Absolute = () => {
       canvas.removeEventListener('click', disableEvent);
       canvas.removeEventListener('contextmenu', disableEvent);
     };
-  }, [resolution, scrollDirection, scrollInterval, mouseJigglerMode, setMouseLastMoveTime]);
+  }, [resolution, scrollDirection, scrollInterval, mouseJigglerMode, setMouseLastMoveTime, videoRotate]);
 
   // disable default events
   function disableEvent(event: any) {

browser/src/components/mouse/relative.tsx

@@ -3,7 +3,7 @@ import { message } from 'antd';
 import { useAtomValue, useSetAtom } from 'jotai';
 import { useTranslation } from 'react-i18next';
 
-import { resolutionAtom } from '@/jotai/device.ts';
+import { resolutionAtom, videoRotateAtom } from '@/jotai/device.ts';
 import {
   mouseJigglerModeAtom,
   mouseLastMoveTimeAtom,
@@ -18,6 +18,7 @@ export const Relative = () => {
   const [messageApi, contextHolder] = message.useMessage();
 
   const resolution = useAtomValue(resolutionAtom);
+  const videoRotate = useAtomValue(videoRotateAtom);
   const scrollDirection = useAtomValue(scrollDirectionAtom);
   const scrollInterval = useAtomValue(scrollIntervalAtom);
   const mouseJigglerMode = useAtomValue(mouseJigglerModeAtom);
@@ -41,7 +42,7 @@ export const Relative = () => {
 
   // listen mouse events
   useEffect(() => {
-    const canvas = document.getElementById('video-canvas');
+    const canvas = document.getElementById(videoRotate == 0 ? 'video': 'video-canvas');
     if (!canvas) return;
 
     document.addEventListener('pointerlockchange', handlePointerLockChange);
@@ -150,7 +151,7 @@ export const Relative = () => {
       canvas.removeEventListener('wheel', handleWheel);
       canvas.removeEventListener('contextmenu', disableEvent);
     };
-  }, [resolution, scrollDirection, scrollInterval, mouseJigglerMode, setMouseLastMoveTime]);
+  }, [resolution, scrollDirection, scrollInterval, mouseJigglerMode, setMouseLastMoveTime, videoRotate]);
 
   async function send(x: number, y: number, scroll: number) {
     await device.sendMouseRelativeData(keyRef.current, x, y, scroll);