feat: add video rotate

Author: imguoguo

browser/src/App.tsx

@@ -1,21 +1,15 @@
-import { useEffect, useState } from 'react';
+import { useEffect, useRef, useState } from 'react';
 import { Alert, Result, Spin } from 'antd';
 import clsx from 'clsx';
-import { useAtomValue, useSetAtom } from 'jotai';
+import { useAtomValue, useSetAtom, useAtom } from 'jotai';
 import { useTranslation } from 'react-i18next';
 import { useMediaQuery } from 'react-responsive';
-
 import { DeviceModal } from '@/components/device-modal';
 import { Keyboard } from '@/components/keyboard';
 import { Menu } from '@/components/menu';
 import { Mouse } from '@/components/mouse';
 import { VirtualKeyboard } from '@/components/virtual-keyboard';
-import {
-  resolutionAtom,
-  serialStateAtom,
-  videoScaleAtom,
-  videoStateAtom
-} from '@/jotai/device.ts';
+import { resolutionAtom, serialStateAtom, videoRotateAtom, videoScaleAtom, videoStateAtom } from '@/jotai/device.ts';
 import { isKeyboardEnableAtom } from '@/jotai/keyboard.ts';
 import { mouseStyleAtom } from '@/jotai/mouse.ts';
 import { camera } from '@/libs/camera';
@@ -28,15 +22,81 @@ const App = () => {
   const isBigScreen = useMediaQuery({ minWidth: 850 });
 
   const mouseStyle = useAtomValue(mouseStyleAtom);
-  const videoScale = useAtomValue(videoScaleAtom);
+  const [videoScale, setVideoScale] = useAtom(videoScaleAtom);
+  const [videoRotate, setVideoRotate] = useAtom(videoRotateAtom);
   const videoState = useAtomValue(videoStateAtom);
   const serialState = useAtomValue(serialStateAtom);
   const isKeyboardEnable = useAtomValue(isKeyboardEnableAtom);
   const setResolution = useSetAtom(resolutionAtom);
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const videoRef = useRef<HTMLVideoElement>(null);
+  const canvasContextRef = useRef<CanvasRenderingContext2D | null>(null);
 
   const [isLoading, setIsLoading] = useState(true);
   const [isCameraAvailable, setIsCameraAvailable] = useState(false);
 
+  const videoStyle = clsx('block select-none origin-center max-w-full max-h-full object-scale-down', mouseStyle)
+
+  const renderFrame = (frame: VideoFrame) => {
+    const canvas = canvasRef.current;
+    const ctx = canvasContextRef.current;
+    if (!canvas || !ctx) return;
+
+    ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+    ctx.save();
+    ctx.translate(canvas.width / 2, canvas.height / 2);
+    ctx.rotate((videoRotate * Math.PI) / 180);
+
+    ctx.drawImage(
+      frame,
+      -frame.displayWidth / 2,
+      -frame.displayHeight / 2,
+      frame.displayWidth,
+      frame.displayHeight
+    );
+
+    ctx.restore();
+  };
+
+  const setCanvas = () => {
+    const video = videoRef.current;
+    const canvas = canvasRef.current;
+
+    if (!video || !canvas || video.videoWidth === 0) return;
+
+    if (videoRotate % 180 === 0) {
+      canvas.width = video.videoWidth;
+      canvas.height = video.videoHeight;
+    } else {
+      canvas.width = video.videoHeight;
+      canvas.height = video.videoWidth;
+    }
+
+    if (videoRotate !== 0) {
+      processVideoFrames();
+    }
+  }
+
+  const processVideoFrames = () => {
+    const video = videoRef.current;
+    if (video == null || videoRotate === 0) return;
+    video.requestVideoFrameCallback(() => {
+      const frame = new VideoFrame(video);
+      renderFrame(frame);
+      frame.close();
+      processVideoFrames();
+    })
+  }
+
+  useEffect(() => {
+    if (videoRotate !== 0) {
+      setCanvas();
+      if (canvasRef.current !== null)
+        canvasContextRef.current = canvasRef.current.getContext('2d');
+    }
+  }, [videoRotate]);
+
   useEffect(() => {
     const resolution = storage.getVideoResolution();
     if (resolution) {
@@ -45,6 +105,16 @@ const App = () => {
 
     requestMediaPermissions(resolution);
 
+    const rotate = storage.getVideoRotate();
+    if (rotate) {
+      setVideoRotate(rotate);
+    }
+
+    const scale = storage.getVideoScale();
+    if (scale) {
+      setVideoScale(scale);
+    }
+
     return () => {
       camera.close();
       device.serialPort.close();
@@ -112,18 +182,24 @@ const App = () => {
 
       <video
         id="video"
-        className={clsx('block min-h-[480px] min-w-[640px] select-none', mouseStyle)}
-        style={{
-          transform: `scale(${videoScale})`,
-          transformOrigin: 'center',
-          maxWidth: '100%',
-          maxHeight: '100%',
-          objectFit: 'scale-down'
-        }}
+        className={clsx(videoRotate === 0 ? [videoStyle, "min-h-[480px] min-w-[640px]"] : "hidden")}
+        ref={videoRef}
         autoPlay
         playsInline
+        onLoadedMetadata={setCanvas}
       />
 
+      {videoRotate !== 0 &&
+        <canvas
+          id="video-canvas"
+          ref={canvasRef}
+          className={videoStyle}
+          style={{
+            transform: `scale(${videoScale})`,
+          }}
+        />
+      }
+
       <VirtualKeyboard isBigScreen={isBigScreen} />
     </>
   );

browser/src/components/menu/video/index.tsx

@@ -4,12 +4,14 @@ import { MonitorIcon } from 'lucide-react';
 import { Device } from './device.tsx';
 import { Resolution } from './resolution.tsx';
 import { Scale } from './scale.tsx';
+import { Rotate } from './rotate.tsx';
 
 export const Video = () => {
   const content = (
     <div className="flex flex-col space-y-1">
       <Resolution />
       <Scale />
+      <Rotate />
       <Device />
     </div>
   );

browser/src/components/menu/video/rotate.tsx

@@ -0,0 +1,51 @@
+import { ReactElement } from 'react'
+import { Popover } from 'antd'
+import { useSetAtom } from 'jotai'
+import { RotateCcwIcon } from 'lucide-react'
+import { useTranslation } from 'react-i18next'
+
+import { videoRotateAtom } from '@/jotai/device.ts';
+import * as storage from '@/libs/storage';
+
+export const Rotate = (): ReactElement => {
+  const { t } = useTranslation()
+  const rotates = new Map([
+      ["0", t('video.noRotation')],
+      ["90", "90°"],
+      ["180", "180°"],
+      ["270", "270°"]
+  ]);
+
+  const setVideoRotate = useSetAtom(videoRotateAtom)
+
+  async function updateRotate(rotate: number): Promise<void> {
+    setVideoRotate(rotate)
+    storage.setVideoRotate(rotate)
+  }
+
+  const content = (
+    <>
+      {Array.from(rotates).map(([degree, label]) => (
+        <div
+          key={degree}
+          className={
+            'flex cursor-pointer select-none items-center space-x-1 rounded px-3 py-1.5 hover:bg-neutral-700/60'}
+          onClick={() => updateRotate(parseInt(degree))}
+        >
+          <span>{label}</span>
+        </div>
+      ))}
+    </>
+  )
+
+  return (
+    <Popover content={content} placement="rightTop" arrow={false} align={{ offset: [13, 0] }}>
+      <div className="flex h-[30px] cursor-pointer items-center space-x-1 rounded px-3 text-neutral-300 hover:bg-neutral-700/50">
+        <div className="flex h-[14px] w-[20px] items-end">
+          <RotateCcwIcon size={16} />
+        </div>
+        <span>{t('video.rotate')}</span>
+      </div>
+    </Popover>
+  )
+}

browser/src/components/mouse/absolute.tsx

@@ -1,7 +1,7 @@
 import { useEffect, useRef } from 'react';
 import { useAtomValue, useSetAtom } from 'jotai';
 
-import { resolutionAtom } from '@/jotai/device.ts';
+import { resolutionAtom, videoRotateAtom } from '@/jotai/device.ts';
 import {
   mouseJigglerModeAtom,
   mouseLastMoveTimeAtom,
@@ -13,6 +13,7 @@ import { Key } from '@/libs/device/mouse.ts';
 
 export const Absolute = () => {
   const resolution = useAtomValue(resolutionAtom);
+  const videoRotate = useAtomValue(videoRotateAtom);
   const scrollDirection = useAtomValue(scrollDirectionAtom);
   const scrollInterval = useAtomValue(scrollIntervalAtom);
   const mouseJigglerMode = useAtomValue(mouseJigglerModeAtom);
@@ -23,7 +24,7 @@ export const Absolute = () => {
 
   // listen mouse events
   useEffect(() => {
-    const canvas = document.getElementById('video');
+    const canvas = document.getElementById(videoRotate === 0 ? 'video': 'video-canvas');
     if (!canvas) return;
 
     canvas.addEventListener('mousedown', handleMouseDown);
@@ -157,7 +158,7 @@ export const Absolute = () => {
       canvas.removeEventListener('click', disableEvent);
       canvas.removeEventListener('contextmenu', disableEvent);
     };
-  }, [resolution, scrollDirection, scrollInterval, mouseJigglerMode, setMouseLastMoveTime]);
+  }, [resolution, scrollDirection, scrollInterval, mouseJigglerMode, setMouseLastMoveTime, videoRotate]);
 
   // disable default events
   function disableEvent(event: any) {

browser/src/components/mouse/relative.tsx

@@ -3,7 +3,7 @@ import { message } from 'antd';
 import { useAtomValue, useSetAtom } from 'jotai';
 import { useTranslation } from 'react-i18next';
 
-import { resolutionAtom } from '@/jotai/device.ts';
+import { resolutionAtom, videoRotateAtom } from '@/jotai/device.ts';
 import {
   mouseJigglerModeAtom,
   mouseLastMoveTimeAtom,
@@ -18,6 +18,7 @@ export const Relative = () => {
   const [messageApi, contextHolder] = message.useMessage();
 
   const resolution = useAtomValue(resolutionAtom);
+  const videoRotate = useAtomValue(videoRotateAtom);
   const scrollDirection = useAtomValue(scrollDirectionAtom);
   const scrollInterval = useAtomValue(scrollIntervalAtom);
   const mouseJigglerMode = useAtomValue(mouseJigglerModeAtom);
@@ -41,7 +42,7 @@ export const Relative = () => {
 
   // listen mouse events
   useEffect(() => {
-    const canvas = document.getElementById('video');
+    const canvas = document.getElementById(videoRotate === 0 ? 'video': 'video-canvas');
     if (!canvas) return;
 
     document.addEventListener('pointerlockchange', handlePointerLockChange);
@@ -150,7 +151,7 @@ export const Relative = () => {
       canvas.removeEventListener('wheel', handleWheel);
       canvas.removeEventListener('contextmenu', disableEvent);
     };
-  }, [resolution, scrollDirection, scrollInterval, mouseJigglerMode, setMouseLastMoveTime]);
+  }, [resolution, scrollDirection, scrollInterval, mouseJigglerMode, setMouseLastMoveTime, videoRotate]);
 
   async function send(x: number, y: number, scroll: number) {
     await device.sendMouseRelativeData(keyRef.current, x, y, scroll);

browser/src/i18n/locales/en.ts

@@ -25,6 +25,8 @@ const en = {
     video: {
       resolution: 'Resolution',
       scale: 'Scale',
+      rotate: 'Rotate',
+      noRotation: 'No Rotation',
       customResolution: 'Custom',
       device: 'Device',
       custom: {

browser/src/i18n/locales/zh.ts

@@ -23,6 +23,8 @@ const zh = {
     video: {
       resolution: '分辨率',
       scale: '缩放',
+      rotate: '旋转',
+      noRotation: '不旋转',
       customResolution: '自定义',
       device: '设备',
       custom: {

browser/src/jotai/device.ts

@@ -10,7 +10,8 @@ export const resolutionAtom = atom<Resolution>({
   height: 1080
 });
 
-export const videoScaleAtom = atom<number>(1.0)
+export const videoScaleAtom = atom<number>(1.0);
+export const videoRotateAtom = atom<number>(0);
 
 export const videoDeviceIdAtom = atom('');
 export const videoStateAtom = atom<VideoState>('disconnected');

browser/src/libs/storage/index.ts

@@ -5,6 +5,7 @@ const VIDEO_DEVICE_ID_KEY = 'nanokvm-usb-video-device-id';
 const VIDEO_RESOLUTION_KEY = 'nanokvm-usb-video-resolution';
 const CUSTOM_RESOLUTION_KEY = 'nanokvm-usb-custom-resolution';
 const VIDEO_SCALE_KEY = 'nanokvm-usb-video-scale'
+const VIDEO_ROTATE_KEY = 'nanokvm-usb-video-rotate'
 const IS_MENU_OPEN_KEY = 'nanokvm-is-menu-open';
 const MOUSE_STYLE_KEY = 'nanokvm-usb-mouse-style';
 const MOUSE_MODE_KEY = 'nanokvm-usb-mouse-mode';
@@ -70,6 +71,18 @@ export function setVideoScale(scale: number): void {
   localStorage.setItem(VIDEO_SCALE_KEY, String(scale))
 }
 
+export function getVideoRotate(): number | null {
+  const scale = localStorage.getItem(VIDEO_ROTATE_KEY)
+  if (scale && Number(scale)) {
+    return Number(scale)
+  }
+  return null
+}
+
+export function setVideoRotate(scale: number): void {
+  localStorage.setItem(VIDEO_ROTATE_KEY, String(scale))
+}
+
 export function getIsMenuOpen(): boolean {
   const state = localStorage.getItem(IS_MENU_OPEN_KEY);
   if (!state) {