feat(desktop): add video rotate

Author: imguoguo

desktop/src/renderer/src/App.tsx

@@ -1,7 +1,7 @@
-import { ReactElement, useEffect, useState } from 'react'
+import { ReactElement, useEffect, useRef, useState } from 'react'
 import { Result, Spin } from 'antd'
 import clsx from 'clsx'
-import { useAtomValue, useSetAtom } from 'jotai'
+import { useAtom, useAtomValue, useSetAtom } from 'jotai'
 import { useTranslation } from 'react-i18next'
 import { useMediaQuery } from 'react-responsive'
 
@@ -14,13 +14,14 @@ import { VirtualKeyboard } from '@renderer/components/virtual-keyboard'
 import {
   resolutionAtom,
   serialPortStateAtom,
+  videoRotateAtom,
   videoScaleAtom,
   videoStateAtom
 } from '@renderer/jotai/device'
 import { isKeyboardEnableAtom } from '@renderer/jotai/keyboard'
 import { mouseStyleAtom } from '@renderer/jotai/mouse'
 import { camera } from '@renderer/libs/camera'
-import { getVideoResolution } from '@renderer/libs/storage'
+import * as storage from '@renderer/libs/storage'
 import type { Resolution } from '@renderer/types'
 
 type State = 'loading' | 'success' | 'failed'
@@ -30,17 +31,93 @@ const App = (): ReactElement => {
   const isBigScreen = useMediaQuery({ minWidth: 850 })
 
   const videoScale = useAtomValue(videoScaleAtom)
+  const [videoRotate, setVideoRotate] = useAtom(videoRotateAtom)
   const videoState = useAtomValue(videoStateAtom)
   const serialPortState = useAtomValue(serialPortStateAtom)
   const mouseStyle = useAtomValue(mouseStyleAtom)
   const isKeyboardEnable = useAtomValue(isKeyboardEnableAtom)
   const setResolution = useSetAtom(resolutionAtom)
+  const canvasRef = useRef<HTMLCanvasElement>(null)
+  const videoRef = useRef<HTMLVideoElement>(null)
+  const canvasContextRef = useRef<CanvasRenderingContext2D | null>(null)
 
   const [state, setState] = useState<State>('loading')
   const [isConnected, setIsConnected] = useState(false)
 
+  const videoStyle = clsx(
+    'block select-none origin-center max-w-full max-h-full object-scale-down',
+    mouseStyle
+  )
+
+  const renderFrame = (frame: VideoFrame) => {
+    const canvas = canvasRef.current
+    const ctx = canvasContextRef.current
+    if (!canvas || !ctx) return
+
+    ctx.clearRect(0, 0, canvas.width, canvas.height)
+
+    ctx.save()
+    ctx.translate(canvas.width / 2, canvas.height / 2)
+    ctx.rotate((videoRotate * Math.PI) / 180)
+
+    ctx.drawImage(
+      frame,
+      -frame.displayWidth / 2,
+      -frame.displayHeight / 2,
+      frame.displayWidth,
+      frame.displayHeight
+    )
+
+    ctx.restore()
+  }
+
+  const setCanvas = () => {
+    const video = videoRef.current
+    const canvas = canvasRef.current
+
+    if (!video || !canvas || video.videoWidth === 0) return
+
+    if (videoRotate % 180 === 0) {
+      canvas.width = video.videoWidth
+      canvas.height = video.videoHeight
+    } else {
+      canvas.width = video.videoHeight
+      canvas.height = video.videoWidth
+    }
+
+    if (canvasRef.current !== null) {
+      canvasContextRef.current = canvasRef.current.getContext('2d')
+    }
+
+    if (videoRotate !== 0) {
+      processVideoFrames()
+    }
+  }
+
+  const processVideoFrames = () => {
+    const video = videoRef.current
+    if (video == null || videoRotate === 0) return
+    video.requestVideoFrameCallback(() => {
+      const frame = new VideoFrame(video)
+      renderFrame(frame)
+      frame.close()
+      processVideoFrames()
+    })
+  }
+
   useEffect(() => {
-    const resolution = getVideoResolution()
+    if (videoRotate !== 0) {
+      setCanvas()
+    }
+  }, [videoRotate])
+
+  useEffect(() => {
+    const rotate = storage.getVideoRotate()
+    if (rotate) {
+      setVideoRotate(rotate)
+    }
+
+    const resolution = storage.getVideoResolution()
     if (resolution) {
       setResolution(resolution)
     }
@@ -120,18 +197,24 @@ const App = (): ReactElement => {
 
       <video
         id="video"
-        className={clsx('block min-h-[480px] min-w-[640px] select-none', mouseStyle)}
-        style={{
-          transform: `scale(${videoScale})`,
-          transformOrigin: 'center',
-          maxWidth: '100%',
-          maxHeight: '100%',
-          objectFit: 'scale-down'
-        }}
+        className={clsx(videoRotate === 0 ? [videoStyle, 'min-h-[480px] min-w-[640px]'] : 'hidden')}
+        ref={videoRef}
         autoPlay
         playsInline
+        onLoadedMetadata={setCanvas}
       />
 
+      {videoRotate !== 0 && (
+        <canvas
+          id="video-canvas"
+          ref={canvasRef}
+          className={videoStyle}
+          style={{
+            transform: `scale(${videoScale})`
+          }}
+        />
+      )}
+
       <VirtualKeyboard isBigScreen={isBigScreen} />
     </>
   )

desktop/src/renderer/src/components/menu/video/index.tsx

@@ -4,12 +4,14 @@ import { MonitorIcon } from 'lucide-react'
 
 import { Device } from './device'
 import { Resolution } from './resolution'
+import { Rotate } from './rotate'
 import { Scale } from './scale'
 
 export const Video = (): ReactElement => {
   const content = (
     <div className="flex flex-col space-y-1">
       <Resolution />
+      <Rotate />
       <Scale />
       <Device />
     </div>

desktop/src/renderer/src/components/menu/video/rotate.tsx

@@ -0,0 +1,54 @@
+import { ReactElement } from 'react'
+import { Popover } from 'antd'
+import clsx from 'clsx'
+import { useAtom } from 'jotai'
+import { RotateCcwIcon } from 'lucide-react'
+import { useTranslation } from 'react-i18next'
+
+import { videoRotateAtom } from '@renderer/jotai/device'
+import * as storage from '@renderer/libs/storage'
+
+export const Rotate = (): ReactElement => {
+  const { t } = useTranslation()
+  const rotates = new Map([
+    ['0', t('video.noRotation')],
+    ['90', '90°'],
+    ['180', '180°'],
+    ['270', '270°']
+  ])
+
+  const [videoRotate, setVideoRotate] = useAtom(videoRotateAtom)
+
+  async function updateRotate(rotate: number): Promise<void> {
+    setVideoRotate(rotate)
+    storage.setVideoRotate(rotate)
+  }
+
+  const content = (
+    <>
+      {Array.from(rotates).map(([degree, label]) => (
+        <div
+          key={degree}
+          className={clsx(
+            'flex cursor-pointer items-center space-x-1 rounded px-3 py-1.5 select-none hover:bg-neutral-700/60',
+            videoRotate === parseInt(degree) ? 'text-blue-500' : 'text-white'
+          )}
+          onClick={() => updateRotate(parseInt(degree))}
+        >
+          <span>{label}</span>
+        </div>
+      ))}
+    </>
+  )
+
+  return (
+    <Popover content={content} placement="rightTop" arrow={false} align={{ offset: [13, 0] }}>
+      <div className="flex h-[30px] cursor-pointer items-center space-x-1 rounded px-3 text-neutral-300 hover:bg-neutral-700/50">
+        <div className="flex h-[14px] w-[20px] items-end">
+          <RotateCcwIcon size={16} />
+        </div>
+        <span>{t('video.rotate')}</span>
+      </div>
+    </Popover>
+  )
+}

desktop/src/renderer/src/components/mouse/absolute.tsx

@@ -2,7 +2,7 @@ import { ReactElement, useEffect, useRef } from 'react'
 import { useAtomValue } from 'jotai'
 
 import { IpcEvents } from '@common/ipc-events'
-import { resolutionAtom } from '@renderer/jotai/device'
+import { resolutionAtom, videoRotateAtom } from '@renderer/jotai/device'
 import { scrollDirectionAtom, scrollIntervalAtom } from '@renderer/jotai/mouse'
 import { Key } from '@renderer/libs/device/mouse'
 import { mouseJiggler } from '@renderer/libs/mouse-jiggler'
@@ -11,12 +11,13 @@ export const Absolute = (): ReactElement => {
   const resolution = useAtomValue(resolutionAtom)
   const scrollDirection = useAtomValue(scrollDirectionAtom)
   const scrollInterval = useAtomValue(scrollIntervalAtom)
+  const videoRotate = useAtomValue(videoRotateAtom)
 
   const keyRef = useRef<Key>(new Key())
   const lastScrollTimeRef = useRef(0)
 
   useEffect(() => {
-    const canvas = document.getElementById('video')
+    const canvas = document.getElementById(videoRotate === 0 ? 'video': 'video-canvas')
     if (!canvas) return
 
     canvas.addEventListener('mousedown', handleMouseDown)
@@ -150,7 +151,7 @@ export const Absolute = (): ReactElement => {
       canvas.removeEventListener('click', disableEvent)
       canvas.removeEventListener('contextmenu', disableEvent)
     }
-  }, [resolution, scrollDirection, scrollInterval])
+  }, [resolution, scrollDirection, scrollInterval, videoRotate])
 
   function disableEvent(event: MouseEvent): void {
     event.preventDefault()

desktop/src/renderer/src/components/mouse/relative.tsx

@@ -4,7 +4,7 @@ import { useAtomValue } from 'jotai'
 import { useTranslation } from 'react-i18next'
 
 import { IpcEvents } from '@common/ipc-events'
-import { resolutionAtom } from '@renderer/jotai/device'
+import { resolutionAtom, videoRotateAtom } from '@renderer/jotai/device'
 import { scrollDirectionAtom, scrollIntervalAtom } from '@renderer/jotai/mouse'
 import { Key } from '@renderer/libs/device/mouse'
 import { mouseJiggler } from '@renderer/libs/mouse-jiggler'
@@ -14,6 +14,7 @@ export const Relative = (): ReactElement => {
   const [messageApi, contextHolder] = message.useMessage()
 
   const resolution = useAtomValue(resolutionAtom)
+  const videoRotate = useAtomValue(videoRotateAtom);
   const scrollDirection = useAtomValue(scrollDirectionAtom)
   const scrollInterval = useAtomValue(scrollIntervalAtom)
 
@@ -34,7 +35,7 @@ export const Relative = (): ReactElement => {
   }, [])
 
   useEffect(() => {
-    const canvas = document.getElementById('video')
+    const canvas = document.getElementById(videoRotate === 0 ? 'video': 'video-canvas');
     if (!canvas) return
 
     document.addEventListener('pointerlockchange', handlePointerLockChange)
@@ -140,7 +141,7 @@ export const Relative = (): ReactElement => {
       canvas.removeEventListener('wheel', handleWheel)
       canvas.removeEventListener('contextmenu', disableEvent)
     }
-  }, [resolution, scrollDirection, scrollInterval])
+  }, [resolution, scrollDirection, scrollInterval, videoRotate])
 
   function disableEvent(event: MouseEvent): void {
     event.preventDefault()

desktop/src/renderer/src/i18n/locales/en.ts

@@ -27,6 +27,8 @@ const en = {
     video: {
       resolution: 'Resolution',
       scale: 'Scale',
+      rotate: 'Rotate',
+      noRotation: 'No Rotation',
       customResolution: 'Custom',
       device: 'Device',
       custom: {

desktop/src/renderer/src/i18n/locales/zh.ts

@@ -26,6 +26,8 @@ const zh = {
     video: {
       resolution: '分辨率',
       scale: '缩放',
+      rotate: '旋转',
+      noRotation: '不旋转',
       customResolution: '自定义',
       device: '设备',
       custom: {

desktop/src/renderer/src/jotai/device.ts

@@ -11,6 +11,7 @@ export const resolutionAtom = atom<Resolution>({
 })
 
 export const videoScaleAtom = atom<number>(1.0)
+export const videoRotateAtom = atom<number>(0)
 
 export const videoDeviceIdAtom = atom('')
 export const videoStateAtom = atom<VideoState>('disconnected')

desktop/src/renderer/src/libs/storage/index.ts

@@ -6,6 +6,7 @@ const LANGUAGE_KEY = 'nanokvm-usb-language'
 const VIDEO_DEVICE_ID_KEY = 'nanokvm-usb-video-device-id'
 const VIDEO_RESOLUTION_KEY = 'nanokvm-usb-video-resolution'
 const VIDEO_SCALE_KEY = 'nanokvm-usb-video-scale'
+const VIDEO_ROTATE_KEY = 'nanokvm-usb-video-rotate'
 const CUSTOM_RESOLUTION_KEY = 'nanokvm-usb-custom-resolution'
 const SERIAL_PORT_KEY = 'nanokvm-serial-port'
 const IS_MENU_OPEN_KEY = 'nanokvm-is-menu-open'
@@ -79,6 +80,18 @@ export function setVideoScale(scale: number): void {
   localStorage.setItem(VIDEO_SCALE_KEY, String(scale))
 }
 
+export function getVideoRotate(): number | null {
+  const scale = localStorage.getItem(VIDEO_ROTATE_KEY)
+  if (scale && Number(scale)) {
+    return Number(scale)
+  }
+  return null
+}
+
+export function setVideoRotate(scale: number): void {
+  localStorage.setItem(VIDEO_ROTATE_KEY, String(scale))
+}
+
 export function getSerialPort(): string | null {
   return localStorage.getItem(SERIAL_PORT_KEY)
 }