diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7cbc202e1..88b2d1a05f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -39,7 +39,7 @@ jobs: - name: "Install system dependencies" run: | sudo apt update -qq - sudo apt install -y clang-15 cmake freeglut3-dev libgcrypt20-dev libglm-dev libgtk-3-dev libpulse-dev libsecret-1-dev libsystemd-dev libudev-dev nasm ninja-build libbluetooth-dev + sudo apt install -y clang-15 cmake freeglut3-dev libbluetooth-dev libgcrypt20-dev libglm-dev libgtk-3-dev libturbojpeg0-dev libpulse-dev libsecret-1-dev libsystemd-dev libudev-dev nasm ninja-build - name: "Setup cmake" uses: jwlawson/actions-setup-cmake@v2 @@ -96,7 +96,7 @@ jobs: - name: "Install system dependencies" run: | sudo apt update -qq - sudo apt install -y clang-15 cmake freeglut3-dev libgcrypt20-dev libglm-dev libgtk-3-dev libpulse-dev libsecret-1-dev libsystemd-dev nasm ninja-build appstream libbluetooth-dev + sudo apt install -y appstream clang-15 cmake freeglut3-dev libbluetooth-dev libgcrypt20-dev libglm-dev libgtk-3-dev libturbojpeg0-dev libpulse-dev libsecret-1-dev libsystemd-dev nasm ninja-build - name: "Build AppImage" run: | diff --git a/.gitmodules b/.gitmodules index 8f9772d342..718f825196 100644 --- a/.gitmodules +++ b/.gitmodules @@ -18,6 +18,10 @@ path = dependencies/imgui url = https://github.com/ocornut/imgui shallow = true +[submodule "dependencies/openpnp-capture"] + path = dependencies/openpnp-capture + url = https://github.com/capitalistspz/openpnp-capture + branch = dev [submodule "dependencies/xbyak_aarch64"] path = dependencies/xbyak_aarch64 url = https://github.com/fujitsu/xbyak_aarch64 diff --git a/BUILD.md b/BUILD.md index 31c265313f..0a746a21f4 100644 --- a/BUILD.md +++ b/BUILD.md @@ -46,10 +46,10 @@ To compile Cemu, a recent enough compiler and STL with C++20 support is required ### Dependencies #### For Arch and derivatives: -`sudo pacman -S --needed base-devel bluez-libs clang cmake freeglut git glm gtk3 libgcrypt libpulse libsecret linux-headers llvm nasm ninja systemd unzip zip` +`sudo pacman -S --needed base-devel bluez-libs clang cmake freeglut git glm gtk3 libgcrypt libjpeg-turbo libpulse libsecret linux-headers llvm nasm ninja systemd unzip zip` #### For Debian, Ubuntu and derivatives: -`sudo apt install -y cmake curl clang-15 freeglut3-dev git libbluetooth-dev libgcrypt20-dev libglm-dev libgtk-3-dev libpulse-dev libsecret-1-dev libsystemd-dev libtool nasm ninja-build` +`sudo apt install -y cmake curl clang-15 freeglut3-dev git libbluetooth-dev libgcrypt20-dev libglm-dev libgtk-3-dev libturbojpeg0-dev libpulse-dev libsecret-1-dev libsystemd-dev libtool nasm ninja-build` You may also need to install `libusb-1.0-0-dev` as a workaround for an issue with the vcpkg hidapi package. @@ -57,7 +57,7 @@ At Step 3 in [Build Cemu using cmake and clang](#build-cemu-using-cmake-and-clan `cmake -S . -B build -DCMAKE_BUILD_TYPE=release -DCMAKE_C_COMPILER=/usr/bin/clang-15 -DCMAKE_CXX_COMPILER=/usr/bin/clang++-15 -G Ninja -DCMAKE_MAKE_PROGRAM=/usr/bin/ninja` #### For Fedora and derivatives: -`sudo dnf install bluez-libs-devel clang cmake cubeb-devel freeglut-devel git glm-devel gtk3-devel kernel-headers libgcrypt-devel libsecret-devel libtool libusb1-devel llvm nasm ninja-build perl-core systemd-devel wayland-protocols-devel zlib-devel zlib-static` +`sudo dnf install bluez-libs-devel clang cmake cubeb-devel freeglut-devel git glm-devel gtk3-devel kernel-headers libgcrypt-devel libjpeg-turbo-devel libsecret-devel libtool libusb1-devel llvm nasm ninja-build perl-core systemd-devel wayland-protocols-devel zlib-devel zlib-static` ### Build Cemu diff --git a/CMakeLists.txt b/CMakeLists.txt index eb848ce71f..ca1f2062ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -222,6 +222,10 @@ endif() add_subdirectory("dependencies/ih264d" EXCLUDE_FROM_ALL) +set(BUILD_SHARED_LIBS OFF) +add_subdirectory("dependencies/openpnp-capture" EXCLUDE_FROM_ALL SYSTEM) +set(BUILD_SHARED_LIBS "") + if(CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(AARCH64)") add_subdirectory("dependencies/xbyak_aarch64" EXCLUDE_FROM_ALL) endif() diff --git a/dependencies/openpnp-capture b/dependencies/openpnp-capture new file mode 160000 index 0000000000..fd9d6fac12 --- /dev/null +++ b/dependencies/openpnp-capture @@ -0,0 +1 @@ +Subproject commit fd9d6fac12d6258f07c788abfe543e4ba4323bf9 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ee7f8610df..62fd97dfcc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -46,6 +46,7 @@ add_subdirectory(Cemu) add_subdirectory(config) add_subdirectory(input) add_subdirectory(audio) +add_subdirectory(camera) add_subdirectory(util) add_subdirectory(imgui) add_subdirectory(resource) @@ -124,6 +125,7 @@ set_target_properties(CemuBin PROPERTIES target_link_libraries(CemuBin PRIVATE CemuAudio CemuCafe + CemuCamera CemuCommon CemuComponents CemuConfig diff --git a/src/Cafe/CMakeLists.txt b/src/Cafe/CMakeLists.txt index 71866b2155..ca477c15eb 100644 --- a/src/Cafe/CMakeLists.txt +++ b/src/Cafe/CMakeLists.txt @@ -557,6 +557,7 @@ endif() target_link_libraries(CemuCafe PRIVATE CemuAudio + CemuCamera CemuCommon CemuComponents CemuConfig diff --git a/src/Cafe/OS/libs/camera/camera.cpp b/src/Cafe/OS/libs/camera/camera.cpp index efb8013d8c..e19dc3ff87 100644 --- a/src/Cafe/OS/libs/camera/camera.cpp +++ b/src/Cafe/OS/libs/camera/camera.cpp @@ -1,257 +1,291 @@ #include "Common/precompiled.h" #include "Cafe/OS/common/OSCommon.h" #include "camera.h" + #include "Cafe/OS/RPL/rpl.h" -#include "Cafe/OS/libs/coreinit/coreinit_Alarm.h" -#include "Cafe/OS/libs/coreinit/coreinit_Time.h" #include "Cafe/HW/Espresso/PPCCallback.h" +#include "util/helpers/helpers.h" + +#include "util/helpers/ringbuffer.h" +#include "camera/CameraManager.h" +#include "Cafe/HW/Espresso/Const.h" +#include "Common/CafeString.h" namespace camera { + constexpr unsigned CAMERA_WIDTH = 640; + constexpr unsigned CAMERA_HEIGHT = 480; - struct CAMInitInfo_t + enum CAMStatus : sint32 { - /* +0x00 */ uint32be ukn00; - /* +0x04 */ uint32be width; - /* +0x08 */ uint32be height; - - /* +0x0C */ uint32be workMemorySize; - /* +0x10 */ MEMPTR workMemory; - - /* +0x14 */ uint32be handlerFuncPtr; - - /* +0x18 */ uint32be ukn18; - /* +0x1C */ uint32be fps; - - /* +0x20 */ uint32be ukn20; + CAM_STATUS_SUCCESS = 0, + CAM_STATUS_INVALID_ARG = -1, + CAM_STATUS_INVALID_HANDLE = -2, + CAM_STATUS_SURFACE_QUEUE_FULL = -4, + CAM_STATUS_INSUFFICIENT_MEMORY = -5, + CAM_STATUS_NOT_READY = -6, + CAM_STATUS_UNINITIALIZED = -8, + CAM_STATUS_UVC_ERROR = -9, + CAM_STATUS_DECODER_INIT_INIT_FAILED = -10, + CAM_STATUS_DEVICE_IN_USE = -12, + CAM_STATUS_DECODER_SESSION_FAILED = -13, + CAM_STATUS_INVALID_PROPERTY = -14, + CAM_STATUS_SEGMENT_VIOLATION = -15 }; - struct CAMTargetSurface + enum class CAMFps : uint32 { - /* +0x00 */ uint32be surfaceSize; - /* +0x04 */ MEMPTR surfacePtr; - /* +0x08 */ uint32be ukn08; - /* +0x0C */ uint32be ukn0C; - /* +0x10 */ uint32be ukn10; - /* +0x14 */ uint32be ukn14; - /* +0x18 */ uint32be ukn18; - /* +0x1C */ uint32be ukn1C; + _15 = 0, + _30 = 1 }; - struct CAMCallbackParam + enum class CAMEventType : uint32 { - // type 0 - frame decoded | field1 - imagePtr, field2 - imageSize, field3 - ukn (0) - // type 1 - ??? - - - /* +0x0 */ uint32be type; // 0 -> Frame decoded - /* +0x4 */ uint32be field1; - /* +0x8 */ uint32be field2; - /* +0xC */ uint32be field3; + Decode = 0, + Detached = 1 }; + enum class CAMForceDisplay + { + None = 0, + DRC = 1 + }; - #define CAM_ERROR_SUCCESS 0 - #define CAM_ERROR_INVALID_HANDLE -8 + enum class CAMImageType : uint32 + { + Default = 0 + }; - std::vector g_table_cameraHandles; - std::vector g_activeCameraInstances; - std::recursive_mutex g_mutex_camera; - std::atomic_int g_cameraCounter{ 0 }; - SysAllocator g_alarm_camera; - SysAllocator g_cameraHandlerParam; + struct CAMImageInfo + { + betype type; + uint32be height; + uint32be width; + }; + static_assert(sizeof(CAMImageInfo) == 0x0C); - CameraInstance* GetCameraInstanceByHandle(sint32 camHandle) + struct CAMInitInfo_t { - std::unique_lock _lock(g_mutex_camera); - if (camHandle <= 0) - return nullptr; - camHandle -= 1; - if (camHandle >= g_table_cameraHandles.size()) - return nullptr; - return g_table_cameraHandles[camHandle]; - } + CAMImageInfo imageInfo; + uint32be workMemorySize; + MEMPTR workMemoryData; + MEMPTR callback; + betype forceDisplay; + betype fps; + uint32be threadFlags; + uint8 unk[0x10]; + }; + static_assert(sizeof(CAMInitInfo_t) == 0x34); - struct CameraInstance + struct CAMTargetSurface { - CameraInstance(uint32 frameWidth, uint32 frameHeight, MPTR handlerFunc) : width(frameWidth), height(frameHeight), handlerFunc(handlerFunc) { AcquireHandle(); }; - ~CameraInstance() { if (isOpen) { CloseCam(); } ReleaseHandle(); }; + sint32be size; + MEMPTR data; + uint8 unused[0x18]; + }; + static_assert(sizeof(CAMTargetSurface) == 0x20); - sint32 handle{ 0 }; - uint32 width; - uint32 height; - bool isOpen{false}; - std::queue queue_targetSurfaces; - MPTR handlerFunc; + struct CAMDecodeEventParam + { + betype type; + MEMPTR data; + uint32be channel; + uint32be errored; + }; + static_assert(sizeof(CAMDecodeEventParam) == 0x10); - bool OpenCam() - { - if (isOpen) - return false; - isOpen = true; - g_activeCameraInstances.push_back(this); - return true; - } + constexpr static int32_t CAM_HANDLE = 0; - bool CloseCam() - { - if (!isOpen) - return false; - isOpen = false; - vectorRemoveByValue(g_activeCameraInstances, this); - return true; - } - - void QueueTargetSurface(CAMTargetSurface* targetSurface) - { - std::unique_lock _lock(g_mutex_camera); - cemu_assert_debug(queue_targetSurfaces.size() < 100); // check for sane queue length - queue_targetSurfaces.push(*targetSurface); - } + struct + { + std::recursive_mutex mutex{}; + bool initialized = false; + bool shouldTriggerCallback = false; + std::atomic_bool isOpen = false; + std::atomic_bool isExiting = false; + bool isWorking = false; + unsigned fps = 30; + MEMPTR eventCallback = nullptr; + RingBuffer, 20> inTargetBuffers{}; + RingBuffer, 20> outTargetBuffers{}; + } s_instance; + + SysAllocator s_cameraEventData; + SysAllocator s_cameraWorkerThread; + SysAllocator s_cameraWorkerThreadStack; + SysAllocator> s_cameraWorkerThreadNameBuffer; + SysAllocator s_cameraOpenEvent; + + void WorkerThread(PPCInterpreter_t*) + { + s_cameraEventData->type = CAMEventType::Decode; + s_cameraEventData->channel = 0; + s_cameraEventData->data = nullptr; + s_cameraEventData->errored = false; + PPCCoreCallback(s_instance.eventCallback, s_cameraEventData.GetMPTR()); - private: - void AcquireHandle() + while (!s_instance.isExiting) { - std::unique_lock _lock(g_mutex_camera); - for (uint32 i = 0; i < g_table_cameraHandles.size(); i++) + coreinit::OSWaitEvent(s_cameraOpenEvent); + while (true) { - if (g_table_cameraHandles[i] == nullptr) + if (!s_instance.isOpen || s_instance.isExiting) { - g_table_cameraHandles[i] = this; - this->handle = i + 1; - return; + // Fill leftover buffers before stopping + if (!s_instance.inTargetBuffers.HasData()) + break; } - } - this->handle = (sint32)(g_table_cameraHandles.size() + 1); - g_table_cameraHandles.push_back(this); - } + s_cameraEventData->type = CAMEventType::Decode; + s_cameraEventData->channel = 0; - void ReleaseHandle() - { - for (uint32 i = 0; i < g_table_cameraHandles.size(); i++) - { - if (g_table_cameraHandles[i] == this) + const auto surfaceBuffer = s_instance.inTargetBuffers.Pop(); + if (surfaceBuffer.IsNull()) + { + s_cameraEventData->data = nullptr; + s_cameraEventData->errored = true; + } + else { - g_table_cameraHandles[i] = nullptr; - return; + CameraManager::FillNV12Buffer(surfaceBuffer.GetPtr()); + s_cameraEventData->data = surfaceBuffer; + s_cameraEventData->errored = false; } + PPCCoreCallback(s_instance.eventCallback, s_cameraEventData.GetMPTR()); + coreinit::OSSleepTicks(Espresso::TIMER_CLOCK / (s_instance.fps - 1)); } - cemu_assert_debug(false); } - }; + coreinit::OSExitThread(0); + } - sint32 CAMGetMemReq(void* ukn) + sint32 CAMGetMemReq(const CAMImageInfo* info) { + if (!info) + return CAM_STATUS_INVALID_ARG; return 1 * 1024; // always return 1KB } - sint32 CAMCheckMemSegmentation(void* base, uint32 size) + CAMStatus CAMCheckMemSegmentation(void* base, uint32 size) { - return CAM_ERROR_SUCCESS; // always return success + if (!base || size == 0) + return CAM_STATUS_INVALID_ARG; + return CAM_STATUS_SUCCESS; } - void ppcCAMUpdate60(PPCInterpreter_t* hCPU) + sint32 CAMInit(uint32 cameraId, const CAMInitInfo_t* initInfo, betype* error) { - // update all open camera instances - size_t numCamInstances = g_activeCameraInstances.size(); - //for (auto& itr : g_activeCameraInstances) - for(size_t i=0; i _lock(g_mutex_camera); - if (i >= g_activeCameraInstances.size()) - break; - CameraInstance* camInstance = g_activeCameraInstances[i]; - // todo - handle 30 / 60 FPS - if (camInstance->queue_targetSurfaces.empty()) - continue; - auto& targetSurface = camInstance->queue_targetSurfaces.front(); - g_cameraHandlerParam->type = 0; - g_cameraHandlerParam->field1 = targetSurface.surfacePtr.GetMPTR(); - g_cameraHandlerParam->field2 = targetSurface.surfaceSize; - g_cameraHandlerParam->field3 = 0; - cemu_assert_debug(camInstance->handlerFunc != MPTR_NULL); - camInstance->queue_targetSurfaces.pop(); - _lock.unlock(); - PPCCoreCallback(camInstance->handlerFunc, g_cameraHandlerParam.GetPtr()); + *error = CAM_STATUS_DEVICE_IN_USE; + return -1; } - osLib_returnFromFunction(hCPU, 0); - } - - sint32 CAMInit(uint32 cameraId, CAMInitInfo_t* camInitInfo, uint32be* error) - { - CameraInstance* camInstance = new CameraInstance(camInitInfo->width, camInitInfo->height, camInitInfo->handlerFuncPtr); - *error = 0; // Hunter's Trophy 2 will fail to boot if we don't set this - std::unique_lock _lock(g_mutex_camera); - if (g_cameraCounter == 0) + if (!initInfo || !initInfo->workMemoryData || + !match_any_of(initInfo->forceDisplay, CAMForceDisplay::None, CAMForceDisplay::DRC) || + !match_any_of(initInfo->fps, CAMFps::_15, CAMFps::_30) || + initInfo->imageInfo.type != CAMImageType::Default) { - coreinit::OSCreateAlarm(g_alarm_camera.GetPtr()); - coreinit::OSSetPeriodicAlarm(g_alarm_camera.GetPtr(), coreinit::OSGetTime(), (uint64)ESPRESSO_TIMER_CLOCK / 60ull, RPLLoader_MakePPCCallable(ppcCAMUpdate60)); + *error = CAM_STATUS_INVALID_ARG; + return -1; } - g_cameraCounter++; - - return camInstance->handle; + CameraManager::Init(); + + cemu_assert_debug(initInfo->forceDisplay != CAMForceDisplay::DRC); + cemu_assert_debug(initInfo->workMemorySize != 0); + cemu_assert_debug(initInfo->imageInfo.type == CAMImageType::Default); + + s_instance.isExiting = false; + s_instance.fps = initInfo->fps == CAMFps::_15 ? 15 : 30; + s_instance.initialized = true; + s_instance.eventCallback = initInfo->callback; + + coreinit::OSInitEvent(s_cameraOpenEvent, coreinit::OSEvent::EVENT_STATE::STATE_NOT_SIGNALED, coreinit::OSEvent::EVENT_MODE::MODE_AUTO); + + coreinit::__OSCreateThreadType( + s_cameraWorkerThread, RPLLoader_MakePPCCallable(WorkerThread), 0, nullptr, + s_cameraWorkerThreadStack.GetPtr() + s_cameraWorkerThreadStack.GetByteSize(), s_cameraWorkerThreadStack.GetByteSize(), + 0x10, initInfo->threadFlags & 7, OSThread_t::THREAD_TYPE::TYPE_DRIVER); + s_cameraWorkerThreadNameBuffer->assign("CameraWorkerThread"); + coreinit::OSSetThreadName(s_cameraWorkerThread.GetPtr(), s_cameraWorkerThreadNameBuffer->c_str()); + coreinit::OSResumeThread(s_cameraWorkerThread.GetPtr()); + return CAM_STATUS_SUCCESS; } - sint32 CAMExit(sint32 camHandle) + CAMStatus CAMClose(sint32 camHandle) { - CameraInstance* camInstance = GetCameraInstanceByHandle(camHandle); - if (!camInstance) - return CAM_ERROR_INVALID_HANDLE; - CAMClose(camHandle); - delete camInstance; - - std::unique_lock _lock(g_mutex_camera); - g_cameraCounter--; - if (g_cameraCounter == 0) - coreinit::OSCancelAlarm(g_alarm_camera.GetPtr()); - return CAM_ERROR_SUCCESS; + if (camHandle != CAM_HANDLE) + return CAM_STATUS_INVALID_HANDLE; + { + std::scoped_lock lock(s_instance.mutex); + if (!s_instance.initialized || !s_instance.isOpen) + return CAM_STATUS_UNINITIALIZED; + s_instance.isOpen = false; + } + CameraManager::Close(); + return CAM_STATUS_SUCCESS; } - sint32 CAMOpen(sint32 camHandle) + CAMStatus CAMOpen(sint32 camHandle) { - CameraInstance* camInstance = GetCameraInstanceByHandle(camHandle); - if (!camInstance) - return CAM_ERROR_INVALID_HANDLE; - camInstance->OpenCam(); - return CAM_ERROR_SUCCESS; + if (camHandle != CAM_HANDLE) + return CAM_STATUS_INVALID_HANDLE; + auto lock = std::scoped_lock(s_instance.mutex); + if (!s_instance.initialized) + return CAM_STATUS_UNINITIALIZED; + if (s_instance.isOpen) + return CAM_STATUS_DEVICE_IN_USE; + CameraManager::Open(); + s_instance.isOpen = true; + coreinit::OSSignalEvent(s_cameraOpenEvent); + s_instance.inTargetBuffers.Clear(); + s_instance.outTargetBuffers.Clear(); + return CAM_STATUS_SUCCESS; } - sint32 CAMClose(sint32 camHandle) + CAMStatus CAMSubmitTargetSurface(sint32 camHandle, CAMTargetSurface* targetSurface) { - CameraInstance* camInstance = GetCameraInstanceByHandle(camHandle); - if (!camInstance) - return CAM_ERROR_INVALID_HANDLE; - camInstance->CloseCam(); - return CAM_ERROR_SUCCESS; + if (camHandle != CAM_HANDLE) + return CAM_STATUS_INVALID_HANDLE; + if (!targetSurface || targetSurface->data.IsNull() || targetSurface->size < 1) + return CAM_STATUS_INVALID_ARG; + auto lock = std::scoped_lock(s_instance.mutex); + if (!s_instance.initialized) + return CAM_STATUS_UNINITIALIZED; + if (!s_instance.inTargetBuffers.Push(targetSurface->data)) + return CAM_STATUS_SURFACE_QUEUE_FULL; + return CAM_STATUS_SUCCESS; } - sint32 CAMSubmitTargetSurface(sint32 camHandle, CAMTargetSurface* targetSurface) + void CAMExit(sint32 camHandle) { - CameraInstance* camInstance = GetCameraInstanceByHandle(camHandle); - if (!camInstance) - return CAM_ERROR_INVALID_HANDLE; - - camInstance->QueueTargetSurface(targetSurface); - - return CAM_ERROR_SUCCESS; + if (camHandle != CAM_HANDLE) + return; + std::scoped_lock lock(s_instance.mutex); + if (!s_instance.initialized) + return; + s_instance.isExiting = true; + if (s_instance.isOpen) + CAMClose(camHandle); + coreinit::OSSignalEvent(s_cameraOpenEvent.GetPtr()); + coreinit::OSJoinThread(s_cameraWorkerThread, nullptr); + s_instance.initialized = false; } void reset() { - g_cameraCounter = 0; + CAMExit(0); } void load() { reset(); - cafeExportRegister("camera", CAMGetMemReq, LogType::Placeholder); - cafeExportRegister("camera", CAMCheckMemSegmentation, LogType::Placeholder); - cafeExportRegister("camera", CAMInit, LogType::Placeholder); - cafeExportRegister("camera", CAMExit, LogType::Placeholder); - cafeExportRegister("camera", CAMOpen, LogType::Placeholder); - cafeExportRegister("camera", CAMClose, LogType::Placeholder); - cafeExportRegister("camera", CAMSubmitTargetSurface, LogType::Placeholder); + cafeExportRegister("camera", CAMGetMemReq, LogType::CameraAPI); + cafeExportRegister("camera", CAMCheckMemSegmentation, LogType::CameraAPI); + cafeExportRegister("camera", CAMInit, LogType::CameraAPI); + cafeExportRegister("camera", CAMExit, LogType::CameraAPI); + cafeExportRegister("camera", CAMOpen, LogType::CameraAPI); + cafeExportRegister("camera", CAMClose, LogType::CameraAPI); + cafeExportRegister("camera", CAMSubmitTargetSurface, LogType::CameraAPI); } -} - +} // namespace camera diff --git a/src/Cafe/OS/libs/camera/camera.h b/src/Cafe/OS/libs/camera/camera.h index 04248bbc20..175edb625e 100644 --- a/src/Cafe/OS/libs/camera/camera.h +++ b/src/Cafe/OS/libs/camera/camera.h @@ -2,9 +2,5 @@ namespace camera { - - sint32 CAMOpen(sint32 camHandle); - sint32 CAMClose(sint32 camHandle); - void load(); }; \ No newline at end of file diff --git a/src/Cemu/Logging/CemuLogging.h b/src/Cemu/Logging/CemuLogging.h index d729d36403..c0debfb8cc 100644 --- a/src/Cemu/Logging/CemuLogging.h +++ b/src/Cemu/Logging/CemuLogging.h @@ -14,6 +14,7 @@ enum class LogType : sint32 UnsupportedAPI = 2, SoundAPI = 4, // any audio related API InputAPI = 5, // any input related API + CameraAPI = 27, Socket = 6, Save = 7, H264 = 9, diff --git a/src/camera/CMakeLists.txt b/src/camera/CMakeLists.txt new file mode 100644 index 0000000000..efe3f99fe2 --- /dev/null +++ b/src/camera/CMakeLists.txt @@ -0,0 +1,16 @@ +add_library(CemuCamera + CameraManager.cpp + CameraManager.h + Rgb2Nv12.cpp + Rgb2Nv12.h +) + +set_property(TARGET CemuCamera PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + +target_include_directories(CemuCamera PUBLIC "../") +target_link_libraries(CemuCamera PRIVATE CemuCommon CemuUtil openpnp-capture) + + +if (ENABLE_WXWIDGETS) + target_link_libraries(CemuCamera PRIVATE wx::base) +endif() \ No newline at end of file diff --git a/src/camera/CameraManager.cpp b/src/camera/CameraManager.cpp new file mode 100644 index 0000000000..9f40e190eb --- /dev/null +++ b/src/camera/CameraManager.cpp @@ -0,0 +1,228 @@ +#include "CameraManager.h" + +#include "config/CemuConfig.h" +#include "util/helpers/helpers.h" +#include "Rgb2Nv12.h" + +#include +#include +#include +#include + +#include + +constexpr unsigned CAMERA_WIDTH = 640; +constexpr unsigned CAMERA_HEIGHT = 480; +constexpr unsigned CAMERA_PITCH = 768; + +namespace CameraManager +{ + std::mutex s_mutex; + CapContext s_ctx; + std::optional s_device; + std::optional s_stream; + std::array s_rgbBuffer; + std::array s_nv12Buffer; + int s_refCount = 0; + std::thread s_captureThread; + std::atomic_bool s_capturing = false; + std::atomic_bool s_running = false; + + std::string FourCC(uint32le value) + { + return { + static_cast((value >> 0) & 0xFF), + static_cast((value >> 8) & 0xFF), + static_cast((value >> 16) & 0xFF), + static_cast((value >> 24) & 0xFF)}; + } + + void CaptureLogFunction(uint32_t level, const char* string) + { + cemuLog_log(LogType::CameraAPI, "OpenPNPCapture: {}: {}", level, string); + } + + std::optional FindCorrectFormat() + { + const auto device = *s_device; + cemuLog_log(LogType::CameraAPI, "Video capture device '{}' available formats:", Cap_getDeviceName(s_ctx, device)); + const auto formatCount = Cap_getNumFormats(s_ctx, device); + for (int32_t formatId = 0; formatId < formatCount; ++formatId) + { + CapFormatInfo formatInfo; + if (Cap_getFormatInfo(s_ctx, device, formatId, &formatInfo) != CAPRESULT_OK) + continue; + cemuLog_log(LogType::CameraAPI, "{}: {} {}x{} @ {} fps, {} bpp", formatId, FourCC(formatInfo.fourcc), formatInfo.width, formatInfo.height, formatInfo.fps, formatInfo.bpp); + if (formatInfo.width == CAMERA_WIDTH && formatInfo.height == CAMERA_HEIGHT) + { + cemuLog_log(LogType::CameraAPI, "Selected video format {}", formatId); + return formatId; + } + } + cemuLog_log(LogType::CameraAPI, "Failed to find suitable video format"); + return std::nullopt; + } + + void CaptureWorker() + { + SetThreadName("CameraManager"); + while (s_running) + { + while (s_capturing) + { + s_mutex.lock(); + if (s_stream && Cap_hasNewFrame(s_ctx, *s_stream) && + Cap_captureFrame(s_ctx, *s_stream, s_rgbBuffer.data(), s_rgbBuffer.size()) == CAPRESULT_OK) + Rgb2Nv12(s_rgbBuffer.data(), CAMERA_WIDTH, CAMERA_HEIGHT, s_nv12Buffer.data(), CAMERA_PITCH); + s_mutex.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(30)); + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::yield(); + } + } + void OpenStream() + { + const auto formatId = FindCorrectFormat(); + if (!formatId) + return; + const auto stream = Cap_openStream(s_ctx, *s_device, *formatId); + if (stream == -1) + return; + s_capturing = true; + s_stream = stream; + } + void CloseStream() + { + s_capturing = false; + if (s_stream) + { + Cap_closeStream(s_ctx, *s_stream); + s_stream = std::nullopt; + } + } + void ResetBuffers() + { + std::ranges::fill(s_rgbBuffer, 0); + constexpr auto pixCount = CAMERA_HEIGHT * CAMERA_PITCH; + std::ranges::fill_n(s_nv12Buffer.begin(), pixCount, 16); + std::ranges::fill_n(s_nv12Buffer.begin() + pixCount, (pixCount / 2), 128); + } + + void Init() + { + { + std::scoped_lock lock(s_mutex); + if (s_running) + return; + s_running = true; + s_ctx = Cap_createContext(); + Cap_setLogLevel(4); + Cap_installCustomLogFunction(CaptureLogFunction); + } + + s_captureThread = std::thread(&CaptureWorker); + + const auto uniqueId = GetConfig().camera_id.GetValue(); + if (!uniqueId.empty()) + { + const auto devices = EnumerateDevices(); + for (CapDeviceID deviceId = 0; deviceId < devices.size(); ++deviceId) + { + if (devices[deviceId].uniqueId == uniqueId) + { + s_device = deviceId; + return; + } + } + } + ResetBuffers(); + } + void Deinit() + { + CloseStream(); + Cap_releaseContext(s_ctx); + s_running = false; + s_captureThread.join(); + } + void FillNV12Buffer(uint8* nv12Buffer) + { + std::scoped_lock lock(s_mutex); + std::ranges::copy(s_nv12Buffer, nv12Buffer); + } + + void FillRGBBuffer(uint8* rgbBuffer) + { + std::scoped_lock lock(s_mutex); + std::ranges::copy(s_rgbBuffer, rgbBuffer); + } + void SetDevice(uint32 deviceNo) + { + std::scoped_lock lock(s_mutex); + CloseStream(); + if (deviceNo == DEVICE_NONE) + { + s_device = std::nullopt; + ResetBuffers(); + return; + } + s_device = deviceNo; + if (s_refCount != 0) + OpenStream(); + } + void Open() + { + std::scoped_lock lock(s_mutex); + if (s_device && s_refCount == 0) + { + OpenStream(); + } + s_refCount += 1; + } + void Close() + { + std::scoped_lock lock(s_mutex); + if (s_refCount == 0) + return; + s_refCount -= 1; + if (s_refCount != 0) + return; + CloseStream(); + } + std::vector EnumerateDevices() + { + std::scoped_lock lock(s_mutex); + std::vector infos; + const auto deviceCount = Cap_getDeviceCount(s_ctx); + cemuLog_log(LogType::CameraAPI, "Available video capture devices:"); + for (CapDeviceID deviceNo = 0; deviceNo < deviceCount; ++deviceNo) + { + const auto uniqueId = Cap_getDeviceUniqueID(s_ctx, deviceNo); + const auto name = Cap_getDeviceName(s_ctx, deviceNo); + DeviceInfo info; + info.uniqueId = uniqueId; + + if (name) + info.name = fmt::format("{}: {}", deviceNo, name); + else + info.name = fmt::format("{}: Unknown", deviceNo); + infos.push_back(info); + cemuLog_log(LogType::CameraAPI, "{}", info.name); + } + if (infos.empty()) + cemuLog_log(LogType::CameraAPI, "No available video capture devices"); + return infos; + } + void SaveDevice() + { + std::scoped_lock lock(s_mutex); + auto& config = GetConfig(); + const auto cameraId = s_device ? Cap_getDeviceUniqueID(s_ctx, *s_device) : ""; + config.camera_id = cameraId; + } + + std::optional GetCurrentDevice() + { + return s_device; + } +} // namespace CameraManager \ No newline at end of file diff --git a/src/camera/CameraManager.h b/src/camera/CameraManager.h new file mode 100644 index 0000000000..28e751a0a9 --- /dev/null +++ b/src/camera/CameraManager.h @@ -0,0 +1,25 @@ +#pragma once +#include + +namespace CameraManager +{ + struct DeviceInfo + { + std::string uniqueId; + std::string name; + }; + constexpr static uint32 DEVICE_NONE = std::numeric_limits::max(); + + void Init(); + void Deinit(); + void Open(); + void Close(); + + void FillNV12Buffer(uint8* nv12Buffer); + void FillRGBBuffer(uint8* rgbBuffer); + + void SetDevice(uint32 deviceNo); + std::vector EnumerateDevices(); + void SaveDevice(); + std::optional GetCurrentDevice(); +} // namespace CameraManager diff --git a/src/camera/Rgb2Nv12.cpp b/src/camera/Rgb2Nv12.cpp new file mode 100644 index 0000000000..2390be0aaf --- /dev/null +++ b/src/camera/Rgb2Nv12.cpp @@ -0,0 +1,65 @@ +// Based on https://github.com/cohenrotem/Rgb2NV12 +#include "Rgb2Nv12.h" + +constexpr static glm::mat3x3 coefficientMatrix = + { + +0.257f, -0.148f, 0.439f, + +0.504f, -0.291f, -0.368f, + +0.098f, +0.439f, -0.071f}; + +constexpr static glm::mat4x3 offsetMatrix = { + 16.0f + 0.5f, 128.0f + 2.0f, 128.0f + 2.0f, + 16.0f + 0.5f, 128.0f + 2.0f, 128.0f + 2.0f, + 16.0f + 0.5f, 128.0f + 2.0f, 128.0f + 2.0f, + 16.0f + 0.5f, 128.0f + 2.0f, 128.0f + 2.0f}; + +static void Rgb2Nv12TwoRows(const uint8* topLine, + const uint8* bottomLine, + unsigned imageWidth, + uint8* topLineY, + uint8* bottomLineY, + uint8* uv) +{ + auto* topIn = reinterpret_cast(topLine); + auto* botIn = reinterpret_cast(bottomLine); + + for (auto x = 0u; x < imageWidth; x += 2) + { + const glm::mat4x3 rgbMatrix{ + topIn[x], + topIn[x + 1], + botIn[x], + botIn[x + 1], + }; + + const auto result = coefficientMatrix * rgbMatrix + offsetMatrix; + + topLineY[x + 0] = result[0].s; + topLineY[x + 1] = result[1].s; + bottomLineY[x + 0] = result[2].s; + bottomLineY[x + 1] = result[3].s; + + uv[x + 0] = (result[0].t + result[1].t + result[2].t + result[3].t) * 0.25f; + uv[x + 1] = (result[0].p + result[1].p + result[2].p + result[3].p) * 0.25f; + } +} + +void Rgb2Nv12(const uint8* rgbImage, + unsigned imageWidth, + unsigned imageHeight, + uint8* outNv12Image, + unsigned nv12Pitch) +{ + cemu_assert_debug(!((imageWidth | imageHeight) & 1)); + unsigned char* UV = outNv12Image + nv12Pitch * imageHeight; + + for (auto row = 0u; row < imageHeight; row += 2) + { + Rgb2Nv12TwoRows(&rgbImage[row * imageWidth * 3], + &rgbImage[(row + 1) * imageWidth * 3], + imageWidth, + &outNv12Image[row * nv12Pitch], + &outNv12Image[(row + 1) * nv12Pitch], + &UV[(row / 2) * nv12Pitch]); + } +} diff --git a/src/camera/Rgb2Nv12.h b/src/camera/Rgb2Nv12.h new file mode 100644 index 0000000000..d0e01d01ac --- /dev/null +++ b/src/camera/Rgb2Nv12.h @@ -0,0 +1,7 @@ +#pragma once + +void Rgb2Nv12(const uint8* rgbImage, + unsigned imageWidth, + unsigned imageHeight, + uint8* outNv12Image, + unsigned nv12Pitch); \ No newline at end of file diff --git a/src/config/CemuConfig.cpp b/src/config/CemuConfig.cpp index 6bb7ac34cf..232563706f 100644 --- a/src/config/CemuConfig.cpp +++ b/src/config/CemuConfig.cpp @@ -343,6 +343,9 @@ void CemuConfig::Load(XMLConfigParser& parser) dsu_client.host = dsuc.get_attribute("host", dsu_client.host); dsu_client.port = dsuc.get_attribute("port", dsu_client.port); + auto camera = parser.get("Camera"); + camera_id = camera.get("Id", ""); + // emulatedusbdevices auto usbdevices = parser.get("EmulatedUsbDevices"); emulated_usb_devices.emulate_skylander_portal = usbdevices.get("EmulateSkylanderPortal", emulated_usb_devices.emulate_skylander_portal); @@ -544,6 +547,9 @@ void CemuConfig::Save(XMLConfigParser& parser) dsuc.set_attribute("host", dsu_client.host); dsuc.set_attribute("port", dsu_client.port); + auto camera = config.set("Camera"); + camera.set("Id", camera_id); + // emulated usb devices auto usbdevices = config.set("EmulatedUsbDevices"); usbdevices.set("EmulateSkylanderPortal", emulated_usb_devices.emulate_skylander_portal.GetValue()); diff --git a/src/config/CemuConfig.h b/src/config/CemuConfig.h index 191614a272..c4b01a5767 100644 --- a/src/config/CemuConfig.h +++ b/src/config/CemuConfig.h @@ -499,6 +499,9 @@ struct CemuConfig ConfigValue port{ 26760 }; }dsu_client{}; + // camera + ConfigValue camera_id; + // debug ConfigValueBounds crash_dump{ CrashDump::Disabled }; ConfigValue gdb_port{ 1337 }; diff --git a/src/gui/CMakeLists.txt b/src/gui/CMakeLists.txt index 7cdc208eba..aa302fdf81 100644 --- a/src/gui/CMakeLists.txt +++ b/src/gui/CMakeLists.txt @@ -127,6 +127,8 @@ add_library(CemuGui wxcomponents/unchecked.xpm wxgui.h wxHelper.h + CameraSettingsWindow.cpp + CameraSettingsWindow.h ) set_property(TARGET CemuGui PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") @@ -139,6 +141,7 @@ target_include_directories(CemuGui PUBLIC ${RAPIDJSON_INCLUDE_DIRS}) target_link_libraries(CemuGui PRIVATE CemuAudio CemuCafe + CemuCamera CemuCommon CemuComponents CemuConfig diff --git a/src/gui/CameraSettingsWindow.cpp b/src/gui/CameraSettingsWindow.cpp new file mode 100644 index 0000000000..24cd96dbfa --- /dev/null +++ b/src/gui/CameraSettingsWindow.cpp @@ -0,0 +1,97 @@ +#include "CameraSettingsWindow.h" + +#include "camera/CameraManager.h" + +#include +#include +#include +#include + +constexpr unsigned CAMERA_WIDTH = 640; +constexpr unsigned CAMERA_HEIGHT = 480; + +CameraSettingsWindow::CameraSettingsWindow(wxWindow* parent) + : wxDialog(parent, wxID_ANY, _("Camera settings"), wxDefaultPosition), + m_imageBitmap(CAMERA_WIDTH, CAMERA_HEIGHT, 24), m_imageBuffer(CAMERA_WIDTH * CAMERA_HEIGHT * 3) +{ + + CameraManager::Init(); + CameraManager::Open(); + auto* rootSizer = new wxBoxSizer(wxVERTICAL); + { + auto* topSizer = new wxBoxSizer(wxHORIZONTAL); + { + m_cameraChoice = new wxChoice(this, wxID_ANY, wxDefaultPosition, {300, -1}); + m_cameraChoice->Bind(wxEVT_CHOICE, &CameraSettingsWindow::OnSelectCameraChoice, this); + + m_refreshButton = new wxButton(this, wxID_ANY, wxString::FromUTF8("⟳")); + m_refreshButton->Fit(); + m_refreshButton->Bind(wxEVT_BUTTON, &CameraSettingsWindow::OnRefreshPressed, this); + wxQueueEvent(m_refreshButton, new wxCommandEvent{wxEVT_BUTTON}); + + topSizer->Add(m_cameraChoice); + topSizer->Add(m_refreshButton); + } + + m_imageWindow = new wxWindow(this, wxID_ANY, wxDefaultPosition, {CAMERA_WIDTH, CAMERA_HEIGHT}); + rootSizer->Add(topSizer); + rootSizer->Add(m_imageWindow, wxEXPAND); + } + SetSizerAndFit(rootSizer); + m_imageUpdateTimer.Bind(wxEVT_TIMER, &CameraSettingsWindow::UpdateImage, this); + m_imageUpdateTimer.Start(33, wxTIMER_CONTINUOUS); + this->Bind(wxEVT_CLOSE_WINDOW, &CameraSettingsWindow::OnClose, this); +} +void CameraSettingsWindow::OnSelectCameraChoice(wxCommandEvent&) +{ + const auto selection = m_cameraChoice->GetSelection(); + if (selection < 0) + return; + if (selection == 0) + CameraManager::SetDevice(CameraManager::DEVICE_NONE); + else + CameraManager::SetDevice(selection - 1); +} +void CameraSettingsWindow::OnRefreshPressed(wxCommandEvent&) +{ + wxArrayString choices = {_("None")}; + for (const auto& entry : CameraManager::EnumerateDevices()) + { + choices.push_back(entry.name); + } + m_cameraChoice->Set(choices); + if (auto currentDevice = CameraManager::GetCurrentDevice()) + m_cameraChoice->SetSelection(*currentDevice + 1); +} +void CameraSettingsWindow::UpdateImage(const wxTimerEvent&) +{ + CameraManager::FillRGBBuffer(m_imageBuffer.data()); + + wxNativePixelData data{m_imageBitmap}; + if (!data) + return; + wxNativePixelData::Iterator p{data}; + for (auto row = 0u; row < CAMERA_HEIGHT; ++row) + { + const auto* rowPtr = m_imageBuffer.data() + row * CAMERA_WIDTH * 3; + wxNativePixelData::Iterator rowStart = p; + for (auto col = 0u; col < CAMERA_WIDTH; ++col, ++p) + { + auto* colour = rowPtr + col * 3; + p.Red() = colour[0]; + p.Green() = colour[1]; + p.Blue() = colour[2]; + } + p = rowStart; + p.OffsetY(data, 1); + } + + wxClientDC dc{m_imageWindow}; + dc.DrawBitmap(m_imageBitmap, 0, 0); +} +void CameraSettingsWindow::OnClose(wxCloseEvent& event) +{ + CameraManager::Close(); + CameraManager::SaveDevice(); + event.Skip(); +} \ No newline at end of file diff --git a/src/gui/CameraSettingsWindow.h b/src/gui/CameraSettingsWindow.h new file mode 100644 index 0000000000..1a6dc6df61 --- /dev/null +++ b/src/gui/CameraSettingsWindow.h @@ -0,0 +1,21 @@ +#pragma once +#include +#include +#include +#include + +class CameraSettingsWindow : public wxDialog +{ + wxChoice* m_cameraChoice; + wxButton* m_refreshButton; + wxWindow* m_imageWindow; + wxBitmap m_imageBitmap; + wxTimer m_imageUpdateTimer; + std::vector m_imageBuffer; + public: + explicit CameraSettingsWindow(wxWindow* parent); + void OnSelectCameraChoice(wxCommandEvent&); + void OnRefreshPressed(wxCommandEvent&); + void UpdateImage(const wxTimerEvent&); + void OnClose(wxCloseEvent& event); +}; \ No newline at end of file diff --git a/src/gui/MainWindow.cpp b/src/gui/MainWindow.cpp index 2f63b46089..92ac331a1f 100644 --- a/src/gui/MainWindow.cpp +++ b/src/gui/MainWindow.cpp @@ -61,6 +61,7 @@ #include "gamemode_client.h" #endif +#include "CameraSettingsWindow.h" #include "Cafe/TitleList/TitleInfo.h" #include "Cafe/TitleList/TitleList.h" #include "wxHelper.h" @@ -91,6 +92,7 @@ enum MAINFRAME_MENU_ID_OPTIONS_GENERAL2, MAINFRAME_MENU_ID_OPTIONS_AUDIO, MAINFRAME_MENU_ID_OPTIONS_INPUT, + MAINFRAME_MENU_ID_OPTIONS_CAMERA, // options -> account MAINFRAME_MENU_ID_OPTIONS_ACCOUNT_1 = 20350, MAINFRAME_MENU_ID_OPTIONS_ACCOUNT_12 = 20350 + 11, @@ -187,6 +189,7 @@ EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_GENERAL, MainWindow::OnOptionsInput) EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_GENERAL2, MainWindow::OnOptionsInput) EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_AUDIO, MainWindow::OnOptionsInput) EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_INPUT, MainWindow::OnOptionsInput) +EVT_MENU(MAINFRAME_MENU_ID_OPTIONS_CAMERA, MainWindow::OnOptionsInput) // tools menu EVT_MENU(MAINFRAME_MENU_ID_TOOLS_MEMORY_SEARCHER, MainWindow::OnToolsInput) EVT_MENU(MAINFRAME_MENU_ID_TOOLS_TITLE_MANAGER, MainWindow::OnToolsInput) @@ -923,6 +926,12 @@ void MainWindow::OnOptionsInput(wxCommandEvent& event) frame->Destroy(); break; } + case MAINFRAME_MENU_ID_OPTIONS_CAMERA: + { + auto* frame = new CameraSettingsWindow(this); + frame->ShowModal(); + frame->Destroy(); + } } } @@ -2167,6 +2176,7 @@ void MainWindow::RecreateMenu() optionsMenu->AppendSeparator(); optionsMenu->Append(MAINFRAME_MENU_ID_OPTIONS_GENERAL2, _("&General settings")); optionsMenu->Append(MAINFRAME_MENU_ID_OPTIONS_INPUT, _("&Input settings")); + optionsMenu->Append(MAINFRAME_MENU_ID_OPTIONS_CAMERA, _("&Camera settings")); optionsMenu->AppendSeparator(); optionsMenu->AppendSubMenu(m_optionsAccountMenu, _("&Active account")); @@ -2232,6 +2242,7 @@ void MainWindow::RecreateMenu() logCosModulesMenu->AppendCheckItem(MAINFRAME_MENU_ID_DEBUG_LOGGING0 + stdx::to_underlying(LogType::GX2), _("gx2 API"), wxEmptyString)->Check(cemuLog_isLoggingEnabled(LogType::GX2)); logCosModulesMenu->AppendCheckItem(MAINFRAME_MENU_ID_DEBUG_LOGGING0 + stdx::to_underlying(LogType::SoundAPI), _("Audio API"), wxEmptyString)->Check(cemuLog_isLoggingEnabled(LogType::SoundAPI)); logCosModulesMenu->AppendCheckItem(MAINFRAME_MENU_ID_DEBUG_LOGGING0 + stdx::to_underlying(LogType::InputAPI), _("Input API"), wxEmptyString)->Check(cemuLog_isLoggingEnabled(LogType::InputAPI)); + logCosModulesMenu->AppendCheckItem(MAINFRAME_MENU_ID_DEBUG_LOGGING0 + stdx::to_underlying(LogType::CameraAPI), _("Camera API"), wxEmptyString)->Check(cemuLog_isLoggingEnabled(LogType::CameraAPI)); debugLoggingMenu->AppendSubMenu(logCosModulesMenu, _("&CafeOS modules logging")); debugLoggingMenu->AppendSeparator();