Dynamically adjust libplacebo swapchain depth on macOS

Direct-to-display mode on some Macs (M4 Mac Mini) can retain our drawables, which cuts the frame rate in half if we're in double-buffered mode.

We don't want to use triple-buffered mode all the time because it increases latency when used if not required, so detect delayed presentation and enable it on the fly if needed.

Author: cgutman

app/streaming/video/ffmpeg-renderers/plvk.cpp

@@ -421,6 +421,7 @@ bool PlVkRenderer::isExtensionSupportedByPhysicalDevice(VkPhysicalDevice device,
 bool PlVkRenderer::initialize(PDECODER_PARAMETERS params)
 {
     m_Window = params->window;
+    m_MaxVideoFps = params->frameRate;
 
     unsigned int instanceExtensionCount = 0;
     if (!SDL_Vulkan_GetInstanceExtensions(params->window, &instanceExtensionCount, nullptr)) {
@@ -488,18 +489,17 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params)
         return false;
     }
 
-    VkPresentModeKHR presentMode;
     if (params->enableVsync) {
         // FIFO mode improves frame pacing compared with Mailbox, especially for
         // platforms like X11 that lack a VSyncSource implementation for Pacer.
-        presentMode = VK_PRESENT_MODE_FIFO_KHR;
+        m_VkPresentMode = VK_PRESENT_MODE_FIFO_KHR;
     }
     else {
         // We want immediate mode for V-Sync disabled if possible
         if (isPresentModeSupportedByPhysicalDevice(m_Vulkan->phys_device, VK_PRESENT_MODE_IMMEDIATE_KHR)) {
             SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                         "Using Immediate present mode with V-Sync disabled");
-            presentMode = VK_PRESENT_MODE_IMMEDIATE_KHR;
+            m_VkPresentMode = VK_PRESENT_MODE_IMMEDIATE_KHR;
         }
         else {
             SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
@@ -509,41 +509,26 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params)
             if (isPresentModeSupportedByPhysicalDevice(m_Vulkan->phys_device, VK_PRESENT_MODE_FIFO_RELAXED_KHR)) {
                 SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                             "Using FIFO Relaxed present mode with V-Sync disabled");
-                presentMode = VK_PRESENT_MODE_FIFO_RELAXED_KHR;
+                m_VkPresentMode = VK_PRESENT_MODE_FIFO_RELAXED_KHR;
             }
             // Mailbox at least provides non-blocking behavior
             else if (isPresentModeSupportedByPhysicalDevice(m_Vulkan->phys_device, VK_PRESENT_MODE_MAILBOX_KHR)) {
                 SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                             "Using Mailbox present mode with V-Sync disabled");
-                presentMode = VK_PRESENT_MODE_MAILBOX_KHR;
+                m_VkPresentMode = VK_PRESENT_MODE_MAILBOX_KHR;
             }
             // FIFO is always supported
             else {
                 SDL_LogInfo(SDL_LOG_CATEGORY_APPLICATION,
                             "Using FIFO present mode with V-Sync disabled");
-                presentMode = VK_PRESENT_MODE_FIFO_KHR;
+                m_VkPresentMode = VK_PRESENT_MODE_FIFO_KHR;
             }
         }
     }
 
-    pl_vulkan_swapchain_params vkSwapchainParams = {};
-    vkSwapchainParams.surface = m_VkSurface;
-    vkSwapchainParams.present_mode = presentMode;
-    vkSwapchainParams.swapchain_depth = 1; // No queued frames
-#if PL_API_VER >= 338
-    vkSwapchainParams.disable_10bit_sdr = true; // Some drivers don't dither 10-bit SDR output correctly
-#endif
-
-    {
-        // Don't let Qt take DRM master from us during pl_vulkan_create_swapchain()
-        DrmMasterLocker locker;
-
-        m_Swapchain = pl_vulkan_create_swapchain(m_Vulkan, &vkSwapchainParams);
-        if (m_Swapchain == nullptr) {
-            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
-                         "pl_vulkan_create_swapchain() failed");
-            return false;
-        }
+    // Start with a swapchain that is double-buffered for lowest display latency
+    if (!createSwapchain(1)) {
+        return false;
     }
 
     m_Renderer = pl_renderer_create(m_Log, m_Vulkan->gpu);
@@ -622,6 +607,35 @@ bool PlVkRenderer::initialize(PDECODER_PARAMETERS params)
     return true;
 }
 
+
+bool PlVkRenderer::createSwapchain(int depth)
+{
+    pl_swapchain_destroy(&m_Swapchain);
+
+    pl_vulkan_swapchain_params vkSwapchainParams = {};
+    vkSwapchainParams.surface = m_VkSurface;
+    vkSwapchainParams.present_mode = m_VkPresentMode;
+    vkSwapchainParams.swapchain_depth = depth;
+#if PL_API_VER >= 338
+    vkSwapchainParams.disable_10bit_sdr = true; // Some drivers don't dither 10-bit SDR output correctly
+#endif
+
+    {
+        // Don't let Qt take DRM master from us during pl_vulkan_create_swapchain()
+        DrmMasterLocker locker;
+
+        m_Swapchain = pl_vulkan_create_swapchain(m_Vulkan, &vkSwapchainParams);
+        if (m_Swapchain == nullptr) {
+            SDL_LogError(SDL_LOG_CATEGORY_APPLICATION,
+                         "pl_vulkan_create_swapchain() failed");
+            return false;
+        }
+    }
+
+    m_SwapchainDepth = depth;
+    return true;
+}
+
 bool PlVkRenderer::prepareDecoderContext(AVCodecContext *context, AVDictionary **)
 {
     if (m_HwDeviceCtx) {
@@ -991,6 +1005,10 @@ void PlVkRenderer::renderFrame(AVFrame *frame)
     targetFrame.crop.x1 = dst.x + dst.w;
     targetFrame.crop.y1 = dst.y + dst.h;
 
+#ifdef PLVK_USE_DYNAMIC_SWAPCHAIN_DEPTH
+    Uint32 renderStartTime = SDL_GetTicks();
+#endif
+
     // Render the video image and overlays into the swapchain buffer
     targetFrame.num_overlays = (int)overlays.size();
     targetFrame.overlays = overlays.data();
@@ -1013,6 +1031,32 @@ void PlVkRenderer::renderFrame(AVFrame *frame)
         goto UnmapExit;
     }
 
+#ifdef PLVK_USE_DYNAMIC_SWAPCHAIN_DEPTH
+    // Trigger a switch to triple-buffered mode if our frame presentation time
+    // exceeds 110% of the frame interval for half a second of frames.
+    if (SDL_GetTicks() - renderStartTime > (1100U / m_MaxVideoFps)) {
+        m_DelayedPresents++;
+    }
+    else if (m_DelayedPresents > 0) {
+        m_DelayedPresents--;
+    }
+
+    if (m_DelayedPresents == m_MaxVideoFps / 2 && m_SwapchainDepth < 2) {
+        SDL_LogWarn(SDL_LOG_CATEGORY_APPLICATION,
+                    "Switching to triple-buffered swapchain after delayed presentations");
+        if (!createSwapchain(2)) {
+            // Recreate the renderer
+            SDL_Event event;
+            event.type = SDL_RENDER_DEVICE_RESET;
+            SDL_PushEvent(&event);
+            goto UnmapExit;
+        }
+
+        // Restore the swapchain's colorspace from the previous swapchain frame
+        pl_swapchain_colorspace_hint(m_Swapchain, &targetFrame.color);
+    }
+#endif
+
 #ifdef Q_OS_WIN32
     // On Windows, we swap buffers here instead of waitToRender()
     // to avoid some performance problems on Nvidia GPUs.

app/streaming/video/ffmpeg-renderers/plvk.h

@@ -23,6 +23,11 @@ class MetalVulkanTextureFactory {
     pl_vulkan m_Vulkan;
     /* CVMetalTextureCacheRef */ void* m_TextureCache = nullptr;
 };
+
+// Work around direct-to-display mode sometimes (but not always!) blocking us
+// from getting a new drawable while the current one is getting scanned out.
+#define PLVK_USE_DYNAMIC_SWAPCHAIN_DEPTH 1
+
 #endif
 
 class PlVkRenderer : public IFFmpegRenderer {
@@ -49,6 +54,7 @@ class PlVkRenderer : public IFFmpegRenderer {
     static void unlockQueue(AVHWDeviceContext *dev_ctx, uint32_t queue_family, uint32_t index);
     static void overlayUploadComplete(void* opaque);
 
+    bool createSwapchain(int depth);
     bool mapAvFrameToPlacebo(const AVFrame *frame, pl_frame* mappedFrame);
     void unmapAvFrameFromPlacebo(const AVFrame *frame, pl_frame* mappedFrame);
     bool populateQueues(int videoFormat);
@@ -68,13 +74,22 @@ class PlVkRenderer : public IFFmpegRenderer {
     std::unique_ptr<MetalVulkanTextureFactory> m_MetalTextureFactory;
 #endif
 
+#ifdef PLVK_USE_DYNAMIC_SWAPCHAIN_DEPTH
+    int m_DelayedPresents = 0;
+#endif
+
     // SDL state
     SDL_Window* m_Window = nullptr;
 
+    // Stream state
+    int m_MaxVideoFps;
+
     // The libplacebo rendering state
     pl_log m_Log = nullptr;
     pl_vk_inst m_PlVkInstance = nullptr;
     VkSurfaceKHR m_VkSurface = VK_NULL_HANDLE;
+    int m_SwapchainDepth = 0;
+    VkPresentModeKHR m_VkPresentMode;
     pl_vulkan m_Vulkan = nullptr;
     pl_swapchain m_Swapchain = nullptr;
     pl_renderer m_Renderer = nullptr;