fix: memory leak in d3d texture interop

Author: Julusian

src/accelerator/d3d/d3d_texture2d.cpp

@@ -6,12 +6,12 @@
 #include <GL/wglew.h>
 #include <atlcomcli.h>
 
+#include "../ogl/util/device.h"
+
 namespace caspar { namespace accelerator { namespace d3d {
+
 d3d_texture2d::d3d_texture2d(ID3D11Texture2D* tex)
-    : texture_(std::shared_ptr<ID3D11Texture2D>(tex, [](ID3D11Texture2D* p) {
-        if (p)
-            p->Release();
-    }))
+    : texture_(tex)
 {
     share_handle_ = nullptr;
 
@@ -26,46 +26,81 @@ d3d_texture2d::d3d_texture2d(ID3D11Texture2D* tex)
     }
 
     {
-        CComQIPtr<IDXGIResource1> res = texture_.get();
+        CComQIPtr<IDXGIResource1> res = texture_;
         if (res) {
             res->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ, nullptr, &share_handle_);
         }
     }
 
-    if (share_handle_ == nullptr || !wglDXSetResourceShareHandleNV(texture_.get(), share_handle_)) {
+    if (share_handle_ == nullptr || !wglDXSetResourceShareHandleNV(texture_, share_handle_)) {
         CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("Failed to setup shared d3d texture."));
     }
 }
 
 d3d_texture2d::~d3d_texture2d()
 {
-    if (gl_texture_id_ != 0)
-        glDeleteTextures(1, &gl_texture_id_);
+    const std::shared_ptr<ogl::device> ogl = ogl_.lock();
+    if (ogl != nullptr) {
+        // The cleanup must happen be done on the opengl thread
+        ogl->dispatch_sync([&] {
+            const std::shared_ptr<void> interop = ogl->d3d_interop();
+            if (texture_handle_ != nullptr && interop != nullptr) {
+                wglDXUnlockObjectsNV(interop.get(), 1, &texture_handle_);
+                wglDXUnregisterObjectNV(interop.get(), texture_handle_);
+                texture_handle_ = nullptr;
+            }
+
+            if (gl_texture_id_ != 0) {
+                GL(glDeleteTextures(1, &gl_texture_id_));
+                gl_texture_id_ = 0;
+            }
+
+            // TODO: This appears to be leaking something opengl, but it is not clear what that is.
+
+            if (share_handle_ != nullptr) {
+                CloseHandle(share_handle_);
+                share_handle_ = nullptr;
+            }
+        });
+    }
+
+    if (texture_ != nullptr) {
+        texture_->Release();
+        texture_ = nullptr;
+    }
 }
 
-void d3d_texture2d::gen_gl_texture(const std::shared_ptr<void>& interop)
+void d3d_texture2d::gen_gl_texture(std::shared_ptr<ogl::device> ogl)
 {
-    if (gl_texture_id_ == 0) {
+    if (gl_texture_id_ != 0 || texture_ == nullptr)
+        return;
+
+    ogl_ = ogl;
+
+    const std::shared_ptr<void> interop = ogl->d3d_interop();
+    if (!interop) {
+        CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("d3d interop not setup to bind shared d3d texture."));
+    }
+
+    ogl->dispatch_sync([&] {
         GL(glGenTextures(1, &gl_texture_id_));
-        void* tex_handle = wglDXRegisterObjectNV(
-            interop.get(), texture_.get(), gl_texture_id_, GL_TEXTURE_2D, WGL_ACCESS_READ_ONLY_NV);
-        if (!tex_handle) {
+
+        texture_handle_ =
+            wglDXRegisterObjectNV(interop.get(), texture_, gl_texture_id_, GL_TEXTURE_2D, WGL_ACCESS_READ_ONLY_NV);
+        if (!texture_handle_) {
             GL(glDeleteTextures(1, &gl_texture_id_));
             gl_texture_id_ = 0;
             CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("Failed to bind shared d3d texture."));
         }
 
-        texture_handle_ = std::shared_ptr<void>(tex_handle, [=](void* p) {
-            wglDXUnlockObjectsNV(interop.get(), 1, &p);
-            wglDXUnregisterObjectNV(interop.get(), p);
-        });
-
-        if (!wglDXLockObjectsNV(interop.get(), 1, &tex_handle)) {
+        if (!wglDXLockObjectsNV(interop.get(), 1, &texture_handle_)) {
+            wglDXUnregisterObjectNV(interop.get(), texture_handle_);
+            texture_handle_ = nullptr;
             GL(glDeleteTextures(1, &gl_texture_id_));
             gl_texture_id_ = 0;
             CASPAR_THROW_EXCEPTION(gl::ogl_exception() << msg_info("Failed to lock shared d3d texture."));
         }
-    }
+    });
 }
 
 }}} // namespace caspar::accelerator::d3d

src/accelerator/d3d/d3d_texture2d.h

@@ -8,6 +8,7 @@
 #include <memory>
 
 namespace caspar { namespace accelerator { namespace d3d {
+
 class d3d_texture2d
 {
   public:
@@ -26,21 +27,22 @@ class d3d_texture2d
 
     void* share_handle() const { return share_handle_; }
 
-    ID3D11Texture2D* texture() const { return texture_.get(); }
+    ID3D11Texture2D* texture() const { return texture_; }
 
     uint32_t gl_texture_id() const { return gl_texture_id_; }
 
-    void gen_gl_texture(const std::shared_ptr<void>& interop);
+    void gen_gl_texture(std::shared_ptr<ogl::device>);
 
   private:
     HANDLE share_handle_;
 
-    std::shared_ptr<ID3D11Texture2D> const texture_;
-    uint32_t                               width_  = 0;
-    uint32_t                               height_ = 0;
-    DXGI_FORMAT                            format_ = DXGI_FORMAT::DXGI_FORMAT_UNKNOWN;
+    ID3D11Texture2D* texture_;
+    uint32_t         width_  = 0;
+    uint32_t         height_ = 0;
+    DXGI_FORMAT      format_ = DXGI_FORMAT::DXGI_FORMAT_UNKNOWN;
 
-    std::shared_ptr<void> texture_handle_;
-    uint32_t              gl_texture_id_ = 0;
+    std::weak_ptr<ogl::device> ogl_;
+    HANDLE                     texture_handle_ = nullptr;
+    uint32_t                   gl_texture_id_  = 0;
 };
 }}} // namespace caspar::accelerator::d3d

src/accelerator/ogl/image/image_mixer.cpp

@@ -340,7 +340,7 @@ struct image_mixer::impl
     {
         // map directx texture with wgl texture
         if (d3d_texture->gl_texture_id() == 0)
-            ogl_->dispatch_sync([=] { d3d_texture->gen_gl_texture(ogl_->d3d_interop()); });
+            d3d_texture->gen_gl_texture(ogl_);
 
         // copy directx texture to gl texture
         auto gl_texture = ogl_->dispatch_sync([=] {