Linux: Add Vulkan support for wayland (#553)

Author: SSimco

src/CMakeLists.txt

@@ -27,6 +27,7 @@ elseif(UNIX)
 		add_compile_definitions(
 			VK_USE_PLATFORM_XLIB_KHR # legacy. Do we need to support XLIB surfaces?
 			VK_USE_PLATFORM_XCB_KHR
+			VK_USE_PLATFORM_WAYLAND_KHR
 		)
 	endif()
 	add_compile_options(-maes)

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanAPI.h

@@ -129,6 +129,7 @@ VKFUNC_DEVICE(vkCmdBindPipeline);
 #if BOOST_OS_LINUX
 VKFUNC_INSTANCE(vkCreateXlibSurfaceKHR);
 VKFUNC_INSTANCE(vkCreateXcbSurfaceKHR);
+VKFUNC_INSTANCE(vkCreateWaylandSurfaceKHR);
 #endif
 
 #if BOOST_OS_WINDOWS

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanRenderer.cpp

@@ -107,7 +107,11 @@ std::vector<VulkanRenderer::DeviceInfo> VulkanRenderer::GetDevices()
 	#if BOOST_OS_WINDOWS
 	requiredExtensions.emplace_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
 	#elif BOOST_OS_LINUX
-	requiredExtensions.emplace_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
+	auto backend = gui_getWindowInfo().window_main.backend;
+	if(backend == WindowHandleInfo::Backend::X11)
+		requiredExtensions.emplace_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
+	else if (backend == WindowHandleInfo::Backend::WAYLAND)
+		requiredExtensions.emplace_back(VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME);
 	#elif BOOST_OS_MACOS
 	requiredExtensions.emplace_back(VK_EXT_METAL_SURFACE_EXTENSION_NAME);
 	#endif
@@ -1149,7 +1153,11 @@ std::vector<const char*> VulkanRenderer::CheckInstanceExtensionSupport(FeatureCo
 	#if BOOST_OS_WINDOWS
 	requiredInstanceExtensions.emplace_back(VK_KHR_WIN32_SURFACE_EXTENSION_NAME);
 	#elif BOOST_OS_LINUX
-	requiredInstanceExtensions.emplace_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
+	auto backend = gui_getWindowInfo().window_main.backend;
+	if(backend == WindowHandleInfo::Backend::X11)
+		requiredInstanceExtensions.emplace_back(VK_KHR_XLIB_SURFACE_EXTENSION_NAME);
+	else if (backend == WindowHandleInfo::Backend::WAYLAND)
+		requiredInstanceExtensions.emplace_back(VK_KHR_WAYLAND_SURFACE_EXTENSION_NAME);
 	#elif BOOST_OS_MACOS
 	requiredInstanceExtensions.emplace_back(VK_EXT_METAL_SURFACE_EXTENSION_NAME);
 	#endif
@@ -1267,14 +1275,37 @@ VkSurfaceKHR VulkanRenderer::CreateXcbSurface(VkInstance instance, xcb_connectio
 
     return result;
 }
+
+VkSurfaceKHR VulkanRenderer::CreateWaylandSurface(VkInstance instance, wl_display* display, wl_surface* surface)
+{
+    VkWaylandSurfaceCreateInfoKHR sci{};
+    sci.sType = VK_STRUCTURE_TYPE_WAYLAND_SURFACE_CREATE_INFO_KHR;
+    sci.flags = 0;
+	sci.display = display;
+	sci.surface = surface;
+
+    VkSurfaceKHR result;
+    VkResult err;
+    if ((err = vkCreateWaylandSurfaceKHR(instance, &sci, nullptr, &result)) != VK_SUCCESS)
+    {
+        forceLog_printf("Cannot create a Wayland Vulkan surface: %d", (sint32)err);
+        throw std::runtime_error(fmt::format("Cannot create a Wayland Vulkan surface: {}", err));
+    }
+
+    return result;
+}
 #endif
 
 VkSurfaceKHR VulkanRenderer::CreateFramebufferSurface(VkInstance instance, struct WindowHandleInfo& windowInfo)
 {
 #if BOOST_OS_WINDOWS
 	return CreateWinSurface(instance, windowInfo.hwnd);
 #elif BOOST_OS_LINUX
-	return CreateXlibSurface(instance, windowInfo.xlib_display, windowInfo.xlib_window);
+	if(windowInfo.backend == WindowHandleInfo::Backend::X11)
+		return CreateXlibSurface(instance, windowInfo.xlib_display, windowInfo.xlib_window);
+	if(windowInfo.backend == WindowHandleInfo::Backend::WAYLAND)
+		return CreateWaylandSurface(instance, windowInfo.display, windowInfo.surface);
+	return {};
 #elif BOOST_OS_MACOS
 	return CreateCocoaSurface(instance, windowInfo.handle);
 #endif
@@ -2596,6 +2627,15 @@ bool VulkanRenderer::UpdateSwapchainProperties(bool mainWindow)
 	if (chainInfo.m_usesSRGB != latteBufferUsesSRGB)
 		stateChanged = true;
 
+	int width, height;
+	if (mainWindow)
+		gui_getWindowSize(width, height);
+	else
+		gui_getPadWindowSize(width, height);
+	auto extent = chainInfo.getExtent();
+	if (width != extent.width || height != extent.height)
+		stateChanged = true;
+
 	if(stateChanged)
 	{
 		try

src/Cafe/HW/Latte/Renderer/Vulkan/VulkanRenderer.h

@@ -201,6 +201,7 @@ class VulkanRenderer : public Renderer
 #if BOOST_OS_LINUX
 	static VkSurfaceKHR CreateXlibSurface(VkInstance instance, Display* dpy, Window window);
     static VkSurfaceKHR CreateXcbSurface(VkInstance instance, xcb_connection_t* connection, xcb_window_t window);
+	static VkSurfaceKHR CreateWaylandSurface(VkInstance instance, wl_display* display, wl_surface* surface);
 #endif
 
 	static VkSurfaceKHR CreateFramebufferSurface(VkInstance instance, struct WindowHandleInfo& windowInfo);

src/gui/PadViewFrame.cpp

@@ -87,6 +87,7 @@ void PadViewFrame::InitializeRenderCanvas()
 	m_render_canvas->Bind(wxEVT_GESTURE_PAN, &PadViewFrame::OnGesturePan, this);
 
 	m_render_canvas->SetFocus();
+	SendSizeEvent();
 }
 
 void PadViewFrame::OnSizeEvent(wxSizeEvent& event)

src/gui/canvas/VulkanCanvas.cpp

@@ -11,7 +11,17 @@ VulkanCanvas::VulkanCanvas(wxWindow* parent, const wxSize& size, bool is_main_wi
 	Bind(wxEVT_SIZE, &VulkanCanvas::OnResize, this);
 
 	if(is_main_window)
-		gui_initHandleContextFromWxWidgetsWindow(gui_getWindowInfo().canvas_main, this);
+	{
+		WindowHandleInfo& canvasMain = gui_getWindowInfo().canvas_main;
+		gui_initHandleContextFromWxWidgetsWindow(canvasMain, this);
+		#if BOOST_OS_LINUX
+		if(canvasMain.backend == WindowHandleInfo::Backend::WAYLAND)
+		{	
+			m_subsurface = std::make_unique<wxWlSubsurface>(this);
+			canvasMain.surface = m_subsurface->getSurface();
+		}
+		#endif
+	}
 	else
 		gui_initHandleContextFromWxWidgetsWindow(gui_getWindowInfo().canvas_pad, this);
 
@@ -55,6 +65,14 @@ void VulkanCanvas::OnPaint(wxPaintEvent& event)
 
 void VulkanCanvas::OnResize(wxSizeEvent& event)
 {
+#if BOOST_OS_LINUX
+	if(m_subsurface)
+	{
+		int32_t x,y;
+		GetScreenPosition(&x,&y);
+		m_subsurface->setPosition(x, y);
+	}
+#endif
 	const wxSize size = GetSize();
 	if (size.GetWidth() == 0 || size.GetHeight() == 0)
 		return;

src/gui/canvas/VulkanCanvas.h

@@ -6,11 +6,15 @@
 
 #include "Cafe/HW/Latte/Renderer/Vulkan/VulkanAPI.h"
 #include <set>
-
-
+#if BOOST_OS_LINUX
+#include "gui/helpers/wxWayland.h"
+#endif
 
 class VulkanCanvas : public IRenderCanvas, public wxWindow
 {
+#if BOOST_OS_LINUX
+	std::unique_ptr<wxWlSubsurface> m_subsurface;
+#endif
 public:
 	VulkanCanvas(wxWindow* parent, const wxSize& size, bool is_main_window);
 	~VulkanCanvas();

src/gui/guiWrapper.cpp

@@ -3,6 +3,7 @@
 #include <gdk/gdk.h>
 #include <gdk/gdkwindow.h>
 #include <gdk/gdkx.h>
+#include <gdk/gdkwayland.h>
 #endif
 
 #include "gui/wxgui.h"
@@ -204,19 +205,31 @@ void gui_initHandleContextFromWxWidgetsWindow(WindowHandleInfo& handleInfoOut, c
 #if BOOST_OS_WINDOWS
 	handleInfoOut.hwnd = wxw->GetHWND();
 #elif BOOST_OS_LINUX
-    // get window
     GtkWidget* gtkWidget = (GtkWidget*)wxw->GetHandle(); // returns GtkWidget
     gtk_widget_realize(gtkWidget);
     GdkWindow* gdkWindow = gtk_widget_get_window(gtkWidget);
-    handleInfoOut.xlib_window = gdk_x11_window_get_xid(gdkWindow);
-
-    // get display
-    GdkDisplay* gdkDisplay = gdk_window_get_display(gdkWindow);
-    handleInfoOut.xlib_display = gdk_x11_display_get_xdisplay(gdkDisplay);
-    if(!handleInfoOut.xlib_display)
-    {
-        cemuLog_log(LogType::Force, "Unable to get xlib display");
-    }
+	GdkDisplay* gdkDisplay = gdk_window_get_display(gdkWindow);
+	if(GDK_IS_X11_WINDOW(gdkWindow))
+	{
+		handleInfoOut.backend = WindowHandleInfo::Backend::X11;
+		handleInfoOut.xlib_window = gdk_x11_window_get_xid(gdkWindow);
+		handleInfoOut.xlib_display = gdk_x11_display_get_xdisplay(gdkDisplay);
+		if(!handleInfoOut.xlib_display)
+		{
+			cemuLog_log(LogType::Force, "Unable to get xlib display");
+		}
+	}
+	else if(GDK_IS_WAYLAND_WINDOW(gdkWindow))
+	{
+		handleInfoOut.backend = WindowHandleInfo::Backend::WAYLAND;
+		handleInfoOut.surface = gdk_wayland_window_get_wl_surface(gdkWindow);
+		handleInfoOut.display = gdk_wayland_display_get_wl_display(gdkDisplay);
+	}
+	else
+	{
+		cemuLog_log(LogType::Force, "Unsuported GTK backend");
+
+	}
 #else
 	handleInfoOut.handle = wxw->GetHandle();
 #endif

src/gui/guiWrapper.h

@@ -5,6 +5,7 @@
 #if BOOST_OS_LINUX
 #include "xcb/xproto.h"
 #include <gdk/gdkkeysyms.h>
+#include <wayland-client.h>
 #endif
 
 #if BOOST_OS_MACOS
@@ -16,6 +17,11 @@ struct WindowHandleInfo
 #if BOOST_OS_WINDOWS
 	std::atomic<HWND> hwnd;
 #elif BOOST_OS_LINUX
+	enum class Backend
+	{
+		X11,
+		WAYLAND,
+	} backend;
     // XLIB
     Display* xlib_display{};
     Window xlib_window{};
@@ -24,7 +30,8 @@ struct WindowHandleInfo
 	//xcb_connection_t* xcb_con{};
 	//xcb_window_t xcb_window{};
 	// Wayland
-	// todo
+	wl_display* display;
+	wl_surface* surface;
 #else
 	void* handle;
 #endif

src/gui/helpers/wxWayland.h

@@ -0,0 +1,75 @@
+#pragma once
+
+#if BOOST_OS_LINUX
+
+#include <gdk/gdk.h>
+#include <gdk/gdkwayland.h>
+#include <gtk/gtk.h>
+#include <wayland-client.h>
+#include <wx/wx.h>
+
+class wxWlSubsurface
+{
+	static void registry_add_object(void* data, struct wl_registry* registry, uint32_t name, const char* interface, uint32_t version)
+	{
+		if (!strcmp(interface, wl_subcompositor_interface.name))
+		{
+			auto wlSubsurface = static_cast<wxWlSubsurface*>(data);
+			wlSubsurface->m_subcompositor = static_cast<wl_subcompositor*>(wl_registry_bind(registry, name, &wl_subcompositor_interface, 1));
+		}
+	}
+	static void registry_remove_object(void* /*data*/, struct wl_registry* /*registry*/, uint32_t /*name*/) {}
+
+	wl_registry_listener m_registry_listener = {&registry_add_object, &registry_remove_object};
+	wl_subcompositor* m_subcompositor;
+	wl_surface* m_surface;
+	wl_subsurface* m_subsurface;
+	int32_t m_xPos = 0;
+	int32_t m_yPos = 0;
+
+   public:
+	wxWlSubsurface(wxWindow* window)
+	{
+		GtkWidget* widget = static_cast<GtkWidget*>(window->GetHandle());
+		gtk_widget_realize(widget);
+		GdkWindow* gdkWindow = gtk_widget_get_window(widget);
+		GdkDisplay* gdkDisplay = gdk_window_get_display(gdkWindow);
+		wl_display* display = gdk_wayland_display_get_wl_display(gdkDisplay);
+		wl_surface* surface = gdk_wayland_window_get_wl_surface(gdkWindow);
+		wl_compositor* compositor = gdk_wayland_display_get_wl_compositor(gdkDisplay);
+		wl_registry* registry = wl_display_get_registry(display);
+		wl_registry_add_listener(registry, &m_registry_listener, this);
+		wl_display_roundtrip(display);
+		m_surface = wl_compositor_create_surface(compositor);
+		wl_region* region = wl_compositor_create_region(compositor);
+		wl_surface_set_input_region(m_surface, region);
+		m_subsurface = wl_subcompositor_get_subsurface(m_subcompositor, m_surface, surface);
+		wl_subsurface_set_desync(m_subsurface);
+		window->GetScreenPosition(&m_xPos, &m_yPos);
+		wl_subsurface_set_position(m_subsurface, m_xPos, m_yPos);
+		wl_surface_commit(m_surface);
+		wl_region_destroy(region);
+	}
+
+	wl_surface* getSurface() const { return m_surface; }
+
+	void setPosition(int32_t xPos, int32_t yPos)
+	{
+		if (xPos != m_xPos || m_yPos != yPos)
+		{
+			m_xPos = xPos;
+			m_yPos = yPos;
+			wl_subsurface_set_position(m_subsurface, m_xPos, m_yPos);
+			wl_surface_commit(m_surface);
+		}
+	}
+
+	~wxWlSubsurface()
+	{
+		wl_subsurface_destroy(m_subsurface);
+		wl_surface_destroy(m_surface);
+		wl_subcompositor_destroy(m_subcompositor);
+	}
+};
+
+#endif	// BOOST_OS_LINUX