wayland: Handle min/max sizes in fixed-size windows with viewports

Wayland is sometimes at-odds with clients that want to enforce an aspect ratio or min/max window size, as certain window states have dimensions that either must be obeyed (maximized), or will give terrible results if they aren't (tiled). Use viewports and/or a masking subsurface to handle cases where surfaces are unable to match the exact window size.

The masking system also allows for switching the emulated fullscreen modes to aspect-correct by default, as it will handle masking non-native aspect ratios on compositors that don't automatically mask around fullscreen windows.

Author: Kontrabant

include/SDL3/SDL_hints.h

@@ -3805,7 +3805,7 @@ extern "C" {
 #define SDL_HINT_VIDEO_WAYLAND_MODE_EMULATION "SDL_VIDEO_WAYLAND_MODE_EMULATION"
 
 /**
- * A variable controlling how modes with a non-native aspect ratio are
+ * A variable controlling how video modes with a non-native aspect ratio are
  * displayed under Wayland.
  *
  * When this hint is set, the requested scaling will be used when displaying
@@ -3815,9 +3815,8 @@ extern "C" {
  * The variable can be set to the following values:
  *
  * - "aspect" - Video modes will be displayed scaled, in their proper aspect
- *   ratio, with black bars.
+ *   ratio, with black bars. (default)
  * - "stretch" - Video modes will be scaled to fill the entire display.
- *   (default)
  * - "none" - Video modes will be displayed as 1:1 with no scaling.
  *
  * This hint should be set before creating a window.

src/video/wayland/SDL_waylandshmbuffer.c

@@ -32,6 +32,7 @@
 
 #include "SDL_waylandshmbuffer.h"
 #include "SDL_waylandvideo.h"
+#include "single-pixel-buffer-v1-client-protocol.h"
 
 static bool SetTempFileSize(int fd, off_t size)
 {
@@ -186,4 +187,28 @@ void Wayland_ReleaseSHMPool(Wayland_SHMPool *shmPool)
     }
 }
 
+struct wl_buffer *Wayland_CreateSinglePixelBuffer(Uint32 r, Uint32 g, Uint32 b, Uint32 a)
+{
+    SDL_VideoData *viddata = SDL_GetVideoDevice()->internal;
+
+    // THe single-pixel buffer protocol is preferred, as the compositor can choose an optimal format.
+    if (viddata->single_pixel_buffer_manager) {
+        return wp_single_pixel_buffer_manager_v1_create_u32_rgba_buffer(viddata->single_pixel_buffer_manager, r, g, b, a);
+    } else {
+        Wayland_SHMPool *pool = Wayland_AllocSHMPool(4);
+        if (!pool) {
+            return NULL;
+        }
+
+        void *mem;
+        struct wl_buffer *wl_buffer = Wayland_AllocBufferFromPool(pool, 1, 1, &mem);
+
+        const Uint8 pixel[4] = { r >> 24, g >> 24, b >> 24, a >> 24 };
+        SDL_memcpy(mem, pixel, sizeof(pixel));
+
+        Wayland_ReleaseSHMPool(pool);
+        return wl_buffer;
+    }
+}
+
 #endif

src/video/wayland/SDL_waylandshmbuffer.h

@@ -30,4 +30,6 @@ extern Wayland_SHMPool *Wayland_AllocSHMPool(int size);
 extern struct wl_buffer *Wayland_AllocBufferFromPool(Wayland_SHMPool *shmPool, int width, int height, void **data);
 extern void Wayland_ReleaseSHMPool(Wayland_SHMPool *shmPool);
 
+extern struct wl_buffer *Wayland_CreateSinglePixelBuffer(Uint32 r, Uint32 g, Uint32 b, Uint32 a);
+
 #endif

src/video/wayland/SDL_waylandvideo.c

@@ -68,6 +68,7 @@
 #include "color-management-v1-client-protocol.h"
 #include "pointer-warp-v1-client-protocol.h"
 #include "pointer-gestures-unstable-v1-client-protocol.h"
+#include "single-pixel-buffer-v1-client-protocol.h"
 
 #ifdef HAVE_LIBDECOR_H
 #include <libdecor.h>
@@ -651,6 +652,7 @@ static SDL_VideoDevice *Wayland_CreateDevice(bool require_preferred_protocols)
     device->SetWindowResizable = Wayland_SetWindowResizable;
     device->SetWindowPosition = Wayland_SetWindowPosition;
     device->SetWindowSize = Wayland_SetWindowSize;
+    device->SetWindowAspectRatio = Wayland_SetWindowAspectRatio;
     device->SetWindowMinimumSize = Wayland_SetWindowMinimumSize;
     device->SetWindowMaximumSize = Wayland_SetWindowMaximumSize;
     device->SetWindowParent = Wayland_SetWindowParent;
@@ -1272,6 +1274,8 @@ static void handle_registry_global(void *data, struct wl_registry *registry, uin
 
     if (SDL_strcmp(interface, "wl_compositor") == 0) {
         d->compositor = wl_registry_bind(d->registry, id, &wl_compositor_interface, SDL_min(SDL_WL_COMPOSITOR_VERSION, version));
+    } else if (SDL_strcmp(interface, "wl_subcompositor") == 0) {
+        d->subcompositor = wl_registry_bind(d->registry, id, &wl_subcompositor_interface, 1);
     } else if (SDL_strcmp(interface, "wl_output") == 0) {
         Wayland_add_display(d, id, SDL_min(version, SDL_WL_OUTPUT_VERSION));
     } else if (SDL_strcmp(interface, "wl_seat") == 0) {
@@ -1337,6 +1341,8 @@ static void handle_registry_global(void *data, struct wl_registry *registry, uin
     } else if (SDL_strcmp(interface, "zwp_pointer_gestures_v1") == 0) {
         d->zwp_pointer_gestures = wl_registry_bind(d->registry, id, &zwp_pointer_gestures_v1_interface, SDL_min(version, 3));
         Wayland_DisplayInitPointerGestureManager(d);
+    } else if (SDL_strcmp(interface, "wp_single_pixel_buffer_manager_v1") == 0) {
+        d->single_pixel_buffer_manager = wl_registry_bind(d->registry, id, &wp_single_pixel_buffer_manager_v1_interface, 1);
     }
 #ifdef SDL_WL_FIXES_VERSION
     else if (SDL_strcmp(interface, "wl_fixes") == 0) {
@@ -1665,6 +1671,11 @@ static void Wayland_VideoCleanup(SDL_VideoDevice *_this)
         data->zwp_pointer_gestures = NULL;
     }
 
+    if (data->single_pixel_buffer_manager) {
+        wp_single_pixel_buffer_manager_v1_destroy(data->single_pixel_buffer_manager);
+        data->single_pixel_buffer_manager = NULL;
+    }
+
     if (data->compositor) {
         wl_compositor_destroy(data->compositor);
         data->compositor = NULL;

src/video/wayland/SDL_waylandvideo.h

@@ -61,6 +61,7 @@ struct SDL_VideoData
         struct libdecor *libdecor;
 #endif
     } shell;
+    struct wl_subcompositor *subcompositor;
     struct zwp_relative_pointer_manager_v1 *relative_pointer_manager;
     struct zwp_pointer_constraints_v1 *pointer_constraints;
     struct wp_pointer_warp_v1 *wp_pointer_warp_v1;
@@ -85,6 +86,7 @@ struct SDL_VideoData
     struct zwp_tablet_manager_v2 *tablet_manager;
     struct wl_fixes *wl_fixes;
     struct zwp_pointer_gestures_v1 *zwp_pointer_gestures;
+    struct wp_single_pixel_buffer_manager_v1 *single_pixel_buffer_manager;
 
     struct xkb_context *xkb_context;