wayland: Add make_modal protocol method

Author: leolost2605

protocol/pantheon-desktop-shell-v1.xml

@@ -151,5 +151,13 @@
         by the compositor.
       </description>
     </request>
+
+    <request name="make_modal">
+      <description summary="requests to make a surface system modal">
+        This will block all user input outside the surface and most system shortcuts.
+      </description>
+
+      <arg name="dim" type="uint" summary="1 to dim, 0 to not dim"/>
+    </request>
   </interface>
 </protocol>

protocol/pantheon-desktop-shell.vapi

@@ -61,6 +61,7 @@ namespace Pantheon.Desktop {
         public SetKeepAbove set_keep_above;
         public MakeCentered make_centered;
         public Focus focus;
+        public MakeModal make_modal;
     }
 
     [CCode (has_target = false, has_typedef = false)]
@@ -88,5 +89,7 @@ namespace Pantheon.Desktop {
     [CCode (has_target = false, has_typedef = false)]
     public delegate void MakeCentered (Wl.Client client, Wl.Resource resource);
     [CCode (has_target = false, has_typedef = false)]
+    public delegate void MakeModal (Wl.Client client, Wl.Resource resource, uint dim);
+    [CCode (has_target = false, has_typedef = false)]
     public delegate void Destroy (Wl.Client client, Wl.Resource resource);
 }

src/PantheonShell.vala

@@ -53,6 +53,7 @@ namespace Gala {
             set_keep_above,
             make_centered,
             focus_extended_behavior,
+            make_modal,
         };
 
         PanelSurface.quark = GLib.Quark.from_string ("-gala-wayland-panel-surface-data");
@@ -376,6 +377,21 @@ namespace Gala {
         ShellClientsManager.get_instance ().make_centered (window);
     }
 
+    internal static void make_modal (Wl.Client client, Wl.Resource resource, uint dim) {
+        unowned ExtendedBehaviorSurface? eb_surface = resource.get_user_data<ExtendedBehaviorSurface> ();
+        if (eb_surface.wayland_surface == null) {
+            return;
+        }
+
+        Meta.Window? window;
+        eb_surface.wayland_surface.get ("window", out window, null);
+        if (window == null) {
+            return;
+        }
+
+        ShellClientsManager.get_instance ().make_modal (window, dim == 1);
+    }
+
     internal static void destroy_panel_surface (Wl.Client client, Wl.Resource resource) {
         resource.destroy ();
     }

src/ShellClients/ExtendedBehaviorWindow.vala

@@ -6,11 +6,19 @@
  */
 
 public class Gala.ExtendedBehaviorWindow : ShellWindow {
+    public bool modal { get; private set; default = false; }
+    public bool dim { get; private set; default = false; }
+
     public ExtendedBehaviorWindow (Meta.Window window) {
         var target = new PropertyTarget (CUSTOM, window.get_compositor_private (), "opacity", typeof (uint), 255u, 0u);
         Object (window: window, hide_target: target);
     }
 
+    public void make_modal (bool dim) {
+        modal = true;
+        this.dim = dim;
+    }
+
     protected override void get_window_position (
         Mtk.Rectangle window_rect, Mtk.Rectangle monitor_rect, out int x, out int y
     ) {

src/ShellClients/ShellClientsManager.vala

@@ -26,7 +26,7 @@ public class Gala.ShellClientsManager : Object, GestureTarget {
     private ManagedClient[] protocol_clients = {};
 
     private GLib.HashTable<Meta.Window, PanelWindow> panel_windows = new GLib.HashTable<Meta.Window, PanelWindow> (null, null);
-    private GLib.HashTable<Meta.Window, ShellWindow> positioned_windows = new GLib.HashTable<Meta.Window, ShellWindow> (null, null);
+    private GLib.HashTable<Meta.Window, ExtendedBehaviorWindow> positioned_windows = new GLib.HashTable<Meta.Window, ExtendedBehaviorWindow> (null, null);
 
     private ShellClientsManager (WindowManager wm) {
         Object (wm: wm);
@@ -205,6 +205,10 @@ public class Gala.ShellClientsManager : Object, GestureTarget {
         window.unmanaging.connect_after ((_window) => positioned_windows.remove (_window));
     }
 
+    public void make_modal (Meta.Window window, bool dim) requires (window in positioned_windows) {
+        positioned_windows[window].make_modal (dim);
+    }
+
     public void propagate (UpdateType update_type, GestureAction action, double progress) {
         foreach (var window in positioned_windows.get_values ()) {
             window.propagate (update_type, action, progress);
@@ -232,6 +236,27 @@ public class Gala.ShellClientsManager : Object, GestureTarget {
         return positioned;
     }
 
+    private bool is_itself_system_modal (Meta.Window window) {
+        return (window in positioned_windows) && positioned_windows[window].modal;
+    }
+
+    public bool is_system_modal_window (Meta.Window window) {
+        var modal = is_itself_system_modal (window);
+        window.foreach_ancestor ((ancestor) => {
+            if (is_itself_system_modal (ancestor)) {
+                modal = true;
+            }
+
+            return !modal;
+        });
+
+        return modal;
+    }
+
+    public bool is_system_modal_dimmed (Meta.Window window) {
+        return is_itself_system_modal (window) && positioned_windows[window].dim;
+    }
+
     //X11 only
     private void parse_mutter_hints (Meta.Window window) requires (!Meta.Util.is_wayland_compositor ()) {
         if (window.mutter_hints == null) {

src/Widgets/ModalGroup.vala

@@ -0,0 +1,77 @@
+/*
+ * Copyright 2025 elementary, Inc. (https://elementary.io)
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ *
+ * Authored by: Leonhard Kargl <[email protected]>
+ */
+
+/**
+ * This class allows to make windows system modal i.e. dim
+ * the desktop behind them and only allow interaction with them.
+ * Not to be confused with WindowManager.push_modal which only
+ * works for our own Clutter.Actors.
+ */
+public class Gala.ModalGroup : Clutter.Actor {
+    public WindowManager wm { private get; construct; }
+    public ShellClientsManager shell_clients { private get; construct; }
+
+    private Gee.Set<Clutter.Actor> dimmed;
+    private ModalProxy? modal_proxy = null;
+
+    public ModalGroup (WindowManager wm, ShellClientsManager shell_clients) {
+        Object (wm: wm, shell_clients: shell_clients);
+    }
+
+    construct {
+        dimmed = new Gee.HashSet<Clutter.Actor> ();
+
+        visible = false;
+        reactive = true;
+
+        child_added.connect (on_child_added);
+        child_removed.connect (on_child_removed);
+    }
+
+    private void on_child_added (Clutter.Actor child) {
+        if (child is Meta.WindowActor && shell_clients.is_system_modal_dimmed (child.meta_window)) {
+            dimmed.add (child);
+        }
+
+        if (get_n_children () == 1) {
+            assert (modal_proxy == null);
+
+            visible = true;
+            modal_proxy = wm.push_modal (this, false);
+        }
+
+        if (dimmed.size == 1) {
+            save_easing_state ();
+            set_easing_duration (Utils.get_animation_duration (AnimationDuration.OPEN));
+            background_color = { 0, 0, 0, 200 };
+            restore_easing_state ();
+        }
+    }
+
+    private void on_child_removed (Clutter.Actor child) {
+        dimmed.remove (child);
+
+        if (dimmed.size == 0) {
+            save_easing_state ();
+            set_easing_duration (Utils.get_animation_duration (AnimationDuration.CLOSE));
+            background_color = { 0, 0, 0, 0 };
+            restore_easing_state ();
+        }
+
+        if (get_n_children () == 0) {
+            wm.pop_modal (modal_proxy);
+            modal_proxy = null;
+
+            var transition = get_transition ("background-color");
+            if (transition != null) {
+                transition.completed.connect (() => visible = false);
+            } else {
+                visible = false;
+            }
+        }
+    }
+}

src/WindowManager.vala

@@ -49,6 +49,12 @@ namespace Gala {
          */
         public Clutter.Actor shell_group { get; private set; }
 
+        /**
+         * The group that contains all WindowActors that are system modal.
+         * See {@link ShellClientsManager.is_system_modal_window}.
+         */
+        public ModalGroup modal_group { get; private set; }
+
         /**
          * {@inheritDoc}
          */
@@ -223,6 +229,7 @@ namespace Gala {
              * +-- window switcher
              * +-- window overview
              * +-- shell group
+             * +-- modal group
              * +-- feedback group (e.g. DND icons)
              * +-- pointer locator
              * +-- dwell click timer
@@ -291,6 +298,10 @@ namespace Gala {
             shell_group = new Clutter.Actor ();
             ui_group.add_child (shell_group);
 
+            modal_group = new ModalGroup (this, ShellClientsManager.get_instance ());
+            modal_group.add_constraint (new Clutter.BindConstraint (stage, ALL, 0));
+            ui_group.add_child (modal_group);
+
             var feedback_group = display.get_compositor ().get_feedback_group ();
             stage.remove_child (feedback_group);
             ui_group.add_child (feedback_group);
@@ -1011,6 +1022,12 @@ namespace Gala {
 
         private void check_shell_window (Meta.WindowActor actor) {
             unowned var window = actor.get_meta_window ();
+
+            if (ShellClientsManager.get_instance ().is_system_modal_window (window)) {
+                InternalUtils.clutter_actor_reparent (actor, modal_group);
+                return;
+            }
+
             if (ShellClientsManager.get_instance ().is_positioned_window (window)) {
                 InternalUtils.clutter_actor_reparent (actor, shell_group);
             }

src/meson.build

@@ -60,6 +60,7 @@ gala_bin_sources = files(
     'Widgets/MultitaskingView/WindowCloneContainer.vala',
     'Widgets/MultitaskingView/WorkspaceClone.vala',
     'Widgets/MultitaskingView/WorkspaceRow.vala',
+    'Widgets/ModalGroup.vala',
     'Widgets/PixelPicker.vala',
     'Widgets/PointerLocator.vala',
     'Widgets/SessionLocker.vala',