fix(linux): fire hotkey regardless of NumLock/CapsLock state (#39)

An X11 XGrabKey matches an exact modifier mask, but an active NumLock
(Mod2) or CapsLock (Lock) adds bits to the event state, so the grab no
longer matches and the hotkey silently stops firing (issue #25).

Grab the key once for every on/off combination of the NumLock and
CapsLock masks (deduplicated, so we never grab the same key+mask twice
and trigger BadAccess). The mask set is computed in Go by lockVariants
and unit-tested.

Fixes #25.

Author: changkun

hotkey_linux.c

@@ -47,7 +47,11 @@ int displayTest() {
 
 // waitHotkey blocks until the hotkey is triggered.
 // this function crashes the program if the hotkey already grabbed by others.
-int waitHotkey(uintptr_t hkhandle, unsigned int mod, int key) {
+//
+// mods points to nmods modifier masks: the same hotkey is grabbed once per
+// mask so that it still fires while NumLock/CapsLock are active (those locks
+// add bits to the event state that an exact-mask grab would not match).
+int waitHotkey(uintptr_t hkhandle, unsigned int* mods, int nmods, int key) {
 	Display* d = NULL;
 	for (int i = 0; i < 42; i++) {
 		d = XOpenDisplay(0);
@@ -58,7 +62,9 @@ int waitHotkey(uintptr_t hkhandle, unsigned int mod, int key) {
 		return -1;
 	}
 	int keycode = XKeysymToKeycode(d, key);
-	XGrabKey(d, keycode, mod, DefaultRootWindow(d), False, GrabModeAsync, GrabModeAsync);
+	for (int i = 0; i < nmods; i++) {
+		XGrabKey(d, keycode, mods[i], DefaultRootWindow(d), False, GrabModeAsync, GrabModeAsync);
+	}
 	XSelectInput(d, DefaultRootWindow(d), KeyPressMask);
 	XEvent ev;
 	while(1) {
@@ -69,7 +75,9 @@ int waitHotkey(uintptr_t hkhandle, unsigned int mod, int key) {
 			continue;
 		case KeyRelease:
 			hotkeyUp(hkhandle);
-			XUngrabKey(d, keycode, mod, DefaultRootWindow(d));
+			for (int i = 0; i < nmods; i++) {
+				XUngrabKey(d, keycode, mods[i], DefaultRootWindow(d));
+			}
 			XCloseDisplay(d);
 			return 0;
 		}

hotkey_linux.go

@@ -14,7 +14,7 @@ package hotkey
 #include <stdint.h>
 
 int displayTest();
-int waitHotkey(uintptr_t hkhandle, unsigned int mod, int key);
+int waitHotkey(uintptr_t hkhandle, unsigned int* mods, int nmods, int key);
 */
 import "C"
 import (
@@ -95,15 +95,48 @@ func (hk *Hotkey) handle() {
 	h := cgo.NewHandle(hk)
 	defer h.Delete()
 
+	// Grab the hotkey once per NumLock/CapsLock state so it fires
+	// regardless of those locks (see lockVariants).
+	variants := lockVariants(mod)
+	cmods := make([]C.uint, len(variants))
+	for i, v := range variants {
+		cmods[i] = C.uint(v)
+	}
+
 	for {
 		select {
 		case <-hk.ctx.Done():
 			close(hk.canceled)
 			return
 		default:
-			_ = C.waitHotkey(C.uintptr_t(h), C.uint(mod), C.int(hk.key))
+			_ = C.waitHotkey(C.uintptr_t(h), &cmods[0], C.int(len(cmods)), C.int(hk.key))
+		}
+	}
+}
+
+// X11 lock modifier masks (see /usr/include/X11/X.h).
+const (
+	x11LockMask Modifier = 1 << 1 // CapsLock
+	x11Mod2Mask Modifier = 1 << 4 // usually NumLock
+)
+
+// lockVariants returns mod combined with every on/off combination of the
+// CapsLock and NumLock masks, deduplicated. An XGrabKey uses an exact
+// modifier mask, so without these variants a hotkey would stop firing
+// whenever NumLock or CapsLock is toggled on. Duplicates are removed so we
+// never grab the same key+mask twice (which itself raises BadAccess).
+func lockVariants(mod Modifier) []uint32 {
+	extra := []Modifier{0, x11LockMask, x11Mod2Mask, x11LockMask | x11Mod2Mask}
+	seen := make(map[uint32]bool, len(extra))
+	out := make([]uint32, 0, len(extra))
+	for _, e := range extra {
+		v := uint32(mod | e)
+		if !seen[v] {
+			seen[v] = true
+			out = append(out, v)
 		}
 	}
+	return out
 }
 
 //export hotkeyDown

hotkey_linux_internal_test.go

@@ -0,0 +1,44 @@
+// Copyright 2021 The golang.design Initiative Authors.
+// All rights reserved. Use of this source code is governed
+// by a MIT license that can be found in the LICENSE file.
+//
+// Written by Changkun Ou <changkun.de>
+
+//go:build linux && cgo
+
+package hotkey
+
+import (
+	"reflect"
+	"testing"
+)
+
+// TestLockVariants verifies that a hotkey is grabbed for every NumLock/CapsLock
+// state. Before this fix only the exact modifier mask was grabbed, so a hotkey
+// stopped firing whenever NumLock or CapsLock was toggled on (issue #25).
+func TestLockVariants(t *testing.T) {
+	const (
+		lock = 1 << 1 // CapsLock
+		num  = 1 << 4 // NumLock
+	)
+	for _, tt := range []struct {
+		name string
+		mod  Modifier
+		want []uint32
+	}{
+		{
+			name: "ctrl+shift grabs all four lock combinations",
+			mod:  ModCtrl | ModShift, // 0b101 = 5
+			want: []uint32{5, 5 | lock, 5 | num, 5 | lock | num},
+		},
+		{
+			name: "mod already containing NumLock is deduplicated",
+			mod:  ModCtrl | Mod2, // Mod2 == NumLock mask
+			want: []uint32{uint32(ModCtrl | Mod2), uint32(ModCtrl|Mod2) | lock},
+		},
+	} {
+		if got := lockVariants(tt.mod); !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("%s: lockVariants(%#b) = %v, want %v", tt.name, tt.mod, got, tt.want)
+		}
+	}
+}