Skip to content

Commit a200ed7

Browse files
CI: Troubleshooting AT-SPI2 bridge startup
1 parent f5b705f commit a200ed7

7 files changed

Lines changed: 245 additions & 24 deletions

File tree

.github/actions/setup/action.yml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,44 @@ runs:
138138
sudo update-alternatives --install /usr/bin/lld lld /usr/bin/lld-${{ inputs.llvm_version }} 100
139139
sudo update-alternatives --install /usr/bin/llvm-symbolizer llvm-symbolizer /usr/bin/llvm-symbolizer-${{ inputs.llvm_version }} 100
140140
141+
# Ubuntu 24.04 noble's stock qt6-base-dev is 6.4.2, but our AT-SPI2 bridge needs the
142+
# QAccessibleAttributesInterface that was added in Qt 6.8 — see UI/Qt/AccessibilityInterface.{h,cpp} (the impl is
143+
# gated on QT_VERSION >= QT_VERSION_CHECK(6, 8, 0)). Install Qt via the official Qt CI binaries (not from apt —
144+
# no Ubuntu/Kubuntu PPA currently ships Qt 6.8+ for noble) and let CMake's find_package(Qt6) prefer it via
145+
# Qt6_DIR. The pinned version matches what the AT-SPI2 tests have been verified against locally on Fedora 43
146+
# (qt6-qtbase 6.10.3). We keep qt6-base-dev installed via apt above so the runtime libs (libxcb, fontconfig, etc.)
147+
# Qt was built against are present; only the dev headers / CMake config files are shadowed by the action's install.
148+
- name: Install Qt 6.10.3
149+
if: ${{ inputs.os == 'Linux' && inputs.type == 'build' }}
150+
uses: jurplel/install-qt-action@v4
151+
with:
152+
version: '6.10.3'
153+
host: ${{ inputs.arch == 'arm64' && 'linux_arm64' || 'linux' }}
154+
target: 'desktop'
155+
arch: ${{ inputs.arch == 'arm64' && 'linux_gcc_arm64' || 'linux_gcc_64' }}
156+
cache: true
157+
cache-key-prefix: 'install-qt-action'
158+
159+
# Match the Orca version we run AT-SPI2 tests against locally on Fedora 43 (49.7). apt's noble package is 46.1,
160+
# which still uses the camelCase Python API (settings_manager.getManager); 47+ renamed it to snake_case
161+
# (settings_manager.get_manager), and our harness/orca.py expects the new name. apt already pulled in the C-level
162+
# runtime deps (python3-gi, gir1.2-atspi-2.0, python3-louis, python3-brlapi, speech-dispatcher, …), so we just
163+
# overlay the upstream Python module on top of apt's install. No new build-time deps required.
164+
- name: Install Orca 49.7 (overlay on apt's 46.1)
165+
if: ${{ inputs.os == 'Linux' && inputs.type == 'build' }}
166+
shell: bash
167+
run: |
168+
ORCA_VERSION="49.7"
169+
ORCA_SERIES="${ORCA_VERSION%%.*}"
170+
WORKDIR="$(mktemp -d)"
171+
cd "${WORKDIR}"
172+
curl -fsSL "https://download.gnome.org/sources/orca/${ORCA_SERIES}/orca-${ORCA_VERSION}.tar.xz" | tar xJ
173+
APT_ORCA_DIR="$(/usr/bin/python3 -c 'import orca, os; print(os.path.dirname(orca.__file__))')"
174+
echo "Replacing apt's Orca Python module at ${APT_ORCA_DIR} with upstream ${ORCA_VERSION}"
175+
sudo rm -rf "${APT_ORCA_DIR}"
176+
sudo cp -a "orca-${ORCA_VERSION}/src/orca" "${APT_ORCA_DIR}"
177+
/usr/bin/python3 -c 'import orca; from orca import settings_manager; print("Orca module path:", orca.__file__); assert hasattr(settings_manager, "get_manager"), "get_manager API not present after overlay"'
178+
141179
- name: Install Dependencies
142180
if: ${{ inputs.os == 'Linux' && inputs.type == 'lint' }}
143181
shell: bash

.github/workflows/ci.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,13 @@ jobs:
5858
clang_plugins: false
5959
runner_labels: '["blacksmith-16vcpu-ubuntu-2404"]'
6060

61+
- os_name: 'Linux'
62+
arch: 'x86_64'
63+
build_preset: 'Release'
64+
toolchain: 'Clang'
65+
clang_plugins: false
66+
runner_labels: '["blacksmith-16vcpu-ubuntu-2404"]'
67+
6168
- os_name: 'Windows'
6269
arch: 'x86_64'
6370
build_preset: 'Sanitizer'

.github/workflows/lagom-template.yml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,18 @@ jobs:
221221
env:
222222
TESTS_ONLY: 1
223223

224+
# The AT-SPI2 tests need a Linux build that is neither Sanitizer nor All_Debug:
225+
# - ASan-instrumented Qt-on-XCB hits a buffer overflow in libxkbcommon-x11.
226+
# - All_Debug's ENABLE_ALL_THE_DEBUG_MACROS log volume starves the Qt event loop, so AT-SPI2 registration never
227+
# completes within any reasonable wait window.
228+
# The Release runner has neither problem, so we run them there.
229+
- name: Test - LibWeb AT-SPI2 bridge (Release)
230+
if: ${{ inputs.os_name == 'Linux' && inputs.build_preset == 'Release' }}
231+
working-directory: ${{ github.workspace }}
232+
run: ctest --output-on-failure --test-dir Build -R LibWebAccessibilityBridge --timeout 1800
233+
env:
234+
TESTS_ONLY: 1
235+
224236
- name: Upload LibWeb Test Artifacts
225237
if: ${{ always() && inputs.build_preset != 'Fuzzers' && inputs.build_preset != 'All_Debug' }}
226238
uses: actions/upload-artifact@v7

Tests/LibWeb/AccessibilityBridge/CMakeLists.txt

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,17 @@ if (NOT LADYBIRD_GUI_FRAMEWORK STREQUAL "Qt")
1111
return()
1212
endif()
1313

14+
if (ENABLE_ADDRESS_SANITIZER)
15+
# For running these tests under CI:
16+
# 1. We need to actually launch a Qt-UI Ladybird process, and assert on the output from the Qt AT-SPI2 bridge.
17+
# 2. Qt's AT-SPI2 bridge only activates with a real platform plugin (offscreen doesn’t qualify).
18+
# 3. On a headless Linux runner in our CI, the only viable Qt platform plugin is xcb.
19+
# 4. Qt's xcb plugin pulls libxkbcommon-x11 in as a transitive runtime dependency.
20+
# But, under ASan, the QXcbConnection startup path triggers a buffer overflow in libxkbcommon-x11. Therefore, we
21+
# skip these tests under ASan — and for CI, we run them on the Release runner instead.
22+
return()
23+
endif()
24+
1425
find_package(Python3 REQUIRED)
1526

1627
# run_tests.sh reads LADYBIRD_BINARY to locate the Ladybird executable. Point it at the ladybird target — so tests don't

Tests/LibWeb/AccessibilityBridge/harness/ladybird.py

Lines changed: 73 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import os
66
import pathlib
77
import subprocess
8+
import tempfile
89
import time
910

1011
from contextlib import contextmanager
@@ -60,6 +61,7 @@ def __init__(self, url: str, binary: Optional[pathlib.Path] = None):
6061
self._binary = DEFAULT_BINARY
6162
self._process: Optional[subprocess.Popen] = None
6263
self._app: Optional[Atspi.Accessible] = None
64+
self._stderr_file: Optional[tempfile.NamedTemporaryFile] = None
6365

6466
def __enter__(self) -> "LadybirdContext":
6567
self.start()
@@ -80,12 +82,27 @@ def start(self) -> None:
8082
env.setdefault("QT_QPA_PLATFORM", "xcb")
8183
# Silence Qt's a11y logging noise — so failures are readable.
8284
env.setdefault("QT_LOGGING_RULES", "qt.accessibility.atspi=false")
83-
85+
# By default, Qt only activates its AT-SPI2 bridge when something signals that AT is "enabled" — usually a
86+
# desktop-environment toolkit setting (e.g. GNOME's org.gnome.desktop.interface.toolkit-accessibility, which
87+
# gets reflected onto the org.a11y.Status D-Bus property). On a headless CI runner there's no DE backend
88+
# setting that flag, so even with at-spi-bus-launcher and at-spi2-registryd both alive, Qt declines to attach.
89+
# This env var tells Qt to skip the IsEnabled check and bring the bridge up unconditionally.
90+
env.setdefault("QT_LINUX_ACCESSIBILITY_ALWAYS_ON", "1")
91+
92+
# Send stderr to a regular file rather than subprocess.PIPE: Ladybird debug builds (ENABLE_ALL_THE_DEBUG_MACROS)
93+
# produce huge stderr volumes (PROMISE_DEBUG, etc.). A pipe's kernel buffer is ~64 KB; once it fills, every
94+
# dbgln in WebContent blocks on write — which can wedge the engine before AT-SPI2 registration completes. A
95+
# regular file never backpressures the writer, and we still get the bytes for diagnostics on failure.
96+
self._stderr_file = tempfile.NamedTemporaryFile(prefix="ladybird-a11y-stderr-", suffix=".log", delete=False)
97+
98+
# --force-cpu-painting: skip the GPU/Vulkan path. Headless CI runners have no Vulkan ICD, and per
99+
# Services/WebContent/main.cpp the CPU backend is also what other test harnesses use ("the GPU backend is not
100+
# deterministic"). The Vulkan probe failure itself is non-fatal — we just don't want it cluttering diagnostics.
84101
self._process = subprocess.Popen(
85-
[str(self._binary), self._url],
102+
[str(self._binary), "--force-cpu-painting", self._url],
86103
env=env,
87104
stdout=subprocess.DEVNULL,
88-
stderr=subprocess.PIPE,
105+
stderr=self._stderr_file,
89106
)
90107

91108
# Start a fresh Atspi connection for this context. Atspi.init() is idempotent — subsequent tests share the
@@ -113,21 +130,54 @@ def _app_appeared() -> bool:
113130
self._app = app
114131
return True
115132

116-
if not wait_for(_app_appeared, timeout=15.0, description="Ladybird AT-SPI2 app to appear"):
117-
self.stop()
133+
startup_timeout = float(os.environ.get("LADYBIRD_AT_SPI2_STARTUP_TIMEOUT", "60"))
134+
if not wait_for(_app_appeared, timeout=startup_timeout, description="Ladybird AT-SPI2 app to appear"):
135+
# Snapshot diagnostics *before* stop() tears the process and stderr file down. The point of this branch is
136+
# to make the next CI run's failure say *why* the app didn't appear: did Ladybird die? Is something else
137+
# registered on the desktop? Did Ladybird register under a different name?
138+
expected_pid = self._process.pid if self._process is not None else None
139+
process_alive = self._process is not None and self._process.poll() is None
140+
process_returncode = self._process.returncode if self._process is not None else None
141+
desktop_children: list[str] = []
142+
try:
143+
desktop = Atspi.get_desktop(0)
144+
for i in range(desktop.get_child_count()):
145+
child = desktop.get_child_at_index(i)
146+
if child is None:
147+
desktop_children.append(f"[{i}] <None>")
148+
continue
149+
try:
150+
child_name = child.get_name() or "<unnamed>"
151+
except Exception as e:
152+
child_name = f"<get_name failed: {e}>"
153+
try:
154+
child_pid = child.get_process_id()
155+
except Exception:
156+
child_pid = None
157+
desktop_children.append(f"[{i}] name={child_name!r} pid={child_pid}")
158+
except Exception as e:
159+
desktop_children.append(f"<desktop enumeration failed: {e}>")
160+
118161
stderr = ""
119-
if self._process and self._process.stderr:
162+
if self._stderr_file is not None:
120163
try:
121-
stderr = self._process.stderr.read().decode(errors="replace")
164+
with open(self._stderr_file.name, "rb") as f:
165+
stderr = f.read().decode(errors="replace")
122166
except Exception:
123167
pass
168+
self.stop()
124169
raise RuntimeError(
125-
"Ladybird did not register on AT-SPI2 within 15s.\n"
126-
f"URL: {self._url}\nBinary: {self._binary}\nStderr: {stderr[-2000:]}"
170+
f"Ladybird did not register on AT-SPI2 within {startup_timeout:g}s.\n"
171+
f"URL: {self._url}\nBinary: {self._binary}\n"
172+
f"Expected PID: {expected_pid}, alive: {process_alive}, returncode: {process_returncode}\n"
173+
f"AT-SPI2 desktop children ({len(desktop_children)}):\n "
174+
+ "\n ".join(desktop_children)
175+
+ f"\nStderr: {stderr[-2000:]}"
127176
)
128177

129178
def stop(self) -> None:
130179
if self._process is None:
180+
self._cleanup_stderr_file()
131181
return
132182
try:
133183
self._process.terminate()
@@ -139,6 +189,20 @@ def stop(self) -> None:
139189
finally:
140190
self._process = None
141191
self._app = None
192+
self._cleanup_stderr_file()
193+
194+
def _cleanup_stderr_file(self) -> None:
195+
if self._stderr_file is None:
196+
return
197+
try:
198+
self._stderr_file.close()
199+
except Exception:
200+
pass
201+
try:
202+
os.unlink(self._stderr_file.name)
203+
except Exception:
204+
pass
205+
self._stderr_file = None
142206

143207
def _find_app_on_desktop(self) -> Optional[Atspi.Accessible]:
144208
expected_pid = self._process.pid if self._process is not None else None

Tests/LibWeb/AccessibilityBridge/run_tests.sh

Lines changed: 72 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,10 +131,50 @@ fi
131131
export DISPLAY="${XVFB_DISPLAY}"
132132
export QT_QPA_PLATFORM=xcb
133133

134+
# Ladybird's IPC sockets are bound under XDG_RUNTIME_DIR (Core::StandardPaths::runtime_directory). On a CI runner with
135+
# no login session, XDG_RUNTIME_DIR is unset and /run/user/<uid> doesn't exist — so the socket bind() fails with ENOENT
136+
# and Ladybird never reaches the AT-SPI2 registration we wait on. dbus-run-session would have the same problem for its
137+
# bus socket. Fix both by pointing XDG_RUNTIME_DIR at a per-run scratch directory we own.
138+
if [[ -z "${XDG_RUNTIME_DIR:-}" ]] || [[ ! -d "${XDG_RUNTIME_DIR}" ]]; then
139+
XDG_RUNTIME_DIR="$(mktemp -d -t "ladybird-a11y-runtime-XXXXXX")"
140+
chmod 0700 "${XDG_RUNTIME_DIR}"
141+
export XDG_RUNTIME_DIR
142+
LADYBIRD_RUNTIME_DIR_OWNED=1
143+
fi
144+
145+
# Outer EXIT trap: kill anything we own — Xvfb (if we started it), and clean up the runtime dir we created. We don't
146+
# exec into dbus-run-session below; that would wipe this trap and orphan Xvfb past the test's wall-clock budget.
147+
cleanup_outer() {
148+
local rc=$?
149+
if [[ -n "${XVFB_PID:-}" ]]; then
150+
kill "${XVFB_PID}" 2>/dev/null || true
151+
fi
152+
if [[ -n "${LADYBIRD_RUNTIME_DIR_OWNED:-}" && -n "${XDG_RUNTIME_DIR:-}" ]]; then
153+
rm -rf "${XDG_RUNTIME_DIR}" 2>/dev/null || true
154+
fi
155+
return $rc
156+
}
157+
trap cleanup_outer EXIT
158+
134159
# shellcheck disable=SC2016 # The single-quoted body is intentional — the
135160
# inner bash -c expands its own $SCRIPT_DIR / $at_spi_bus_launcher etc.
136-
exec dbus-run-session -- bash -c '
161+
dbus-run-session -- bash -c '
137162
set -euo pipefail
163+
164+
# Inner EXIT trap: SIGTERM the AT-SPI2 daemons we backgrounded. Without this, when python exits the daemons
165+
# remain alive (orphaned to dbus-run-session), nothing reaps them, and the runner hangs until ctest hits its
166+
# wall-clock TIMEOUT — which we observed as ~600 s of dead time per Layer in CI logs.
167+
AT_SPI_BUS_LAUNCHER_PID=""
168+
AT_SPI2_REGISTRYD_PID=""
169+
cleanup_inner() {
170+
local rc=$?
171+
for pid in "$AT_SPI2_REGISTRYD_PID" "$AT_SPI_BUS_LAUNCHER_PID"; do
172+
[[ -n "$pid" ]] && kill "$pid" 2>/dev/null || true
173+
done
174+
return $rc
175+
}
176+
trap cleanup_inner EXIT
177+
138178
at_spi_bus_launcher=""
139179
if command -v /usr/libexec/at-spi-bus-launcher >/dev/null 2>&1; then
140180
at_spi_bus_launcher=/usr/libexec/at-spi-bus-launcher
@@ -143,6 +183,7 @@ exec dbus-run-session -- bash -c '
143183
fi
144184
if [[ -n "$at_spi_bus_launcher" ]]; then
145185
"$at_spi_bus_launcher" --launch-immediately &
186+
AT_SPI_BUS_LAUNCHER_PID=$!
146187
for _ in $(seq 1 50); do
147188
if dbus-send --session --dest=org.freedesktop.DBus --print-reply=literal /org/freedesktop/DBus org.freedesktop.DBus.NameHasOwner string:org.a11y.Bus 2>/dev/null | grep -q "true"; then
148189
break
@@ -152,7 +193,36 @@ exec dbus-run-session -- bash -c '
152193
fi
153194
if [[ -x /usr/libexec/at-spi2-registryd ]]; then
154195
/usr/libexec/at-spi2-registryd &
196+
AT_SPI2_REGISTRYD_PID=$!
197+
fi
198+
199+
# Give the registry a moment to claim org.a11y.atspi.Registry on the a11y bus, then dump a one-shot diagnostic of
200+
# the stack state. If a CI run later fails with "AT-SPI2 desktop children (0)", this snapshot is what tells us
201+
# which piece is broken: at-spi-bus-launcher missing, registryd not running, a11y bus address unreachable, etc.
202+
sleep 0.5
203+
echo "=== AT-SPI2 stack diagnostic (one-shot) ==="
204+
echo "DBUS_SESSION_BUS_ADDRESS=${DBUS_SESSION_BUS_ADDRESS:-<unset>}"
205+
echo "DISPLAY=${DISPLAY:-<unset>}"
206+
echo "XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-<unset>}"
207+
echo "--- relevant processes ---"
208+
ps -eo pid,ppid,cmd | grep -Ei "(at-spi|atspi|dbus-daemon|dbus-launch|Xvfb)" | grep -v grep || echo " (none)"
209+
echo "--- org.a11y.Bus.GetAddress on session bus ---"
210+
if command -v gdbus >/dev/null 2>&1; then
211+
gdbus call --session --dest=org.a11y.Bus --object-path=/org/a11y/bus --method=org.a11y.Bus.GetAddress 2>&1 || echo " gdbus call failed"
212+
elif command -v dbus-send >/dev/null 2>&1; then
213+
dbus-send --session --print-reply --dest=org.a11y.Bus /org/a11y/bus org.a11y.Bus.GetAddress 2>&1 || echo " dbus-send call failed"
214+
else
215+
echo " (no gdbus or dbus-send on PATH)"
216+
fi
217+
echo "--- org.a11y.atspi.Registry presence ---"
218+
if command -v gdbus >/dev/null 2>&1; then
219+
# Note: this asks the *session* bus whether it knows the registry name. The registry actually lives on the a11y
220+
# bus (a separate D-Bus daemon). A session-bus false-result is expected; this is just a sanity print.
221+
gdbus call --session --dest=org.freedesktop.DBus --object-path=/org/freedesktop/DBus --method=org.freedesktop.DBus.NameHasOwner org.a11y.atspi.Registry 2>&1 || echo " gdbus name-has-owner failed"
155222
fi
223+
echo "=== end diagnostic ==="
224+
156225
cd "'"${SCRIPT_DIR}"'"
157-
exec '"${PYTHON_BIN}"' -m unittest discover '"${discover_args[*]}"' -v "$@"
226+
# Run python (no exec) so the EXIT trap above fires after unittest finishes and reaps the AT-SPI2 daemons.
227+
'"${PYTHON_BIN}"' -m unittest discover '"${discover_args[*]}"' -v "$@"
158228
' -- "${args[@]}"

0 commit comments

Comments
 (0)