fix: stage Chromium runtime libraries so they survive chroot shadowing

Author: Copilot

containers/agent/entrypoint.sh

@@ -1560,6 +1560,7 @@ run_chroot_command() {
   mount_host_cgroupfs
   check_chroot_prereqs
   copy_preload_libs
+  copy_browser_libs
   copy_agent_helper_scripts
   copy_dind_runner_binary
   ensure_usr_local_bin_shims
@@ -1625,8 +1626,8 @@ run_chroot_command() {
     CLEANUP_CMD="${CLEANUP_CMD}; sed -i '/^[0-9.]\\+[[:space:]]\\+host\\.docker\\.internal\$/d' /etc/hosts 2>/dev/null || true"
     echo "[entrypoint] host.docker.internal will be removed from /etc/hosts on exit"
   fi
-  # Clean up /run/awf-lib if anything was copied (one-shot-token, CA cert, key helper)
-  if [ -n "${ONE_SHOT_TOKEN_LIB}" ] || [ -n "${AWF_CA_CHROOT}" ] || [ -n "${SYSTEM_CA_CHROOT}" ] || [ -n "${CHROOT_KEY_HELPER}" ] || [ -n "${STAGED_RUNNER_BINARY_CHROOT}" ]; then
+  # Clean up /run/awf-lib if anything was copied (one-shot-token, CA cert, key helper, browser libs)
+  if [ -n "${ONE_SHOT_TOKEN_LIB}" ] || [ -n "${AWF_CA_CHROOT}" ] || [ -n "${SYSTEM_CA_CHROOT}" ] || [ -n "${CHROOT_KEY_HELPER}" ] || [ -n "${STAGED_RUNNER_BINARY_CHROOT}" ] || [ -n "${BROWSER_LD_LIBRARY_PATH}" ]; then
     CLEANUP_CMD="${CLEANUP_CMD}; rm -rf /run/awf-lib 2>/dev/null || true"
   fi
   # NOTE: the /usr/local/bin overlay is torn down by cleanup_usr_local_bin_overlay(),
@@ -1682,10 +1683,18 @@ run_chroot_command() {
     LD_PRELOAD_CMD="export LD_PRELOAD=${ONE_SHOT_TOKEN_LIB};"
   fi
 
+  # Build LD_LIBRARY_PATH command so Chromium/Playwright can resolve the
+  # staged browser runtime libraries (see copy_browser_libs above).
+  LD_LIBRARY_PATH_CMD=""
+  if [ -n "${BROWSER_LD_LIBRARY_PATH}" ]; then
+    LD_LIBRARY_PATH_CMD="export LD_LIBRARY_PATH=${BROWSER_LD_LIBRARY_PATH}\${LD_LIBRARY_PATH:+:\$LD_LIBRARY_PATH};"
+  fi
+
   run_command_with_stdout run_agent_with_token_protection chroot /host /bin/bash -c "
     cd '${CHROOT_WORKDIR}' 2>/dev/null || cd / 2>/dev/null || true
     trap '${CLEANUP_CMD}' EXIT
     ${LD_PRELOAD_CMD}
+    ${LD_LIBRARY_PATH_CMD}
     exec capsh --drop=${CAPS_TO_DROP} ${CAPSH_IDENTITY_ARGS} -- -c 'exec ${SCRIPT_FILE}'
   "
 }

docs-site/src/content/docs/reference/agent-images.md

@@ -79,7 +79,7 @@ The default `agent` image (based on Ubuntu 22.04) includes the following pre-ins
 | Chromium runtime libraries | — | see below | Shared libraries required by Playwright-managed browsers |
 
 :::note[Chromium/Playwright runtime libraries]
-Because the agent uses selective bind mounts instead of a full host filesystem mount, browsers downloaded by Playwright cannot rely on host libraries. The image therefore preinstalls `libasound2`, `libatk-bridge2.0-0`, `libatk1.0-0`, `libatspi2.0-0`, `libcairo2`, `libcups2`, `libdbus-1-3`, `libdrm2`, `libexpat1`, `libgbm1`, `libglib2.0-0`, `libnspr4`, `libnss3`, `libpango-1.0-0`, `libpangocairo-1.0-0`, `libx11-6`, `libxcb1`, `libxcomposite1`, `libxdamage1`, `libxext6`, `libxfixes3`, `libxkbcommon0`, `libxrandr2`, `libxrender1`, `libxshmfence1`, and `fonts-liberation` (plus their transitive dependencies). On Ubuntu 24.04 base images the `t64` variants of these packages are installed automatically.
+Because the agent uses selective bind mounts instead of a full host filesystem mount, browsers downloaded by Playwright cannot rely on host libraries. The image therefore preinstalls `libasound2`, `libatk-bridge2.0-0`, `libatk1.0-0`, `libatspi2.0-0`, `libcairo2`, `libcups2`, `libdbus-1-3`, `libdrm2`, `libexpat1`, `libgbm1`, `libglib2.0-0`, `libnspr4`, `libnss3`, `libpango-1.0-0`, `libpangocairo-1.0-0`, `libx11-6`, `libxcb1`, `libxcomposite1`, `libxdamage1`, `libxext6`, `libxfixes3`, `libxkbcommon0`, `libxrandr2`, `libxrender1`, `libxshmfence1`, and `fonts-liberation` (plus their transitive dependencies). On Ubuntu 24.04 base images the `t64` variants of these packages are installed automatically. In chroot mode (the default), these image-installed libraries would otherwise be shadowed by the host's own `/usr`, `/lib`, etc.; `entrypoint.sh` stages copies of the files under `/run/awf-lib/browser-libs` (on the container's own writable rootfs, unaffected by the chroot bind mounts) and sets `LD_LIBRARY_PATH` so Chromium resolves them there.
 :::
 
 :::caution[Docker CLI Stub]

docs/agent-images.md

@@ -73,7 +73,7 @@ The `default` preset image (based on Ubuntu 22.04) includes the following pre-in
 | ca-certificates | — | `ca-certificates` | Trusted root certificates |
 | Chromium runtime libraries | — | see below | Shared libraries required by Playwright-managed browsers |
 
-**Chromium/Playwright runtime libraries:** Because the agent uses selective bind mounts instead of a full host filesystem mount, browsers downloaded by Playwright cannot rely on host libraries. The image therefore preinstalls `libasound2`, `libatk-bridge2.0-0`, `libatk1.0-0`, `libatspi2.0-0`, `libcairo2`, `libcups2`, `libdbus-1-3`, `libdrm2`, `libexpat1`, `libgbm1`, `libglib2.0-0`, `libnspr4`, `libnss3`, `libpango-1.0-0`, `libpangocairo-1.0-0`, `libx11-6`, `libxcb1`, `libxcomposite1`, `libxdamage1`, `libxext6`, `libxfixes3`, `libxkbcommon0`, `libxrandr2`, `libxrender1`, `libxshmfence1`, and `fonts-liberation` (plus their transitive dependencies). On Ubuntu 24.04 base images the `t64` variants of these packages are installed automatically.
+**Chromium/Playwright runtime libraries:** Because the agent uses selective bind mounts instead of a full host filesystem mount, browsers downloaded by Playwright cannot rely on host libraries. The image therefore preinstalls `libasound2`, `libatk-bridge2.0-0`, `libatk1.0-0`, `libatspi2.0-0`, `libcairo2`, `libcups2`, `libdbus-1-3`, `libdrm2`, `libexpat1`, `libgbm1`, `libglib2.0-0`, `libnspr4`, `libnss3`, `libpango-1.0-0`, `libpangocairo-1.0-0`, `libx11-6`, `libxcb1`, `libxcomposite1`, `libxdamage1`, `libxext6`, `libxfixes3`, `libxkbcommon0`, `libxrandr2`, `libxrender1`, `libxshmfence1`, and `fonts-liberation` (plus their transitive dependencies). On Ubuntu 24.04 base images the `t64` variants of these packages are installed automatically. In chroot mode (the default), these image-installed libraries would otherwise be shadowed by the host's own `/usr`, `/lib`, etc.; `entrypoint.sh` stages copies of the files under `/run/awf-lib/browser-libs` (on the container's own writable rootfs, unaffected by the chroot bind mounts) and sets `LD_LIBRARY_PATH` so Chromium resolves them there.
 
 **⚠️ Docker CLI Stub:** The `docker` command is present but is a stub—there is no Docker daemon running inside the container. Docker-in-Docker is not supported. Use `--mount` to access Docker sockets from the host if needed.
 

docs/troubleshooting.md

@@ -377,7 +377,7 @@ Host system is missing dependencies to run browsers.
 
 **Cause:** The agent container uses selective bind mounts rather than a full host filesystem mount, so browsers downloaded by Playwright cannot pick up native libraries installed on the host.
 
-**Solution:** The agent image preinstalls Chromium's native runtime libraries (`libnspr4`, `libnss3`, `libatk1.0-0`, `libatk-bridge2.0-0`, `libatspi2.0-0`, `libcups2`, `libdrm2`, `libgbm1`, `libpango-1.0-0`, `libxkbcommon0`, `libxcomposite1`, `libxdamage1`, `libxrandr2`, `libasound2`, `fonts-liberation`, and related packages). Use a current agent image (`--image-tag latest`, or `--build-local` when building from source) and the browser will launch.
+**Solution:** The agent image preinstalls Chromium's native runtime libraries (`libnspr4`, `libnss3`, `libatk1.0-0`, `libatk-bridge2.0-0`, `libatspi2.0-0`, `libcups2`, `libdrm2`, `libgbm1`, `libpango-1.0-0`, `libxkbcommon0`, `libxcomposite1`, `libxdamage1`, `libxrandr2`, `libasound2`, `fonts-liberation`, and related packages). In chroot mode (the default), `entrypoint.sh` stages copies of these libraries under `/run/awf-lib/browser-libs` — on the container's own writable rootfs, so they are not shadowed by the host `/usr`/`/lib` bind mounts — and sets `LD_LIBRARY_PATH` so Chromium resolves them there. Use a current agent image (`--image-tag latest`, or `--build-local` when building from source) and the browser will launch.
 
 Verify the libraries are present inside the sandbox:
 

scripts/ci/agent-dockerfile-browser-deps.test.ts

@@ -37,10 +37,10 @@ apt-cache() {
   done
   return 100
 }
-PKGS="iptables"
+BROWSER_ACTUAL_PKGS=""
 BROWSER_PKGS="${readBrowserPackages().join(' ')}"
 ${readResolutionScript()}
-echo "$PKGS"
+echo "$BROWSER_ACTUAL_PKGS"
 `;
 
   const result = spawnSync('bash', ['-c', script], { encoding: 'utf-8' });
@@ -85,7 +85,7 @@ describe('agent Dockerfile Chromium runtime dependencies', () => {
   it('keeps the 22.04 names when the base image has no t64 packages', () => {
     const resolved = runResolution([]);
 
-    expect(resolved.split(/\s+/)).toEqual(['iptables', ...readBrowserPackages()]);
+    expect(resolved.split(/\s+/).filter(Boolean)).toEqual(readBrowserPackages());
   });
 
   it('prefers t64 packages when the base image provides them', () => {
@@ -98,7 +98,14 @@ describe('agent Dockerfile Chromium runtime dependencies', () => {
     expect(resolved).toContain('libnss3');
   });
 
-  it('installs the resolved browser packages with the other agent packages', () => {
+  it('installs the resolved browser packages separately so their transitive deps can be diffed', () => {
     expect(readDockerfile()).toMatch(/apt_install_retry \$PKGS/);
+    expect(readDockerfile()).toMatch(/apt_install_retry \$BROWSER_ACTUAL_PKGS/);
+  });
+
+  it('records the installed .so files into a manifest entrypoint.sh stages for the chroot', () => {
+    const dockerfile = readDockerfile();
+    expect(dockerfile).toMatch(/\/usr\/local\/share\/awf\/browser-libs\.manifest/);
+    expect(dockerfile).toMatch(/dpkg -L \$\(cat \/tmp\/browser-pkgs-installed\.txt\)/);
   });
 });

tests/entrypoint-phase-functions.test.sh

@@ -30,6 +30,7 @@ required_functions=(
   mount_host_procfs
   mount_host_cgroupfs
   copy_preload_libs
+  copy_browser_libs
   copy_agent_helper_scripts
   copy_dind_runner_binary
   resolve_chroot_binary_path
@@ -122,6 +123,7 @@ chroot_helpers=(
   'mount_host_cgroupfs'
   'check_chroot_prereqs'
   'copy_preload_libs'
+  'copy_browser_libs'
   'copy_agent_helper_scripts'
   'copy_dind_runner_binary'
   'ensure_usr_local_bin_shims'
@@ -222,6 +224,81 @@ else
   fail "run_chroot_command() does not clean up copied system CA bundles"
 fi
 
+# copy_browser_libs() must stage the Chromium/Playwright runtime libraries
+# recorded in the Dockerfile's manifest (containers/agent/Dockerfile:
+# /usr/local/share/awf/browser-libs.manifest) under /run/awf-lib/browser-libs
+# -- on the container's own writable rootfs, not the /host bind mount -- and
+# export their directories via BROWSER_LD_LIBRARY_PATH. This directly
+# exercises the staging mechanism that makes the packages installed by
+# BROWSER_PKGS visible inside the chroot, where buildSystemMounts() otherwise
+# shadows the image's /usr, /lib, etc. with the runner's own.
+run_copy_browser_libs_fixture() {
+  local tmp_dir
+  tmp_dir="$(mktemp -d)"
+  local host_root="${tmp_dir}/host-root"
+  local fixture_entrypoint="${tmp_dir}/entrypoint-fixture.sh"
+  local fake_lib_dir="${tmp_dir}/image-rootfs/usr/lib/x86_64-linux-gnu"
+  local fake_lib="${fake_lib_dir}/libnspr4.so"
+  local manifest="${tmp_dir}/browser-libs.manifest"
+
+  mkdir -p "${host_root}" "${fake_lib_dir}"
+  printf 'real-libnspr4-bytes' > "${fake_lib}"
+  printf '%s\n' "${fake_lib}" > "${manifest}"
+
+  awk '$0 != "main \"$@\""' "${ENTRYPOINT}" > "${fixture_entrypoint}"
+  sed -i "s#/host#\${AWF_TEST_HOST_ROOT}#g" "${fixture_entrypoint}"
+  sed -i "s#/usr/local/share/awf/browser-libs.manifest#\${AWF_TEST_MANIFEST}#g" "${fixture_entrypoint}"
+
+  (
+    set -e
+    # shellcheck disable=SC1090
+    . "${fixture_entrypoint}"
+
+    AWF_TEST_HOST_ROOT="${host_root}"
+    AWF_TEST_MANIFEST="${manifest}"
+
+    unset BROWSER_LD_LIBRARY_PATH
+    copy_browser_libs
+
+    staged_lib="${host_root}/run/awf-lib/browser-libs${fake_lib}"
+    [ -f "${staged_lib}" ]
+    [ "$(cat "${staged_lib}")" = "real-libnspr4-bytes" ]
+    [ "${BROWSER_LD_LIBRARY_PATH}" = "/run/awf-lib/browser-libs${fake_lib_dir}" ]
+
+    # Simulate buildSystemMounts() shadowing the image's own /usr/lib with the
+    # runner's: overwrite the original library path with different content
+    # (as if the host's own libnspr4.so.0 had been bind-mounted over it).
+    printf 'shadowed-by-host-bind-mount' > "${fake_lib}"
+
+    # The staged copy under /run/awf-lib must be unaffected by the shadowing,
+    # proving Chromium resolves the image-provided library via
+    # LD_LIBRARY_PATH rather than being silently broken by the chroot bind mount.
+    [ "$(cat "${staged_lib}")" = "real-libnspr4-bytes" ]
+  )
+  local result=$?
+  rm -rf "${tmp_dir}"
+  return "${result}"
+}
+
+if run_copy_browser_libs_fixture; then
+  pass "copy_browser_libs() stages manifest libraries under /run/awf-lib and survives /usr shadowing"
+else
+  fail "copy_browser_libs() does not stage Chromium/Playwright runtime libraries correctly"
+fi
+
+if grep -Eq '\[ -n "\$\{BROWSER_LD_LIBRARY_PATH\}" \]' "${ENTRYPOINT}"; then
+  pass "run_chroot_command() cleans up staged browser runtime libraries"
+else
+  fail "run_chroot_command() does not clean up staged browser runtime libraries"
+fi
+
+if grep -Fq 'LD_LIBRARY_PATH_CMD' "${ENTRYPOINT}" && \
+   grep -Fq 'export LD_LIBRARY_PATH=${BROWSER_LD_LIBRARY_PATH}' "${ENTRYPOINT}"; then
+  pass "run_chroot_command() exports LD_LIBRARY_PATH from BROWSER_LD_LIBRARY_PATH before exec"
+else
+  fail "run_chroot_command() does not export LD_LIBRARY_PATH from BROWSER_LD_LIBRARY_PATH"
+fi
+
 # configure_jvm_proxy must not abort the entrypoint (set -e) when $HOME is
 # read-only, including when .m2/.gradle already exist but cannot be written.
 run_configure_jvm_proxy_readonly_home_fixture() {