test(hw): add driver-probe-error check to dmesg verification

``assert_no_kernel_faults`` only catches hard kernel faults (panic,
oops, BUG, SError).  A driver can silently fail to probe — e.g. a
bad DT overlay apply, a phandle that doesn't resolve, a clock
provider that never appears — and the test reaches the IIO-device
assertion with a missing device, producing a confusing "not found"
message that obscures the real root cause.

Add ``assert_no_probe_errors(dmesg_txt)`` in ``hw_helpers.py`` that
scans for the canonical probe-failure signatures:
- ``probe of <dev> failed with error``  (classic ``device_probe``
  return-errno message)
- ``Error applying overlay`` / ``failed to apply overlay``  (DT
  overlay apply failure)
- ``Error resolving``  (phandle resolution failure)

Reuses ``_DMESG_BENIGN_SUBSTRINGS`` so known-benign probe chatter
(firmware loads, ``-EPROBE_DEFER`` retries, ZynqMP early-boot
warnings) doesn't fire.  ``-EPROBE_DEFER`` was already allowlisted;
this commit also adds the raw ``error -517`` string for callers
that log the numeric errno instead of the symbolic name.

Wire the new check in next to every existing
``assert_no_kernel_faults`` site:
- ``test_ad9081_zcu102_xsa_hw.py``
- ``test_ad9081_zcu102_system_hw.py``
- ``test_adrv9009_zcu102_hw.py``   (boot dmesg + post-profile-reload
  dmesg in the outer loop)
- ``test_adrv9371_zc706_hw.py``

Also restructure ``test_fmcdaq3_vcu118_hw.py`` so it actually
exercises the new check: switch the fixture from ``target`` to
``board`` (gets teardown power-off for free — VCU118 was previously
left powered on) and swap the bespoke ``grep -Ei fail|error``
snippet for ``collect_dmesg`` + the two assertions, matching the
four other hw tests.

Author: tfcollins

test/hw/hw_helpers.py

@@ -294,8 +294,11 @@ def require_hw_prereqs() -> None:
     "failed to load firmware",
     # Wifi/USB hotplug noise seen on some Kuiper releases:
     "cfg80211: failed to load",
-    # Harmless driver-level probe deferrals re-tried later:
+    # Harmless driver-level probe deferrals re-tried later.  The kernel
+    # surfaces these both symbolically (``-EPROBE_DEFER``) and as the
+    # raw errno (``-517``) depending on the caller.
     "EPROBE_DEFER",
+    "error -517",
     # ZynqMP early-boot WARNING: the kernel logs a Call trace through
     # gic_of_init / of_irq_init because the RPU-bus interrupt-controller
     # cannot be initialized from Linux on ZynqMP.  Always benign; the
@@ -346,6 +349,41 @@ def assert_no_kernel_faults(dmesg_txt: str) -> None:
     assert not bad, "Kernel fault(s) detected in dmesg:\n" + "\n".join(bad)
 
 
+# Driver-probe-failure patterns in dmesg.  These appear when a probe()
+# callback returns a negative errno other than -EPROBE_DEFER (the defer
+# path is the normal retry-until-resolved dance and is allowlisted via
+# _DMESG_BENIGN_SUBSTRINGS above).  Regex, not plain substrings —
+# ``probe of <dev> failed with error <N>`` is the canonical kernel
+# message.  Overlay-apply errors fall in the same bucket because a
+# failed overlay almost always cascades into silent probe misses.
+_DMESG_PROBE_ERROR_PATTERNS = (
+    r"probe of \S+ failed with error",
+    r"Error applying overlay",
+    r"failed to apply overlay",
+    r"Error resolving",
+)
+
+
+def assert_no_probe_errors(dmesg_txt: str) -> None:
+    """Fail the calling test if *dmesg_txt* contains driver-probe errors.
+
+    Complements :func:`assert_no_kernel_faults` — a driver can fail to
+    probe without ever producing a kernel fault (e.g. a DT overlay
+    apply error, a regulator not showing up, a phandle mismatch).
+    Reuses :data:`_DMESG_BENIGN_SUBSTRINGS` so known-benign probe
+    chatter (firmware loads, ``-EPROBE_DEFER`` retries, ZynqMP early-
+    boot warnings) does not fire.
+    """
+    compiled = [_re.compile(p) for p in _DMESG_PROBE_ERROR_PATTERNS]
+    bad: list[str] = []
+    for line in dmesg_txt.splitlines():
+        if any(s in line for s in _DMESG_BENIGN_SUBSTRINGS):
+            continue
+        if any(rx.search(line) for rx in compiled):
+            bad.append(line)
+    assert not bad, "Driver probe errors detected in dmesg:\n" + "\n".join(bad)
+
+
 def shell_out(shell, cmd: str) -> str:
     """Run *cmd* via an ``ADIShellDriver`` and return the output as a string.
 

test/hw/test_ad9081_zcu102_system_hw.py

@@ -46,6 +46,7 @@
     acquire_xsa,
     assert_jesd_links_data,
     assert_no_kernel_faults,
+    assert_no_probe_errors,
     collect_dmesg,
     compile_dts_to_dtb,
     deploy_and_boot,
@@ -266,6 +267,7 @@ def test_ad9081_zcu102_system_hw(board, built_kernel_image_zynqmp, tmp_path):
 
     # --- 9. Verify: kernel probe + IIO context + JESD DATA state ---
     assert_no_kernel_faults(dmesg_txt)
+    assert_no_probe_errors(dmesg_txt)
     assert "AD9081 Rev." in dmesg_txt or "probed ADC AD9081" in dmesg_txt, (
         "AD9081 probe signature was not found in kernel dmesg output"
     )

test/hw/test_ad9081_zcu102_xsa_hw.py

@@ -35,6 +35,7 @@
     acquire_xsa,
     assert_jesd_links_data,
     assert_no_kernel_faults,
+    assert_no_probe_errors,
     collect_dmesg,
     compile_dts_to_dtb,
     deploy_and_boot,
@@ -184,6 +185,7 @@ def test_ad9081_zcu102_xsa_hw(board, built_kernel_image_zynqmp, tmp_path):
 
     # --- 7. Verify: kernel probe + IIO context + JESD DATA state ---
     assert_no_kernel_faults(dmesg_txt)
+    assert_no_probe_errors(dmesg_txt)
     assert "AD9081 Rev." in dmesg_txt or "probed ADC AD9081" in dmesg_txt, (
         "AD9081 probe signature was not found in kernel dmesg output"
     )

test/hw/test_adrv9009_zcu102_hw.py

@@ -38,6 +38,7 @@
     acquire_xsa,
     assert_jesd_links_data,
     assert_no_kernel_faults,
+    assert_no_probe_errors,
     collect_dmesg,
     compile_dts_to_dtb,
     deploy_and_boot,
@@ -214,6 +215,7 @@ def test_adrv9009_zcu102_hw(board, built_kernel_image_zynqmp, tmp_path):
 
     # --- 7. Verify: kernel probe + IIO context + JESD DATA state ---
     assert_no_kernel_faults(dmesg_txt)
+    assert_no_probe_errors(dmesg_txt)
     assert "adrv9009-phy" in dmesg_txt or "Talise" in dmesg_txt, (
         "ADRV9009 phy probe signature was not found in kernel dmesg output"
     )
@@ -279,5 +281,7 @@ def test_adrv9009_zcu102_hw(board, built_kernel_image_zynqmp, tmp_path):
         # relock both links before re-reading sysfs status.
         time.sleep(3.0)
         assert_jesd_links_data(shell, context=f"after {filename}")
-        assert_no_kernel_faults(shell_out(shell, "dmesg"))
+        dmesg = shell_out(shell, "dmesg")
+        assert_no_kernel_faults(dmesg)
+        assert_no_probe_errors(dmesg)
         print(f"  {filename}: RX+TX JESD DATA OK")

test/hw/test_adrv9371_zc706_hw.py

@@ -36,6 +36,7 @@
     DEFAULT_OUT_DIR,
     acquire_xsa,
     assert_no_kernel_faults,
+    assert_no_probe_errors,
     collect_dmesg,
     compile_dts_to_dtb,
     deploy_and_boot,
@@ -137,6 +138,7 @@ def test_adrv9371_zc706_xsa_hw(board, built_kernel_image_zynq, tmp_path):
         grep_pattern="ad9371|ad9528|jesd204|mykonos|probe|failed|error",
     )
     assert_no_kernel_faults(dmesg_txt)
+    assert_no_probe_errors(dmesg_txt)
     assert "ad9371" in dmesg_txt.lower() or "mykonos" in dmesg_txt.lower(), (
         "AD9371 driver probe signature not found in dmesg"
     )

test/hw/test_fmcdaq3_vcu118_hw.py

@@ -33,34 +33,37 @@
         allow_module_level=True,
     )
 
+from test.hw.hw_helpers import (  # noqa: E402
+    DEFAULT_OUT_DIR,
+    assert_no_kernel_faults,
+    assert_no_probe_errors,
+    collect_dmesg,
+)
+
 
 @pytest.mark.lg_feature(["fmcdaq3", "vcu118"])
-def test_fmcdaq3_vcu118_boot_hw(target):
+def test_fmcdaq3_vcu118_boot_hw(board):
     """Boot FMCDAQ3+VCU118 with the prebuilt Kuiper image and verify IIO."""
-    shell = _boot_and_get_shell(target)
-    _assert_probed_drivers(shell)
-    _assert_iio_devices(shell)
+    out_dir = DEFAULT_OUT_DIR
+    out_dir.mkdir(parents=True, exist_ok=True)
 
+    board.transition("shell")
+    shell = board.target.get_driver("ADIShellDriver")
 
-def _boot_and_get_shell(target):
-    """Drive ``BootFabric`` through ``powered_off`` → ``shell`` and return shell."""
-    strategy = target.get_driver("Strategy")
-    strategy.transition("powered_off")
-    strategy.transition("shell")
-    return target.get_driver("ADIShellDriver")
+    dmesg_txt = collect_dmesg(
+        shell,
+        out_dir,
+        label="fmcdaq3_vcu118",
+        grep_pattern="ad9680|ad9152|ad9528|jesd204|probe|failed|error",
+    )
+    assert_no_kernel_faults(dmesg_txt)
+    assert_no_probe_errors(dmesg_txt)
 
+    lowered = dmesg_txt.lower()
+    assert "ad9680" in lowered, "AD9680 driver messages not seen in dmesg"
+    assert "ad9152" in lowered, "AD9152 driver messages not seen in dmesg"
 
-def _assert_probed_drivers(shell) -> None:
-    """Fail unless dmesg shows AD9680 / AD9152 / AD9528 / JESD driver probes."""
-    out = shell.run_check(
-        "dmesg | grep -Ei 'ad9680|ad9152|ad9528|jesd204|fail|error' | tail -n 200; true"
-    )
-    dmesg = "\n".join(out) if isinstance(out, list) else str(out)
-    print("\n=== FMCDAQ3 probe-relevant dmesg ===")
-    print(dmesg)
-    print("====================================")
-    assert "ad9680" in dmesg.lower(), "AD9680 driver messages not seen in dmesg"
-    assert "ad9152" in dmesg.lower(), "AD9152 driver messages not seen in dmesg"
+    _assert_iio_devices(shell)
 
 
 def _assert_iio_devices(shell) -> None: