test(hw): driver-probe-error check + rx-capture data-flow assertion

``assert_no_kernel_faults`` only catches panic/oops/BUG/SError —
drivers can fail to probe silently (bad DT overlay apply, phandle
mismatch, missing regulator) and the suite would reach the
IIO-device assertion with a confusing "not found" message instead
of the probe-error root cause.  Similarly, ``Link status: DATA``
only says JESD trained at link-up — DMA / TPL / clock-path can
silently stop delivering samples and the test wouldn't notice.

Add two helpers in ``test/hw/hw_helpers.py`` and wire them into
every hw test that has IIO verification:

1. ``assert_no_probe_errors(dmesg_txt)`` — scans for
   ``probe of <dev> failed with error``,
   ``Error applying overlay`` / ``failed to apply overlay``,
   ``Error resolving``.  Reuses ``_DMESG_BENIGN_SUBSTRINGS`` (now
   also allowlisting the stock-Kuiper ZCU102 / ZynqMP watchdog,
   DisplayPort, and Ceva AHCI probes that fire on every boot
   regardless of overlay).

2. ``assert_rx_capture_valid(ctx, device_candidates, …)`` — uses
   raw libiio (works with any buffered AXI ADC regardless of
   whether a pyadi-iio wrapper exists for its device name),
   enables every non-output scan channel, refills a one-shot
   buffer, and asserts at least one channel is non-zero + at
   least one channel's ``|std|`` >= 1 LSB.  Device selection
   tries an ordered candidate list, then falls back to any
   ``axi-*`` / ``cf-*`` / TPL frontend on the context.
   ``TimeoutError`` from the refill path gets remapped to a
   clear ``AssertionError`` pointing at the stalled DMA.

Wired into:

- ``test_ad9081_zcu102_xsa_hw.py``       → ``axi-ad9081-rx-hpc`` /
  ``ad_ip_jesd204_tpl_adc``.
- ``test_ad9081_zcu102_system_hw.py``    → same.
- ``test_adrv9009_zcu102_hw.py``         → ``axi-adrv9009-rx-hpc``.
  Also restructured to use ``board`` fixture (from the
  ``target``-swap fix) so the VCU118-style teardown power-off
  runs.
- ``test_fmcdaq3_vcu118_hw.py``          → ``axi-ad9680-core-lpc``
  / ``axi-ad9680-hpc``.

Four findings surfaced via this check, all fixed:

- ``assert_no_probe_errors`` tripped on stock Kuiper ZynqMP boot
  noise (``cdns-wdt: probe of ffcb0000.watchdog failed with
  error -2``, DisplayPort + Ceva AHCI) → three specific
  device-node addresses added to the benign list so a real
  watchdog/display/sata regression elsewhere still trips.
- ``adi.ad9081(uri=…)`` fails with ``'NoneType' object has no
  attribute 'channels'`` when the design exposes the TPL core
  rather than ``axi-ad9081-rx-hpc`` → moved to raw libiio.
- Fallback initially picked the control-plane device
  (``ad9528-1``) which isn't AXI-DMA-backed → narrowed fallback
  to ``axi-*`` / ``cf-*`` / TPL only.

Author: tfcollins

test/hw/hw_helpers.py

@@ -294,8 +294,11 @@ def require_hw_prereqs() -> None:
     "failed to load firmware",
     # Wifi/USB hotplug noise seen on some Kuiper releases:
     "cfg80211: failed to load",
-    # Harmless driver-level probe deferrals re-tried later:
+    # Harmless driver-level probe deferrals re-tried later.  The kernel
+    # surfaces these both symbolically (``-EPROBE_DEFER``) and as the
+    # raw errno (``-517``) depending on the caller.
     "EPROBE_DEFER",
+    "error -517",
     # ZynqMP early-boot WARNING: the kernel logs a Call trace through
     # gic_of_init / of_irq_init because the RPU-bus interrupt-controller
     # cannot be initialized from Linux on ZynqMP.  Always benign; the
@@ -305,6 +308,14 @@ def require_hw_prereqs() -> None:
     "irqchip_init",
     "__primary_switched",
     "rpu-bus/interrupt-controller",
+    # Stock Kuiper ZynqMP (ZCU102) probes these SoC peripherals from the
+    # base DTS regardless of the overlay we merge in — the hardware is
+    # either unconfigured (no DisplayPort monitor attached) or not wired
+    # out on the board (no SATA).  Match by device-node address so a
+    # genuine regression on the same driver elsewhere still trips.
+    "ffcb0000.watchdog",  # Cadence WDT — unroutable clocks
+    "fd4a0000.display",   # ZynqMP DisplayPort — no monitor + DPMS pipe
+    "fd0c0000.ahci",      # Ceva AHCI/SATA — not routed on ZCU102
 )
 
 # Hard-fail patterns — these indicate a genuine kernel fault.
@@ -346,6 +357,41 @@ def assert_no_kernel_faults(dmesg_txt: str) -> None:
     assert not bad, "Kernel fault(s) detected in dmesg:\n" + "\n".join(bad)
 
 
+# Driver-probe-failure patterns in dmesg.  These appear when a probe()
+# callback returns a negative errno other than -EPROBE_DEFER (the defer
+# path is the normal retry-until-resolved dance and is allowlisted via
+# _DMESG_BENIGN_SUBSTRINGS above).  Regex, not plain substrings —
+# ``probe of <dev> failed with error <N>`` is the canonical kernel
+# message.  Overlay-apply errors fall in the same bucket because a
+# failed overlay almost always cascades into silent probe misses.
+_DMESG_PROBE_ERROR_PATTERNS = (
+    r"probe of \S+ failed with error",
+    r"Error applying overlay",
+    r"failed to apply overlay",
+    r"Error resolving",
+)
+
+
+def assert_no_probe_errors(dmesg_txt: str) -> None:
+    """Fail the calling test if *dmesg_txt* contains driver-probe errors.
+
+    Complements :func:`assert_no_kernel_faults` — a driver can fail to
+    probe without ever producing a kernel fault (e.g. a DT overlay
+    apply error, a regulator not showing up, a phandle mismatch).
+    Reuses :data:`_DMESG_BENIGN_SUBSTRINGS` so known-benign probe
+    chatter (firmware loads, ``-EPROBE_DEFER`` retries, ZynqMP early-
+    boot warnings) does not fire.
+    """
+    compiled = [_re.compile(p) for p in _DMESG_PROBE_ERROR_PATTERNS]
+    bad: list[str] = []
+    for line in dmesg_txt.splitlines():
+        if any(s in line for s in _DMESG_BENIGN_SUBSTRINGS):
+            continue
+        if any(rx.search(line) for rx in compiled):
+            bad.append(line)
+    assert not bad, "Driver probe errors detected in dmesg:\n" + "\n".join(bad)
+
+
 def shell_out(shell, cmd: str) -> str:
     """Run *cmd* via an ``ADIShellDriver`` and return the output as a string.
 
@@ -475,6 +521,133 @@ def assert_jesd_links_data(
     return rx_status, tx_status
 
 
+def assert_rx_capture_valid(
+    ctx,
+    device_candidates: str | tuple[str, ...],
+    n_samples: int = 2**12,
+    min_std: float = 1.0,
+    context: str = "",
+) -> dict:
+    """Capture ``n_samples`` from an IIO device and verify data is flowing.
+
+    Covers the "IIO device probed but no samples actually arrive" failure
+    mode: the JESD204 link reports DATA, drivers probe cleanly, IIO
+    devices appear, but the DMA / JESD transport / clock path silently
+    stops delivering samples.  The buffer comes back, but every sample
+    is zero, or every sample is latched to one value.
+
+    Asserts:
+
+    - At least one RX channel is not all-zero (DMA actually transferred
+      bytes).
+    - At least one RX channel's |std| is ``>= min_std`` LSBs (samples
+      actually vary — noise floor alone clears this threshold easily,
+      but a latched converter does not).
+
+    Uses raw libiio so it works with any buffered IIO device, including
+    AD9081 designs that expose the buffered frontend as the TPL core
+    (``ad_ip_jesd204_tpl_adc``) rather than ``axi-ad9081-rx-hpc``.
+
+    Args:
+        ctx: A live ``iio.Context`` (e.g. from :func:`open_iio_context`).
+        device_candidates: IIO device name to capture from, or a tuple
+            of candidate names — the first one present on *ctx* wins.
+        n_samples: buffer depth for the capture.
+        min_std: minimum |std| across all channels, in raw-LSB units.
+        context: tag prepended to assertion-failure messages.
+
+    Returns:
+        ``dict`` mapping channel id → captured ``numpy.ndarray``.
+    """
+    import iio
+    import numpy as np
+
+    suffix = f" ({context})" if context else ""
+    candidates = (
+        (device_candidates,) if isinstance(device_candidates, str) else tuple(device_candidates)
+    )
+    all_names = sorted(d.name for d in ctx.devices if d.name)
+
+    def _has_rx_scan(d):
+        return any(c.scan_element and not c.output for c in d.channels)
+
+    dev = next((d for d in (ctx.find_device(n) for n in candidates) if d is not None), None)
+    if dev is None or not _has_rx_scan(dev):
+        # No named candidate is RX-buffered — fall back to the first
+        # *AXI DMA frontend* on the context (name starts with ``axi-`` /
+        # ``cf-`` or contains ``tpl``).  Control-plane devices like
+        # ``ad9528`` or ``ad9371-phy`` may expose scan channels too, but
+        # they aren't wired to an AXI-DMA and ``buf.refill()`` would just
+        # time out on them.
+        buffered = [
+            d
+            for d in ctx.devices
+            if d.name
+            and (d.name.startswith("axi-") or d.name.startswith("cf-") or "tpl" in d.name)
+            and _has_rx_scan(d)
+        ]
+        dev = buffered[0] if buffered else None
+    assert dev is not None, (
+        f"No RX-buffered IIO device found{suffix}. "
+        f"Tried: {list(candidates)}. Present: {all_names}"
+    )
+
+    scan_channels = [c for c in dev.channels if c.scan_element and not c.output]
+    assert scan_channels, (
+        f"No RX scan channels on {dev.name!r}{suffix}. Present: {all_names}"
+    )
+
+    print(f"rx capture{suffix}: selected IIO device {dev.name!r}")
+    buf = None
+    try:
+        for ch in scan_channels:
+            ch.enabled = True
+        buf = iio.Buffer(dev, n_samples, False)
+        try:
+            buf.refill()
+        except TimeoutError as exc:
+            raise AssertionError(
+                f"Buffer refill timed out on {dev.name!r}{suffix} — "
+                "AXI DMA is not delivering samples (JESD or DMA path "
+                "stalled).  Present devices: " + ", ".join(all_names)
+            ) from exc
+        per_channel: dict[str, np.ndarray] = {}
+        for ch in scan_channels:
+            raw = ch.read(buf)
+            # AXI ADC frontends emit signed int16 (or sign-extended
+            # int14/int12); dtype=int16 is correct for every chip this
+            # suite currently runs against.
+            per_channel[ch.id] = np.frombuffer(raw, dtype=np.int16)
+    finally:
+        if buf is not None:
+            del buf
+        for ch in scan_channels:
+            try:
+                ch.enabled = False
+            except Exception:
+                pass
+
+    nonzero = [name for name, arr in per_channel.items() if arr.any()]
+    assert nonzero, (
+        f"All channels on {dev.name!r} returned zero samples{suffix} — "
+        "JESD/DMA/clock path is likely stalled."
+    )
+
+    stds = {name: float(np.abs(arr).std()) for name, arr in per_channel.items()}
+    max_std = max(stds.values())
+    assert max_std >= min_std, (
+        f"All channels on {dev.name!r} latched to a constant value{suffix} "
+        f"(max |std|={max_std:.3g} < {min_std}) — data path stuck."
+    )
+
+    print(
+        f"rx capture{suffix}: device={dev.name}, "
+        f"{len(per_channel)} channel(s), {n_samples} samples, "
+        f"non-zero={list(nonzero)}, max |std|={max_std:.2f}"
+    )
+    return per_channel
+
+
 def _kernel_cache_key(platform_arch: str, config_path: Path) -> str:
     """Return a short sha256 over *platform_arch* and the config file bytes."""
     import hashlib

test/hw/test_ad9081_zcu102_system_hw.py

@@ -46,6 +46,8 @@
     acquire_xsa,
     assert_jesd_links_data,
     assert_no_kernel_faults,
+    assert_no_probe_errors,
+    assert_rx_capture_valid,
     collect_dmesg,
     compile_dts_to_dtb,
     deploy_and_boot,
@@ -266,6 +268,7 @@ def test_ad9081_zcu102_system_hw(board, built_kernel_image_zynqmp, tmp_path):
 
     # --- 9. Verify: kernel probe + IIO context + JESD DATA state ---
     assert_no_kernel_faults(dmesg_txt)
+    assert_no_probe_errors(dmesg_txt)
     assert "AD9081 Rev." in dmesg_txt or "probed ADC AD9081" in dmesg_txt, (
         "AD9081 probe signature was not found in kernel dmesg output"
     )
@@ -292,3 +295,11 @@ def test_ad9081_zcu102_system_hw(board, built_kernel_image_zynqmp, tmp_path):
     )
     print(f"$ cat .../84a90000.axi?jesd204?rx/status\n{rx_status}")
     print(f"$ cat .../84b90000.axi?jesd204?tx/status\n{tx_status}")
+
+    # Data-path smoke test: capture a real buffer and verify samples flow.
+    assert_rx_capture_valid(
+        ctx,
+        ("axi-ad9081-rx-hpc", "ad_ip_jesd204_tpl_adc"),
+        n_samples=2**12,
+        context="ad9081 system",
+    )

test/hw/test_ad9081_zcu102_xsa_hw.py

@@ -35,6 +35,8 @@
     acquire_xsa,
     assert_jesd_links_data,
     assert_no_kernel_faults,
+    assert_no_probe_errors,
+    assert_rx_capture_valid,
     collect_dmesg,
     compile_dts_to_dtb,
     deploy_and_boot,
@@ -184,6 +186,7 @@ def test_ad9081_zcu102_xsa_hw(board, built_kernel_image_zynqmp, tmp_path):
 
     # --- 7. Verify: kernel probe + IIO context + JESD DATA state ---
     assert_no_kernel_faults(dmesg_txt)
+    assert_no_probe_errors(dmesg_txt)
     assert "AD9081 Rev." in dmesg_txt or "probed ADC AD9081" in dmesg_txt, (
         "AD9081 probe signature was not found in kernel dmesg output"
     )
@@ -210,3 +213,11 @@ def test_ad9081_zcu102_xsa_hw(board, built_kernel_image_zynqmp, tmp_path):
     )
     print(f"$ cat .../84a90000.axi?jesd204?rx/status\n{rx_status}")
     print(f"$ cat .../84b90000.axi?jesd204?tx/status\n{tx_status}")
+
+    # Data-path smoke test: capture a real buffer and verify samples flow.
+    assert_rx_capture_valid(
+        ctx,
+        ("axi-ad9081-rx-hpc", "ad_ip_jesd204_tpl_adc"),
+        n_samples=2**12,
+        context="ad9081 xsa",
+    )

test/hw/test_adrv9009_zcu102_hw.py

@@ -38,6 +38,8 @@
     acquire_xsa,
     assert_jesd_links_data,
     assert_no_kernel_faults,
+    assert_no_probe_errors,
+    assert_rx_capture_valid,
     collect_dmesg,
     compile_dts_to_dtb,
     deploy_and_boot,
@@ -214,6 +216,7 @@ def test_adrv9009_zcu102_hw(board, built_kernel_image_zynqmp, tmp_path):
 
     # --- 7. Verify: kernel probe + IIO context + JESD DATA state ---
     assert_no_kernel_faults(dmesg_txt)
+    assert_no_probe_errors(dmesg_txt)
     assert "adrv9009-phy" in dmesg_txt or "Talise" in dmesg_txt, (
         "ADRV9009 phy probe signature was not found in kernel dmesg output"
     )
@@ -238,6 +241,14 @@ def test_adrv9009_zcu102_hw(board, built_kernel_image_zynqmp, tmp_path):
     print(f"$ cat .../axi?jesd204?rx/status\n{rx_status}")
     print(f"$ cat .../axi?jesd204?tx/status\n{tx_status}")
 
+    # --- 8b. Data-path smoke test: capture a real RX buffer. ---
+    assert_rx_capture_valid(
+        ctx,
+        ("axi-adrv9009-rx-hpc", "axi-adrv9009-rx-obs-hpc"),
+        n_samples=2**12,
+        context="adrv9009 initial boot",
+    )
+
     # --- 9. Load all four canonical Talise filter profiles ---
     # The remote libiio write path drops its TCP socket when the driver
     # holds the CPU for Talise re-init, surfacing as ``BrokenPipeError``.
@@ -279,5 +290,7 @@ def test_adrv9009_zcu102_hw(board, built_kernel_image_zynqmp, tmp_path):
         # relock both links before re-reading sysfs status.
         time.sleep(3.0)
         assert_jesd_links_data(shell, context=f"after {filename}")
-        assert_no_kernel_faults(shell_out(shell, "dmesg"))
+        dmesg = shell_out(shell, "dmesg")
+        assert_no_kernel_faults(dmesg)
+        assert_no_probe_errors(dmesg)
         print(f"  {filename}: RX+TX JESD DATA OK")

test/hw/test_fmcdaq3_vcu118_hw.py

@@ -33,34 +33,48 @@
         allow_module_level=True,
     )
 
+from test.hw.hw_helpers import (  # noqa: E402
+    DEFAULT_OUT_DIR,
+    assert_no_kernel_faults,
+    assert_no_probe_errors,
+    assert_rx_capture_valid,
+    collect_dmesg,
+    open_iio_context,
+)
+
 
 @pytest.mark.lg_feature(["fmcdaq3", "vcu118"])
-def test_fmcdaq3_vcu118_boot_hw(target):
+def test_fmcdaq3_vcu118_boot_hw(board):
     """Boot FMCDAQ3+VCU118 with the prebuilt Kuiper image and verify IIO."""
-    shell = _boot_and_get_shell(target)
-    _assert_probed_drivers(shell)
-    _assert_iio_devices(shell)
+    out_dir = DEFAULT_OUT_DIR
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    board.transition("shell")
+    shell = board.target.get_driver("ADIShellDriver")
 
+    dmesg_txt = collect_dmesg(
+        shell,
+        out_dir,
+        label="fmcdaq3_vcu118",
+        grep_pattern="ad9680|ad9152|ad9528|jesd204|probe|failed|error",
+    )
+    assert_no_kernel_faults(dmesg_txt)
+    assert_no_probe_errors(dmesg_txt)
 
-def _boot_and_get_shell(target):
-    """Drive ``BootFabric`` through ``powered_off`` → ``shell`` and return shell."""
-    strategy = target.get_driver("Strategy")
-    strategy.transition("powered_off")
-    strategy.transition("shell")
-    return target.get_driver("ADIShellDriver")
+    lowered = dmesg_txt.lower()
+    assert "ad9680" in lowered, "AD9680 driver messages not seen in dmesg"
+    assert "ad9152" in lowered, "AD9152 driver messages not seen in dmesg"
 
+    _assert_iio_devices(shell)
 
-def _assert_probed_drivers(shell) -> None:
-    """Fail unless dmesg shows AD9680 / AD9152 / AD9528 / JESD driver probes."""
-    out = shell.run_check(
-        "dmesg | grep -Ei 'ad9680|ad9152|ad9528|jesd204|fail|error' | tail -n 200; true"
+    # Data-path smoke test: capture a real AD9680 RX buffer.
+    ctx, _ = open_iio_context(shell)
+    assert_rx_capture_valid(
+        ctx,
+        ("axi-ad9680-core-lpc", "axi-ad9680-hpc", "axi-ad9680-rx-hpc"),
+        n_samples=2**12,
+        context="fmcdaq3 boot",
     )
-    dmesg = "\n".join(out) if isinstance(out, list) else str(out)
-    print("\n=== FMCDAQ3 probe-relevant dmesg ===")
-    print(dmesg)
-    print("====================================")
-    assert "ad9680" in dmesg.lower(), "AD9680 driver messages not seen in dmesg"
-    assert "ad9152" in dmesg.lower(), "AD9152 driver messages not seen in dmesg"
 
 
 def _assert_iio_devices(shell) -> None: