Skip to content

Commit 357626c

Browse files
tbv 0.2.15: fix per-rail netdev teardown crash on peer disconnect
The per-rail GID netdev parented to xd->dev, making it a sibling of the tb_services in xd->dev child list. On disconnect, icm_tr_xdomain_disconnected -> tb_xdomain_remove iterates that child list; our service .remove unregister_netdev deletes the netdev mid-iteration -> kernfs_find_and_get_ns NULL deref -> a chain-wide crash CASCADE (one node crashes, drops its link, neighbours see a disconnect, they crash too). kdump appmana-002 2026-06-22. Fix (the thunderbolt_net pattern): parent the netdev to the stable, driver-owned NHI/ring device the ib_device already uses, never xd->dev (tbv_ibdev_netdev_parent, red-first KUnit, 34/34). The peer identity udev/GID needed from the parent is now COPIED onto the netdev at attach as tbv_peer_uuid/tbv_peer_name sysfs attrs (no lifetime tie to xd); the udev helpers read those and resolve the live ifname by IFINDEX. peer->xd is refcounted + NULL-checked everywhere else -- this netdev parent was the only teardown hazard.
1 parent 29e7561 commit 357626c

6 files changed

Lines changed: 154 additions & 44 deletions

File tree

drivers/thunderbolt_ibverbs/dkms.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
PACKAGE_NAME="thunderbolt-ibverbs"
2-
PACKAGE_VERSION="0.2.14"
2+
PACKAGE_VERSION="0.2.15"
33

44
BUILT_MODULE_NAME[0]="thunderbolt_ibverbs"
55
BUILT_MODULE_LOCATION[0]="kernel"

drivers/thunderbolt_ibverbs/kernel/ibdev.c

Lines changed: 83 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,14 @@ struct tbv_ibdev {
399399
* rails for the same peer by tbv_select_qp_rail_locked().
400400
*/
401401
struct tbv_rail *rail;
402+
/*
403+
* Peer identity COPIED from the XDomain at attach (no lifetime tie to
404+
* xd) and exposed as netdev sysfs attrs (tbv_peer_uuid / tbv_peer_name).
405+
* The netdev now parents to the stable NHI, not xd, so udev naming and
406+
* the per-link GID helper read the peer from the netdev itself.
407+
*/
408+
char peer_uuid[UUID_STRING_LEN + 1];
409+
char peer_name[64];
402410
};
403411

404412
struct tbv_send_ctx {
@@ -2362,6 +2370,55 @@ static void tbv_netdev_setup(struct net_device *ndev)
23622370
* to the rail's Thunderbolt device for matching) so both ends of a link share a
23632371
* subnet. The netdev carries no traffic; the rail's data path is native TB.
23642372
*/
2373+
/*
2374+
* Per-rail netdev sysfs: the COPIED peer identity, read back from the netdev's
2375+
* drvdata (the tbv_ibdev). Lets udev/GID tooling resolve the peer from the
2376+
* netdev directly, now that it no longer parents to the XDomain device.
2377+
*/
2378+
static ssize_t tbv_peer_uuid_show(struct device *d,
2379+
struct device_attribute *a, char *buf)
2380+
{
2381+
struct tbv_ibdev *idev = dev_get_drvdata(d);
2382+
2383+
return sysfs_emit(buf, "%s\n", idev ? idev->peer_uuid : "");
2384+
}
2385+
static DEVICE_ATTR(tbv_peer_uuid, 0444, tbv_peer_uuid_show, NULL);
2386+
2387+
static ssize_t tbv_peer_name_show(struct device *d,
2388+
struct device_attribute *a, char *buf)
2389+
{
2390+
struct tbv_ibdev *idev = dev_get_drvdata(d);
2391+
2392+
return sysfs_emit(buf, "%s\n", idev ? idev->peer_name : "");
2393+
}
2394+
static DEVICE_ATTR(tbv_peer_name, 0444, tbv_peer_name_show, NULL);
2395+
2396+
static struct attribute *tbv_rail_netdev_attrs[] = {
2397+
&dev_attr_tbv_peer_uuid.attr,
2398+
&dev_attr_tbv_peer_name.attr,
2399+
NULL,
2400+
};
2401+
static const struct attribute_group tbv_rail_netdev_group = {
2402+
.attrs = tbv_rail_netdev_attrs,
2403+
};
2404+
2405+
struct device *tbv_ibdev_netdev_parent(struct device *ib_parent,
2406+
struct tb_xdomain *xd)
2407+
{
2408+
/*
2409+
* Parent the per-rail GID netdev to the stable, driver-owned NHI/ring
2410+
* device the ib_device already uses -- NOT the XDomain device. A direct
2411+
* child of xd->dev is a sibling of the tb_services in xd->dev's child
2412+
* list, so unregister_netdev() in our service .remove (called from
2413+
* tb_xdomain_remove's device_for_each_child_reverse on peer disconnect)
2414+
* mutates that list mid-iteration and NULL-derefs kernfs -- the crash
2415+
* cascade of 2026-06-22. Decoupled, the netdev is torn down in order by
2416+
* tbv_ibdev_detach_netdev() before its parent.
2417+
*/
2418+
(void)xd;
2419+
return ib_parent;
2420+
}
2421+
23652422
static int tbv_ibdev_attach_netdev(struct tbv_ibdev *dev)
23662423
{
23672424
struct net_device *ndev;
@@ -2376,13 +2433,16 @@ static int tbv_ibdev_attach_netdev(struct tbv_ibdev *dev)
23762433
eth_hw_addr_set(ndev, mac);
23772434

23782435
/*
2379-
* Parent to the per-link XDomain device (NOT the shared NHI ring device
2380-
* that ib_device uses): it is unique per cabled neighbour and carries the
2381-
* peer hostname in its `device_name` attribute, so udev can resolve the
2382-
* peer and rename the netdev per link (mirrors thunderbolt_net).
2436+
* Parent to the stable NHI/ring device the ib_device uses, NOT the
2437+
* XDomain device: a netdev that is a direct child of xd->dev races
2438+
* tb_xdomain_remove()'s child-list teardown on peer disconnect (see
2439+
* tbv_ibdev_netdev_parent). udev/GID resolution of the peer now comes
2440+
* from the netdev itself, not its parent device.
23832441
*/
2384-
if (dev->rail && dev->rail->peer && dev->rail->peer->xd)
2385-
SET_NETDEV_DEV(ndev, &dev->rail->peer->xd->dev);
2442+
SET_NETDEV_DEV(ndev,
2443+
tbv_ibdev_netdev_parent(dev->base.dev.parent,
2444+
(dev->rail && dev->rail->peer) ? dev->rail->peer->xd :
2445+
NULL));
23862446

23872447
ret = register_netdev(ndev);
23882448
if (ret) {
@@ -2405,6 +2465,22 @@ static int tbv_ibdev_attach_netdev(struct tbv_ibdev *dev)
24052465
* from a warm reload (see drivers/thunderbolt/test.c icm_fw model), so a
24062466
* load-time hang on these nodes is a driver-software fault, not the ICM.
24072467
*/
2468+
/* Copy peer identity from the XDomain and expose it on the netdev. */
2469+
{
2470+
struct tb_xdomain *xd = (dev->rail && dev->rail->peer) ?
2471+
dev->rail->peer->xd : NULL;
2472+
2473+
if (xd && xd->remote_uuid)
2474+
snprintf(dev->peer_uuid, sizeof(dev->peer_uuid), "%pUb",
2475+
xd->remote_uuid);
2476+
if (xd && xd->device_name)
2477+
strscpy(dev->peer_name, xd->device_name,
2478+
sizeof(dev->peer_name));
2479+
}
2480+
dev_set_drvdata(&ndev->dev, dev);
2481+
if (sysfs_create_group(&ndev->dev.kobj, &tbv_rail_netdev_group))
2482+
pr_warn("rail netdev peer-attr group create failed\n");
2483+
24082484
netif_carrier_on(ndev);
24092485
dev->netdev = ndev;
24102486
return 0;
@@ -2418,6 +2494,7 @@ static void tbv_ibdev_detach_netdev(struct tbv_ibdev *dev)
24182494
return;
24192495

24202496
dev->netdev = NULL;
2497+
sysfs_remove_group(&ndev->dev.kobj, &tbv_rail_netdev_group);
24212498
ib_device_set_netdev(&dev->base, NULL, 1);
24222499
unregister_netdev(ndev);
24232500
free_netdev(ndev);

drivers/thunderbolt_ibverbs/kernel/tbv.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -872,6 +872,16 @@ int tbv_ibdev_rail_name_index(const struct tbv_rail *rail);
872872
* deadlocks (unregister_netdev under the rename's rtnl). Exposed for kunit.
873873
*/
874874
bool tbv_netdev_rename_keep(const char *expected_name, const char *new_name);
875+
/*
876+
* tbv_ibdev_netdev_parent returns the device the per-rail GID netdev must be
877+
* parented to. It MUST be a stable, driver-owned device (the NHI/ring device the
878+
* ib_device already uses, dev->base.dev.parent), NOT the XDomain device: the
879+
* netdev must not be a direct child of xd->dev, or unregister_netdev() races
880+
* tb_xdomain_remove()'s child-list iteration and NULL-derefs kernfs. Exposed for
881+
* kunit.
882+
*/
883+
struct device *tbv_ibdev_netdev_parent(struct device *ib_parent,
884+
struct tb_xdomain *xd);
875885
bool tbv_gid_matches_identity(const u8 gid[16], u64 eui64, u32 ipv4_be);
876886
/*
877887
* tbv_gid_identity_verdict classifies a dgid against a stored peer identity.

drivers/thunderbolt_ibverbs/kernel/tests/ibdev_naming_test.c

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,13 +106,37 @@ static void tbv_rename_detach_pinned_diverged(struct kunit *test)
106106
KUNIT_EXPECT_FALSE(test, tbv_netdev_rename_keep("br0.lan", "eth9"));
107107
}
108108

109+
/*
110+
* The per-rail GID netdev MUST NOT be parented to the XDomain device. xd->dev is
111+
* owned/removed by the thunderbolt core; parenting the netdev there makes it a
112+
* *sibling* of the tb_services in xd->dev's child list, so unregister_netdev()
113+
* (in our service .remove, run from inside tb_xdomain_remove's
114+
* device_for_each_child_reverse) mutates that child list mid-iteration ->
115+
* kernfs_find_and_get_ns NULL deref -> chain-wide crash cascade on any neighbour
116+
* disconnect (kdump appmana-002 2026-06-22). It must instead use the stable
117+
* driver-owned NHI/ring device that the ib_device already parents to
118+
* (dev->base.dev.parent), so its teardown is ordered and self-contained.
119+
*/
120+
static void tbv_netdev_parent_is_stable_not_xdomain(struct kunit *test)
121+
{
122+
struct tb_xdomain xd = {0};
123+
struct device nhi_dev = {0}; /* tb_ring_dma_device: stable, driver-owned */
124+
125+
/* red while it returns &xd.dev; green once it returns the stable parent */
126+
KUNIT_EXPECT_PTR_NE(test,
127+
tbv_ibdev_netdev_parent(&nhi_dev, &xd), &xd.dev);
128+
KUNIT_EXPECT_PTR_EQ(test,
129+
tbv_ibdev_netdev_parent(&nhi_dev, &xd), &nhi_dev);
130+
}
131+
109132
static struct kunit_case tbv_ibdev_naming_test_cases[] = {
110133
KUNIT_CASE(tbv_naming_two_neighbours_must_differ),
111134
KUNIT_CASE(tbv_naming_lanes_disambiguate),
112135
KUNIT_CASE(tbv_naming_errors),
113136
KUNIT_CASE(tbv_rename_keep_own_gid_netdev),
114137
KUNIT_CASE(tbv_rename_keep_pinned_match),
115138
KUNIT_CASE(tbv_rename_detach_pinned_diverged),
139+
KUNIT_CASE(tbv_netdev_parent_is_stable_not_xdomain),
116140
{}
117141
};
118142

Lines changed: 20 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,14 @@
11
#!/bin/sh
22
# tbv-rdma-addr <ifname> -- assign the deterministic per-link routable /64 to a
33
# usb4_rdma GID-only rail netdev, so both ends of a cabled link share a subnet.
4-
# Invoked from udev (RUN+=) after the rail netdev appears, and safe to re-run by
5-
# hand. Pure derivation lives in tbv-rdma-addr-lib.sh; this does the sysfs
6-
# lookups and the `ip` assignment.
4+
# Invoked from udev (RUN+=) and safe to re-run by hand. Pure derivation is in
5+
# tbv-rdma-addr-lib.sh; this does the sysfs lookups and the `ip` assignment.
76
#
8-
# The rail netdev is parented (SET_NETDEV_DEV) to its XDomain *device* (e.g.
9-
# "0-1"), whose unique_id is the remote peer's host-router UUID. The local
10-
# host-router UUID is "<domain>-0" in the same thunderbolt domain. Both ends
11-
# therefore hold the same unordered UUID pair -> the same /64 (see the lib).
12-
#
13-
# udev renames the netdev (u4r<N> -> tbr-<peer>) BEFORE running RUN+=, so the
14-
# passed kernel name is stale by the time we run. We therefore resolve the live
15-
# interface from the stable XDomain parent device (via DEVPATH), not the name.
7+
# As of 0.2.15 the rail netdev parents to the stable NHI device (NOT the XDomain
8+
# device -- a direct child of xd->dev crashes on disconnect), so the peer UUID is
9+
# read from the netdev's own tbv_peer_uuid attribute, and the live interface is
10+
# resolved by the stable IFINDEX udev provides (robust to udev's concurrent
11+
# rename). The local host-router UUID is the same for every rail on the host.
1612
set -eu
1713
export PATH=/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}
1814

@@ -21,28 +17,26 @@ LIB="$(dirname "$0")/tbv-rdma-addr-lib.sh"
2117
. "$LIB"
2218

2319
ifc="${1:-}"
24-
xdev=""
25-
# Prefer the passed name if it still resolves; else recover the XDomain parent
26-
# from DEVPATH (stable across the rename) and read the live name back out of it.
27-
if [ -n "$ifc" ] && [ -e "/sys/class/net/${ifc}/device" ]; then
28-
xdev="$(readlink -f "/sys/class/net/${ifc}/device" 2>/dev/null || true)"
29-
elif [ -n "${DEVPATH:-}" ]; then
30-
xdev="$(readlink -f "/sys${DEVPATH%/net/*}" 2>/dev/null || true)"
31-
[ -n "$xdev" ] && ifc="$(ls "${xdev}/net" 2>/dev/null | head -n1)"
20+
# Resolve the live ifname by the stable ifindex (survives the udev rename).
21+
if [ -n "${IFINDEX:-}" ]; then
22+
for f in /sys/class/net/*/ifindex; do
23+
[ "$(cat "$f" 2>/dev/null)" = "$IFINDEX" ] || continue
24+
ifc="$(basename "$(dirname "$f")")"
25+
break
26+
done
3227
fi
33-
[ -n "$xdev" ] && [ -n "$ifc" ] && [ -e "/sys/class/net/${ifc}" ] || exit 0
28+
[ -n "$ifc" ] && [ -e "/sys/class/net/${ifc}" ] || exit 0
3429

35-
remote="$(cat "${xdev}/unique_id" 2>/dev/null || true)"
36-
[ -n "$remote" ] || exit 0 # not an XDomain-parented rail
30+
remote="$(cat "/sys/class/net/${ifc}/tbv_peer_uuid" 2>/dev/null | tr -d '[:space:]')"
31+
[ -n "$remote" ] || exit 0 # not a tbv rail netdev
3732

38-
domain="$(basename "$xdev")"; domain="${domain%%-*}"
39-
local="$(cat "/sys/bus/thunderbolt/devices/${domain}-0/unique_id" 2>/dev/null || true)"
33+
# local host-router UUID (identical for every rail on this host)
34+
local="$(cat /sys/bus/thunderbolt/devices/0-0/unique_id 2>/dev/null | tr -d '[:space:]')"
35+
[ -n "$local" ] || local="$(cat /sys/bus/thunderbolt/devices/*-0/unique_id 2>/dev/null | head -n1 | tr -d '[:space:]')"
4036
[ -n "$local" ] || exit 0
4137

4238
addr="$(tbv_link_addr "$local" "$remote")"
4339

44-
# nodad: link-local DAD is pointless on a point-to-point GID-only netdev and only
45-
# adds latency before the GID appears. `replace` keeps the helper idempotent.
4640
ip -6 addr replace "$addr" dev "$ifc" nodad 2>/dev/null || \
4741
ip -6 addr replace "$addr" dev "$ifc"
4842
ip link set "$ifc" up

drivers/thunderbolt_ibverbs/packaging/udev/tbv-rdma-ifname

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1,32 @@
11
#!/bin/sh
22
# tbv-rdma-ifname: udev PROGRAM helper. Prints a stable, per-link name for the
3-
# usb4_rdma GID-only netdev that thunderbolt_ibverbs creates per rail. The name
4-
# is derived ONLY from what the kernel already exposes on the netdev's parent
5-
# XDomain device, so it needs NO site configuration and works on any host.
3+
# usb4_rdma GID-only netdev that thunderbolt_ibverbs creates per rail (kernel
4+
# name u4r<N>, enumeration-order and unstable).
65
#
7-
# Naming priority (all from sysfs, none site-specific):
8-
# 1. tbr-<peer-hostname-tail> -- device_name (peer kernel hostname)
9-
# 2. tbr-<peer-uuid-prefix> -- unique_id (stable peer hardware UUID)
10-
# 3. (no rename) -- keep the kernel's u4r<N> name
6+
# As of 0.2.15 the peer identity lives on the netdev itself (tbv_peer_name /
7+
# tbv_peer_uuid), because the netdev parents to the stable NHI device rather
8+
# than the XDomain device. Falls back to the parent XDomain attrs for older
9+
# (xd-parented) netdevs. Needs NO site configuration.
1110
#
12-
# Names are kept within the 15-char ifname limit ("tbr-" + up to 11).
11+
# Naming priority:
12+
# 1. tbr-<peer-hostname-tail> -- peer kernel hostname
13+
# 2. tbr-<peer-uuid-prefix> -- stable peer hardware UUID
14+
# 3. (no rename) -- keep the kernel's u4r<N> name
15+
# Names stay within the 15-char ifname limit ("tbr-" + up to 11).
1316
set -u
1417
iface="${1:-}"
1518
[ -n "$iface" ] || exit 0
16-
dev="/sys/class/net/${iface}/device"
19+
nd="/sys/class/net/${iface}"
1720

18-
host=$(cat "${dev}/device_name" 2>/dev/null | tr -cd 'A-Za-z0-9')
21+
host=$(cat "${nd}/tbv_peer_name" 2>/dev/null | tr -cd 'A-Za-z0-9')
22+
[ -n "$host" ] || host=$(cat "${nd}/device/device_name" 2>/dev/null | tr -cd 'A-Za-z0-9')
1923
if [ -n "$host" ]; then
2024
printf 'tbr-%s' "$(printf '%s' "$host" | tail -c 11)"
2125
exit 0
2226
fi
2327

24-
uid=$(cat "${dev}/unique_id" 2>/dev/null | tr -cd '0-9a-f')
28+
uid=$(cat "${nd}/tbv_peer_uuid" 2>/dev/null | tr -cd '0-9a-f')
29+
[ -n "$uid" ] || uid=$(cat "${nd}/device/unique_id" 2>/dev/null | tr -cd '0-9a-f')
2530
if [ -n "$uid" ]; then
2631
printf 'tbr-%s' "$(printf '%s' "$uid" | cut -c1-12)"
2732
exit 0

0 commit comments

Comments
 (0)