From 17e7e4acae9ef7784a15f1e41247e68290a8624c Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Thu, 30 Apr 2026 12:16:19 +0200 Subject: [PATCH 01/14] chore(log): add relay session diagnostic logging --- src/nat_traversal_api.rs | 33 ++++++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 2188424e..50cacad5 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -3822,12 +3822,21 @@ impl NatTraversalEndpoint { // DashMap provides lock-free .get() that returns Option> if let Some(session) = self.relay_sessions.get(&relay_addr) { if session.is_active() { - debug!("Reusing existing relay session to {}", relay_addr); + debug!( + relay = %relay_addr, + public_address = ?session.public_address, + "relay session: reusing active CONNECT-UDP session" + ); return Ok((session.public_address, None)); } } - info!("Establishing relay session to {}", relay_addr); + info!( + relay = %relay_addr, + tracked_connections = self.connections.len(), + active_relay_sessions = self.relay_sessions.len(), + "relay session: establishing CONNECT-UDP Bind session" + ); // Prefer reusing an existing peer connection to the relay. // The relay server's handle_relay_requests is spawned for each ACCEPTED @@ -3835,15 +3844,28 @@ impl NatTraversalEndpoint { // already listening for bidi streams. let connection = if let Some(existing) = self.connections.get(&relay_addr) { if existing.close_reason().is_none() { - info!("Reusing existing peer connection to relay {}", relay_addr); + info!( + relay = %relay_addr, + stable_id = existing.stable_id(), + "relay session: reusing existing peer connection to relay" + ); existing.clone() } else { // Existing connection is dead — fall back to creating a new one + debug!( + relay = %relay_addr, + close_reason = ?existing.close_reason(), + "relay session: existing peer connection is closed; cold-dialing relay" + ); drop(existing); self.connect_new_to_relay(relay_addr).await? } } else { // No existing connection — create one + debug!( + relay = %relay_addr, + "relay session: no existing peer connection; cold-dialing relay" + ); self.connect_new_to_relay(relay_addr).await? }; @@ -3992,6 +4014,11 @@ impl NatTraversalEndpoint { })?; let server_name = relay_addr.ip().to_string(); + debug!( + relay = %relay_addr, + timeout = ?self.config.coordination_timeout, + "relay session: cold-dialing advertised relay address" + ); let connecting = endpoint.connect(relay_addr, &server_name).map_err(|e| { NatTraversalError::ConnectionFailed(format!( "Failed to initiate relay connection: {}", From ea7773a3f45902db4631bf2d57a22cb63cb8f8e9 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Fri, 1 May 2026 09:42:34 +0200 Subject: [PATCH 02/14] chore(log): elevate relay diagnostic logging to debug level - Enhanced relay diagnostic logs for clarity during inbound and outbound traffic handling. - Refined UDP relay logging paths to include detailed source, target, and payload information. --- src/masque/relay_server.rs | 64 ++++++++++++++++++++++++++------------ src/masque/relay_socket.rs | 15 +++++++++ 2 files changed, 59 insertions(+), 20 deletions(-) diff --git a/src/masque/relay_server.rs b/src/masque/relay_server.rs index fc8b650b..692db8d6 100644 --- a/src/masque/relay_server.rs +++ b/src/masque/relay_server.rs @@ -888,11 +888,11 @@ impl MasqueRelayServer { match socket.recv_from(&mut buf).await { Ok((len, source)) => { let payload = Bytes::copy_from_slice(&buf[..len]); - tracing::trace!( + tracing::debug!( session_id, source = %source, len, - "Relay: received UDP from target" + "RELAY_TUNNEL[srv]: dgram-loop dir1 recv UDP → forwarding to relay-client" ); // Encode as uncompressed datagram (includes source address @@ -961,21 +961,32 @@ impl MasqueRelayServer { }; match resolved { Some((target, payload)) => { - tracing::trace!( + tracing::debug!( session_id, target = %target, len = payload.len(), - "Relay: forwarding to target via UDP" + "RELAY_TUNNEL[srv]: dgram-loop dir2 recv from relay-client → sendto target" ); server2.stats.record_bytes(payload.len() as u64); server2.stats.record_datagram(); - if let Err(e) = socket2.send_to(&payload, target).await { - tracing::warn!( - session_id, - target = %target, - error = %e, - "Failed to send UDP to target" - ); + match socket2.send_to(&payload, target).await { + Ok(n) => { + tracing::debug!( + session_id, + target = %target, + len = payload.len(), + sent = n, + "RELAY_TUNNEL[srv]: dgram-loop dir2 sendto OK" + ); + } + Err(e) => { + tracing::warn!( + session_id, + target = %target, + error = %e, + "Failed to send UDP to target" + ); + } } } None => { @@ -1072,9 +1083,9 @@ impl MasqueRelayServer { match socket.recv_from(&mut buf).await { Ok((len, source)) => { let payload = Bytes::copy_from_slice(&buf[..len]); - tracing::trace!( + tracing::debug!( session_id, source = %source, len, - "Stream relay: received UDP from target" + "RELAY_TUNNEL[srv]: stream-loop dir1 recv UDP → forwarding to relay-client" ); let datagram = UncompressedDatagram::new(VarInt::from_u32(0), source, payload); @@ -1167,18 +1178,31 @@ impl MasqueRelayServer { let mut cursor = Bytes::from(frame_buf); match UncompressedDatagram::decode(&mut cursor) { Ok(datagram) => { - tracing::trace!( + tracing::debug!( session_id, target = %datagram.target, len = datagram.payload.len(), - "Stream relay: forwarding to target via UDP" + "RELAY_TUNNEL[srv]: stream-loop dir2 recv from relay-client → sendto target" ); stats2.record_bytes(datagram.payload.len() as u64); stats2.record_datagram(); - if let Err(e) = socket2.send_to(&datagram.payload, datagram.target).await { - tracing::warn!( - session_id, target = %datagram.target, error = %e, - "Failed to send UDP to target" - ); + let target = datagram.target; + let payload_len = datagram.payload.len(); + match socket2.send_to(&datagram.payload, target).await { + Ok(n) => { + tracing::debug!( + session_id, + target = %target, + len = payload_len, + sent = n, + "RELAY_TUNNEL[srv]: stream-loop dir2 sendto OK" + ); + } + Err(e) => { + tracing::warn!( + session_id, target = %target, error = %e, + "Failed to send UDP to target" + ); + } } } Err(_) => { diff --git a/src/masque/relay_socket.rs b/src/masque/relay_socket.rs index e7a56a98..9edc83cd 100644 --- a/src/masque/relay_socket.rs +++ b/src/masque/relay_socket.rs @@ -206,6 +206,14 @@ impl MasqueRelaySocket { Ok(datagram) => { // `datagram.payload` is a zero-copy slice of // the original frame buffer — no clone needed. + let inbound_source = datagram.target; + let inbound_len = datagram.payload.len(); + tracing::debug!( + relay = %relay_public_addr, + source = %inbound_source, + len = inbound_len, + "RELAY_TUNNEL[clt]: decoded inbound frame → enqueue for Quinn poll_recv" + ); if recv_tx .send((datagram.payload, datagram.target)) .await @@ -336,6 +344,13 @@ impl AsyncUdpSocket for MasqueRelaySocket { // of `segment_size` bytes. Each segment must be sent as its // own tunnel frame — the relay server has a per-frame size // limit and cannot handle the entire batch as one. + tracing::debug!( + relay = %self.relay_public_addr, + destination = %transmit.destination, + len = transmit.contents.len(), + segment_size = ?transmit.segment_size, + "RELAY_TUNNEL[clt]: try_send → enqueue outbound for relay-server" + ); if let Some(segment_size) = transmit.segment_size { for chunk in transmit.contents.chunks(segment_size) { let datagram = UncompressedDatagram::new( From 3ecfee0aefbeec6f4af21f4931bf50d798157150 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Fri, 1 May 2026 21:08:00 +0200 Subject: [PATCH 03/14] fix(relay,pqc): cap relay-tunnelled MTU and surface path-MTU feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - connection: bound PQC `min_initial_size` by the current path MTU so coalesced Initial+Handshake packets never exceed what the path has proven; restart MTUD from the established MTU instead of jumping to the PQC-preferred floor. - masque: add out-of-band `TunnelControlFrame::PmtuUpdate` emitted by the relay-server on egress `EMSGSIZE` and enforced by the relay-client `MasqueRelaySocket` as a per-target send cap that drives Quinn's DPLPMTUD downward. - masque/relay_server: enable `IP_DONTFRAG` / `IP_PMTUDISC_DO` on the bound socket so oversized egress fails fast instead of silently fragmenting and getting dropped by intermediate NATs. - nat_traversal_api, p2p_endpoint: clamp the inner endpoint's transport config to a 1200-byte initial/min MTU on both accept- and dial-through-relay paths, with DPLPMTUD enabled to probe upward. - p2p_endpoint, config/nat_timeouts: treat `send_ack_timeout` as a best-effort observation window rather than a hard failure — data is already queued to QUIC; later connection state or application-level timeouts handle retries. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/config/nat_timeouts.rs | 17 ++- src/connection/mod.rs | 82 ++++++++++--- src/masque/mod.rs | 1 + src/masque/relay_server.rs | 226 +++++++++++++++++++++++++++++++++-- src/masque/relay_socket.rs | 92 +++++++++++++- src/masque/tunnel_control.rs | 199 ++++++++++++++++++++++++++++++ src/nat_traversal_api.rs | 32 ++++- src/p2p_endpoint.rs | 45 +++++-- 8 files changed, 645 insertions(+), 49 deletions(-) create mode 100644 src/masque/tunnel_control.rs diff --git a/src/config/nat_timeouts.rs b/src/config/nat_timeouts.rs index 3177ff59..cc14ba31 100644 --- a/src/config/nat_timeouts.rs +++ b/src/config/nat_timeouts.rs @@ -131,10 +131,10 @@ impl Default for RelayTimeouts { } } -/// Default time to wait for the peer to acknowledge stream data after a send. +/// Default best-effort window to observe stream-data acknowledgement after a send. const DEFAULT_SEND_ACK_TIMEOUT: Duration = Duration::from_secs(1); -/// Fast-network send ACK timeout (halved from default, matching the fast profile pattern). +/// Fast-network best-effort send ACK window (halved from default). const FAST_SEND_ACK_TIMEOUT: Duration = Duration::from_millis(500); /// Master timeout configuration @@ -149,14 +149,13 @@ pub struct TimeoutConfig { /// Relay timeouts pub relay: RelayTimeouts, - /// Maximum time to wait for the peer to acknowledge stream data after - /// `finish()`. If this expires the send is treated as failed and the - /// connection is considered dead. + /// Best-effort time to wait for Quinn to observe acknowledgement of stream + /// data after `finish()`. /// - /// This must be **shorter** than any outer send timeout applied by the - /// caller (e.g. saorsa-core's `connection_timeout`) so that the - /// transport layer can surface the error before the caller's timeout - /// fires. + /// Explicit stream stop or connection loss is still returned as a send + /// error. Expiry of this window only means the data has been queued to + /// QUIC but not confirmed locally yet; later connection state or + /// application-level timeouts are responsible for retries. pub send_ack_timeout: Duration, } diff --git a/src/connection/mod.rs b/src/connection/mod.rs index 0ded7792..c32e87c2 100644 --- a/src/connection/mod.rs +++ b/src/connection/mod.rs @@ -888,7 +888,9 @@ impl Connection { // Finish current packet if let Some(mut builder) = builder_storage.take() { if pad_datagram { - let min_size = self.pqc_state.min_initial_size(); + let min_size = self + .pqc_state + .min_initial_size_for_path_mtu(self.path.current_mtu()); builder.pad_to(min_size); } @@ -1145,7 +1147,9 @@ impl Connection { } buf.write(token); self.stats.frame_tx.path_response += 1; - let min_size = self.pqc_state.min_initial_size(); + let min_size = self + .pqc_state + .min_initial_size_for_path_mtu(self.path.current_mtu()); builder.pad_to(min_size); builder.finish_and_track( now, @@ -1232,7 +1236,9 @@ impl Connection { // Finish the last packet if let Some(mut builder) = builder_storage { if pad_datagram { - let min_size = self.pqc_state.min_initial_size(); + let min_size = self + .pqc_state + .min_initial_size_for_path_mtu(self.path.current_mtu()); builder.pad_to(min_size); } @@ -1435,7 +1441,10 @@ impl Connection { "PATH_CHALLENGE queued without 1-RTT keys" ); - buf.reserve(self.pqc_state.min_initial_size() as usize); + let min_size = self + .pqc_state + .min_initial_size_for_path_mtu(self.path.current_mtu()); + buf.reserve(min_size as usize); let buf_capacity = buf.capacity(); let mut builder = PacketBuilder::new( @@ -1463,7 +1472,9 @@ impl Connection { buf.write(challenge); self.stats.frame_tx.path_challenge += 1; - let min_size = self.pqc_state.min_initial_size(); + let min_size = self + .pqc_state + .min_initial_size_for_path_mtu(self.path.current_mtu()); builder.pad_to(min_size); builder.finish_and_track(now, self, None, buf); @@ -1531,7 +1542,10 @@ impl Connection { "PATH_CHALLENGE queued without 1-RTT keys" ); - buf.reserve(self.pqc_state.min_initial_size() as usize); + let min_size = self + .pqc_state + .min_initial_size_for_path_mtu(self.path.current_mtu()); + buf.reserve(min_size as usize); let buf_capacity = buf.capacity(); // Use current connection ID for NAT traversal PATH_CHALLENGE @@ -1561,7 +1575,9 @@ impl Connection { self.stats.frame_tx.path_challenge += 1; // PATH_CHALLENGE frames must be padded to at least 1200 bytes - let min_size = self.pqc_state.min_initial_size(); + let min_size = self + .pqc_state + .min_initial_size_for_path_mtu(self.path.current_mtu()); builder.pad_to(min_size); builder.finish_and_track(now, self, None, buf); @@ -1595,7 +1611,10 @@ impl Connection { SpaceId::Data, "PATH_CHALLENGE queued without 1-RTT keys" ); - buf.reserve(self.pqc_state.min_initial_size() as usize); + let min_size = self + .pqc_state + .min_initial_size_for_path_mtu(prev_path.current_mtu()); + buf.reserve(min_size as usize); let buf_capacity = buf.capacity(); @@ -1629,7 +1648,6 @@ impl Connection { // to at least the smallest allowed maximum datagram size of 1200 bytes, // unless the anti-amplification limit for the path does not permit // sending a datagram of this size - let min_size = self.pqc_state.min_initial_size(); builder.pad_to(min_size); builder.finish(self, buf); @@ -2712,10 +2730,12 @@ impl Connection { // Check if we should trigger MTU discovery for PQC if self.pqc_state.should_trigger_mtu_discovery() { - // Request larger MTU for PQC handshakes - self.path - .mtud - .reset(self.pqc_state.min_initial_size(), self.config.min_mtu); + // Restart MTU discovery from the current path MTU. PQC handshakes + // may benefit from larger packets, but raising `current_mtu` + // directly would let the first server flight exceed the path + // before DPLPMTUD has proven it. + let current_mtu = self.path.current_mtu(); + self.path.mtud.reset(current_mtu, self.config.min_mtu); trace!("Triggered MTU discovery for PQC handshake"); } @@ -2776,10 +2796,14 @@ impl Connection { if use_pqc_fragmentation { // Fragment large CRYPTO data for PQC handshakes + let max_crypto_frame_size = usize::from( + self.pqc_state + .min_initial_size_for_path_mtu(self.path.current_mtu()), + ); let frames = self.pqc_state.packet_handler.fragment_crypto_data( &outgoing, offset, - self.pqc_state.min_initial_size() as usize, + max_crypto_frame_size, ); for frame in frames { self.spaces[space].pending.crypto.push_back(frame); @@ -6498,6 +6522,12 @@ impl PqcState { } } + /// Return the PQC-preferred minimum packet size, bounded by the + /// path MTU that has actually been established for this connection. + fn min_initial_size_for_path_mtu(&self, path_mtu: u16) -> u16 { + self.min_initial_size().min(path_mtu) + } + /// Update PQC state based on peer's transport parameters fn update_from_peer_params(&mut self, params: &TransportParameters) { if let Some(ref algorithms) = params.pqc_algorithms { @@ -7039,9 +7069,31 @@ impl AddressObservationRateLimiter { #[cfg(test)] mod tests { use super::*; - use crate::transport_parameters::AddressDiscoveryConfig; + use crate::transport_parameters::{AddressDiscoveryConfig, PqcAlgorithms}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; + #[test] + fn pqc_min_initial_size_is_capped_by_path_mtu() { + let mut state = PqcState::new(); + let params = TransportParameters { + pqc_algorithms: Some(PqcAlgorithms { + ml_kem_768: true, + ml_dsa_65: true, + }), + ..TransportParameters::default() + }; + + state.update_from_peer_params(¶ms); + + assert_eq!(state.min_initial_size(), 4096); + assert_eq!( + state.min_initial_size_for_path_mtu(INITIAL_MTU), + INITIAL_MTU + ); + assert_eq!(state.min_initial_size_for_path_mtu(1452), 1452); + assert_eq!(state.min_initial_size_for_path_mtu(9000), 4096); + } + #[test] fn address_discovery_state_new() { let config = crate::transport_parameters::AddressDiscoveryConfig::SendAndReceive; diff --git a/src/masque/mod.rs b/src/masque/mod.rs index eff8d393..6773be6f 100644 --- a/src/masque/mod.rs +++ b/src/masque/mod.rs @@ -100,6 +100,7 @@ pub mod relay_client; pub mod relay_server; pub mod relay_session; pub mod relay_socket; +pub(crate) mod tunnel_control; // Re-export primary types for convenience pub use capsule::{ diff --git a/src/masque/relay_server.rs b/src/masque/relay_server.rs index 692db8d6..48645fd9 100644 --- a/src/masque/relay_server.rs +++ b/src/masque/relay_server.rs @@ -41,9 +41,13 @@ use std::time::{Duration, Instant}; use tokio::net::UdpSocket; use tokio::sync::RwLock; +#[cfg(any(target_os = "linux", target_os = "macos", target_os = "freebsd"))] +use std::os::unix::io::AsRawFd; + use crate::VarInt; use crate::high_level::Connection as QuicConnection; use crate::masque::ip_policy::IpPolicy; +use crate::masque::tunnel_control::{CONTROL_FRAME_MARKER, TunnelControlFrame}; use crate::masque::{ Capsule, ConnectUdpRequest, ConnectUdpResponse, Datagram, RelaySession, RelaySessionConfig, RelaySessionState, UncompressedDatagram, @@ -65,6 +69,130 @@ const RELAY_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(15); /// (208 KB default on Linux, which holds only ~170 packets). const RELAY_FORWARD_CHANNEL_CAPACITY: usize = 8192; +/// Suggested PMTU sent to the relay-client when the egress UDP send +/// fails with `EMSGSIZE`. Picked to be QUIC's mandatory minimum +/// datagram size (1200 bytes), which any conformant path must carry. +/// Real path MTUs are usually higher; the relay-client's Quinn DPLPMTUD +/// can probe upward from there once it lowers to this floor. +const PMTU_FALLBACK_HINT: u16 = 1200; + +/// Set the local "don't fragment" bit on a freshly-bound UDP socket so +/// that oversized [`UdpSocket::send_to`] calls fail with `EMSGSIZE` +/// instead of being silently fragmented at the IP layer. Without this +/// the kernel happily fragments the egress datagram, the user-side +/// path then drops the fragments (most home NATs / routers refuse +/// fragmented UDP), and the relay-server cannot tell that the path +/// rejected the packet — which is exactly the false-success that lets +/// Quinn's PMTU estimate stay too high. +/// +/// Returns the underlying I/O error so the caller can decide whether +/// to bail or proceed with default fragmentation behaviour. +#[cfg(target_os = "linux")] +fn set_dont_fragment(socket: &UdpSocket) -> std::io::Result<()> { + let fd = socket.as_raw_fd(); + + // Linux: opt into kernel-level PMTU discovery. IP_PMTUDISC_DO + // forces DF=1 on every outbound IPv4 datagram and surfaces + // EMSGSIZE on the send_to that exceeds the path MTU. + let v4_val: libc::c_int = libc::IP_PMTUDISC_DO; + let rc = unsafe { + libc::setsockopt( + fd, + libc::IPPROTO_IP, + libc::IP_MTU_DISCOVER, + std::ptr::from_ref(&v4_val).cast::(), + std::mem::size_of_val(&v4_val) as libc::socklen_t, + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + + // Same for IPv6 — best effort: the socket may be v4-only, in + // which case the setsockopt fails with ENOPROTOOPT and that's + // fine. We deliberately ignore the result to keep the v4 path + // working on dual-stack-incapable kernels. + let v6_val: libc::c_int = libc::IPV6_PMTUDISC_DO; + let _ = unsafe { + libc::setsockopt( + fd, + libc::IPPROTO_IPV6, + libc::IPV6_MTU_DISCOVER, + std::ptr::from_ref(&v6_val).cast::(), + std::mem::size_of_val(&v6_val) as libc::socklen_t, + ) + }; + + Ok(()) +} + +#[cfg(any(target_os = "macos", target_os = "freebsd"))] +fn set_dont_fragment(socket: &UdpSocket) -> std::io::Result<()> { + // BSD-derived kernels expose a simple boolean IP_DONTFRAG / + // IPV6_DONTFRAG. Behaviour matches Linux's IP_PMTUDISC_DO: DF=1 + // on outbound, EMSGSIZE on too-big. + let fd = socket.as_raw_fd(); + let on: libc::c_int = 1; + + let rc = unsafe { + libc::setsockopt( + fd, + libc::IPPROTO_IP, + libc::IP_DONTFRAG, + std::ptr::from_ref(&on).cast::(), + std::mem::size_of_val(&on) as libc::socklen_t, + ) + }; + if rc != 0 { + return Err(std::io::Error::last_os_error()); + } + + let _ = unsafe { + libc::setsockopt( + fd, + libc::IPPROTO_IPV6, + libc::IPV6_DONTFRAG, + std::ptr::from_ref(&on).cast::(), + std::mem::size_of_val(&on) as libc::socklen_t, + ) + }; + + Ok(()) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "freebsd")))] +fn set_dont_fragment(_socket: &UdpSocket) -> std::io::Result<()> { + // Other platforms: best-effort no-op. PMTU discovery falls back + // to default kernel behaviour (silent fragmentation) and the + // tunnel-level PMTU control frame loop never fires. + Ok(()) +} + +/// Did this `send_to` failure mean "datagram too large for path"? +/// Linux returns `EMSGSIZE` (errno 90); BSD returns `EMSGSIZE` as well. +/// Treated as the only signal that warrants emitting a PMTU control +/// frame back through the tunnel. +fn is_message_too_large(err: &std::io::Error) -> bool { + err.raw_os_error() == Some(libc::EMSGSIZE) +} + +/// Item carried over the bounded channel between the UDP reader task +/// and the QUIC stream writer task in [`MasqueRelayServer::run_stream_forwarding_loop`]. +/// +/// Both data frames (UDP arriving on the bound socket) and control +/// frames (out-of-band tunnel-level signals such as PMTU updates) +/// share the writer task so frame ordering is preserved and the +/// keepalive timer treats both equally. +enum WriterItem { + /// A length-prefixed [`UncompressedDatagram`]-encoded payload + /// that originated from a Direction-1 UDP recv. + Data(Bytes), + /// A control frame body (everything after the + /// `[CONTROL_FRAME_MARKER][body_len]` header). The writer + /// prepends the header before sending. + Control(Bytes), +} + /// Configuration for the MASQUE relay server #[derive(Debug, Clone)] pub struct MasqueRelayConfig { @@ -575,6 +703,22 @@ impl MasqueRelayServer { }, })?; + // Force DF=1 on the bound socket so oversized egress send_to + // fails with EMSGSIZE rather than getting silently fragmented. + // The error then drives a PmtuUpdate control frame back to the + // relay-client (see [`run_stream_forwarding_loop`]). If the + // setsockopt itself fails (very old kernel, exotic platform), + // we log and proceed: PMTU control frames will simply never + // fire and the relay falls back to the legacy lossy behaviour. + if let Err(e) = set_dont_fragment(&udp_socket) { + tracing::warn!( + error = %e, + "Failed to enable IP_DONTFRAG on relay-allocated socket — \ + oversized egress will silently fragment instead of \ + surfacing PMTU feedback" + ); + } + let bound_port = udp_socket .local_addr() .map_err(|e| RelayError::SessionError { @@ -1073,8 +1217,14 @@ impl MasqueRelayServer { // acts as a userspace buffer (~10 MB at capacity) that absorbs // full max-message-size bursts that would otherwise overflow the // kernel's tiny 208 KB UDP receive buffer. + // + // The channel carries a tagged item rather than raw bytes so + // the same writer can interleave normal data frames (Direction + // 1's forwarded UDP) with out-of-band control frames emitted + // by Direction 2 when its egress send_to fails with EMSGSIZE. let (fwd_tx, mut fwd_rx) = - tokio::sync::mpsc::channel::(RELAY_FORWARD_CHANNEL_CAPACITY); + tokio::sync::mpsc::channel::(RELAY_FORWARD_CHANNEL_CAPACITY); + let ctrl_tx = fwd_tx.clone(); // Reader: UDP socket → channel (never blocked by stream writes) let reader_handle = tokio::spawn(async move { @@ -1092,7 +1242,7 @@ impl MasqueRelayServer { let encoded = datagram.encode(); stats.record_bytes(encoded.len() as u64); stats.record_datagram(); - if fwd_tx.send(encoded).await.is_err() { + if fwd_tx.send(WriterItem::Data(encoded)).await.is_err() { break; // writer closed } } @@ -1112,15 +1262,38 @@ impl MasqueRelayServer { loop { tokio::select! { item = fwd_rx.recv() => { - let Some(encoded) = item else { break }; - let frame_len = encoded.len() as u32; - if let Err(e) = send_stream.write_all(&frame_len.to_be_bytes()).await { - tracing::debug!(session_id, error = %e, "Stream write error (length)"); - break; - } - if let Err(e) = send_stream.write_all(&encoded).await { - tracing::debug!(session_id, error = %e, "Stream write error (data)"); - break; + let Some(item) = item else { break }; + match item { + WriterItem::Data(encoded) => { + let frame_len = encoded.len() as u32; + if let Err(e) = send_stream.write_all(&frame_len.to_be_bytes()).await { + tracing::debug!(session_id, error = %e, "Stream write error (length)"); + break; + } + if let Err(e) = send_stream.write_all(&encoded).await { + tracing::debug!(session_id, error = %e, "Stream write error (data)"); + break; + } + } + WriterItem::Control(body) => { + // Wire format: [4-byte BE marker][4-byte BE body_len][body] + let body_len = body.len() as u32; + if let Err(e) = send_stream + .write_all(&CONTROL_FRAME_MARKER.to_be_bytes()) + .await + { + tracing::debug!(session_id, error = %e, "Stream write error (control marker)"); + break; + } + if let Err(e) = send_stream.write_all(&body_len.to_be_bytes()).await { + tracing::debug!(session_id, error = %e, "Stream write error (control len)"); + break; + } + if let Err(e) = send_stream.write_all(&body).await { + tracing::debug!(session_id, error = %e, "Stream write error (control body)"); + break; + } + } } } _ = keepalive.tick() => { @@ -1197,6 +1370,37 @@ impl MasqueRelayServer { "RELAY_TUNNEL[srv]: stream-loop dir2 sendto OK" ); } + Err(e) if is_message_too_large(&e) => { + // Path-MTU exceeded. Emit a PmtuUpdate + // control frame back through the tunnel + // so the relay-client's MasqueRelaySocket + // can clamp future sends to this target — + // effectively forcing Quinn's DPLPMTUD + // to lower the connection's MTU estimate. + tracing::debug!( + session_id, + target = %target, + len = payload_len, + suggested_mtu = PMTU_FALLBACK_HINT, + "RELAY_TUNNEL[srv]: stream-loop dir2 EMSGSIZE → emitting PmtuUpdate" + ); + let body = TunnelControlFrame::PmtuUpdate { + target, + mtu: PMTU_FALLBACK_HINT, + } + .encode_body(); + // Best-effort: if the writer is gone or + // its bounded queue is full, avoid + // stalling the client-to-target path. + if let Err(e) = ctrl_tx.try_send(WriterItem::Control(body)) { + tracing::debug!( + session_id, + target = %target, + error = %e, + "RELAY_TUNNEL[srv]: dropped PmtuUpdate control frame" + ); + } + } Err(e) => { tracing::warn!( session_id, target = %target, error = %e, diff --git a/src/masque/relay_socket.rs b/src/masque/relay_socket.rs index 9edc83cd..165aede4 100644 --- a/src/masque/relay_socket.rs +++ b/src/masque/relay_socket.rs @@ -38,6 +38,7 @@ //! when the tunnel cannot keep up. use bytes::Bytes; +use dashmap::DashMap; use parking_lot::Mutex as PlMutex; use std::fmt; use std::future::Future; @@ -54,6 +55,9 @@ use quinn_udp::{RecvMeta, Transmit}; use crate::VarInt; use crate::high_level::{AsyncUdpSocket, UdpPoller}; use crate::masque::UncompressedDatagram; +use crate::masque::tunnel_control::{ + CONTROL_FRAME_MARKER, MAX_CONTROL_FRAME_BODY, TunnelControlFrame, +}; /// Interval at which the relay client sends a zero-length keepalive /// frame through the relay stream. Must be shorter than the NAT @@ -113,6 +117,15 @@ pub struct MasqueRelaySocket { /// `notify_waiters`) so a drain that races with a poller entering /// the wait state stores a permit, avoiding lost wakeups. send_capacity_freed: Arc, + /// Per-target maximum payload size enforced by [`Self::try_send`], + /// populated by [`TunnelControlFrame::PmtuUpdate`] frames decoded by + /// the reader task. When a destination has an entry, any + /// [`Transmit`] whose `contents.len()` exceeds the cap is silently + /// dropped at try_send time, simulating packet loss for Quinn's + /// DPLPMTUD machinery so the inner connection's MTU estimate + /// converges to the true egress path MTU. Targets without an + /// entry are unconstrained by this layer (Quinn governs sizing). + target_mtu: Arc>, /// The original socket is kept alive so the relay connection's own /// QUIC traffic (keepalives, ACKs, stream data) continues to flow /// directly. Without this reference the OS may reclaim the socket. @@ -165,11 +178,15 @@ impl MasqueRelaySocket { let closed = Arc::new(Notify::new()); let send_capacity_freed = Arc::new(Notify::new()); + let target_mtu: Arc> = Arc::new(DashMap::new()); + let target_mtu_reader = Arc::clone(&target_mtu); + let socket = Arc::new(Self { relay_public_addr, recv_rx: PlMutex::new(recv_rx), send_tx: send_tx.clone(), send_capacity_freed: Arc::clone(&send_capacity_freed), + target_mtu, _original_socket: original_socket, }); @@ -184,12 +201,61 @@ impl MasqueRelaySocket { tracing::debug!(error = %e, "MasqueRelaySocket: stream read error (length)"); break; } - let frame_len = u32::from_be_bytes(len_buf) as usize; + let frame_len = u32::from_be_bytes(len_buf); + // Zero-length frame = keepalive ping from the relay // server, skip without trying to decode a datagram. if frame_len == 0 { continue; } + + // Sentinel marker for an out-of-band control frame. + // Wire layout: + // [4-byte BE CONTROL_FRAME_MARKER] + // [4-byte BE body_len] + // [body_len bytes body] + if frame_len == CONTROL_FRAME_MARKER { + let mut body_len_buf = [0u8; 4]; + if let Err(e) = recv_stream.read_exact(&mut body_len_buf).await { + tracing::debug!(error = %e, "MasqueRelaySocket: control frame read error (body_len)"); + break; + } + let body_len = u32::from_be_bytes(body_len_buf); + if body_len > MAX_CONTROL_FRAME_BODY { + tracing::warn!( + body_len, + cap = MAX_CONTROL_FRAME_BODY, + "MasqueRelaySocket: control frame body too large, closing" + ); + break; + } + let mut body = vec![0u8; body_len as usize]; + if let Err(e) = recv_stream.read_exact(&mut body).await { + tracing::debug!(error = %e, "MasqueRelaySocket: control frame read error (body)"); + break; + } + match TunnelControlFrame::decode_body(&body) { + Some(TunnelControlFrame::PmtuUpdate { target, mtu }) => { + tracing::debug!( + relay = %relay_public_addr, + target = %target, + mtu, + "RELAY_TUNNEL[clt]: PmtuUpdate received → clamping per-target MTU" + ); + target_mtu_reader.insert(target, mtu); + } + None => { + tracing::debug!( + relay = %relay_public_addr, + body_len, + "RELAY_TUNNEL[clt]: unknown / malformed control frame, ignoring" + ); + } + } + continue; + } + + let frame_len = frame_len as usize; if frame_len > MAX_RELAY_FRAME { tracing::warn!(frame_len, "MasqueRelaySocket: corrupt frame length"); break; @@ -351,6 +417,30 @@ impl AsyncUdpSocket for MasqueRelaySocket { segment_size = ?transmit.segment_size, "RELAY_TUNNEL[clt]: try_send → enqueue outbound for relay-server" ); + + // Per-target MTU enforcement: if a previous PmtuUpdate control + // frame told us the egress path to this destination caps at + // `mtu` bytes, drop oversized packets here so they never reach + // the relay-server's fragmentation-rejecting socket. Returning + // `Ok(())` makes Quinn treat the packet as successfully sent; + // its loss-detection then observes the missing ACK and lowers + // the connection's MTU estimate via DPLPMTUD's normal path. + // We do NOT return an Err here because that would skip Quinn's + // PMTUD machinery entirely and leave the size unchanged. + if let Some(cap) = self.target_mtu.get(&transmit.destination) { + let segment = transmit.segment_size.unwrap_or(transmit.contents.len()); + if segment > usize::from(*cap) { + tracing::debug!( + relay = %self.relay_public_addr, + destination = %transmit.destination, + segment, + cap = *cap, + "RELAY_TUNNEL[clt]: try_send dropping oversized packet (per-target MTU exceeded)" + ); + return Ok(()); + } + } + if let Some(segment_size) = transmit.segment_size { for chunk in transmit.contents.chunks(segment_size) { let datagram = UncompressedDatagram::new( diff --git a/src/masque/tunnel_control.rs b/src/masque/tunnel_control.rs new file mode 100644 index 00000000..c014129d --- /dev/null +++ b/src/masque/tunnel_control.rs @@ -0,0 +1,199 @@ +// Copyright 2024 Saorsa Labs Ltd. +// +// This Saorsa Network Software is licensed under the General Public License (GPL), version 3. +// Please see the file LICENSE-GPL, or visit for the full text. +// +// Full details available at https://saorsalabs.com/licenses + +//! Out-of-band control frames carried over the MASQUE relay tunnel. +//! +//! The data plane wraps every payload in an [`UncompressedDatagram`] +//! prefixed by a 4-byte big-endian length. Any frame whose length +//! prefix equals the sentinel value [`CONTROL_FRAME_MARKER`] is a +//! control frame instead — the next four bytes are the body length, +//! followed by a 1-byte type tag and a type-specific payload. +//! +//! Currently only one control frame type exists: [`PmtuUpdate`], sent +//! from the relay-server to the relay-client when the relay's egress +//! UDP send to a third party fails with `EMSGSIZE`. The relay-client's +//! [`crate::masque::MasqueRelaySocket`] then enforces the suggested +//! MTU on subsequent [`AsyncUdpSocket::try_send`] calls to that target, +//! simulating packet loss for Quinn's DPLPMTUD machinery so that the +//! inner connection's MTU estimate converges to the path reality +//! without an explicit Quinn-level API. +//! +//! [`UncompressedDatagram`]: crate::masque::UncompressedDatagram +//! [`AsyncUdpSocket::try_send`]: crate::high_level::AsyncUdpSocket::try_send +//! [`PmtuUpdate`]: TunnelControlFrame::PmtuUpdate + +use bytes::{BufMut, Bytes, BytesMut}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; + +/// Sentinel length value that marks a control frame on the wire. +/// Chosen above any plausible data-frame length (the existing data +/// plane caps frames at 512 KiB), so a peer running an older build +/// will treat it as a corrupt-length error and tear the tunnel down +/// — which is the right outcome for a feature-mismatched session. +pub(crate) const CONTROL_FRAME_MARKER: u32 = 0xFFFF_FFFF; + +/// Type tag for a path-MTU update control frame. +const CTRL_TYPE_PMTU_UPDATE: u8 = 0x01; + +/// Address-family tag for the [`SocketAddr`] encoding inside a control +/// frame body. Only IPv4 (`4`) and IPv6 (`6`) are defined. +const ADDR_FAMILY_V4: u8 = 4; +const ADDR_FAMILY_V6: u8 = 6; + +/// One-byte type tag + worst-case body for the largest defined frame +/// (PmtuUpdate over IPv6: 1 family + 16 addr + 2 port + 2 mtu = 21, +/// plus the type tag = 22). Used as a safety cap on inbound control +/// frame length on the relay-client side so a malformed frame can't +/// allocate huge buffers. +pub(crate) const MAX_CONTROL_FRAME_BODY: u32 = 64; + +/// A control frame carried over the MASQUE relay tunnel out-of-band +/// from data datagrams. See the module-level docs for the wire format. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum TunnelControlFrame { + /// Relay-server tells the relay-client: "the egress path to + /// `target` rejected my last datagram for being too large — please + /// stop generating packets larger than `mtu` bytes for that + /// target, so Quinn's DPLPMTUD lowers the connection MTU." + PmtuUpdate { target: SocketAddr, mtu: u16 }, +} + +impl TunnelControlFrame { + /// Encode the body of the frame (everything after the + /// `[CONTROL_FRAME_MARKER][body_len]` header). + pub(crate) fn encode_body(&self) -> Bytes { + match self { + Self::PmtuUpdate { target, mtu } => { + let mut buf = BytesMut::with_capacity(32); + buf.put_u8(CTRL_TYPE_PMTU_UPDATE); + encode_socket_addr(&mut buf, *target); + buf.put_u16(*mtu); + buf.freeze() + } + } + } + + /// Decode the body of a control frame (everything after the + /// `[CONTROL_FRAME_MARKER][body_len]` header). Returns `None` for + /// unknown type tags or malformed bodies — callers should log and + /// skip rather than tearing down the tunnel, so introducing a new + /// control-frame type is not itself a breaking wire-format change. + pub(crate) fn decode_body(body: &[u8]) -> Option { + let (ctype, mut rest) = body.split_first()?; + match *ctype { + CTRL_TYPE_PMTU_UPDATE => { + let target = decode_socket_addr(&mut rest)?; + if rest.len() < 2 { + return None; + } + let mtu = u16::from_be_bytes([rest[0], rest[1]]); + Some(Self::PmtuUpdate { target, mtu }) + } + _ => None, + } + } +} + +fn encode_socket_addr(buf: &mut BytesMut, addr: SocketAddr) { + match addr { + SocketAddr::V4(v4) => { + buf.put_u8(ADDR_FAMILY_V4); + buf.put_slice(&v4.ip().octets()); + buf.put_u16(v4.port()); + } + SocketAddr::V6(v6) => { + buf.put_u8(ADDR_FAMILY_V6); + buf.put_slice(&v6.ip().octets()); + buf.put_u16(v6.port()); + } + } +} + +fn decode_socket_addr(buf: &mut &[u8]) -> Option { + let (family, rest) = buf.split_first()?; + *buf = rest; + match *family { + ADDR_FAMILY_V4 => { + if buf.len() < 6 { + return None; + } + let ip = Ipv4Addr::new(buf[0], buf[1], buf[2], buf[3]); + let port = u16::from_be_bytes([buf[4], buf[5]]); + *buf = &buf[6..]; + Some(SocketAddr::new(IpAddr::V4(ip), port)) + } + ADDR_FAMILY_V6 => { + if buf.len() < 18 { + return None; + } + let mut octets = [0u8; 16]; + octets.copy_from_slice(&buf[..16]); + let ip = Ipv6Addr::from(octets); + let port = u16::from_be_bytes([buf[16], buf[17]]); + *buf = &buf[18..]; + Some(SocketAddr::new(IpAddr::V6(ip), port)) + } + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn v4(a: u8, b: u8, c: u8, d: u8, port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(a, b, c, d)), port) + } + + fn v6_loopback(port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), port) + } + + #[test] + fn roundtrip_pmtu_update_v4() { + let frame = TunnelControlFrame::PmtuUpdate { + target: v4(192, 0, 2, 1, 9000), + mtu: 1252, + }; + let body = frame.encode_body(); + let decoded = TunnelControlFrame::decode_body(&body).expect("decode"); + assert_eq!(frame, decoded); + } + + #[test] + fn roundtrip_pmtu_update_v6() { + let frame = TunnelControlFrame::PmtuUpdate { + target: v6_loopback(443), + mtu: 1452, + }; + let body = frame.encode_body(); + let decoded = TunnelControlFrame::decode_body(&body).expect("decode"); + assert_eq!(frame, decoded); + } + + #[test] + fn unknown_type_returns_none() { + // Body whose first byte is an undefined type tag. + let body = [0xFE_u8, 0, 0, 0, 0]; + assert!(TunnelControlFrame::decode_body(&body).is_none()); + } + + #[test] + fn truncated_body_returns_none() { + let frame = TunnelControlFrame::PmtuUpdate { + target: v4(127, 0, 0, 1, 1234), + mtu: 1200, + }; + let body = frame.encode_body(); + for short_len in 0..body.len() { + assert!( + TunnelControlFrame::decode_body(&body[..short_len]).is_none(), + "truncated to {short_len} bytes should not decode" + ); + } + } +} diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 50cacad5..fac911cf 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -34,6 +34,22 @@ const MAX_RELAY_CLIENTS_PER_PUBLIC_PEER: usize = 4; /// rather than retrying against the same one. See [`NatTraversalError::RelayAtCapacity`]. const MASQUE_RELAY_FULL_STATUS: u16 = 503; +/// Initial UDP payload size for connections accepted through a MASQUE +/// relay tunnel. Set to QUIC's mandatory minimum (1200 bytes) so the +/// inner endpoint never produces a coalesced packet larger than what +/// any conformant Internet path can carry; DPLPMTUD is then enabled +/// to probe upward as the per-target PMTU map (populated by +/// [`crate::masque::tunnel_control::TunnelControlFrame::PmtuUpdate`] +/// frames) refines the egress cap. This neutralises the +/// "server-flight 4KB coalesced packet" first-packet failure mode +/// observed when relay-tunnelled handshakes inherited the regular +/// endpoint's higher MTU estimate. +/// +/// Shared with the client-side dial-through-relay path in +/// `p2p_endpoint.rs` so the same MTU discipline applies to both +/// directions of relay-tunnelled traffic. +pub(crate) const RELAY_TUNNEL_INITIAL_MTU: u16 = 1200; + /// Buffer for completed handshakes. Sized above the peak backlog observed /// in the ant-rc-18 testnet (1 079 connections) so transient consumer /// stalls don't block the accept loop. @@ -282,7 +298,7 @@ use crate::{ }; use crate::{ - ClientConfig, EndpointConfig, ServerConfig, Side, TransportConfig, + ClientConfig, EndpointConfig, MtuDiscoveryConfig, ServerConfig, Side, TransportConfig, high_level::{Connection as InnerConnection, Endpoint as InnerEndpoint}, }; @@ -4698,6 +4714,20 @@ impl NatTraversalEndpoint { .map_err(|_| NatTraversalError::ConfigError("mutex poisoned".to_string()))? .clone(); + // Override the inner endpoint's TransportConfig with a + // relay-tunnel-safe MTU profile while preserving every other + // setting the main endpoint cares about (NAT traversal, + // congestion control, idle timeouts, flow control). We clone + // the existing transport config, clamp MTU, and re-attach. + let server_config = server_config.map(|mut sc| { + let mut tunnel_tc = (*sc.transport).clone(); + tunnel_tc.initial_mtu(RELAY_TUNNEL_INITIAL_MTU); + tunnel_tc.min_mtu(RELAY_TUNNEL_INITIAL_MTU); + tunnel_tc.mtu_discovery_config(Some(MtuDiscoveryConfig::default())); + sc.transport = Arc::new(tunnel_tc); + sc + }); + let runtime = crate::high_level::default_runtime().ok_or_else(|| { NatTraversalError::ConfigError("No async runtime available".to_string()) })?; diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 5003967b..bc2120f6 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -59,6 +59,7 @@ use tokio::time::timeout; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; +use crate::MtuDiscoveryConfig; use crate::Side; use crate::bootstrap_cache::{BootstrapCache, BootstrapTokenStore}; use crate::bounded_pending_buffer::BoundedPendingBuffer; @@ -73,6 +74,7 @@ use crate::happy_eyeballs::{self, HappyEyeballsConfig}; pub use crate::nat_traversal_api::TraversalPhase; use crate::nat_traversal_api::{ NatTraversalEndpoint, NatTraversalError, NatTraversalEvent, NatTraversalStatistics, + RELAY_TUNNEL_INITIAL_MTU, }; use crate::transport::{ProtocolEngine, TransportAddr, TransportRegistry}; use crate::unified_config::P2pConfig; @@ -2113,11 +2115,26 @@ impl P2pEndpoint { original_socket, ); - let client_config = existing_endpoint + let mut client_config = existing_endpoint .default_client_config .clone() .ok_or_else(|| EndpointError::Config("No client config available".to_string()))?; + // Clamp the relay-tunnelled dialer's MTU to QUIC's safe + // minimum so the very first Initial+Handshake we coalesce can + // never exceed any real Internet egress path that the relay + // hands off to. DPLPMTUD then probes upward as + // `TunnelControlFrame::PmtuUpdate` (out-of-band tunnel + // control frames) trim the per-target cap on the relay side. + // Other transport settings — congestion control, idle timeout, + // flow control windows — are inherited from the existing + // client config. + let mut tunnel_tc = (*client_config.transport).clone(); + tunnel_tc.initial_mtu(RELAY_TUNNEL_INITIAL_MTU); + tunnel_tc.min_mtu(RELAY_TUNNEL_INITIAL_MTU); + tunnel_tc.mtu_discovery_config(Some(MtuDiscoveryConfig::default())); + client_config.transport_config(Arc::new(tunnel_tc)); + let runtime = crate::high_level::default_runtime() .ok_or_else(|| EndpointError::Config("No async runtime available".to_string()))?; @@ -2493,15 +2510,18 @@ impl P2pEndpoint { EndpointError::Connection(e.to_string()) })?; - // Wait for the peer to acknowledge receipt of all stream data. - // Without this, finish() only buffers a FIN locally — if the - // connection is dead the caller would see Ok(()) despite the - // data never arriving. + // Give Quinn a short chance to observe stream completion. + // `finish()` only queues the FIN locally, while `stopped()` + // resolves when all stream data has been acknowledged or the + // peer explicitly stops the stream. A missing ACK within this + // local window is not proof of delivery failure, especially on + // busy testnets where ACKs can be delayed behind other work. // - // The base timeout handles small messages and dead-connection - // detection. For large payloads we add time proportional to - // size at an assumed 256 KB/s per connection — conservative - // enough for concurrent uploads sharing the uplink. + // Keep hard errors for explicit stop/connection loss, but + // treat timeout as "queued to QUIC" and let later connection + // state or application-level request timeouts handle retries. + // For large payloads we add time proportional to size at an + // assumed 256 KB/s per connection. let base_timeout = self.config.timeouts.send_ack_timeout; let size_budget = std::time::Duration::from_millis((data.len() as u64).saturating_div(256)); @@ -2519,9 +2539,10 @@ impl P2pEndpoint { ))); } Err(_elapsed) => { - return Err(EndpointError::Connection(format!( - "peer did not acknowledge stream data within {ack_timeout:?}" - ))); + debug!( + "send({}): stream data queued but not fully acknowledged within {:?}", + addr, ack_timeout + ); } } From 0abe007bb30b6fcb1df485b4f72c02e98661064e Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Fri, 1 May 2026 23:01:39 +0200 Subject: [PATCH 04/14] feat(reachability): expose per-(peer, observation) signal via PeerObservedExternal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `NatTraversalEvent::PeerObservedExternal { peer_addr, observed_external }` and the corresponding `P2pEvent::PeerObservedExternal`, fired on every distinct `(peer, observed-address)` pair from the OBSERVED_ADDRESS path — both the regular discovery loop and the bootstrap reflexive-candidate path. The signal is emitted regardless of whether saorsa-transport's pinning quorum (`MIN_OBSERVERS_FOR_QUORUM = 2`) has been reached. Upper layers can now do per-address attribution of cold-dialability proof: the peer's frame is their own statement that they reached us at a specific external. Saorsa-core consumes this in its passive reachability classifier (separate PR) to credit only the externals each inbound peer actually reported observing — closing a per-family-leak in the previous design where two source-disjoint inbounds promoted every same-family pinned external, including stale NAT rebindings and unrelated externals. Additive on the public enums; `Node::convert_event` adds the new variant to its already-ignored arm. Other matchers in saorsa-transport already used catch-all arms. Tests: cargo fmt clean, cargo clippy --all-targets --all-features -- -D warnings clean, 1506 lib tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/nat_traversal_api.rs | 33 +++++++++++++++++++++++++++++++++ src/node.rs | 1 + src/p2p_endpoint.rs | 25 +++++++++++++++++++++++++ 5 files changed, 61 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ce48086..ceabea73 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3012,7 +3012,7 @@ dependencies = [ [[package]] name = "saorsa-transport" -version = "0.33.0" +version = "0.34.0" dependencies = [ "anyhow", "arbitrary", diff --git a/Cargo.toml b/Cargo.toml index 80376215..6e73865c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ [package] name = "saorsa-transport" -version = "0.33.0" +version = "0.34.0" edition = "2024" rust-version = "1.88.0" license = "MIT OR Apache-2.0" diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index fac911cf..1c442c22 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -1237,6 +1237,20 @@ pub enum NatTraversalEvent { /// Our observed external address address: SocketAddr, }, + /// A peer reported an OBSERVED_ADDRESS observation of us, before + /// `MIN_OBSERVERS_FOR_QUORUM` was necessarily reached. + /// + /// Emitted on the first observation from each peer for a given external + /// address (i.e. once per (peer, observed-address) pair). Allows upper + /// layers to do per-address attribution of cold-dialability proof — + /// the peer's frame is their own statement that they reached us at + /// `observed_external`. + PeerObservedExternal { + /// The peer that sent the OBSERVED_ADDRESS frame + peer_addr: SocketAddr, + /// The external address the peer reports observing us at + observed_external: SocketAddr, + }, /// A connected peer advertised a new reachable address (ADD_ADDRESS frame). /// /// The upper layer should update its routing table so that future lookups @@ -3501,6 +3515,18 @@ impl NatTraversalEndpoint { ); } + // Surface every distinct (peer, observed) pair so upper + // layers can attribute cold-dialability proof to the + // specific external the peer reached us at — independent + // of whether quorum is reached. Required for + // saorsa-core's per-address `proven_externals` model. + if check.new_observer { + let _ = event_tx.send(NatTraversalEvent::PeerObservedExternal { + peer_addr: remote_addr, + observed_external: observed_addr, + }); + } + // Broadcast ADD_ADDRESS on the *first* observer, not the // quorum cross. Rationale: peers need our external // address early to coordinate hole-punches back to us. @@ -3620,6 +3646,13 @@ impl NatTraversalEndpoint { ); } + if check.new_observer { + let _ = event_tx.send(NatTraversalEvent::PeerObservedExternal { + peer_addr: *bootstrap_node, + observed_external: candidate.address, + }); + } + // Broadcast ADD_ADDRESS to connected peers on the // first observer so hole-punch coordination can // start before quorum. See the OBSERVED_ADDRESS diff --git a/src/node.rs b/src/node.rs index a051c70c..889ba3d8 100644 --- a/src/node.rs +++ b/src/node.rs @@ -359,6 +359,7 @@ impl Node { | P2pEvent::BootstrapStatus { .. } | P2pEvent::PeerAuthenticated { .. } | P2pEvent::PeerAddressUpdated { .. } + | P2pEvent::PeerObservedExternal { .. } | P2pEvent::RelayEstablished { .. } | P2pEvent::RelayLost { .. } => None, } diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index bc2120f6..95f53601 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -431,6 +431,22 @@ pub enum P2pEvent { addr: TransportAddr, }, + /// A connected peer reported observing one of our external addresses + /// via an OBSERVED_ADDRESS frame. + /// + /// Fires on the first observation from each peer for a given external + /// address (once per `(peer, observed_external)` pair), regardless of + /// whether saorsa-transport's pinning quorum has been reached. Upper + /// layers can use this for per-address attribution of cold-dialability + /// proof: the peer is stating, with their own observation, that they + /// reached us at `observed_external`. + PeerObservedExternal { + /// The peer's socket address (the source of the OBSERVED_ADDRESS frame) + peer_addr: SocketAddr, + /// The external address the peer reports observing us at + observed_external: SocketAddr, + }, + /// A connected peer advertised a new reachable address (relay or migration). PeerAddressUpdated { /// The connected peer that sent the advertisement @@ -770,6 +786,15 @@ impl P2pEndpoint { addr: TransportAddr::Quic(*address), }); } + NatTraversalEvent::PeerObservedExternal { + peer_addr, + observed_external, + } => { + let _ = event_tx.send(P2pEvent::PeerObservedExternal { + peer_addr: *peer_addr, + observed_external: *observed_external, + }); + } _ => {} } drop(stats_guard); From 547574e53c2b6d3ec31d82860fc20cebf42fe5f5 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sat, 2 May 2026 20:09:36 +0200 Subject: [PATCH 05/14] fix(p2p): normalize connection cleanup and backpressure large sends --- src/connection/mod.rs | 9 ++ src/connection_strategy.rs | 11 +- src/endpoint.rs | 10 +- src/happy_eyeballs.rs | 103 ++++++++++++-- src/high_level/endpoint.rs | 15 +-- src/nat_traversal_api.rs | 183 ++++++++++++++++--------- src/p2p_endpoint.rs | 267 +++++++++++++++++++++---------------- src/shared.rs | 31 +++++ 8 files changed, 417 insertions(+), 212 deletions(-) diff --git a/src/connection/mod.rs b/src/connection/mod.rs index c32e87c2..73e79106 100644 --- a/src/connection/mod.rs +++ b/src/connection/mod.rs @@ -4916,12 +4916,21 @@ impl Connection { // Normalize the address to handle IPv4-mapped IPv6 addresses // This is critical for nodes bound to IPv4-only sockets let normalized_addr = crate::shared::normalize_socket_addr(add_address.address); + let peer_addr = crate::shared::normalize_socket_addr(self.path.remote); info!( "handle_add_address: RECEIVED ADD_ADDRESS from peer addr={} (normalized={}) seq={} priority={}", add_address.address, normalized_addr, add_address.sequence, add_address.priority ); + if peer_addr.ip() == normalized_addr.ip() { + debug!( + "handle_add_address: dropping same-IP ADD_ADDRESS from peer={} addr={} seq={}", + peer_addr, normalized_addr, add_address.sequence + ); + return Ok(()); + } + match nat_state.add_remote_candidate( add_address.sequence, normalized_addr, diff --git a/src/connection_strategy.rs b/src/connection_strategy.rs index 2c933c79..0888444b 100644 --- a/src/connection_strategy.rs +++ b/src/connection_strategy.rs @@ -53,11 +53,12 @@ use std::time::{Duration, Instant}; /// Timeout for direct connection attempts (both IPv4 and IPv6). /// Relay-allocated addresses (advertised via the DHT as plain socket /// addresses) are indistinguishable from direct addresses at the -/// transport level. A relay handshake adds one extra RTT through the -/// relay server, so cross-continent relay paths need up to ~1.5 s. -/// 3 s gives comfortable headroom without meaningfully delaying -/// fallback to hole-punching for truly unreachable endpoints. -const DEFAULT_DIRECT_CONNECT_TIMEOUT: Duration = Duration::from_secs(3); +/// transport level. After a congested send timeout, immediately +/// classifying a peer as unreachable on a tiny direct-dial budget creates +/// false negatives. Eight seconds still bounds lookup latency while giving +/// a busy peer or relay path enough time to complete the QUIC + PQC +/// handshake. +const DEFAULT_DIRECT_CONNECT_TIMEOUT: Duration = Duration::from_secs(8); /// How a connection was established #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/endpoint.rs b/src/endpoint.rs index 99cf2417..f3d73d61 100644 --- a/src/endpoint.rs +++ b/src/endpoint.rs @@ -256,24 +256,18 @@ impl Endpoint { pub fn peer_connection_addr(&self, peer_id: &PeerId) -> Option { let handle = self.peer_connections.get(peer_id)?; let meta = self.connections.get(handle.0)?; - Some(meta.addresses.remote) + Some(crate::shared::normalize_socket_addr(meta.addresses.remote)) } /// Find the connection handle for a given remote address. pub fn connection_handle_for_addr(&self, addr: &SocketAddr) -> Option { let normalized = crate::shared::normalize_socket_addr(*addr); - let alt = crate::shared::dual_stack_alternate(addr); for (idx, meta) in self.connections.iter() { - let remote = meta.addresses.remote; + let remote = crate::shared::normalize_socket_addr(meta.addresses.remote); if remote == normalized { return Some(ConnectionHandle(idx)); } - if let Some(ref a) = alt { - if remote == *a { - return Some(ConnectionHandle(idx)); - } - } } None } diff --git a/src/happy_eyeballs.rs b/src/happy_eyeballs.rs index 0f0aedfa..01b694af 100644 --- a/src/happy_eyeballs.rs +++ b/src/happy_eyeballs.rs @@ -229,6 +229,36 @@ enum AttemptResult { Failure(SocketAddr, String), } +/// Join handles for spawned connection attempts. +/// +/// Dropping a `JoinHandle` detaches the task; it does not cancel the QUIC +/// dial. `race_connect` is commonly wrapped in an outer timeout, so this +/// guard ensures timed-out Happy Eyeballs races abort their in-flight +/// handshakes instead of letting orphaned connection drivers complete later. +struct AttemptHandles(Vec>); + +impl AttemptHandles { + fn with_capacity(capacity: usize) -> Self { + Self(Vec::with_capacity(capacity)) + } + + fn push(&mut self, handle: JoinHandle<()>) { + self.0.push(handle); + } + + fn abort_all(&self) { + for handle in &self.0 { + handle.abort(); + } + } +} + +impl Drop for AttemptHandles { + fn drop(&mut self) { + self.abort_all(); + } +} + /// Spawn a single connection attempt as a tokio task. /// /// The task sends its result (success or failure) through the provided channel sender. @@ -319,7 +349,7 @@ where ); let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::>(); - let mut handles: Vec> = Vec::with_capacity(sorted.len()); + let mut handles = AttemptHandles::with_capacity(sorted.len()); let mut errors: Vec<(SocketAddr, String)> = Vec::new(); let mut next_index: usize = 0; let total = sorted.len(); @@ -347,7 +377,7 @@ where match result { Some(AttemptResult::Success(conn, addr)) => { info!(addr = %addr, "Happy Eyeballs: connection succeeded"); - abort_all(&handles); + handles.abort_all(); return Ok((conn, addr)); } Some(AttemptResult::Failure(addr, err)) => { @@ -402,7 +432,7 @@ where match rx.recv().await { Some(AttemptResult::Success(conn, addr)) => { info!(addr = %addr, "Happy Eyeballs: connection succeeded"); - abort_all(&handles); + handles.abort_all(); return Ok((conn, addr)); } Some(AttemptResult::Failure(addr, err)) => { @@ -421,13 +451,6 @@ where Err(HappyEyeballsError::AllAttemptsFailed { errors }) } -/// Abort all spawned task handles. -fn abort_all(handles: &[JoinHandle<()>]) { - for handle in handles { - handle.abort(); - } -} - #[cfg(test)] mod tests { #![allow(clippy::unwrap_used)] @@ -797,6 +820,66 @@ mod tests { assert_eq!(completed.load(Ordering::SeqCst), 1); } + #[tokio::test] + async fn test_race_aborts_in_flight_attempts_when_dropped() { + struct DropGuard { + dropped: Arc, + notify: Arc, + } + + impl Drop for DropGuard { + fn drop(&mut self) { + self.dropped.fetch_add(1, Ordering::SeqCst); + self.notify.notify_one(); + } + } + + let started = Arc::new(tokio::sync::Notify::new()); + let dropped = Arc::new(AtomicUsize::new(0)); + let dropped_notify = Arc::new(tokio::sync::Notify::new()); + + let addrs = vec![v4("10.0.0.1:80")]; + let config = HappyEyeballsConfig { + connection_attempt_delay: Duration::from_secs(60), + ..Default::default() + }; + + { + let started_clone = Arc::clone(&started); + let dropped_clone = Arc::clone(&dropped); + let dropped_notify_clone = Arc::clone(&dropped_notify); + + let race = race_connect(&addrs, &config, move |_addr| { + let started = Arc::clone(&started_clone); + let dropped = Arc::clone(&dropped_clone); + let dropped_notify = Arc::clone(&dropped_notify_clone); + + async move { + let _drop_guard = DropGuard { + dropped, + notify: dropped_notify, + }; + started.notify_one(); + std::future::pending::>().await + } + }); + tokio::pin!(race); + + tokio::select! { + _ = started.notified() => {} + result = &mut race => panic!("race completed unexpectedly: {result:?}"), + _ = tokio::time::sleep(Duration::from_secs(1)) => { + panic!("timed out waiting for connection attempt to start") + } + } + } + + tokio::time::timeout(Duration::from_secs(1), dropped_notify.notified()) + .await + .expect("timed out waiting for aborted connection attempt to be dropped"); + assert_eq!(dropped.load(Ordering::SeqCst), 1); + } + #[test] fn test_error_display() { let err = HappyEyeballsError::NoAddresses; diff --git a/src/high_level/endpoint.rs b/src/high_level/endpoint.rs index b8a4a35b..382244ae 100644 --- a/src/high_level/endpoint.rs +++ b/src/high_level/endpoint.rs @@ -319,18 +319,19 @@ impl Endpoint { ) { if let Ok(mut state) = self.inner.0.state.lock() { // Find the connection handle for this address - let handle = state.inner.connection_handle_for_addr(&addr); + let normalized = crate::shared::normalize_socket_addr(addr); + let handle = state.inner.connection_handle_for_addr(&normalized); if let Some(ch) = handle { state.inner.set_connection_peer_id(ch, peer_id); tracing::info!( "Registered peer ID {} for connection {} at low-level endpoint", hex::encode(&peer_id.0[..8]), - addr + normalized ); } else { tracing::debug!( "No connection handle found for {} — peer ID not registered", - addr + normalized ); } } @@ -501,13 +502,7 @@ impl Endpoint { return None; }; let normalized = crate::shared::normalize_socket_addr(*addr); - let handle = state - .inner - .connection_handle_for_addr(&normalized) - .or_else(|| { - crate::shared::dual_stack_alternate(&normalized) - .and_then(|alt| state.inner.connection_handle_for_addr(&alt)) - }); + let handle = state.inner.connection_handle_for_addr(&normalized); handle.map(|h| state.inner.connection_stable_id(h)) } diff --git a/src/nat_traversal_api.rs b/src/nat_traversal_api.rs index 1c442c22..bcf55516 100644 --- a/src/nat_traversal_api.rs +++ b/src/nat_traversal_api.rs @@ -165,7 +165,7 @@ fn extract_ml_dsa_from_spki(spki: &[u8]) -> Option info!( "accept_connections: sent {} to forwarder channel", - remote_address + connection_key ), Err(e) => error!( "accept_connections: forwarder channel send FAILED for {}: {}", - remote_address, e + connection_key, e ), } // Only emit ConnectionEstablished if we haven't already for this address // DashSet::insert returns true if the value was newly inserted - let should_emit = emitted_events.insert(remote_address); + let should_emit = emitted_events.insert(connection_key); if should_emit { // Background accept = they connected to us = Server side let _ = event_tx.send(NatTraversalEvent::ConnectionEstablished { - remote_address, + remote_address: connection_key, side: Side::Server, public_key, }); @@ -3705,6 +3713,7 @@ impl NatTraversalEndpoint { connection: InnerConnection, event_tx: mpsc::UnboundedSender, ) { + let remote_address = normalize_socket_addr(remote_address); let closed = connection.closed(); tokio::pin!(closed); @@ -3760,7 +3769,7 @@ impl NatTraversalEndpoint { // Send event notification (we initiated = Client side) if let Some(ref event_tx) = self.event_tx { let _ = event_tx.send(NatTraversalEvent::ConnectionEstablished { - remote_address: remote_addr, + remote_address: normalize_socket_addr(remote_addr), side: Side::Client, public_key, }); @@ -4311,14 +4320,15 @@ impl NatTraversalEndpoint { ); return; // new connection is not tracked in `connections` } - connections2.insert(remote_address, connection.clone()); + let connection_key = normalize_socket_addr(remote_address); + connections2.insert(connection_key, connection.clone()); // Only forward to handshake_tx if this is the first time // we've seen this address. Without this guard, a // simultaneous-open (both sides connect at the same time) // sends two entries to handshake_tx, causing duplicate // reader tasks for the same connection address. - if emitted2.insert(remote_address) { + if emitted2.insert(connection_key) { if let Some(ref server) = relay_server2 { let conn_clone = connection.clone(); let server_clone = Arc::clone(server); @@ -4335,14 +4345,14 @@ impl NatTraversalEndpoint { if let Some(ref etx) = event_tx2 { let etx = etx.clone(); - let addr = remote_address; + let addr = connection_key; let conn = connection.clone(); tokio::spawn(async move { Self::handle_connection(addr, conn, etx).await; }); } - let _ = tx2.send(Ok((remote_address, connection))).await; + let _ = tx2.send(Ok((connection_key, connection))).await; } else { debug!( "Duplicate connection from {} already emitted, skipping", @@ -4380,21 +4390,23 @@ impl NatTraversalEndpoint { } pub fn is_connected(&self, addr: &SocketAddr) -> bool { - if let Some(entry) = self.connections.get(addr) { + for candidate in socket_addr_variants(*addr) { + let Some(entry) = self.connections.get(&candidate) else { + continue; + }; if let Some(reason) = entry.value().close_reason() { // Connection is dead — remove it and report not connected. info!( "is_connected: {} has close_reason={}, removing from DashMap", - addr, reason + candidate, reason ); drop(entry); // release the DashMap ref before removing - self.connections.remove(addr); + self.connections.remove(&candidate); return false; } - true - } else { - false + return true; } + false } /// Number of tracked connections (for diagnostics). @@ -4408,10 +4420,12 @@ impl NatTraversalEndpoint { addr: &SocketAddr, ) -> Result, NatTraversalError> { // DashMap provides lock-free .get() - Ok(self - .connections - .get(addr) - .map(|entry| entry.value().clone())) + for candidate in socket_addr_variants(*addr) { + if let Some(entry) = self.connections.get(&candidate) { + return Ok(Some(entry.value().clone())); + } + } + Ok(None) } /// Get the receiver for accepted connection addresses. @@ -4435,8 +4449,9 @@ impl NatTraversalEndpoint { addr: SocketAddr, connection: InnerConnection, ) -> Result<(), NatTraversalError> { + let key = normalize_socket_addr(addr); let observed = connection.observed_address(); - info!("add_connection: {} observed_address={:?}", addr, observed); + info!("add_connection: {} observed_address={:?}", key, observed); // Always overwrite with the newer connection. The previous // logic skipped overwrite when the existing connection had no // close_reason, but a connection can become a zombie (driver no @@ -4444,10 +4459,10 @@ impl NatTraversalEndpoint { // Frames queued on such a connection are never transmitted. // The newest connection is the one most likely to have an active // driver, so always use it. - if self.connections.contains_key(&addr) { + if self.connections.contains_key(&key) { info!( "add_connection: {} replacing existing connection with newer one", - addr + key ); } // Symmetric P2P: spawn the relay-request handler so peers on the @@ -4462,7 +4477,7 @@ impl NatTraversalEndpoint { &self.relay_handler_connections, &connection, ); - self.connections.insert(addr, connection); + self.connections.insert(key, connection); info!( "add_connection: now have {} connections", self.connections.len() @@ -4474,9 +4489,9 @@ impl NatTraversalEndpoint { // because they never initiate hole-punching. if self.advertise_external_addresses { let mut nodes = self.bootstrap_nodes.write(); - if !nodes.iter().any(|n| n.address == addr) { + if !nodes.iter().any(|n| n.address == key) { nodes.push(BootstrapNode { - address: addr, + address: key, last_seen: std::time::Instant::now(), can_coordinate: true, rtt: None, @@ -4484,7 +4499,7 @@ impl NatTraversalEndpoint { }); info!( "add_connection: registered {} as NAT traversal coordinator ({} total)", - addr, + key, nodes.len() ); } @@ -4513,11 +4528,12 @@ impl NatTraversalEndpoint { NatTraversalError::ConfigError("NAT traversal event channel not configured".to_string()) })?; - let remote_address = connection.remote_address(); + let event_key = normalize_socket_addr(addr); + let remote_address = normalize_socket_addr(connection.remote_address()); // Only emit ConnectionEstablished if we haven't already for this address // DashSet::insert returns true if this is a new address (not already present) - let should_emit = self.emitted_established_events.insert(addr); + let should_emit = self.emitted_established_events.insert(event_key); if should_emit { let public_key = Self::extract_public_key_from_connection(&connection); @@ -4560,21 +4576,23 @@ impl NatTraversalEndpoint { addr: &SocketAddr, expected_stable_id: Option, ) -> Result, NatTraversalError> { - // Clear emitted event tracking so reconnections can generate new events - // DashSet provides lock-free .remove() - self.emitted_established_events.remove(addr); + let variants = socket_addr_variants(*addr); - if let Some(entry) = self.connections.get(addr) { + let mut remove_keys = Vec::new(); + for candidate in &variants { + let Some(entry) = self.connections.get(candidate) else { + continue; + }; if let Some(expected_id) = expected_stable_id { let current_id = entry.value().stable_id(); if current_id != expected_id { info!( "remove_connection: {} DashMap has a different connection \ (stable_id {} vs expected {}), keeping", - addr, current_id, expected_id + candidate, current_id, expected_id ); drop(entry); - return Ok(None); + continue; } } @@ -4587,8 +4605,27 @@ impl NatTraversalEndpoint { .value() .close(VarInt::from_u32(0), b"saorsa-transport: force-closed"); } + drop(entry); + self.emitted_established_events.remove(candidate); + self.active_sessions.remove(candidate); + self.closed_at.remove(candidate); + self.transport_candidates.remove(candidate); + remove_keys.push(*candidate); + } + + if expected_stable_id.is_some() && remove_keys.is_empty() { + return Ok(None); + } + + let mut removed = None; + for key in remove_keys { + if let Some((_, connection)) = self.connections.remove(&key) { + if removed.is_none() { + removed = Some(connection); + } + } } - Ok(self.connections.remove(addr).map(|(_, v)| v)) + Ok(removed) } /// List all active connections @@ -4603,9 +4640,13 @@ impl NatTraversalEndpoint { /// Returns the raw SPKI bytes if the connection has a valid ML-DSA-65 public key, /// `None` otherwise. pub fn peer_public_key(&self, addr: &SocketAddr) -> Option> { - self.connections - .get(addr) - .and_then(|entry| Self::extract_public_key_from_connection(entry.value())) + socket_addr_variants(*addr) + .into_iter() + .find_map(|candidate| { + self.connections + .get(&candidate) + .and_then(|entry| Self::extract_public_key_from_connection(entry.value())) + }) } /// Get the external/reflexive address as observed by remote peers @@ -5653,11 +5694,12 @@ impl NatTraversalEndpoint { match connecting.await { Ok(connection) => { let remote = connection.remote_address(); + let remote_key = normalize_socket_addr(remote); // Check if another task already inserted a connection - if connections.contains_key(&remote) { + if connections.contains_key(&remote_key) { debug!( "Connection already exists for {}, discarding duplicate from {}", - remote, address + remote_key, address ); // Close the duplicate connection to free resources connection.close(0u32.into(), b"duplicate connection"); @@ -5673,20 +5715,21 @@ impl NatTraversalEndpoint { // live connection. The reader task may have already // registered the incoming connection from the same peer. let mut inserted = false; - if let Some(existing) = connections.get(&remote) { + let key = normalize_socket_addr(remote); + if let Some(existing) = connections.get(&key) { if existing.value().close_reason().is_none() { info!( "attempt_hole_punch: {} already has live connection, skipping insert", - remote + key ); drop(existing); } else { drop(existing); - connections.insert(remote, connection.clone()); + connections.insert(key, connection.clone()); inserted = true; } } else { - connections.insert(remote, connection.clone()); + connections.insert(key, connection.clone()); inserted = true; } if inserted { @@ -6442,10 +6485,11 @@ impl NatTraversalEndpoint { info!("Connected to coordinator {}", coordinator); // Check if another task already established a coordinator connection - if connections.contains_key(&coordinator) { + let coordinator_key = normalize_socket_addr(coordinator); + if connections.contains_key(&coordinator_key) { debug!( "Coordinator connection already exists for {}, discarding duplicate", - coordinator + coordinator_key ); // Close the duplicate connection to free resources connection.close(0u32.into(), b"duplicate coordinator"); @@ -6454,7 +6498,7 @@ impl NatTraversalEndpoint { // Store the connection keyed by SocketAddr // DashMap provides lock-free .insert() - connections.insert(coordinator, connection.clone()); + connections.insert(coordinator_key, connection.clone()); // Symmetric P2P: ensure the coordinator (which accepted) // can open CONNECT-UDP bidi streams toward us too. Self::spawn_relay_handler_task( @@ -6721,11 +6765,13 @@ impl NatTraversalEndpoint { fn check_punch_results(&self, addr: &SocketAddr) -> Option { // Check if we have an established connection to this address // DashMap provides lock-free .get() - if let Some(entry) = self.connections.get(addr) { - // We have a connection! Return its address - let remote = entry.value().remote_address(); - info!("Found successful connection to {} at {}", addr, remote); - return Some(remote); + for candidate in socket_addr_variants(*addr) { + if let Some(entry) = self.connections.get(&candidate) { + // We have a connection! Return its address + let remote = entry.value().remote_address(); + info!("Found successful connection to {} at {}", addr, remote); + return Some(remote); + } } // No connection found, check if we have any validated candidates @@ -6776,7 +6822,10 @@ impl NatTraversalEndpoint { // Check if we have a connection to validate // DashMap provides lock-free .get() - if let Some(entry) = self.connections.get(&target_addr) { + for candidate_addr in socket_addr_variants(target_addr) { + let Some(entry) = self.connections.get(&candidate_addr) else { + continue; + }; let conn = entry.value(); // Connection exists, check if it's to the expected address if conn.remote_address() == address { @@ -6819,7 +6868,9 @@ impl NatTraversalEndpoint { /// wasted resources on hole punching attempts. #[inline] fn has_existing_connection(&self, addr: &SocketAddr) -> bool { - self.connections.contains_key(addr) + socket_addr_variants(*addr) + .iter() + .any(|addr| self.connections.contains_key(addr)) } /// Check if path validation succeeded @@ -6854,7 +6905,10 @@ impl NatTraversalEndpoint { fn is_connection_healthy(&self, addr: &SocketAddr) -> bool { // In real implementation, check QUIC connection status // DashMap provides lock-free .get() - if self.connections.get(addr).is_some() { + if socket_addr_variants(*addr) + .iter() + .any(|addr| self.connections.get(addr).is_some()) + { // Check if connection is still active // Note: Connection doesn't have is_closed/is_drained methods // We use the closed() future to check if still active @@ -7156,7 +7210,8 @@ impl NatTraversalEndpoint { // Step 3: Now safe to insert into connections keyed by remote address let remote_address = connection.remote_address(); - self.connections.insert(remote_address, connection.clone()); + let connection_key = normalize_socket_addr(remote_address); + self.connections.insert(connection_key, connection.clone()); // Symmetric P2P: this is a dial-side insert (we initiated via // `endpoint.connect`), so spawn the relay handler to mirror what // the accept-side does. Without this, a later diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 95f53601..5101e09e 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -54,7 +54,7 @@ use std::net::SocketAddr; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::{RwLock, broadcast, mpsc}; +use tokio::sync::{RwLock, Semaphore, broadcast, mpsc}; use tokio::time::timeout; use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; @@ -76,6 +76,7 @@ use crate::nat_traversal_api::{ NatTraversalEndpoint, NatTraversalError, NatTraversalEvent, NatTraversalStatistics, RELAY_TUNNEL_INITIAL_MTU, }; +use crate::shared::{normalize_socket_addr, socket_addr_variants}; use crate::transport::{ProtocolEngine, TransportAddr, TransportRegistry}; use crate::unified_config::P2pConfig; use rustls; @@ -89,6 +90,12 @@ const EVENT_CHANNEL_CAPACITY: usize = 256; /// event-driven reader-exit detection. const STALE_REAPER_INTERVAL: Duration = Duration::from_secs(10); +/// Payload size above which QUIC sends are serialized per remote address. +const LARGE_SEND_BACKPRESSURE_THRESHOLD: usize = 1024 * 1024; + +/// Number of large sends allowed in flight per remote address. +const LARGE_SEND_PER_ADDR_PERMITS: usize = 1; + /// Quick direct connection attempt after a failed hole-punch round. /// If the target's outgoing packets created a NAT binding, a QUIC handshake /// through the pinhole needs only 1-2 RTTs (~600ms at 300ms worst-case RTT). @@ -231,6 +238,9 @@ pub struct P2pEndpoint { pending_dials: Arc< tokio::sync::Mutex>>>, >, + + /// Per-address semaphore used to serialize large QUIC stream messages. + large_send_permits: Arc>>, } impl std::fmt::Debug for P2pEndpoint { @@ -592,10 +602,17 @@ pub enum EndpointError { fn broadcast_peer_connected_once( emitted: &dashmap::DashSet, event_tx: &broadcast::Sender, - addr: TransportAddr, + mut addr: TransportAddr, public_key: Option>, side: Side, ) { + addr = match addr { + TransportAddr::Quic(socket_addr) => TransportAddr::Quic(normalize_socket_addr(socket_addr)), + TransportAddr::Tcp(socket_addr) => TransportAddr::Tcp(normalize_socket_addr(socket_addr)), + TransportAddr::Udp(socket_addr) => TransportAddr::Udp(normalize_socket_addr(socket_addr)), + other => other, + }; + if let Some(socket_addr) = addr.as_socket_addr() { if !emitted.insert(socket_addr) { debug!( @@ -636,38 +653,74 @@ async fn do_cleanup_connection( stats: &RwLock, event_tx: &broadcast::Sender, emitted_peer_connected: &dashmap::DashSet, + large_send_permits: &dashmap::DashMap>, addr: &SocketAddr, reason: DisconnectReason, expected_stable_id: Option, ) -> bool { + let variants = socket_addr_variants(*addr); + // Step 1: Try to remove from the NAT traversal layer first (lock-free // DashMap). When the caller is a reader-exit firing for an older // connection that has since been replaced via simultaneous-open, the // stable_id guard rejects the removal — in that case the current // DashMap entry belongs to a newer connection and the per-addr // tracking (connected_peers, reader_handles) must NOT be torn down. - let inner_removed = matches!( - inner.remove_connection(addr, expected_stable_id), - Ok(Some(_)) - ); - if expected_stable_id.is_some() && !inner_removed { + let inner_removed = variants.iter().any(|candidate| { + matches!( + inner.remove_connection(candidate, expected_stable_id), + Ok(Some(_)) + ) + }); + if let Some(expected_id) = expected_stable_id + && !inner_removed + { + if let Ok(Some(current)) = inner.get_connection(addr) { + debug!( + "do_cleanup_connection: {} — current connection stable_id {} \ + differs from reader-exit stable_id {}, leaving state intact", + addr, + current.stable_id(), + expected_id + ); + return false; + } debug!( - "do_cleanup_connection: {} — DashMap holds a newer connection, \ - leaving connected_peers/reader_handles intact", - addr + "do_cleanup_connection: {} — no current connection for reader-exit stable_id {}, \ + cleaning remaining endpoint state", + addr, expected_id ); - return false; } // Step 2: Remove from connected_peers (canonical lock #1) - let removed = connected_peers.write().await.remove(addr); + let removed = { + let mut peers = connected_peers.write().await; + let mut removed = None; + for candidate in &variants { + if let Some(peer) = peers.remove(candidate) { + if removed.is_none() { + removed = Some(peer); + } + } + } + removed + }; // Allow a future reconnect to broadcast a fresh PeerConnected event. - emitted_peer_connected.remove(addr); + for candidate in &variants { + emitted_peer_connected.remove(candidate); + large_send_permits.remove(candidate); + } // Step 3: Remove and abort reader task (canonical lock #2) - let abort_handle = reader_handles.write().await.remove(addr); - if let Some(handle) = abort_handle { + let abort_handles: Vec<_> = { + let mut handles = reader_handles.write().await; + variants + .iter() + .filter_map(|candidate| handles.remove(candidate)) + .collect() + }; + for handle in abort_handles { handle.abort(); } @@ -906,6 +959,7 @@ impl P2pEndpoint { reader_handles, reader_exit_tx, pending_dials: Arc::new(tokio::sync::Mutex::new(HashMap::new())), + large_send_permits: Arc::new(dashmap::DashMap::new()), }; // Spawn background constrained poller task @@ -946,13 +1000,14 @@ impl P2pEndpoint { &self, addr: &SocketAddr, ) -> Result, EndpointError> { + let addr = normalize_socket_addr(*addr); let peers = self.connected_peers.read().await; - if !peers.contains_key(addr) { + if !peers.contains_key(&addr) { return Ok(None); } drop(peers); self.inner - .get_connection(addr) + .get_connection(&addr) .map_err(EndpointError::NatTraversal) } @@ -996,6 +1051,7 @@ impl P2pEndpoint { if self.shutdown.is_cancelled() { return Err(EndpointError::ShuttingDown); } + let addr = normalize_socket_addr(addr); // Dedup check: if we already have a live connection to this address, return it. { @@ -1851,6 +1907,7 @@ impl P2pEndpoint { connection: crate::high_level::Connection, addr: SocketAddr, ) -> Result { + let addr = normalize_socket_addr(addr); // Extract public key from TLS let remote_public_key = extract_public_key_bytes_from_connection(&connection); @@ -1956,6 +2013,8 @@ impl P2pEndpoint { target: SocketAddr, coordinator: SocketAddr, ) -> Result { + let target = normalize_socket_addr(target); + let coordinator = normalize_socket_addr(coordinator); info!( "try_hole_punch: ENTER target={} coordinator={}", target, coordinator @@ -2046,6 +2105,7 @@ impl P2pEndpoint { }; if let Some(actual_addr) = connected_addr { + let actual_addr = normalize_socket_addr(actual_addr); info!( "try_hole_punch: connection to {} established (actual addr: {})!", target, actual_addr @@ -2244,7 +2304,7 @@ impl P2pEndpoint { /// Check if we're connected to a specific address async fn is_connected_to_addr(&self, addr: SocketAddr) -> bool { - let transport_addr = TransportAddr::Quic(addr); + let transport_addr = TransportAddr::Quic(normalize_socket_addr(addr)); let peers = self.connected_peers.read().await; peers.values().any(|p| p.remote_addr == transport_addr) } @@ -2357,6 +2417,7 @@ impl P2pEndpoint { &*self.stats, &self.event_tx, &self.emitted_peer_connected, + &self.large_send_permits, addr, reason, None, @@ -2366,7 +2427,14 @@ impl P2pEndpoint { /// Disconnect from a peer by address pub async fn disconnect(&self, addr: &SocketAddr) -> Result<(), EndpointError> { - if self.connected_peers.read().await.contains_key(addr) { + let variants = socket_addr_variants(*addr); + if self + .connected_peers + .read() + .await + .keys() + .any(|known| variants.contains(known)) + { self.cleanup_connection(addr, DisconnectReason::Normal) .await; Ok(()) @@ -2427,10 +2495,10 @@ impl P2pEndpoint { // plain IPv4. Try both forms when looking up the peer. let (transport_addr, cached_connection) = { let peer_info = self.connected_peers.read().await; - let alt = crate::shared::dual_stack_alternate(addr); - let found = peer_info - .get(addr) - .or_else(|| alt.as_ref().and_then(|a| peer_info.get(a))); + let variants = socket_addr_variants(*addr); + let found = variants + .iter() + .find_map(|candidate| peer_info.get(candidate)); if let Some(peer_conn) = found { (peer_conn.remote_addr.clone(), None) } else { @@ -2438,31 +2506,31 @@ impl P2pEndpoint { // address (e.g. from a hole-punch that bypassed the normal path). // Capture the connection now before it can be cleaned up. drop(peer_info); - let conn = self.inner.get_connection(addr).ok().flatten().or_else(|| { - alt.as_ref() - .and_then(|a| self.inner.get_connection(a).ok().flatten()) - }); + let conn = variants + .iter() + .find_map(|candidate| self.inner.get_connection(candidate).ok().flatten()); if let Some(conn) = conn { + let key = normalize_socket_addr(*addr); info!( "send: found hole-punched connection to {}, registering", - addr + key ); let peer_conn = PeerConnection { public_key: None, - remote_addr: TransportAddr::Quic(*addr), + remote_addr: TransportAddr::Quic(key), authenticated: true, connected_at: Instant::now(), last_activity: Instant::now(), }; - self.connected_peers.write().await.insert(*addr, peer_conn); + self.connected_peers.write().await.insert(key, peer_conn); broadcast_peer_connected_once( &self.emitted_peer_connected, &self.event_tx, - TransportAddr::Quic(*addr), + TransportAddr::Quic(key), None, Side::Server, ); - (TransportAddr::Quic(*addr), Some(conn)) + (TransportAddr::Quic(key), Some(conn)) } else { return Err(EndpointError::PeerNotFound(*addr)); } @@ -2515,6 +2583,31 @@ impl P2pEndpoint { return Err(EndpointError::PeerNotFound(*addr)); } + let _large_send_permit = if data.len() >= LARGE_SEND_BACKPRESSURE_THRESHOLD { + let permit_key = match transport_addr { + TransportAddr::Quic(addr) => normalize_socket_addr(addr), + _ => normalize_socket_addr(*addr), + }; + let semaphore = self + .large_send_permits + .entry(permit_key) + .or_insert_with(|| Arc::new(Semaphore::new(LARGE_SEND_PER_ADDR_PERMITS))) + .clone(); + debug!( + "send({}): waiting for large-send permit ({} bytes)", + permit_key, + data.len() + ); + Some( + semaphore + .acquire_owned() + .await + .map_err(|_| EndpointError::ShuttingDown)?, + ) + } else { + None + }; + let mut send_stream = connection.open_uni().await.map_err(|e| { warn!("send({}): open_uni failed: {}", addr, e); EndpointError::Connection(e.to_string()) @@ -2812,7 +2905,10 @@ impl P2pEndpoint { /// Check if an address is connected pub async fn is_connected(&self, addr: &SocketAddr) -> bool { - self.connected_peers.read().await.contains_key(addr) + self.connected_peers + .read() + .await + .contains_key(&normalize_socket_addr(*addr)) } /// Check if a live QUIC connection exists at the NatTraversalEndpoint level. @@ -2838,6 +2934,11 @@ impl P2pEndpoint { .register_connection_peer_id(addr, crate::nat_traversal_api::PeerId(peer_id)); } + /// Return whether this endpoint owns an active QUIC connection for `addr`. + pub fn has_active_connection(&self, addr: &SocketAddr) -> bool { + self.inner.is_connected(addr) + } + /// Check if a peer is connected at the transport level. pub fn inner_is_connected(&self, addr: &SocketAddr) -> bool { if self.inner.is_connected(addr) { @@ -2879,6 +2980,7 @@ impl P2pEndpoint { // Abort all background reader tasks self.reader_tasks.lock().await.abort_all(); self.reader_handles.write().await.clear(); + self.large_send_permits.clear(); // Disconnect all peers let addrs: Vec = self.connected_peers.read().await.keys().copied().collect(); @@ -2915,21 +3017,10 @@ impl P2pEndpoint { let event_tx = self.event_tx.clone(); let max_read_bytes = self.config.max_message_size; let exit_tx = self.reader_exit_tx.clone(); - let inner = Arc::clone(&self.inner); let abort_handle = self.reader_tasks.lock().await.spawn(async move { info!("Reader task STARTED for {}", addr); - // Ensure the connection is in the NatTraversalEndpoint's DashMap - // so the send path can find it. This is critical for NAT-traversed - // connections where the accept-time DashMap entry may be missing - // or removed by competing accept paths. - debug!("Reader task: calling add_connection for {}", addr); - match inner.add_connection(addr, connection.clone()) { - Ok(()) => debug!("Reader task: add_connection OK for {}", addr), - Err(e) => warn!("Reader task: add_connection FAILED for {}: {:?}", addr, e), - } - loop { // Accept the next unidirectional stream let mut recv_stream = match connection.accept_uni().await { @@ -3001,7 +3092,10 @@ impl P2pEndpoint { addr }); - self.reader_handles.write().await.insert(addr, abort_handle); + let key = normalize_socket_addr(addr); + if let Some(old) = self.reader_handles.write().await.insert(key, abort_handle) { + old.abort(); + } } /// Spawn a single background task that polls constrained transport events @@ -3195,6 +3289,7 @@ impl P2pEndpoint { let stats = Arc::clone(&self.stats); let reader_handles = Arc::clone(&self.reader_handles); let emitted_peer_connected = Arc::clone(&self.emitted_peer_connected); + let large_send_permits = Arc::clone(&self.large_send_permits); let shutdown = self.shutdown.clone(); tokio::spawn(async move { @@ -3224,6 +3319,7 @@ impl P2pEndpoint { &stats, &event_tx, &emitted_peer_connected, + &large_send_permits, &addr, DisconnectReason::Timeout, Some(stable_id), @@ -3254,6 +3350,7 @@ impl P2pEndpoint { let stats = Arc::clone(&self.stats); let reader_handles = Arc::clone(&self.reader_handles); let emitted_peer_connected = Arc::clone(&self.emitted_peer_connected); + let large_send_permits = Arc::clone(&self.large_send_permits); let shutdown = self.shutdown.clone(); tokio::spawn(async move { @@ -3295,6 +3392,7 @@ impl P2pEndpoint { &stats, &event_tx, &emitted_peer_connected, + &large_send_permits, addr, DisconnectReason::Timeout, None, @@ -3462,14 +3560,9 @@ impl P2pEndpoint { let data_tx = data_tx.clone(); let event_tx = event_tx.clone(); let exit_tx = reader_exit_tx.clone(); - let inner2 = Arc::clone(&inner); let abort_handle = reader_tasks.lock().await.spawn(async move { info!("Reader task STARTED for {} (via forwarder)", addr); - match inner2.add_connection(addr, conn.clone()) { - Ok(()) => debug!("Reader task (forwarder): add_connection OK for {}", addr), - Err(e) => warn!("Reader task (forwarder): add_connection FAILED for {}: {:?}", addr, e), - } loop { let mut recv_stream = match conn.accept_uni().await { @@ -3507,7 +3600,10 @@ impl P2pEndpoint { addr }); - reader_handles.write().await.insert(addr, abort_handle); + let key = normalize_socket_addr(addr); + if let Some(old) = reader_handles.write().await.insert(key, abort_handle) { + old.abort(); + } } else { warn!( "Incoming connection forwarder: no connection found for {} in DashMap", @@ -3515,7 +3611,10 @@ impl P2pEndpoint { ); } - connected_peers.write().await.insert(addr, peer_conn); + connected_peers + .write() + .await + .insert(normalize_socket_addr(addr), peer_conn); broadcast_peer_connected_once( &emitted_peer_connected, &event_tx, @@ -3523,69 +3622,6 @@ impl P2pEndpoint { None, Side::Server, ); - - // Spawn a reader task for the connection so incoming streams - // (DHT, chunk protocol) are actually read. Without this, relayed - // connections are registered but never processed. - match inner.get_connection(&addr) { - Ok(Some(conn)) => { - info!( - "Incoming connection forwarder: spawning reader task for {}", - addr - ); - let data_tx = data_tx.clone(); - let event_tx_for_reader = event_tx.clone(); - let exit_tx = reader_exit_tx.clone(); - let inner_for_reader = Arc::clone(&inner); - reader_tasks.lock().await.spawn(async move { - info!("Reader task STARTED for {} (via forwarder)", addr); - match inner_for_reader.add_connection(addr, conn.clone()) { - Ok(()) => {} - Err(e) => { - warn!("Reader task: add_connection FAILED for {}: {:?}", addr, e); - } - } - loop { - let mut recv_stream = match conn.accept_uni().await { - Ok(stream) => stream, - Err(e) => { - info!("Reader task for {} (forwarder) ending: {}", addr, e); - break; - } - }; - let data = match recv_stream.read_to_end(max_read_bytes).await { - Ok(data) if data.is_empty() => continue, - Ok(data) => data, - Err(e) => { - info!("Reader task for {} (forwarder): read error: {}", addr, e); - break; - } - }; - let data_len = data.len(); - let _ = event_tx_for_reader.send(P2pEvent::DataReceived { - addr, bytes: data_len, - }); - if data_tx.try_send((addr, data)).is_err() { - warn!("Reader task for {} (forwarder): data channel full, dropping {} bytes", addr, data_len); - } - } - let _ = exit_tx.send((addr, conn.stable_id())); - addr - }); - } - Ok(None) => { - warn!( - "Incoming connection forwarder: get_connection({}) returned None — no reader task", - addr - ); - } - Err(e) => { - warn!( - "Incoming connection forwarder: get_connection({}) failed: {} — no reader task", - addr, e - ); - } - } } }); } @@ -3620,6 +3656,7 @@ impl Clone for P2pEndpoint { reader_handles: Arc::clone(&self.reader_handles), reader_exit_tx: self.reader_exit_tx.clone(), pending_dials: Arc::clone(&self.pending_dials), + large_send_permits: Arc::clone(&self.large_send_permits), } } } diff --git a/src/shared.rs b/src/shared.rs index 44e47530..449b0b32 100644 --- a/src/shared.rs +++ b/src/shared.rs @@ -273,6 +273,37 @@ pub fn dual_stack_alternate(addr: &SocketAddr) -> Option { } } +/// Maximum number of address forms produced by [`socket_addr_variants`]: +/// the normalized form, the original (if it differs from normalized), and +/// the dual-stack alternate (if any). +pub const MAX_ADDR_VARIANTS: usize = 3; + +/// Return every address form that may key a connection lookup for `addr`. +/// +/// On dual-stack sockets the same peer can show up under multiple +/// representations (plain IPv4 vs. IPv4-mapped IPv6), and connections may +/// be inserted under either. Lookups must therefore probe all live forms: +/// +/// 1. The normalized address (IPv4-mapped IPv6 collapsed to IPv4). +/// 2. The original `addr` if it differs from the normalized form. +/// 3. The dual-stack alternate, if one exists. +/// +/// The result has at most [`MAX_ADDR_VARIANTS`] entries with no duplicates. +pub fn socket_addr_variants(addr: SocketAddr) -> Vec { + let normalized = normalize_socket_addr(addr); + let mut variants = Vec::with_capacity(MAX_ADDR_VARIANTS); + variants.push(normalized); + if addr != normalized { + variants.push(addr); + } + if let Some(alt) = dual_stack_alternate(&normalized) + && !variants.contains(&alt) + { + variants.push(alt); + } + variants +} + /// Deterministic 32-byte wire identifier from a `SocketAddr`. /// /// Used to correlate PUNCH_ME_NOW relay targets across connections. From 3527d1841a7cf91196b638e5743c6847f145927c Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sun, 3 May 2026 14:58:14 +0200 Subject: [PATCH 06/14] feat(p2p): add staged SendFailed error with per-write progress timeout The previous send path used `write_all` which could block indefinitely on a stuck stream, and collapsed every failure into the opaque `EndpointError::Connection(String)`. Upper layers had no way to tell "open failed" from "stopped by peer" from "stalled mid-write" and so could not pick a sensible retry strategy. This introduces: - `SendFailureStage` enum identifying where a send broke (open, open progress timeout, write, write progress timeout, finish, stopped, acknowledgement). - `EndpointError::SendFailed { stage, bytes_written, reason }` so callers can branch on stage and know how many bytes the QUIC stream accepted before the failure. - `write_stream_with_progress_timeout` helper that writes in 64 KiB chunks with a per-write progress timeout, returning the byte count for diagnostics and resetting the stream on failure. - A wrapping `timeout` around `open_uni` to surface stuck stream opens the same way. Constants `STREAM_WRITE_CHUNK_SIZE`, `STREAM_WRITE_PROGRESS_TIMEOUT` and `STREAM_RESET_ABORT_CODE` are introduced rather than inlined. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/lib.rs | 2 +- src/p2p_endpoint.rs | 203 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 184 insertions(+), 21 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index a05613c9..8536af1d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -354,7 +354,7 @@ pub use node_event::{DisconnectReason as NodeDisconnectReason, NodeEvent, Traver /// P2P endpoint - for advanced use, prefer Node for most applications pub use p2p_endpoint::{ ConnectionMetrics, DisconnectReason, EndpointError, EndpointStats, P2pEndpoint, P2pEvent, - PeerConnection, TraversalPhase, + PeerConnection, SendFailureStage, TraversalPhase, }; /// P2P configuration with builder pattern diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 5101e09e..3db8e7b0 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -96,6 +96,17 @@ const LARGE_SEND_BACKPRESSURE_THRESHOLD: usize = 1024 * 1024; /// Number of large sends allowed in flight per remote address. const LARGE_SEND_PER_ADDR_PERMITS: usize = 1; +/// Per-write chunk size used for stream write progress accounting. +const STREAM_WRITE_CHUNK_SIZE: usize = 64 * 1024; + +/// Maximum time a QUIC stream write may make no forward progress. +const STREAM_WRITE_PROGRESS_TIMEOUT: Duration = Duration::from_secs(2); + +/// Application error code used when aborting a stuck/failed send stream. +/// `0` is the conventional "generic abort, no specific application error" +/// QUIC application error code. +const STREAM_RESET_ABORT_CODE: u32 = 0; + /// Quick direct connection attempt after a failed hole-punch round. /// If the target's outgoing packets created a NAT binding, a QUIC handshake /// through the pinhole needs only 1-2 RTTs (~600ms at 300ms worst-case RTT). @@ -543,6 +554,40 @@ pub enum DisconnectReason { // TraversalPhase is re-exported from nat_traversal_api +/// Stage at which a QUIC send failed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SendFailureStage { + /// The send failed before a stream could be opened. + OpenStream, + /// Opening a stream made no progress within the progress timeout. + OpenStreamProgressTimeout, + /// A stream write made no progress within the progress timeout. + WriteProgressTimeout, + /// A stream write returned an error. + Write, + /// The stream could not be finished after all bytes were queued. + Finish, + /// The peer explicitly stopped the stream. + Stopped, + /// The stream failed while waiting for acknowledgement/stop state. + Acknowledgement, +} + +impl std::fmt::Display for SendFailureStage { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let label = match self { + Self::OpenStream => "open_stream", + Self::OpenStreamProgressTimeout => "open_stream_progress_timeout", + Self::WriteProgressTimeout => "write_progress_timeout", + Self::Write => "write", + Self::Finish => "finish", + Self::Stopped => "stopped", + Self::Acknowledgement => "acknowledgement", + }; + f.write_str(label) + } +} + /// Error type for P2pEndpoint operations #[derive(Debug, thiserror::Error)] pub enum EndpointError { @@ -554,6 +599,17 @@ pub enum EndpointError { #[error("Connection error: {0}")] Connection(String), + /// Send failure with enough context for upper layers to decide retry policy. + #[error("Send failed at {stage}: {reason} (bytes_written={bytes_written})")] + SendFailed { + /// Stage at which the send failed. + stage: SendFailureStage, + /// Bytes accepted by the QUIC stream before failure. + bytes_written: usize, + /// Human-readable failure reason. + reason: String, + }, + /// NAT traversal error #[error("NAT traversal error: {0}")] NatTraversal(#[from] NatTraversalError), @@ -587,6 +643,79 @@ pub enum EndpointError { NoAddress, } +fn send_failed( + stage: SendFailureStage, + bytes_written: usize, + reason: impl Into, +) -> EndpointError { + EndpointError::SendFailed { + stage, + bytes_written, + reason: reason.into(), + } +} + +async fn write_stream_with_progress_timeout( + send_stream: &mut crate::high_level::SendStream, + addr: SocketAddr, + data: &[u8], +) -> Result { + let mut bytes_written = 0usize; + + while bytes_written < data.len() { + let end = bytes_written + .saturating_add(STREAM_WRITE_CHUNK_SIZE) + .min(data.len()); + let slice = &data[bytes_written..end]; + + match timeout(STREAM_WRITE_PROGRESS_TIMEOUT, send_stream.write(slice)).await { + Ok(Ok(0)) => { + return Err(send_failed( + SendFailureStage::Write, + bytes_written, + "stream write returned zero bytes", + )); + } + Ok(Ok(written)) => { + bytes_written += written; + } + Ok(Err(e)) => { + warn!( + "send({}): stream write failed after {}/{} bytes: {}", + addr, + bytes_written, + data.len(), + e + ); + return Err(send_failed( + SendFailureStage::Write, + bytes_written, + e.to_string(), + )); + } + Err(_elapsed) => { + warn!( + "send({}): stream write made no progress for {:?} after {}/{} bytes", + addr, + STREAM_WRITE_PROGRESS_TIMEOUT, + bytes_written, + data.len() + ); + return Err(send_failed( + SendFailureStage::WriteProgressTimeout, + bytes_written, + format!( + "stream write made no progress for {:?}", + STREAM_WRITE_PROGRESS_TIMEOUT + ), + )); + } + } + } + + Ok(bytes_written) +} + /// Broadcast `P2pEvent::PeerConnected` for `addr` exactly once per /// disconnect cycle. /// @@ -2608,24 +2737,46 @@ impl P2pEndpoint { None }; - let mut send_stream = connection.open_uni().await.map_err(|e| { - warn!("send({}): open_uni failed: {}", addr, e); - EndpointError::Connection(e.to_string()) - })?; + let mut send_stream = + match timeout(STREAM_WRITE_PROGRESS_TIMEOUT, connection.open_uni()).await { + Ok(Ok(stream)) => stream, + Ok(Err(e)) => { + warn!("send({}): open_uni failed: {}", addr, e); + return Err(send_failed( + SendFailureStage::OpenStream, + 0, + e.to_string(), + )); + } + Err(_elapsed) => { + warn!( + "send({}): open_uni made no progress for {:?}", + addr, STREAM_WRITE_PROGRESS_TIMEOUT + ); + return Err(send_failed( + SendFailureStage::OpenStreamProgressTimeout, + 0, + format!( + "open_uni made no progress for {:?}", + STREAM_WRITE_PROGRESS_TIMEOUT + ), + )); + } + }; - send_stream.write_all(data).await.map_err(|e| { - warn!( - "send({}): write_all ({} bytes) failed: {}", - addr, - data.len(), - e - ); - EndpointError::Connection(e.to_string()) - })?; + let bytes_written = + match write_stream_with_progress_timeout(&mut send_stream, *addr, data).await { + Ok(bytes_written) => bytes_written, + Err(e) => { + let _ = + send_stream.reset(crate::VarInt::from_u32(STREAM_RESET_ABORT_CODE)); + return Err(e); + } + }; send_stream.finish().map_err(|e| { warn!("send({}): finish failed: {}", addr, e); - EndpointError::Connection(e.to_string()) + send_failed(SendFailureStage::Finish, bytes_written, e.to_string()) })?; // Give Quinn a short chance to observe stream completion. @@ -2647,14 +2798,18 @@ impl P2pEndpoint { match timeout(ack_timeout, send_stream.stopped()).await { Ok(Ok(None)) => {} Ok(Ok(Some(stop_code))) => { - return Err(EndpointError::Connection(format!( - "peer stopped stream with code {stop_code}" - ))); + return Err(send_failed( + SendFailureStage::Stopped, + bytes_written, + format!("peer stopped stream with code {stop_code}"), + )); } Ok(Err(e)) => { - return Err(EndpointError::Connection(format!( - "peer did not acknowledge stream data: {e}" - ))); + return Err(send_failed( + SendFailureStage::Acknowledgement, + bytes_written, + format!("peer did not acknowledge stream data: {e}"), + )); } Err(_elapsed) => { debug!( @@ -3719,6 +3874,14 @@ mod tests { let addr: SocketAddr = "127.0.0.1:8080".parse().expect("valid addr"); let err = EndpointError::PeerNotFound(addr); assert!(err.to_string().contains("not found")); + + let err = EndpointError::SendFailed { + stage: SendFailureStage::WriteProgressTimeout, + bytes_written: 1024, + reason: "no progress".to_string(), + }; + assert!(err.to_string().contains("write_progress_timeout")); + assert!(err.to_string().contains("bytes_written=1024")); } #[tokio::test] From e081f1f786141ed8593cda554233fadb010a64b8 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sun, 3 May 2026 15:36:08 +0200 Subject: [PATCH 07/14] feat(p2p): add separate progress and handshake timeouts for direct connections - Introduced distinct timeouts for direct connection stages: progress and handshake. - Enhanced `HappyEyeballs` logging to reflect both timeouts. - Modified `race_connect` for explicit timeout handling at each stage and surface errors appropriately. - Updated `StrategyConfig` to include configurable `direct_handshake_timeout`. - Adjusted defaults and augmented tests to validate new timeout logic. --- src/connection_strategy.rs | 71 +++++++++++++++++++++----------------- src/p2p_endpoint.rs | 51 +++++++++++++++------------ 2 files changed, 67 insertions(+), 55 deletions(-) diff --git a/src/connection_strategy.rs b/src/connection_strategy.rs index 0888444b..e0eeebf1 100644 --- a/src/connection_strategy.rs +++ b/src/connection_strategy.rs @@ -50,15 +50,15 @@ use std::net::SocketAddr; use std::time::{Duration, Instant}; -/// Timeout for direct connection attempts (both IPv4 and IPv6). -/// Relay-allocated addresses (advertised via the DHT as plain socket -/// addresses) are indistinguishable from direct addresses at the -/// transport level. After a congested send timeout, immediately -/// classifying a peer as unreachable on a tiny direct-dial budget creates -/// false negatives. Eight seconds still bounds lookup latency while giving -/// a busy peer or relay path enough time to complete the QUIC + PQC -/// handshake. -const DEFAULT_DIRECT_CONNECT_TIMEOUT: Duration = Duration::from_secs(8); +/// Timeout for observing direct connection progress (both IPv4 and IPv6). +/// +/// This bounds how long we wait for a peer to show handshake progress before +/// treating the address as dead. Full QUIC + PQC handshake completion is +/// governed separately by [`DEFAULT_DIRECT_HANDSHAKE_TIMEOUT`]. +const DEFAULT_DIRECT_CONNECT_TIMEOUT: Duration = Duration::from_secs(1); + +/// Timeout for full direct QUIC + PQC handshake completion after progress. +const DEFAULT_DIRECT_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(4); /// How a connection was established #[derive(Debug, Clone, PartialEq, Eq)] @@ -165,10 +165,10 @@ pub enum ConnectionStage { /// Configuration for connection strategy timeouts and behavior #[derive(Debug, Clone)] pub struct StrategyConfig { - /// Timeout for direct IPv4 connection attempts - pub ipv4_timeout: Duration, - /// Timeout for direct IPv6 connection attempts - pub ipv6_timeout: Duration, + /// Timeout for direct connection attempts (applies to both IPv4 and IPv6) + pub direct_connect_timeout: Duration, + /// Timeout for full direct handshake completion after progress + pub direct_handshake_timeout: Duration, /// Timeout for each hole-punch round pub holepunch_timeout: Duration, /// Timeout for relay connection @@ -190,8 +190,8 @@ pub struct StrategyConfig { impl Default for StrategyConfig { fn default() -> Self { Self { - ipv4_timeout: DEFAULT_DIRECT_CONNECT_TIMEOUT, - ipv6_timeout: DEFAULT_DIRECT_CONNECT_TIMEOUT, + direct_connect_timeout: DEFAULT_DIRECT_CONNECT_TIMEOUT, + direct_handshake_timeout: DEFAULT_DIRECT_HANDSHAKE_TIMEOUT, holepunch_timeout: Duration::from_secs(8), relay_timeout: Duration::from_secs(10), max_holepunch_rounds: 2, @@ -223,15 +223,15 @@ impl StrategyConfig { } } - /// Set the IPv4 timeout - pub fn with_ipv4_timeout(mut self, timeout: Duration) -> Self { - self.ipv4_timeout = timeout; + /// Set the direct connection timeout (applies to both IPv4 and IPv6) + pub fn with_direct_connect_timeout(mut self, timeout: Duration) -> Self { + self.direct_connect_timeout = timeout; self } - /// Set the IPv6 timeout - pub fn with_ipv6_timeout(mut self, timeout: Duration) -> Self { - self.ipv6_timeout = timeout; + /// Set the direct handshake timeout + pub fn with_direct_handshake_timeout(mut self, timeout: Duration) -> Self { + self.direct_handshake_timeout = timeout; self } @@ -323,14 +323,14 @@ impl ConnectionStrategy { &self.config } - /// Get the IPv4 timeout - pub fn ipv4_timeout(&self) -> Duration { - self.config.ipv4_timeout + /// Get the direct connection timeout + pub fn direct_connect_timeout(&self) -> Duration { + self.config.direct_connect_timeout } - /// Get the IPv6 timeout - pub fn ipv6_timeout(&self) -> Duration { - self.config.ipv6_timeout + /// Get the direct handshake timeout + pub fn direct_handshake_timeout(&self) -> Duration { + self.config.direct_handshake_timeout } /// Get the hole-punch timeout @@ -520,8 +520,14 @@ mod tests { #[test] fn test_default_config() { let config = StrategyConfig::default(); - assert_eq!(config.ipv4_timeout, DEFAULT_DIRECT_CONNECT_TIMEOUT); - assert_eq!(config.ipv6_timeout, DEFAULT_DIRECT_CONNECT_TIMEOUT); + assert_eq!( + config.direct_connect_timeout, + DEFAULT_DIRECT_CONNECT_TIMEOUT + ); + assert_eq!( + config.direct_handshake_timeout, + DEFAULT_DIRECT_HANDSHAKE_TIMEOUT + ); assert_eq!(config.holepunch_timeout, Duration::from_secs(8)); assert_eq!(config.relay_timeout, Duration::from_secs(10)); assert_eq!(config.max_holepunch_rounds, 2); @@ -532,12 +538,13 @@ mod tests { #[test] fn test_config_builder() { let config = StrategyConfig::new() - .with_ipv4_timeout(Duration::from_secs(2)) - .with_ipv6_timeout(Duration::from_secs(2)) + .with_direct_connect_timeout(Duration::from_secs(2)) + .with_direct_handshake_timeout(Duration::from_secs(4)) .with_max_holepunch_rounds(5) .with_ipv6_enabled(false); - assert_eq!(config.ipv4_timeout, Duration::from_secs(2)); + assert_eq!(config.direct_connect_timeout, Duration::from_secs(2)); + assert_eq!(config.direct_handshake_timeout, Duration::from_secs(4)); assert_eq!(config.max_holepunch_rounds, 5); assert!(!config.ipv6_enabled); } diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 3db8e7b0..439bff51 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -1819,12 +1819,14 @@ impl P2pEndpoint { } let he_config = HappyEyeballsConfig::default(); - let direct_timeout = strategy.ipv4_timeout().max(strategy.ipv6_timeout()); + let connect_timeout = strategy.direct_connect_timeout(); + let handshake_timeout = strategy.direct_handshake_timeout(); info!( - "Happy Eyeballs: racing {} direct addresses (timeout: {:?})", + "Happy Eyeballs: racing {} direct addresses (connect timeout: {:?}, handshake timeout: {:?})", direct_addresses.len(), - direct_timeout + connect_timeout, + handshake_timeout ); // Clone the QUIC endpoint for use in the Happy Eyeballs closure. @@ -1840,24 +1842,32 @@ impl P2pEndpoint { }; let addrs = direct_addresses.clone(); - let he_result = timeout(direct_timeout, async { - happy_eyeballs::race_connect(&addrs, &he_config, |addr| { - let ep = quic_endpoint.clone(); - async move { - let connecting = ep - .connect(addr, "peer") - .map_err(|e| format!("connect error: {e}"))?; - connecting - .await - .map_err(|e| format!("handshake error: {e}")) - } - }) - .await + let he_result = happy_eyeballs::race_connect(&addrs, &he_config, move |addr| { + let ep = quic_endpoint.clone(); + async move { + let mut connecting = ep + .connect(addr, "peer") + .map_err(|e| format!("connect error: {e}"))?; + + timeout(connect_timeout, connecting.handshake_data()) + .await + .map_err(|_| { + format!("direct connect timed out after {:?}", connect_timeout) + })? + .map_err(|e| format!("direct connect error: {e}"))?; + + timeout(handshake_timeout, connecting) + .await + .map_err(|_| { + format!("handshake timed out after {:?}", handshake_timeout) + })? + .map_err(|e| format!("handshake error: {e}")) + } }) .await; match he_result { - Ok(Ok((connection, winning_addr))) => { + Ok((connection, winning_addr)) => { let method = if winning_addr.is_ipv6() { ConnectionMethod::DirectIPv6 } else { @@ -1874,16 +1884,11 @@ impl P2pEndpoint { .await?; return Ok((peer_conn, method)); } - Ok(Err(e)) => { + Err(e) => { debug!("Happy Eyeballs: all direct attempts failed: {}", e); strategy.transition_to_ipv6(e.to_string()); strategy.transition_to_holepunch("Happy Eyeballs exhausted"); } - Err(_) => { - debug!("Happy Eyeballs: direct connection timed out"); - strategy.transition_to_ipv6("Timeout"); - strategy.transition_to_holepunch("Happy Eyeballs timed out"); - } } } From 7c526545e5fd951e4e5c745fda6f5ee1901ce472 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sun, 3 May 2026 16:49:43 +0200 Subject: [PATCH 08/14] fix(quic): align bbr2 packet accounting --- src/congestion.rs | 9 ++ src/congestion/bbr2/README.md | 58 +++----- src/congestion/bbr2/adapter.rs | 69 +++++++-- src/congestion/bbr2/smoke_tests.rs | 42 +++++- src/connection/mod.rs | 215 ++++++++++++++++++++++++----- src/connection/packet_builder.rs | 7 +- 6 files changed, 304 insertions(+), 96 deletions(-) diff --git a/src/congestion.rs b/src/congestion.rs index 5cbb5c2b..2e940749 100644 --- a/src/congestion.rs +++ b/src/congestion.rs @@ -87,6 +87,15 @@ pub trait Controller: Send + Sync { let _ = pn; } + /// A packet left the recovery state without being acked or declared lost. + /// + /// This includes discarded packet-number spaces, rejected 0-RTT packets, + /// and PLPMTUD probes that should not trigger a congestion response. + fn on_packet_abandoned(&mut self, pn: u64, bytes: u64) { + let _ = bytes; + self.on_packet_neutered(pn); + } + /// Called when the known in-flight packet count has decreased (should be called exactly once per on_ack_received) fn on_end_acks( &mut self, diff --git a/src/congestion/bbr2/README.md b/src/congestion/bbr2/README.md index 851285c8..4424b812 100644 --- a/src/congestion/bbr2/README.md +++ b/src/congestion/bbr2/README.md @@ -20,7 +20,7 @@ Or via the raw factory: ```rust let mut cc = crate::congestion::Bbr2Config::default(); -cc.initial_window(200 * 1200); +cc.initial_window(10 * 1200); let factory: Arc = Arc::new(cc); ``` @@ -62,45 +62,18 @@ The vendored BBRv2 wants a single batched call: The adapter buffers `Acked`/`Lost` entries and flushes at the two natural batch boundaries in saorsa's flow: -1. `on_end_acks` → flush the ack batch (lost list is usually empty here; - saorsa's `detect_lost_packets` runs afterwards). -2. `on_congestion_event` → append to the loss list, then flush. - -Because saorsa's ack-then-loss split spans two flushes within the same -ack-processing round, the adapter issues up to two `BBRv2::on_congestion_event` -calls per RTT instead of one batched call. BBRv2 handles this — the minor -fidelity cost is that the sampler can't correlate same-round acks and -losses in a single event. +1. `on_congestion_event` → flushes ack/loss batches when loss detection has + just produced a congestion event. +2. `on_end_acks` → flushes any remaining ack batch and resyncs the adapter's + shadow bytes-in-flight value with the connection's authoritative counter. ## Known fidelity gaps vs upstream quiche -These are the places where the adapter trades accuracy for interface -compatibility with saorsa's existing `Controller` trait. Fixing them -requires extending saorsa's trait (cross-cutting changes in -`src/congestion.rs` + `src/connection/mod.rs`): - -1. **No packet number on ack.** saorsa's `Controller::on_ack` doesn't carry - `pn: u64`. The adapter falls back to using `max_sent_packet_number` - as a proxy key when pushing `Acked { pkt_num, time_sent }`. The - sampler is robust to unknown pkt_nums (it just skips them) but loses - per-packet delivery-rate precision as a result. - -2. **No per-packet loss info.** saorsa's `on_congestion_event` gives a - single aggregate `lost_bytes`. The adapter wraps this in a single - synthetic `Lost` entry keyed on `max_sent_packet_number`. BBRv2's - `inflight_hi_on_loss` reduction still fires correctly from the - aggregate byte count; only the sampler's per-packet loss-rate - tracking is degraded. - -3. **No `bytes_in_flight` passed through.** saorsa's `on_sent` and - `on_ack` don't include the connection's in-flight counter. The - adapter tracks its own `bytes_in_flight` by summing sent/acked/lost - bytes and resyncs to the authoritative value in `on_end_acks`. - -Both (1) and (2) are blocked on a single trait change — add `pn: u64` to -`Controller::on_ack` and take `&[(u64, u64)]` (pn, bytes) in -`on_congestion_event` — which would be a small cross-cutting refactor in -`src/congestion.rs` and `src/connection/mod.rs`. +The adapter mirrors bytes-in-flight by summing packet sends, acks, losses, +and abandoned packets, then resyncs from the connection's authoritative +counter in `on_end_acks`. The controller trait still does not pass +bytes-in-flight directly into `on_sent`, so this shadow counter is the main +remaining fidelity compromise versus quiche's native recovery manager. ## What this gets you over saorsa's BBRv1 @@ -119,8 +92,9 @@ Both (1) and (2) are blocked on a single trait change — add `pn: u64` to ## Default -BBRv2 is the default as of this port (`CongestionAlgorithm::Bbr2`). To -opt back into BBRv1, set `congestion_algorithm: CongestionAlgorithm::Bbr` -in the `P2pConfig`; CUBIC is available as -`CongestionAlgorithm::Cubic` for comparison or when probing an unknown -path. +BBRv2 is the default as of this port (`CongestionAlgorithm::Bbr2`). Its +default initial congestion window is 10 packets, matching RFC 9002/quiche +style startup behavior. To opt back into BBRv1, set +`congestion_algorithm: CongestionAlgorithm::Bbr` in the `P2pConfig`; CUBIC is +available as `CongestionAlgorithm::Cubic` for comparison or when probing an +unknown path. diff --git a/src/congestion/bbr2/adapter.rs b/src/congestion/bbr2/adapter.rs index ef7ecb8e..da108b6f 100644 --- a/src/congestion/bbr2/adapter.rs +++ b/src/congestion/bbr2/adapter.rs @@ -17,6 +17,7 @@ // no-op when both buffers are empty. use std::any::Any; +use std::collections::BTreeSet; use std::sync::Arc; use std::time::Duration; @@ -27,9 +28,11 @@ use crate::connection::RttEstimator; use super::BBRv2; use super::types::{Acked, BbrParams, CongestionControl, Lost, RecoveryStats, RttStats}; -/// Default packet count for the initial congestion window — matches BBRv1 -/// in this crate (200 × BASE_DATAGRAM_SIZE). -const K_INITIAL_WINDOW_PACKETS: u64 = 200; +/// Default packet count for the initial congestion window. +/// +/// Keep this aligned with RFC 9002/quiche-style defaults; callers that want a +/// more aggressive startup can still override `Bbr2Config::initial_window`. +const K_INITIAL_WINDOW_PACKETS: u64 = 10; /// Cap the max congestion window at something generous but not /// unbounded. 20k packets × 1500 = 30 MB, enough for 10 Gbps×100ms BDP. @@ -108,10 +111,15 @@ pub(crate) struct Bbr2Adapter { pending_acked: Vec, /// Packets lost during the current batch. pending_lost: Vec, - /// Smallest packet number still unacked, tracked for BBRv2's - /// sampler-GC hint. Updated every time we see a pn we haven't - /// before — either at send or at ack. - least_unacked: u64, + /// Ack-eliciting packets still outstanding in the BBRv2 sampler. + /// + /// BBRv2 uses the smallest of these as the sampler-GC boundary. This must + /// not be approximated with highest-acked+1, because out-of-order ACKs can + /// leave lower packets legitimately outstanding. + unacked_packets: BTreeSet, + /// Highest acked/lost/neutered packet number observed. Used only when no + /// sampler-tracked packets remain outstanding. + largest_removed_packet: Option, /// Unused `RttStats` placeholder — the vendored BBRv2 only takes it /// as a typed parameter and never reads it. rtt_stats: RttStats, @@ -139,12 +147,38 @@ impl Bbr2Adapter { prior_in_flight: 0, pending_acked: Vec::new(), pending_lost: Vec::new(), - least_unacked: 0, + unacked_packets: BTreeSet::new(), + largest_removed_packet: None, rtt_stats: RttStats, recovery_stats: RecoveryStats::default(), } } + fn least_unacked(&self) -> u64 { + self.unacked_packets + .iter() + .next() + .copied() + .or_else(|| self.largest_removed_packet.map(|pn| pn.saturating_add(1))) + .unwrap_or(0) + } + + fn mark_packet_removed(&mut self, pn: u64) { + self.unacked_packets.remove(&pn); + self.largest_removed_packet = + Some(self.largest_removed_packet.map_or(pn, |old| old.max(pn))); + } + + #[cfg(test)] + pub(super) fn debug_least_unacked(&self) -> u64 { + self.least_unacked() + } + + #[cfg(test)] + pub(super) fn debug_is_unacked(&self, pn: u64) -> bool { + self.unacked_packets.contains(&pn) + } + /// Flush the pending acks/losses into a single BBRv2 congestion /// event. No-op if both buffers are empty. `event_time` should be /// the most recent ack/loss-detection time; `bytes_in_flight` is @@ -157,7 +191,7 @@ impl Bbr2Adapter { // newly-acked packet update RTT? We approximate by checking the // batch has any ack-eliciting acks (which is what's buffered). let rtt_updated = !self.pending_acked.is_empty(); - let least_unacked = self.least_unacked; + let least_unacked = self.least_unacked(); self.inner.on_congestion_event( rtt_updated, self.prior_in_flight as usize, @@ -194,6 +228,7 @@ impl Controller for Bbr2Adapter { is_retransmissible, ); if is_retransmissible { + self.unacked_packets.insert(last_packet_number); self.bytes_in_flight = self.bytes_in_flight.saturating_add(bytes); } } @@ -217,11 +252,7 @@ impl Controller for Bbr2Adapter { pkt_num: pn, time_sent: sent, }); - // Advance least_unacked monotonically past the highest pn we've - // seen acked. The sampler uses this as a GC hint; overestimating - // would cause premature state deletion, so we bump only when the - // ack is at or beyond the current mark. - self.least_unacked = self.least_unacked.max(pn.saturating_add(1)); + self.mark_packet_removed(pn); } fn on_packet_lost(&mut self, _now: Instant, pn: u64, bytes: u64) { @@ -233,6 +264,7 @@ impl Controller for Bbr2Adapter { packet_number: pn, bytes_lost: bytes as usize, }); + self.mark_packet_removed(pn); } fn on_app_limited(&mut self, bytes_in_flight: u64) { @@ -241,6 +273,13 @@ impl Controller for Bbr2Adapter { fn on_packet_neutered(&mut self, pn: u64) { self.inner.on_packet_neutered(pn); + self.mark_packet_removed(pn); + } + + fn on_packet_abandoned(&mut self, pn: u64, bytes: u64) { + self.bytes_in_flight = self.bytes_in_flight.saturating_sub(bytes); + self.inner.on_packet_neutered(pn); + self.mark_packet_removed(pn); } fn on_end_acks( @@ -316,7 +355,7 @@ impl std::fmt::Debug for Bbr2Adapter { f.debug_struct("Bbr2Adapter") .field("mss", &self.mss) .field("bytes_in_flight", &self.bytes_in_flight) - .field("least_unacked", &self.least_unacked) + .field("least_unacked", &self.least_unacked()) .field("cwnd", &self.inner.get_congestion_window()) .finish() } diff --git a/src/congestion/bbr2/smoke_tests.rs b/src/congestion/bbr2/smoke_tests.rs index 07988f71..612ba207 100644 --- a/src/congestion/bbr2/smoke_tests.rs +++ b/src/congestion/bbr2/smoke_tests.rs @@ -33,9 +33,7 @@ mod tests { let a = make_adapter(); let w = a.window(); assert!(w > 0, "initial window must be positive, got {w}"); - // Default initial_window is 200 packets × 1200 bytes. The cwnd is - // derived as initial_cwnd_pkts × mss, so it should land in that - // ballpark (allow slack for BBRv2's internal clamping). + assert_eq!(a.initial_window(), 10 * TEST_MTU as u64); assert!(w >= 4 * TEST_MTU as u64, "got {w}"); } @@ -96,6 +94,44 @@ mod tests { assert!(a.window() > 0); } + #[test] + fn out_of_order_ack_keeps_least_unacked_at_oldest_outstanding_packet() { + let mut a = make_adapter(); + let t0 = Instant::now(); + let rtt = rtt_of(Duration::from_millis(50)); + + for pn in 1..=10_u64 { + a.on_sent(t0 + Duration::from_micros(pn), 1200, pn, true); + } + + a.on_ack(t0 + Duration::from_millis(50), t0, 1200, false, &rtt, 10); + + assert_eq!(a.debug_least_unacked(), 1); + assert!(!a.debug_is_unacked(10)); + assert!(a.debug_is_unacked(1)); + + for pn in 1..=9_u64 { + a.on_ack(t0 + Duration::from_millis(50), t0, 1200, false, &rtt, pn); + } + + assert_eq!(a.debug_least_unacked(), 11); + } + + #[test] + fn abandoned_packet_leaves_sampler_tracking_without_congestion_event() { + let mut a = make_adapter(); + let t0 = Instant::now(); + + a.on_sent(t0, 1200, 1, true); + a.on_sent(t0, 1200, 2, true); + + a.on_packet_abandoned(1, 1200); + + assert_eq!(a.debug_least_unacked(), 2); + assert!(!a.debug_is_unacked(1)); + assert!(a.debug_is_unacked(2)); + } + #[test] fn mtu_update_propagates() { let mut a = make_adapter(); diff --git a/src/connection/mod.rs b/src/connection/mod.rs index 73e79106..0af3baae 100644 --- a/src/connection/mod.rs +++ b/src/connection/mod.rs @@ -115,6 +115,10 @@ mod timer; use crate::congestion::Controller; use timer::{Timer, TimerTable}; +fn would_exceed_congestion_window(in_flight: u64, bytes_to_send: u64, window: u64) -> bool { + in_flight.saturating_add(bytes_to_send) > window +} + /// Protocol state and logic for a single QUIC connection /// /// Objects of this type receive [`ConnectionEvent`]s and emit [`EndpointEvent`]s and application @@ -853,7 +857,11 @@ impl Connection { debug_assert!(untracked_bytes <= segment_size as u64); let bytes_to_send = segment_size as u64 + untracked_bytes; - if self.path.in_flight.bytes + bytes_to_send >= self.path.congestion.window() { + if would_exceed_congestion_window( + self.path.in_flight.bytes, + bytes_to_send, + self.path.congestion.window(), + ) { space_idx += 1; congestion_blocked = true; // We continue instead of breaking here in order to avoid @@ -1160,7 +1168,7 @@ impl Connection { }), buf, ); - self.stats.udp_tx.on_sent(1, buf.len()); + self.account_udp_transmit(1, buf.len()); // Trace packet sent #[cfg(feature = "trace")] @@ -1251,12 +1259,7 @@ impl Connection { builder.pad_to(segment_size as u16); } - let sample_pn = builder.sample_pn; - let ack_eliciting = builder.ack_eliciting; builder.finish_and_track(now, self, sent_frames, buf); - self.path - .congestion - .on_sent(now, buf.len() as u64, sample_pn, ack_eliciting); #[cfg(feature = "__qlog")] self.emit_qlog_recovery_metrics(now); @@ -1330,9 +1333,7 @@ impl Connection { } trace!("sending {} bytes in {} datagrams", buf.len(), num_datagrams); - self.path.total_sent = self.path.total_sent.saturating_add(buf.len() as u64); - - self.stats.udp_tx.on_sent(num_datagrams as u64, buf.len()); + self.account_udp_transmit(num_datagrams as u64, buf.len()); // Trace packets sent #[cfg(feature = "trace")] @@ -1367,6 +1368,11 @@ impl Connection { }) } + fn account_udp_transmit(&mut self, datagrams: u64, bytes: usize) { + self.path.total_sent = self.path.total_sent.saturating_add(bytes as u64); + self.stats.udp_tx.on_sent(datagrams, bytes); + } + /// Send PUNCH_ME_NOW for coordination if necessary fn send_coordination_request(&mut self, _now: Instant, _buf: &mut Vec) -> Option { // Get coordination info without borrowing mutably @@ -1454,7 +1460,7 @@ impl Connection { buf, buf_capacity, 0, - false, + true, self, )?; @@ -1476,7 +1482,16 @@ impl Connection { .pqc_state .min_initial_size_for_path_mtu(self.path.current_mtu()); builder.pad_to(min_size); - builder.finish_and_track(now, self, None, buf); + builder.finish_and_track( + now, + self, + Some(SentFrames { + non_retransmits: true, + ..SentFrames::default() + }), + buf, + ); + self.account_udp_transmit(1, buf.len()); // Mark coordination as validating after packet is built if let Some(nat_traversal) = &mut self.nat_traversal { @@ -1556,7 +1571,7 @@ impl Connection { buf, buf_capacity, 0, - false, + true, self, )?; @@ -1580,7 +1595,16 @@ impl Connection { .min_initial_size_for_path_mtu(self.path.current_mtu()); builder.pad_to(min_size); - builder.finish_and_track(now, self, None, buf); + builder.finish_and_track( + now, + self, + Some(SentFrames { + non_retransmits: true, + ..SentFrames::default() + }), + buf, + ); + self.account_udp_transmit(1, buf.len()); Some(Transmit { destination: remote_addr, @@ -1630,7 +1654,7 @@ impl Connection { buf, buf_capacity, 0, - false, + true, self, )?; trace!("validating previous path with PATH_CHALLENGE {:08x}", token); @@ -1650,8 +1674,16 @@ impl Connection { // sending a datagram of this size builder.pad_to(min_size); - builder.finish(self, buf); - self.stats.udp_tx.on_sent(1, buf.len()); + builder.finish_and_track( + now, + self, + Some(SentFrames { + non_retransmits: true, + ..SentFrames::default() + }), + buf, + ); + self.account_udp_transmit(1, buf.len()); Some(Transmit { destination, @@ -2431,6 +2463,9 @@ impl Connection { // Handle a lost MTU probe if let Some(packet) = lost_mtu_probe { let info = self.spaces[SpaceId::Data].take(packet).unwrap(); // safe: lost_mtu_probe is omitted from lost_packets, and therefore must not have been removed yet + self.path + .congestion + .on_packet_abandoned(info.sample_pn, info.size.into()); self.remove_in_flight(packet, &info); self.path.mtud.on_probe_lost(); self.stats.path.lost_plpmtud_probes += 1; @@ -2648,7 +2683,7 @@ impl Connection { false, ); - self.process_decrypted_packet(now, remote, Some(packet_number), packet.into())?; + self.process_decrypted_packet(now, remote, ecn, Some(packet_number), packet.into())?; if let Some(data) = remaining { self.handle_coalesced(now, remote, ecn, data); } @@ -2862,7 +2897,9 @@ impl Connection { // Neuter the packet in the congestion controller too — BBRv2's // per-packet sampler would otherwise keep the send-time state // around indefinitely as a zombie entry. - self.path.congestion.on_packet_neutered(packet.sample_pn); + self.path + .congestion + .on_packet_abandoned(packet.sample_pn, packet.size.into()); self.remove_in_flight(pn, &packet); } self.set_loss_detection_timer(now) @@ -3013,7 +3050,7 @@ impl Connection { } } - if !self.state.is_closed() { + if !self.state.is_closed() && number.is_some() { let spin = match packet.header { Header::Short { spin, .. } => spin, _ => false, @@ -3028,7 +3065,7 @@ impl Connection { ); } - self.process_decrypted_packet(now, remote, number, packet) + self.process_decrypted_packet(now, remote, ecn, number, packet) } } }; @@ -3084,6 +3121,7 @@ impl Connection { &mut self, now: Instant, remote: SocketAddr, + ecn: Option, number: Option, packet: Packet, ) -> Result<(), ConnectionError> { @@ -3134,7 +3172,7 @@ impl Connection { return Err(TransportError::PROTOCOL_VIOLATION("client sent Retry").into()); } - if self.total_authed_packets > 1 + if self.total_authed_packets > 0 || packet.payload.len() <= 16 // token + 16 byte tag || !self.crypto.is_valid_retry( &self.rem_cids.active(), @@ -3155,6 +3193,7 @@ impl Connection { trace!("retrying with CID {}", rem_cid); let client_hello = state.client_hello.take().unwrap(); + self.on_packet_authenticated(now, SpaceId::Initial, ecn, None, false, false); self.retry_src_cid = Some(rem_cid); self.rem_cids.update_initial_cid(rem_cid); self.rem_handshake_cid = rem_cid; @@ -3182,6 +3221,9 @@ impl Connection { // Retransmit all 0-RTT data let zero_rtt = mem::take(&mut self.spaces[SpaceId::Data].sent_packets); for (pn, info) in zero_rtt { + self.path + .congestion + .on_packet_abandoned(info.sample_pn, info.size.into()); self.remove_in_flight(pn, &info); self.spaces[SpaceId::Data].pending |= info.retransmits; } @@ -3248,6 +3290,9 @@ impl Connection { let sent_packets = mem::take(&mut self.spaces[SpaceId::Data].sent_packets); for (pn, packet) in sent_packets { + self.path + .congestion + .on_packet_abandoned(packet.sample_pn, packet.size.into()); self.remove_in_flight(pn, &packet); } } else { @@ -3320,7 +3365,7 @@ impl Connection { Ok(()) } Header::VersionNegotiate { .. } => { - if self.total_authed_packets > 1 { + if self.total_authed_packets > 0 { return Ok(()); } let supported = packet @@ -3342,6 +3387,47 @@ impl Connection { } } + fn illegal_frame(frame: &Frame, reason: &'static str) -> TransportError { + let mut err = TransportError::PROTOCOL_VIOLATION(reason); + err.frame = Some(frame.ty()); + err + } + + fn ensure_early_frame_allowed(frame: &Frame) -> Result<(), TransportError> { + match frame { + Frame::Padding + | Frame::Ping + | Frame::Crypto(_) + | Frame::Ack(_) + | Frame::Close(Close::Connection(_)) => Ok(()), + _ => Err(Self::illegal_frame( + frame, + "illegal frame type in handshake", + )), + } + } + + fn ensure_0rtt_frame_allowed(frame: &Frame) -> Result<(), TransportError> { + match frame { + Frame::Padding + | Frame::Ping + | Frame::ResetStream(_) + | Frame::StopSending(_) + | Frame::Stream(_) + | Frame::MaxData(_) + | Frame::MaxStreamData { .. } + | Frame::MaxStreams { .. } + | Frame::DataBlocked { .. } + | Frame::StreamDataBlocked { .. } + | Frame::StreamsBlocked { .. } + | Frame::NewConnectionId(_) + | Frame::PathChallenge(_) + | Frame::Close(_) + | Frame::Datagram(_) => Ok(()), + _ => Err(Self::illegal_frame(frame, "illegal frame type in 0-RTT")), + } + } + /// Process an Initial or Handshake packet payload fn process_early_payload( &mut self, @@ -3361,6 +3447,7 @@ impl Connection { self.stats.frame_rx.record(&frame); let _guard = span.as_ref().map(|x| x.enter()); + Self::ensure_early_frame_allowed(&frame)?; ack_eliciting |= frame.is_ack_eliciting(); // Process frames @@ -3378,10 +3465,10 @@ impl Connection { return Ok(()); } _ => { - let mut err = - TransportError::PROTOCOL_VIOLATION("illegal frame type in handshake"); - err.frame = Some(frame.ty()); - return Err(err); + return Err(Self::illegal_frame( + &frame, + "illegal frame type in handshake", + )); } } } @@ -3436,14 +3523,7 @@ impl Connection { let _guard = span.as_ref().map(|x| x.enter()); if packet.header.is_0rtt() { - match frame { - Frame::Crypto(_) | Frame::Close(Close::Application(_)) => { - return Err(TransportError::PROTOCOL_VIOLATION( - "illegal frame type in 0-RTT", - )); - } - _ => {} - } + Self::ensure_0rtt_frame_allowed(&frame)?; } ack_eliciting |= frame.is_ack_eliciting(); @@ -7337,6 +7417,71 @@ mod tests { } } + fn ack_frame() -> Frame { + Frame::Ack(frame::Ack { + largest: 0, + delay: 0, + additional: Bytes::new(), + ecn: None, + }) + } + + fn app_close_frame() -> Frame { + Frame::Close(Close::Application(frame::ApplicationClose { + error_code: VarInt::from_u32(0), + reason: Bytes::new(), + })) + } + + fn transport_close_frame() -> Frame { + Frame::Close(Close::Connection(frame::ConnectionClose { + error_code: TransportErrorCode::NO_ERROR, + frame_type: None, + reason: Bytes::new(), + })) + } + + #[test] + fn zero_rtt_rejects_frames_forbidden_by_rfc9000() { + let forbidden = vec![ + ack_frame(), + Frame::Crypto(frame::Crypto { + offset: 0, + data: Bytes::new(), + }), + Frame::NewToken(NewToken { + token: Bytes::from_static(b"token"), + }), + Frame::PathResponse(1), + Frame::RetireConnectionId { sequence: 0 }, + Frame::HandshakeDone, + ]; + + for frame in forbidden { + let err = Connection::ensure_0rtt_frame_allowed(&frame).unwrap_err(); + assert_eq!(err.code, TransportErrorCode::PROTOCOL_VIOLATION); + assert_eq!(err.frame, Some(frame.ty())); + } + } + + #[test] + fn zero_rtt_allows_application_data_close_but_early_packets_do_not() { + assert!(Connection::ensure_0rtt_frame_allowed(&app_close_frame()).is_ok()); + + let err = Connection::ensure_early_frame_allowed(&app_close_frame()).unwrap_err(); + assert_eq!(err.code, TransportErrorCode::PROTOCOL_VIOLATION); + assert_eq!(err.frame, Some(frame::FrameType::APPLICATION_CLOSE)); + + assert!(Connection::ensure_early_frame_allowed(&transport_close_frame()).is_ok()); + } + + #[test] + fn congestion_gate_allows_exact_window_fill() { + assert!(!would_exceed_congestion_window(10_800, 1_200, 12_000)); + assert!(would_exceed_congestion_window(10_801, 1_200, 12_000)); + assert!(would_exceed_congestion_window(12_000, 1, 12_000)); + } + #[test] fn path_creation_initializes_address_discovery() { let config = TransportConfig::default(); diff --git a/src/connection/packet_builder.rs b/src/connection/packet_builder.rs index 2bc0c840..ca674982 100644 --- a/src/connection/packet_builder.rs +++ b/src/connection/packet_builder.rs @@ -236,6 +236,12 @@ impl PacketBuilder { conn.path .sent(exact_number, packet, &mut conn.spaces[space_id]); + if size != 0 { + conn.path + .congestion + .on_sent(now, u64::from(size), sample_pn, ack_eliciting); + conn.path.pacing.on_transmit(size); + } conn.stats.path.sent_packets += 1; conn.reset_keep_alive(now); if size != 0 { @@ -247,7 +253,6 @@ impl PacketBuilder { conn.permit_idle_reset = false; } conn.set_loss_detection_timer(now); - conn.path.pacing.on_transmit(size); // Update PQC state for packet tracking conn.pqc_state.on_packet_sent(space_id, size); From 623352c073b5c1603e980d78c322f3a62a2b09a1 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Sun, 3 May 2026 17:46:39 +0200 Subject: [PATCH 09/14] feat(p2p): gate direct connect timeout on first authenticated peer packet Add Connecting::first_peer_response() that resolves once the first QUIC packet authenticated by the peer is received, distinct from full handshake completion. P2pEndpoint's Happy Eyeballs direct race now uses it instead of handshake_data() so direct_connect_timeout bounds basic peer reachability while direct_handshake_timeout continues to govern full handshake completion. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/connection/mod.rs | 5 +++ src/connection_strategy.rs | 8 ++--- src/high_level/connection.rs | 67 ++++++++++++++++++++++++++++++++++++ src/p2p_endpoint.rs | 7 ++-- tests/smoke_quic_connect.rs | 51 +++++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 6 deletions(-) diff --git a/src/connection/mod.rs b/src/connection/mod.rs index 0af3baae..39ca1d75 100644 --- a/src/connection/mod.rs +++ b/src/connection/mod.rs @@ -1899,6 +1899,11 @@ impl Connection { stats } + /// Number of packets received from the peer that authenticated successfully. + pub(crate) fn authenticated_packets(&self) -> u64 { + self.total_authed_packets + } + /// Set the bound peer identity for token v2 issuance. pub fn set_token_binding_peer_id(&mut self, pid: PeerId) { self.peer_id_for_tokens = Some(pid); diff --git a/src/connection_strategy.rs b/src/connection_strategy.rs index e0eeebf1..5b4cde16 100644 --- a/src/connection_strategy.rs +++ b/src/connection_strategy.rs @@ -52,9 +52,9 @@ use std::time::{Duration, Instant}; /// Timeout for observing direct connection progress (both IPv4 and IPv6). /// -/// This bounds how long we wait for a peer to show handshake progress before -/// treating the address as dead. Full QUIC + PQC handshake completion is -/// governed separately by [`DEFAULT_DIRECT_HANDSHAKE_TIMEOUT`]. +/// This bounds how long we wait for the first authenticated QUIC packet from a +/// peer before treating the address as dead. Full QUIC + PQC handshake +/// completion is governed separately by [`DEFAULT_DIRECT_HANDSHAKE_TIMEOUT`]. const DEFAULT_DIRECT_CONNECT_TIMEOUT: Duration = Duration::from_secs(1); /// Timeout for full direct QUIC + PQC handshake completion after progress. @@ -165,7 +165,7 @@ pub enum ConnectionStage { /// Configuration for connection strategy timeouts and behavior #[derive(Debug, Clone)] pub struct StrategyConfig { - /// Timeout for direct connection attempts (applies to both IPv4 and IPv6) + /// Timeout for receiving the first authenticated QUIC packet on direct attempts pub direct_connect_timeout: Duration, /// Timeout for full direct handshake completion after progress pub direct_handshake_timeout: Duration, diff --git a/src/high_level/connection.rs b/src/high_level/connection.rs index e3ffd0fe..e810cd0b 100644 --- a/src/high_level/connection.rs +++ b/src/high_level/connection.rs @@ -42,6 +42,7 @@ use crate::{ pub struct Connecting { conn: Option, connected: oneshot::Receiver, + first_peer_response: Option>, handshake_data_ready: Option>, } @@ -54,6 +55,7 @@ impl Connecting { socket: Arc, runtime: Arc, ) -> Self { + let (on_first_peer_response_send, on_first_peer_response_recv) = oneshot::channel(); let (on_handshake_data_send, on_handshake_data_recv) = oneshot::channel(); let (on_connected_send, on_connected_recv) = oneshot::channel(); let conn = ConnectionRef::new( @@ -61,6 +63,7 @@ impl Connecting { conn, endpoint_events, conn_events, + on_first_peer_response_send, on_handshake_data_send, on_connected_send, socket, @@ -80,6 +83,7 @@ impl Connecting { Self { conn: Some(conn), connected: on_connected_recv, + first_peer_response: Some(on_first_peer_response_recv), handshake_data_ready: Some(on_handshake_data_recv), } } @@ -186,6 +190,45 @@ impl Connecting { }) } + /// Wait for the first authenticated QUIC packet from the peer. + /// + /// This completes before the full handshake and can be used to distinguish basic peer + /// reachability from handshake completion. + pub async fn first_peer_response(&mut self) -> Result<(), ConnectionError> { + if !self.has_peer_response()? { + if let Some(x) = self.first_peer_response.take() { + let _ = x.await; + } + } + + let conn = self.conn.as_ref().ok_or_else(|| { + tracing::error!("Connection state missing while waiting for first peer response"); + ConnectionError::LocallyClosed + })?; + let inner = conn.state.lock("first_peer_response"); + if inner.peer_response_received { + Ok(()) + } else { + Err(inner.error.clone().unwrap_or_else(|| { + error!("Spurious peer response notification with no error"); + ConnectionError::TransportError(crate::transport_error::Error::INTERNAL_ERROR( + "Spurious peer response notification".to_string(), + )) + })) + } + } + + fn has_peer_response(&self) -> Result { + let conn = self.conn.as_ref().ok_or_else(|| { + tracing::error!("Connection state missing while checking first peer response"); + ConnectionError::LocallyClosed + })?; + Ok(conn + .state + .lock("first_peer_response") + .peer_response_received) + } + /// The local IP address which was used when the peer established /// the connection /// @@ -1160,20 +1203,24 @@ impl ConnectionRef { conn: crate::Connection, endpoint_events: mpsc::UnboundedSender<(ConnectionHandle, EndpointEvent)>, conn_events: mpsc::Receiver, + on_first_peer_response: oneshot::Sender<()>, on_handshake_data: oneshot::Sender<()>, on_connected: oneshot::Sender, socket: Arc, runtime: Arc, ) -> Self { let remote_addr = conn.remote_address(); + let peer_response_received = conn.authenticated_packets() > 0; Self(Arc::new(ConnectionInner { initial_remote_addr: remote_addr, state: Mutex::new(State { inner: conn, driver: None, handle, + on_first_peer_response: Some(on_first_peer_response), on_handshake_data: Some(on_handshake_data), on_connected: Some(on_connected), + peer_response_received, connected: false, timer: None, timer_deadline: None, @@ -1256,8 +1303,10 @@ pub(crate) struct State { pub(crate) inner: crate::Connection, driver: Option, handle: ConnectionHandle, + on_first_peer_response: Option>, on_handshake_data: Option>, on_connected: Option>, + peer_response_received: bool, connected: bool, timer: Option>>, timer_deadline: Option, @@ -1389,7 +1438,11 @@ impl State { self.inner.local_address_changed(); } Poll::Ready(Some(ConnectionEvent::Proto(event))) => { + let needs_first_response_check = !self.peer_response_received; self.inner.handle_event(event); + if needs_first_response_check && self.inner.authenticated_packets() > 0 { + self.notify_first_peer_response(); + } } Poll::Ready(Some(ConnectionEvent::Close { reason, error_code })) => { self.close(error_code, reason, shared); @@ -1408,16 +1461,27 @@ impl State { } } + fn notify_first_peer_response(&mut self) { + if !self.peer_response_received { + self.peer_response_received = true; + if let Some(x) = self.on_first_peer_response.take() { + let _ = x.send(()); + } + } + } + fn forward_app_events(&mut self, shared: &Shared) { while let Some(event) = self.inner.poll() { use crate::Event::*; match event { HandshakeDataReady => { + self.notify_first_peer_response(); if let Some(x) = self.on_handshake_data.take() { let _ = x.send(()); } } Connected => { + self.notify_first_peer_response(); self.connected = true; if let Some(x) = self.on_connected.take() { // We don't care if the on-connected future was dropped @@ -1533,6 +1597,9 @@ impl State { /// Used to wake up all blocked futures when the connection becomes closed for any reason fn terminate(&mut self, reason: ConnectionError, shared: &Shared) { self.error = Some(reason.clone()); + if let Some(x) = self.on_first_peer_response.take() { + let _ = x.send(()); + } if let Some(x) = self.on_handshake_data.take() { let _ = x.send(()); } diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index 439bff51..ca3e5794 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -1849,10 +1849,13 @@ impl P2pEndpoint { .connect(addr, "peer") .map_err(|e| format!("connect error: {e}"))?; - timeout(connect_timeout, connecting.handshake_data()) + timeout(connect_timeout, connecting.first_peer_response()) .await .map_err(|_| { - format!("direct connect timed out after {:?}", connect_timeout) + format!( + "direct connect timed out after {:?} waiting for first QUIC response", + connect_timeout + ) })? .map_err(|e| format!("direct connect error: {e}"))?; diff --git a/tests/smoke_quic_connect.rs b/tests/smoke_quic_connect.rs index 2fc6c30d..880966ad 100644 --- a/tests/smoke_quic_connect.rs +++ b/tests/smoke_quic_connect.rs @@ -80,6 +80,57 @@ async fn connect_classical_tls_loopback() { do_connect_classical_tls_loopback().await; } +#[tokio::test] +async fn first_peer_response_completes_before_full_connect_wait() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let (chain, key) = gen_self_signed_cert(); + let server_cfg = + ServerConfig::with_single_cert(chain.clone(), key).expect("failed to build ServerConfig"); + + let server_addr: SocketAddr = ([127, 0, 0, 1], 0).into(); + let server_ep = Endpoint::server(server_cfg, server_addr).expect("server endpoint"); + let listen_addr = server_ep.local_addr().expect("obtain server local addr"); + + let accept_task = tokio::spawn(async move { + let inc = timeout(Duration::from_secs(10), server_ep.accept()) + .await + .expect("server accept wait") + .expect("incoming"); + timeout(Duration::from_secs(10), inc) + .await + .expect("server handshake wait") + .expect("server handshake ok") + }); + + let mut roots = rustls::RootCertStore::empty(); + for c in chain { + roots.add(c).expect("add server cert to roots"); + } + let client_cfg = ClientConfig::with_root_certificates(Arc::new(roots)).expect("client config"); + + let client_addr: SocketAddr = ([127, 0, 0, 1], 0).into(); + let mut client_ep = Endpoint::client(client_addr).expect("client endpoint"); + client_ep.set_default_client_config(client_cfg); + + let mut connecting = client_ep + .connect(listen_addr, "localhost") + .expect("start connect"); + + timeout(Duration::from_secs(10), connecting.first_peer_response()) + .await + .expect("first peer response wait") + .expect("first peer response"); + + let conn = timeout(Duration::from_secs(10), connecting) + .await + .expect("client connect wait") + .expect("client connected"); + + let _server_conn = accept_task.await.expect("accept task join"); + assert_eq!(conn.remote_address(), listen_addr); +} + // PQC capability + connection smoke: ensure PQC primitives work and a classical QUIC // handshake still succeeds on the same runtime. This validates local readiness for // enabling hybrid KEX in CI or dockerized envs. From 0b58a6627added04c67f4bdff8d1480ac0910d65 Mon Sep 17 00:00:00 2001 From: Warm Beer Date: Mon, 4 May 2026 10:46:41 +0200 Subject: [PATCH 10/14] fix(p2p): drop best-effort post-finish ack wait on QUIC sends The post-`finish()` `stopped()` wait was a 1s+payload best-effort window that tied each send to a remote acknowledgement that may never come on busy or torn-down connections. Per-step progress timeouts (open_uni and write) plus close_reason short-circuiting already cover real failure modes; the extra wait only added latency without delivery guarantees. BREAKING CHANGE: removes the TimeoutConfig::send_ack_timeout field and the SendFailureStage::Stopped/Acknowledgement variants that existed only to drive that path. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/config/nat_timeouts.rs | 30 +--------------------- src/p2p_endpoint.rs | 52 +------------------------------------- 2 files changed, 2 insertions(+), 80 deletions(-) diff --git a/src/config/nat_timeouts.rs b/src/config/nat_timeouts.rs index cc14ba31..c73481f1 100644 --- a/src/config/nat_timeouts.rs +++ b/src/config/nat_timeouts.rs @@ -131,14 +131,8 @@ impl Default for RelayTimeouts { } } -/// Default best-effort window to observe stream-data acknowledgement after a send. -const DEFAULT_SEND_ACK_TIMEOUT: Duration = Duration::from_secs(1); - -/// Fast-network best-effort send ACK window (halved from default). -const FAST_SEND_ACK_TIMEOUT: Duration = Duration::from_millis(500); - /// Master timeout configuration -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct TimeoutConfig { /// NAT traversal timeouts pub nat_traversal: NatTraversalTimeouts, @@ -148,26 +142,6 @@ pub struct TimeoutConfig { /// Relay timeouts pub relay: RelayTimeouts, - - /// Best-effort time to wait for Quinn to observe acknowledgement of stream - /// data after `finish()`. - /// - /// Explicit stream stop or connection loss is still returned as a send - /// error. Expiry of this window only means the data has been queued to - /// QUIC but not confirmed locally yet; later connection state or - /// application-level timeouts are responsible for retries. - pub send_ack_timeout: Duration, -} - -impl Default for TimeoutConfig { - fn default() -> Self { - Self { - nat_traversal: NatTraversalTimeouts::default(), - discovery: DiscoveryTimeouts::default(), - relay: RelayTimeouts::default(), - send_ack_timeout: DEFAULT_SEND_ACK_TIMEOUT, - } - } } impl TimeoutConfig { @@ -177,7 +151,6 @@ impl TimeoutConfig { nat_traversal: NatTraversalTimeouts::fast(), discovery: DiscoveryTimeouts::default(), relay: RelayTimeouts::default(), - send_ack_timeout: FAST_SEND_ACK_TIMEOUT, } } @@ -187,7 +160,6 @@ impl TimeoutConfig { nat_traversal: NatTraversalTimeouts::conservative(), discovery: DiscoveryTimeouts::default(), relay: RelayTimeouts::default(), - send_ack_timeout: DEFAULT_SEND_ACK_TIMEOUT, } } } diff --git a/src/p2p_endpoint.rs b/src/p2p_endpoint.rs index ca3e5794..f1d33e3a 100644 --- a/src/p2p_endpoint.rs +++ b/src/p2p_endpoint.rs @@ -567,10 +567,6 @@ pub enum SendFailureStage { Write, /// The stream could not be finished after all bytes were queued. Finish, - /// The peer explicitly stopped the stream. - Stopped, - /// The stream failed while waiting for acknowledgement/stop state. - Acknowledgement, } impl std::fmt::Display for SendFailureStage { @@ -581,8 +577,6 @@ impl std::fmt::Display for SendFailureStage { Self::WriteProgressTimeout => "write_progress_timeout", Self::Write => "write", Self::Finish => "finish", - Self::Stopped => "stopped", - Self::Acknowledgement => "acknowledgement", }; f.write_str(label) } @@ -2701,11 +2695,7 @@ impl P2pEndpoint { // // If the QUIC connection has a close_reason, every step of // the send path below is guaranteed to either fail or hang - // until the per-step timeout. In particular, - // `send_stream.stopped()` waits the full ack_timeout - // (~1 s + payload budget) for a peer that can never - // acknowledge — 1 s of dead latency per attempted send - // against a torn-down connection. Short-circuiting here + // until a per-step progress timeout. Short-circuiting here // collapses that to a microsecond-scale error. // // The upper layer (transport_handle channel recovery) @@ -2787,46 +2777,6 @@ impl P2pEndpoint { send_failed(SendFailureStage::Finish, bytes_written, e.to_string()) })?; - // Give Quinn a short chance to observe stream completion. - // `finish()` only queues the FIN locally, while `stopped()` - // resolves when all stream data has been acknowledged or the - // peer explicitly stops the stream. A missing ACK within this - // local window is not proof of delivery failure, especially on - // busy testnets where ACKs can be delayed behind other work. - // - // Keep hard errors for explicit stop/connection loss, but - // treat timeout as "queued to QUIC" and let later connection - // state or application-level request timeouts handle retries. - // For large payloads we add time proportional to size at an - // assumed 256 KB/s per connection. - let base_timeout = self.config.timeouts.send_ack_timeout; - let size_budget = - std::time::Duration::from_millis((data.len() as u64).saturating_div(256)); - let ack_timeout = base_timeout + size_budget; - match timeout(ack_timeout, send_stream.stopped()).await { - Ok(Ok(None)) => {} - Ok(Ok(Some(stop_code))) => { - return Err(send_failed( - SendFailureStage::Stopped, - bytes_written, - format!("peer stopped stream with code {stop_code}"), - )); - } - Ok(Err(e)) => { - return Err(send_failed( - SendFailureStage::Acknowledgement, - bytes_written, - format!("peer did not acknowledge stream data: {e}"), - )); - } - Err(_elapsed) => { - debug!( - "send({}): stream data queued but not fully acknowledged within {:?}", - addr, ack_timeout - ); - } - } - debug!("Sent {} bytes to {} via QUIC", data.len(), addr); } crate::transport::ProtocolEngine::Constrained => { From e0f7bcd2c7011f298a112f242656b2d3269aef43 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Mon, 4 May 2026 22:15:56 +0100 Subject: [PATCH 11/14] fix(masque): gate is_message_too_large on cfg(unix) for Windows build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit is_message_too_large referenced libc::EMSGSIZE unconditionally, but the libc crate is declared as [target.'cfg(unix)'.dependencies] and so is unavailable on Windows targets. The Windows build broke with E0433: "cannot find module or crate `libc` in this scope". Match the platform pattern set_dont_fragment already follows in this module: keep the libc-backed implementation behind cfg(unix), and add a non-Unix fallback that returns false. The PMTU control-frame loop simply does not fire on those targets — same outcome as the existing no-op set_dont_fragment fallback for non-Linux/BSD targets. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/masque/relay_server.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/masque/relay_server.rs b/src/masque/relay_server.rs index 48645fd9..55f374fd 100644 --- a/src/masque/relay_server.rs +++ b/src/masque/relay_server.rs @@ -172,10 +172,21 @@ fn set_dont_fragment(_socket: &UdpSocket) -> std::io::Result<()> { /// Linux returns `EMSGSIZE` (errno 90); BSD returns `EMSGSIZE` as well. /// Treated as the only signal that warrants emitting a PMTU control /// frame back through the tunnel. +#[cfg(unix)] fn is_message_too_large(err: &std::io::Error) -> bool { err.raw_os_error() == Some(libc::EMSGSIZE) } +/// Non-Unix targets do not link `libc`, and the corresponding +/// `set_dont_fragment` is a best-effort no-op there. Mirror that +/// behaviour: never claim a send error is PMTU-related, so the +/// tunnel-level PMTU control frame loop simply does not fire and we +/// fall back to the kernel's default fragmentation behaviour. +#[cfg(not(unix))] +fn is_message_too_large(_err: &std::io::Error) -> bool { + false +} + /// Item carried over the bounded channel between the UDP reader task /// and the QUIC stream writer task in [`MasqueRelayServer::run_stream_forwarding_loop`]. /// From f2e10969a44db56e4e5e3cadbcfc8abdc24a0d52 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Mon, 4 May 2026 23:06:21 +0100 Subject: [PATCH 12/14] chore(release): roll rc-2026.4.4 to 0.33.1-rc.1 --- Cargo.lock | 133 ++++++++++++++++++++++++++++++++++++++++------------- Cargo.toml | 2 +- 2 files changed, 101 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ceabea73..2d527065 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -290,9 +290,9 @@ checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "blake3" -version = "1.8.4" +version = "1.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d2d5991425dfd0785aed03aedcf0b321d61975c9b5b3689c774a2610ae0b51e" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" dependencies = [ "arrayref", "arrayvec", @@ -1334,9 +1334,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" dependencies = [ "atomic-waker", "bytes", @@ -1698,9 +1698,9 @@ dependencies = [ [[package]] name = "idna_adapter" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" dependencies = [ "icu_normalizer", "icu_properties", @@ -1834,6 +1834,36 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn", +] + [[package]] name = "jni-sys" version = "0.3.1" @@ -1889,9 +1919,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.95" +version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" +checksum = "a1840c94c045fbcf8ba2812c95db44499f7c64910a912551aaaa541decebcacf" dependencies = [ "cfg-if", "futures-util", @@ -2743,9 +2773,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "reqwest" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801" +checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" dependencies = [ "base64", "bytes", @@ -2763,7 +2793,7 @@ dependencies = [ "quinn", "rustls", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.7.0", "serde", "serde_json", "sync_wrapper", @@ -2831,9 +2861,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.39" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c2c118cb077cca2822033836dfb1b975355dfb784b5e8da48f7b6c5db74e60e" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -2896,6 +2926,27 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.22.4", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + [[package]] name = "rustls-platform-verifier-android" version = "0.1.1" @@ -3012,7 +3063,7 @@ dependencies = [ [[package]] name = "saorsa-transport" -version = "0.34.0" +version = "0.33.1-rc.1" dependencies = [ "anyhow", "arbitrary", @@ -3057,7 +3108,7 @@ dependencies = [ "rustls", "rustls-native-certs", "rustls-pemfile", - "rustls-platform-verifier", + "rustls-platform-verifier 0.6.2", "rustls-post-quantum", "saorsa-pqc", "saorsa-transport-workspace-hack", @@ -3220,9 +3271,9 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.18.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd5414fad8e6907dbdd5bc441a50ae8d6e26151a03b1de04d89a5576de61d01f" +checksum = "f05839ce67618e14a09b286535c0d9c94e85ef25469b0e13cb4f844e5593eb19" dependencies = [ "serde_core", "serde_with_macros", @@ -3230,9 +3281,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.18.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3db8978e608f1fe7357e211969fd9abdcae80bac1ba7a3369bb7eb6b404eb65" +checksum = "cf2ebbe86054f9b45bc3881e865683ccfaccce97b9b4cb53f3039d67f355a334" dependencies = [ "darling", "proc-macro2", @@ -3308,6 +3359,22 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -3570,9 +3637,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.52.1" +version = "1.52.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67dee974fe86fd92cc45b7a95fdd2f99a36a6d7b0d431a231178d3d670bbcc6" +checksum = "110a78583f19d5cdb2c5ccf321d1290344e71313c6c37d43520d386027d18386" dependencies = [ "bytes", "libc", @@ -3899,9 +3966,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.118" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" +checksum = "df52b6d9b87e0c74c9edfa1eb2d9bf85e5d63515474513aa50fa181b3c4f5db1" dependencies = [ "cfg-if", "once_cell", @@ -3912,9 +3979,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.68" +version = "0.4.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" +checksum = "af934872acec734c2d80e6617bbb5ff4f12b052dd8e6332b0817bce889516084" dependencies = [ "js-sys", "wasm-bindgen", @@ -3922,9 +3989,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.118" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" +checksum = "78b1041f495fb322e64aca85f5756b2172e35cd459376e67f2a6c9dffcedb103" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3932,9 +3999,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.118" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" +checksum = "9dcd0ff20416988a18ac686d4d4d0f6aae9ebf08a389ff5d29012b05af2a1b41" dependencies = [ "bumpalo", "proc-macro2", @@ -3945,9 +4012,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.118" +version = "0.2.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" +checksum = "49757b3c82ebf16c57d69365a142940b384176c24df52a087fb748e2085359ea" dependencies = [ "unicode-ident", ] @@ -3988,9 +4055,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.95" +version = "0.3.97" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" +checksum = "2eadbac71025cd7b0834f20d1fe8472e8495821b4e9801eb0a60bd1f19827602" dependencies = [ "js-sys", "wasm-bindgen", diff --git a/Cargo.toml b/Cargo.toml index 6e73865c..a49f1dfa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ [package] name = "saorsa-transport" -version = "0.34.0" +version = "0.33.1-rc.1" edition = "2024" rust-version = "1.88.0" license = "MIT OR Apache-2.0" From 09fa0703d4e456a70e6f82d30dd8aab35a99ddf5 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Mon, 4 May 2026 23:15:15 +0100 Subject: [PATCH 13/14] chore(release): roll rc-2026.4.4 to 0.34.1-rc.1 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d527065..d4a89b4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3063,7 +3063,7 @@ dependencies = [ [[package]] name = "saorsa-transport" -version = "0.33.1-rc.1" +version = "0.34.1-rc.1" dependencies = [ "anyhow", "arbitrary", diff --git a/Cargo.toml b/Cargo.toml index a49f1dfa..60d06fea 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ [package] name = "saorsa-transport" -version = "0.33.1-rc.1" +version = "0.34.1-rc.1" edition = "2024" rust-version = "1.88.0" license = "MIT OR Apache-2.0" From eed2a3fd7253891aecdbedff1993282539f0cdaa Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 5 May 2026 23:26:37 +0100 Subject: [PATCH 14/14] chore(release): promote rc-2026.4.4 to 0.34.1 --- Cargo.lock | 18 ++++-------------- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4a89b4e..68e848c7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1753,16 +1753,6 @@ version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" -[[package]] -name = "iri-string" -version = "0.7.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25e659a4bb38e810ebc252e53b5814ff908a8c58c2a9ce2fae1bbec24cbf4e20" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is-terminal" version = "0.4.17" @@ -3063,7 +3053,7 @@ dependencies = [ [[package]] name = "saorsa-transport" -version = "0.34.1-rc.1" +version = "0.34.1" dependencies = [ "anyhow", "arbitrary", @@ -3715,20 +3705,20 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.8" +version = "0.6.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +checksum = "a28f0d049ccfaa566e14e9663d304d8577427b368cb4710a20528690287a738b" dependencies = [ "bitflags", "bytes", "futures-util", "http", "http-body", - "iri-string", "pin-project-lite", "tower", "tower-layer", "tower-service", + "url", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 60d06fea..ac957e69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,7 @@ members = [ [package] name = "saorsa-transport" -version = "0.34.1-rc.1" +version = "0.34.1" edition = "2024" rust-version = "1.88.0" license = "MIT OR Apache-2.0"