diff --git a/quinn-udp/src/lib.rs b/quinn-udp/src/lib.rs index d1b1d6342..179a02b69 100644 --- a/quinn-udp/src/lib.rs +++ b/quinn-udp/src/lib.rs @@ -161,6 +161,7 @@ impl Transmit<'_> { /// This case is actually quite common when splitting up a prepared GSO batch /// again after GSO has been disabled because the last datagram in a GSO /// batch is allowed to be smaller than the segment size. + #[cfg_attr(apple_fast, allow(dead_code))] // Used by prepare_msg, which is unused when apple_fast fn effective_segment_size(&self) -> Option { match self.segment_size? { size if size >= self.contents.len() => None, diff --git a/quinn-udp/src/unix.rs b/quinn-udp/src/unix.rs index 031e78845..7b7818f1f 100644 --- a/quinn-udp/src/unix.rs +++ b/quinn-udp/src/unix.rs @@ -397,7 +397,7 @@ fn send(state: &UdpSocketState, io: SockRef<'_>, transmit: &Transmit<'_>) -> io: .enumerate() .take(BATCH_SIZE) { - prepare_msg( + prepare_msg_x( &Transmit { destination: transmit.destination, ecn: transmit.ecn, @@ -433,6 +433,12 @@ fn send(state: &UdpSocketState, io: SockRef<'_>, transmit: &Transmit<'_>) -> io: #[cfg(any(target_os = "openbsd", target_os = "netbsd", apple_slow))] fn send(state: &UdpSocketState, io: SockRef<'_>, transmit: &Transmit<'_>) -> io::Result<()> { + send_single(state, io, transmit) +} + +#[cfg(any(target_os = "openbsd", target_os = "netbsd", apple))] +#[cfg_attr(apple_fast, allow(dead_code))] // Unused when apple_fast is enabled +fn send_single(state: &UdpSocketState, io: SockRef<'_>, transmit: &Transmit<'_>) -> io::Result<()> { let mut hdr: libc::msghdr = unsafe { mem::zeroed() }; let mut iov: libc::iovec = unsafe { mem::zeroed() }; let mut ctrl = cmsg::Aligned([0u8; CMSG_LEN]); @@ -523,7 +529,7 @@ fn recv(io: SockRef<'_>, bufs: &mut [IoSliceMut<'_>], meta: &mut [RecvMeta]) -> let mut hdrs = unsafe { mem::zeroed::<[msghdr_x; BATCH_SIZE]>() }; let max_msg_count = bufs.len().min(BATCH_SIZE); for i in 0..max_msg_count { - prepare_recv(&mut bufs[i], &mut names[i], &mut ctrls[i], &mut hdrs[i]); + prepare_recv_x(&mut bufs[i], &mut names[i], &mut ctrls[i], &mut hdrs[i]); } let msg_count = loop { let n = unsafe { recvmsg_x(io.as_raw_fd(), hdrs.as_mut_ptr(), max_msg_count as _, 0) }; @@ -553,6 +559,22 @@ fn recv(io: SockRef<'_>, bufs: &mut [IoSliceMut<'_>], meta: &mut [RecvMeta]) -> apple_slow ))] fn recv(io: SockRef<'_>, bufs: &mut [IoSliceMut<'_>], meta: &mut [RecvMeta]) -> io::Result { + recv_single(io, bufs, meta) +} + +#[cfg(any( + target_os = "openbsd", + target_os = "netbsd", + target_os = "dragonfly", + solarish, + apple +))] +#[cfg_attr(apple_fast, allow(dead_code))] // Unused when apple_fast is enabled +fn recv_single( + io: SockRef<'_>, + bufs: &mut [IoSliceMut<'_>], + meta: &mut [RecvMeta], +) -> io::Result { let mut name = MaybeUninit::::uninit(); let mut ctrl = cmsg::Aligned(MaybeUninit::<[u8; CMSG_LEN]>::uninit()); let mut hdr = unsafe { mem::zeroed::() }; @@ -581,11 +603,11 @@ fn recv(io: SockRef<'_>, bufs: &mut [IoSliceMut<'_>], meta: &mut [RecvMeta]) -> const CMSG_LEN: usize = 88; +#[cfg_attr(apple_fast, allow(dead_code))] // Unused when apple_fast is enabled fn prepare_msg( transmit: &Transmit<'_>, dst_addr: &socket2::SockAddr, - #[cfg(not(apple_fast))] hdr: &mut libc::msghdr, - #[cfg(apple_fast)] hdr: &mut msghdr_x, + hdr: &mut libc::msghdr, iov: &mut libc::iovec, ctrl: &mut cmsg::Aligned<[u8; CMSG_LEN]>, #[allow(unused_variables)] // only used on FreeBSD & macOS @@ -625,6 +647,10 @@ fn prepare_msg( encoder.push(libc::IPPROTO_IPV6, libc::IPV6_TCLASS, ecn); } + // On apple_fast, prepare_msg is only compiled for send_single (fallback path), while the main + // send path uses prepare_msg_x with msghdr_x. gso::set_segment_size has a different signature + // when apple_fast is enabled, and it's a no-op on non-Linux platforms anyway. + #[cfg(not(apple_fast))] if let Some(segment_size) = transmit.effective_segment_size() { gso::set_segment_size(&mut encoder, segment_size as u16); } @@ -668,7 +694,67 @@ fn prepare_msg( encoder.finish(); } -#[cfg(not(apple_fast))] +/// Prepares an `msghdr_x` for use with `sendmsg_x`. +#[cfg(apple_fast)] +fn prepare_msg_x( + transmit: &Transmit<'_>, + dst_addr: &socket2::SockAddr, + hdr: &mut msghdr_x, + iov: &mut libc::iovec, + ctrl: &mut cmsg::Aligned<[u8; CMSG_LEN]>, + #[allow(unused_variables)] encode_src_ip: bool, + sendmsg_einval: bool, +) { + iov.iov_base = transmit.contents.as_ptr() as *const _ as *mut _; + iov.iov_len = transmit.contents.len(); + + let name = dst_addr.as_ptr() as *mut libc::c_void; + let namelen = dst_addr.len(); + hdr.msg_name = name as *mut _; + hdr.msg_namelen = namelen; + hdr.msg_iov = iov; + hdr.msg_iovlen = 1; + + hdr.msg_control = ctrl.0.as_mut_ptr() as _; + hdr.msg_controllen = CMSG_LEN as _; + let mut encoder = unsafe { cmsg::Encoder::new(hdr) }; + let ecn = transmit.ecn.map_or(0, |x| x as libc::c_int); + let is_ipv4 = transmit.destination.is_ipv4() + || matches!(transmit.destination.ip(), IpAddr::V6(addr) if addr.to_ipv4_mapped().is_some()); + if is_ipv4 { + if !sendmsg_einval { + encoder.push(libc::IPPROTO_IP, libc::IP_TOS, ecn as IpTosTy); + } + } else { + encoder.push(libc::IPPROTO_IPV6, libc::IPV6_TCLASS, ecn); + } + + if let Some(ip) = &transmit.src_ip { + match ip { + IpAddr::V4(v4) => { + if encode_src_ip { + let addr = libc::in_addr { + s_addr: u32::from_ne_bytes(v4.octets()), + }; + encoder.push(libc::IPPROTO_IP, libc::IP_RECVDSTADDR, addr); + } + } + IpAddr::V6(v6) => { + let pktinfo = libc::in6_pktinfo { + ipi6_ifindex: 0, + ipi6_addr: libc::in6_addr { + s6_addr: v6.octets(), + }, + }; + encoder.push(libc::IPPROTO_IPV6, libc::IPV6_PKTINFO, pktinfo); + } + } + } + + encoder.finish(); +} + +#[cfg_attr(apple_fast, allow(dead_code))] // Unused when apple_fast is enabled fn prepare_recv( buf: &mut IoSliceMut<'_>, name: &mut MaybeUninit, @@ -684,8 +770,9 @@ fn prepare_recv( hdr.msg_flags = 0; } +/// Prepares an `msghdr_x` for receiving with `recvmsg_x`. #[cfg(apple_fast)] -fn prepare_recv( +fn prepare_recv_x( buf: &mut IoSliceMut<'_>, name: &mut MaybeUninit, ctrl: &mut cmsg::Aligned<[u8; CMSG_LEN]>, @@ -1006,6 +1093,7 @@ mod gso { } } + #[cfg_attr(apple_fast, allow(dead_code))] // Unused when apple_fast is enabled pub(super) fn set_segment_size( #[cfg(not(apple_fast))] _encoder: &mut cmsg::Encoder<'_, libc::msghdr>, #[cfg(apple_fast)] _encoder: &mut cmsg::Encoder<'_, msghdr_x>,