From 65e8dec280efa4554d93fb15b0ce9ccf0c0bb6a7 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sat, 17 Dec 2022 20:46:42 +0000 Subject: [PATCH 01/77] Bump to CE 3.5-4f9e57b --- build.sbt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 3599a5ecae..b8d7c616ac 100644 --- a/build.sbt +++ b/build.sbt @@ -209,9 +209,9 @@ lazy val core = crossProject(JVMPlatform, JSPlatform, NativePlatform) libraryDependencies ++= Seq( "org.typelevel" %%% "cats-core" % "2.9.0", "org.typelevel" %%% "cats-laws" % "2.9.0" % Test, - "org.typelevel" %%% "cats-effect" % "3.4.2", - "org.typelevel" %%% "cats-effect-laws" % "3.4.2" % Test, - "org.typelevel" %%% "cats-effect-testkit" % "3.4.2" % Test, + "org.typelevel" %%% "cats-effect" % "3.5-4f9e57b", + "org.typelevel" %%% "cats-effect-laws" % "3.5-4f9e57b" % Test, + "org.typelevel" %%% "cats-effect-testkit" % "3.5-4f9e57b" % Test, "org.scodec" %%% "scodec-bits" % "1.1.34", "org.typelevel" %%% "scalacheck-effect-munit" % "2.0.0-M2" % Test, "org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3" % Test, From 073723bb0d5d6271f6b351c1ad9a0ab56f559780 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sat, 17 Dec 2022 21:10:19 +0000 Subject: [PATCH 02/77] Sketch `FdPollingSocket` --- .../scala/fs2/io/net/FdPollingSocket.scala | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala new file mode 100644 index 0000000000..32480881c5 --- /dev/null +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2 +package io.net + +import cats.effect.std.Mutex +import cats.effect.unsafe.FileDescriptorPoller +import com.comcast.ip4s.IpAddress +import com.comcast.ip4s.SocketAddress +import fs2.io.internal.ResizableBuffer + +import java.util.concurrent.atomic.AtomicReference + +import FdPollingSocket._ + +private final class FdPollingSocket[F[_]]( + fd: Int, + readBuffer: ResizableBuffer[F], + readMutex: Mutex[F], + writeMutex: Mutex[F] +) extends Socket[F] + with FileDescriptorPoller.Callback { + + def isOpen: F[Boolean] = ??? + + def localAddress: F[SocketAddress[IpAddress]] = ??? + def remoteAddress: F[SocketAddress[IpAddress]] = ??? + + def endOfInput: F[Unit] = ??? + def endOfOutput: F[Unit] = ??? + + private[this] val readCallback = new AtomicReference[Either[Throwable, Unit] => Unit] + private[this] val writeCallback = new AtomicReference[Either[Throwable, Unit] => Unit] + + def notifyFileDescriptorEvents(readReady: Boolean, writeReady: Boolean): Unit = ??? + + def read(maxBytes: Int): F[Option[Chunk[Byte]]] = ??? + def readN(numBytes: Int): F[Chunk[Byte]] = ??? + def reads: Stream[F, Byte] = ??? + + def write(bytes: Chunk[Byte]): F[Unit] = ??? + def writes: Pipe[F, Byte, Nothing] = ??? + +} + +private object FdPollingSocket { + + private val ReadySentinel: Either[Throwable, Unit] => Unit = _ => () + +} From ce77f22421642bf0dfa3752587d7a3c157696940 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sat, 17 Dec 2022 21:40:56 +0000 Subject: [PATCH 03/77] Better error-handling in `ResizableBuffer` --- .../main/scala/fs2/io/internal/ResizableBuffer.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala b/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala index 51f4808cf6..1ec4310bd0 100644 --- a/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala +++ b/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala @@ -23,10 +23,10 @@ package fs2.io.internal import cats.effect.kernel.Resource import cats.effect.kernel.Sync -import cats.syntax.all._ import scala.scalanative.libc.errno._ import scala.scalanative.libc.stdlib._ +import scala.scalanative.posix.string._ import scala.scalanative.unsafe._ import scala.scalanative.unsigned._ @@ -37,15 +37,15 @@ private[io] final class ResizableBuffer[F[_]] private ( def get(size: Int): F[Ptr[Byte]] = F.delay { if (size <= this.size) - F.pure(ptr) + ptr else { ptr = realloc(ptr, size.toUInt) this.size = size if (ptr == null) - F.raiseError[Ptr[Byte]](new RuntimeException(s"realloc: ${errno}")) - else F.pure(ptr) + throw new RuntimeException(fromCString(strerror(errno))) + else ptr } - }.flatten + } } @@ -56,7 +56,7 @@ private[io] object ResizableBuffer { F.delay { val ptr = malloc(size.toUInt) if (ptr == null) - throw new RuntimeException(s"malloc: ${errno}") + throw new RuntimeException(fromCString(strerror(errno))) else new ResizableBuffer(ptr, size) } }(buf => F.delay(free(buf.ptr))) From 3bc141099b0030cbc7bb10029a10636d34b6ce01 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sat, 17 Dec 2022 21:57:01 +0000 Subject: [PATCH 04/77] Impl socket close, add native util --- .../scala/fs2/io/internal/NativeUtil.scala | 45 +++++++++++++++++++ .../scala/fs2/io/net/FdPollingSocket.scala | 17 +++++-- 2 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 io/native/src/main/scala/fs2/io/internal/NativeUtil.scala diff --git a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala new file mode 100644 index 0000000000..8b3062d5ec --- /dev/null +++ b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.io.internal + +import scala.scalanative.annotation.alwaysinline +import scala.scalanative.libc.errno._ +import scala.scalanative.posix.string._ +import scala.scalanative.unsafe._ +import java.io.IOException + +private[io] object NativeUtil { + + @alwaysinline def guard_(thunk: => CInt): Unit = { + guard(thunk) + () + } + + @alwaysinline def guard(thunk: => CInt): CInt = { + val rtn = thunk + if (rtn < 0) + throw new IOException(fromCString(strerror(errno))) + else + rtn + } + +} diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 32480881c5..6b25d18a77 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -22,12 +22,15 @@ package fs2 package io.net +import cats.effect.kernel.Async import cats.effect.std.Mutex import cats.effect.unsafe.FileDescriptorPoller import com.comcast.ip4s.IpAddress import com.comcast.ip4s.SocketAddress +import fs2.io.internal.NativeUtil._ import fs2.io.internal.ResizableBuffer +import scala.scalanative.posix.unistd import java.util.concurrent.atomic.AtomicReference import FdPollingSocket._ @@ -37,10 +40,18 @@ private final class FdPollingSocket[F[_]]( readBuffer: ResizableBuffer[F], readMutex: Mutex[F], writeMutex: Mutex[F] -) extends Socket[F] +)(implicit F: Async[F]) + extends Socket[F] with FileDescriptorPoller.Callback { - def isOpen: F[Boolean] = ??? + @volatile private[this] var open = true + + def isOpen: F[Boolean] = F.delay(open) + + def close: F[Unit] = F.delay { + open = false + guard_(unistd.close(fd)) + } def localAddress: F[SocketAddress[IpAddress]] = ??? def remoteAddress: F[SocketAddress[IpAddress]] = ??? @@ -49,7 +60,7 @@ private final class FdPollingSocket[F[_]]( def endOfOutput: F[Unit] = ??? private[this] val readCallback = new AtomicReference[Either[Throwable, Unit] => Unit] - private[this] val writeCallback = new AtomicReference[Either[Throwable, Unit] => Unit] + private[this] val writeCallback = new AtomicReference[Either[Throwable, Unit] => Unit] def notifyFileDescriptorEvents(readReady: Boolean, writeReady: Boolean): Unit = ??? From 862ae581e0a5828f4c60d4be89ee9b9f0ebc1c43 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sat, 17 Dec 2022 21:57:23 +0000 Subject: [PATCH 05/77] Tidy unused type param --- io/native/src/main/scala/fs2/io/net/tls/s2nutil.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/native/src/main/scala/fs2/io/net/tls/s2nutil.scala b/io/native/src/main/scala/fs2/io/net/tls/s2nutil.scala index 66590f5d3b..c52eeca612 100644 --- a/io/native/src/main/scala/fs2/io/net/tls/s2nutil.scala +++ b/io/native/src/main/scala/fs2/io/net/tls/s2nutil.scala @@ -44,7 +44,7 @@ private[tls] object s2nutil { throw new S2nException(error) } - @alwaysinline def guard[A](thunk: => CInt): CInt = { + @alwaysinline def guard(thunk: => CInt): CInt = { val rtn = thunk if (rtn < 0) { val error = !s2n_errno_location() From 9e475e84239d7acf2f5971c9ca84f410b42f4644 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sat, 17 Dec 2022 22:37:38 +0000 Subject: [PATCH 06/77] Add socket address helpers Co-authored-by: Lee Tibbert --- .../io/internal/SocketAddressHelpers.scala | 193 ++++++++++++++++++ .../main/scala/fs2/io/internal/netinet.scala | 90 ++++++++ .../scala/fs2/io/net/FdPollingSocket.scala | 6 +- 3 files changed, 287 insertions(+), 2 deletions(-) create mode 100644 io/native/src/main/scala/fs2/io/internal/SocketAddressHelpers.scala create mode 100644 io/native/src/main/scala/fs2/io/internal/netinet.scala diff --git a/io/native/src/main/scala/fs2/io/internal/SocketAddressHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketAddressHelpers.scala new file mode 100644 index 0000000000..a0a3ad2e76 --- /dev/null +++ b/io/native/src/main/scala/fs2/io/internal/SocketAddressHelpers.scala @@ -0,0 +1,193 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.io.internal + +import cats.effect.kernel.Resource +import cats.effect.kernel.Sync +import cats.syntax.all._ +import com.comcast.ip4s.IpAddress +import com.comcast.ip4s.Ipv4Address +import com.comcast.ip4s.Ipv6Address +import com.comcast.ip4s.Port +import com.comcast.ip4s.SocketAddress + +import java.io.IOException +import scala.scalanative.posix.arpa.inet._ +import scala.scalanative.posix.sys.socket._ +import scala.scalanative.posix.sys.socketOps._ +import scala.scalanative.unsafe._ +import scala.scalanative.unsigned._ + +import NativeUtil._ +import netinetin._ +import netinetinOps._ + +private[io] object SocketAddressHelpers { + + def getLocalAddress[F[_]](fd: Int)(implicit F: Sync[F]): F[SocketAddress[IpAddress]] = + SocketAddressHelpers.toSocketAddress { (addr, len) => + F.delay(guard_(getsockname(fd, addr, len))) + } + + def allocateSockaddr[F[_]](implicit F: Sync[F]): Resource[F, (Ptr[sockaddr], Ptr[socklen_t])] = + Resource + .make(F.delay(Zone.open()))(z => F.delay(z.close())) + .evalMap { implicit z => + F.delay { + val addr = // allocate enough for an IPv6 + alloc[sockaddr_in6]().asInstanceOf[Ptr[sockaddr]] + val len = alloc[socklen_t]() + (addr, len) + } + } + + def toSockaddr[A]( + address: SocketAddress[IpAddress] + )(f: (Ptr[sockaddr], socklen_t) => A): A = + address.host.fold( + _ => + toSockaddrIn(address.asInstanceOf[SocketAddress[Ipv4Address]])( + f.asInstanceOf[(Ptr[sockaddr_in], socklen_t) => A] + ), + _ => + toSockaddrIn6(address.asInstanceOf[SocketAddress[Ipv6Address]])( + f.asInstanceOf[(Ptr[sockaddr_in6], socklen_t) => A] + ) + ) + + private[this] def toSockaddrIn[A]( + address: SocketAddress[Ipv4Address] + )(f: (Ptr[sockaddr_in], socklen_t) => A): A = { + val addr = stackalloc[sockaddr_in]() + val len = stackalloc[socklen_t]() + + toSockaddrIn(address, addr, len) + + f(addr, !len) + } + + private[this] def toSockaddrIn6[A]( + address: SocketAddress[Ipv6Address] + )(f: (Ptr[sockaddr_in6], socklen_t) => A): A = { + val addr = stackalloc[sockaddr_in6]() + val len = stackalloc[socklen_t]() + + toSockaddrIn6(address, addr, len) + + f(addr, !len) + } + + def toSockaddr( + address: SocketAddress[IpAddress], + addr: Ptr[sockaddr], + len: Ptr[socklen_t] + ): Unit = + address.host.fold( + _ => + toSockaddrIn( + address.asInstanceOf[SocketAddress[Ipv4Address]], + addr.asInstanceOf[Ptr[sockaddr_in]], + len + ), + _ => + toSockaddrIn6( + address.asInstanceOf[SocketAddress[Ipv6Address]], + addr.asInstanceOf[Ptr[sockaddr_in6]], + len + ) + ) + + private[this] def toSockaddrIn( + address: SocketAddress[Ipv4Address], + addr: Ptr[sockaddr_in], + len: Ptr[socklen_t] + ): Unit = { + !len = sizeof[sockaddr_in].toUInt + addr.sin_family = AF_INET.toUShort + addr.sin_port = htons(address.port.value.toUShort) + addr.sin_addr.s_addr = htonl(address.host.toLong.toUInt) + } + + private[this] def toSockaddrIn6[A]( + address: SocketAddress[Ipv6Address], + addr: Ptr[sockaddr_in6], + len: Ptr[socklen_t] + ): Unit = { + !len = sizeof[sockaddr_in6].toUInt + + addr.sin6_family = AF_INET6.toUShort + addr.sin6_port = htons(address.port.value.toUShort) + + val bytes = address.host.toBytes + var i = 0 + while (i < 0) { + addr.sin6_addr.s6_addr(i) = bytes(i).toUByte + i += 1 + } + } + + def toSocketAddress[F[_]]( + f: (Ptr[sockaddr], Ptr[socklen_t]) => F[Unit] + )(implicit F: Sync[F]): F[SocketAddress[IpAddress]] = { + val addr = // allocate enough for an IPv6 + stackalloc[sockaddr_in6]().asInstanceOf[Ptr[sockaddr]] + val len = stackalloc[socklen_t]() + !len = sizeof[sockaddr_in6].toUInt + + f(addr, len) *> toSocketAddress(addr) + } + + def toSocketAddress[F[_]](addr: Ptr[sockaddr])(implicit F: Sync[F]): F[SocketAddress[IpAddress]] = + if (addr.sa_family.toInt == AF_INET) + F.pure(toIpv4SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in]])) + else if (addr.sa_family.toInt == AF_INET6) + F.pure(toIpv6SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in6]])) + else + F.raiseError(new IOException(s"Unsupported sa_family: ${addr.sa_family}")) + + private[this] def toIpv4SocketAddress(addr: Ptr[sockaddr_in]): SocketAddress[Ipv4Address] = { + val port = Port.fromInt(ntohs(addr.sin_port).toInt).get + val addrBytes = addr.sin_addr.at1.asInstanceOf[Ptr[Byte]] + val host = Ipv4Address.fromBytes( + addrBytes(0).toInt, + addrBytes(1).toInt, + addrBytes(2).toInt, + addrBytes(3).toInt + ) + SocketAddress(host, port) + } + + private[this] def toIpv6SocketAddress(addr: Ptr[sockaddr_in6]): SocketAddress[Ipv6Address] = { + val port = Port.fromInt(ntohs(addr.sin6_port).toInt).get + val addrBytes = addr.sin6_addr.at1.asInstanceOf[Ptr[Byte]] + val host = Ipv6Address.fromBytes { + val addr = new Array[Byte](16) + var i = 0 + while (i < addr.length) { + addr(i) = addrBytes(i.toLong) + i += 1 + } + addr + }.get + SocketAddress(host, port) + } +} diff --git a/io/native/src/main/scala/fs2/io/internal/netinet.scala b/io/native/src/main/scala/fs2/io/internal/netinet.scala new file mode 100644 index 0000000000..fdb1dc4afc --- /dev/null +++ b/io/native/src/main/scala/fs2/io/internal/netinet.scala @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.io.internal + +import scalanative.unsafe._ +import scalanative.posix.inttypes._ +import scalanative.posix.sys.socket._ + +private[io] object netinetin { + import Nat._ + type _16 = Digit2[_1, _6] + + type in_port_t = uint16_t + + type in_addr = CStruct1[uint32_t] + + type sockaddr_in = CStruct4[ + sa_family_t, + in_port_t, + in_addr, + CArray[Byte, _8] + ] + + type in6_addr = CStruct1[CArray[CUnsignedChar, _16]] + + type sockaddr_in6 = CStruct5[ + sa_family_t, + in_port_t, + uint32_t, + in6_addr, + uint32_t + ] + +} + +private[io] object netinetinOps { + import netinetin._ + + implicit final class in_addrOps(val in_addr: in_addr) extends AnyVal { + def s_addr: uint32_t = in_addr._1 + def s_addr_=(s_addr: uint32_t): Unit = in_addr._1 = s_addr + } + + implicit final class sockaddr_inOps(val sockaddr_in: Ptr[sockaddr_in]) extends AnyVal { + def sin_family: sa_family_t = sockaddr_in._1 + def sin_family_=(sin_family: sa_family_t): Unit = sockaddr_in._1 = sin_family + def sin_port: in_port_t = sockaddr_in._2 + def sin_port_=(sin_port: in_port_t): Unit = sockaddr_in._2 = sin_port + def sin_addr: in_addr = sockaddr_in._3 + def sin_addr_=(sin_addr: in_addr) = sockaddr_in._3 = sin_addr + } + + implicit final class in6_addrOps(val in6_addr: in6_addr) extends AnyVal { + def s6_addr: CArray[uint8_t, _16] = in6_addr._1 + def s6_addr_=(s6_addr: CArray[uint8_t, _16]): Unit = in6_addr._1 = s6_addr + } + + implicit final class sockaddr_in6Ops(val sockaddr_in6: Ptr[sockaddr_in6]) extends AnyVal { + def sin6_family: sa_family_t = sockaddr_in6._1 + def sin6_family_=(sin6_family: sa_family_t): Unit = sockaddr_in6._1 = sin6_family + def sin6_port: in_port_t = sockaddr_in6._2 + def sin6_port_=(sin6_port: in_port_t): Unit = sockaddr_in6._2 = sin6_port + def sin6_flowinfo: uint32_t = sockaddr_in6._3 + def sin6_flowinfo_=(sin6_flowinfo: uint32_t): Unit = sockaddr_in6._3 = sin6_flowinfo + def sin6_addr: in6_addr = sockaddr_in6._4 + def sin6_addr_=(sin6_addr: in6_addr) = sockaddr_in6._4 = sin6_addr + def sin6_scope_id: uint32_t = sockaddr_in6._5 + def sin6_scope_id_=(sin6_scope_id: uint32_t): Unit = sockaddr_in6._5 = sin6_scope_id + } + +} diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 6b25d18a77..f8a57719f9 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -29,6 +29,7 @@ import com.comcast.ip4s.IpAddress import com.comcast.ip4s.SocketAddress import fs2.io.internal.NativeUtil._ import fs2.io.internal.ResizableBuffer +import fs2.io.internal.SocketAddressHelpers._ import scala.scalanative.posix.unistd import java.util.concurrent.atomic.AtomicReference @@ -37,6 +38,7 @@ import FdPollingSocket._ private final class FdPollingSocket[F[_]]( fd: Int, + _remoteAddress: SocketAddress[IpAddress], readBuffer: ResizableBuffer[F], readMutex: Mutex[F], writeMutex: Mutex[F] @@ -53,8 +55,8 @@ private final class FdPollingSocket[F[_]]( guard_(unistd.close(fd)) } - def localAddress: F[SocketAddress[IpAddress]] = ??? - def remoteAddress: F[SocketAddress[IpAddress]] = ??? + def localAddress: F[SocketAddress[IpAddress]] = getLocalAddress(fd) + def remoteAddress: F[SocketAddress[IpAddress]] = F.pure(_remoteAddress) def endOfInput: F[Unit] = ??? def endOfOutput: F[Unit] = ??? From f93c66265dc1d45eb54596518c31aacdba453cf1 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sat, 17 Dec 2022 22:48:03 +0000 Subject: [PATCH 07/77] Implement `endOfInput`, `endOfOutput` --- io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index f8a57719f9..43d4bba24b 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -31,6 +31,7 @@ import fs2.io.internal.NativeUtil._ import fs2.io.internal.ResizableBuffer import fs2.io.internal.SocketAddressHelpers._ +import scala.scalanative.posix.sys.socket._ import scala.scalanative.posix.unistd import java.util.concurrent.atomic.AtomicReference @@ -58,8 +59,8 @@ private final class FdPollingSocket[F[_]]( def localAddress: F[SocketAddress[IpAddress]] = getLocalAddress(fd) def remoteAddress: F[SocketAddress[IpAddress]] = F.pure(_remoteAddress) - def endOfInput: F[Unit] = ??? - def endOfOutput: F[Unit] = ??? + def endOfInput: F[Unit] = F.delay(guard_(shutdown(fd, 0))) + def endOfOutput: F[Unit] = F.delay(guard_(shutdown(fd, 1))) private[this] val readCallback = new AtomicReference[Either[Throwable, Unit] => Unit] private[this] val writeCallback = new AtomicReference[Either[Throwable, Unit] => Unit] From 06c0fe76d77e728a49dc6bff419e4dc47604ad17 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sat, 17 Dec 2022 23:14:53 +0000 Subject: [PATCH 08/77] Wip socket reading --- .../scala/fs2/io/net/FdPollingSocket.scala | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 43d4bba24b..a9035cd4bb 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -25,6 +25,7 @@ package io.net import cats.effect.kernel.Async import cats.effect.std.Mutex import cats.effect.unsafe.FileDescriptorPoller +import cats.syntax.all._ import com.comcast.ip4s.IpAddress import com.comcast.ip4s.SocketAddress import fs2.io.internal.NativeUtil._ @@ -33,6 +34,7 @@ import fs2.io.internal.SocketAddressHelpers._ import scala.scalanative.posix.sys.socket._ import scala.scalanative.posix.unistd +import scala.scalanative.unsafe._ import java.util.concurrent.atomic.AtomicReference import FdPollingSocket._ @@ -65,7 +67,27 @@ private final class FdPollingSocket[F[_]]( private[this] val readCallback = new AtomicReference[Either[Throwable, Unit] => Unit] private[this] val writeCallback = new AtomicReference[Either[Throwable, Unit] => Unit] - def notifyFileDescriptorEvents(readReady: Boolean, writeReady: Boolean): Unit = ??? + def notifyFileDescriptorEvents(readReady: Boolean, writeReady: Boolean): Unit = { + if (readReady) { + val cb = readCallback.getAndSet(ReadySentinel) + if (cb ne null) cb(Either.unit) + } + if (writeReady) { + val cb = writeCallback.getAndSet(ReadySentinel) + if (cb ne null) cb(Either.unit) + } + } + + def awaitReadReady: F[Unit] = F.async { cb => + F.delay { + if (readCallback.compareAndSet(null, cb)) + Some(F.delay(readCallback.compareAndSet(cb, null))) + else { + cb(Either.unit) + None + } + } + } def read(maxBytes: Int): F[Option[Chunk[Byte]]] = ??? def readN(numBytes: Int): F[Chunk[Byte]] = ??? From a75a5abbdb6653a16f707509454b98de2caefecb Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sun, 18 Dec 2022 00:50:01 +0000 Subject: [PATCH 09/77] Take address as ctor args --- io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index a9035cd4bb..1084464a26 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -41,10 +41,11 @@ import FdPollingSocket._ private final class FdPollingSocket[F[_]]( fd: Int, - _remoteAddress: SocketAddress[IpAddress], readBuffer: ResizableBuffer[F], readMutex: Mutex[F], - writeMutex: Mutex[F] + writeMutex: Mutex[F], + val localAddress: F[SocketAddress[IpAddress]], + val remoteAddress: F[SocketAddress[IpAddress]] )(implicit F: Async[F]) extends Socket[F] with FileDescriptorPoller.Callback { @@ -58,9 +59,6 @@ private final class FdPollingSocket[F[_]]( guard_(unistd.close(fd)) } - def localAddress: F[SocketAddress[IpAddress]] = getLocalAddress(fd) - def remoteAddress: F[SocketAddress[IpAddress]] = F.pure(_remoteAddress) - def endOfInput: F[Unit] = F.delay(guard_(shutdown(fd, 0))) def endOfOutput: F[Unit] = F.delay(guard_(shutdown(fd, 1))) From c104cd71c4148c5b4aa5be43110d6bdc4d31652c Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sun, 18 Dec 2022 01:24:52 +0000 Subject: [PATCH 10/77] Implement reading --- .../scala/fs2/io/internal/NativeUtil.scala | 11 +++-- .../scala/fs2/io/net/FdPollingSocket.scala | 41 +++++++++++++++++-- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala index 8b3062d5ec..9f0fe5cbe5 100644 --- a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala +++ b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala @@ -23,6 +23,7 @@ package fs2.io.internal import scala.scalanative.annotation.alwaysinline import scala.scalanative.libc.errno._ +import scala.scalanative.posix.errno._ import scala.scalanative.posix.string._ import scala.scalanative.unsafe._ import java.io.IOException @@ -36,9 +37,13 @@ private[io] object NativeUtil { @alwaysinline def guard(thunk: => CInt): CInt = { val rtn = thunk - if (rtn < 0) - throw new IOException(fromCString(strerror(errno))) - else + if (rtn < 0) { + val en = errno + if (en == EAGAIN || en == EWOULDBLOCK) + rtn + else + throw new IOException(fromCString(strerror(errno))) + } else rtn } diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 1084464a26..c8919fb760 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -35,6 +35,7 @@ import fs2.io.internal.SocketAddressHelpers._ import scala.scalanative.posix.sys.socket._ import scala.scalanative.posix.unistd import scala.scalanative.unsafe._ +import scala.scalanative.unsigned._ import java.util.concurrent.atomic.AtomicReference import FdPollingSocket._ @@ -87,9 +88,41 @@ private final class FdPollingSocket[F[_]]( } } - def read(maxBytes: Int): F[Option[Chunk[Byte]]] = ??? - def readN(numBytes: Int): F[Chunk[Byte]] = ??? - def reads: Stream[F, Byte] = ??? + def read(maxBytes: Int): F[Option[Chunk[Byte]]] = readMutex.lock.surround { + readBuffer.get(maxBytes).flatMap { buf => + def go: F[Option[Chunk[Byte]]] = + F.delay(guard(unistd.read(fd, buf, maxBytes.toULong))).flatMap { rtn => + if (rtn > 0) + F.delay(Some(Chunk.fromBytePtr(buf, rtn))) + else if (rtn == 0) + F.pure(None) + else + awaitReadReady *> go + } + + go + } + } + + def readN(numBytes: Int): F[Chunk[Byte]] = readMutex.lock.surround { + readBuffer.get(numBytes).flatMap { buf => + def go(pos: Int): F[Chunk[Byte]] = + F.delay(guard(unistd.read(fd, buf + pos.toLong, (numBytes - pos).toULong))).flatMap { rtn => + if (rtn > 0) { + val newPos = pos + rtn + if (newPos < numBytes) go(newPos) + else F.delay(Chunk.fromBytePtr(buf, newPos)) + } else if (rtn == 0) + F.delay(Chunk.fromBytePtr(buf, pos)) + else + awaitReadReady *> go(pos) + } + + go(0) + } + } + + def reads: Stream[F, Byte] = Stream.repeatEval(read(DefaultReadSize)).unNoneTerminate.unchunks def write(bytes: Chunk[Byte]): F[Unit] = ??? def writes: Pipe[F, Byte, Nothing] = ??? @@ -98,6 +131,8 @@ private final class FdPollingSocket[F[_]]( private object FdPollingSocket { + private final val DefaultReadSize = 8192 + private val ReadySentinel: Either[Throwable, Unit] => Unit = _ => () } From c8e23167126727f97cd7150b4e93d0d8dba524c0 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sun, 18 Dec 2022 01:27:07 +0000 Subject: [PATCH 11/77] Address warnings --- .../src/main/scala/fs2/io/net/FdPollingSocket.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index c8919fb760..8ca28b94a0 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -30,11 +30,9 @@ import com.comcast.ip4s.IpAddress import com.comcast.ip4s.SocketAddress import fs2.io.internal.NativeUtil._ import fs2.io.internal.ResizableBuffer -import fs2.io.internal.SocketAddressHelpers._ import scala.scalanative.posix.sys.socket._ import scala.scalanative.posix.unistd -import scala.scalanative.unsafe._ import scala.scalanative.unsigned._ import java.util.concurrent.atomic.AtomicReference @@ -80,7 +78,12 @@ private final class FdPollingSocket[F[_]]( def awaitReadReady: F[Unit] = F.async { cb => F.delay { if (readCallback.compareAndSet(null, cb)) - Some(F.delay(readCallback.compareAndSet(cb, null))) + Some( + F.delay { + readCallback.compareAndSet(cb, null) + () + } + ) else { cb(Either.unit) None From f0fd81d836f941b4e1768d7a7696530fcbdf5f41 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sun, 18 Dec 2022 01:43:45 +0000 Subject: [PATCH 12/77] Bikeshed --- .../scala/fs2/io/net/FdPollingSocket.scala | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 8ca28b94a0..1b31245f8e 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -94,10 +94,10 @@ private final class FdPollingSocket[F[_]]( def read(maxBytes: Int): F[Option[Chunk[Byte]]] = readMutex.lock.surround { readBuffer.get(maxBytes).flatMap { buf => def go: F[Option[Chunk[Byte]]] = - F.delay(guard(unistd.read(fd, buf, maxBytes.toULong))).flatMap { rtn => - if (rtn > 0) - F.delay(Some(Chunk.fromBytePtr(buf, rtn))) - else if (rtn == 0) + F.delay(guard(unistd.read(fd, buf, maxBytes.toULong))).flatMap { readed => + if (readed > 0) + F.delay(Some(Chunk.fromBytePtr(buf, readed))) + else if (readed == 0) F.pure(None) else awaitReadReady *> go @@ -110,16 +110,17 @@ private final class FdPollingSocket[F[_]]( def readN(numBytes: Int): F[Chunk[Byte]] = readMutex.lock.surround { readBuffer.get(numBytes).flatMap { buf => def go(pos: Int): F[Chunk[Byte]] = - F.delay(guard(unistd.read(fd, buf + pos.toLong, (numBytes - pos).toULong))).flatMap { rtn => - if (rtn > 0) { - val newPos = pos + rtn - if (newPos < numBytes) go(newPos) - else F.delay(Chunk.fromBytePtr(buf, newPos)) - } else if (rtn == 0) - F.delay(Chunk.fromBytePtr(buf, pos)) - else - awaitReadReady *> go(pos) - } + F.delay(guard(unistd.read(fd, buf + pos.toLong, (numBytes - pos).toULong))) + .flatMap { readed => + if (readed > 0) { + val newPos = pos + readed + if (newPos < numBytes) go(newPos) + else F.delay(Chunk.fromBytePtr(buf, newPos)) + } else if (readed == 0) + F.delay(Chunk.fromBytePtr(buf, pos)) + else + awaitReadReady *> go(pos) + } go(0) } From 86ceb060960cb9625256fa37f424837ed4c00835 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Sun, 18 Dec 2022 01:51:00 +0000 Subject: [PATCH 13/77] Implement writing --- .../scala/fs2/io/net/FdPollingSocket.scala | 44 ++++++++++++++++++- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 1b31245f8e..15cd93aa09 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -31,8 +31,10 @@ import com.comcast.ip4s.SocketAddress import fs2.io.internal.NativeUtil._ import fs2.io.internal.ResizableBuffer +import scala.scalanative.meta.LinktimeInfo import scala.scalanative.posix.sys.socket._ import scala.scalanative.posix.unistd +import scala.scalanative.unsafe._ import scala.scalanative.unsigned._ import java.util.concurrent.atomic.AtomicReference @@ -128,8 +130,46 @@ private final class FdPollingSocket[F[_]]( def reads: Stream[F, Byte] = Stream.repeatEval(read(DefaultReadSize)).unNoneTerminate.unchunks - def write(bytes: Chunk[Byte]): F[Unit] = ??? - def writes: Pipe[F, Byte, Nothing] = ??? + def awaitWriteReady: F[Unit] = F.async { cb => + F.delay { + if (writeCallback.compareAndSet(null, cb)) + Some( + F.delay { + writeCallback.compareAndSet(cb, null) + () + } + ) + else { + cb(Either.unit) + None + } + } + } + + def write(bytes: Chunk[Byte]): F[Unit] = writeMutex.lock.surround { + val Chunk.ArraySlice(buf, offset, length) = bytes.toArraySlice + + def go(pos: Int): F[Unit] = + F.delay { + if (LinktimeInfo.isLinux) + send(fd, buf.at(offset + pos), (length - pos).toULong, MSG_NOSIGNAL).toInt + else + unistd.write(fd, buf.at(offset + pos), (length - pos).toULong) + }.flatMap { wrote => + if (wrote > 0) { + val newPos = pos + wrote + if (newPos < length) + go(newPos) + else + F.unit + } else + awaitWriteReady *> go(pos) + } + + go(0) + } + + def writes: Pipe[F, Byte, Nothing] = _.chunks.foreach(write(_)) } From 3eb0e18fd8e56965b4ca6dcd9de5a28d3a410dbd Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Wed, 21 Dec 2022 05:03:01 +0000 Subject: [PATCH 14/77] Bump CE snapshot, adopt new fd polling api --- build.sbt | 6 +- .../fs2/io/internal/ResizableBuffer.scala | 42 +++--- .../scala/fs2/io/net/FdPollingSocket.scala | 135 +++++------------- .../scala/fs2/io/net/tls/S2nConnection.scala | 2 +- 4 files changed, 67 insertions(+), 118 deletions(-) diff --git a/build.sbt b/build.sbt index b8d7c616ac..c532395f18 100644 --- a/build.sbt +++ b/build.sbt @@ -209,9 +209,9 @@ lazy val core = crossProject(JVMPlatform, JSPlatform, NativePlatform) libraryDependencies ++= Seq( "org.typelevel" %%% "cats-core" % "2.9.0", "org.typelevel" %%% "cats-laws" % "2.9.0" % Test, - "org.typelevel" %%% "cats-effect" % "3.5-4f9e57b", - "org.typelevel" %%% "cats-effect-laws" % "3.5-4f9e57b" % Test, - "org.typelevel" %%% "cats-effect-testkit" % "3.5-4f9e57b" % Test, + "org.typelevel" %%% "cats-effect" % "3.5-9ba870f", + "org.typelevel" %%% "cats-effect-laws" % "3.5-9ba870f" % Test, + "org.typelevel" %%% "cats-effect-testkit" % "3.5-9ba870f" % Test, "org.scodec" %%% "scodec-bits" % "1.1.34", "org.typelevel" %%% "scalacheck-effect-munit" % "2.0.0-M2" % Test, "org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3" % Test, diff --git a/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala b/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala index 1ec4310bd0..52b946c4b4 100644 --- a/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala +++ b/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala @@ -21,8 +21,10 @@ package fs2.io.internal +import cats.effect.kernel.Async import cats.effect.kernel.Resource -import cats.effect.kernel.Sync +import cats.effect.std.Semaphore +import cats.syntax.all._ import scala.scalanative.libc.errno._ import scala.scalanative.libc.stdlib._ @@ -31,19 +33,22 @@ import scala.scalanative.unsafe._ import scala.scalanative.unsigned._ private[io] final class ResizableBuffer[F[_]] private ( + semaphore: Semaphore[F], private var ptr: Ptr[Byte], private[this] var size: Int -)(implicit F: Sync[F]) { +)(implicit F: Async[F]) { - def get(size: Int): F[Ptr[Byte]] = F.delay { - if (size <= this.size) - ptr - else { - ptr = realloc(ptr, size.toUInt) - this.size = size - if (ptr == null) - throw new RuntimeException(fromCString(strerror(errno))) - else ptr + def get(size: Int): Resource[F, Ptr[Byte]] = semaphore.permit.evalMap { _ => + F.delay { + if (size <= this.size) + ptr + else { + ptr = realloc(ptr, size.toUInt) + this.size = size + if (ptr == null) + throw new RuntimeException(fromCString(strerror(errno))) + else ptr + } } } @@ -51,14 +56,17 @@ private[io] final class ResizableBuffer[F[_]] private ( private[io] object ResizableBuffer { - def apply[F[_]](size: Int)(implicit F: Sync[F]): Resource[F, ResizableBuffer[F]] = + def apply[F[_]](size: Int)(implicit F: Async[F]): Resource[F, ResizableBuffer[F]] = Resource.make { - F.delay { - val ptr = malloc(size.toUInt) - if (ptr == null) - throw new RuntimeException(fromCString(strerror(errno))) - else new ResizableBuffer(ptr, size) + Semaphore[F](1).flatMap { semaphore => + F.delay { + val ptr = malloc(size.toUInt) + if (ptr == null) + throw new RuntimeException(fromCString(strerror(errno))) + else new ResizableBuffer(semaphore, ptr, size) + } } + }(buf => F.delay(free(buf.ptr))) } diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 15cd93aa09..49bad882f0 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -22,9 +22,10 @@ package fs2 package io.net +import cats.effect.FileDescriptorPollHandle +import cats.effect.IO +import cats.effect.LiftIO import cats.effect.kernel.Async -import cats.effect.std.Mutex -import cats.effect.unsafe.FileDescriptorPoller import cats.syntax.all._ import com.comcast.ip4s.IpAddress import com.comcast.ip4s.SocketAddress @@ -36,20 +37,15 @@ import scala.scalanative.posix.sys.socket._ import scala.scalanative.posix.unistd import scala.scalanative.unsafe._ import scala.scalanative.unsigned._ -import java.util.concurrent.atomic.AtomicReference -import FdPollingSocket._ - -private final class FdPollingSocket[F[_]]( +private final class FdPollingSocket[F[_]: LiftIO]( fd: Int, + handle: FileDescriptorPollHandle, readBuffer: ResizableBuffer[F], - readMutex: Mutex[F], - writeMutex: Mutex[F], val localAddress: F[SocketAddress[IpAddress]], val remoteAddress: F[SocketAddress[IpAddress]] )(implicit F: Async[F]) - extends Socket[F] - with FileDescriptorPoller.Callback { + extends Socket[F] { @volatile private[this] var open = true @@ -63,120 +59,65 @@ private final class FdPollingSocket[F[_]]( def endOfInput: F[Unit] = F.delay(guard_(shutdown(fd, 0))) def endOfOutput: F[Unit] = F.delay(guard_(shutdown(fd, 1))) - private[this] val readCallback = new AtomicReference[Either[Throwable, Unit] => Unit] - private[this] val writeCallback = new AtomicReference[Either[Throwable, Unit] => Unit] - - def notifyFileDescriptorEvents(readReady: Boolean, writeReady: Boolean): Unit = { - if (readReady) { - val cb = readCallback.getAndSet(ReadySentinel) - if (cb ne null) cb(Either.unit) - } - if (writeReady) { - val cb = writeCallback.getAndSet(ReadySentinel) - if (cb ne null) cb(Either.unit) - } - } - - def awaitReadReady: F[Unit] = F.async { cb => - F.delay { - if (readCallback.compareAndSet(null, cb)) - Some( - F.delay { - readCallback.compareAndSet(cb, null) - () - } - ) - else { - cb(Either.unit) - None + def read(maxBytes: Int): F[Option[Chunk[Byte]]] = readBuffer.get(maxBytes).use { buf => + handle + .pollReadRec(()) { _ => + IO(guard(unistd.read(fd, buf, maxBytes.toULong))).flatMap { readed => + if (readed > 0) + IO(Right(Some(Chunk.fromBytePtr(buf, readed)))) + else if (readed == 0) + IO.pure(Right(None)) + else + IO.pure(Left(())) + } } - } + .to } - def read(maxBytes: Int): F[Option[Chunk[Byte]]] = readMutex.lock.surround { - readBuffer.get(maxBytes).flatMap { buf => - def go: F[Option[Chunk[Byte]]] = - F.delay(guard(unistd.read(fd, buf, maxBytes.toULong))).flatMap { readed => - if (readed > 0) - F.delay(Some(Chunk.fromBytePtr(buf, readed))) - else if (readed == 0) - F.pure(None) + def readN(numBytes: Int): F[Chunk[Byte]] = + readBuffer.get(numBytes).use { buf => + def go(pos: Int): IO[Either[Int, Chunk[Byte]]] = + IO(guard(unistd.read(fd, buf + pos.toLong, (numBytes - pos).toULong))).flatMap { readed => + if (readed > 0) { + val newPos = pos + readed + if (newPos < numBytes) go(newPos) + else IO(Right(Chunk.fromBytePtr(buf, newPos))) + } else if (readed == 0) + IO(Right(Chunk.fromBytePtr(buf, pos))) else - awaitReadReady *> go + IO.pure(Left(pos)) } - go + handle.pollReadRec(0)(go(_)).to } - } - def readN(numBytes: Int): F[Chunk[Byte]] = readMutex.lock.surround { - readBuffer.get(numBytes).flatMap { buf => - def go(pos: Int): F[Chunk[Byte]] = - F.delay(guard(unistd.read(fd, buf + pos.toLong, (numBytes - pos).toULong))) - .flatMap { readed => - if (readed > 0) { - val newPos = pos + readed - if (newPos < numBytes) go(newPos) - else F.delay(Chunk.fromBytePtr(buf, newPos)) - } else if (readed == 0) - F.delay(Chunk.fromBytePtr(buf, pos)) - else - awaitReadReady *> go(pos) - } - - go(0) - } - } + private[this] final val DefaultReadSize = 8192 def reads: Stream[F, Byte] = Stream.repeatEval(read(DefaultReadSize)).unNoneTerminate.unchunks - def awaitWriteReady: F[Unit] = F.async { cb => - F.delay { - if (writeCallback.compareAndSet(null, cb)) - Some( - F.delay { - writeCallback.compareAndSet(cb, null) - () - } - ) - else { - cb(Either.unit) - None - } - } - } - - def write(bytes: Chunk[Byte]): F[Unit] = writeMutex.lock.surround { + def write(bytes: Chunk[Byte]): F[Unit] = { val Chunk.ArraySlice(buf, offset, length) = bytes.toArraySlice - def go(pos: Int): F[Unit] = - F.delay { + def go(pos: Int): IO[Either[Int, Unit]] = + IO { if (LinktimeInfo.isLinux) send(fd, buf.at(offset + pos), (length - pos).toULong, MSG_NOSIGNAL).toInt else unistd.write(fd, buf.at(offset + pos), (length - pos).toULong) }.flatMap { wrote => - if (wrote > 0) { + if (wrote >= 0) { val newPos = pos + wrote if (newPos < length) go(newPos) else - F.unit + IO.pure(Either.unit) } else - awaitWriteReady *> go(pos) + IO.pure(Left(pos)) } - go(0) + handle.pollWriteRec(0)(go(_)).to } def writes: Pipe[F, Byte, Nothing] = _.chunks.foreach(write(_)) } - -private object FdPollingSocket { - - private final val DefaultReadSize = 8192 - - private val ReadySentinel: Either[Throwable, Unit] => Unit = _ => () - -} diff --git a/io/native/src/main/scala/fs2/io/net/tls/S2nConnection.scala b/io/native/src/main/scala/fs2/io/net/tls/S2nConnection.scala index 67dbd605dd..0eac89ff81 100644 --- a/io/native/src/main/scala/fs2/io/net/tls/S2nConnection.scala +++ b/io/native/src/main/scala/fs2/io/net/tls/S2nConnection.scala @@ -136,7 +136,7 @@ private[tls] object S2nConnection { }.iterateUntil(_.toInt == S2N_NOT_BLOCKED) *> F.delay(guard_(s2n_connection_free_handshake(conn))) - def read(n: Int) = readBuffer.get(n).flatMap { buf => + def read(n: Int) = readBuffer.get(n).use { buf => def go(i: Int): F[Option[Chunk[Byte]]] = F.delay { readTasks.set(F.unit) From 8889e05f05b11676fa7b83c543496909a6492c75 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 00:19:33 +0000 Subject: [PATCH 15/77] First attempt at unix sockets --- .../scala/fs2/io/internal/NativeUtil.scala | 7 + ...dressHelpers.scala => SocketHelpers.scala} | 43 ++++- .../scala/fs2/io/internal/syssocket.scala | 44 +++++ .../main/scala/fs2/io/internal/sysun.scala | 48 ++++++ .../src/main/scala/fs2/io/ioplatform.scala | 18 +- .../scala/fs2/io/net/FdPollingSocket.scala | 31 ++-- .../net/unixsocket/FdPollingUnixSockets.scala | 158 ++++++++++++++++++ 7 files changed, 334 insertions(+), 15 deletions(-) rename io/native/src/main/scala/fs2/io/internal/{SocketAddressHelpers.scala => SocketHelpers.scala} (83%) create mode 100644 io/native/src/main/scala/fs2/io/internal/syssocket.scala create mode 100644 io/native/src/main/scala/fs2/io/internal/sysun.scala create mode 100644 io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala diff --git a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala index 9f0fe5cbe5..80a0a89446 100644 --- a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala +++ b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala @@ -21,8 +21,11 @@ package fs2.io.internal +import cats.effect.Sync + import scala.scalanative.annotation.alwaysinline import scala.scalanative.libc.errno._ +import scala.scalanative.posix.fcntl._ import scala.scalanative.posix.errno._ import scala.scalanative.posix.string._ import scala.scalanative.unsafe._ @@ -47,4 +50,8 @@ private[io] object NativeUtil { rtn } + def setNonBlocking[F[_]](fd: CInt)(implicit F: Sync[F]): F[Unit] = F.delay { + guard_(fcntl(fd, F_SETFL, O_NONBLOCK)) + } + } diff --git a/io/native/src/main/scala/fs2/io/internal/SocketAddressHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala similarity index 83% rename from io/native/src/main/scala/fs2/io/internal/SocketAddressHelpers.scala rename to io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index a0a3ad2e76..b0521ea82b 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketAddressHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -31,20 +31,59 @@ import com.comcast.ip4s.Port import com.comcast.ip4s.SocketAddress import java.io.IOException +import scala.scalanative.meta.LinktimeInfo import scala.scalanative.posix.arpa.inet._ import scala.scalanative.posix.sys.socket._ import scala.scalanative.posix.sys.socketOps._ +import scala.scalanative.posix.unistd._ import scala.scalanative.unsafe._ import scala.scalanative.unsigned._ import NativeUtil._ import netinetin._ import netinetinOps._ +import syssocket._ -private[io] object SocketAddressHelpers { +private[io] object SocketHelpers { + + def openNonBlocking[F[_]](domain: CInt, `type`: CInt)(implicit F: Sync[F]): Resource[F, CInt] = + Resource + .make { + F.delay { + val SOCK_NONBLOCK = + if (LinktimeInfo.isLinux) + syssocket.SOCK_NONBLOCK + else 0 + + guard(socket(domain, `type` | SOCK_NONBLOCK, 0)) + } + }(fd => F.delay(guard_(close(fd)))) + .evalTap { fd => + (if (!LinktimeInfo.isLinux) setNonBlocking(fd) else F.unit) *> + (if (LinktimeInfo.isMac) setNoSigPipe(fd) else F.unit) + } + + // macOS-only + def setNoSigPipe[F[_]: Sync](fd: CInt): F[Unit] = + setOption(fd, SO_NOSIGPIPE, true) + + def setOption[F[_]](fd: CInt, option: CInt, value: Boolean)(implicit F: Sync[F]): F[Unit] = + F.delay { + val ptr = stackalloc[CInt]() + !ptr = if (value.asInstanceOf[java.lang.Boolean]) 1 else 0 + guard_( + setsockopt( + fd, + SOL_SOCKET, + option, + ptr.asInstanceOf[Ptr[Byte]], + sizeof[CInt].toUInt + ) + ) + } def getLocalAddress[F[_]](fd: Int)(implicit F: Sync[F]): F[SocketAddress[IpAddress]] = - SocketAddressHelpers.toSocketAddress { (addr, len) => + SocketHelpers.toSocketAddress { (addr, len) => F.delay(guard_(getsockname(fd, addr, len))) } diff --git a/io/native/src/main/scala/fs2/io/internal/syssocket.scala b/io/native/src/main/scala/fs2/io/internal/syssocket.scala new file mode 100644 index 0000000000..f8af29ee8e --- /dev/null +++ b/io/native/src/main/scala/fs2/io/internal/syssocket.scala @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.io.internal + +import scala.scalanative.posix.sys.socket._ +import scala.scalanative.unsafe._ + +@extern +private[io] object syssocket { + // only in Linux and FreeBSD, but not macOS + final val SOCK_NONBLOCK = 2048 + + // only on macOS and some BSDs (?) + final val SO_NOSIGPIPE = 0x1022 /* APPLE: No SIGPIPE on EPIPE */ + + def bind(sockfd: CInt, addr: Ptr[sockaddr], addrlen: socklen_t): CInt = + extern + + def accept(sockfd: CInt, addr: Ptr[sockaddr], addrlen: Ptr[socklen_t]): CInt = + extern + + // only supported on Linux and FreeBSD, but not macOS + def accept4(sockfd: CInt, addr: Ptr[sockaddr], addrlen: Ptr[socklen_t], flags: CInt): CInt = + extern +} diff --git a/io/native/src/main/scala/fs2/io/internal/sysun.scala b/io/native/src/main/scala/fs2/io/internal/sysun.scala new file mode 100644 index 0000000000..951a1ca346 --- /dev/null +++ b/io/native/src/main/scala/fs2/io/internal/sysun.scala @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2.io.internal + +import scala.scalanative.posix.sys.socket._ +import scala.scalanative.unsafe._ + +private[io] object sysun { + import Nat._ + type _108 = Digit3[_1, _0, _8] + + type sockaddr_un = CStruct2[ + sa_family_t, + CArray[CChar, _108] + ] + +} + +private[io] object sysunOps { + import sysun._ + + implicit final class sockaddr_unOps(val sockaddr_un: Ptr[sockaddr_un]) extends AnyVal { + def sun_family: sa_family_t = sockaddr_un._1 + def sun_family_=(sun_family: sa_family_t): Unit = sockaddr_un._1 = sun_family + def sun_path: CArray[CChar, _108] = sockaddr_un._2 + def sun_path_=(sun_path: CArray[CChar, _108]): Unit = sockaddr_un._2 = sun_path + } + +} diff --git a/io/native/src/main/scala/fs2/io/ioplatform.scala b/io/native/src/main/scala/fs2/io/ioplatform.scala index 4cdd4d1f8f..43c8807c99 100644 --- a/io/native/src/main/scala/fs2/io/ioplatform.scala +++ b/io/native/src/main/scala/fs2/io/ioplatform.scala @@ -22,4 +22,20 @@ package fs2 package io -private[fs2] trait ioplatform extends iojvmnative +import cats.effect.FileDescriptorPoller +import cats.effect.IO +import cats.effect.LiftIO +import cats.syntax.all._ + +private[fs2] trait ioplatform extends iojvmnative { + + private[fs2] def fileDescriptorPoller[F[_]: LiftIO]: F[FileDescriptorPoller] = + IO.poller[FileDescriptorPoller] + .flatMap( + _.liftTo[IO]( + new RuntimeException("Installed PollingSystem does not provide a FileDescriptorPoller") + ) + ) + .to + +} diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 49bad882f0..1fb4e7a71e 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -26,6 +26,7 @@ import cats.effect.FileDescriptorPollHandle import cats.effect.IO import cats.effect.LiftIO import cats.effect.kernel.Async +import cats.effect.kernel.Resource import cats.syntax.all._ import com.comcast.ip4s.IpAddress import com.comcast.ip4s.SocketAddress @@ -38,24 +39,18 @@ import scala.scalanative.posix.unistd import scala.scalanative.unsafe._ import scala.scalanative.unsigned._ -private final class FdPollingSocket[F[_]: LiftIO]( +import FdPollingSocket._ + +private final class FdPollingSocket[F[_]: LiftIO] private ( fd: Int, handle: FileDescriptorPollHandle, readBuffer: ResizableBuffer[F], + val isOpen: F[Boolean], val localAddress: F[SocketAddress[IpAddress]], val remoteAddress: F[SocketAddress[IpAddress]] )(implicit F: Async[F]) extends Socket[F] { - @volatile private[this] var open = true - - def isOpen: F[Boolean] = F.delay(open) - - def close: F[Unit] = F.delay { - open = false - guard_(unistd.close(fd)) - } - def endOfInput: F[Unit] = F.delay(guard_(shutdown(fd, 0))) def endOfOutput: F[Unit] = F.delay(guard_(shutdown(fd, 1))) @@ -91,8 +86,6 @@ private final class FdPollingSocket[F[_]: LiftIO]( handle.pollReadRec(0)(go(_)).to } - private[this] final val DefaultReadSize = 8192 - def reads: Stream[F, Byte] = Stream.repeatEval(read(DefaultReadSize)).unNoneTerminate.unchunks def write(bytes: Chunk[Byte]): F[Unit] = { @@ -121,3 +114,17 @@ private final class FdPollingSocket[F[_]: LiftIO]( def writes: Pipe[F, Byte, Nothing] = _.chunks.foreach(write(_)) } + +private object FdPollingSocket { + private final val DefaultReadSize = 8192 + + def apply[F[_]: LiftIO]( + fd: Int, + handle: FileDescriptorPollHandle, + localAddress: F[SocketAddress[IpAddress]], + remoteAddress: F[SocketAddress[IpAddress]] + )(implicit F: Async[F]): Resource[F, Socket[F]] = for { + buffer <- ResizableBuffer(DefaultReadSize) + isOpen <- Resource.make(F.ref(true))(_.set(false)) + } yield new FdPollingSocket(fd, handle, buffer, isOpen.get, localAddress, remoteAddress) +} diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala new file mode 100644 index 0000000000..a4fab6f097 --- /dev/null +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2 +package io +package net +package unixsocket + +import cats.effect.IO +import cats.effect.LiftIO +import cats.effect.kernel.Async +import cats.effect.kernel.Resource +import cats.syntax.all._ +import fs2.io.file.Files +import fs2.io.file.Path +import fs2.io.internal.NativeUtil._ +import fs2.io.internal.SocketHelpers +import fs2.io.internal.syssocket._ +import fs2.io.internal.sysun._ +import fs2.io.internal.sysunOps._ + +import scala.scalanative.libc.errno._ +import scala.scalanative.meta.LinktimeInfo +import scala.scalanative.posix.errno._ +import scala.scalanative.posix.string._ +import scala.scalanative.posix.sys.socket.{bind => _, accept => _, _} +import scala.scalanative.posix.unistd._ +import scala.scalanative.unsafe._ +import scala.scalanative.unsigned._ + +private final class FdPollingUnixSockets[F[_]: Files: LiftIO](implicit F: Async[F]) + extends UnixSockets[F] { + + def client(address: UnixSocketAddress): Resource[F, Socket[F]] = for { + poller <- Resource.eval(fileDescriptorPoller[F]) + fd <- SocketHelpers.openNonBlocking(AF_UNIX, SOCK_STREAM) + handle <- poller.registerFileDescriptor(fd, true, true).mapK(LiftIO.liftK) + _ <- Resource.eval { + toSockaddrUn(address.path).use { addr => + handle + .pollWriteRec(false) { connected => + if (connected) IO.pure(Either.unit) + else + IO { + if (connect(fd, addr, sizeof[sockaddr_un].toUInt) < 0) { + val e = errno + if (e == EINPROGRESS) + Left(true) // we will be connected when we unblock + else if (e == ECONNREFUSED) + throw new ConnectException(fromCString(strerror(errno))) + else + throw new IOException(fromCString(strerror(errno))) + } else + Either.unit + } + } + .to + } + } + socket <- FdPollingSocket[F](fd, handle, raiseIpAddressError, raiseIpAddressError) + } yield socket + + def server( + address: UnixSocketAddress, + deleteIfExists: Boolean, + deleteOnClose: Boolean + ): Stream[F, Socket[F]] = for { + poller <- Stream.eval(fileDescriptorPoller[F]) + + _ <- Stream.bracket(Files[F].deleteIfExists(Path(address.path)).whenA(deleteIfExists)) { _ => + Files[F].deleteIfExists(Path(address.path)).whenA(deleteOnClose) + } + + fd <- Stream.resource(SocketHelpers.openNonBlocking(AF_UNIX, SOCK_STREAM)) + handle <- Stream.resource(poller.registerFileDescriptor(fd, true, false).mapK(LiftIO.liftK)) + + _ <- Stream.eval { + toSockaddrUn(address.path).use { addr => + F.delay(guard_(bind(fd, addr, sizeof[sockaddr_un].toUInt))) + } *> F.delay(guard_(listen(fd, 0))) + } + + socket <- Stream + .resource { + val accepted = for { + fd <- Resource.makeFull[F, Int] { poll => + poll { + handle + .pollReadRec(()) { _ => + IO { + val clientFd = + if (LinktimeInfo.isLinux) + guard(accept(fd, null, null)) + else + guard(accept4(fd, null, null, SOCK_NONBLOCK)) + + if (clientFd >= 0) + Right(clientFd) + else + Left(()) + } + } + .to + } + }(fd => F.delay(guard_(close(fd)))) + _ <- + if (!LinktimeInfo.isLinux) + Resource.eval(setNonBlocking(fd)) + else Resource.unit[F] + handle <- poller.registerFileDescriptor(fd, true, true).mapK(LiftIO.liftK) + socket <- FdPollingSocket[F](fd, handle, raiseIpAddressError, raiseIpAddressError) + } yield socket + + accepted.attempt + .map(_.toOption) + } + .repeat + .unNone + + } yield socket + + private def toSockaddrUn(path: String): Resource[F, Ptr[sockaddr]] = + Resource.make(F.delay(Zone.open()))(z => F.delay(z.close())).evalMap[Ptr[sockaddr]] { + implicit z => + val pathBytes = path.getBytes + if (pathBytes.length > 107) + F.raiseError(new IllegalArgumentException(s"Path too long: $path")) + else + F.delay { + val addr = alloc[sockaddr_un]() + addr.sun_family = AF_UNIX.toUShort + memcpy(addr.sun_path.at(0), pathBytes.at(0), pathBytes.length.toULong) + addr.asInstanceOf[Ptr[sockaddr]] + } + } + + private def raiseIpAddressError[A]: F[A] = + F.raiseError(new UnsupportedOperationException("UnixSockets do not use IP addressing")) + +} From 5c62bc2728a8301dd25af3d14d7b8de51dbb30b1 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 00:37:49 +0000 Subject: [PATCH 16/77] Cross-compile unixsockets tests --- .../src/test/scala/fs2/io/Fs2IoSuite.scala | 25 ------------------- .../net/unixsocket/UnixSocketsPlatform.scala | 8 +++++- .../test/scala/fs2/io/net/tls/TLSSuite.scala | 2 +- .../UnixSocketsSuitePlatform.scala} | 8 +++--- io/shared/src/test/scala/fs2/io/IoSuite.scala | 2 +- .../test/scala/fs2/io/file/FilesSuite.scala | 2 +- .../test/scala/fs2/io/file/PathSuite.scala | 2 +- .../fs2/io/file/PosixPermissionsSuite.scala | 2 +- .../scala/fs2/io/net/tcp/SocketSuite.scala | 2 +- .../io/net/unixsocket/UnixSocketsSuite.scala | 0 10 files changed, 17 insertions(+), 36 deletions(-) delete mode 100644 io/js-jvm/src/test/scala/fs2/io/Fs2IoSuite.scala rename io/native/src/test/scala/fs2/io/{Fs2IoSuite.scala => net/unixsockets/UnixSocketsSuitePlatform.scala} (87%) rename io/{js-jvm => shared}/src/test/scala/fs2/io/net/unixsocket/UnixSocketsSuite.scala (100%) diff --git a/io/js-jvm/src/test/scala/fs2/io/Fs2IoSuite.scala b/io/js-jvm/src/test/scala/fs2/io/Fs2IoSuite.scala deleted file mode 100644 index 76c8042f20..0000000000 --- a/io/js-jvm/src/test/scala/fs2/io/Fs2IoSuite.scala +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (c) 2013 Functional Streams for Scala - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of - * this software and associated documentation files (the "Software"), to deal in - * the Software without restriction, including without limitation the rights to - * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of - * the Software, and to permit persons to whom the Software is furnished to do so, - * subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS - * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER - * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -package fs2 -package io - -abstract class Fs2IoSuite extends Fs2Suite diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/UnixSocketsPlatform.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/UnixSocketsPlatform.scala index e5599efece..689f56e6da 100644 --- a/io/native/src/main/scala/fs2/io/net/unixsocket/UnixSocketsPlatform.scala +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/UnixSocketsPlatform.scala @@ -21,4 +21,10 @@ package fs2.io.net.unixsocket -private[unixsocket] trait UnixSocketsCompanionPlatform +import cats.effect.LiftIO +import cats.effect.kernel.Async + +private[unixsocket] trait UnixSocketsCompanionPlatform { + implicit def forAsync[F[_]: Async: LiftIO]: UnixSockets[F] = + new FdPollingUnixSockets[F] +} diff --git a/io/native/src/test/scala/fs2/io/net/tls/TLSSuite.scala b/io/native/src/test/scala/fs2/io/net/tls/TLSSuite.scala index 4fb261bb27..fce760199e 100644 --- a/io/native/src/test/scala/fs2/io/net/tls/TLSSuite.scala +++ b/io/native/src/test/scala/fs2/io/net/tls/TLSSuite.scala @@ -30,7 +30,7 @@ import fs2.io.file.Files import fs2.io.file.Path import scodec.bits.ByteVector -abstract class TLSSuite extends Fs2IoSuite { +abstract class TLSSuite extends Fs2Suite { def testTlsContext: Resource[IO, TLSContext[IO]] = for { cert <- Resource.eval { Files[IO].readAll(Path("io/shared/src/test/resources/cert.pem")).compile.to(ByteVector) diff --git a/io/native/src/test/scala/fs2/io/Fs2IoSuite.scala b/io/native/src/test/scala/fs2/io/net/unixsockets/UnixSocketsSuitePlatform.scala similarity index 87% rename from io/native/src/test/scala/fs2/io/Fs2IoSuite.scala rename to io/native/src/test/scala/fs2/io/net/unixsockets/UnixSocketsSuitePlatform.scala index 40512de1ab..92ac7a5949 100644 --- a/io/native/src/test/scala/fs2/io/Fs2IoSuite.scala +++ b/io/native/src/test/scala/fs2/io/net/unixsockets/UnixSocketsSuitePlatform.scala @@ -20,10 +20,10 @@ */ package fs2 -package io +package io.net.unixsocket -import epollcat.unsafe.EpollRuntime +import cats.effect.IO -abstract class Fs2IoSuite extends Fs2Suite { - override def munitIORuntime = EpollRuntime.global +trait UnixSocketsSuitePlatform { self: UnixSocketsSuite => + testProvider("native")(UnixSockets.forAsync[IO]) } diff --git a/io/shared/src/test/scala/fs2/io/IoSuite.scala b/io/shared/src/test/scala/fs2/io/IoSuite.scala index 387d32adad..23ffd78022 100644 --- a/io/shared/src/test/scala/fs2/io/IoSuite.scala +++ b/io/shared/src/test/scala/fs2/io/IoSuite.scala @@ -28,7 +28,7 @@ import org.scalacheck.effect.PropF.forAllF import java.io.ByteArrayInputStream import java.io.InputStream -class IoSuite extends io.Fs2IoSuite { +class IoSuite extends Fs2Suite { group("readInputStream") { test("non-buffered") { forAllF { (bytes: Array[Byte], chunkSize0: Int) => diff --git a/io/shared/src/test/scala/fs2/io/file/FilesSuite.scala b/io/shared/src/test/scala/fs2/io/file/FilesSuite.scala index f9f9cf2929..a514344ac1 100644 --- a/io/shared/src/test/scala/fs2/io/file/FilesSuite.scala +++ b/io/shared/src/test/scala/fs2/io/file/FilesSuite.scala @@ -29,7 +29,7 @@ import cats.syntax.all._ import scala.concurrent.duration._ -class FilesSuite extends Fs2IoSuite with BaseFileSuite { +class FilesSuite extends Fs2Suite with BaseFileSuite { group("readAll") { test("retrieves whole content of a file") { diff --git a/io/shared/src/test/scala/fs2/io/file/PathSuite.scala b/io/shared/src/test/scala/fs2/io/file/PathSuite.scala index e0229b32df..d25a80305f 100644 --- a/io/shared/src/test/scala/fs2/io/file/PathSuite.scala +++ b/io/shared/src/test/scala/fs2/io/file/PathSuite.scala @@ -31,7 +31,7 @@ import org.scalacheck.Cogen import org.scalacheck.Gen import org.scalacheck.Prop.forAll -class PathSuite extends Fs2IoSuite { +class PathSuite extends Fs2Suite { implicit val arbitraryPath: Arbitrary[Path] = Arbitrary(for { names <- Gen.listOf(Gen.alphaNumStr) diff --git a/io/shared/src/test/scala/fs2/io/file/PosixPermissionsSuite.scala b/io/shared/src/test/scala/fs2/io/file/PosixPermissionsSuite.scala index 17df211606..79f116bde1 100644 --- a/io/shared/src/test/scala/fs2/io/file/PosixPermissionsSuite.scala +++ b/io/shared/src/test/scala/fs2/io/file/PosixPermissionsSuite.scala @@ -23,7 +23,7 @@ package fs2 package io package file -class PosixPermissionsSuite extends Fs2IoSuite { +class PosixPermissionsSuite extends Fs2Suite { test("construction") { val cases = Seq( "777" -> "rwxrwxrwx", diff --git a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala index b7f0c761b2..d0c28e0c73 100644 --- a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala +++ b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala @@ -31,7 +31,7 @@ import com.comcast.ip4s._ import scala.concurrent.duration._ import scala.concurrent.TimeoutException -class SocketSuite extends Fs2IoSuite with SocketSuitePlatform { +class SocketSuite extends Fs2Suite with SocketSuitePlatform { val timeout = 30.seconds diff --git a/io/js-jvm/src/test/scala/fs2/io/net/unixsocket/UnixSocketsSuite.scala b/io/shared/src/test/scala/fs2/io/net/unixsocket/UnixSocketsSuite.scala similarity index 100% rename from io/js-jvm/src/test/scala/fs2/io/net/unixsocket/UnixSocketsSuite.scala rename to io/shared/src/test/scala/fs2/io/net/unixsocket/UnixSocketsSuite.scala From adba4328380863765e2a7016411d61c001e35cce Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 00:46:34 +0000 Subject: [PATCH 17/77] Workaround another borked method --- io/native/src/main/scala/fs2/io/internal/syssocket.scala | 3 +++ .../scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/io/native/src/main/scala/fs2/io/internal/syssocket.scala b/io/native/src/main/scala/fs2/io/internal/syssocket.scala index f8af29ee8e..984be7acb0 100644 --- a/io/native/src/main/scala/fs2/io/internal/syssocket.scala +++ b/io/native/src/main/scala/fs2/io/internal/syssocket.scala @@ -35,6 +35,9 @@ private[io] object syssocket { def bind(sockfd: CInt, addr: Ptr[sockaddr], addrlen: socklen_t): CInt = extern + def connect(sockfd: CInt, addr: Ptr[sockaddr], addrlen: socklen_t): CInt = + extern + def accept(sockfd: CInt, addr: Ptr[sockaddr], addrlen: Ptr[socklen_t]): CInt = extern diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala index a4fab6f097..0e24f7935c 100644 --- a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala @@ -41,7 +41,7 @@ import scala.scalanative.libc.errno._ import scala.scalanative.meta.LinktimeInfo import scala.scalanative.posix.errno._ import scala.scalanative.posix.string._ -import scala.scalanative.posix.sys.socket.{bind => _, accept => _, _} +import scala.scalanative.posix.sys.socket.{bind => _, connect => _, accept => _, _} import scala.scalanative.posix.unistd._ import scala.scalanative.unsafe._ import scala.scalanative.unsigned._ From 0aa9260604d6efc9035facad986c59da2e4991c6 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 01:48:21 +0000 Subject: [PATCH 18/77] Bump base version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index c532395f18..f4f45284ec 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import com.typesafe.tools.mima.core._ Global / onChangedBuildSource := ReloadOnSourceChanges -ThisBuild / tlBaseVersion := "3.4" +ThisBuild / tlBaseVersion := "3.5" ThisBuild / organization := "co.fs2" ThisBuild / organizationName := "Functional Streams for Scala" From 64d7a761c3ce84932b299e7d2b18bd3b781ecf69 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 04:01:27 +0000 Subject: [PATCH 19/77] Implement non-blocking std{in,out,err} --- .../src/main/scala/fs2/io/iojvmnative.scala | 30 ---- io/jvm/src/main/scala/fs2/io/ioplatform.scala | 30 ++++ .../src/main/scala/fs2/io/ioplatform.scala | 129 ++++++++++++++++++ 3 files changed, 159 insertions(+), 30 deletions(-) diff --git a/io/jvm-native/src/main/scala/fs2/io/iojvmnative.scala b/io/jvm-native/src/main/scala/fs2/io/iojvmnative.scala index 4066bafc78..8bcdeed359 100644 --- a/io/jvm-native/src/main/scala/fs2/io/iojvmnative.scala +++ b/io/jvm-native/src/main/scala/fs2/io/iojvmnative.scala @@ -22,46 +22,16 @@ package fs2 package io -import cats._ import cats.effect.kernel.Sync import cats.effect.kernel.implicits._ import cats.syntax.all._ -import java.nio.charset.Charset -import java.nio.charset.StandardCharsets import scala.reflect.ClassTag private[fs2] trait iojvmnative { type InterruptedIOException = java.io.InterruptedIOException type ClosedChannelException = java.nio.channels.ClosedChannelException - // - // STDIN/STDOUT Helpers - - /** Stream of bytes read asynchronously from standard input. */ - def stdin[F[_]: Sync](bufSize: Int): Stream[F, Byte] = - readInputStream(Sync[F].blocking(System.in), bufSize, false) - - /** Pipe of bytes that writes emitted values to standard output asynchronously. */ - def stdout[F[_]: Sync]: Pipe[F, Byte, Nothing] = - writeOutputStream(Sync[F].blocking(System.out), false) - - /** Pipe of bytes that writes emitted values to standard error asynchronously. */ - def stderr[F[_]: Sync]: Pipe[F, Byte, Nothing] = - writeOutputStream(Sync[F].blocking(System.err), false) - - /** Writes this stream to standard output asynchronously, converting each element to - * a sequence of bytes via `Show` and the given `Charset`. - */ - def stdoutLines[F[_]: Sync, O: Show]( - charset: Charset = StandardCharsets.UTF_8 - ): Pipe[F, O, Nothing] = - _.map(_.show).through(text.encode(charset)).through(stdout) - - /** Stream of `String` read asynchronously from standard input decoded in UTF-8. */ - def stdinUtf8[F[_]: Sync](bufSize: Int): Stream[F, String] = - stdin(bufSize).through(text.utf8.decode) - /** Stream of bytes read asynchronously from the specified resource relative to the class `C`. * @see [[readClassLoaderResource]] for a resource relative to a classloader. */ diff --git a/io/jvm/src/main/scala/fs2/io/ioplatform.scala b/io/jvm/src/main/scala/fs2/io/ioplatform.scala index 1bf51864aa..499e8d5b55 100644 --- a/io/jvm/src/main/scala/fs2/io/ioplatform.scala +++ b/io/jvm/src/main/scala/fs2/io/ioplatform.scala @@ -22,6 +22,7 @@ package fs2 package io +import cats.Show import cats.effect.kernel.{Async, Outcome, Resource, Sync} import cats.effect.kernel.implicits._ import cats.effect.kernel.Deferred @@ -29,9 +30,38 @@ import cats.syntax.all._ import fs2.io.internal.PipedStreamBuffer import java.io.{InputStream, OutputStream} +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets private[fs2] trait ioplatform extends iojvmnative { + // + // STDIN/STDOUT Helpers + + /** Stream of bytes read asynchronously from standard input. */ + def stdin[F[_]: Sync](bufSize: Int): Stream[F, Byte] = + readInputStream(Sync[F].blocking(System.in), bufSize, false) + + /** Pipe of bytes that writes emitted values to standard output asynchronously. */ + def stdout[F[_]: Sync]: Pipe[F, Byte, Nothing] = + writeOutputStream(Sync[F].blocking(System.out), false) + + /** Pipe of bytes that writes emitted values to standard error asynchronously. */ + def stderr[F[_]: Sync]: Pipe[F, Byte, Nothing] = + writeOutputStream(Sync[F].blocking(System.err), false) + + /** Writes this stream to standard output asynchronously, converting each element to + * a sequence of bytes via `Show` and the given `Charset`. + */ + def stdoutLines[F[_]: Sync, O: Show]( + charset: Charset = StandardCharsets.UTF_8 + ): Pipe[F, O, Nothing] = + _.map(_.show).through(text.encode(charset)).through(stdout) + + /** Stream of `String` read asynchronously from standard input decoded in UTF-8. */ + def stdinUtf8[F[_]: Sync](bufSize: Int): Stream[F, String] = + stdin(bufSize).through(text.utf8.decode) + /** Pipe that converts a stream of bytes to a stream that will emit a single `java.io.InputStream`, * that is closed whenever the resulting stream terminates. * diff --git a/io/native/src/main/scala/fs2/io/ioplatform.scala b/io/native/src/main/scala/fs2/io/ioplatform.scala index 43c8807c99..66e96834b8 100644 --- a/io/native/src/main/scala/fs2/io/ioplatform.scala +++ b/io/native/src/main/scala/fs2/io/ioplatform.scala @@ -22,10 +22,23 @@ package fs2 package io +import cats.Show import cats.effect.FileDescriptorPoller import cats.effect.IO import cats.effect.LiftIO +import cats.effect.kernel.Async +import cats.effect.kernel.Resource +import cats.effect.kernel.Sync import cats.syntax.all._ +import fs2.io.internal.NativeUtil._ + +import java.io.OutputStream +import java.nio.charset.Charset +import java.nio.charset.StandardCharsets +import scala.scalanative.meta.LinktimeInfo +import scala.scalanative.posix.unistd._ +import scala.scalanative.unsafe._ +import scala.scalanative.unsigned._ private[fs2] trait ioplatform extends iojvmnative { @@ -38,4 +51,120 @@ private[fs2] trait ioplatform extends iojvmnative { ) .to + // + // STDIN/STDOUT Helpers + + /** Stream of bytes read asynchronously from standard input. */ + def stdin[F[_]: Async: LiftIO](bufSize: Int): Stream[F, Byte] = + if (LinktimeInfo.isLinux || LinktimeInfo.isMac) + Stream + .resource { + Resource + .eval { + setNonBlocking(STDIN_FILENO) *> fileDescriptorPoller[F] + } + .flatMap { poller => + poller.registerFileDescriptor(STDIN_FILENO, true, false).mapK(LiftIO.liftK) + } + } + .flatMap { handle => + Stream.repeatEval { + handle + .pollReadRec(()) { _ => + IO { + val buf = new Array[Byte](bufSize) + val readed = guard(read(STDIN_FILENO, buf.at(0), bufSize.toULong)) + if (readed > 0) + Right(Some(Chunk.array(buf, 0, readed))) + else if (readed == 0) + Right(None) + else + Left(()) + } + } + .to + } + } + .unNoneTerminate + .unchunks + else + readInputStream(Sync[F].blocking(System.in), bufSize, false) + + /** Pipe of bytes that writes emitted values to standard output asynchronously. */ + def stdout[F[_]: Async: LiftIO]: Pipe[F, Byte, Nothing] = + if (LinktimeInfo.isLinux || LinktimeInfo.isMac) + writeFd(STDOUT_FILENO) + else + writeOutputStream(Sync[F].blocking(System.out), false) + + /** Pipe of bytes that writes emitted values to standard error asynchronously. */ + def stderr[F[_]: Async: LiftIO]: Pipe[F, Byte, Nothing] = + if (LinktimeInfo.isLinux || LinktimeInfo.isMac) + writeFd(STDERR_FILENO) + else + writeOutputStream(Sync[F].blocking(System.err), false) + + private[this] def writeFd[F[_]: Async: LiftIO](fd: Int): Pipe[F, Byte, Nothing] = in => + Stream + .resource { + Resource + .eval { + setNonBlocking(fd) *> fileDescriptorPoller[F] + } + .flatMap { poller => + poller.registerFileDescriptor(fd, false, true).mapK(LiftIO.liftK) + } + } + .flatMap { handle => + in.chunks.foreach { bytes => + val Chunk.ArraySlice(buf, offset, length) = bytes.toArraySlice + + def go(pos: Int): IO[Either[Int, Unit]] = + IO(write(fd, buf.at(offset + pos), (length - pos).toULong)).flatMap { wrote => + if (wrote >= 0) { + val newPos = pos + wrote + if (newPos < length) + go(newPos) + else + IO.pure(Either.unit) + } else + IO.pure(Left(pos)) + } + + handle.pollWriteRec(0)(go(_)).to + } + } + + /** Writes this stream to standard output asynchronously, converting each element to + * a sequence of bytes via `Show` and the given `Charset`. + */ + def stdoutLines[F[_]: Async: LiftIO, O: Show]( + charset: Charset = StandardCharsets.UTF_8 + ): Pipe[F, O, Nothing] = + _.map(_.show).through(text.encode(charset)).through(stdout) + + /** Stream of `String` read asynchronously from standard input decoded in UTF-8. */ + def stdinUtf8[F[_]: Async: LiftIO](bufSize: Int): Stream[F, String] = + stdin(bufSize).through(text.utf8.decode) + + @deprecated("Prefer non-blocking, async variant", "3.5.0") + def stdin[F[_]](bufSize: Int, F: Sync[F]): Stream[F, Byte] = + readInputStream(F.blocking(System.in), bufSize, false)(F) + + @deprecated("Prefer non-blocking, async variant", "3.5.0") + def stdout[F[_]](F: Sync[F]): Pipe[F, Byte, Nothing] = + writeOutputStream(F.blocking(System.out: OutputStream), false)(F) + + @deprecated("Prefer non-blocking, async variant", "3.5.0") + def stderr[F[_]](F: Sync[F]): Pipe[F, Byte, Nothing] = + writeOutputStream(F.blocking(System.err: OutputStream), false)(F) + + @deprecated("Prefer non-blocking, async variant", "3.5.0") + def stdoutLines[F[_], O](charset: Charset, F: Sync[F], O: Show[O]): Pipe[F, O, Nothing] = + _.map(O.show(_)).through(text.encode(charset)).through(stdout(F)) + + @deprecated("Prefer non-blocking, async variant", "3.5.0") + def stdinUtf8[F[_]](bufSize: Int, F: Sync[F]): Stream[F, String] = + stdin(bufSize, F).through(text.utf8.decode) + } From c27e03d3989ae705d4fc1d42ed262843817f5097 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 04:27:20 +0000 Subject: [PATCH 20/77] Fix unix socket error handling --- .../scala/fs2/io/internal/SocketHelpers.scala | 17 +++++++++++++++++ .../net/unixsocket/FdPollingUnixSockets.scala | 4 ++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index b0521ea82b..4523709a0f 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -33,6 +33,7 @@ import com.comcast.ip4s.SocketAddress import java.io.IOException import scala.scalanative.meta.LinktimeInfo import scala.scalanative.posix.arpa.inet._ +import scala.scalanative.posix.string._ import scala.scalanative.posix.sys.socket._ import scala.scalanative.posix.sys.socketOps._ import scala.scalanative.posix.unistd._ @@ -82,6 +83,22 @@ private[io] object SocketHelpers { ) } + def raiseSocketError[F[_]](fd: Int)(implicit F: Sync[F]): F[Unit] = F.delay { + val optval = stackalloc[CInt]() + val optlen = stackalloc[socklen_t]() + guard_ { + getsockopt( + fd, + SOL_SOCKET, + SO_ERROR, + optval.asInstanceOf[Ptr[Byte]], + optlen + ) + } + if (!optval != 0) + throw new IOException(fromCString(strerror(!optval))) + } + def getLocalAddress[F[_]](fd: Int)(implicit F: Sync[F]): F[SocketAddress[IpAddress]] = SocketHelpers.toSocketAddress { (addr, len) => F.delay(guard_(getsockname(fd, addr, len))) diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala index 0e24f7935c..5587ce2bc4 100644 --- a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala @@ -57,12 +57,12 @@ private final class FdPollingUnixSockets[F[_]: Files: LiftIO](implicit F: Async[ toSockaddrUn(address.path).use { addr => handle .pollWriteRec(false) { connected => - if (connected) IO.pure(Either.unit) + if (connected) SocketHelpers.raiseSocketError[IO](fd).as(Either.unit) else IO { if (connect(fd, addr, sizeof[sockaddr_un].toUInt) < 0) { val e = errno - if (e == EINPROGRESS) + if (e == EAGAIN) Left(true) // we will be connected when we unblock else if (e == ECONNREFUSED) throw new ConnectException(fromCString(strerror(errno))) From ebc0ca52362289d28be487a8728e95db18550099 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 04:29:48 +0000 Subject: [PATCH 21/77] Simplify unix socket connect --- .../io/net/unixsocket/FdPollingUnixSockets.scala | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala index 5587ce2bc4..939b8ef28b 100644 --- a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala @@ -37,9 +37,7 @@ import fs2.io.internal.syssocket._ import fs2.io.internal.sysun._ import fs2.io.internal.sysunOps._ -import scala.scalanative.libc.errno._ import scala.scalanative.meta.LinktimeInfo -import scala.scalanative.posix.errno._ import scala.scalanative.posix.string._ import scala.scalanative.posix.sys.socket.{bind => _, connect => _, accept => _, _} import scala.scalanative.posix.unistd._ @@ -60,16 +58,8 @@ private final class FdPollingUnixSockets[F[_]: Files: LiftIO](implicit F: Async[ if (connected) SocketHelpers.raiseSocketError[IO](fd).as(Either.unit) else IO { - if (connect(fd, addr, sizeof[sockaddr_un].toUInt) < 0) { - val e = errno - if (e == EAGAIN) - Left(true) // we will be connected when we unblock - else if (e == ECONNREFUSED) - throw new ConnectException(fromCString(strerror(errno))) - else - throw new IOException(fromCString(strerror(errno))) - } else - Either.unit + guard_(connect(fd, addr, sizeof[sockaddr_un].toUInt)) + Either.unit[Boolean] } } .to From 43b0c06a55e1863952ed331ffe6a7cf18c467257 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 04:58:09 +0000 Subject: [PATCH 22/77] Fix simplified unix socket connect --- .../scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala index 939b8ef28b..cb48186df0 100644 --- a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala @@ -58,8 +58,10 @@ private final class FdPollingUnixSockets[F[_]: Files: LiftIO](implicit F: Async[ if (connected) SocketHelpers.raiseSocketError[IO](fd).as(Either.unit) else IO { - guard_(connect(fd, addr, sizeof[sockaddr_un].toUInt)) - Either.unit[Boolean] + if (guard(connect(fd, addr, sizeof[sockaddr_un].toUInt)) < 0) + Left(true) // we will be connected when unblocked + else + Either.unit[Boolean] } } .to From 6e25eab62bff48af343e65c96d14f2c74342e60b Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 05:05:36 +0000 Subject: [PATCH 23/77] stackalloc `sockaddr_un` ftw --- .../net/unixsocket/FdPollingUnixSockets.scala | 47 +++++++++---------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala index cb48186df0..36e6403b16 100644 --- a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala @@ -52,20 +52,20 @@ private final class FdPollingUnixSockets[F[_]: Files: LiftIO](implicit F: Async[ fd <- SocketHelpers.openNonBlocking(AF_UNIX, SOCK_STREAM) handle <- poller.registerFileDescriptor(fd, true, true).mapK(LiftIO.liftK) _ <- Resource.eval { - toSockaddrUn(address.path).use { addr => - handle - .pollWriteRec(false) { connected => - if (connected) SocketHelpers.raiseSocketError[IO](fd).as(Either.unit) - else - IO { + handle + .pollWriteRec(false) { connected => + if (connected) SocketHelpers.raiseSocketError[IO](fd).as(Either.unit) + else + IO { + toSockaddrUn(address.path) { addr => if (guard(connect(fd, addr, sizeof[sockaddr_un].toUInt)) < 0) Left(true) // we will be connected when unblocked else Either.unit[Boolean] } - } - .to - } + } + } + .to } socket <- FdPollingSocket[F](fd, handle, raiseIpAddressError, raiseIpAddressError) } yield socket @@ -85,8 +85,8 @@ private final class FdPollingUnixSockets[F[_]: Files: LiftIO](implicit F: Async[ handle <- Stream.resource(poller.registerFileDescriptor(fd, true, false).mapK(LiftIO.liftK)) _ <- Stream.eval { - toSockaddrUn(address.path).use { addr => - F.delay(guard_(bind(fd, addr, sizeof[sockaddr_un].toUInt))) + F.delay { + toSockaddrUn(address.path)(addr => guard_(bind(fd, addr, sizeof[sockaddr_un].toUInt))) } *> F.delay(guard_(listen(fd, 0))) } @@ -129,20 +129,17 @@ private final class FdPollingUnixSockets[F[_]: Files: LiftIO](implicit F: Async[ } yield socket - private def toSockaddrUn(path: String): Resource[F, Ptr[sockaddr]] = - Resource.make(F.delay(Zone.open()))(z => F.delay(z.close())).evalMap[Ptr[sockaddr]] { - implicit z => - val pathBytes = path.getBytes - if (pathBytes.length > 107) - F.raiseError(new IllegalArgumentException(s"Path too long: $path")) - else - F.delay { - val addr = alloc[sockaddr_un]() - addr.sun_family = AF_UNIX.toUShort - memcpy(addr.sun_path.at(0), pathBytes.at(0), pathBytes.length.toULong) - addr.asInstanceOf[Ptr[sockaddr]] - } - } + private def toSockaddrUn[A](path: String)(f: Ptr[sockaddr] => A): A = { + val pathBytes = path.getBytes + if (pathBytes.length > 107) + throw new IllegalArgumentException(s"Path too long: $path") + + val addr = stackalloc[sockaddr_un]() + addr.sun_family = AF_UNIX.toUShort + memcpy(addr.sun_path.at(0), pathBytes.at(0), pathBytes.length.toULong) + + f(addr.asInstanceOf[Ptr[sockaddr]]) + } private def raiseIpAddressError[A]: F[A] = F.raiseError(new UnsupportedOperationException("UnixSockets do not use IP addressing")) From 1cf0db4e4860408fa79c73298781b26a6546a617 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 05:31:16 +0000 Subject: [PATCH 24/77] Add more socket option helpers --- .../scala/fs2/io/internal/SocketHelpers.scala | 76 +++++++++++++++---- 1 file changed, 62 insertions(+), 14 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index 4523709a0f..5149b14369 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -31,8 +31,12 @@ import com.comcast.ip4s.Port import com.comcast.ip4s.SocketAddress import java.io.IOException +import java.net.SocketOption +import java.net.StandardSocketOptions import scala.scalanative.meta.LinktimeInfo import scala.scalanative.posix.arpa.inet._ +import scala.scalanative.posix.netinet.in.IPPROTO_TCP +import scala.scalanative.posix.netinet.tcp._ import scala.scalanative.posix.string._ import scala.scalanative.posix.sys.socket._ import scala.scalanative.posix.sys.socketOps._ @@ -68,14 +72,70 @@ private[io] object SocketHelpers { def setNoSigPipe[F[_]: Sync](fd: CInt): F[Unit] = setOption(fd, SO_NOSIGPIPE, true) + def setOption[F[_]: Sync, T](fd: CInt, name: SocketOption[T], value: T): F[Unit] = name match { + case StandardSocketOptions.SO_SNDBUF => + setOption( + fd, + SO_SNDBUF, + value.asInstanceOf[java.lang.Integer] + ) + case StandardSocketOptions.SO_RCVBUF => + setOption( + fd, + SO_RCVBUF, + value.asInstanceOf[java.lang.Integer] + ) + case StandardSocketOptions.SO_REUSEADDR => + setOption( + fd, + SO_REUSEADDR, + value.asInstanceOf[java.lang.Boolean] + ) + case StandardSocketOptions.SO_REUSEPORT => + SocketHelpers.setOption( + fd, + SO_REUSEPORT, + value.asInstanceOf[java.lang.Boolean] + ) + case StandardSocketOptions.SO_KEEPALIVE => + SocketHelpers.setOption( + fd, + SO_KEEPALIVE, + value.asInstanceOf[java.lang.Boolean] + ) + case StandardSocketOptions.TCP_NODELAY => + setTcpOption( + fd, + TCP_NODELAY, + value.asInstanceOf[java.lang.Boolean] + ) + case _ => throw new IllegalArgumentException + } + def setOption[F[_]](fd: CInt, option: CInt, value: Boolean)(implicit F: Sync[F]): F[Unit] = + setOptionImpl(fd, SOL_SOCKET, option, if (value) 1 else 0) + + def setOption[F[_]](fd: CInt, option: CInt, value: CInt)(implicit F: Sync[F]): F[Unit] = + setOptionImpl(fd, SOL_SOCKET, option, value) + + def setTcpOption[F[_]](fd: CInt, option: CInt, value: Boolean)(implicit F: Sync[F]): F[Unit] = + setOptionImpl( + fd, + IPPROTO_TCP, // aka SOL_TCP + option, + if (value) 1 else 0 + ) + + def setOptionImpl[F[_]](fd: CInt, level: CInt, option: CInt, value: CInt)(implicit + F: Sync[F] + ): F[Unit] = F.delay { val ptr = stackalloc[CInt]() - !ptr = if (value.asInstanceOf[java.lang.Boolean]) 1 else 0 + !ptr = value guard_( setsockopt( fd, - SOL_SOCKET, + level, option, ptr.asInstanceOf[Ptr[Byte]], sizeof[CInt].toUInt @@ -104,18 +164,6 @@ private[io] object SocketHelpers { F.delay(guard_(getsockname(fd, addr, len))) } - def allocateSockaddr[F[_]](implicit F: Sync[F]): Resource[F, (Ptr[sockaddr], Ptr[socklen_t])] = - Resource - .make(F.delay(Zone.open()))(z => F.delay(z.close())) - .evalMap { implicit z => - F.delay { - val addr = // allocate enough for an IPv6 - alloc[sockaddr_in6]().asInstanceOf[Ptr[sockaddr]] - val len = alloc[socklen_t]() - (addr, len) - } - } - def toSockaddr[A]( address: SocketAddress[IpAddress] )(f: (Ptr[sockaddr], socklen_t) => A): A = From 74b80f90ae8197fdb28e42260d2169b0a8c488c7 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 06:45:29 +0000 Subject: [PATCH 25/77] `accept4` is a linux thing --- .../scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala index 36e6403b16..7802f5b81c 100644 --- a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala @@ -100,9 +100,9 @@ private final class FdPollingUnixSockets[F[_]: Files: LiftIO](implicit F: Async[ IO { val clientFd = if (LinktimeInfo.isLinux) - guard(accept(fd, null, null)) - else guard(accept4(fd, null, null, SOCK_NONBLOCK)) + else + guard(accept(fd, null, null)) if (clientFd >= 0) Right(clientFd) From 84fc9b1b5783e1c6335baeeaf2e837edb67efafa Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 07:31:12 +0000 Subject: [PATCH 26/77] First attempt at fd polling socket group --- .../scala/fs2/io/internal/SocketHelpers.scala | 29 ++-- .../fs2/io/net/FdPollingSocketGroup.scala | 153 ++++++++++++++++++ 2 files changed, 172 insertions(+), 10 deletions(-) create mode 100644 io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index 5149b14369..85c70e6383 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -160,8 +160,10 @@ private[io] object SocketHelpers { } def getLocalAddress[F[_]](fd: Int)(implicit F: Sync[F]): F[SocketAddress[IpAddress]] = - SocketHelpers.toSocketAddress { (addr, len) => - F.delay(guard_(getsockname(fd, addr, len))) + F.delay { + SocketHelpers.toSocketAddress { (addr, len) => + guard_(getsockname(fd, addr, len)) + } } def toSockaddr[A]( @@ -249,24 +251,31 @@ private[io] object SocketHelpers { } } - def toSocketAddress[F[_]]( - f: (Ptr[sockaddr], Ptr[socklen_t]) => F[Unit] - )(implicit F: Sync[F]): F[SocketAddress[IpAddress]] = { + def allocateSockaddr[A]( + f: (Ptr[sockaddr], Ptr[socklen_t]) => A + ): A = { val addr = // allocate enough for an IPv6 stackalloc[sockaddr_in6]().asInstanceOf[Ptr[sockaddr]] val len = stackalloc[socklen_t]() !len = sizeof[sockaddr_in6].toUInt - f(addr, len) *> toSocketAddress(addr) + f(addr, len) + } + + def toSocketAddress[A]( + f: (Ptr[sockaddr], Ptr[socklen_t]) => Unit + ): SocketAddress[IpAddress] = allocateSockaddr { (addr, len) => + f(addr, len) + toSocketAddress(addr) } - def toSocketAddress[F[_]](addr: Ptr[sockaddr])(implicit F: Sync[F]): F[SocketAddress[IpAddress]] = + def toSocketAddress(addr: Ptr[sockaddr]): SocketAddress[IpAddress] = if (addr.sa_family.toInt == AF_INET) - F.pure(toIpv4SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in]])) + toIpv4SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in]]) else if (addr.sa_family.toInt == AF_INET6) - F.pure(toIpv6SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in6]])) + toIpv6SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in6]]) else - F.raiseError(new IOException(s"Unsupported sa_family: ${addr.sa_family}")) + throw new IOException(s"Unsupported sa_family: ${addr.sa_family}") private[this] def toIpv4SocketAddress(addr: Ptr[sockaddr_in]): SocketAddress[Ipv4Address] = { val port = Port.fromInt(ntohs(addr.sin_port).toInt).get diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala new file mode 100644 index 0000000000..7496395c98 --- /dev/null +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala @@ -0,0 +1,153 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2 +package io +package net + +import cats.effect.IO +import cats.effect.LiftIO +import cats.effect.kernel.Async +import cats.effect.kernel.Resource +import cats.syntax.all._ +import com.comcast.ip4s._ +import fs2.io.internal.NativeUtil._ +import fs2.io.internal.SocketHelpers +import fs2.io.internal.syssocket._ + +import scala.scalanative.libc.errno._ +import scala.scalanative.meta.LinktimeInfo +import scala.scalanative.posix.errno._ +import scala.scalanative.posix.string._ +import scala.scalanative.posix.sys.socket.{bind => _, connect => _, accept => _, _} +import scala.scalanative.posix.unistd._ +import scala.scalanative.unsafe._ + +private final class FdPollingSocketGroup[F[_]: Dns: LiftIO](implicit F: Async[F]) + extends SocketGroup[F] { + + def client(to: SocketAddress[Host], options: List[SocketOption]): Resource[F, Socket[F]] = for { + poller <- Resource.eval(fileDescriptorPoller[F]) + address <- Resource.eval(to.resolve) + ipv4 = address.host.isInstanceOf[Ipv4Address] + fd <- SocketHelpers.openNonBlocking(if (ipv4) AF_INET else AF_INET6, SOCK_STREAM) + _ <- Resource.eval(options.traverse(so => SocketHelpers.setOption(fd, so.key, so.value))) + handle <- poller.registerFileDescriptor(fd, true, true).mapK(LiftIO.liftK) + _ <- Resource.eval { + handle + .pollWriteRec(false) { connected => + if (connected) SocketHelpers.raiseSocketError[IO](fd).as(Either.unit) + else + IO { + SocketHelpers.toSockaddr(address) { (addr, len) => + if (connect(fd, addr, len) < 0) { + val e = errno + if (e == EINPROGRESS) + Left(true) // we will be connected when we unblock + else if (e == ECONNREFUSED) + throw new ConnectException(fromCString(strerror(errno))) + else + throw new IOException(fromCString(strerror(errno))) + } else + Either.unit + } + } + } + .to + } + socket <- FdPollingSocket[F](fd, handle, SocketHelpers.getLocalAddress(fd), F.pure(address)) + } yield socket + + def server( + address: Option[Host], + port: Option[Port], + options: List[SocketOption] + ): Stream[F, Socket[F]] = + Stream.resource(serverResource(address, port, options)).flatMap(_._2) + + def serverResource( + address: Option[Host], + port: Option[Port], + options: List[SocketOption] + ): Resource[F, (SocketAddress[IpAddress], Stream[F, Socket[F]])] = for { + poller <- Resource.eval(fileDescriptorPoller[F]) + address <- Resource.eval(address.fold(IpAddress.loopback)(_.resolve)) + ipv4 = address.isInstanceOf[Ipv4Address] + fd <- SocketHelpers.openNonBlocking(if (ipv4) AF_INET else AF_INET6, SOCK_STREAM) + handle <- poller.registerFileDescriptor(fd, true, false).mapK(LiftIO.liftK) + _ <- Resource.eval { + F.delay { + val socketAddress = SocketAddress(address, port.getOrElse(port"0")) + SocketHelpers.toSockaddr(socketAddress) { (addr, len) => + guard_(bind(fd, addr, len)) + } + } *> F.delay(guard_(listen(fd, 0))) + } + + sockets = Stream + .resource { + val accepted = for { + addrFd <- Resource.makeFull[F, (SocketAddress[IpAddress], Int)] { poll => + poll { + handle + .pollReadRec(()) { _ => + IO { + SocketHelpers.allocateSockaddr { (addr, len) => + val clientFd = + if (LinktimeInfo.isLinux) + guard(accept4(fd, addr, len, SOCK_NONBLOCK)) + else + guard(accept(fd, addr, len)) + + if (clientFd >= 0) { + val address = SocketHelpers.toSocketAddress(addr) + Right((address, clientFd)) + } else + Left(()) + } + } + } + .to + } + }(addrFd => F.delay(guard_(close(addrFd._2)))) + (address, fd) = addrFd + _ <- + if (!LinktimeInfo.isLinux) + Resource.eval(setNonBlocking(fd)) + else Resource.unit[F] + handle <- poller.registerFileDescriptor(fd, true, true).mapK(LiftIO.liftK) + socket <- FdPollingSocket[F]( + fd, + handle, + SocketHelpers.getLocalAddress(fd), + F.pure(address) + ) + } yield socket + + accepted.attempt.map(_.toOption) + } + .repeat + .unNone + + serverAddress <- Resource.eval(SocketHelpers.getLocalAddress(fd)) + } yield (serverAddress, sockets) + +} From 1678d527dafec9be2f7a61c7275541cb20a422ec Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 07:47:30 +0000 Subject: [PATCH 27/77] `raiseSocketError` -> `checkSocketError` --- io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala | 2 +- io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala | 2 +- .../main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index 85c70e6383..8ebeda78bf 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -143,7 +143,7 @@ private[io] object SocketHelpers { ) } - def raiseSocketError[F[_]](fd: Int)(implicit F: Sync[F]): F[Unit] = F.delay { + def checkSocketError[F[_]](fd: Int)(implicit F: Sync[F]): F[Unit] = F.delay { val optval = stackalloc[CInt]() val optlen = stackalloc[socklen_t]() guard_ { diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala index 7496395c98..85591db730 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala @@ -54,7 +54,7 @@ private final class FdPollingSocketGroup[F[_]: Dns: LiftIO](implicit F: Async[F] _ <- Resource.eval { handle .pollWriteRec(false) { connected => - if (connected) SocketHelpers.raiseSocketError[IO](fd).as(Either.unit) + if (connected) SocketHelpers.checkSocketError[IO](fd).as(Either.unit) else IO { SocketHelpers.toSockaddr(address) { (addr, len) => diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala index 7802f5b81c..a5dbd0858a 100644 --- a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala @@ -54,7 +54,7 @@ private final class FdPollingUnixSockets[F[_]: Files: LiftIO](implicit F: Async[ _ <- Resource.eval { handle .pollWriteRec(false) { connected => - if (connected) SocketHelpers.raiseSocketError[IO](fd).as(Either.unit) + if (connected) SocketHelpers.checkSocketError[IO](fd).as(Either.unit) else IO { toSockaddrUn(address.path) { addr => From b5b3a049ed64397ae5fca02a0f48d93c231cd64b Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 07:55:20 +0000 Subject: [PATCH 28/77] Expose new polling system `Network` --- .../scala/fs2/io/net/NetworkPlatform.scala | 33 +++++++++++++++++-- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/NetworkPlatform.scala b/io/native/src/main/scala/fs2/io/net/NetworkPlatform.scala index f2aba3b524..7536484da5 100644 --- a/io/native/src/main/scala/fs2/io/net/NetworkPlatform.scala +++ b/io/native/src/main/scala/fs2/io/net/NetworkPlatform.scala @@ -23,9 +23,10 @@ package fs2 package io package net +import cats.effect.LiftIO import cats.effect.kernel.{Async, Resource} -import com.comcast.ip4s.{Host, IpAddress, Port, SocketAddress} +import com.comcast.ip4s.{Dns, Host, IpAddress, Port, SocketAddress} import fs2.io.net.tls.TLSContext @@ -33,9 +34,9 @@ private[net] trait NetworkPlatform[F[_]] private[net] trait NetworkCompanionPlatform { self: Network.type => - implicit def forAsync[F[_]](implicit F: Async[F]): Network[F] = + implicit def forAsync[F[_]: Async: Dns: LiftIO]: Network[F] = new UnsealedNetwork[F] { - private lazy val globalSocketGroup = SocketGroup.unsafe[F](null) + private lazy val globalSocketGroup = new FdPollingSocketGroup[F] def client( to: SocketAddress[Host], @@ -58,4 +59,30 @@ private[net] trait NetworkCompanionPlatform { self: Network.type => def tlsContext: TLSContext.Builder[F] = TLSContext.Builder.forAsync } + @deprecated("Prefer the IO polling system-based implementation", "3.5.0") + def forAsync[F[_]](F: Async[F]): Network[F] = + new UnsealedNetwork[F] { + private lazy val globalSocketGroup = SocketGroup.unsafe[F](null)(F) + + def client( + to: SocketAddress[Host], + options: List[SocketOption] + ): Resource[F, Socket[F]] = globalSocketGroup.client(to, options) + + def server( + address: Option[Host], + port: Option[Port], + options: List[SocketOption] + ): Stream[F, Socket[F]] = globalSocketGroup.server(address, port, options) + + def serverResource( + address: Option[Host], + port: Option[Port], + options: List[SocketOption] + ): Resource[F, (SocketAddress[IpAddress], Stream[F, Socket[F]])] = + globalSocketGroup.serverResource(address, port, options) + + def tlsContext: TLSContext.Builder[F] = TLSContext.Builder.forAsync(F) + } + } From e682a3777365a8d5a406007b8b87e4b0d792f297 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 08:05:47 +0000 Subject: [PATCH 29/77] Fix exceptions, tweak test --- .../scala/fs2/io/internal/NativeUtil.scala | 19 ++++++++++++++----- .../scala/fs2/io/net/tcp/SocketSuite.scala | 4 ++-- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala index 80a0a89446..8887e8409c 100644 --- a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala +++ b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala @@ -23,13 +23,15 @@ package fs2.io.internal import cats.effect.Sync +import java.io.IOException +import java.net.BindException +import java.net.ConnectException import scala.scalanative.annotation.alwaysinline import scala.scalanative.libc.errno._ import scala.scalanative.posix.fcntl._ import scala.scalanative.posix.errno._ import scala.scalanative.posix.string._ import scala.scalanative.unsafe._ -import java.io.IOException private[io] object NativeUtil { @@ -41,11 +43,18 @@ private[io] object NativeUtil { @alwaysinline def guard(thunk: => CInt): CInt = { val rtn = thunk if (rtn < 0) { - val en = errno - if (en == EAGAIN || en == EWOULDBLOCK) + val e = errno + if (e == EAGAIN || e == EWOULDBLOCK) rtn - else - throw new IOException(fromCString(strerror(errno))) + else { + val msg = fromCString(strerror(e)) + if (e == EADDRINUSE /* || e == EADDRNOTAVAIL */ ) + throw new BindException(msg) + else if (e == ECONNREFUSED) + throw new ConnectException(msg) + else + throw new IOException(msg) + } } else rtn } diff --git a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala index d0c28e0c73..fa240abd60 100644 --- a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala +++ b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala @@ -218,7 +218,7 @@ class SocketSuite extends Fs2Suite with SocketSuitePlatform { } } - test("read after timed out read not allowed on JVM or Native") { + test("read after timed out read not allowed on JVM") { val setup = for { serverSetup <- Network[IO].serverResource(Some(ip"127.0.0.1")) (bindAddress, server) = serverSetup @@ -239,7 +239,7 @@ class SocketSuite extends Fs2Suite with SocketSuitePlatform { client .readN(msg.size) .flatMap { c => - if (isJVM || isNative) { + if (isJVM) { assertEquals(c.size, 0) // Read again now that the pending read is no longer pending client.readN(msg.size).map(c => assertEquals(c.size, 0)) From 220f6bcdb746d00131b6e77b942bb1d3d9baa170 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 17:12:10 +0000 Subject: [PATCH 30/77] Add forgotten `guard`s --- .../scala/fs2/io/internal/NativeUtil.scala | 31 +++++++++++++------ .../scala/fs2/io/net/FdPollingSocket.scala | 4 +-- .../fs2/io/net/FdPollingSocketGroup.scala | 6 +--- 3 files changed, 25 insertions(+), 16 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala index 8887e8409c..00850c5660 100644 --- a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala +++ b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala @@ -46,19 +46,32 @@ private[io] object NativeUtil { val e = errno if (e == EAGAIN || e == EWOULDBLOCK) rtn - else { - val msg = fromCString(strerror(e)) - if (e == EADDRINUSE /* || e == EADDRNOTAVAIL */ ) - throw new BindException(msg) - else if (e == ECONNREFUSED) - throw new ConnectException(msg) - else - throw new IOException(msg) - } + else throw errnoToThrowable(e) } else rtn } + @alwaysinline def guardSSize(thunk: => CSSize): CSSize = { + val rtn = thunk + if (rtn < 0) { + val e = errno + if (e == EAGAIN || e == EWOULDBLOCK) + rtn + else throw errnoToThrowable(e) + } else + rtn + } + + @alwaysinline def errnoToThrowable(e: CInt): Throwable = { + val msg = fromCString(strerror(e)) + if (e == EADDRINUSE /* || e == EADDRNOTAVAIL */ ) + new BindException(msg) + else if (e == ECONNREFUSED) + new ConnectException(msg) + else + new IOException(msg) + } + def setNonBlocking[F[_]](fd: CInt)(implicit F: Sync[F]): F[Unit] = F.delay { guard_(fcntl(fd, F_SETFL, O_NONBLOCK)) } diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 1fb4e7a71e..821268de46 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -94,9 +94,9 @@ private final class FdPollingSocket[F[_]: LiftIO] private ( def go(pos: Int): IO[Either[Int, Unit]] = IO { if (LinktimeInfo.isLinux) - send(fd, buf.at(offset + pos), (length - pos).toULong, MSG_NOSIGNAL).toInt + guardSSize(send(fd, buf.at(offset + pos), (length - pos).toULong, MSG_NOSIGNAL)).toInt else - unistd.write(fd, buf.at(offset + pos), (length - pos).toULong) + guard(unistd.write(fd, buf.at(offset + pos), (length - pos).toULong)) }.flatMap { wrote => if (wrote >= 0) { val newPos = pos + wrote diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala index 85591db730..aa9a947276 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala @@ -36,10 +36,8 @@ import fs2.io.internal.syssocket._ import scala.scalanative.libc.errno._ import scala.scalanative.meta.LinktimeInfo import scala.scalanative.posix.errno._ -import scala.scalanative.posix.string._ import scala.scalanative.posix.sys.socket.{bind => _, connect => _, accept => _, _} import scala.scalanative.posix.unistd._ -import scala.scalanative.unsafe._ private final class FdPollingSocketGroup[F[_]: Dns: LiftIO](implicit F: Async[F]) extends SocketGroup[F] { @@ -62,10 +60,8 @@ private final class FdPollingSocketGroup[F[_]: Dns: LiftIO](implicit F: Async[F] val e = errno if (e == EINPROGRESS) Left(true) // we will be connected when we unblock - else if (e == ECONNREFUSED) - throw new ConnectException(fromCString(strerror(errno))) else - throw new IOException(fromCString(strerror(errno))) + throw errnoToThrowable(e) } else Either.unit } From ef299db90ad01ae40f643c96a5bcb695439c9e14 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 19:37:00 +0000 Subject: [PATCH 31/77] Workaround BSD `sa_family` quirk --- .../scala/fs2/io/internal/SocketHelpers.scala | 13 ++++++-- .../main/scala/fs2/io/internal/netinet.scala | 31 ++++++++++++++----- .../scala/fs2/io/internal/syssocket.scala | 5 +++ 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index 8ebeda78bf..1fd40ce28c 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -269,13 +269,20 @@ private[io] object SocketHelpers { toSocketAddress(addr) } - def toSocketAddress(addr: Ptr[sockaddr]): SocketAddress[IpAddress] = - if (addr.sa_family.toInt == AF_INET) + def toSocketAddress(addr: Ptr[sockaddr]): SocketAddress[IpAddress] = { + val sa_family = + if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) + addr.sa_family.asInstanceOf[bsd_len_family]._2.toInt + else + addr.sa_family.toInt + + if (sa_family == AF_INET) toIpv4SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in]]) - else if (addr.sa_family.toInt == AF_INET6) + else if (sa_family == AF_INET6) toIpv6SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in6]]) else throw new IOException(s"Unsupported sa_family: ${addr.sa_family}") + } private[this] def toIpv4SocketAddress(addr: Ptr[sockaddr_in]): SocketAddress[Ipv4Address] = { val port = Port.fromInt(ntohs(addr.sin_port).toInt).get diff --git a/io/native/src/main/scala/fs2/io/internal/netinet.scala b/io/native/src/main/scala/fs2/io/internal/netinet.scala index fdb1dc4afc..29b07f9990 100644 --- a/io/native/src/main/scala/fs2/io/internal/netinet.scala +++ b/io/native/src/main/scala/fs2/io/internal/netinet.scala @@ -21,9 +21,12 @@ package fs2.io.internal -import scalanative.unsafe._ -import scalanative.posix.inttypes._ -import scalanative.posix.sys.socket._ +import scala.scalanative.meta.LinktimeInfo +import scala.scalanative.posix.inttypes._ +import scala.scalanative.posix.sys.socket._ +import scala.scalanative.unsafe._ + +import syssocket.bsd_len_family private[io] object netinetin { import Nat._ @@ -61,8 +64,16 @@ private[io] object netinetinOps { } implicit final class sockaddr_inOps(val sockaddr_in: Ptr[sockaddr_in]) extends AnyVal { - def sin_family: sa_family_t = sockaddr_in._1 - def sin_family_=(sin_family: sa_family_t): Unit = sockaddr_in._1 = sin_family + def sin_family: sa_family_t = + if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) + sockaddr_in.at1.asInstanceOf[bsd_len_family]._2 + else + sockaddr_in._1 + def sin_family_=(sin_family: sa_family_t): Unit = + if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) + sockaddr_in.at1.asInstanceOf[bsd_len_family]._2 = sin_family.toUByte + else + sockaddr_in._1 = sin_family def sin_port: in_port_t = sockaddr_in._2 def sin_port_=(sin_port: in_port_t): Unit = sockaddr_in._2 = sin_port def sin_addr: in_addr = sockaddr_in._3 @@ -75,8 +86,14 @@ private[io] object netinetinOps { } implicit final class sockaddr_in6Ops(val sockaddr_in6: Ptr[sockaddr_in6]) extends AnyVal { - def sin6_family: sa_family_t = sockaddr_in6._1 - def sin6_family_=(sin6_family: sa_family_t): Unit = sockaddr_in6._1 = sin6_family + def sin6_family: sa_family_t = if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) + sockaddr_in6.asInstanceOf[bsd_len_family]._2 + else + sockaddr_in6._1 + def sin6_family_=(sin6_family: sa_family_t): Unit = + if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) + sockaddr_in6.at1.asInstanceOf[bsd_len_family]._2 = sin6_family.toUByte + else sockaddr_in6._1 = sin6_family def sin6_port: in_port_t = sockaddr_in6._2 def sin6_port_=(sin6_port: in_port_t): Unit = sockaddr_in6._2 = sin6_port def sin6_flowinfo: uint32_t = sockaddr_in6._3 diff --git a/io/native/src/main/scala/fs2/io/internal/syssocket.scala b/io/native/src/main/scala/fs2/io/internal/syssocket.scala index 984be7acb0..631d4f69b1 100644 --- a/io/native/src/main/scala/fs2/io/internal/syssocket.scala +++ b/io/native/src/main/scala/fs2/io/internal/syssocket.scala @@ -21,11 +21,16 @@ package fs2.io.internal +import scala.scalanative.posix.inttypes._ import scala.scalanative.posix.sys.socket._ import scala.scalanative.unsafe._ +import syssocket._ + @extern private[io] object syssocket { + type bsd_len_family = CStruct2[uint8_t, uint8_t] + // only in Linux and FreeBSD, but not macOS final val SOCK_NONBLOCK = 2048 From f6ecd2b1c3021bce175dd24e41e6c62a9f0af671 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 19:40:58 +0000 Subject: [PATCH 32/77] Unused import --- io/native/src/main/scala/fs2/io/internal/syssocket.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/syssocket.scala b/io/native/src/main/scala/fs2/io/internal/syssocket.scala index 631d4f69b1..46b4782d85 100644 --- a/io/native/src/main/scala/fs2/io/internal/syssocket.scala +++ b/io/native/src/main/scala/fs2/io/internal/syssocket.scala @@ -25,8 +25,6 @@ import scala.scalanative.posix.inttypes._ import scala.scalanative.posix.sys.socket._ import scala.scalanative.unsafe._ -import syssocket._ - @extern private[io] object syssocket { type bsd_len_family = CStruct2[uint8_t, uint8_t] From 830564afb39a8db9e261c98622b75d2e3def3b6a Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 15:23:11 -0500 Subject: [PATCH 33/77] Fixing+debugging --- .../src/main/scala/fs2/io/internal/SocketHelpers.scala | 9 ++------- io/native/src/main/scala/fs2/io/internal/netinet.scala | 8 ++++---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index 1fd40ce28c..e5b7e04f08 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -270,18 +270,13 @@ private[io] object SocketHelpers { } def toSocketAddress(addr: Ptr[sockaddr]): SocketAddress[IpAddress] = { - val sa_family = - if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) - addr.sa_family.asInstanceOf[bsd_len_family]._2.toInt - else - addr.sa_family.toInt - + val sa_family = addr.sa_family.toInt if (sa_family == AF_INET) toIpv4SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in]]) else if (sa_family == AF_INET6) toIpv6SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in6]]) else - throw new IOException(s"Unsupported sa_family: ${addr.sa_family}") + throw new IOException(s"Unsupported sa_family: $sa_family") } private[this] def toIpv4SocketAddress(addr: Ptr[sockaddr_in]): SocketAddress[Ipv4Address] = { diff --git a/io/native/src/main/scala/fs2/io/internal/netinet.scala b/io/native/src/main/scala/fs2/io/internal/netinet.scala index 29b07f9990..3f24773444 100644 --- a/io/native/src/main/scala/fs2/io/internal/netinet.scala +++ b/io/native/src/main/scala/fs2/io/internal/netinet.scala @@ -66,12 +66,12 @@ private[io] object netinetinOps { implicit final class sockaddr_inOps(val sockaddr_in: Ptr[sockaddr_in]) extends AnyVal { def sin_family: sa_family_t = if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) - sockaddr_in.at1.asInstanceOf[bsd_len_family]._2 + sockaddr_in.at1.asInstanceOf[Ptr[bsd_len_family]]._2 else sockaddr_in._1 def sin_family_=(sin_family: sa_family_t): Unit = if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) - sockaddr_in.at1.asInstanceOf[bsd_len_family]._2 = sin_family.toUByte + sockaddr_in.at1.asInstanceOf[Ptr[bsd_len_family]]._2 = sin_family.toUByte else sockaddr_in._1 = sin_family def sin_port: in_port_t = sockaddr_in._2 @@ -87,12 +87,12 @@ private[io] object netinetinOps { implicit final class sockaddr_in6Ops(val sockaddr_in6: Ptr[sockaddr_in6]) extends AnyVal { def sin6_family: sa_family_t = if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) - sockaddr_in6.asInstanceOf[bsd_len_family]._2 + sockaddr_in6.asInstanceOf[Ptr[bsd_len_family]]._2 else sockaddr_in6._1 def sin6_family_=(sin6_family: sa_family_t): Unit = if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) - sockaddr_in6.at1.asInstanceOf[bsd_len_family]._2 = sin6_family.toUByte + sockaddr_in6.at1.asInstanceOf[Ptr[bsd_len_family]]._2 = sin6_family.toUByte else sockaddr_in6._1 = sin6_family def sin6_port: in_port_t = sockaddr_in6._2 def sin6_port_=(sin6_port: in_port_t): Unit = sockaddr_in6._2 = sin6_port From 0ece5fc809346506054ddba059287621853b3b76 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Dec 2022 22:53:57 +0000 Subject: [PATCH 34/77] Revert "Workaround BSD `sa_family` quirk" This reverts commit ef299db90ad01ae40f643c96a5bcb695439c9e14. --- .../scala/fs2/io/internal/SocketHelpers.scala | 10 +++--- .../main/scala/fs2/io/internal/netinet.scala | 31 +++++-------------- .../scala/fs2/io/internal/syssocket.scala | 3 -- 3 files changed, 11 insertions(+), 33 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index e5b7e04f08..8ebeda78bf 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -269,15 +269,13 @@ private[io] object SocketHelpers { toSocketAddress(addr) } - def toSocketAddress(addr: Ptr[sockaddr]): SocketAddress[IpAddress] = { - val sa_family = addr.sa_family.toInt - if (sa_family == AF_INET) + def toSocketAddress(addr: Ptr[sockaddr]): SocketAddress[IpAddress] = + if (addr.sa_family.toInt == AF_INET) toIpv4SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in]]) - else if (sa_family == AF_INET6) + else if (addr.sa_family.toInt == AF_INET6) toIpv6SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in6]]) else - throw new IOException(s"Unsupported sa_family: $sa_family") - } + throw new IOException(s"Unsupported sa_family: ${addr.sa_family}") private[this] def toIpv4SocketAddress(addr: Ptr[sockaddr_in]): SocketAddress[Ipv4Address] = { val port = Port.fromInt(ntohs(addr.sin_port).toInt).get diff --git a/io/native/src/main/scala/fs2/io/internal/netinet.scala b/io/native/src/main/scala/fs2/io/internal/netinet.scala index 3f24773444..fdb1dc4afc 100644 --- a/io/native/src/main/scala/fs2/io/internal/netinet.scala +++ b/io/native/src/main/scala/fs2/io/internal/netinet.scala @@ -21,12 +21,9 @@ package fs2.io.internal -import scala.scalanative.meta.LinktimeInfo -import scala.scalanative.posix.inttypes._ -import scala.scalanative.posix.sys.socket._ -import scala.scalanative.unsafe._ - -import syssocket.bsd_len_family +import scalanative.unsafe._ +import scalanative.posix.inttypes._ +import scalanative.posix.sys.socket._ private[io] object netinetin { import Nat._ @@ -64,16 +61,8 @@ private[io] object netinetinOps { } implicit final class sockaddr_inOps(val sockaddr_in: Ptr[sockaddr_in]) extends AnyVal { - def sin_family: sa_family_t = - if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) - sockaddr_in.at1.asInstanceOf[Ptr[bsd_len_family]]._2 - else - sockaddr_in._1 - def sin_family_=(sin_family: sa_family_t): Unit = - if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) - sockaddr_in.at1.asInstanceOf[Ptr[bsd_len_family]]._2 = sin_family.toUByte - else - sockaddr_in._1 = sin_family + def sin_family: sa_family_t = sockaddr_in._1 + def sin_family_=(sin_family: sa_family_t): Unit = sockaddr_in._1 = sin_family def sin_port: in_port_t = sockaddr_in._2 def sin_port_=(sin_port: in_port_t): Unit = sockaddr_in._2 = sin_port def sin_addr: in_addr = sockaddr_in._3 @@ -86,14 +75,8 @@ private[io] object netinetinOps { } implicit final class sockaddr_in6Ops(val sockaddr_in6: Ptr[sockaddr_in6]) extends AnyVal { - def sin6_family: sa_family_t = if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) - sockaddr_in6.asInstanceOf[Ptr[bsd_len_family]]._2 - else - sockaddr_in6._1 - def sin6_family_=(sin6_family: sa_family_t): Unit = - if (LinktimeInfo.isMac || LinktimeInfo.isFreeBSD) - sockaddr_in6.at1.asInstanceOf[Ptr[bsd_len_family]]._2 = sin6_family.toUByte - else sockaddr_in6._1 = sin6_family + def sin6_family: sa_family_t = sockaddr_in6._1 + def sin6_family_=(sin6_family: sa_family_t): Unit = sockaddr_in6._1 = sin6_family def sin6_port: in_port_t = sockaddr_in6._2 def sin6_port_=(sin6_port: in_port_t): Unit = sockaddr_in6._2 = sin6_port def sin6_flowinfo: uint32_t = sockaddr_in6._3 diff --git a/io/native/src/main/scala/fs2/io/internal/syssocket.scala b/io/native/src/main/scala/fs2/io/internal/syssocket.scala index 46b4782d85..984be7acb0 100644 --- a/io/native/src/main/scala/fs2/io/internal/syssocket.scala +++ b/io/native/src/main/scala/fs2/io/internal/syssocket.scala @@ -21,14 +21,11 @@ package fs2.io.internal -import scala.scalanative.posix.inttypes._ import scala.scalanative.posix.sys.socket._ import scala.scalanative.unsafe._ @extern private[io] object syssocket { - type bsd_len_family = CStruct2[uint8_t, uint8_t] - // only in Linux and FreeBSD, but not macOS final val SOCK_NONBLOCK = 2048 From 26a4e4f73fae9f4eaf86ebe24f66201fe4f6543d Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Fri, 23 Dec 2022 00:05:03 +0000 Subject: [PATCH 35/77] Explicitly track if ipv4/ipv6 socket --- .../scala/fs2/io/internal/SocketHelpers.scala | 19 +++++++++---------- .../fs2/io/net/FdPollingSocketGroup.scala | 13 +++++++++---- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index 8ebeda78bf..f8e8e82ff7 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -39,7 +39,6 @@ import scala.scalanative.posix.netinet.in.IPPROTO_TCP import scala.scalanative.posix.netinet.tcp._ import scala.scalanative.posix.string._ import scala.scalanative.posix.sys.socket._ -import scala.scalanative.posix.sys.socketOps._ import scala.scalanative.posix.unistd._ import scala.scalanative.unsafe._ import scala.scalanative.unsigned._ @@ -159,9 +158,11 @@ private[io] object SocketHelpers { throw new IOException(fromCString(strerror(!optval))) } - def getLocalAddress[F[_]](fd: Int)(implicit F: Sync[F]): F[SocketAddress[IpAddress]] = + def getLocalAddress[F[_]](fd: Int, ipv4: Boolean)(implicit + F: Sync[F] + ): F[SocketAddress[IpAddress]] = F.delay { - SocketHelpers.toSocketAddress { (addr, len) => + SocketHelpers.toSocketAddress(ipv4) { (addr, len) => guard_(getsockname(fd, addr, len)) } } @@ -262,20 +263,18 @@ private[io] object SocketHelpers { f(addr, len) } - def toSocketAddress[A]( + def toSocketAddress[A](ipv4: Boolean)( f: (Ptr[sockaddr], Ptr[socklen_t]) => Unit ): SocketAddress[IpAddress] = allocateSockaddr { (addr, len) => f(addr, len) - toSocketAddress(addr) + toSocketAddress(addr, ipv4) } - def toSocketAddress(addr: Ptr[sockaddr]): SocketAddress[IpAddress] = - if (addr.sa_family.toInt == AF_INET) + def toSocketAddress(addr: Ptr[sockaddr], ipv4: Boolean): SocketAddress[IpAddress] = + if (ipv4) toIpv4SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in]]) - else if (addr.sa_family.toInt == AF_INET6) - toIpv6SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in6]]) else - throw new IOException(s"Unsupported sa_family: ${addr.sa_family}") + toIpv6SocketAddress(addr.asInstanceOf[Ptr[sockaddr_in6]]) private[this] def toIpv4SocketAddress(addr: Ptr[sockaddr_in]): SocketAddress[Ipv4Address] = { val port = Port.fromInt(ntohs(addr.sin_port).toInt).get diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala index aa9a947276..fab3d28e65 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala @@ -69,7 +69,12 @@ private final class FdPollingSocketGroup[F[_]: Dns: LiftIO](implicit F: Async[F] } .to } - socket <- FdPollingSocket[F](fd, handle, SocketHelpers.getLocalAddress(fd), F.pure(address)) + socket <- FdPollingSocket[F]( + fd, + handle, + SocketHelpers.getLocalAddress(fd, ipv4), + F.pure(address) + ) } yield socket def server( @@ -114,7 +119,7 @@ private final class FdPollingSocketGroup[F[_]: Dns: LiftIO](implicit F: Async[F] guard(accept(fd, addr, len)) if (clientFd >= 0) { - val address = SocketHelpers.toSocketAddress(addr) + val address = SocketHelpers.toSocketAddress(addr, ipv4) Right((address, clientFd)) } else Left(()) @@ -133,7 +138,7 @@ private final class FdPollingSocketGroup[F[_]: Dns: LiftIO](implicit F: Async[F] socket <- FdPollingSocket[F]( fd, handle, - SocketHelpers.getLocalAddress(fd), + SocketHelpers.getLocalAddress(fd, ipv4), F.pure(address) ) } yield socket @@ -143,7 +148,7 @@ private final class FdPollingSocketGroup[F[_]: Dns: LiftIO](implicit F: Async[F] .repeat .unNone - serverAddress <- Resource.eval(SocketHelpers.getLocalAddress(fd)) + serverAddress <- Resource.eval(SocketHelpers.getLocalAddress(fd, ipv4)) } yield (serverAddress, sockets) } From b0f71fe606f8c377145fda23cb98948f72c88c7c Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Fri, 23 Dec 2022 01:26:29 +0000 Subject: [PATCH 36/77] Fix Scala 3 compile --- io/native/src/main/scala/fs2/io/ioplatform.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/native/src/main/scala/fs2/io/ioplatform.scala b/io/native/src/main/scala/fs2/io/ioplatform.scala index 66e96834b8..66b7248be6 100644 --- a/io/native/src/main/scala/fs2/io/ioplatform.scala +++ b/io/native/src/main/scala/fs2/io/ioplatform.scala @@ -141,7 +141,7 @@ private[fs2] trait ioplatform extends iojvmnative { def stdoutLines[F[_]: Async: LiftIO, O: Show]( charset: Charset = StandardCharsets.UTF_8 ): Pipe[F, O, Nothing] = - _.map(_.show).through(text.encode(charset)).through(stdout) + _.map(_.show).through(text.encode(charset)).through(stdout(implicitly, implicitly)) /** Stream of `String` read asynchronously from standard input decoded in UTF-8. */ def stdinUtf8[F[_]: Async: LiftIO](bufSize: Int): Stream[F, String] = From 8c2131252026159520421f04180744ab4b7c11ab Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 26 Dec 2022 19:01:25 +0000 Subject: [PATCH 37/77] Implement `SelectorPollingSocket` --- build.sbt | 6 +- .../fs2/io/net/SelectorPollingSocket.scala | 103 ++++++++++++++++++ 2 files changed, 106 insertions(+), 3 deletions(-) create mode 100644 io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala diff --git a/build.sbt b/build.sbt index 217b529d8c..11becadd8c 100644 --- a/build.sbt +++ b/build.sbt @@ -209,9 +209,9 @@ lazy val core = crossProject(JVMPlatform, JSPlatform, NativePlatform) libraryDependencies ++= Seq( "org.typelevel" %%% "cats-core" % "2.9.0", "org.typelevel" %%% "cats-laws" % "2.9.0" % Test, - "org.typelevel" %%% "cats-effect" % "3.4.3", - "org.typelevel" %%% "cats-effect-laws" % "3.4.3" % Test, - "org.typelevel" %%% "cats-effect-testkit" % "3.4.3" % Test, + "org.typelevel" %%% "cats-effect" % "3.5-01c4a03", + "org.typelevel" %%% "cats-effect-laws" % "3.5-01c4a03" % Test, + "org.typelevel" %%% "cats-effect-testkit" % "3.5-01c4a03" % Test, "org.scodec" %%% "scodec-bits" % "1.1.34", "org.typelevel" %%% "scalacheck-effect-munit" % "2.0.0-M2" % Test, "org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3" % Test, diff --git a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala new file mode 100644 index 0000000000..6c1c3b9dcc --- /dev/null +++ b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2 +package io.net + +import cats.effect.LiftIO +import cats.effect.SelectorPoller +import cats.effect.kernel.Async +import cats.effect.std.Semaphore +import cats.syntax.all._ +import com.comcast.ip4s.IpAddress +import com.comcast.ip4s.SocketAddress + +import java.nio.ByteBuffer +import java.nio.channels.SelectionKey.OP_READ +import java.nio.channels.SelectionKey.OP_WRITE +import java.nio.channels.SocketChannel + +private final class SelectorPollingSocket[F[_]: LiftIO] private ( + poller: SelectorPoller, + ch: SocketChannel, + readSemaphore: Semaphore[F], + writeSemaphore: Semaphore[F], + val localAddress: F[SocketAddress[IpAddress]], + val remoteAddress: F[SocketAddress[IpAddress]] +)(implicit F: Async[F]) + extends Socket.BufferedReads(readSemaphore) { + + protected def readChunk(buf: ByteBuffer): F[Int] = + F.delay(ch.read(buf)).flatMap { readed => + if (readed == 0) poller.select(ch, OP_READ).to *> readChunk(buf) + else F.pure(readed) + } + + def write(bytes: Chunk[Byte]): F[Unit] = { + def go(buf: ByteBuffer): F[Unit] = + F.delay { + ch.write(buf) + buf.remaining() + }.flatMap { remaining => + if (remaining > 0) { + poller.select(ch, OP_WRITE).to *> go(buf) + } else F.unit + } + writeSemaphore.permit.use { _ => + go(bytes.toByteBuffer) + } + } + + def isOpen: F[Boolean] = F.delay(ch.isOpen) + + def endOfOutput: F[Unit] = + F.delay { + ch.shutdownOutput(); () + } + + def endOfInput: F[Unit] = + F.delay { + ch.shutdownInput(); () + } + +} + +private object SelectorPollingSocket { + def apply[F[_]: LiftIO]( + poller: SelectorPoller, + ch: SocketChannel, + localAddress: F[SocketAddress[IpAddress]], + remoteAddress: F[SocketAddress[IpAddress]] + )(implicit F: Async[F]): F[Socket[F]] = + (Semaphore[F](1), Semaphore[F](1)).flatMapN { (readSemaphore, writeSemaphore) => + F.delay { + ch.configureBlocking(false) + new SelectorPollingSocket[F]( + poller, + ch, + readSemaphore, + writeSemaphore, + localAddress, + remoteAddress + ) + } + } +} From 283eda45034b6ce8f6d62b778fc6cee8781c4891 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 26 Dec 2022 21:20:17 +0000 Subject: [PATCH 38/77] Implement `SelectorPollingSocketGroup` --- .../fs2/io/net/SelectorPollingSocket.scala | 1 - .../io/net/SelectorPollingSocketGroup.scala | 173 ++++++++++++++++++ 2 files changed, 173 insertions(+), 1 deletion(-) create mode 100644 io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala diff --git a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala index 6c1c3b9dcc..9c2ba2641a 100644 --- a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala +++ b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala @@ -89,7 +89,6 @@ private object SelectorPollingSocket { )(implicit F: Async[F]): F[Socket[F]] = (Semaphore[F](1), Semaphore[F](1)).flatMapN { (readSemaphore, writeSemaphore) => F.delay { - ch.configureBlocking(false) new SelectorPollingSocket[F]( poller, ch, diff --git a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala new file mode 100644 index 0000000000..e33ff06d7b --- /dev/null +++ b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2013 Functional Streams for Scala + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +package fs2 +package io.net + +import cats.effect.LiftIO +import cats.effect.SelectorPoller +import cats.effect.kernel.Async +import cats.effect.kernel.Resource +import cats.syntax.all._ +import com.comcast.ip4s.Dns +import com.comcast.ip4s.Host +import com.comcast.ip4s.IpAddress +import com.comcast.ip4s.Port +import com.comcast.ip4s.SocketAddress + +import java.net.InetSocketAddress +import java.nio.channels.AsynchronousCloseException +import java.nio.channels.ClosedChannelException +import java.nio.channels.SelectionKey.OP_ACCEPT +import java.nio.channels.SelectionKey.OP_CONNECT +import java.nio.channels.SocketChannel + +private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: SelectorPoller)(implicit + F: Async[F] +) extends SocketGroup[F] { + + def client( + to: SocketAddress[Host], + options: List[SocketOption] + ): Resource[F, Socket[F]] = + Resource + .make(F.delay(poller.provider.openSocketChannel())) { ch => + F.delay(ch.close()) + } + .evalMap { ch => + val configure = F.delay { + ch.configureBlocking(false) + options.foreach(opt => ch.setOption(opt.key, opt.value)) + } + + val connect = to.resolve.flatMap { ip => + F.delay(ch.connect(ip.toInetSocketAddress)).flatMap { connected => + poller + .select(ch, OP_CONNECT) + .to + .untilM_(F.delay(ch.finishConnect())) + .unlessA(connected) + } + } + + val make = SelectorPollingSocket[F]( + poller, + ch, + localAddress(ch), + remoteAddress(ch) + ) + + configure *> connect *> make + } + + def server( + address: Option[Host], + port: Option[Port], + options: List[SocketOption] + ): Stream[F, Socket[F]] = + Stream + .resource( + serverResource( + address, + port, + options + ) + ) + .flatMap { case (_, clients) => clients } + + def serverResource( + address: Option[Host], + port: Option[Port], + options: List[SocketOption] + ): Resource[F, (SocketAddress[IpAddress], Stream[F, Socket[F]])] = + Resource + .make(F.delay(poller.provider.openServerSocketChannel())) { ch => + F.delay(ch.close()) + } + .evalTap { ch => + address.traverse(_.resolve).flatMap { ip => + F.delay { + ch.configureBlocking(false) + ch.bind( + new InetSocketAddress( + ip.map(_.toInetAddress).orNull, + port.map(_.value).getOrElse(0) + ) + ) + } + } + } + .evalMap { serverCh => + def acceptLoop: Stream[F, SocketChannel] = Stream + .bracket { + def go: F[SocketChannel] = + F.delay(serverCh.accept()).flatMap { + case null => poller.select(serverCh, OP_ACCEPT).to *> go + case ch => F.pure(ch) + } + go + }(ch => F.delay(ch.close())) + .attempt + .flatMap { + case Right(ch) => + Stream.emit(ch) ++ acceptLoop + case Left(_: AsynchronousCloseException) | Left(_: ClosedChannelException) => + Stream.empty + case _ => + acceptLoop + } + + val clients = acceptLoop.evalMap { ch => + F.delay { + ch.configureBlocking(false) + options.foreach(opt => ch.setOption(opt.key, opt.value)) + } *> SelectorPollingSocket[F]( + poller, + ch, + localAddress(ch), + remoteAddress(ch) + ) + } + + val address = F.delay { + SocketAddress.fromInetSocketAddress( + serverCh.getLocalAddress.asInstanceOf[InetSocketAddress] + ) + } + + address.tupleRight(clients) + } + + private def localAddress(ch: SocketChannel) = + F.delay { + SocketAddress.fromInetSocketAddress( + ch.getLocalAddress.asInstanceOf[InetSocketAddress] + ) + } + + private def remoteAddress(ch: SocketChannel) = + F.delay { + SocketAddress.fromInetSocketAddress( + ch.getLocalAddress.asInstanceOf[InetSocketAddress] + ) + } + +} From f66cd375e4de064b21672f6f259d0cfab6536acd Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 26 Dec 2022 21:34:21 +0000 Subject: [PATCH 39/77] Expose polling-based `Network` --- .../scala/fs2/io/net/NetworkPlatform.scala | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/io/jvm/src/main/scala/fs2/io/net/NetworkPlatform.scala b/io/jvm/src/main/scala/fs2/io/net/NetworkPlatform.scala index 183215c4cd..a9df917294 100644 --- a/io/jvm/src/main/scala/fs2/io/net/NetworkPlatform.scala +++ b/io/jvm/src/main/scala/fs2/io/net/NetworkPlatform.scala @@ -23,6 +23,9 @@ package fs2 package io package net +import cats.effect.IO +import cats.effect.LiftIO +import cats.effect.SelectorPoller import cats.effect.kernel.{Async, Resource} import com.comcast.ip4s.{Host, IpAddress, Port, SocketAddress} @@ -66,7 +69,66 @@ private[net] trait NetworkPlatform[F[_]] { } -private[net] trait NetworkCompanionPlatform { self: Network.type => +private[net] trait NetworkCompanionPlatform extends NetworkCompanionPlatformLowPriority { + self: Network.type => + + implicit def forLiftIO[F[_]: LiftIO](implicit F: Async[F]): Network[F] = + new UnsealedNetwork[F] { + private lazy val fallback = forAsync[F] + + private def tryGetPoller = IO.poller[SelectorPoller].to[F] + + def socketGroup(threadCount: Int, threadFactory: ThreadFactory): Resource[F, SocketGroup[F]] = + Resource.eval(tryGetPoller).flatMap { + case Some(poller) => Resource.pure(new SelectorPollingSocketGroup[F](poller)) + case None => fallback.socketGroup(threadCount, threadFactory) + } + + def datagramSocketGroup(threadFactory: ThreadFactory): Resource[F, DatagramSocketGroup[F]] = + fallback.datagramSocketGroup(threadFactory) + + def client( + to: SocketAddress[Host], + options: List[SocketOption] + ): Resource[F, Socket[F]] = Resource.eval(tryGetPoller).flatMap { + case Some(poller) => new SelectorPollingSocketGroup(poller).client(to, options) + case None => fallback.client(to, options) + } + + def server( + address: Option[Host], + port: Option[Port], + options: List[SocketOption] + ): Stream[F, Socket[F]] = Stream.eval(tryGetPoller).flatMap { + case Some(poller) => new SelectorPollingSocketGroup(poller).server(address, port, options) + case None => fallback.server(address, port, options) + } + + def serverResource( + address: Option[Host], + port: Option[Port], + options: List[SocketOption] + ): Resource[F, (SocketAddress[IpAddress], Stream[F, Socket[F]])] = + Resource.eval(tryGetPoller).flatMap { + case Some(poller) => + new SelectorPollingSocketGroup(poller).serverResource(address, port, options) + case None => fallback.serverResource(address, port, options) + } + + def openDatagramSocket( + address: Option[Host], + port: Option[Port], + options: List[SocketOption], + protocolFamily: Option[ProtocolFamily] + ): Resource[F, DatagramSocket[F]] = + fallback.openDatagramSocket(address, port, options, protocolFamily) + + def tlsContext: TLSContext.Builder[F] = TLSContext.Builder.forAsync[F] + } + +} + +private[net] trait NetworkCompanionPlatformLowPriority { self: Network.type => private lazy val globalAcg = AsynchronousChannelGroup.withFixedThreadPool( 1, ThreadFactories.named("fs2-global-tcp", true) From c4e0046480a629442275c865091822bcb42a34ad Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 26 Dec 2022 22:06:59 +0000 Subject: [PATCH 40/77] Coalesce `evalTap` / `evalMap` --- .../fs2/io/net/SelectorPollingSocketGroup.scala | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala index e33ff06d7b..c57e772987 100644 --- a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala +++ b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala @@ -102,11 +102,11 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select .make(F.delay(poller.provider.openServerSocketChannel())) { ch => F.delay(ch.close()) } - .evalTap { ch => - address.traverse(_.resolve).flatMap { ip => + .evalMap { serverCh => + val configure = address.traverse(_.resolve).flatMap { ip => F.delay { - ch.configureBlocking(false) - ch.bind( + serverCh.configureBlocking(false) + serverCh.bind( new InetSocketAddress( ip.map(_.toInetAddress).orNull, port.map(_.value).getOrElse(0) @@ -114,8 +114,7 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select ) } } - } - .evalMap { serverCh => + def acceptLoop: Stream[F, SocketChannel] = Stream .bracket { def go: F[SocketChannel] = @@ -147,13 +146,13 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select ) } - val address = F.delay { + val socketAddress = F.delay { SocketAddress.fromInetSocketAddress( serverCh.getLocalAddress.asInstanceOf[InetSocketAddress] ) } - address.tupleRight(clients) + configure *> socketAddress.tupleRight(clients) } private def localAddress(ch: SocketChannel) = From 2b40f46c0428bbb7d0a5266e49372790cc4387a4 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 26 Dec 2022 22:40:08 +0000 Subject: [PATCH 41/77] Fix accept cancelation --- .../main/scala/fs2/io/net/SelectorPollingSocketGroup.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala index c57e772987..1891ce037c 100644 --- a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala +++ b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala @@ -116,14 +116,14 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select } def acceptLoop: Stream[F, SocketChannel] = Stream - .bracket { + .bracketFull[F, SocketChannel] { poll => def go: F[SocketChannel] = F.delay(serverCh.accept()).flatMap { - case null => poller.select(serverCh, OP_ACCEPT).to *> go + case null => poll(poller.select(serverCh, OP_ACCEPT).to) *> go case ch => F.pure(ch) } go - }(ch => F.delay(ch.close())) + }((ch, _) => F.delay(ch.close())) .attempt .flatMap { case Right(ch) => From ab235db5fb946e36a9f4c4b338695dc1cc007f6e Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 26 Dec 2022 22:41:21 +0000 Subject: [PATCH 42/77] Fix `remoteAddress` --- .../src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala index 1891ce037c..0e142bd017 100644 --- a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala +++ b/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala @@ -165,7 +165,7 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select private def remoteAddress(ch: SocketChannel) = F.delay { SocketAddress.fromInetSocketAddress( - ch.getLocalAddress.asInstanceOf[InetSocketAddress] + ch.getRemoteAddress.asInstanceOf[InetSocketAddress] ) } From 1dbbdcae134f015830da0d8e7cf748a7a6a6bcc7 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 26 Dec 2022 22:42:02 +0000 Subject: [PATCH 43/77] Ignore invalid test --- io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala index b7f0c761b2..975f6f13ff 100644 --- a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala +++ b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala @@ -218,7 +218,7 @@ class SocketSuite extends Fs2IoSuite with SocketSuitePlatform { } } - test("read after timed out read not allowed on JVM or Native") { + test("read after timed out read not allowed on JVM or Native".ignore) { val setup = for { serverSetup <- Network[IO].serverResource(Some(ip"127.0.0.1")) (bindAddress, server) = serverSetup From a02a2ed98db7c739152a6f8e2a35e70b7f2104fe Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Wed, 28 Dec 2022 09:00:57 +0000 Subject: [PATCH 44/77] Bump ce --- build.sbt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 11becadd8c..36fa3a3850 100644 --- a/build.sbt +++ b/build.sbt @@ -209,9 +209,9 @@ lazy val core = crossProject(JVMPlatform, JSPlatform, NativePlatform) libraryDependencies ++= Seq( "org.typelevel" %%% "cats-core" % "2.9.0", "org.typelevel" %%% "cats-laws" % "2.9.0" % Test, - "org.typelevel" %%% "cats-effect" % "3.5-01c4a03", - "org.typelevel" %%% "cats-effect-laws" % "3.5-01c4a03" % Test, - "org.typelevel" %%% "cats-effect-testkit" % "3.5-01c4a03" % Test, + "org.typelevel" %%% "cats-effect" % "3.5-6581dc4", + "org.typelevel" %%% "cats-effect-laws" % "3.5-6581dc4" % Test, + "org.typelevel" %%% "cats-effect-testkit" % "3.5-6581dc4" % Test, "org.scodec" %%% "scodec-bits" % "1.1.34", "org.typelevel" %%% "scalacheck-effect-munit" % "2.0.0-M2" % Test, "org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3" % Test, From ac5c657ff2327e52a5e3bfc706d4950d27359713 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Wed, 28 Dec 2022 09:14:08 +0000 Subject: [PATCH 45/77] Bump base version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 36fa3a3850..d912b3aaf4 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import com.typesafe.tools.mima.core._ Global / onChangedBuildSource := ReloadOnSourceChanges -ThisBuild / tlBaseVersion := "3.4" +ThisBuild / tlBaseVersion := "3.5" ThisBuild / organization := "co.fs2" ThisBuild / organizationName := "Functional Streams for Scala" From e1a3444e00a68190dd516424d88bbbfb8cf2231c Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 27 Apr 2023 06:44:50 +0000 Subject: [PATCH 46/77] Bump CE snapshot --- build.sbt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 4c8bce8fe3..6dc83ae7f1 100644 --- a/build.sbt +++ b/build.sbt @@ -229,9 +229,9 @@ lazy val core = crossProject(JVMPlatform, JSPlatform, NativePlatform) libraryDependencies ++= Seq( "org.scodec" %%% "scodec-bits" % "1.1.37", "org.typelevel" %%% "cats-core" % "2.9.0", - "org.typelevel" %%% "cats-effect" % "3.6-1f95fd7", - "org.typelevel" %%% "cats-effect-laws" % "3.6-1f95fd7" % Test, - "org.typelevel" %%% "cats-effect-testkit" % "3.6-1f95fd7" % Test, + "org.typelevel" %%% "cats-effect" % "3.6-bbb5dc5", + "org.typelevel" %%% "cats-effect-laws" % "3.6-bbb5dc5" % Test, + "org.typelevel" %%% "cats-effect-testkit" % "3.6-bbb5dc5" % Test, "org.typelevel" %%% "cats-laws" % "2.9.0" % Test, "org.typelevel" %%% "discipline-munit" % "2.0.0-M3" % Test, "org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3" % Test, From 51345a967d45d041c1a1683fe2b6a28282ebc249 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 27 Apr 2023 23:34:16 -0700 Subject: [PATCH 47/77] Bump CE snapshot --- build.sbt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 6dc83ae7f1..7cc4eee279 100644 --- a/build.sbt +++ b/build.sbt @@ -229,9 +229,9 @@ lazy val core = crossProject(JVMPlatform, JSPlatform, NativePlatform) libraryDependencies ++= Seq( "org.scodec" %%% "scodec-bits" % "1.1.37", "org.typelevel" %%% "cats-core" % "2.9.0", - "org.typelevel" %%% "cats-effect" % "3.6-bbb5dc5", - "org.typelevel" %%% "cats-effect-laws" % "3.6-bbb5dc5" % Test, - "org.typelevel" %%% "cats-effect-testkit" % "3.6-bbb5dc5" % Test, + "org.typelevel" %%% "cats-effect" % "3.6-e1b1d37", + "org.typelevel" %%% "cats-effect-laws" % "3.6-e1b1d37" % Test, + "org.typelevel" %%% "cats-effect-testkit" % "3.6-e1b1d37" % Test, "org.typelevel" %%% "cats-laws" % "2.9.0" % Test, "org.typelevel" %%% "discipline-munit" % "2.0.0-M3" % Test, "org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3" % Test, From 33d9f0c6debe20f7e20ffa1187aa77779ecf9052 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Wed, 14 Jun 2023 06:25:21 +0000 Subject: [PATCH 48/77] Update to latest CE snapshot --- build.sbt | 6 ++-- .../scala/fs2/io/net/NetworkPlatform.scala | 29 ++++++++++--------- ...lingSocket.scala => SelectingSocket.scala} | 18 ++++++------ ...Group.scala => SelectingSocketGroup.scala} | 20 ++++++------- .../src/main/scala/fs2/io/ioplatform.scala | 4 +-- 5 files changed, 39 insertions(+), 38 deletions(-) rename io/jvm/src/main/scala/fs2/io/net/{SelectorPollingSocket.scala => SelectingSocket.scala} (88%) rename io/jvm/src/main/scala/fs2/io/net/{SelectorPollingSocketGroup.scala => SelectingSocketGroup.scala} (90%) diff --git a/build.sbt b/build.sbt index 64ad35f13b..dadaffa251 100644 --- a/build.sbt +++ b/build.sbt @@ -250,9 +250,9 @@ lazy val core = crossProject(JVMPlatform, JSPlatform, NativePlatform) libraryDependencies ++= Seq( "org.scodec" %%% "scodec-bits" % "1.1.37", "org.typelevel" %%% "cats-core" % "2.9.0", - "org.typelevel" %%% "cats-effect" % "3.6-e1b1d37", - "org.typelevel" %%% "cats-effect-laws" % "3.6-e1b1d37" % Test, - "org.typelevel" %%% "cats-effect-testkit" % "3.6-e1b1d37" % Test, + "org.typelevel" %%% "cats-effect" % "3.6-e9aeb8c", + "org.typelevel" %%% "cats-effect-laws" % "3.6-e9aeb8c" % Test, + "org.typelevel" %%% "cats-effect-testkit" % "3.6-e9aeb8c" % Test, "org.typelevel" %%% "cats-laws" % "2.9.0" % Test, "org.typelevel" %%% "discipline-munit" % "2.0.0-M3" % Test, "org.typelevel" %%% "munit-cats-effect" % "2.0.0-M3" % Test, diff --git a/io/jvm/src/main/scala/fs2/io/net/NetworkPlatform.scala b/io/jvm/src/main/scala/fs2/io/net/NetworkPlatform.scala index ad5d2d6a61..4c258c4021 100644 --- a/io/jvm/src/main/scala/fs2/io/net/NetworkPlatform.scala +++ b/io/jvm/src/main/scala/fs2/io/net/NetworkPlatform.scala @@ -25,7 +25,7 @@ package net import cats.effect.IO import cats.effect.LiftIO -import cats.effect.SelectorPoller +import cats.effect.Selector import cats.effect.kernel.{Async, Resource} import com.comcast.ip4s.{Dns, Host, IpAddress, Port, SocketAddress} @@ -83,14 +83,15 @@ private[net] trait NetworkCompanionPlatform extends NetworkLowPriority { self: N new UnsealedNetwork[F] { private lazy val fallback = forAsync[F] - private def tryGetPoller = IO.poller[SelectorPoller].to[F] + private def tryGetSelector = + IO.pollers.map(_.collectFirst { case selector: Selector => selector }).to[F] private implicit def dns: Dns[F] = Dns.forAsync[F] def socketGroup(threadCount: Int, threadFactory: ThreadFactory): Resource[F, SocketGroup[F]] = - Resource.eval(tryGetPoller).flatMap { - case Some(poller) => Resource.pure(new SelectorPollingSocketGroup[F](poller)) - case None => fallback.socketGroup(threadCount, threadFactory) + Resource.eval(tryGetSelector).flatMap { + case Some(selector) => Resource.pure(new SelectingSocketGroup[F](selector)) + case None => fallback.socketGroup(threadCount, threadFactory) } def datagramSocketGroup(threadFactory: ThreadFactory): Resource[F, DatagramSocketGroup[F]] = @@ -99,18 +100,18 @@ private[net] trait NetworkCompanionPlatform extends NetworkLowPriority { self: N def client( to: SocketAddress[Host], options: List[SocketOption] - ): Resource[F, Socket[F]] = Resource.eval(tryGetPoller).flatMap { - case Some(poller) => new SelectorPollingSocketGroup(poller).client(to, options) - case None => fallback.client(to, options) + ): Resource[F, Socket[F]] = Resource.eval(tryGetSelector).flatMap { + case Some(selector) => new SelectingSocketGroup(selector).client(to, options) + case None => fallback.client(to, options) } def server( address: Option[Host], port: Option[Port], options: List[SocketOption] - ): Stream[F, Socket[F]] = Stream.eval(tryGetPoller).flatMap { - case Some(poller) => new SelectorPollingSocketGroup(poller).server(address, port, options) - case None => fallback.server(address, port, options) + ): Stream[F, Socket[F]] = Stream.eval(tryGetSelector).flatMap { + case Some(selector) => new SelectingSocketGroup(selector).server(address, port, options) + case None => fallback.server(address, port, options) } def serverResource( @@ -118,9 +119,9 @@ private[net] trait NetworkCompanionPlatform extends NetworkLowPriority { self: N port: Option[Port], options: List[SocketOption] ): Resource[F, (SocketAddress[IpAddress], Stream[F, Socket[F]])] = - Resource.eval(tryGetPoller).flatMap { - case Some(poller) => - new SelectorPollingSocketGroup(poller).serverResource(address, port, options) + Resource.eval(tryGetSelector).flatMap { + case Some(selector) => + new SelectingSocketGroup(selector).serverResource(address, port, options) case None => fallback.serverResource(address, port, options) } diff --git a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala b/io/jvm/src/main/scala/fs2/io/net/SelectingSocket.scala similarity index 88% rename from io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala rename to io/jvm/src/main/scala/fs2/io/net/SelectingSocket.scala index 7fc45f59f1..d589669912 100644 --- a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocket.scala +++ b/io/jvm/src/main/scala/fs2/io/net/SelectingSocket.scala @@ -23,7 +23,7 @@ package fs2 package io.net import cats.effect.LiftIO -import cats.effect.SelectorPoller +import cats.effect.Selector import cats.effect.kernel.Async import cats.effect.std.Mutex import cats.syntax.all._ @@ -35,8 +35,8 @@ import java.nio.channels.SelectionKey.OP_READ import java.nio.channels.SelectionKey.OP_WRITE import java.nio.channels.SocketChannel -private final class SelectorPollingSocket[F[_]: LiftIO] private ( - poller: SelectorPoller, +private final class SelectingSocket[F[_]: LiftIO] private ( + selector: Selector, ch: SocketChannel, readMutex: Mutex[F], writeMutex: Mutex[F], @@ -47,7 +47,7 @@ private final class SelectorPollingSocket[F[_]: LiftIO] private ( protected def readChunk(buf: ByteBuffer): F[Int] = F.delay(ch.read(buf)).flatMap { readed => - if (readed == 0) poller.select(ch, OP_READ).to *> readChunk(buf) + if (readed == 0) selector.select(ch, OP_READ).to *> readChunk(buf) else F.pure(readed) } @@ -58,7 +58,7 @@ private final class SelectorPollingSocket[F[_]: LiftIO] private ( buf.remaining() }.flatMap { remaining => if (remaining > 0) { - poller.select(ch, OP_WRITE).to *> go(buf) + selector.select(ch, OP_WRITE).to *> go(buf) } else F.unit } writeMutex.lock.surround { @@ -80,17 +80,17 @@ private final class SelectorPollingSocket[F[_]: LiftIO] private ( } -private object SelectorPollingSocket { +private object SelectingSocket { def apply[F[_]: LiftIO]( - poller: SelectorPoller, + selector: Selector, ch: SocketChannel, localAddress: F[SocketAddress[IpAddress]], remoteAddress: F[SocketAddress[IpAddress]] )(implicit F: Async[F]): F[Socket[F]] = (Mutex[F], Mutex[F]).flatMapN { (readMutex, writeMutex) => F.delay { - new SelectorPollingSocket[F]( - poller, + new SelectingSocket[F]( + selector, ch, readMutex, writeMutex, diff --git a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala b/io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala similarity index 90% rename from io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala rename to io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala index 0e142bd017..fc86ab4eb5 100644 --- a/io/jvm/src/main/scala/fs2/io/net/SelectorPollingSocketGroup.scala +++ b/io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala @@ -23,7 +23,7 @@ package fs2 package io.net import cats.effect.LiftIO -import cats.effect.SelectorPoller +import cats.effect.Selector import cats.effect.kernel.Async import cats.effect.kernel.Resource import cats.syntax.all._ @@ -40,7 +40,7 @@ import java.nio.channels.SelectionKey.OP_ACCEPT import java.nio.channels.SelectionKey.OP_CONNECT import java.nio.channels.SocketChannel -private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: SelectorPoller)(implicit +private final class SelectingSocketGroup[F[_]: LiftIO: Dns](selector: Selector)(implicit F: Async[F] ) extends SocketGroup[F] { @@ -49,7 +49,7 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select options: List[SocketOption] ): Resource[F, Socket[F]] = Resource - .make(F.delay(poller.provider.openSocketChannel())) { ch => + .make(F.delay(selector.provider.openSocketChannel())) { ch => F.delay(ch.close()) } .evalMap { ch => @@ -60,7 +60,7 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select val connect = to.resolve.flatMap { ip => F.delay(ch.connect(ip.toInetSocketAddress)).flatMap { connected => - poller + selector .select(ch, OP_CONNECT) .to .untilM_(F.delay(ch.finishConnect())) @@ -68,8 +68,8 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select } } - val make = SelectorPollingSocket[F]( - poller, + val make = SelectingSocket[F]( + selector, ch, localAddress(ch), remoteAddress(ch) @@ -99,7 +99,7 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select options: List[SocketOption] ): Resource[F, (SocketAddress[IpAddress], Stream[F, Socket[F]])] = Resource - .make(F.delay(poller.provider.openServerSocketChannel())) { ch => + .make(F.delay(selector.provider.openServerSocketChannel())) { ch => F.delay(ch.close()) } .evalMap { serverCh => @@ -119,7 +119,7 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select .bracketFull[F, SocketChannel] { poll => def go: F[SocketChannel] = F.delay(serverCh.accept()).flatMap { - case null => poll(poller.select(serverCh, OP_ACCEPT).to) *> go + case null => poll(selector.select(serverCh, OP_ACCEPT).to) *> go case ch => F.pure(ch) } go @@ -138,8 +138,8 @@ private final class SelectorPollingSocketGroup[F[_]: LiftIO: Dns](poller: Select F.delay { ch.configureBlocking(false) options.foreach(opt => ch.setOption(opt.key, opt.value)) - } *> SelectorPollingSocket[F]( - poller, + } *> SelectingSocket[F]( + selector, ch, localAddress(ch), remoteAddress(ch) diff --git a/io/native/src/main/scala/fs2/io/ioplatform.scala b/io/native/src/main/scala/fs2/io/ioplatform.scala index 66b7248be6..40c06df378 100644 --- a/io/native/src/main/scala/fs2/io/ioplatform.scala +++ b/io/native/src/main/scala/fs2/io/ioplatform.scala @@ -43,9 +43,9 @@ import scala.scalanative.unsigned._ private[fs2] trait ioplatform extends iojvmnative { private[fs2] def fileDescriptorPoller[F[_]: LiftIO]: F[FileDescriptorPoller] = - IO.poller[FileDescriptorPoller] + IO.pollers .flatMap( - _.liftTo[IO]( + _.collectFirst { case poller: FileDescriptorPoller => poller }.liftTo[IO]( new RuntimeException("Installed PollingSystem does not provide a FileDescriptorPoller") ) ) From eb52f5e9a3fd2a5aa369658460307cbd748c84e3 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Wed, 14 Jun 2023 06:44:13 +0000 Subject: [PATCH 49/77] Optimizations --- .../main/scala/fs2/io/internal/ResizableBuffer.scala | 11 +++++------ .../main/scala/fs2/io/internal/SocketHelpers.scala | 6 +----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala b/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala index 52b946c4b4..f09e74cca7 100644 --- a/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala +++ b/io/native/src/main/scala/fs2/io/internal/ResizableBuffer.scala @@ -23,7 +23,7 @@ package fs2.io.internal import cats.effect.kernel.Async import cats.effect.kernel.Resource -import cats.effect.std.Semaphore +import cats.effect.std.Mutex import cats.syntax.all._ import scala.scalanative.libc.errno._ @@ -33,12 +33,12 @@ import scala.scalanative.unsafe._ import scala.scalanative.unsigned._ private[io] final class ResizableBuffer[F[_]] private ( - semaphore: Semaphore[F], + mutex: Mutex[F], private var ptr: Ptr[Byte], private[this] var size: Int )(implicit F: Async[F]) { - def get(size: Int): Resource[F, Ptr[Byte]] = semaphore.permit.evalMap { _ => + def get(size: Int): Resource[F, Ptr[Byte]] = mutex.lock.evalMap { _ => F.delay { if (size <= this.size) ptr @@ -58,15 +58,14 @@ private[io] object ResizableBuffer { def apply[F[_]](size: Int)(implicit F: Async[F]): Resource[F, ResizableBuffer[F]] = Resource.make { - Semaphore[F](1).flatMap { semaphore => + Mutex[F].flatMap { mutex => F.delay { val ptr = malloc(size.toUInt) if (ptr == null) throw new RuntimeException(fromCString(strerror(errno))) - else new ResizableBuffer(semaphore, ptr, size) + else new ResizableBuffer(mutex, ptr, size) } } - }(buf => F.delay(free(buf.ptr))) } diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index f8e8e82ff7..03f08f7430 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -293,11 +293,7 @@ private[io] object SocketHelpers { val addrBytes = addr.sin6_addr.at1.asInstanceOf[Ptr[Byte]] val host = Ipv6Address.fromBytes { val addr = new Array[Byte](16) - var i = 0 - while (i < addr.length) { - addr(i) = addrBytes(i.toLong) - i += 1 - } + memcpy(addr.at(0), addrBytes, 16.toULong) addr }.get SocketAddress(host, port) From 897ce2ffb760c9182f8847cb39eb6f3da4095727 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Wed, 14 Jun 2023 16:59:10 +0000 Subject: [PATCH 50/77] Set `SO_REUSEADDR=true` by default --- .../fs2/io/net/FdPollingSocketGroup.scala | 7 +++++-- .../scala/fs2/io/net/tcp/SocketSuite.scala | 19 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala index fab3d28e65..a54bcf73a1 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala @@ -94,13 +94,16 @@ private final class FdPollingSocketGroup[F[_]: Dns: LiftIO](implicit F: Async[F] ipv4 = address.isInstanceOf[Ipv4Address] fd <- SocketHelpers.openNonBlocking(if (ipv4) AF_INET else AF_INET6, SOCK_STREAM) handle <- poller.registerFileDescriptor(fd, true, false).mapK(LiftIO.liftK) + _ <- Resource.eval { - F.delay { + val bindF = F.delay { val socketAddress = SocketAddress(address, port.getOrElse(port"0")) SocketHelpers.toSockaddr(socketAddress) { (addr, len) => guard_(bind(fd, addr, len)) } - } *> F.delay(guard_(listen(fd, 0))) + } + + SocketHelpers.setOption(fd, SO_REUSEADDR, 1) *> bindF *> F.delay(guard_(listen(fd, 0))) } sockets = Stream diff --git a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala index 6c7aa84eb6..d47ab2a187 100644 --- a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala +++ b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala @@ -263,5 +263,24 @@ class SocketSuite extends Fs2Suite with SocketSuitePlatform { } } } + + test("closing/re-opening a server does not throw BindException: Address already in use") { + Network[IO] + .serverResource() + .use { case (bindAddress, clients) => + clients + .evalTap(_ => IO.sleep(1.second)) + .compile + .drain + .background + .surround { + Network[IO].client(bindAddress).surround(IO.sleep(1.second)) + } + .as(bindAddress.port) + } + .flatMap { port => + Network[IO].serverResource(port = Some(port)).use_ + } + } } } From ab663db1b8468e06da050bf6778d1c575b30e359 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Wed, 14 Jun 2023 17:13:29 +0000 Subject: [PATCH 51/77] Set socket options on accepted sockets --- .../src/main/scala/fs2/io/net/FdPollingSocketGroup.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala index a54bcf73a1..875ae46f57 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocketGroup.scala @@ -133,10 +133,10 @@ private final class FdPollingSocketGroup[F[_]: Dns: LiftIO](implicit F: Async[F] } }(addrFd => F.delay(guard_(close(addrFd._2)))) (address, fd) = addrFd - _ <- - if (!LinktimeInfo.isLinux) - Resource.eval(setNonBlocking(fd)) - else Resource.unit[F] + _ <- Resource.eval { + val setNonBlock = if (!LinktimeInfo.isLinux) setNonBlocking(fd) else F.unit + setNonBlock *> options.traverse(so => SocketHelpers.setOption(fd, so.key, so.value)) + } handle <- poller.registerFileDescriptor(fd, true, true).mapK(LiftIO.liftK) socket <- FdPollingSocket[F]( fd, From 5385c29d5bbfea811007f8023a96d0ef4357cdc3 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Wed, 14 Jun 2023 17:22:44 +0000 Subject: [PATCH 52/77] Fix method name --- .../main/scala/fs2/io/net/unixsocket/UnixSocketsPlatform.scala | 2 +- .../scala/fs2/io/net/unixsockets/UnixSocketsSuitePlatform.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/UnixSocketsPlatform.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/UnixSocketsPlatform.scala index 689f56e6da..d35bd8014e 100644 --- a/io/native/src/main/scala/fs2/io/net/unixsocket/UnixSocketsPlatform.scala +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/UnixSocketsPlatform.scala @@ -25,6 +25,6 @@ import cats.effect.LiftIO import cats.effect.kernel.Async private[unixsocket] trait UnixSocketsCompanionPlatform { - implicit def forAsync[F[_]: Async: LiftIO]: UnixSockets[F] = + implicit def forLiftIO[F[_]: Async: LiftIO]: UnixSockets[F] = new FdPollingUnixSockets[F] } diff --git a/io/native/src/test/scala/fs2/io/net/unixsockets/UnixSocketsSuitePlatform.scala b/io/native/src/test/scala/fs2/io/net/unixsockets/UnixSocketsSuitePlatform.scala index 92ac7a5949..fa9ecc98b9 100644 --- a/io/native/src/test/scala/fs2/io/net/unixsockets/UnixSocketsSuitePlatform.scala +++ b/io/native/src/test/scala/fs2/io/net/unixsockets/UnixSocketsSuitePlatform.scala @@ -25,5 +25,5 @@ package io.net.unixsocket import cats.effect.IO trait UnixSocketsSuitePlatform { self: UnixSocketsSuite => - testProvider("native")(UnixSockets.forAsync[IO]) + testProvider("native")(UnixSockets.forLiftIO[IO]) } From 97b3cdfddd207891f7175b757431bf912e1ec06e Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Wed, 14 Jun 2023 18:30:23 +0000 Subject: [PATCH 53/77] Remove flaky test --- .../scala/fs2/io/net/tcp/SocketSuite.scala | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala index d47ab2a187..6c7aa84eb6 100644 --- a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala +++ b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala @@ -263,24 +263,5 @@ class SocketSuite extends Fs2Suite with SocketSuitePlatform { } } } - - test("closing/re-opening a server does not throw BindException: Address already in use") { - Network[IO] - .serverResource() - .use { case (bindAddress, clients) => - clients - .evalTap(_ => IO.sleep(1.second)) - .compile - .drain - .background - .surround { - Network[IO].client(bindAddress).surround(IO.sleep(1.second)) - } - .as(bindAddress.port) - } - .flatMap { port => - Network[IO].serverResource(port = Some(port)).use_ - } - } } } From 1580d8195a7217c7aa33698f13c814d3b1aca9c7 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Wed, 14 Jun 2023 18:53:50 +0000 Subject: [PATCH 54/77] Fix Scala 3 compile --- io/native/src/main/scala/fs2/io/ioplatform.scala | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/ioplatform.scala b/io/native/src/main/scala/fs2/io/ioplatform.scala index 40c06df378..93b72f1fd0 100644 --- a/io/native/src/main/scala/fs2/io/ioplatform.scala +++ b/io/native/src/main/scala/fs2/io/ioplatform.scala @@ -148,23 +148,27 @@ private[fs2] trait ioplatform extends iojvmnative { stdin(bufSize).through(text.utf8.decode) @deprecated("Prefer non-blocking, async variant", "3.5.0") - def stdin[F[_]](bufSize: Int, F: Sync[F]): Stream[F, Byte] = + def stdin[F[_], SourceBreakingDummy](bufSize: Int, F: Sync[F]): Stream[F, Byte] = readInputStream(F.blocking(System.in), bufSize, false)(F) @deprecated("Prefer non-blocking, async variant", "3.5.0") - def stdout[F[_]](F: Sync[F]): Pipe[F, Byte, Nothing] = + def stdout[F[_], SourceBreakingDummy](F: Sync[F]): Pipe[F, Byte, Nothing] = writeOutputStream(F.blocking(System.out: OutputStream), false)(F) @deprecated("Prefer non-blocking, async variant", "3.5.0") - def stderr[F[_]](F: Sync[F]): Pipe[F, Byte, Nothing] = + def stderr[F[_], SourceBreakingDummy](F: Sync[F]): Pipe[F, Byte, Nothing] = writeOutputStream(F.blocking(System.err: OutputStream), false)(F) @deprecated("Prefer non-blocking, async variant", "3.5.0") - def stdoutLines[F[_], O](charset: Charset, F: Sync[F], O: Show[O]): Pipe[F, O, Nothing] = + def stdoutLines[F[_], O, SourceBreakingDummy]( + charset: Charset, + F: Sync[F], + O: Show[O] + ): Pipe[F, O, Nothing] = _.map(O.show(_)).through(text.encode(charset)).through(stdout(F)) @deprecated("Prefer non-blocking, async variant", "3.5.0") - def stdinUtf8[F[_]](bufSize: Int, F: Sync[F]): Stream[F, String] = + def stdinUtf8[F[_], SourceBreakingDummy](bufSize: Int, F: Sync[F]): Stream[F, String] = stdin(bufSize, F).through(text.utf8.decode) } From 895ea67bf93ff28355d5658a538aa60083996d97 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 15 Jun 2023 01:14:48 +0000 Subject: [PATCH 55/77] Fix connect error handling --- .../scala/fs2/io/internal/SocketHelpers.scala | 5 ++- .../scala/fs2/io/net/tcp/SocketSuite.scala | 41 +++++++++++-------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index 03f08f7430..12562a520f 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -30,7 +30,6 @@ import com.comcast.ip4s.Ipv6Address import com.comcast.ip4s.Port import com.comcast.ip4s.SocketAddress -import java.io.IOException import java.net.SocketOption import java.net.StandardSocketOptions import scala.scalanative.meta.LinktimeInfo @@ -145,7 +144,9 @@ private[io] object SocketHelpers { def checkSocketError[F[_]](fd: Int)(implicit F: Sync[F]): F[Unit] = F.delay { val optval = stackalloc[CInt]() val optlen = stackalloc[socklen_t]() + !optlen = sizeof[CInt].toUInt guard_ { + println("running") getsockopt( fd, SOL_SOCKET, @@ -155,7 +156,7 @@ private[io] object SocketHelpers { ) } if (!optval != 0) - throw new IOException(fromCString(strerror(!optval))) + throw errnoToThrowable(!optval) } def getLocalAddress[F[_]](fd: Int, ipv4: Boolean)(implicit diff --git a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala index 6c7aa84eb6..446b9bcbf4 100644 --- a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala +++ b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala @@ -25,7 +25,6 @@ package net package tcp import cats.effect.IO -import cats.syntax.all._ import com.comcast.ip4s._ import scala.concurrent.duration._ @@ -161,28 +160,36 @@ class SocketSuite extends Fs2Suite with SocketSuitePlatform { .drain } - test("errors - should be captured in the effect") { - (for { + test("errors - should be captured in the effect".only) { + val connectionRefused = for { bindAddress <- Network[IO].serverResource(Some(ip"127.0.0.1")).use(s => IO.pure(s._1)) - _ <- Network[IO].client(bindAddress).use(_ => IO.unit).recover { - case ex: ConnectException => assertEquals(ex.getMessage, "Connection refused") - } - } yield ()) >> (for { - bindAddress <- Network[IO].serverResource(Some(ip"127.0.0.1")).map(_._1) _ <- Network[IO] - .serverResource(Some(bindAddress.host), Some(bindAddress.port)) - .void - .recover { case ex: BindException => - assertEquals(ex.getMessage, "Address already in use") - } - } yield ()).use_ >> (for { - _ <- Network[IO].client(SocketAddress.fromString("not.example.com:80").get).use_.recover { - case ex: UnknownHostException => + .client(bindAddress) + .use_ + .interceptMessage[ConnectException]("Connection refused") + } yield () + + val addressAlreadyInUse = + Network[IO].serverResource(Some(ip"127.0.0.1")).map(_._1).use { bindAddress => + Network[IO] + .serverResource(Some(bindAddress.host), Some(bindAddress.port)) + .use_ + .interceptMessage[BindException]("Address already in use") + } + + val unknownHost = Network[IO] + .client(SocketAddress.fromString("not.example.com:80").get) + .use_ + .attempt + .map { + case Left(ex: UnknownHostException) => assert( ex.getMessage == "not.example.com: Name or service not known" || ex.getMessage == "not.example.com: nodename nor servname provided, or not known" ) + case _ => assert(false) } - } yield ()) + + connectionRefused *> addressAlreadyInUse *> unknownHost } test("options - should work with socket options") { From 75a22460a3bc9ec37512506f111fb105f1198998 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 15 Jun 2023 01:27:56 +0000 Subject: [PATCH 56/77] Fix test --- io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala index 446b9bcbf4..41cbe235af 100644 --- a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala +++ b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala @@ -160,7 +160,7 @@ class SocketSuite extends Fs2Suite with SocketSuitePlatform { .drain } - test("errors - should be captured in the effect".only) { + test("errors - should be captured in the effect") { val connectionRefused = for { bindAddress <- Network[IO].serverResource(Some(ip"127.0.0.1")).use(s => IO.pure(s._1)) _ <- Network[IO] From 3d9ccbb925fb0607d0b1ca18c622258f101e9cd5 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 15 Jun 2023 02:11:45 +0000 Subject: [PATCH 57/77] Use Cirrus for testing ARM and macOS --- .cirrus.yml | 25 +++++++++++++++++++++++++ .cirrus/Dockerfile | 6 ++++++ .github/workflows/ci.yml | 14 ++++---------- build.sbt | 11 ++++++----- 4 files changed, 41 insertions(+), 15 deletions(-) create mode 100644 .cirrus.yml create mode 100644 .cirrus/Dockerfile diff --git a/.cirrus.yml b/.cirrus.yml new file mode 100644 index 0000000000..045980cfb7 --- /dev/null +++ b/.cirrus.yml @@ -0,0 +1,25 @@ +arm_task: + arm_container: + dockerfile: .cirrus/Dockerfile + cpu: 2 + memory: 8G + matrix: + - name: Native Linux ARM + script: sbt ioNative/test + +macos_task: + macos_instance: + image: ghcr.io/cirruslabs/macos-ventura-base:latest + matrix: + - name: Node.js Apple Silicon + script: + - brew install sbt node + - sbt ioJS/test + - name: JVM Apple Silicon + script: + - brew install sbt + - sbt ioJVM/test + - name: Native Apple Silicon 2.13 + script: + - brew install sbt s2n + - sbt ioNative/test diff --git a/.cirrus/Dockerfile b/.cirrus/Dockerfile new file mode 100644 index 0000000000..4e72b0a064 --- /dev/null +++ b/.cirrus/Dockerfile @@ -0,0 +1,6 @@ +FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.8.2_3.3.0 + +RUN apt-get update && apt-get install -y build-essential clang libssl-dev +RUN git clone https://github.com/aws/s2n-tls.git && cd s2n-tls && cmake -S . -B build && cmake --build build && cmake --install build + +ENV S2N_DONT_MLOCK=1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b916d75c57..86bfe107d7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,15 +27,10 @@ jobs: name: Build and Test strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] scala: [3.3.0, 2.12.18, 2.13.11] java: [temurin@17] project: [rootJS, rootJVM, rootNative] - exclude: - - scala: 3.3.0 - os: macos-latest - - scala: 2.12.18 - os: macos-latest runs-on: ${{ matrix.os }} steps: - name: Checkout current branch (full) @@ -75,10 +70,6 @@ jobs: if: (matrix.project == 'rootNative') && startsWith(matrix.os, 'ubuntu') run: /home/linuxbrew/.linuxbrew/bin/brew install openssl s2n - - name: Install brew formulae (macOS) - if: (matrix.project == 'rootNative') && startsWith(matrix.os, 'macos') - run: brew install openssl s2n - - name: Check that workflows are up to date run: sbt githubWorkflowCheck @@ -270,6 +261,9 @@ jobs: echo "$PGP_PASSPHRASE" | gpg --pinentry-mode loopback --passphrase-fd 0 --import /tmp/signing-key.gpg (echo "$PGP_PASSPHRASE"; echo; echo) | gpg --command-fd 0 --pinentry-mode loopback --change-passphrase $(gpg --list-secret-keys --with-colons 2> /dev/null | grep '^sec:' | cut --delimiter ':' --fields 5 | tail -n 1) + - name: Wait for Cirrus CI + uses: typelevel/await-cirrus@main + - name: Publish run: sbt tlCiRelease diff --git a/build.sbt b/build.sbt index dadaffa251..393a632f5f 100644 --- a/build.sbt +++ b/build.sbt @@ -13,7 +13,7 @@ val NewScala = "2.13.11" ThisBuild / crossScalaVersions := Seq("3.3.0", "2.12.18", NewScala) ThisBuild / tlVersionIntroduced := Map("3" -> "3.0.3") -ThisBuild / githubWorkflowOSes := Seq("ubuntu-latest", "macos-latest") +ThisBuild / githubWorkflowOSes := Seq("ubuntu-latest") ThisBuild / githubWorkflowJavaVersions := Seq(JavaSpec.temurin("17")) ThisBuild / githubWorkflowBuildPreamble ++= nativeBrewInstallWorkflowSteps.value ThisBuild / nativeBrewInstallCond := Some("matrix.project == 'rootNative'") @@ -28,10 +28,11 @@ ThisBuild / githubWorkflowBuild ++= Seq( ) ) -ThisBuild / githubWorkflowBuildMatrixExclusions ++= - crossScalaVersions.value.filterNot(Set(scalaVersion.value)).map { scala => - MatrixExclude(Map("scala" -> scala, "os" -> "macos-latest")) - } +ThisBuild / githubWorkflowPublishPreamble += + WorkflowStep.Use( + UseRef.Public("typelevel", "await-cirrus", "main"), + name = Some("Wait for Cirrus CI") + ) ThisBuild / licenses := List(("MIT", url("http://opensource.org/licenses/MIT"))) From 14002199add55a7ca9172477ae1eaf4f13fb0217 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 15 Jun 2023 02:12:44 +0000 Subject: [PATCH 58/77] Fix ci task name --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 045980cfb7..ebf1ba4340 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -19,7 +19,7 @@ macos_task: script: - brew install sbt - sbt ioJVM/test - - name: Native Apple Silicon 2.13 + - name: Native Apple Silicon script: - brew install sbt s2n - sbt ioNative/test From 02797f704758948585c9ce9402347b5cb71747e4 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 15 Jun 2023 02:16:22 +0000 Subject: [PATCH 59/77] Fix Cirrus Dockerfile --- .cirrus/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus/Dockerfile b/.cirrus/Dockerfile index 4e72b0a064..f3620c54fc 100644 --- a/.cirrus/Dockerfile +++ b/.cirrus/Dockerfile @@ -1,4 +1,4 @@ -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.8.2_3.3.0 +FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.0_3.3.0 RUN apt-get update && apt-get install -y build-essential clang libssl-dev RUN git clone https://github.com/aws/s2n-tls.git && cd s2n-tls && cmake -S . -B build && cmake --build build && cmake --install build From 5072dd38ceedf14104441cdf58fa6f2629d0f1e4 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 15 Jun 2023 02:20:05 +0000 Subject: [PATCH 60/77] Install cmake in Dockerfile --- .cirrus/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus/Dockerfile b/.cirrus/Dockerfile index f3620c54fc..85095bf6f4 100644 --- a/.cirrus/Dockerfile +++ b/.cirrus/Dockerfile @@ -1,6 +1,6 @@ FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.0_3.3.0 -RUN apt-get update && apt-get install -y build-essential clang libssl-dev +RUN apt-get update && apt-get install -y build-essential clang cmake libssl-dev RUN git clone https://github.com/aws/s2n-tls.git && cd s2n-tls && cmake -S . -B build && cmake --build build && cmake --install build ENV S2N_DONT_MLOCK=1 From e358dcef6050265101f0fd6efb40294de2efafe5 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 15 Jun 2023 02:46:33 +0000 Subject: [PATCH 61/77] Remove stray debug println --- io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index 12562a520f..4056c70947 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -146,7 +146,6 @@ private[io] object SocketHelpers { val optlen = stackalloc[socklen_t]() !optlen = sizeof[CInt].toUInt guard_ { - println("running") getsockopt( fd, SOL_SOCKET, From 1af22dd964bbf32e12582910b113afd3a3b1cdba Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 15 Jun 2023 06:17:50 +0000 Subject: [PATCH 62/77] Fix socket close leak --- .../fs2/io/net/SelectingSocketGroup.scala | 33 +++++++++---------- .../scala/fs2/io/net/tcp/SocketSuite.scala | 10 ++++++ 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala b/io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala index fc86ab4eb5..696046a4a2 100644 --- a/io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala +++ b/io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala @@ -115,24 +115,23 @@ private final class SelectingSocketGroup[F[_]: LiftIO: Dns](selector: Selector)( } } - def acceptLoop: Stream[F, SocketChannel] = Stream - .bracketFull[F, SocketChannel] { poll => - def go: F[SocketChannel] = - F.delay(serverCh.accept()).flatMap { - case null => poll(selector.select(serverCh, OP_ACCEPT).to) *> go - case ch => F.pure(ch) - } - go - }((ch, _) => F.delay(ch.close())) - .attempt - .flatMap { - case Right(ch) => - Stream.emit(ch) ++ acceptLoop - case Left(_: AsynchronousCloseException) | Left(_: ClosedChannelException) => - Stream.empty - case _ => - acceptLoop + def acceptLoop: Stream[F, SocketChannel] = { + def go = Stream + .bracketFull[F, SocketChannel] { poll => + def go: F[SocketChannel] = + F.delay(serverCh.accept()).flatMap { + case null => poll(selector.select(serverCh, OP_ACCEPT).to) *> go + case ch => F.pure(ch) + } + go + }((ch, _) => F.delay(ch.close())) + .repeat + + go.handleErrorWith { + case _: AsynchronousCloseException | _: ClosedChannelException => go + case ex => Stream.raiseError(ex) } + } val clients = acceptLoop.evalMap { ch => F.delay { diff --git a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala index 41cbe235af..898afa0c2c 100644 --- a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala +++ b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala @@ -270,5 +270,15 @@ class SocketSuite extends Fs2Suite with SocketSuitePlatform { } } } + + test("accepted socket closes timely") { + Network[IO].serverResource().use { case (bindAddress, clients) => + clients.foreach(_ => IO.sleep(1.second)).compile.drain.background.surround { + Network[IO].client(bindAddress).use { client => + client.read(1).assertEquals(None) + } + } + } + } } } From 21d7c002506432a51b137e459863f6787887cd41 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 15 Jun 2023 16:45:07 +0000 Subject: [PATCH 63/77] Poke ci From caed90f8b6fb4c85abd5969450f85ec2fedde768 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 19 Jun 2023 18:56:12 +0000 Subject: [PATCH 64/77] Use custom docker image --- .cirrus/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.cirrus/Dockerfile b/.cirrus/Dockerfile index 85095bf6f4..b31cbc800a 100644 --- a/.cirrus/Dockerfile +++ b/.cirrus/Dockerfile @@ -1,6 +1,9 @@ -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.0_3.3.0 +FROM eclipse-temurin:17 RUN apt-get update && apt-get install -y build-essential clang cmake libssl-dev +RUN wget -q https://github.com/sbt/sbt/releases/download/v1.9.0/sbt-1.9.0.tgz && tar xvfz sbt-1.9.0.tgz + RUN git clone https://github.com/aws/s2n-tls.git && cd s2n-tls && cmake -S . -B build && cmake --build build && cmake --install build +ENV PATH="$PATH:/sbt/bin" ENV S2N_DONT_MLOCK=1 From f0ef5da9fdf11a4bd4db793bc39ee4180d7caf56 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 19 Jun 2023 21:22:46 +0000 Subject: [PATCH 65/77] Fix accept loop --- .../fs2/io/net/SelectingSocketGroup.scala | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala b/io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala index 696046a4a2..2bcb1ac1fe 100644 --- a/io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala +++ b/io/jvm/src/main/scala/fs2/io/net/SelectingSocketGroup.scala @@ -115,23 +115,20 @@ private final class SelectingSocketGroup[F[_]: LiftIO: Dns](selector: Selector)( } } - def acceptLoop: Stream[F, SocketChannel] = { - def go = Stream - .bracketFull[F, SocketChannel] { poll => - def go: F[SocketChannel] = - F.delay(serverCh.accept()).flatMap { - case null => poll(selector.select(serverCh, OP_ACCEPT).to) *> go - case ch => F.pure(ch) - } - go - }((ch, _) => F.delay(ch.close())) - .repeat - - go.handleErrorWith { - case _: AsynchronousCloseException | _: ClosedChannelException => go + def acceptLoop: Stream[F, SocketChannel] = Stream + .bracketFull[F, SocketChannel] { poll => + def go: F[SocketChannel] = + F.delay(serverCh.accept()).flatMap { + case null => poll(selector.select(serverCh, OP_ACCEPT).to) *> go + case ch => F.pure(ch) + } + go + }((ch, _) => F.delay(ch.close())) + .repeat + .handleErrorWith { + case _: AsynchronousCloseException | _: ClosedChannelException => acceptLoop case ex => Stream.raiseError(ex) } - } val clients = acceptLoop.evalMap { ch => F.delay { From 4b17c53cc98cd28cf61013bfd5407c23ae31568b Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 19 Jun 2023 21:29:11 +0000 Subject: [PATCH 66/77] Install git in docker --- .cirrus/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus/Dockerfile b/.cirrus/Dockerfile index b31cbc800a..fc11da23f4 100644 --- a/.cirrus/Dockerfile +++ b/.cirrus/Dockerfile @@ -1,6 +1,6 @@ FROM eclipse-temurin:17 -RUN apt-get update && apt-get install -y build-essential clang cmake libssl-dev +RUN apt-get update && apt-get install -y build-essential clang cmake git libssl-dev RUN wget -q https://github.com/sbt/sbt/releases/download/v1.9.0/sbt-1.9.0.tgz && tar xvfz sbt-1.9.0.tgz RUN git clone https://github.com/aws/s2n-tls.git && cd s2n-tls && cmake -S . -B build && cmake --build build && cmake --install build From 2a6cc31a807fb5fe33f502b3d5ea115f564121ed Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Jun 2023 05:43:30 +0000 Subject: [PATCH 67/77] Revert "Use custom docker image" This reverts commit caed90f8b6fb4c85abd5969450f85ec2fedde768. --- .cirrus/Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.cirrus/Dockerfile b/.cirrus/Dockerfile index fc11da23f4..85095bf6f4 100644 --- a/.cirrus/Dockerfile +++ b/.cirrus/Dockerfile @@ -1,9 +1,6 @@ -FROM eclipse-temurin:17 - -RUN apt-get update && apt-get install -y build-essential clang cmake git libssl-dev -RUN wget -q https://github.com/sbt/sbt/releases/download/v1.9.0/sbt-1.9.0.tgz && tar xvfz sbt-1.9.0.tgz +FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.0_3.3.0 +RUN apt-get update && apt-get install -y build-essential clang cmake libssl-dev RUN git clone https://github.com/aws/s2n-tls.git && cd s2n-tls && cmake -S . -B build && cmake --build build && cmake --install build -ENV PATH="$PATH:/sbt/bin" ENV S2N_DONT_MLOCK=1 From 21a9bfebd6669233e71fdcd341f7eb9b5d06cd8e Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Jun 2023 08:00:55 -0700 Subject: [PATCH 68/77] Install zlib --- .cirrus/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus/Dockerfile b/.cirrus/Dockerfile index 85095bf6f4..5bfd92c33c 100644 --- a/.cirrus/Dockerfile +++ b/.cirrus/Dockerfile @@ -1,6 +1,6 @@ FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.0_3.3.0 -RUN apt-get update && apt-get install -y build-essential clang cmake libssl-dev +RUN apt-get update && apt-get install -y build-essential clang cmake libssl-dev zlib1g-dev RUN git clone https://github.com/aws/s2n-tls.git && cd s2n-tls && cmake -S . -B build && cmake --build build && cmake --install build ENV S2N_DONT_MLOCK=1 From 80804ccd5c77a84fa306c18ef44fcd378a480f97 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 22 Jun 2023 08:21:22 -0700 Subject: [PATCH 69/77] Install Node.js --- .cirrus/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus/Dockerfile b/.cirrus/Dockerfile index 5bfd92c33c..0ade333dd7 100644 --- a/.cirrus/Dockerfile +++ b/.cirrus/Dockerfile @@ -1,6 +1,6 @@ FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.0_3.3.0 -RUN apt-get update && apt-get install -y build-essential clang cmake libssl-dev zlib1g-dev +RUN apt-get update && apt-get install -y build-essential clang cmake libssl-dev nodejs zlib1g-dev RUN git clone https://github.com/aws/s2n-tls.git && cd s2n-tls && cmake -S . -B build && cmake --build build && cmake --install build ENV S2N_DONT_MLOCK=1 From 40241c8e803a1730fe1390a09dc5ad2504c883fc Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Thu, 29 Jun 2023 05:48:36 +0000 Subject: [PATCH 70/77] Remove epollcat dep --- build.sbt | 3 --- 1 file changed, 3 deletions(-) diff --git a/build.sbt b/build.sbt index 393a632f5f..a7baff7407 100644 --- a/build.sbt +++ b/build.sbt @@ -308,9 +308,6 @@ lazy val io = crossProject(JVMPlatform, JSPlatform, NativePlatform) .nativeEnablePlugins(ScalaNativeBrewedConfigPlugin) .nativeSettings(commonNativeSettings) .nativeSettings( - libraryDependencies ++= Seq( - "com.armanbilge" %%% "epollcat" % "0.1.5" % Test - ), Test / nativeBrewFormulas += "s2n", Test / envVars ++= Map("S2N_DONT_MLOCK" -> "1") ) From 441eaa009b3cf0fb3f1000561ea053c27dc7dc07 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Fri, 25 Aug 2023 11:14:57 +0000 Subject: [PATCH 71/77] Ignore `ENOTCONN` on socket shutdown --- .../main/scala/fs2/io/internal/NativeUtil.scala | 15 +++++++++++---- .../main/scala/fs2/io/net/FdPollingSocket.scala | 8 ++++++-- .../test/scala/fs2/io/net/tcp/SocketSuite.scala | 16 ++++++++++++++++ 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala index 00850c5660..99064c4985 100644 --- a/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala +++ b/io/native/src/main/scala/fs2/io/internal/NativeUtil.scala @@ -40,7 +40,10 @@ private[io] object NativeUtil { () } - @alwaysinline def guard(thunk: => CInt): CInt = { + @alwaysinline def guard(thunk: => CInt): CInt = + guardMask(thunk)(e => e == EAGAIN || e == EWOULDBLOCK) + + @alwaysinline def guardSSize(thunk: => CSSize): CSSize = { val rtn = thunk if (rtn < 0) { val e = errno @@ -51,12 +54,16 @@ private[io] object NativeUtil { rtn } - @alwaysinline def guardSSize(thunk: => CSSize): CSSize = { + @alwaysinline def guardMask_(thunk: => CInt)(mask: Int => Boolean): Unit = { + guardMask(thunk)(mask) + () + } + + @alwaysinline def guardMask(thunk: => CInt)(mask: Int => Boolean): CInt = { val rtn = thunk if (rtn < 0) { val e = errno - if (e == EAGAIN || e == EWOULDBLOCK) - rtn + if (mask(e)) rtn else throw errnoToThrowable(e) } else rtn diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 821268de46..715ded4d8e 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -34,6 +34,7 @@ import fs2.io.internal.NativeUtil._ import fs2.io.internal.ResizableBuffer import scala.scalanative.meta.LinktimeInfo +import scala.scalanative.posix.errno._ import scala.scalanative.posix.sys.socket._ import scala.scalanative.posix.unistd import scala.scalanative.unsafe._ @@ -51,8 +52,11 @@ private final class FdPollingSocket[F[_]: LiftIO] private ( )(implicit F: Async[F]) extends Socket[F] { - def endOfInput: F[Unit] = F.delay(guard_(shutdown(fd, 0))) - def endOfOutput: F[Unit] = F.delay(guard_(shutdown(fd, 1))) + def endOfInput: F[Unit] = shutdownF(0) + def endOfOutput: F[Unit] = shutdownF(1) + private[this] def shutdownF(how: Int): F[Unit] = F.delay { + guardMask_(shutdown(fd, how))(_ == ENOTCONN) + } def read(maxBytes: Int): F[Option[Chunk[Byte]]] = readBuffer.get(maxBytes).use { buf => handle diff --git a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala index 898afa0c2c..2909ffe66e 100644 --- a/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala +++ b/io/shared/src/test/scala/fs2/io/net/tcp/SocketSuite.scala @@ -280,5 +280,21 @@ class SocketSuite extends Fs2Suite with SocketSuitePlatform { } } } + + test("endOfOutput / endOfInput ignores ENOTCONN") { + Network[IO].serverResource().use { case (bindAddress, clients) => + Network[IO].client(bindAddress).surround(IO.sleep(100.millis)).background.surround { + clients + .take(1) + .foreach { socket => + socket.write(Chunk.array("fs2.rocks".getBytes)) *> + IO.sleep(1.second) *> + socket.endOfOutput *> socket.endOfInput + } + .compile + .drain + } + } + } } } From 500e5453230383ac95e2aa183301eae7e86af278 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Tue, 5 Sep 2023 14:00:48 +0000 Subject: [PATCH 72/77] Use `atUnsafe` --- .../src/main/scala/fs2/io/internal/SocketHelpers.scala | 2 +- io/native/src/main/scala/fs2/io/ioplatform.scala | 4 ++-- io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala | 6 ++++-- .../scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala | 2 +- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala index 4056c70947..a1d8535168 100644 --- a/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala +++ b/io/native/src/main/scala/fs2/io/internal/SocketHelpers.scala @@ -293,7 +293,7 @@ private[io] object SocketHelpers { val addrBytes = addr.sin6_addr.at1.asInstanceOf[Ptr[Byte]] val host = Ipv6Address.fromBytes { val addr = new Array[Byte](16) - memcpy(addr.at(0), addrBytes, 16.toULong) + memcpy(addr.atUnsafe(0), addrBytes, 16.toULong) addr }.get SocketAddress(host, port) diff --git a/io/native/src/main/scala/fs2/io/ioplatform.scala b/io/native/src/main/scala/fs2/io/ioplatform.scala index 93b72f1fd0..ffa0006fc4 100644 --- a/io/native/src/main/scala/fs2/io/ioplatform.scala +++ b/io/native/src/main/scala/fs2/io/ioplatform.scala @@ -73,7 +73,7 @@ private[fs2] trait ioplatform extends iojvmnative { .pollReadRec(()) { _ => IO { val buf = new Array[Byte](bufSize) - val readed = guard(read(STDIN_FILENO, buf.at(0), bufSize.toULong)) + val readed = guard(read(STDIN_FILENO, buf.atUnsafe(0), bufSize.toULong)) if (readed > 0) Right(Some(Chunk.array(buf, 0, readed))) else if (readed == 0) @@ -120,7 +120,7 @@ private[fs2] trait ioplatform extends iojvmnative { val Chunk.ArraySlice(buf, offset, length) = bytes.toArraySlice def go(pos: Int): IO[Either[Int, Unit]] = - IO(write(fd, buf.at(offset + pos), (length - pos).toULong)).flatMap { wrote => + IO(write(fd, buf.atUnsafe(offset + pos), (length - pos).toULong)).flatMap { wrote => if (wrote >= 0) { val newPos = pos + wrote if (newPos < length) diff --git a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala index 715ded4d8e..1392a8cdaf 100644 --- a/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala +++ b/io/native/src/main/scala/fs2/io/net/FdPollingSocket.scala @@ -98,9 +98,11 @@ private final class FdPollingSocket[F[_]: LiftIO] private ( def go(pos: Int): IO[Either[Int, Unit]] = IO { if (LinktimeInfo.isLinux) - guardSSize(send(fd, buf.at(offset + pos), (length - pos).toULong, MSG_NOSIGNAL)).toInt + guardSSize( + send(fd, buf.atUnsafe(offset + pos), (length - pos).toULong, MSG_NOSIGNAL) + ).toInt else - guard(unistd.write(fd, buf.at(offset + pos), (length - pos).toULong)) + guard(unistd.write(fd, buf.atUnsafe(offset + pos), (length - pos).toULong)) }.flatMap { wrote => if (wrote >= 0) { val newPos = pos + wrote diff --git a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala index a5dbd0858a..98ef205543 100644 --- a/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala +++ b/io/native/src/main/scala/fs2/io/net/unixsocket/FdPollingUnixSockets.scala @@ -136,7 +136,7 @@ private final class FdPollingUnixSockets[F[_]: Files: LiftIO](implicit F: Async[ val addr = stackalloc[sockaddr_un]() addr.sun_family = AF_UNIX.toUShort - memcpy(addr.sun_path.at(0), pathBytes.at(0), pathBytes.length.toULong) + memcpy(addr.sun_path.at(0), pathBytes.atUnsafe(0), pathBytes.length.toULong) f(addr.asInstanceOf[Ptr[sockaddr]]) } From 1e8d405b6cab65e14b132f8ae83f10fd49b00b88 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Tue, 5 Sep 2023 14:15:30 +0000 Subject: [PATCH 73/77] Add nowarn --- io/native/src/main/scala/fs2/io/internal/syssocket.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/io/native/src/main/scala/fs2/io/internal/syssocket.scala b/io/native/src/main/scala/fs2/io/internal/syssocket.scala index 984be7acb0..3bf9266b9d 100644 --- a/io/native/src/main/scala/fs2/io/internal/syssocket.scala +++ b/io/native/src/main/scala/fs2/io/internal/syssocket.scala @@ -21,9 +21,12 @@ package fs2.io.internal +import org.typelevel.scalaccompat.annotation._ + import scala.scalanative.posix.sys.socket._ import scala.scalanative.unsafe._ +@nowarn212("cat=unused") @extern private[io] object syssocket { // only in Linux and FreeBSD, but not macOS From 4b5f50b826d0a5ae929636a8b167e55d35fcbbac Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Fri, 15 Sep 2023 17:34:22 +0000 Subject: [PATCH 74/77] Bump base version --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index a86d52f87e..2558789c9c 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import com.typesafe.tools.mima.core._ Global / onChangedBuildSource := ReloadOnSourceChanges -ThisBuild / tlBaseVersion := "3.9" +ThisBuild / tlBaseVersion := "3.10" ThisBuild / organization := "co.fs2" ThisBuild / organizationName := "Functional Streams for Scala" From 365636d3bc1e67c7b250b2de531a640c42b80748 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Mon, 8 Jan 2024 01:55:06 +0000 Subject: [PATCH 75/77] Bump CE --- build.sbt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index af5e8318f5..f5cb4d57b2 100644 --- a/build.sbt +++ b/build.sbt @@ -271,9 +271,9 @@ lazy val core = crossProject(JVMPlatform, JSPlatform, NativePlatform) libraryDependencies ++= Seq( "org.scodec" %%% "scodec-bits" % "1.1.38", "org.typelevel" %%% "cats-core" % "2.10.0", - "org.typelevel" %%% "cats-effect" % "3.6-e9aeb8c", - "org.typelevel" %%% "cats-effect-laws" % "3.6-e9aeb8c" % Test, - "org.typelevel" %%% "cats-effect-testkit" % "3.6-e9aeb8c" % Test, + "org.typelevel" %%% "cats-effect" % "3.6-c7ca678", + "org.typelevel" %%% "cats-effect-laws" % "3.6-c7ca678" % Test, + "org.typelevel" %%% "cats-effect-testkit" % "3.6-c7ca678" % Test, "org.typelevel" %%% "cats-laws" % "2.10.0" % Test, "org.typelevel" %%% "discipline-munit" % "2.0.0-M3" % Test, "org.typelevel" %%% "munit-cats-effect" % "2.0.0-M4" % Test, From 518efae7905ab2eec3a649b5cb23d8aece8a05e7 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Fri, 27 Dec 2024 23:48:32 +0000 Subject: [PATCH 76/77] No more Fs2IoSuite --- io/jvm/src/test/scala/fs2/io/file/WalkBenchmark.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/io/jvm/src/test/scala/fs2/io/file/WalkBenchmark.scala b/io/jvm/src/test/scala/fs2/io/file/WalkBenchmark.scala index 28b7132636..155231c7c6 100644 --- a/io/jvm/src/test/scala/fs2/io/file/WalkBenchmark.scala +++ b/io/jvm/src/test/scala/fs2/io/file/WalkBenchmark.scala @@ -27,7 +27,7 @@ import cats.effect.IO import java.io.File import scala.concurrent.duration.* -class WalkBenchmark extends Fs2IoSuite { +class WalkBenchmark extends Fs2Suite { override def munitIOTimeout = 5.minutes From 1a076cd891ee18d992ad90df8a725b00c86fd1b8 Mon Sep 17 00:00:00 2001 From: Arman Bilge Date: Fri, 27 Dec 2024 16:17:42 -0800 Subject: [PATCH 77/77] Delete .cirrus/Dockerfile --- .cirrus/Dockerfile | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 .cirrus/Dockerfile diff --git a/.cirrus/Dockerfile b/.cirrus/Dockerfile deleted file mode 100644 index 0ade333dd7..0000000000 --- a/.cirrus/Dockerfile +++ /dev/null @@ -1,6 +0,0 @@ -FROM sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.5_8_1.9.0_3.3.0 - -RUN apt-get update && apt-get install -y build-essential clang cmake libssl-dev nodejs zlib1g-dev -RUN git clone https://github.com/aws/s2n-tls.git && cd s2n-tls && cmake -S . -B build && cmake --build build && cmake --install build - -ENV S2N_DONT_MLOCK=1