Fix rare silent connection hangs on Linux

The epoll backend silently ignored failures from epoll_ctl when
registering, resubscribing, and unsubscribing event filters. An actor
that thought it was subscribed would wait forever for a notification
that never comes.

Check epoll_ctl returns and send ASIO_ERROR to the owning actor on
subscribe/resubscribe failure. Also fixes the init error check (was
comparing against 0 instead of -1) and adds proper resource cleanup
on partial init failure.

Closes #5119
Closes #5126

Author: SeanTAllen

.release-notes/fix-unchecked-epoll-ctl-returns.md

@@ -0,0 +1,7 @@
+## Fix rare silent connection hangs on Linux
+
+On Linux, if the OS failed to register an I/O event (due to resource exhaustion or an FD race), the failure was silently ignored. Actors waiting for network events that were never registered would hang indefinitely with no error. This could appear as connections that never complete, listeners that stop accepting, or timers that stop firing — with no indication of what went wrong.
+
+The runtime now detects these registration failures and notifies the affected actor, which tears down cleanly — the same as any other I/O failure. Stdlib consumers like `TCPConnection` and `TCPListener` handle this automatically.
+
+Also fixes the ASIO backend init to correctly detect `epoll_create1` and `eventfd` failures (previously checked for 0 instead of -1), and to clean up all resources on partial init failure.

src/libponyrt/asio/epoll.c

@@ -112,8 +112,13 @@ asio_backend_t* ponyint_asio_backend_init()
   b->epfd = epoll_create1(EPOLL_CLOEXEC);
   b->wakeup = eventfd(0, EFD_NONBLOCK);
 
-  if(b->epfd == 0 || b->wakeup == 0)
+  if(b->epfd == -1 || b->wakeup == -1)
   {
+    if(b->epfd != -1)
+      close(b->epfd);
+    if(b->wakeup != -1)
+      close(b->wakeup);
+    ponyint_messageq_destroy(&b->q, true);
     POOL_FREE(asio_backend_t, b);
     return NULL;
   }
@@ -122,7 +127,14 @@ asio_backend_t* ponyint_asio_backend_init()
   ep.data.ptr = b;
   ep.events = EPOLLIN | EPOLLRDHUP | EPOLLET;
 
-  epoll_ctl(b->epfd, EPOLL_CTL_ADD, b->wakeup, &ep);
+  if(epoll_ctl(b->epfd, EPOLL_CTL_ADD, b->wakeup, &ep) == -1)
+  {
+    close(b->epfd);
+    close(b->wakeup);
+    ponyint_messageq_destroy(&b->q, true);
+    POOL_FREE(asio_backend_t, b);
+    return NULL;
+  }
 
 #if !defined(USE_SCHEDULER_SCALING_PTHREADS)
   // Make sure we ignore signals related to scheduler sleeping/waking
@@ -186,7 +198,10 @@ PONY_API void pony_asio_event_resubscribe(asio_event_t* ev)
 
   // only resubscribe if there is something to resubscribe to
   if (something_to_resub)
-    epoll_ctl(b->epfd, EPOLL_CTL_MOD, ev->fd, &ep);
+  {
+    if(epoll_ctl(b->epfd, EPOLL_CTL_MOD, ev->fd, &ep) == -1)
+      pony_asio_event_send(ev, ASIO_ERROR, 0);
+  }
 }
 
 // Kept to maintain backwards compatibility so folks don't
@@ -426,7 +441,8 @@ PONY_API void pony_asio_event_subscribe(asio_event_t* ev)
     ep.events |= EPOLLET;
   }
 
-  epoll_ctl(b->epfd, EPOLL_CTL_ADD, ev->fd, &ep);
+  if(epoll_ctl(b->epfd, EPOLL_CTL_ADD, ev->fd, &ep) == -1)
+    pony_asio_event_send(ev, ASIO_ERROR, 0);
 }
 
 PONY_API void pony_asio_event_setnsec(asio_event_t* ev, uint64_t nsec)
@@ -459,6 +475,8 @@ PONY_API void pony_asio_event_unsubscribe(asio_event_t* ev)
   asio_backend_t* b = ponyint_asio_get_backend();
   pony_assert(b != NULL);
 
+  // Don't send ASIO_ERROR on delete failure — the actor is tearing down,
+  // and ENOENT/EBADF is expected (FD already closed or never registered).
   epoll_ctl(b->epfd, EPOLL_CTL_DEL, ev->fd, NULL);
 
   if(ev->flags & ASIO_TIMER)