Hold a KeepAlive to the main evb

Summary:
There's a folly bug that can cause an event posted to the main evb to get stuck
unless a KeepAlive is held.  When KeepAlive's are held, the internal notification queue is
held as a non-internal event.  In our case, the main evb was left holding the signal handler,
and the event to unregister the handler was never processed.

Holding a KeepAlive works around the folly issue.  This also consolidates shutdown to a single
event posted to the evb, and normalizes some other shutdown paths.

Reviewed By: sharmafb

Differential Revision: D116505024

fbshipit-source-id: d0882a039603a9b9faf740a8214ac5a3d1ef82e8

Author: afrind

proxygen/lib/http/coro/server/HTTPServer.cpp

@@ -140,6 +140,7 @@ void HTTPServer::start(
     }
     return;
   }
+  loopKeepAlive_ = folly::getKeepAliveToken(eventBase_);
   run(std::move(onSuccess));
   // Blocks until all IO Threads have terminated and joined
   threadPool.stop();
@@ -355,8 +356,7 @@ void HTTPServer::drain() {
   XLOG(DBG4) << __func__;
   if (state_ == State::RUNNING) {
     state_ = State::DRAINING;
-    eventBase_.runImmediatelyOrRunInEventBaseThread(
-        [this] { globalDrainImpl(); });
+    eventBase_.runImmediatelyOrRunInEventBaseThread([this] { teardownImpl(); });
     for (auto& it : acceptors_) {
       for (auto& acceptor : it.second) {
         if (auto evb = acceptor.getEventBaseKeepalive()) {
@@ -366,11 +366,16 @@ void HTTPServer::drain() {
           // drain/forceStop
       }
     }
-    eventBase_.runImmediatelyOrRunInEventBaseThread(
-        [this] { unregisterSignalHandlers(); });
   }
 }
 
+void HTTPServer::teardownImpl() {
+  XLOG(DBG4) << __func__;
+  globalDrainImpl();
+  unregisterSignalHandlers();
+  loopKeepAlive_.reset();
+}
+
 void HTTPServer::globalDrainImpl() {
   XLOG(DBG4) << __func__;
   for (const auto& serverSocket : serverSockets_) {
@@ -402,15 +407,7 @@ void HTTPServer::forceStop() {
   auto state = state_.load();
   if (state == State::RUNNING) {
     state_ = state = State::DRAINING;
-    folly::ExecutorKeepAlive keepAlive(&eventBase_);
-    eventBase_.runImmediatelyOrRunInEventBaseThread([this] {
-      for (const auto& serverSocket : serverSockets_) {
-        XCHECK(serverSocket);
-        serverSocket->stopAccepting();
-      }
-    });
-    eventBase_.runImmediatelyOrRunInEventBaseThread(
-        [this] { unregisterSignalHandlers(); });
+    eventBase_.runImmediatelyOrRunInEventBaseThread([this] { teardownImpl(); });
   }
   if (state == State::DRAINING) {
     if (quicServer_) {

proxygen/lib/http/coro/server/HTTPServer.h

@@ -268,6 +268,7 @@ class HTTPServer : public quic::QuicHandshakeSocketHolder::Callback {
     XLOG(ERR) << "Failed to accept QUIC connection: " << error.message;
   }
   void run(std::function<void()> onSuccess);
+  void teardownImpl();
   void globalDrainImpl();
   void unregisterSignalHandlers();
   void drainImpl(HTTPCoroAcceptor& acceptor);
@@ -276,6 +277,7 @@ class HTTPServer : public quic::QuicHandshakeSocketHolder::Callback {
   std::shared_ptr<HTTPHandler> handler_;
   folly::F14FastSet<Observer*> observers_;
   folly::EventBase eventBase_;
+  folly::Executor::KeepAlive<folly::EventBase> loopKeepAlive_;
   std::vector<folly::AsyncServerSocket::UniquePtr> serverSockets_;
   bool setReusePortSocketOption_{false};
   folly::F14NodeMap<folly::EventBase*, std::list<HTTPCoroAcceptor>> acceptors_;