Socket automatic reconnections in dedicated thread (nasa#4136)

* Initial addition of dedicated reconnect task in SocketComponentHelper

* Updates to fix UT errors, address FIXMEs, update comments

* Additional FIXMEs and doc updates

* Spelling fix

* Format

* Format and some delay updates

* Address second request FIXMEs (remove)

* Unconditional stop & join in UDP tester cleanup

* Address FIXMEs and update Task to init singleton in a thread and address sanitizer safe manner

* Spelling

* Address PR review comments

Author: m-aleem

Drv/Ip/SocketComponentHelper.cpp

@@ -24,7 +24,22 @@ SocketComponentHelper::~SocketComponentHelper() {}
 void SocketComponentHelper::start(const Fw::StringBase& name,
                                   const FwTaskPriorityType priority,
                                   const Os::Task::ParamType stack,
-                                  const Os::Task::ParamType cpuAffinity) {
+                                  const Os::Task::ParamType cpuAffinity,
+                                  const FwTaskPriorityType priorityReconnect,
+                                  const Os::Task::ParamType stackReconnect,
+                                  const Os::Task::ParamType cpuAffinityReconnect) {
+    // Reconnect Thread
+    FW_ASSERT(m_reconnectTask.getState() ==
+              Os::Task::State::NOT_STARTED);  // It is a coding error to start this task multiple times
+    this->m_reconnectStop = false;
+    Fw::String reconnectName;
+    reconnectName.format("%s_reconnect", name.toChar());
+    Os::Task::Arguments reconnectArguments(reconnectName, SocketComponentHelper::reconnectTask, this, priorityReconnect,
+                                           stackReconnect, cpuAffinityReconnect);
+    Os::Task::Status reconnectStat = m_reconnectTask.start(reconnectArguments);
+    FW_ASSERT(Os::Task::OP_OK == reconnectStat, static_cast<FwAssertArgType>(reconnectStat));
+
+    // Read Thread
     FW_ASSERT(m_task.getState() ==
               Os::Task::State::NOT_STARTED);  // It is a coding error to start this task multiple times
     this->m_stop = false;
@@ -80,18 +95,19 @@ void SocketComponentHelper::setAutomaticOpen(bool auto_open) {
     this->m_reopen = auto_open;
 }
 
+bool SocketComponentHelper::getAutomaticOpen() {
+    Os::ScopeLock scopedLock(this->m_lock);
+    return this->m_reopen;
+}
+
 SocketIpStatus SocketComponentHelper::reopen() {
     SocketIpStatus status = SOCK_SUCCESS;
     if (not this->isOpened()) {
         // Check for auto-open before attempting to reopen
-        bool reopen = false;
-        {
-            Os::ScopeLock scopedLock(this->m_lock);
-            reopen = this->m_reopen;
-        }
-        // Open a network connection if it has not already been open
+        bool reopen = this->getAutomaticOpen();
         if (not reopen) {
             status = SOCK_AUTO_CONNECT_DISABLED;
+            // Open a network connection if it has not already been open
         } else {
             status = this->open();
             if (status == SocketIpStatus::SOCK_ANOTHER_THREAD_OPENING) {
@@ -109,15 +125,16 @@ SocketIpStatus SocketComponentHelper::send(const U8* const data, const FwSizeTyp
     this->m_lock.unlock();
     // Prevent transmission before connection, or after a disconnect
     if (descriptor.fd == -1) {
-        status = this->reopen();
-        // if reopen wasn't successful, pass the that up to the caller
-        if (status != SOCK_SUCCESS) {
-            return status;
+        this->requestReconnect();
+        SocketIpStatus reconnectStat = this->waitForReconnect();
+        if (reconnectStat == SOCK_SUCCESS) {
+            // Refresh local copy after reopen
+            this->m_lock.lock();
+            descriptor = this->m_descriptor;
+            this->m_lock.unlock();
+        } else {
+            return reconnectStat;
         }
-        // Refresh local copy after reopen
-        this->m_lock.lock();
-        descriptor = this->m_descriptor;
-        this->m_lock.unlock();
     }
     status = this->getSocketHandler().send(descriptor, data, size);
     if (status == SOCK_DISCONNECTED) {
@@ -138,8 +155,15 @@ void SocketComponentHelper::close() {
     this->m_open = OpenState::NOT_OPEN;
 }
 
+/* Read Thread */
+
 Os::Task::Status SocketComponentHelper::join() {
-    return m_task.join();
+    Os::Task::Status stat = m_task.join();
+    Os::Task::Status reconnectStat = this->joinReconnect();
+    if (stat == Os::Task::Status::OP_OK) {
+        return reconnectStat;
+    }
+    return stat;
 }
 
 void SocketComponentHelper::stop() {
@@ -148,6 +172,7 @@ void SocketComponentHelper::stop() {
         Os::ScopeLock scopeLock(m_lock);
         this->m_stop = true;
     }
+    this->stopReconnect();
     this->shutdown();  // Break out of any receives and fully shutdown
 }
 
@@ -178,17 +203,12 @@ void SocketComponentHelper::readLoop() {
     do {
         // Prevent transmission before connection, or after a disconnect
         if ((not this->isOpened()) and this->running()) {
-            status = this->reopen();
+            this->requestReconnect();
+            status = this->waitForReconnect();
             // When reopen is disabled, just break as this is a exit condition for the loop
             if (status == SOCK_AUTO_CONNECT_DISABLED) {
                 break;
             }
-            // If the reconnection failed in any other way, warn, wait, and retry
-            else if (status != SOCK_SUCCESS) {
-                Fw::Logger::log("[WARNING] Failed to open port with status %d and errno %d\n", status, errno);
-                (void)Os::Task::delay(SOCKET_RETRY_INTERVAL);
-                continue;
-            }
         }
         // If the network connection is open, read from it
         if (this->isOpened() and this->running()) {
@@ -221,4 +241,123 @@ void SocketComponentHelper::readTask(void* pointer) {
     SocketComponentHelper* self = reinterpret_cast<SocketComponentHelper*>(pointer);
     self->readLoop();
 }
+
+/* Reconnect Thread */
+
+Os::Task::Status SocketComponentHelper::joinReconnect() {
+    return m_reconnectTask.join();
+}
+
+void SocketComponentHelper::stopReconnect() {
+    Os::ScopeLock scopeLock(this->m_reconnectLock);
+    this->m_reconnectState = ReconnectState::NOT_RECONNECTING;
+    this->m_reconnectStop = true;
+}
+
+bool SocketComponentHelper::runningReconnect() {
+    Os::ScopeLock scopedLock(this->m_reconnectLock);
+    bool running = not this->m_reconnectStop;
+    return running;
+}
+
+void SocketComponentHelper::reconnectLoop() {
+    SocketIpStatus status = SOCK_SUCCESS;
+    while (this->runningReconnect()) {
+        // Check if we need to reconnect
+        bool reconnect = false;
+        {
+            Os::ScopeLock scopedLock(this->m_reconnectLock);
+            if (this->m_reconnectState == ReconnectState::REQUEST_RECONNECT) {
+                this->m_reconnectState = ReconnectState::RECONNECT_IN_PROGRESS;
+                reconnect = true;
+
+            }
+            // If we were already in or are now in RECONNECT_IN_PROGRESS we
+            // need to try to reconnect, again
+            else if (this->m_reconnectState == ReconnectState::RECONNECT_IN_PROGRESS) {
+                reconnect = true;
+            }
+        }
+
+        if (reconnect) {
+            status = this->reopen();
+
+            // Reopen Case 1: Auto Connect is disabled, so no longer
+            // try to reconnect
+            if (status == SOCK_AUTO_CONNECT_DISABLED) {
+                Os::ScopeLock scopedLock(this->m_reconnectLock);
+                this->m_reconnectState = ReconnectState::NOT_RECONNECTING;
+            }
+            // Reopen Case 2: Success, so no longer
+            // try to reconnect
+            else if (status == SOCK_SUCCESS) {
+                Os::ScopeLock scopedLock(this->m_reconnectLock);
+                this->m_reconnectState = ReconnectState::NOT_RECONNECTING;
+            }
+            // Reopen Case 3: Keep trying to reconnect - NO reconnect
+            // state change
+            else {
+                Fw::Logger::log("[WARNING] Failed to open port with status %d and errno %d\n", status, errno);
+                (void)Os::Task::delay(SOCKET_RETRY_INTERVAL);
+            }
+        } else {
+            // After a brief delay, we will loop again
+            (void)Os::Task::delay(this->m_reconnectCheckInterval);
+        }
+    }
+}
+
+void SocketComponentHelper::reconnectTask(void* pointer) {
+    FW_ASSERT(pointer);
+    SocketComponentHelper* self = reinterpret_cast<SocketComponentHelper*>(pointer);
+    self->reconnectLoop();
+}
+
+void SocketComponentHelper::requestReconnect() {
+    Os::ScopeLock scopedLock(this->m_reconnectLock);
+    if (m_reconnectState == ReconnectState::NOT_RECONNECTING) {
+        m_reconnectState = ReconnectState::REQUEST_RECONNECT;
+    }
+    return;
+}
+
+SocketIpStatus SocketComponentHelper::waitForReconnect(Fw::TimeInterval timeout) {
+    // Do not attempt to reconnect if auto reconnect config flag is disabled
+    if (!this->getAutomaticOpen()) {
+        return SOCK_AUTO_CONNECT_DISABLED;
+    }
+
+    Fw::TimeInterval elapsed = Fw::TimeInterval(0, 0);
+
+    while (elapsed < timeout) {
+        // If the reconnect thread is NOT reconnecting, we are done waiting
+        // If we are no longer running the reconnect thread, we are done waiting
+        {
+            Os::ScopeLock scopedLock(this->m_reconnectLock);
+            if (this->m_reconnectState == ReconnectState::NOT_RECONNECTING) {
+                break;
+            }
+            if (this->m_reconnectStop) {
+                break;
+            }
+        }
+        // Wait a bit before checking again
+        (void)Os::Task::delay(this->m_reconnectWaitInterval);
+        elapsed.add(this->m_reconnectWaitInterval.getSeconds(), this->m_reconnectWaitInterval.getUSeconds());
+    }
+
+    // If we have completed our loop, check if we are connected or if
+    // auto connect was disabled during our wait
+    if (this->isOpened()) {
+        return SOCK_SUCCESS;
+    }
+
+    // Check one more time if auto reconnect config flag got disabled
+    if (!this->getAutomaticOpen()) {
+        return SOCK_AUTO_CONNECT_DISABLED;
+    }
+
+    return SOCK_DISCONNECTED;  // Indicates failure of this attempt, another reopen needed
+}
+
 }  // namespace Drv