Add SIGALRM to kill stuck workers

Author: LauJosefsen

composer.json

@@ -28,7 +28,8 @@
         "illuminate/encryption": "^7.0|^8.0|^9.0|^10.0|^11.0|^12.0",
         "guzzlehttp/guzzle": "^6.5.5|^7.2",
         "jfcherng/php-diff": "^6.13",
-        "doctrine/dbal": "^3.3|^4.0"
+        "doctrine/dbal": "^3.3|^4.0",
+        "ext-posix": "*"
     },
     "require-dev": {
         "cego/php-cs-fixer": "^2.0",

src/RequestInsurance/RequestInsuranceWorker.php

@@ -69,6 +69,8 @@ public function run(bool $runOnlyOnce = false): void
 
         do {
             try {
+                $this->registerTimeoutHandler();
+
                 if (Config::get('request-insurance.useDbReconnect')) {
                     DB::reconnect();
                 }
@@ -84,16 +86,18 @@ public function run(bool $runOnlyOnce = false): void
 
                 usleep($waitTime);
             } catch (Throwable $throwable) {
+                $this->resetTimeoutHandler(); // We need to reset here before logging the error and sleeping, otherwise the timeout handler might trigger while we are sleeping/logging, which is not desirable.
+
                 Log::error($throwable);
 
                 if ($runOnlyOnce) {
                     throw $throwable;
                 }
 
-                sleep(5); // Sleep to avoid spamming the log
+                usleep(100_000); // Sleep to avoid spamming the log
+            } finally {
+                $this->resetTimeoutHandler();
             }
-
-            pcntl_signal_dispatch();
         } while ( ! $runOnlyOnce && ! $this->shutdownSignalReceived);
 
         Log::info(sprintf('RequestInsurance Worker (#%s) has gracefully stopped', $this->runningHash));
@@ -107,6 +111,7 @@ public function run(bool $runOnlyOnce = false): void
      */
     protected function setupShutdownSignalHandler(): void
     {
+        pcntl_async_signals(true);
         pcntl_signal(SIGQUIT, [$this, 'sig_handler']); // Code 3
         pcntl_signal(SIGTERM, [$this, 'sig_handler']); // Code 15
     }
@@ -327,4 +332,22 @@ public function getIdsOfReadyRequests()
 
         return $builder->pluck('id');
     }
+
+    private function registerTimeoutHandler()
+    {
+        pcntl_signal(SIGALRM, function () {
+            Log::debug('Timeout handler was triggered indicating stuck worker, exiting...');
+
+            if (($pid = getmypid()) === false) {
+                posix_kill($pid, SIGKILL);
+            }
+            exit(1);
+        });
+        pcntl_alarm(Config::integer('request-insurance.maximumSecondsPerWorkerCycle', 60));
+    }
+
+    private function resetTimeoutHandler(): void
+    {
+        pcntl_alarm(0);
+    }
 }