Make database heartbeat more robust

Should fix #21713

Author: mvdbeek

lib/galaxy/model/database_heartbeat.py

@@ -11,6 +11,7 @@
 )
 
 from galaxy.model import WorkerProcess
+from galaxy.model.base import check_database_connection
 from galaxy.model.orm.now import now
 
 log = logging.getLogger(__name__)
@@ -106,7 +107,11 @@ def update_watcher_designation(self):
     def send_database_heartbeat(self):
         if self.active:
             while not self.exit.is_set():
-                self.update_watcher_designation()
+                check_database_connection(self.sa_session)
+                try:
+                    self.update_watcher_designation()
+                except Exception:
+                    log.exception("Error sending database heartbeat for server '%s'", self.server_name)
                 self.exit.wait(self.heartbeat_interval)
 
     def _delete_worker_process(self):