Degrade failing instances at startup instead of exiting the whole app

Previously any single instance failing its startup check terminated
decluttarr via wait_and_exit(), taking healthy instances down with it
and causing container crash-loops on slow servers (ManiMatter#317 follow-up).

Now each unit (arr, qBittorrent, SABnzbd) records a readiness state:
- Transient failures (timeout, connection, 5xx, unknown) degrade the
  instance; setup is re-attempted every timer cycle and the instance
  rejoins automatically (including its detect_deletions watchers).
- Definitive config errors (401/403, non-English UI, client version
  too old) degrade the instance with a per-cycle ERROR + tip; they are
  not retried since they cannot heal without user action.
- The app only exits when nothing is configured (unchanged) or when
  every configured unit failed definitively.

Wrong-arr-type and arr-version-too-old keep their existing
log-and-continue behavior. Repeated identical setup failures log a
single-line skip instead of the full error block each cycle.

Also fixes main.py handing the deletion watchers to a throwaway
WatcherManager, which left terminate() stopping an instance that owned
no observers.

Author: jrhager84

main.py

@@ -8,7 +8,7 @@
 from src.job_manager import JobManager
 from src.settings.settings import Settings
 from src.utils.log_setup import logger
-from src.utils.startup import launch_steps
+from src.utils.startup import launch_steps, retry_degraded_instances
 
 settings = Settings()
 job_manager = JobManager(settings)
@@ -53,13 +53,18 @@ async def main():
     await launch_steps(settings)
 
     if settings.jobs.detect_deletions.enabled:
-        await WatcherManager(settings).setup()
+        await watch_manager.setup()
     # Start Cleaning
     while True:
         logger.info("-" * 50)
 
+        # Give degraded instances a chance to rejoin before this cycle's jobs
+        await retry_degraded_instances(settings, watch_manager)
+
         # Refresh qBit Cookies (SABnzbd doesn't need cookie refresh)
         for qbit in settings.download_clients.qbittorrent:
+            if not qbit.ready:
+                continue
             try:
                 await qbit.refresh_cookie()
             except Exception as err:  # noqa: BLE001
@@ -70,6 +75,8 @@ async def main():
 
         # Run script for each instance
         for arr in settings.instances:
+            if not arr.ready:  # skip was already logged by retry_degraded_instances
+                continue
             await job_manager.run_jobs(arr)
             logger.verbose("")
 

src/deletion_handler/deletion_handler.py

@@ -99,35 +99,52 @@ async def setup(self):
         for arr, folder_path in folders_to_watch:
             self.set_watcher(arr, folder_path)
 
+    async def setup_for_arr(self, arr):
+        """Set up deletion watchers for a single arr (e.g. one that rejoined after a degraded startup)."""
+        if self.loop is None:
+            self.loop = asyncio.get_running_loop()
+        for folder_path in await self.get_folders_to_watch_for_arr(arr):
+            self.set_watcher(arr, folder_path)
+
     async def get_folders_to_watch(self):
         """Gets from all arrs the root folders and lists those that are accessible for the arr, and have present for decluttarr."""
         folders_to_watch = []
         logger.verbose("")
         logger.verbose("*** Setting up monitoring for deletions ***")
         for arr in self.settings.instances:
-            if arr.arr_type not in (
-                "sonarr",
-                "radarr",
-            ):  # only working for sonarr / radarr for now
-                continue
-            root_folders = await arr.get_root_folders()
-
-            for folder in root_folders:
-                if folder.get("accessible") and "path" in folder:
-                    path = Path(folder["path"])
-                    if path.exists():
-                        folders_to_watch.append((arr, folder["path"]))
-                    else:
-                        logger.warning(
-                            f"Job 'detect_deletions' on {arr.name} ({arr.base_url}) does not have access to this path and will not monitor it: '{path}'"
-                        )
+            for folder_path in await self.get_folders_to_watch_for_arr(arr):
+                folders_to_watch.append((arr, folder_path))
+
+        return folders_to_watch
+
+    async def get_folders_to_watch_for_arr(self, arr):
+        """Root folder paths of one arr that are accessible for both the arr and decluttarr."""
+        folders_to_watch = []
+        if arr.arr_type not in (
+            "sonarr",
+            "radarr",
+        ):  # only working for sonarr / radarr for now
+            return folders_to_watch
+        if not arr.ready:  # degraded instance; watchers are added when it rejoins
+            return folders_to_watch
+        root_folders = await arr.get_root_folders()
+
+        for folder in root_folders:
+            if folder.get("accessible") and "path" in folder:
+                path = Path(folder["path"])
+                if path.exists():
+                    folders_to_watch.append(folder["path"])
+                else:
+                    logger.warning(
+                        f"Job 'detect_deletions' on {arr.name} ({arr.base_url}) does not have access to this path and will not monitor it: '{path}'"
+                    )
+                    logger.info(
+                        ">>> 💡 Tip: Make sure that the paths in decluttarr and in your arr instance are identical."
+                    )
+                    if self.settings.envs.in_docker:
                         logger.info(
-                            ">>> 💡 Tip: Make sure that the paths in decluttarr and in your arr instance are identical."
+                            ">>> 💡 Tip: Make sure decluttarr and your arr instance have the same mount points"
                         )
-                        if self.settings.envs.in_docker:
-                            logger.info(
-                                ">>> 💡 Tip: Make sure decluttarr and your arr instance have the same mount points"
-                            )
 
         return folders_to_watch
 

src/job_manager.py

@@ -56,6 +56,9 @@ async def run_download_client_jobs(self):
             )
 
             for client in download_clients:
+                if not client.ready:
+                    continue
+
                 # Get jobs for this client
                 download_client_jobs = self._get_download_client_jobs_for_client(
                     client,
@@ -150,6 +153,8 @@ async def _download_clients_connected(self):
 
     async def _check_client_connection_status(self, clients):
         for client in clients:
+            if not client.ready:  # never-set-up client must not veto or be polled
+                continue
             logger.debug(
                 f"job_manager.py/_check_client_connection_status: Checking if {client.name} is connected",
             )

src/settings/_download_clients_qbit.py

@@ -2,12 +2,19 @@
 from requests.cookies import RequestsCookieJar
 
 from src.settings._constants import ApiEndpoints, MinVersions
-from src.utils.common import extract_json_from_response, make_request, wait_and_exit
+from src.utils.common import (
+    extract_json_from_response,
+    is_definitive_setup_error,
+    make_request,
+)
 from src.utils.log_setup import logger
 
 
 class QbitError(Exception):
-    pass
+    def __init__(self, message, tip="", definitive=False):
+        super().__init__(message)
+        self.tip = tip
+        self.definitive = definitive
 
 
 class QbitClients(list):
@@ -39,6 +46,10 @@ class QbitClient:
     cookie: dict[str, str] = None
     version: str = None
     bandwidth_usage: int = 0
+    ready: bool = False
+    failure_kind: str = None  # None | "transient" | "definitive"
+    last_error: str = None
+    setup_tip: str = ""
 
     def __init__(
         self,
@@ -149,7 +160,7 @@ async def validate_version(self):
                 f"Please update qBittorrent to at least version {min_version}. Current version: {self.version}",
             )
             error = f"qBittorrent version {self.version} is too old. Please update."
-            raise QbitError(error)
+            raise QbitError(error, definitive=True)
         if version.parse(self.version) < version.parse("5.0.0"):
             logger.info(
                 "[Tip!] Consider upgrading to qBittorrent v5.0.0 or newer to reduce network overhead.",
@@ -237,8 +248,9 @@ async def check_qbit_reachability(self):
 
         except Exception as e:  # noqa: BLE001
             tip = "💡 Tip: Did you specify the URL (and username/password if required) correctly?"
-            logger.error(f"-- | qBittorrent\n❗️ {e}\n{tip}\n")
-            wait_and_exit()
+            if str(e) != self.last_error:  # Only report new failure modes in full
+                logger.error(f"-- | qBittorrent\n❗️ {e}\n{tip}\n")
+            raise QbitError(e, tip=tip) from e
 
     async def check_connected(self):
         """Check if the qBittorrent is connected to internet."""
@@ -260,26 +272,38 @@ async def check_connected(self):
         return True
 
     async def setup(self):
-        """Perform the qBittorrent setup by calling relevant managers."""
-        # Check reachabilty
-        await self.check_qbit_reachability()
+        """Perform the qBittorrent setup; degrade instead of exiting on failure."""
+        try:
+            await self.check_qbit_reachability()
 
-        # Refresh the qBittorrent cookie first
-        await self.refresh_cookie()
+            # Refresh the qBittorrent cookie first
+            await self.refresh_cookie()
 
-        try:
             # Fetch version and validate it
             await self.fetch_version()
             await self.validate_version()
-            logger.info(f"OK | qBittorrent ({self.base_url})")
-        except QbitError as e:
-            logger.error(f"qBittorrent version check failed: {e}")
-            wait_and_exit()  # Exit if version check fails
 
-        # Continue with other setup tasks regardless of version check result
-        await self.create_required_tags()
-        await self.set_unwanted_folder()
-        await self.warn_no_bandwidth_limit_set()
+            await self.create_required_tags()
+            await self.set_unwanted_folder()
+            await self.warn_no_bandwidth_limit_set()
+
+            logger.info(f"OK | qBittorrent ({self.base_url})")
+            self.ready = True
+            self.failure_kind = None
+            self.last_error = None
+            self.setup_tip = ""
+        except Exception as e:  # noqa: BLE001
+            if not isinstance(e, QbitError) and str(e) != self.last_error:
+                logger.error(
+                    f"Unhandled error during qBittorrent setup: {e}", exc_info=True
+                )
+            self.ready = False
+            self.failure_kind = (
+                "definitive" if is_definitive_setup_error(e) else "transient"
+            )
+            self.last_error = str(e)
+            self.setup_tip = getattr(e, "tip", "")
+        return self.ready
 
     async def get_protected_and_private(self):
         """Fetch torrents from qBittorrent and checks for protected and private status."""
@@ -395,15 +419,14 @@ async def get_qbit_items(self, hashes: list[str] | str | None = None) -> list[di
     async def get_torrent_properties(self, qbit_hash):
         params = {"hash": qbit_hash.lower()}
         response = await make_request(
-                        "get",
-                        self.api_url + "/torrents/properties",
-                        self.settings,
-                        params=params,
-                        cookies=self.cookie,
-                    )
+            "get",
+            self.api_url + "/torrents/properties",
+            self.settings,
+            params=params,
+            cookies=self.cookie,
+        )
         return response.json()
 
-
     async def get_torrent_files(self, download_id):
         # this may not work if the wrong qbit
         logger.debug("_download_clients_qBit/get_torrent_files: Getting torrent files")

src/settings/_download_clients_sabnzbd.py

@@ -1,12 +1,15 @@
 from packaging import version
 
 from src.settings._constants import MinVersions
-from src.utils.common import make_request, wait_and_exit
+from src.utils.common import is_definitive_setup_error, make_request
 from src.utils.log_setup import logger
 
 
 class SabnzbdError(Exception):
-    pass
+    def __init__(self, message, tip="", definitive=False):
+        super().__init__(message)
+        self.tip = tip
+        self.definitive = definitive
 
 
 class SabnzbdClients(list):
@@ -36,6 +39,10 @@ class SabnzbdClient:
     """Represents a single SABnzbd client."""
 
     version: str = None
+    ready: bool = False
+    failure_kind: str = None  # None | "transient" | "definitive"
+    last_error: str = None
+    setup_tip: str = ""
 
     def __init__(
         self,
@@ -96,7 +103,7 @@ async def validate_version(self):
                 f"Please update SABnzbd to at least version {min_version}. Current version: {self.version}",
             )
             error = f"SABnzbd version {self.version} is too old. Please update."
-            raise SabnzbdError(error)
+            raise SabnzbdError(error, definitive=True)
 
     async def check_sabnzbd_reachability(self):
         """Check if the SABnzbd URL is reachable."""
@@ -116,8 +123,9 @@ async def check_sabnzbd_reachability(self):
 
         except Exception as e:  # noqa: BLE001
             tip = "💡 Tip: Did you specify the URL and API key correctly?"
-            logger.error(f"-- | SABnzbd\n❗️ {e}\n{tip}\n")
-            wait_and_exit()
+            if str(e) != self.last_error:  # Only report new failure modes in full
+                logger.error(f"-- | SABnzbd\n❗️ {e}\n{tip}\n")
+            raise SabnzbdError(e, tip=tip) from e
 
     async def check_connected(self):
         """Check if SABnzbd is connected and operational."""
@@ -137,18 +145,31 @@ async def check_connected(self):
         return "status" in status_data
 
     async def setup(self):
-        """Perform the SABnzbd setup by calling relevant managers."""
-        # Check reachability
-        await self.check_sabnzbd_reachability()
-
+        """Perform the SABnzbd setup; degrade instead of exiting on failure."""
         try:
+            await self.check_sabnzbd_reachability()
+
             # Fetch version and validate it
             await self.fetch_version()
             await self.validate_version()
+
             logger.info(f"OK | SABnzbd ({self.base_url})")
-        except SabnzbdError as e:
-            logger.error(f"SABnzbd version check failed: {e}")
-            wait_and_exit()  # Exit if version check fails
+            self.ready = True
+            self.failure_kind = None
+            self.last_error = None
+            self.setup_tip = ""
+        except Exception as e:  # noqa: BLE001
+            if not isinstance(e, SabnzbdError) and str(e) != self.last_error:
+                logger.error(
+                    f"Unhandled error during SABnzbd setup: {e}", exc_info=True
+                )
+            self.ready = False
+            self.failure_kind = (
+                "definitive" if is_definitive_setup_error(e) else "transient"
+            )
+            self.last_error = str(e)
+            self.setup_tip = getattr(e, "tip", "")
+        return self.ready
 
     async def get_queue_items(self):
         """Fetch queue items from SABnzbd."""