Fix detect_deletions gating and harden timeout handling

- Make the request timeout configurable via general.request_timeout /
  REQUEST_TIMEOUT (defaults to the previous hardcoded 15 seconds)
- Catch request errors per job group so a slow or unreachable server
  skips the current run instead of crashing the container (ManiMatter#317)
- Fix detect_deletions truthiness so the deletion watchers only start
  when the job is actually enabled

Author: tomerh2001

README.md

@@ -20,6 +20,7 @@ Looking to **upgrade from V1 to V2**? Look [here](#upgrading-from-v1-to-v2)
     - [LOG_LEVEL](#log_level)
     - [TEST_RUN](#test_run)
     - [TIMER](#timer)
+    - [REQUEST_TIMEOUT](#request_timeout)
     - [SSL_VERIFICATION](#ssl_verification)
     - [IGNORE_DOWNLOAD_CLIENTS](#ignore_download_clients)
     - [PRIVATE_TRACKER_HANDLING / PUBLIC_TRACKER_HANDLING](#private_tracker_handling--public_tracker_handling)
@@ -193,6 +194,7 @@ services:
       LOG_LEVEL: INFO
       TEST_RUN: True
       TIMER: 10
+      # REQUEST_TIMEOUT: 15
       # IGNORED_DOWNLOAD_CLIENTS: >
       #   - emulerr
       # SSL_VERIFICATION: true
@@ -399,6 +401,13 @@ Configures the general behavior of the application (across all features)
 -   Unit: Minutes
 -   Is Mandatory: No (Defaults to 10)
 
+#### REQUEST_TIMEOUT
+
+-   Timeout used for HTTP/API requests to *arr and download clients
+-   Type: Integer or Float
+-   Unit: Seconds
+-   Is Mandatory: No (Defaults to 15)
+
 #### SSL_VERIFICATION
 
 -   Turns SSL certificate verification on or off for all API calls

config/config_example.yaml

@@ -2,6 +2,7 @@ general:
   log_level: INFO
   test_run: true
   timer: 10
+  # request_timeout: 15 # Optional: timeout for all HTTP/API calls in seconds
   # ignored_download_clients: ["emulerr"]
   # ssl_verification: false # Optional: Defaults to true
   # private_tracker_handling: "obsolete_tag" # remove, skip, obsolete_tag. Optional. Default: remove

main.py

@@ -1,79 +1,91 @@
-import asyncio
-import datetime
-import signal
-import sys
-import types
-
-from src.deletion_handler.deletion_handler import WatcherManager
-from src.job_manager import JobManager
-from src.settings.settings import Settings
-from src.utils.log_setup import logger
-from src.utils.startup import launch_steps
-
-settings = Settings()
-job_manager = JobManager(settings)
-watch_manager = WatcherManager(settings)
-
-
-def terminate(
-    sigterm: signal.SIGTERM,  # noqa: ARG001, pylint: disable=unused-argument
-    frame: types.FrameType,  # noqa: ARG001, pylint: disable=unused-argument
-) -> None:
-    """Terminate cleanly. Needed for respecting 'docker stop'.
-
-    Args:
-    ----
-        sigterm (signal.Signal): The termination signal.
-        frame: The execution frame.
-
-    """
-
-    logger.info(
-        f"Termination signal received at {datetime.datetime.now()}."
-    )  # noqa: DTZ005
-    watch_manager.stop()
-    sys.exit(0)
-
-
-async def wait_next_run():
-    # Calculate next run time dynamically (to display)
-    next_run = datetime.datetime.now() + datetime.timedelta(
-        minutes=settings.general.timer
-    )
-    formatted_next_run = next_run.strftime("%Y-%m-%d %H:%M")
-
-    logger.verbose(f"*** Done - Next run at {formatted_next_run} ****")
-
-    # Wait for the next run
-    await asyncio.sleep(settings.general.timer * 60)
-
-
-# Main function
-async def main():
-    await launch_steps(settings)
-
-    if settings.jobs.detect_deletions:
-        await WatcherManager(settings).setup()
-    # Start Cleaning
-    while True:
-        logger.info("-" * 50)
-
-        # Refresh qBit Cookies (SABnzbd doesn't need cookie refresh)
-        for qbit in settings.download_clients.qbittorrent:
-            await qbit.refresh_cookie()
-
-        # Run script for each instance
-        for arr in settings.instances:
-            await job_manager.run_jobs(arr)
-            logger.verbose("")
-
-        # Run download client jobs (these run independently of *arr instances)
-        await job_manager.run_download_client_jobs()
-
-        # Wait for the next run
-        await wait_next_run()
-
-
-if __name__ == "__main__":
-    signal.signal(signal.SIGTERM, terminate)
-    asyncio.run(main())
+import asyncio
+import datetime
+import signal
+import sys
+import types
+
+from src.deletion_handler.deletion_handler import WatcherManager
+from src.job_manager import JobManager
+from src.settings.settings import Settings
+from src.utils.log_setup import logger
+from src.utils.startup import launch_steps
+
+settings = Settings()
+job_manager = JobManager(settings)
+watch_manager = WatcherManager(settings)
+
+
+def terminate(
+    sigterm: signal.SIGTERM,  # noqa: ARG001, pylint: disable=unused-argument
+    frame: types.FrameType,  # noqa: ARG001, pylint: disable=unused-argument
+) -> None:
+    """Terminate cleanly. Needed for respecting 'docker stop'.
+
+    Args:
+    ----
+        sigterm (signal.Signal): The termination signal.
+        frame: The execution frame.
+
+    """
+
+    logger.info(
+        f"Termination signal received at {datetime.datetime.now()}."
+    )  # noqa: DTZ005
+    watch_manager.stop()
+    sys.exit(0)
+
+
+async def wait_next_run():
+    # Calculate next run time dynamically (to display)
+    next_run = datetime.datetime.now() + datetime.timedelta(
+        minutes=settings.general.timer
+    )
+    formatted_next_run = next_run.strftime("%Y-%m-%d %H:%M")
+
+    logger.verbose(f"*** Done - Next run at {formatted_next_run} ****")
+
+    # Wait for the next run
+    await asyncio.sleep(settings.general.timer * 60)
+
+
+# Main function
+async def main():
+    await launch_steps(settings)
+
+    if settings.jobs.detect_deletions.enabled:
+        await WatcherManager(settings).setup()
+    # Start Cleaning
+    while True:
+        logger.info("-" * 50)
+
+        # Refresh qBit Cookies (SABnzbd doesn't need cookie refresh)
+        for qbit in settings.download_clients.qbittorrent:
+            try:
+                await qbit.refresh_cookie()
+            except Exception as err:  # noqa: BLE001
+                logger.error(
+                    f"Error while refreshing cookie for {qbit.name} ({qbit.base_url}): {err}",
+                    exc_info=True,
+                )
+
+        # Run script for each instance
+        for arr in settings.instances:
+            await job_manager.run_jobs(arr)
+            logger.verbose("")
+
+        # Run download client jobs (these run independently of *arr instances)
+        try:
+            await job_manager.run_download_client_jobs()
+        except Exception as err:  # noqa: BLE001
+            logger.error(
+                f"Error while running download-client jobs: {err}",
+                exc_info=True,
+            )
+
+        # Wait for the next run
+        await wait_next_run()
+
+
+if __name__ == "__main__":
+    signal.signal(signal.SIGTERM, terminate)
+    asyncio.run(main())

src/job_manager.py

@@ -1,4 +1,6 @@
 # Cleans the download queue
+import requests
+
 from src.jobs.remove_bad_files import RemoveBadFiles
 from src.jobs.remove_done_seeding import RemoveDoneSeeding
 from src.jobs.remove_failed_downloads import RemoveFailedDownloads
@@ -24,12 +26,25 @@ def __init__(self, settings):
     async def run_jobs(self, arr):
         self.arr = arr
         logger.info(f"*** Running jobs on {self.arr.name} ({self.arr.base_url}) ***")
-        await self.removal_jobs()
-        await self.search_jobs()
+        await self._run_arr_job_group("Removal Jobs", self.removal_jobs)
+        await self._run_arr_job_group("Search Jobs", self.search_jobs)
 
     async def run_download_client_jobs(self):
         """Run jobs that operate on download clients directly."""
-        if not await self._download_clients_connected():
+        try:
+            if not await self._download_clients_connected():
+                return None
+        except requests.exceptions.RequestException as err:
+            logger.error(
+                f"Download client connectivity check failed with request error: {err}",
+                exc_info=True,
+            )
+            return None
+        except Exception as err:  # noqa: BLE001
+            logger.error(
+                f"Download client connectivity check failed: {err}",
+                exc_info=True,
+            )
             return None
 
         items_detected = 0
@@ -56,7 +71,9 @@ async def run_download_client_jobs(self):
 
                 for download_client_job in download_client_jobs:
                     if download_client_job.job.enabled:
-                        items_detected += await download_client_job.run()
+                        items_detected += await self._run_download_client_job(
+                            download_client_job, client
+                        )
 
         return items_detected
 
@@ -143,6 +160,37 @@ async def _check_client_connection_status(self, clients):
                 return False
         return True
 
+    async def _run_arr_job_group(self, label, job_runner):
+        """Run one ARR job group and keep the main loop alive on failures."""
+        try:
+            await job_runner()
+        except requests.exceptions.RequestException as err:
+            logger.error(
+                f"{label}: request error on {self.arr.name} ({self.arr.base_url}): {err}",
+                exc_info=True,
+            )
+        except Exception as err:  # noqa: BLE001
+            logger.error(
+                f"{label}: unexpected error on {self.arr.name} ({self.arr.base_url}): {err}",
+                exc_info=True,
+            )
+
+    async def _run_download_client_job(self, download_client_job, client):
+        """Run one download-client job and keep the loop alive on failures."""
+        try:
+            return await download_client_job.run()
+        except requests.exceptions.RequestException as err:
+            logger.error(
+                f"Download-client job '{download_client_job.job_name}' failed on {client.name} ({client.base_url}) due to request error: {err}",
+                exc_info=True,
+            )
+        except Exception as err:  # noqa: BLE001
+            logger.error(
+                f"Download-client job '{download_client_job.job_name}' failed on {client.name} ({client.base_url}): {err}",
+                exc_info=True,
+            )
+        return 0
+
     def _get_removal_jobs(self):
         """
         Return a list of enabled removal job instances based on the provided settings.

src/settings/_general.py

@@ -11,6 +11,7 @@ class General:
     log_level: str = "INFO"
     test_run: bool = False
     timer: float = 10.0
+    request_timeout: float = 15.0
     ssl_verification: bool = True
     ignored_download_clients: list = []
     private_tracker_handling: str = "remove"
@@ -23,6 +24,9 @@ def __init__(self, config):
         self.log_level = general_config.get("log_level", self.log_level.upper())
         self.test_run = general_config.get("test_run", self.test_run)
         self.timer = general_config.get("timer", self.timer)
+        self.request_timeout = general_config.get(
+            "request_timeout", self.request_timeout
+        )
         self.ssl_verification = general_config.get(
             "ssl_verification", self.ssl_verification
         )

src/settings/_jobs.py

@@ -44,6 +44,10 @@ def _remove_none_attributes(self):
             if getattr(self, attr) is None:
                 delattr(self, attr)
 
+    def __bool__(self):
+        """Allow direct truthiness checks to reflect whether this job is enabled."""
+        return bool(getattr(self, "enabled", False))
+
 
 class JobDefaults:
     """Represents default job settings."""

src/settings/_user_config.py

@@ -13,6 +13,7 @@
         "LOG_LEVEL",
         "TEST_RUN",
         "TIMER",
+        "REQUEST_TIMEOUT",
         "SSL_VERIFICATION",
         "IGNORED_DOWNLOAD_CLIENTS",
         "PRIVATE_TRACKER_HANDLING",

src/utils/common.py

@@ -42,7 +42,7 @@ async def make_request(
     method: str,
     endpoint: str,
     settings,
-    timeout: int = 15,
+    timeout: float | None = None,
     *,
     log_error=True,
     **kwargs,
@@ -51,6 +51,11 @@ async def make_request(
     A utility function to make HTTP requests (GET, POST, DELETE, PUT).
     """
     ignore_test_run = kwargs.pop("ignore_test_run", False)
+    request_timeout = timeout
+    if request_timeout is None:
+        request_timeout = getattr(
+            getattr(settings, "general", None), "request_timeout", 15
+        )
 
     if settings.general.test_run and not ignore_test_run:
         if method.lower() in ("put", "post", "delete"):
@@ -74,7 +79,7 @@ async def make_request(
             endpoint,
             **kwargs,
             verify=settings.general.ssl_verification,
-            timeout=timeout,
+            timeout=request_timeout,
         )
         response.raise_for_status()
         return response