[BACKEND] Enable throttle protection by default (#1257)

To protect new users, all prebuilt preset (excluding Soundcloud and Bandcamp) will include throttle protection by default.


> How do I disable?

Set the override variable `enable_throttle_protection: False`.
This can be done on a per-subscription basis and/or in the top __preset__ section to apply to all presets:

```
__preset__:
  overrides:
    enable_throttle_protection: False
```

> What if I already use throttle protection?

Your settings will override whatever values are set in the defaults.

Closes:
- #1121
- #1186

Author: jmbannon

src/ytdl_sub/plugins/throttle_protection.py

@@ -185,6 +185,13 @@ def subscription_download_probability(self) -> Optional[ProbabilityValidator]:
 class ThrottleProtectionPlugin(Plugin[ThrottleProtectionOptions]):
     plugin_options_type = ThrottleProtectionOptions
 
+    @classmethod
+    def perform_sleep(cls, sleep_time: float) -> None:
+        """
+        Wrapper to be able to mock
+        """
+        time.sleep(sleep_time)
+
     def __init__(
         self,
         options: ThrottleProtectionOptions,
@@ -239,7 +246,7 @@ def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
             self._subscription_max_downloads is not None
             and self._subscription_download_counter == 0
         ):
-            logger.debug(
+            logger.info(
                 "Setting subscription max downloads to %d", self._subscription_max_downloads
             )
 
@@ -248,8 +255,8 @@ def post_process_entry(self, entry: Entry) -> Optional[FileMetadata]:
 
         if self.plugin_options.sleep_per_download_s:
             sleep_time = self.plugin_options.sleep_per_download_s.randomized_float()
-            logger.debug("Sleeping between downloads for %0.2f seconds", sleep_time)
-            time.sleep(sleep_time)
+            logger.info("Sleeping between downloads for %0.2f seconds", sleep_time)
+            self.perform_sleep(sleep_time)
 
         return None
 
@@ -265,5 +272,5 @@ def post_process_subscription(self):
 
         if self.plugin_options.sleep_per_subscription_s:
             sleep_time = self.plugin_options.sleep_per_subscription_s.randomized_float()
-            logger.debug("Sleeping between subscriptions for %0.2f seconds", sleep_time)
-            time.sleep(sleep_time)
+            logger.info("Sleeping between subscriptions for %0.2f seconds", sleep_time)
+            self.perform_sleep(sleep_time)

src/ytdl_sub/prebuilt_presets/helpers/throttle_protection.yaml

@@ -0,0 +1,26 @@
+presets:
+
+  _throttle_protection:
+    throttle_protection:
+      enable: >-
+        {
+          %print(
+            %if(
+              enable_throttle_protection,
+              "Throttle protection is enabled. Disable using the override variable `enable_throttle_protection: False`",
+              "Throttle protection is disabled. Use at your own risk!"
+            ),
+            enable_throttle_protection
+          )
+        }
+      sleep_per_request_s:
+        min: 3.5
+        max: 3.5
+      sleep_per_download_s:
+        min: 13.8
+        max: 28.4
+      sleep_per_subscription_s:
+        min: 16.3
+        max: 26.1
+    overrides:
+      enable_throttle_protection: True

src/ytdl_sub/prebuilt_presets/music/albums_from_chapters.yaml

@@ -22,4 +22,5 @@ presets:
 
   "YouTube Full Albums":
     preset:
-      - "_albums_from_chapters"
+      - "_albums_from_chapters"
+      - "_throttle_protection"

src/ytdl_sub/prebuilt_presets/music/albums_from_playlists.yaml

@@ -18,6 +18,7 @@ presets:
   "YouTube Releases":
     preset:
       - "_albums_from_playlists"
+      - "_throttle_protection"
 
   "Bandcamp":
     preset:

src/ytdl_sub/prebuilt_presets/music/singles.yaml

@@ -2,6 +2,7 @@ presets:
 
 
   _music_base:
+
     output_options:
       output_directory: "{music_directory}"
       file_name: "{track_full_path}"

…ebuilt_presets/music/other_websites.yaml …b/prebuilt_presets/music/soundcloud.yamlsrc/ytdl_sub/prebuilt_presets/music/other_websites.yaml renamed to src/ytdl_sub/prebuilt_presets/music/soundcloud.yaml src/ytdl_sub/prebuilt_presets/music/other_websites.yaml renamed to src/ytdl_sub/prebuilt_presets/music/soundcloud.yaml

@@ -2,6 +2,9 @@ presets:
   _music_video_base:
 
     preset:
+      # ytdl-sub includes throttle prootection by default or all prebuilt presets.
+      # Can be disabled with override variable `enable_throttle_protection: False`
+      - "_throttle_protection"
       - "_url_categorized"
 
     output_options:

src/ytdl_sub/prebuilt_presets/music_videos/music_video_base.yaml

@@ -3,6 +3,11 @@ presets:
 ####################################################################################################
 
   _episode_base:
+    preset:
+      # ytdl-sub includes throttle prootection by default or all prebuilt presets.
+      # Can be disabled with override variable `enable_throttle_protection: False`
+      - "_throttle_protection"
+
     output_options:
       output_directory: "{tv_show_directory}/{tv_show_name_sanitized}"
       file_name: "{episode_file_path}.{ext}"

src/ytdl_sub/prebuilt_presets/tv_show/episode.yaml

@@ -9,6 +9,9 @@
 
 
 def _log(message: AnyArgument, level: Optional[Integer]) -> None:
+    if not str(message):
+        return
+
     if level is None:
         logger.info(str(message))
         return

src/ytdl_sub/script/functions/print_functions.py

@@ -15,6 +15,7 @@
 from ytdl_sub.downloaders.ytdlp import YTDLP
 from ytdl_sub.entries.script.variable_definitions import VARIABLES
 from ytdl_sub.entries.script.variable_definitions import VariableDefinitions
+from ytdl_sub.plugins.throttle_protection import ThrottleProtectionPlugin
 
 v: VariableDefinitions = VARIABLES
 
@@ -264,6 +265,9 @@ def _write_entries_to_working_dir(*args, **kwargs) -> List[Dict]:
             patch.object(
                 MultiUrlDownloader, "_extract_entry_info_with_retry", new=lambda _, entry: entry
             ),
+            # Throttle protection is included in all prebuilt presets. Mock the sleep avoid
+            # actual sleeps
+            patch.object(ThrottleProtectionPlugin, "perform_sleep", new=lambda _1, _2: None),
         ):
             # Stub out metadata. TODO: update this if we do metadata plugins
             yield