[BACKEND][v2] More flexible formatter post_process (#1424)

Makes the underlying formatting process better handle native (non-string) types in a more flexible way. 2nd attempt since the last one got reverted due to breaking changes.

Author: jmbannon

src/ytdl_sub/config/overrides.py

@@ -3,6 +3,8 @@
 from typing import Iterable
 from typing import Optional
 from typing import Set
+from typing import Type
+from typing import TypeVar
 
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.entries.script.variable_definitions import VARIABLES
@@ -20,10 +22,11 @@
 from ytdl_sub.utils.exceptions import ValidationException
 from ytdl_sub.utils.script import ScriptUtils
 from ytdl_sub.utils.scriptable import Scriptable
-from ytdl_sub.validators.string_formatter_validators import OverridesStringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import StringFormatterValidator
 from ytdl_sub.validators.string_formatter_validators import UnstructuredDictFormatterValidator
 
+ExpectedT = TypeVar("ExpectedT")
+
 
 class Overrides(UnstructuredDictFormatterValidator, Scriptable):
     """
@@ -207,7 +210,8 @@ def apply_formatter(
         formatter: StringFormatterValidator,
         entry: Optional[Entry] = None,
         function_overrides: Optional[Dict[str, str]] = None,
-    ) -> str:
+        expected_type: Type[ExpectedT] = str,
+    ) -> ExpectedT:
         """
         Parameters
         ----------
@@ -217,6 +221,8 @@ def apply_formatter(
             Optional. Entry to add source variables to the formatter
         function_overrides
             Optional. Explicit values to override the overrides themselves and source variables
+        expected_type
+            The expected type that should return. Defaults to string.
 
         Returns
         -------
@@ -227,42 +233,15 @@ def apply_formatter(
         StringFormattingException
             If the formatter that is trying to be resolved cannot
         """
-        return formatter.post_process(
-            str(
-                self._apply_to_resolvable(
-                    formatter=formatter, entry=entry, function_overrides=function_overrides
-                )
-            )
-        )
-
-    def apply_overrides_formatter_to_native(
-        self,
-        formatter: OverridesStringFormatterValidator,
-        function_overrides: Optional[Dict[str, str]] = None,
-    ) -> Any:
-        """
-        Parameters
-        ----------
-        formatter
-            Overrides formatter to apply
-        function_overrides
-            Optional. Explicit values to override the overrides themselves and source variables
-
-        Returns
-        -------
-        The native python form of the resolved variable
-        """
-        return formatter.post_process_native(
+        out = formatter.post_process(
             self._apply_to_resolvable(
-                formatter=formatter, entry=None, function_overrides=function_overrides
+                formatter=formatter, entry=entry, function_overrides=function_overrides
             ).native
         )
 
-    def evaluate_boolean(
-        self, formatter: StringFormatterValidator, entry: Optional[Entry] = None
-    ) -> bool:
-        """
-        Apply a formatter, and evaluate it to a boolean
-        """
-        output = self.apply_formatter(formatter=formatter, entry=entry)
-        return ScriptUtils.bool_formatter_output(output)
+        if not isinstance(out, expected_type):
+            raise StringFormattingException(
+                f"Expected type {expected_type.__name__}, but received '{out.__class__.__name__}'"
+            )
+
+        return out

src/ytdl_sub/config/plugin/plugin.py

@@ -48,7 +48,7 @@ def is_enabled(self) -> bool:
         Returns True if enabled, False if disabled.
         """
         if isinstance(self.plugin_options, ToggleableOptionsDictValidator):
-            return self.overrides.evaluate_boolean(self.plugin_options.enable)
+            return self.overrides.apply_formatter(self.plugin_options.enable, expected_type=bool)
         return True
 
     def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:

src/ytdl_sub/config/preset_options.py

@@ -63,7 +63,7 @@ def to_native_dict(self, overrides: Overrides) -> Dict:
         native python.
         """
         out = {
-            key: overrides.apply_overrides_formatter_to_native(val)
+            key: overrides.apply_formatter(val, expected_type=object)
             for key, val in self.dict.items()
         }
         if "cookiefile" in out:

src/ytdl_sub/downloaders/url/downloader.py

@@ -52,12 +52,12 @@ def _match_entry_to_url_validator(self, entry: Entry) -> UrlValidator:
 
         if 0 <= input_url_idx < len(self.plugin_options.urls.list):
             validator = self.plugin_options.urls.list[input_url_idx]
-            if entry_input_url in self.overrides.apply_overrides_formatter_to_native(validator.url):
+            if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list):
                 return validator
 
         # Match the first validator based on the URL, if one exists
         for validator in self.plugin_options.urls.list:
-            if entry_input_url in self.overrides.apply_overrides_formatter_to_native(validator.url):
+            if entry_input_url in self.overrides.apply_formatter(validator.url, expected_type=list):
                 return validator
 
         # Return the first validator if none exist
@@ -382,7 +382,7 @@ def _iterate_child_entries(
         entries_to_iter: List[Optional[Entry]] = entries
 
         indices = list(range(len(entries_to_iter)))
-        if self.overrides.evaluate_boolean(validator.download_reverse):
+        if self.overrides.apply_formatter(validator.download_reverse, expected_type=bool):
             indices = reversed(indices)
 
         for idx in indices:
@@ -461,8 +461,8 @@ def _download_metadata(self, url: str, validator: UrlValidator) -> Iterable[Entr
             ytdl_option_overrides=validator.ytdl_options.to_native_dict(self.overrides)
         )
 
-        include_sibling_metadata = self.overrides.evaluate_boolean(
-            validator.include_sibling_metadata
+        include_sibling_metadata = self.overrides.apply_formatter(
+            validator.include_sibling_metadata, expected_type=bool
         )
 
         parents, orphan_entries = self._download_url_metadata(
@@ -487,11 +487,9 @@ def download_metadata(self) -> Iterable[Entry]:
         # download the bottom-most urls first since they are top-priority
         for idx, url_validator in reversed(list(enumerate(self.collection.urls.list))):
             # URLs can be empty. If they are, then skip
-            if not (urls := self.overrides.apply_overrides_formatter_to_native(url_validator.url)):
+            if not (urls := self.overrides.apply_formatter(url_validator.url, expected_type=list)):
                 continue
 
-            assert isinstance(urls, list)
-
             for url in reversed(urls):
                 assert isinstance(url, str)
 

src/ytdl_sub/downloaders/url/validators.py

@@ -1,5 +1,6 @@
 from typing import Any
 from typing import Dict
+from typing import List
 from typing import Optional
 from typing import Set
 
@@ -44,7 +45,7 @@ class UrlThumbnailListValidator(ListValidator[UrlThumbnailValidator]):
 
 
 class OverridesOneOrManyUrlValidator(OverridesStringFormatterValidator):
-    def post_process_native(self, resolved: Any) -> Any:
+    def post_process(self, resolved: Any) -> List[str]:
         if isinstance(resolved, str):
             return [resolved]
         if isinstance(resolved, list):

src/ytdl_sub/plugins/date_range.py

@@ -116,7 +116,7 @@ def ytdl_options_match_filters(self) -> Tuple[List[str], List[str]]:
                 date_validator=self.plugin_options.after, overrides=self.overrides
             )
             after_filter = f"{date_type} >= {after_str}"
-            if self.overrides.evaluate_boolean(self.plugin_options.breaks):
+            if self.overrides.apply_formatter(self.plugin_options.breaks, expected_type=bool):
                 breaking_match_filters.append(after_filter)
             else:
                 match_filters.append(after_filter)

src/ytdl_sub/plugins/embed_thumbnail.py

@@ -33,7 +33,7 @@ class EmbedThumbnailPlugin(Plugin[EmbedThumbnailOptions]):
 
     @property
     def _embed_thumbnail(self) -> bool:
-        return self.overrides.evaluate_boolean(self.plugin_options)
+        return self.overrides.apply_formatter(self.plugin_options, expected_type=bool)
 
     @classmethod
     def _embed_video_thumbnail(cls, entry: Entry) -> None:

src/ytdl_sub/plugins/filter_exclude.py

@@ -7,13 +7,14 @@
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.utils.exceptions import StringFormattingException
 from ytdl_sub.utils.logger import Logger
-from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator
+from ytdl_sub.validators.validators import ListValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 logger = Logger.get("filter-exclude")
 
 
-class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
+class FilterExcludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator):
     """
     Applies a conditional OR on any number of filters comprised of either variables or scripts.
     If any filter evaluates to True, the entry will be excluded.
@@ -29,6 +30,8 @@ class FilterExcludeOptions(ListFormatterValidator, OptionsValidator):
            { %contains( %lower(description), '#short' ) }
     """
 
+    _inner_list_type = BooleanFormatterValidator
+
 
 class FilterExcludePlugin(Plugin[FilterExcludeOptions]):
     plugin_options_type = FilterExcludeOptions
@@ -52,7 +55,9 @@ def modify_entry(self, entry: Entry) -> Optional[Entry]:
             return entry
 
         for formatter in self.plugin_options.list:
-            should_exclude = self.overrides.evaluate_boolean(formatter=formatter, entry=entry)
+            should_exclude = self.overrides.apply_formatter(
+                formatter=formatter, entry=entry, expected_type=bool
+            )
 
             if should_exclude:
                 logger.info(

src/ytdl_sub/plugins/filter_include.py

@@ -7,14 +7,14 @@
 from ytdl_sub.entries.entry import Entry
 from ytdl_sub.utils.exceptions import StringFormattingException
 from ytdl_sub.utils.logger import Logger
-from ytdl_sub.utils.script import ScriptUtils
-from ytdl_sub.validators.string_formatter_validators import ListFormatterValidator
+from ytdl_sub.validators.string_formatter_validators import BooleanFormatterValidator
+from ytdl_sub.validators.validators import ListValidator
 from ytdl_sub.ytdl_additions.enhanced_download_archive import EnhancedDownloadArchive
 
 logger = Logger.get("filter-include")
 
 
-class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
+class FilterIncludeOptions(ListValidator[BooleanFormatterValidator], OptionsValidator):
     """
     Applies a conditional AND on any number of filters comprised of either variables or scripts.
     If all filters evaluate to True, the entry will be included.
@@ -38,6 +38,8 @@ class FilterIncludeOptions(ListFormatterValidator, OptionsValidator):
            }
     """
 
+    _inner_list_type = BooleanFormatterValidator
+
 
 class FilterIncludePlugin(Plugin[FilterIncludeOptions]):
     plugin_options_type = FilterIncludeOptions
@@ -61,8 +63,8 @@ def modify_entry(self, entry: Entry) -> Optional[Entry]:
             return entry
 
         for formatter in self.plugin_options.list:
-            should_exclude = ScriptUtils.bool_formatter_output(
-                self.overrides.apply_formatter(formatter=formatter, entry=entry)
+            should_exclude = self.overrides.apply_formatter(
+                formatter=formatter, entry=entry, expected_type=bool
             )
             if not should_exclude:
                 logger.info(

src/ytdl_sub/plugins/nfo_tags.py

@@ -140,7 +140,7 @@ def _create_nfo(self, entry: Entry, save_to_entry: bool = True) -> None:
         if not nfo_tags:
             return
 
-        if self.overrides.evaluate_boolean(self.plugin_options.kodi_safe):
+        if self.overrides.apply_formatter(self.plugin_options.kodi_safe, expected_type=bool):
             nfo_root = to_max_3_byte_utf8_string(nfo_root)
             nfo_tags = {
                 to_max_3_byte_utf8_string(key): [