Add sync_with_source option to delete files removed from the source

Adds an opt-in output_options field that deletes local files when their
source entry is no longer present in a subscription's URL(s). After the
metadata pass, any download archive entry whose ID is absent from the
source is removed along with all of its files.

Reuses the existing keep_files/keep_max_files deletion path. The new
predicate calls the same EnhancedDownloadArchive._remove_entry, so media,
.nfo, thumbnail and .info.json files, the archive entry, and the
transaction log are all handled the same, and no re-downloading occurs.

Because ytdl-sub cannot distinguish "this video was removed" from
"metadata collection stopped early", enabling sync_with_source forces a
full enumeration of the source during the metadata pass:
break_on_existing is overridden to False, and the max_downloads cap that
keep_max_files injects is suppressed, only if the flag is set to True.
For truncation ytdl-sub cannot override, YTDLP now reports which
early-stop exception fired, and any such run skips pruning entirely with
a warning. A source that returns zero entries, or a run that never
enumerated the source at all (update_with_info_json), also never deletes.

Defaults to False, requires maintain_download_archive, and leaves
existing behavior unchanged.

Refs #1038

Author: sapphimars

docs/source/config_reference/plugins.rst

@@ -632,6 +632,7 @@ Defines where to output files and thumbnails after all post-processing has compl
          keep_files_after: 19000101
          keep_max_files: 1000
          keep_files_date_eval: "{upload_date_standardized}"
+         sync_with_source: False
 
 ``download_archive_name``
 
@@ -750,6 +751,29 @@ Defines where to output files and thumbnails after all post-processing has compl
   When True, sets the file's mtime to match the video's upload_date from
   yt-dlp metadata. Defaults to False.
 
+``sync_with_source``
+
+:expected type: Optional[OverridesFormatter]
+:description:
+    Requires ``maintain_download_archive`` set to True.
+
+    Deletes files whose source entry is no longer present in the subscription's URL(s).
+    After the metadata pass, any entry in the download archive whose ID is absent from
+    the source is removed, along with all of its files.
+
+    This forces a metadata fetch of every URL on each invocation. ytdl-sub cannot
+    tell the difference between "this video was removed" and "metadata collection stopped
+    early", so ``break_on_existing`` and ``keep_max_files``' download cap are both
+    disabled during the metadata pass. Only enable this on sources you expect to change,
+    and expect slower runs on large playlists.
+
+    If metadata collection is truncated for a reason ytdl-sub cannot override (such as
+    ``date_range`` with ``breaks`` enabled, or a user-set ``max_downloads``), or if the
+    source returns no entries at all, syncing is skipped for that run and a warning is
+    logged rather than deleting files.
+
+    Defaults to False.
+
 ``thumbnail_name``
 
 :expected type: Optional[EntryFormatter]

src/ytdl_sub/config/preset_options.py

@@ -14,6 +14,7 @@
 from ytdl_sub.validators.sort_by_validator import KeepMaxFilesSortByValidator
 from ytdl_sub.validators.string_datetime import StringDatetimeValidator
 from ytdl_sub.validators.string_formatter_validators import (
+    OverridesBooleanFormatterValidator,
     OverridesIntegerFormatterValidator,
     OverridesStringFormatterValidator,
     StandardizedDateValidator,
@@ -107,6 +108,7 @@ class OutputOptions(OptionsDictValidator):
              keep_files_after: 19000101
              keep_max_files: 1000
              keep_files_date_eval: "{upload_date_standardized}"
+             sync_with_source: False
     """
 
     _required_keys = {"output_directory", "file_name"}
@@ -123,6 +125,7 @@ class OutputOptions(OptionsDictValidator):
         "keep_files_date_eval",
         "keep_max_files_sort_by",
         "preserve_mtime",
+        "sync_with_source",
     }
 
     @classmethod
@@ -195,11 +198,18 @@ def __init__(self, name, value):
             key="preserve_mtime", validator=BoolValidator, default=False
         )
 
+        self._sync_with_source = self._validate_key_if_present(
+            key="sync_with_source", validator=OverridesBooleanFormatterValidator
+        )
+
         if (
-            self._keep_files_before or self._keep_files_after or self._keep_max_files
+            self._keep_files_before
+            or self._keep_files_after
+            or self._keep_max_files
+            or self._sync_with_source
         ) and not self.maintain_download_archive:
             raise self._validation_exception(
-                "keep_files/keep_max requires maintain_download_archive set to True"
+                "keep_files/keep_max/sync_with_source requires maintain_download_archive set to True"
             )
 
     @property
@@ -355,6 +365,32 @@ def keep_max_files_sort_by(self) -> Optional[KeepMaxFilesSortByValidator]:
         """
         return self._keep_max_files_sort_by
 
+    @property
+    def sync_with_source(self) -> Optional[OverridesBooleanFormatterValidator]:
+        """
+        :expected type: Optional[OverridesFormatter]
+        :description:
+            Requires ``maintain_download_archive`` set to True.
+
+            Deletes files whose source entry is no longer present in the subscription's URL(s).
+            After the metadata pass, any entry in the download archive whose ID is absent from
+            the source is removed, along with all of its files.
+
+            This forces a metadata fetch of every URL on each invocation. ytdl-sub cannot
+            tell the difference between "this video was removed" and "metadata collection stopped
+            early", so ``break_on_existing`` and ``keep_max_files``' download cap are both
+            disabled during the metadata pass. Only enable this on sources you expect to change,
+            and expect slower runs on large playlists.
+
+            If metadata collection is truncated for a reason ytdl-sub cannot override (such as
+            ``date_range`` with ``breaks`` enabled, or a user-set ``max_downloads``), or if the
+            source returns no entries at all, syncing is skipped for that run and a warning is
+            logged rather than deleting files.
+
+            Defaults to False.
+        """
+        return self._sync_with_source
+
     @property
     def preserve_mtime(self) -> bool:
         """

src/ytdl_sub/downloaders/url/downloader.py

@@ -381,6 +381,9 @@ def _iterate_child_entries(
             indices = reversed(indices)
 
         for idx in indices:
+            self._enhanced_download_archive.record_source_entry_id(
+                entry_id=entries_to_iter[idx].uid
+            )
             self._url_state.entries_downloaded += 1
 
             if self._is_downloaded(entries_to_iter[idx]):
@@ -413,14 +416,21 @@ def _download_url_metadata(
         """
         Downloads only info.json files and forms EntryParent trees
         """
+        truncation_reasons: List[str] = []
         with self._separate_download_archives():
             entry_dicts = YTDLP.extract_info_via_info_json(
                 working_directory=self.working_directory,
                 ytdl_options_overrides=ytdl_options_overrides,
                 log_prefix_on_info_json_dl="Downloading metadata for",
+                truncation_reasons=truncation_reasons,
                 url=url,
             )
 
+        for reason in truncation_reasons:
+            self._enhanced_download_archive.mark_source_enumeration_truncated(
+                reason=f"{reason} while collecting metadata for {url}"
+            )
+
         parents = EntryParent.from_entry_dicts(
             url=url,
             entry_dicts=entry_dicts,

src/ytdl_sub/downloaders/ytdlp.py

@@ -189,6 +189,7 @@ def extract_info_via_info_json(
         working_directory: str,
         ytdl_options_overrides: Dict,
         log_prefix_on_info_json_dl: Optional[str] = None,
+        truncation_reasons: Optional[List[str]] = None,
         **kwargs,
     ) -> List[Dict]:
         """
@@ -208,6 +209,10 @@ def extract_info_via_info_json(
         log_prefix_on_info_json_dl
             Optional. Spin a new thread to listen for new info.json files. Log
             f'{log_prefix_on_info_json_dl} {title}' when a new one appears
+        truncation_reasons
+            Optional. Appends the name of any exception that stopped metadata
+            collection early. A non-empty list means the returned entries are
+            an incomplete view of the source.
         **kwargs
             arguments passed directory to YoutubeDL extract_info
         """
@@ -217,16 +222,22 @@ def extract_info_via_info_json(
             ):
                 cls.extract_info(ytdl_options_overrides=ytdl_options_overrides, **kwargs)
         except RejectedVideoReached:
+            if truncation_reasons is not None:
+                truncation_reasons.append("RejectedVideoReached")
             cls.logger.debug(
                 "RejectedVideoReached, stopping additional downloads "
                 "(Can be disable by setting `date_range.breaks` to False)."
             )
         except ExistingVideoReached:
+            if truncation_reasons is not None:
+                truncation_reasons.append("ExistingVideoReached")
             cls.logger.debug(
                 "ExistingVideoReached, stopping additional downloads. "
                 "(Can be disable by setting `ytdl_options.break_on_existing` to False)."
             )
         except MaxDownloadsReached:
+            if truncation_reasons is not None:
+                truncation_reasons.append("MaxDownloadsReached")
             cls.logger.info("MaxDownloadsReached, stopping additional downloads.")
 
         parent_dicts: List[Dict] = []

src/ytdl_sub/subscriptions/subscription_download.py

@@ -144,6 +144,27 @@ def _maintain_archive_file(self):
         # If output options maintains stale file deletion, perform the delete here prior to saving
         # the download archive
         if self.maintain_download_archive:
+            if self.output_options.sync_with_source and self.overrides.apply_formatter(
+                self.output_options.sync_with_source, expected_type=bool
+            ):
+                source_entry_ids = self.download_archive.source_entry_ids
+
+                if source_entry_ids is None:
+                    logger.warning(
+                        "sync_with_source: the source was not fully enumerated, skipping sync "
+                        "to avoid deleting files. This happens when metadata collection stops "
+                        "early, i.e. from `date_range.breaks` or a user set `max_downloads`."
+                    )
+                elif not source_entry_ids:
+                    logger.warning(
+                        "sync_with_source: the source returned zero entries, skipping sync. "
+                        "An empty fetch is not treated as an emptied source."
+                    )
+                else:
+                    self.download_archive.remove_entries_not_in_source(
+                        source_entry_ids=source_entry_ids
+                    )
+
             date_range_to_keep = to_date_range(
                 before=self.output_options.keep_files_before,
                 after=self.output_options.keep_files_after,

src/ytdl_sub/subscriptions/subscription_ytdl_options.py

@@ -79,6 +79,15 @@ def _info_json_only_options(self) -> Dict:
             "extract_flat": "discard",  # do not store info.json in mem since its in file
         }
 
+    @property
+    def _sync_with_source(self) -> bool:
+        if not self._preset.output_options.sync_with_source:
+            return False
+
+        return self._overrides.apply_formatter(
+            self._preset.output_options.sync_with_source, expected_type=bool
+        )
+
     @property
     def _output_options(self) -> Dict:
         ytdl_options = {}
@@ -87,7 +96,11 @@ def _output_options(self) -> Dict:
             ytdl_options["download_archive"] = (
                 self._enhanced_download_archive.working_ytdl_file_path
             )
-        if self._preset.output_options.keep_max_files:
+
+        # sync_with_source needs a full enumeration of the source,
+        # max_downloads would truncate the metadata pass and make
+        # present entries look removed.
+        if self._preset.output_options.keep_max_files and not self._sync_with_source:
             keep_max_files = self._overrides.apply_formatter(
                 self._preset.output_options.keep_max_files, expected_type=int
             )
@@ -97,6 +110,16 @@ def _output_options(self) -> Dict:
 
         return ytdl_options
 
+    @property
+    def _sync_with_source_options(self) -> Dict:
+        if not self._sync_with_source:
+            return {}
+
+        # stopping at the first alreadt downloaded entry would hide
+        # the rest of the soure, which sync_with_source would then
+        # interpret as deleted entries
+        return {"break_on_existing": False}
+
     def _plugin_ytdl_options(self, plugin: Type[PluginT]) -> Dict:
         if plugin_obj := self._get_plugin(plugin):
             return plugin_obj.ytdl_options()
@@ -175,6 +198,7 @@ def metadata_builder(self) -> YTDLOptionsBuilder:
             self._plugin_ytdl_options(AudioExtractPlugin),  # will override format
             self._user_ytdl_options,  # user ytdl options...
             self._info_json_only_options,  # then info_json_only options
+            self._sync_with_source_options,  # then sync_with_source overrides
         )
 
     def download_builder(self) -> YTDLOptionsBuilder:

src/ytdl_sub/ytdl_additions/enhanced_download_archive.py

@@ -406,6 +406,11 @@ def __init__(
         self.num_entries_added: int = 0
         self.num_entries_modified: int = 0
         self.num_entries_removed: int = 0
+        # None means no source enum has happened, which
+        # is not the same as a source that enumed to
+        # zero entries
+        self._source_entry_ids: Optional[Set[str]] = None
+        self._source_enumeration_truncated: bool = False
 
     @property
     def num_entries(self) -> int:
@@ -439,6 +444,8 @@ def reinitialize(self, dry_run: bool) -> "EnhancedDownloadArchive":
             mapping_file_path=self._output_file_path,
             migrated_mapping_file_path=self._migrated_file_path,
         )
+        self._source_entry_ids = None
+        self._source_enumeration_truncated = False
         return self
 
     @property
@@ -550,6 +557,46 @@ def prepare_download_archive(self) -> "EnhancedDownloadArchive":
 
         return self
 
+    def record_source_entry_id(self, entry_id: str) -> None:
+        """
+        Records an entry ID seen in the source during the metadata pass.
+
+        Parameters
+        ----------
+        entry_id
+            Entry ID from the source.
+        """
+        if self._source_entry_ids is None:
+            self._source_entry_ids = set()
+
+        self._source_entry_ids.add(entry_id)
+
+    def mark_source_enumeration_truncated(self, reason: str) -> None:
+        """
+        Marks the source enumeration as an incomplete view of the source,
+        which disables any pruning that relies on knowing every entry.
+
+        Parameters
+        ----------
+        reason
+            Why metadata collection stopped early.
+        """
+        self._source_enumeration_truncated = True
+        logger.debug("Source enumeration truncated: %s", reason)
+
+    @property
+    def source_entry_ids(self) -> Optional[Set[str]]:
+        """
+        Returns
+        -------
+        Every entry ID seen in the source during metadata pass, or
+        None if the source was never enumerated or stopped early.
+        """
+        if self._source_enumeration_truncated:
+            return None
+
+        return self._source_entry_ids
+
     def _remove_entry(self, uid: str, mapping: DownloadMapping) -> None:
         for file_name in mapping.file_names:
             self._file_handler.delete_file_from_output_directory(file_name=file_name)
@@ -637,6 +684,32 @@ def remove_stale_files(
 
         return self
 
+    def remove_entries_not_in_source(self, source_entry_ids: Set[str]) -> "EnhancedDownloadArchive":
+        """
+        Checks all entries within mappings. If any entry is no longer present
+        in the source, delete it.
+
+        Parameters
+        ----------
+        source_entry_ids
+            Every entry ID present in the source
+
+        Returns
+        -------
+        self
+        """
+        stale_mappings: Dict[str, DownloadMapping] = {
+            uid: mapping
+            for uid, mapping in self.mapping.entry_mappings.items()
+            if uid not in source_entry_ids
+        }
+
+        for uid, mapping in stale_mappings.items():
+            logger.info("Entry %s is no longer in the source, deleting its files", uid)
+            self._remove_entry(uid=uid, mapping=mapping)
+
+        return self
+
     def save_download_mappings(self) -> "EnhancedDownloadArchive":
         """
         Saves the updated download mappings to the output directory if any files were changed.