tubedl: improve behavior of when YouTube does not provide metadata during initial tubeadd

Author: chapmanjacobd

library/createdb/tube_add.py

@@ -93,8 +93,10 @@ def tube_update(args=None) -> None:
 
     tube_playlists = db_playlists.get_all(
         args,
-        sql_filters=["AND extractor_key NOT IN ('Local', 'reddit_praw_redditor', 'reddit_praw_subreddit')"],
+        sql_filters=["AND coalesce(extractor_key, '') NOT IN ('Local', 'reddit_praw_redditor', 'reddit_praw_subreddit')"],
     )
+
+    print("Checking", len(tube_playlists), "playlists for updates")
     for d in tube_playlists:
         tube_backend.get_playlist_metadata(
             args,

library/createdb/tube_backend.py

@@ -139,6 +139,7 @@ def run(self, info) -> tuple[list, dict]:  # pylint: disable=arguments-renamed
                 )
 
                 # TODO: webpage_url_basename seems like a weird property to check
+                # or is this just extra guarding for YouTube behavior
                 if webpath != playlist_path and info.get("webpage_url_basename") == "playlist":
                     if playlist_root:
                         if not info.get("playlist_id") or webpath == playlist_path:
@@ -187,16 +188,23 @@ def run(self, info) -> tuple[list, dict]:  # pylint: disable=arguments-renamed
             pl = ydl.extract_info(playlist_path, download=False, process=True)
             log.debug("ydl.extract_info done %s", t.elapsed())
         except yt_dlp.DownloadError:
-            log.error("DownloadError skipping %s", playlist_path)
+            if args.safe:
+                log.error("DownloadError skipping %s", playlist_path)
+            else:
+                log.warning("Could not scrape playlist metadata successfully (will try again [in a few days] during tubeupdate)")
+                db_playlists.save_undownloadable(args, playlist_path)
             return
         except ExistingPlaylistVideoReached:
             if added_media_count > count_before_extract:
                 sys.stderr.write("\n")
             db_playlists.log_problem(args, playlist_path)
         else:
-            if not pl and not args.safe:
-                log.warning("Logging undownloadable media")
-                db_playlists.save_undownloadable(args, playlist_path)
+            if not pl:
+                if args.safe:
+                    log.error("DownloadError skipping %s", playlist_path)
+                else:
+                    log.warning("Could not scrape playlist metadata successfully (will try again [in a few days] during tubeupdate)")
+                    db_playlists.save_undownloadable(args, playlist_path)
 
         if args.action == consts.SC.tube_update:
             if added_media_count > count_before_extract:

library/mediadb/db_playlists.py

@@ -306,4 +306,7 @@ def log_problem(args, playlist_path) -> None:
 
 def save_undownloadable(args, playlist_path) -> None:
     entry = {"path": playlist_path, "extractor_config": args.extractor_config}
-    _add(args, objects.dict_filter_bool(entry) or {})
+    _add(args, consolidate(args, entry))
+
+    if args.action == consts.SC.tube_update:
+        update_less_frequently(args, playlist_path)