2.5.001

Author: chapmanjacobd

.github/README.md

@@ -95,7 +95,7 @@ To stop playing press Ctrl+C in either the terminal or mpv
 <details><summary>List all subcommands</summary>
 
     $ library
-    xk media library subcommands (v2.4.011)
+    xk media library subcommands (v2.5.001)
 
     Create database subcommands:
     ╭───────────────┬────────────────────────────────────────────────────╮

pdm.lock

@@ -1 +1 @@
-__version__ = "2.4.011"
+__version__ = "2.5.001"

xklb/__init__.py

@@ -15,7 +15,7 @@ def exists(args, path) -> bool:
     m_columns = db_utils.columns(args, "media")
     try:
         known = args.db.execute(
-            f"select 1 from media where path=? or {'web' if 'webpath' in m_columns else ''}path=?",
+            f"select 1 from media where path=? or {'webpath' if 'webpath' in m_columns else 'path'}=?",
             [str(path), str(path)],
         ).fetchone()
     except sqlite3.OperationalError as e:
@@ -71,7 +71,7 @@ def consolidate(v: dict) -> Optional[dict]:
                 upload_date = None
 
     cv = {}
-    cv["playlist_id"] = v.pop("playlist_id", None)
+    cv["playlists_id"] = v.pop("playlists_id", None)  # not to be confused with yt-dlp playlist_id
     cv["webpath"] = iterables.safe_unpack(
         v.pop("webpath", None),
         v.pop("webpage_url", None),
@@ -238,8 +238,8 @@ def download_add(
     if local_path and Path(local_path).exists():
         local_path = str(Path(local_path).resolve())
         fs_args = argparse.Namespace(
-            profiles=args.profiles,
-            scan_subtitles=DBType.video in args.profiles,
+            profile=args.profile,
+            scan_subtitles=args.profile == DBType.video,
             ocr=False,
             speech_recognition=False,
             delete_unplayable=False,

xklb/db_media.py

@@ -9,11 +9,11 @@
 playlists table
     profile = Type of extractor -- consts.DBType
     extractor_key = Name of the Extractor -- "Local", "Imgur", "YouTube"
-    extractor_playlist_id = ID that the extractor uses to refer to playlists
+    extractor_playlist_id = ID that the extractor uses to refer to playlists (yt-dlp playlist_id)
 
 media table
     extractor_id = ID that the Extractor uses to refer to media
-    playlist_id = Foreign key to playlists table
+    playlists_id = Foreign key to playlists table
 """
 
 
@@ -96,10 +96,10 @@ def exists(args, playlist_path) -> bool:
     return True
 
 
-def get_parentpath_playlist_id(args, playlist_path) -> Optional[int]:
+def get_parentpath_playlists_id(args, playlist_path) -> Optional[int]:
     try:
         known = args.db.pop(
-            "select path from playlists where ? like path || '%' and path != ?",
+            "select id from playlists where ? like path || '%' and path != ?",
             [str(playlist_path), str(playlist_path)],
         )
     except sqlite3.OperationalError as e:
@@ -127,7 +127,7 @@ def delete_subpath_playlists(args, playlist_path) -> Optional[int]:
 def add(args, playlist_path: str, info: dict, check_subpath=False, extractor_key=None) -> int:
     playlist_path = playlist_path.strip()
     if check_subpath:
-        parentpath_playlist_id = get_parentpath_playlist_id(args, playlist_path)
+        parentpath_playlist_id = get_parentpath_playlists_id(args, playlist_path)
         if parentpath_playlist_id:
             return parentpath_playlist_id
         else:
@@ -144,7 +144,7 @@ def media_exists(args, playlist_path, path) -> bool:
     m_columns = db_utils.columns(args, "media")
     try:
         known = args.db.execute(
-            f"select 1 from media where playlist_id in (select id from playlists where path = ?) and (path=? or {'web' if 'webpath' in m_columns else ''}path=?)",
+            f"select 1 from media where playlists_id in (select id from playlists where path = ?) and (path=? or {'webpath' if 'webpath' in m_columns else 'path'}=?)",
             [str(playlist_path), str(path), str(path)],
         ).fetchone()
     except sqlite3.OperationalError as e:

xklb/db_playlists.py

@@ -168,9 +168,10 @@ def construct_query(args) -> Tuple[str, dict]:
         ORDER BY RANDOM()
         LIMIT 1
     )"""
-    if "playlist_id" in m_columns:
+    if "playlists_id" in m_columns:
         query = f"""select
                 m.id
+                , m.playlists_id
                 , m.path
                 , p.path playlist_path
                 {', m.title' if 'title' in m_columns else ''}
@@ -184,7 +185,7 @@ def construct_query(args) -> Tuple[str, dict]:
                 {', p.extractor_config' if 'extractor_config' in pl_columns else ''}
                 , p.extractor_key
             FROM media m
-            LEFT JOIN playlists p on p.id = m.playlist_id
+            LEFT JOIN playlists p on p.id = m.playlists_id
             WHERE 1=1
                 {'and COALESCE(m.time_downloaded,0) = 0' if 'time_downloaded' in m_columns else ''}
                 and COALESCE(m.time_deleted,0) = 0
@@ -342,7 +343,7 @@ def dl_download(args=None) -> None:
                 gdl_backend.download(args, m)
             elif args.profile == DBType.filesystem:
                 local_path = web.download_url(m["path"], output_prefix=args.prefix, relative=args.relative)
-                db_media.download_add(args, m["path"], {}, local_path)
+                db_media.download_add(args, m["path"], m, local_path)
             else:
                 raise NotImplementedError
         except Exception:

xklb/dl_extract.py

@@ -168,12 +168,13 @@ def extract_metadata(mp_args, path) -> Optional[Dict[str, int]]:
     }
 
     ext = path.rsplit(".", 1)[-1].lower()
+    is_scan_all_files = getattr(mp_args, "scan_all_files", False)
 
-    if DBType.audio in mp_args.profiles and (
-        ext in (consts.AUDIO_ONLY_EXTENSIONS | consts.VIDEO_EXTENSIONS) or mp_args.scan_all_files
+    if objects.is_profile(mp_args, DBType.audio) and (
+        ext in (consts.AUDIO_ONLY_EXTENSIONS | consts.VIDEO_EXTENSIONS) or is_scan_all_files
     ):
         media |= av.munge_av_tags(mp_args, path)
-    elif DBType.video in mp_args.profiles and (ext in consts.VIDEO_EXTENSIONS or mp_args.scan_all_files):
+    elif objects.is_profile(mp_args, DBType.video) and (ext in consts.VIDEO_EXTENSIONS or is_scan_all_files):
         media |= av.munge_av_tags(mp_args, path)
 
     if not Path(path).exists():
@@ -184,7 +185,7 @@ def extract_metadata(mp_args, path) -> Optional[Dict[str, int]]:
         text_exts |= consts.OCR_EXTENSIONS
     if mp_args.speech_recognition:
         text_exts |= consts.SPEECH_RECOGNITION_EXTENSIONS
-    if DBType.text in mp_args.profiles and (ext in text_exts or mp_args.scan_all_files):
+    if objects.is_profile(mp_args, DBType.text) and (ext in text_exts or is_scan_all_files):
         try:
             start = timer()
             if any([mp_args.ocr, mp_args.speech_recognition]):
@@ -201,7 +202,7 @@ def extract_metadata(mp_args, path) -> Optional[Dict[str, int]]:
         media["hash"] = sample_hash.sample_hash_file(path)
 
     if getattr(mp_args, "process", False):
-        if DBType.audio in mp_args.profiles and Path(path).suffix not in [".opus", ".mka"]:
+        if objects.is_profile(mp_args, DBType.audio) and Path(path).suffix not in [".opus", ".mka"]:
             path = media["path"] = process_audio.process_path(
                 path, split_longer_than=2160 if "audiobook" in path.lower() else None
             )
@@ -223,7 +224,7 @@ def clean_up_temp_dirs():
 
 
 def extract_chunk(args, media) -> None:
-    if DBType.image in args.profiles:
+    if objects.is_profile(args, DBType.image):
         media = books.extract_image_metadata_chunk(media)
 
     if args.scan_subtitles:
@@ -246,7 +247,7 @@ def extract_chunk(args, media) -> None:
 
         captions.append(caption)
 
-    media = [{"playlist_id": args.playlist_id, **d} for d in media]
+    media = [{"playlists_id": args.playlists_id, **d} for d in media]
     media = iterables.list_dict_filter_bool(media)
     args.db["media"].insert_all(media, pk="id", alter=True, replace=True)
 
@@ -393,7 +394,7 @@ def scan_path(args, path_str: str) -> int:
         ),
         "time_deleted": 0,
     }
-    args.playlist_id = db_playlists.add(args, str(path), info, check_subpath=True)
+    args.playlists_id = db_playlists.add(args, str(path), info, check_subpath=True)
 
     print(f"[{path}] Building file list...")
     new_files = find_new_files(args, path)

xklb/fs_extract.py

@@ -121,14 +121,15 @@ def download(args, m):
         job = gallery_dl.job.DownloadJob(webpath)
     except gallery_dl.exception.NoExtractorError:
         log.info("[%s]: NoExtractorError", webpath)  # RecoverableError
-        db_media.download_add(args, webpath, error="NoExtractorError")
+        db_media.download_add(args, webpath, m, error="NoExtractorError")
         return
 
     job_status = job.run()
     errors = parse_gdl_job_status(job_status, webpath, ignore_errors=args.ignore_errors)
 
     info = getattr(job.pathfmt, "kwdict", None)
     if info:
+        info = {**m, **info}
         info["path"] = webpath
 
     local_path = getattr(job.pathfmt, "path", "") or None
@@ -219,16 +220,16 @@ def get_playlist_metadata(args, playlist_path):
 
         log.debug("webpath == playlist_path" if webpath == playlist_path else "webpath != playlist_path")
 
-        playlist_id = None
+        playlists_id = None
         if is_playlist:
-            playlist_id = db_playlists.add(args, playlist_path, info)
+            playlists_id = db_playlists.add(args, playlist_path, info)
         else:
             log.warning("Importing playlist-less media %s", playlist_path)
 
         if db_media.exists(args, webpath):
             log.warning("Media already exists")
 
-        info = {**info, "playlist_id": playlist_id, "webpath": webpath}
+        info = {**info, "playlists_id": playlists_id, "webpath": webpath}
         db_media.playlist_media_add(
             args,
             playlist_path,

xklb/gdl_backend.py

@@ -217,11 +217,11 @@ def since_last_created(args, playlist_path):
         """
         select max(time_created)
         from (
-            select time_created, playlist_id from reddit_posts
+            select time_created, playlists_id from reddit_posts
             UNION ALL
-            select time_created, playlist_id from media
+            select time_created, playlists_id from media
         )
-        where playlist_id = (select id from playlists where path = ?)
+        where playlists_id = (select id from playlists where path = ?)
     """,
         [playlist_path],
         ignore_errors=["no such column", "no such table"],

xklb/reddit_extract.py

@@ -172,7 +172,7 @@ def block(args=None) -> None:
                 f"""UPDATE media
                 SET time_deleted={consts.APPLICATION_START}
                 WHERE path LIKE 'http%'
-                AND playlist_id in (
+                AND playlists_id in (
                     SELECT id from playlists
                     WHERE path IN ("""
                 + ",".join(["?"] * len(playlist_paths))
@@ -185,7 +185,7 @@ def block(args=None) -> None:
                 """SELECT path, size FROM media
                 WHERE coalesce(time_deleted, 0)=0
                 AND time_downloaded > 0
-                AND playlist_id in (
+                AND playlists_id in (
                     SELECT id from playlists
                     WHERE path IN ("""
                 + ",".join(["?"] * len(playlist_paths))

xklb/scripts/block.py

@@ -160,7 +160,7 @@ def add_playlist(args, path):
         },
         "time_deleted": 0,
     }
-    args.playlist_id = db_playlists.add(args, str(path), info)
+    args.playlists_id = db_playlists.add(args, str(path), info)
 
 
 def consolidate_media(args, path: str) -> dict:

xklb/scripts/links_db.py

@@ -32,7 +32,7 @@ def get_duplicates(args) -> List[dict]:
             and extractor_id is not null
             and extractor_id != ""
             and coalesce(time_deleted, 0)=0
-            and playlist_id in (
+            and playlists_id in (
                 SELECT id from playlists
                 WHERE extractor_key
                     NOT IN ('Local', 'NBCStations', 'TedTalk', 'ThisAmericanLife', 'InfoQ', 'NFB', 'KickStarter')
@@ -54,7 +54,7 @@ def get_duplicates(args) -> List[dict]:
             and title is null
     ) m2
     JOIN media_fts on m2.id = media_fts.rowid
-    JOIN playlists p2 on p2.id = m2.playlist_id
+    JOIN playlists p2 on p2.id = m2.playlists_id
     WHERE p2.extractor_key = 'Local'
         AND media_fts.path MATCH '"'||m1.extractor_id||'"'
         AND m2.PATH LIKE '%['||m1.extractor_id||']%'

xklb/scripts/merge_online_local.py

@@ -119,15 +119,15 @@ def delete_playlists(args, playlists) -> None:
             with args.db.conn:
                 cursor = args.db.conn.execute(
                     """DELETE from media where
-                    playlist_id in (
+                    playlists_id in (
                         SELECT id from playlists
                         WHERE path IN ("""
                     + ",".join(["?"] * len(online_media))
                     + "))",
                     (*online_media,),
                 )
                 deleted_media_count += cursor.rowcount
-    except sqlite3.OperationalError:  # no such column: playlist_id
+    except sqlite3.OperationalError:  # no such column: playlists_id
         pass
 
     local_media = [p.rstrip(os.sep) for p in playlists if not p.startswith("http")]
@@ -146,7 +146,7 @@ def playlists() -> None:
     m_columns = db_utils.columns(args, "media")
     query, bindings = construct_query(args)
 
-    if "playlist_id" in m_columns:
+    if "playlists_id" in m_columns:
         query = f"""
         select
             coalesce(p.path, "Playlist-less media") path
@@ -158,8 +158,8 @@ def playlists() -> None:
             {', sum(m.size) size' if 'size' in m_columns else ''}
             , count(*) count
         from media m
-        join ({query}) p on p.id = m.playlist_id
-        group by m.playlist_id, coalesce(p.path, "Playlist-less media")
+        join ({query}) p on p.id = m.playlists_id
+        group by m.playlists_id, coalesce(p.path, "Playlist-less media")
         order by count, p.path
         """
 

xklb/scripts/playlists.py

@@ -139,8 +139,7 @@ def run(self, info) -> Tuple[list, dict]:  # pylint: disable=arguments-renamed
                         log.warning("Importing playlist-less media %s", playlist_path)
                     else:
                         # add sub-playlist
-                        playlist_id = db_playlists.add(args, playlist_path, info, extractor_key=extractor_key)
-                        entry["playlist_id"] = playlist_id
+                        entry["playlists_id"] = db_playlists.add(args, playlist_path, info, extractor_key=extractor_key)
                         log.info("playlists.add2 %s", t.elapsed())
 
                     db_media.playlist_media_add(args, webpath, entry)  # type: ignore
@@ -213,13 +212,13 @@ def get_extra_metadata(args, playlist_path, playlist_dl_opts=None) -> Optional[L
                 SELECT
                 id
                 , path
-                , playlist_id
+                , playlists_id
                 FROM media
                 WHERE 1=1
                     AND COALESCE(time_deleted, 0)=0
                     {'and width is null' if 'width' in m_columns else ''}
                     and path not like '%playlist%'
-                    and playlist_id = (select id from playlists where path = ?)
+                    and playlists_id = (select id from playlists where path = ?)
                 ORDER by random()
                 """,
                 [playlist_path],
@@ -228,7 +227,7 @@ def get_extra_metadata(args, playlist_path, playlist_dl_opts=None) -> Optional[L
             videos = []
 
         current_video_count = 0
-        for id, path, playlist_id in videos:
+        for id, path, playlists_id in videos:
             entry = ydl.extract_info(path)
             if entry is None:
                 continue
@@ -259,7 +258,7 @@ def get_extra_metadata(args, playlist_path, playlist_dl_opts=None) -> Optional[L
                     args.db["captions"].insert_all(captions, alter=True)
 
             entry["id"] = id
-            entry["playlist_id"] = playlist_id
+            entry["playlists_id"] = playlists_id
             entry["chapter_count"] = chapter_count
 
             db_media.playlist_media_add(args, path, entry)
@@ -457,7 +456,9 @@ def blocklist_check(info, *pargs, incomplete):
 
         if info is None:
             log.debug("[%s]: yt-dlp returned no info", webpath)
+            info = m
         else:
+            info = {**m, **info}
             local_path = info.get("local_path", None)
             if args.profile == DBType.audio:
                 local_path = ydl.prepare_filename({**info, "ext": args.ext})

xklb/tube_backend.py

@@ -351,7 +351,7 @@ def reddit_frequency(frequency) -> str:
     "requested_downloads",
     "thumbnails",
     "playlist_count",
-    "playlist_id",
+    "playlists_id",
     "playlist_title",
     "playlist_uploader",
     "audio_channels",