2.4.005

Author: chapmanjacobd

.github/README.md

@@ -95,14 +95,16 @@ To stop playing press Ctrl+C in either the terminal or mpv
 <details><summary>List all subcommands</summary>
 
     $ library
-    xk media library subcommands (v2.4.004)
+    xk media library subcommands (v2.4.005)
 
     Create database subcommands:
     ╭───────────────┬────────────────────────────────────────────────────╮
     │ fsadd         │ Add local media                                    │
     ├───────────────┼────────────────────────────────────────────────────┤
     │ tubeadd       │ Add online video media (yt-dlp)                    │
     ├───────────────┼────────────────────────────────────────────────────┤
+    │ webadd        │ Add open-directory media                           │
+    ├───────────────┼────────────────────────────────────────────────────┤
     │ galleryadd    │ Add online gallery media (gallery-dl)              │
     ├───────────────┼────────────────────────────────────────────────────┤
     │ tabsadd       │ Create a tabs database; Add URLs                   │
@@ -256,6 +258,11 @@ To stop playing press Ctrl+C in either the terminal or mpv
     │ nouns          │ Unstructured text -> compound nouns (stdin) │
     ╰────────────────┴─────────────────────────────────────────────╯
 
+    Other subcommands:
+    ╭───────────┬───────────╮
+    │ webupdate │ webupdate │
+    ╰───────────┴───────────╯
+
 
 </details>
 
@@ -575,6 +582,19 @@ BTW, for some cols like time_deleted you'll need to specify a where clause so th
         library tubeupdate tw.db --extra
 
 
+</details>
+
+###### webadd
+
+<details><summary>Add open-directory media</summary>
+
+    $ library webadd -h
+    usage: library web-add [(--filesystem) | --video | --audio | --image | --text] DATABASE URL ...
+
+    Scan open directories
+
+
+
 </details>
 
 ###### galleryadd
@@ -2573,6 +2593,21 @@ BTW, for some cols like time_deleted you'll need to specify a where clause so th
         library process-audio --split-longer-than 36mins audiobook.m4b audiobook2.mp3
 
 
+</details>
+
+### Other subcommands
+
+###### webupdate
+
+<details><summary>webupdate</summary>
+
+    $ library webupdate -h
+    usage: library web-update DATABASE
+
+    Update saved open directories
+
+
+
 </details>
 
 

pdm.lock

@@ -16,8 +16,6 @@ def create_args(test_name):
         verbose=2,
         profile={},
         extractor_config={},
-        extra_playlist_data={},
-        extra_media_data={},
     )
     args.db = connect(args)
     return args

tests/test_gdl_backend.py

@@ -1 +1 @@
-__version__ = "2.4.004"
+__version__ = "2.4.005"

xklb/__init__.py

@@ -296,8 +296,8 @@ def filter_args_sql(args, m_columns):
         {'and COALESCE(time_deleted,0) = 0' if 'time_deleted' in m_columns and "deleted" not in (getattr(args, 'sort_groups_by',None) or '') and "time_deleted" not in args.where else ''}
         {'AND (score IS NULL OR score > 7)' if 'score' in m_columns else ''}
         {'AND (upvote_ratio IS NULL OR upvote_ratio > 0.73)' if 'upvote_ratio' in m_columns else ''}
-        {'AND COALESCE(time_downloaded,0) = 0' if args.online_media_only else ''}
-        {'AND COALESCE(time_downloaded,1)!= 0 AND path not like "http%"' if args.local_media_only else ''}
+        {'AND COALESCE(time_downloaded,0) = 0' if args.online_media_only and 'time_downloaded' in m_columns else ''}
+        {'AND COALESCE(time_downloaded,1)!= 0 AND path not like "http%"' if args.local_media_only and 'time_downloaded' in m_columns else ''}
     """
 
 

xklb/db_media.py

@@ -134,7 +134,7 @@ def add(args, playlist_path: str, info: dict, check_subpath=False, extractor_key
             delete_subpath_playlists(args, playlist_path)
 
     pl = consolidate(args, objects.dumbcopy(info))
-    playlist = {**pl, "path": playlist_path, **args.extra_playlist_data}
+    playlist = {**pl, "path": playlist_path}
     if extractor_key:
         playlist["extractor_key"] = extractor_key
     return _add(args, objects.dict_filter_bool(playlist))
@@ -240,6 +240,5 @@ def save_undownloadable(args, playlist_path) -> None:
     entry = {
         "path": playlist_path,
         "extractor_config": args.extractor_config,
-        **args.extra_playlist_data,
     }
     _add(args, objects.dict_filter_bool(entry))

xklb/db_playlists.py

@@ -3,7 +3,7 @@
 
 from xklb import gdl_backend, tube_backend, usage
 from xklb.media import media_printer
-from xklb.utils import arg_utils, consts, db_utils, iterables, nums, objects, printing, processes, sql_utils
+from xklb.utils import arg_utils, consts, db_utils, iterables, nums, objects, printing, processes, sql_utils, web
 from xklb.utils.consts import SC, DBType
 from xklb.utils.log_utils import log
 
@@ -43,6 +43,14 @@ def parse_args():
         const=DBType.image,
         help="Use image downloader",
     )
+    subp_profile.add_argument(
+        "--filesystem",
+        "--fs",
+        action="store_const",
+        dest="profile",
+        const=DBType.filesystem,
+        help="Use filesystem downloader",
+    )
 
     parser.add_argument(
         "--extractor-config",
@@ -54,14 +62,7 @@ def parse_args():
         help="Add key/value pairs to override or extend downloader configuration",
     )
     parser.add_argument("--download-archive")
-    parser.add_argument("--extra-media-data", default={}, nargs=1, action=arg_utils.ArgparseDict, metavar="KEY=VALUE")
-    parser.add_argument(
-        "--extra-playlist-data",
-        default={},
-        nargs=1,
-        action=arg_utils.ArgparseDict,
-        metavar="KEY=VALUE",
-    )
+
     parser.add_argument("--safe", "-safe", action="store_true", help="Skip generic URLs")
     parser.add_argument("--same-domain", action="store_true", help="Choose a random domain to focus on")
 
@@ -184,7 +185,7 @@ def construct_query(args) -> Tuple[str, dict]:
             FROM media m
             LEFT JOIN playlists p on p.id = m.playlist_id
             WHERE 1=1
-                and COALESCE(m.time_downloaded,0) = 0
+                {'and COALESCE(m.time_downloaded,0) = 0' if 'time_downloaded' in m_columns else ''}
                 and COALESCE(m.time_deleted,0) = 0
                 {'and COALESCE(p.time_deleted, 0) = 0' if 'time_deleted' in pl_columns else ''}
                 and m.path like "http%"
@@ -273,7 +274,15 @@ def dl_download(args=None) -> None:
     if "limit" in args.defaults and "media" in args.db.table_names() and "webpath" in m_columns:
         if args.db.pop("SELECT 1 from media WHERE webpath is NULL and path in (select webpath from media) LIMIT 1"):
             with args.db.conn:
-                args.db.conn.execute("DELETE from media WHERE webpath is NULL and path in (select webpath from media)")
+                args.db.conn.execute(
+                    """
+                    DELETE from media WHERE webpath is NULL
+                    AND path in (
+                        select webpath from media
+                        WHERE error IS NULL OR error != 'Media check failed'
+                    )
+                    """
+                )
 
     args.blocklist_rules = []
     if "blocklist" in args.db.table_names():
@@ -330,6 +339,8 @@ def dl_download(args=None) -> None:
                 tube_backend.download(args, m)
             elif args.profile == DBType.image:
                 gdl_backend.download(args, m)
+            elif args.profile == DBType.filesystem:
+                web.download_url(m["path"], output_prefix=args.prefix)
             else:
                 raise NotImplementedError
         except Exception: