2.5.002

Author: chapmanjacobd

.github/README.md

@@ -95,7 +95,7 @@ To stop playing press Ctrl+C in either the terminal or mpv
 <details><summary>List all subcommands</summary>
 
     $ library
-    xk media library subcommands (v2.5.001)
+    xk media library subcommands (v2.5.002)
 
     Create database subcommands:
     ╭───────────────┬────────────────────────────────────────────────────╮
@@ -240,7 +240,7 @@ To stop playing press Ctrl+C in either the terminal or mpv
     ╭────────────────────┬────────────────────────────────────────╮
     │ dedupe-db          │ Dedupe SQLITE tables                   │
     ├────────────────────┼────────────────────────────────────────┤
-    │ dedupe             │ Dedupe similar media                   │
+    │ dedupe-media       │ Dedupe similar media                   │
     ├────────────────────┼────────────────────────────────────────┤
     │ merge-online-local │ Merge online and local data            │
     ├────────────────────┼────────────────────────────────────────┤
@@ -1584,10 +1584,15 @@ BTW, for some cols like time_deleted you'll need to specify a where clause so th
 
         library download dl.db --prefix ~/output/path/root/
 
-    Limit downloads to a specified playlist URLs or substring
+    Limit downloads to a specified playlist URLs or substring (TODO: https://github.com/chapmanjacobd/library/issues/31)
 
         library download dl.db https://www.youtube.com/c/BlenderFoundation/videos
 
+    Limit downloads to a specified video URLs or substring
+
+        library download dl.db --include https://www.youtube.com/watch?v=YE7VzlLtp-4
+        library download dl.db -s https://www.youtube.com/watch?v=YE7VzlLtp-4  # equivalent
+
     Maximizing the variety of subdomains
 
         library download photos.db --photos --image --sort "ROW_NUMBER() OVER ( PARTITION BY SUBSTR(m.path, INSTR(m.path, '//') + 2, INSTR( SUBSTR(m.path, INSTR(m.path, '//') + 2), '/') - 1) )"
@@ -2519,11 +2524,11 @@ BTW, for some cols like time_deleted you'll need to specify a where clause so th
 
 </details>
 
-###### dedupe
+###### dedupe-media
 
 <details><summary>Dedupe similar media</summary>
 
-    $ library dedupe -h
+    $ library dedupe-media -h
     usage: library [--audio | --id | --title | --filesystem] [--only-soft-delete] [--limit LIMIT] DATABASE
 
     Dedupe your files (not to be confused with the dedupe-db subcommand)

xklb/__init__.py

@@ -1 +1 @@
-__version__ = "2.5.001"
+__version__ = "2.5.002"

xklb/dl_extract.py

@@ -171,11 +171,11 @@ def construct_query(args) -> Tuple[str, dict]:
     if "playlists_id" in m_columns:
         # TODO: filter out downloads based on args.playlists
         '''AND playlists_id in (
-                SELECT id from playlists
-                WHERE path IN ("""
-            + ",".join(["?"] * len(playlist_paths))
-            + "))",
-            (*playlist_paths,),
+            SELECT id from playlists
+            WHERE path IN ("""
+        + ",".join(["?"] * len(playlist_paths))
+        + "))",
+        (*playlist_paths,),
         '''
         # TODO --- https://github.com/chapmanjacobd/library/issues/31
 

xklb/lb.py

@@ -83,7 +83,7 @@
     },
     "Single database enrichment subcommands": {
         "dedupe_db": "Dedupe SQLITE tables",
-        "dedupe": "Dedupe similar media",
+        "dedupe_media": "Dedupe similar media",
         "merge_online_local": "Merge online and local data",
         "mpv_watchlater": "Import mpv watchlater files to history",
         "reddit_selftext": "Copy selftext links to media table",

xklb/scripts/web_add.py

@@ -1,4 +1,4 @@
-import argparse, json, random, sys, time
+import argparse, json, random, re, sys, time
 from pathlib import Path
 from typing import Set
 from urllib.parse import urlparse
@@ -204,6 +204,22 @@ def add_media(args, media):
     args.db["media"].insert_all(media, pk="id", alter=True, replace=True)
 
 
+def is_web_dir(path):
+    if path.endswith("/"):
+        return True
+
+    patterns = [
+        r"/index\.php\?dir=",
+        r"/index\.php$",
+        r"/index\.html?$",
+    ]
+    for pattern in patterns:
+        if re.search(pattern, path, re.IGNORECASE):
+            return True
+
+    return False
+
+
 def spider(args, paths: Set):
     get_urls = iterables.return_unique(extract_links.get_inner_urls)
 
@@ -220,7 +236,7 @@ def spider(args, paths: Set):
             f"Pages to scan {len(paths)} link scan: {new_media_count} new [{len(known_paths)} known]"
         )
 
-        if path.endswith("/"):
+        if is_web_dir(path):
             for a_ref in get_urls(args, path):
                 if a_ref is None:
                     break
@@ -231,10 +247,9 @@ def spider(args, paths: Set):
 
                 if link in (paths | traversed_paths):
                     continue
-                if not web.is_subpath(path, link):
-                    continue
-                if link.endswith("/"):
-                    paths.add(link)
+                if is_web_dir(link):
+                    if web.is_subpath(path, link):
+                        paths.add(link)
                     continue
 
                 if db_media.exists(args, link):
@@ -295,7 +310,6 @@ def spider(args, paths: Set):
 def add_playlist(args, path):
     info = {
         "hostname": urlparse(path).hostname,
-        "time_created": consts.APPLICATION_START,
         "extractor_key": "WebFolder",
         "extractor_config": {k: v for k, v in args.__dict__.items() if k not in ["db", "database", "verbose", "paths"]},
         "time_deleted": 0,
@@ -325,7 +339,7 @@ def web_add(args=None) -> None:
         try:
             for playlist_path in arg_utils.gen_paths(args):
                 spider(args, {playlist_path})
-                if playlist_path.endswith("/"):
+                if is_web_dir(playlist_path):
                     add_playlist(args, playlist_path)
 
         finally:

xklb/usage.py

@@ -1044,7 +1044,7 @@ def play(action) -> str:
 
         library copy-play-counts audio.db phone.db --source-prefix /storage/6E7B-7DCE/d --target-prefix /mnt/d
 """
-dedupe = """library [--audio | --id | --title | --filesystem] [--only-soft-delete] [--limit LIMIT] DATABASE
+dedupe_media = """library [--audio | --id | --title | --filesystem] [--only-soft-delete] [--limit LIMIT] DATABASE
 
     Dedupe your files (not to be confused with the dedupe-db subcommand)
 

xklb/utils/web.py

@@ -610,7 +610,7 @@ def is_subpath(parent_url, child_url):
     if child.scheme != parent.scheme or child.netloc != parent.netloc:
         return False
 
-    return child.path.startswith(parent.path.rstrip("/") + "/")
+    return child_url.startswith(parent_url)
 
 
 def remove_apache_sorting_params(url):