Skip to content

Commit 623bd90

Browse files
chapmanjacobdchapmanjacobd
committed
2.5.001
1 parent b850097 commit 623bd90

15 files changed

+124
-104
lines changed

‎.github/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ To stop playing press Ctrl+C in either the terminal or mpv
9595
<details><summary>List all subcommands</summary>
9696

9797
$ library
98-
xk media library subcommands (v2.4.011)
98+
xk media library subcommands (v2.5.001)
9999

100100
Create database subcommands:
101101
╭───────────────┬────────────────────────────────────────────────────╮

‎pdm.lock

Lines changed: 73 additions & 57 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎xklb/__init__.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
__version__ = "2.4.011"
1+
__version__ = "2.5.001"

‎xklb/db_media.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ def exists(args, path) -> bool:
1515
m_columns = db_utils.columns(args, "media")
1616
try:
1717
known = args.db.execute(
18-
f"select 1 from media where path=? or {'web' if 'webpath' in m_columns else ''}path=?",
18+
f"select 1 from media where path=? or {'webpath' if 'webpath' in m_columns else 'path'}=?",
1919
[str(path), str(path)],
2020
).fetchone()
2121
except sqlite3.OperationalError as e:
@@ -71,7 +71,7 @@ def consolidate(v: dict) -> Optional[dict]:
7171
upload_date = None
7272

7373
cv = {}
74-
cv["playlist_id"] = v.pop("playlist_id", None)
74+
cv["playlists_id"] = v.pop("playlists_id", None) # not to be confused with yt-dlp playlist_id
7575
cv["webpath"] = iterables.safe_unpack(
7676
v.pop("webpath", None),
7777
v.pop("webpage_url", None),
@@ -238,8 +238,8 @@ def download_add(
238238
if local_path and Path(local_path).exists():
239239
local_path = str(Path(local_path).resolve())
240240
fs_args = argparse.Namespace(
241-
profiles=args.profiles,
242-
scan_subtitles=DBType.video in args.profiles,
241+
profile=args.profile,
242+
scan_subtitles=args.profile == DBType.video,
243243
ocr=False,
244244
speech_recognition=False,
245245
delete_unplayable=False,

‎xklb/db_playlists.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,11 @@
99
playlists table
1010
profile = Type of extractor -- consts.DBType
1111
extractor_key = Name of the Extractor -- "Local", "Imgur", "YouTube"
12-
extractor_playlist_id = ID that the extractor uses to refer to playlists
12+
extractor_playlist_id = ID that the extractor uses to refer to playlists (yt-dlp playlist_id)
1313
1414
media table
1515
extractor_id = ID that the Extractor uses to refer to media
16-
playlist_id = Foreign key to playlists table
16+
playlists_id = Foreign key to playlists table
1717
"""
1818

1919

@@ -96,10 +96,10 @@ def exists(args, playlist_path) -> bool:
9696
return True
9797

9898

99-
def get_parentpath_playlist_id(args, playlist_path) -> Optional[int]:
99+
def get_parentpath_playlists_id(args, playlist_path) -> Optional[int]:
100100
try:
101101
known = args.db.pop(
102-
"select path from playlists where ? like path || '%' and path != ?",
102+
"select id from playlists where ? like path || '%' and path != ?",
103103
[str(playlist_path), str(playlist_path)],
104104
)
105105
except sqlite3.OperationalError as e:
@@ -127,7 +127,7 @@ def delete_subpath_playlists(args, playlist_path) -> Optional[int]:
127127
def add(args, playlist_path: str, info: dict, check_subpath=False, extractor_key=None) -> int:
128128
playlist_path = playlist_path.strip()
129129
if check_subpath:
130-
parentpath_playlist_id = get_parentpath_playlist_id(args, playlist_path)
130+
parentpath_playlist_id = get_parentpath_playlists_id(args, playlist_path)
131131
if parentpath_playlist_id:
132132
return parentpath_playlist_id
133133
else:
@@ -144,7 +144,7 @@ def media_exists(args, playlist_path, path) -> bool:
144144
m_columns = db_utils.columns(args, "media")
145145
try:
146146
known = args.db.execute(
147-
f"select 1 from media where playlist_id in (select id from playlists where path = ?) and (path=? or {'web' if 'webpath' in m_columns else ''}path=?)",
147+
f"select 1 from media where playlists_id in (select id from playlists where path = ?) and (path=? or {'webpath' if 'webpath' in m_columns else 'path'}=?)",
148148
[str(playlist_path), str(path), str(path)],
149149
).fetchone()
150150
except sqlite3.OperationalError as e:

‎xklb/dl_extract.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -168,9 +168,10 @@ def construct_query(args) -> Tuple[str, dict]:
168168
ORDER BY RANDOM()
169169
LIMIT 1
170170
)"""
171-
if "playlist_id" in m_columns:
171+
if "playlists_id" in m_columns:
172172
query = f"""select
173173
m.id
174+
, m.playlists_id
174175
, m.path
175176
, p.path playlist_path
176177
{', m.title' if 'title' in m_columns else ''}
@@ -184,7 +185,7 @@ def construct_query(args) -> Tuple[str, dict]:
184185
{', p.extractor_config' if 'extractor_config' in pl_columns else ''}
185186
, p.extractor_key
186187
FROM media m
187-
LEFT JOIN playlists p on p.id = m.playlist_id
188+
LEFT JOIN playlists p on p.id = m.playlists_id
188189
WHERE 1=1
189190
{'and COALESCE(m.time_downloaded,0) = 0' if 'time_downloaded' in m_columns else ''}
190191
and COALESCE(m.time_deleted,0) = 0
@@ -342,7 +343,7 @@ def dl_download(args=None) -> None:
342343
gdl_backend.download(args, m)
343344
elif args.profile == DBType.filesystem:
344345
local_path = web.download_url(m["path"], output_prefix=args.prefix, relative=args.relative)
345-
db_media.download_add(args, m["path"], {}, local_path)
346+
db_media.download_add(args, m["path"], m, local_path)
346347
else:
347348
raise NotImplementedError
348349
except Exception:

‎xklb/fs_extract.py

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -168,12 +168,13 @@ def extract_metadata(mp_args, path) -> Optional[Dict[str, int]]:
168168
}
169169

170170
ext = path.rsplit(".", 1)[-1].lower()
171+
is_scan_all_files = getattr(mp_args, "scan_all_files", False)
171172

172-
if DBType.audio in mp_args.profiles and (
173-
ext in (consts.AUDIO_ONLY_EXTENSIONS | consts.VIDEO_EXTENSIONS) or mp_args.scan_all_files
173+
if objects.is_profile(mp_args, DBType.audio) and (
174+
ext in (consts.AUDIO_ONLY_EXTENSIONS | consts.VIDEO_EXTENSIONS) or is_scan_all_files
174175
):
175176
media |= av.munge_av_tags(mp_args, path)
176-
elif DBType.video in mp_args.profiles and (ext in consts.VIDEO_EXTENSIONS or mp_args.scan_all_files):
177+
elif objects.is_profile(mp_args, DBType.video) and (ext in consts.VIDEO_EXTENSIONS or is_scan_all_files):
177178
media |= av.munge_av_tags(mp_args, path)
178179

179180
if not Path(path).exists():
@@ -184,7 +185,7 @@ def extract_metadata(mp_args, path) -> Optional[Dict[str, int]]:
184185
text_exts |= consts.OCR_EXTENSIONS
185186
if mp_args.speech_recognition:
186187
text_exts |= consts.SPEECH_RECOGNITION_EXTENSIONS
187-
if DBType.text in mp_args.profiles and (ext in text_exts or mp_args.scan_all_files):
188+
if objects.is_profile(mp_args, DBType.text) and (ext in text_exts or is_scan_all_files):
188189
try:
189190
start = timer()
190191
if any([mp_args.ocr, mp_args.speech_recognition]):
@@ -201,7 +202,7 @@ def extract_metadata(mp_args, path) -> Optional[Dict[str, int]]:
201202
media["hash"] = sample_hash.sample_hash_file(path)
202203

203204
if getattr(mp_args, "process", False):
204-
if DBType.audio in mp_args.profiles and Path(path).suffix not in [".opus", ".mka"]:
205+
if objects.is_profile(mp_args, DBType.audio) and Path(path).suffix not in [".opus", ".mka"]:
205206
path = media["path"] = process_audio.process_path(
206207
path, split_longer_than=2160 if "audiobook" in path.lower() else None
207208
)
@@ -223,7 +224,7 @@ def clean_up_temp_dirs():
223224

224225

225226
def extract_chunk(args, media) -> None:
226-
if DBType.image in args.profiles:
227+
if objects.is_profile(args, DBType.image):
227228
media = books.extract_image_metadata_chunk(media)
228229

229230
if args.scan_subtitles:
@@ -246,7 +247,7 @@ def extract_chunk(args, media) -> None:
246247

247248
captions.append(caption)
248249

249-
media = [{"playlist_id": args.playlist_id, **d} for d in media]
250+
media = [{"playlists_id": args.playlists_id, **d} for d in media]
250251
media = iterables.list_dict_filter_bool(media)
251252
args.db["media"].insert_all(media, pk="id", alter=True, replace=True)
252253

@@ -393,7 +394,7 @@ def scan_path(args, path_str: str) -> int:
393394
),
394395
"time_deleted": 0,
395396
}
396-
args.playlist_id = db_playlists.add(args, str(path), info, check_subpath=True)
397+
args.playlists_id = db_playlists.add(args, str(path), info, check_subpath=True)
397398

398399
print(f"[{path}] Building file list...")
399400
new_files = find_new_files(args, path)

‎xklb/gdl_backend.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -121,14 +121,15 @@ def download(args, m):
121121
job = gallery_dl.job.DownloadJob(webpath)
122122
except gallery_dl.exception.NoExtractorError:
123123
log.info("[%s]: NoExtractorError", webpath) # RecoverableError
124-
db_media.download_add(args, webpath, error="NoExtractorError")
124+
db_media.download_add(args, webpath, m, error="NoExtractorError")
125125
return
126126

127127
job_status = job.run()
128128
errors = parse_gdl_job_status(job_status, webpath, ignore_errors=args.ignore_errors)
129129

130130
info = getattr(job.pathfmt, "kwdict", None)
131131
if info:
132+
info = {**m, **info}
132133
info["path"] = webpath
133134

134135
local_path = getattr(job.pathfmt, "path", "") or None
@@ -219,16 +220,16 @@ def get_playlist_metadata(args, playlist_path):
219220

220221
log.debug("webpath == playlist_path" if webpath == playlist_path else "webpath != playlist_path")
221222

222-
playlist_id = None
223+
playlists_id = None
223224
if is_playlist:
224-
playlist_id = db_playlists.add(args, playlist_path, info)
225+
playlists_id = db_playlists.add(args, playlist_path, info)
225226
else:
226227
log.warning("Importing playlist-less media %s", playlist_path)
227228

228229
if db_media.exists(args, webpath):
229230
log.warning("Media already exists")
230231

231-
info = {**info, "playlist_id": playlist_id, "webpath": webpath}
232+
info = {**info, "playlists_id": playlists_id, "webpath": webpath}
232233
db_media.playlist_media_add(
233234
args,
234235
playlist_path,

‎xklb/reddit_extract.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -217,11 +217,11 @@ def since_last_created(args, playlist_path):
217217
"""
218218
select max(time_created)
219219
from (
220-
select time_created, playlist_id from reddit_posts
220+
select time_created, playlists_id from reddit_posts
221221
UNION ALL
222-
select time_created, playlist_id from media
222+
select time_created, playlists_id from media
223223
)
224-
where playlist_id = (select id from playlists where path = ?)
224+
where playlists_id = (select id from playlists where path = ?)
225225
""",
226226
[playlist_path],
227227
ignore_errors=["no such column", "no such table"],

‎xklb/scripts/block.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,7 +172,7 @@ def block(args=None) -> None:
172172
f"""UPDATE media
173173
SET time_deleted={consts.APPLICATION_START}
174174
WHERE path LIKE 'http%'
175-
AND playlist_id in (
175+
AND playlists_id in (
176176
SELECT id from playlists
177177
WHERE path IN ("""
178178
+ ",".join(["?"] * len(playlist_paths))
@@ -185,7 +185,7 @@ def block(args=None) -> None:
185185
"""SELECT path, size FROM media
186186
WHERE coalesce(time_deleted, 0)=0
187187
AND time_downloaded > 0
188-
AND playlist_id in (
188+
AND playlists_id in (
189189
SELECT id from playlists
190190
WHERE path IN ("""
191191
+ ",".join(["?"] * len(playlist_paths))

‎xklb/scripts/links_db.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ def add_playlist(args, path):
160160
},
161161
"time_deleted": 0,
162162
}
163-
args.playlist_id = db_playlists.add(args, str(path), info)
163+
args.playlists_id = db_playlists.add(args, str(path), info)
164164

165165

166166
def consolidate_media(args, path: str) -> dict:

‎xklb/scripts/merge_online_local.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ def get_duplicates(args) -> List[dict]:
3232
and extractor_id is not null
3333
and extractor_id != ""
3434
and coalesce(time_deleted, 0)=0
35-
and playlist_id in (
35+
and playlists_id in (
3636
SELECT id from playlists
3737
WHERE extractor_key
3838
NOT IN ('Local', 'NBCStations', 'TedTalk', 'ThisAmericanLife', 'InfoQ', 'NFB', 'KickStarter')
@@ -54,7 +54,7 @@ def get_duplicates(args) -> List[dict]:
5454
and title is null
5555
) m2
5656
JOIN media_fts on m2.id = media_fts.rowid
57-
JOIN playlists p2 on p2.id = m2.playlist_id
57+
JOIN playlists p2 on p2.id = m2.playlists_id
5858
WHERE p2.extractor_key = 'Local'
5959
AND media_fts.path MATCH '"'||m1.extractor_id||'"'
6060
AND m2.PATH LIKE '%['||m1.extractor_id||']%'

‎xklb/scripts/playlists.py

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -119,15 +119,15 @@ def delete_playlists(args, playlists) -> None:
119119
with args.db.conn:
120120
cursor = args.db.conn.execute(
121121
"""DELETE from media where
122-
playlist_id in (
122+
playlists_id in (
123123
SELECT id from playlists
124124
WHERE path IN ("""
125125
+ ",".join(["?"] * len(online_media))
126126
+ "))",
127127
(*online_media,),
128128
)
129129
deleted_media_count += cursor.rowcount
130-
except sqlite3.OperationalError: # no such column: playlist_id
130+
except sqlite3.OperationalError: # no such column: playlists_id
131131
pass
132132

133133
local_media = [p.rstrip(os.sep) for p in playlists if not p.startswith("http")]
@@ -146,7 +146,7 @@ def playlists() -> None:
146146
m_columns = db_utils.columns(args, "media")
147147
query, bindings = construct_query(args)
148148

149-
if "playlist_id" in m_columns:
149+
if "playlists_id" in m_columns:
150150
query = f"""
151151
select
152152
coalesce(p.path, "Playlist-less media") path
@@ -158,8 +158,8 @@ def playlists() -> None:
158158
{', sum(m.size) size' if 'size' in m_columns else ''}
159159
, count(*) count
160160
from media m
161-
join ({query}) p on p.id = m.playlist_id
162-
group by m.playlist_id, coalesce(p.path, "Playlist-less media")
161+
join ({query}) p on p.id = m.playlists_id
162+
group by m.playlists_id, coalesce(p.path, "Playlist-less media")
163163
order by count, p.path
164164
"""
165165

‎xklb/tube_backend.py

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -139,8 +139,7 @@ def run(self, info) -> Tuple[list, dict]: # pylint: disable=arguments-renamed
139139
log.warning("Importing playlist-less media %s", playlist_path)
140140
else:
141141
# add sub-playlist
142-
playlist_id = db_playlists.add(args, playlist_path, info, extractor_key=extractor_key)
143-
entry["playlist_id"] = playlist_id
142+
entry["playlists_id"] = db_playlists.add(args, playlist_path, info, extractor_key=extractor_key)
144143
log.info("playlists.add2 %s", t.elapsed())
145144

146145
db_media.playlist_media_add(args, webpath, entry) # type: ignore
@@ -213,13 +212,13 @@ def get_extra_metadata(args, playlist_path, playlist_dl_opts=None) -> Optional[L
213212
SELECT
214213
id
215214
, path
216-
, playlist_id
215+
, playlists_id
217216
FROM media
218217
WHERE 1=1
219218
AND COALESCE(time_deleted, 0)=0
220219
{'and width is null' if 'width' in m_columns else ''}
221220
and path not like '%playlist%'
222-
and playlist_id = (select id from playlists where path = ?)
221+
and playlists_id = (select id from playlists where path = ?)
223222
ORDER by random()
224223
""",
225224
[playlist_path],
@@ -228,7 +227,7 @@ def get_extra_metadata(args, playlist_path, playlist_dl_opts=None) -> Optional[L
228227
videos = []
229228

230229
current_video_count = 0
231-
for id, path, playlist_id in videos:
230+
for id, path, playlists_id in videos:
232231
entry = ydl.extract_info(path)
233232
if entry is None:
234233
continue
@@ -259,7 +258,7 @@ def get_extra_metadata(args, playlist_path, playlist_dl_opts=None) -> Optional[L
259258
args.db["captions"].insert_all(captions, alter=True)
260259

261260
entry["id"] = id
262-
entry["playlist_id"] = playlist_id
261+
entry["playlists_id"] = playlists_id
263262
entry["chapter_count"] = chapter_count
264263

265264
db_media.playlist_media_add(args, path, entry)
@@ -457,7 +456,9 @@ def blocklist_check(info, *pargs, incomplete):
457456

458457
if info is None:
459458
log.debug("[%s]: yt-dlp returned no info", webpath)
459+
info = m
460460
else:
461+
info = {**m, **info}
461462
local_path = info.get("local_path", None)
462463
if args.profile == DBType.audio:
463464
local_path = ydl.prepare_filename({**info, "ext": args.ext})

‎xklb/utils/consts.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -351,7 +351,7 @@ def reddit_frequency(frequency) -> str:
351351
"requested_downloads",
352352
"thumbnails",
353353
"playlist_count",
354-
"playlist_id",
354+
"playlists_id",
355355
"playlist_title",
356356
"playlist_uploader",
357357
"audio_channels",

0 commit comments

Comments
 (0)