forked from mrusse/soularr
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathsoularr.py
More file actions
1467 lines (1259 loc) · 58.6 KB
/
Copy pathsoularr.py
File metadata and controls
1467 lines (1259 loc) · 58.6 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/usr/bin/env python
import argparse
import math
import re
import os
import sys
import time
import shutil
import difflib
import operator
import configparser
import logging
import json
from datetime import datetime
import copy
import music_tag
import slskd_api
from pyarr import LidarrAPI
from slskd_api.apis import users
class EnvInterpolation(configparser.ExtendedInterpolation):
"""
Interpolation which expands environment variables in values.
Borrowed from https://stackoverflow.com/a/68068943
"""
def before_read(self, parser, section, option, value):
value = super().before_read(parser, section, option, value)
return os.path.expandvars(value)
# Allows backwards compatibility for users updating an older version of Soularr
# without using the new [Logging] section in the config.ini file.
DEFAULT_LOGGING_CONF = {
"level": "INFO",
"format": "[%(levelname)s|%(module)s|L%(lineno)d] %(asctime)s: %(message)s",
"datefmt": "%Y-%m-%dT%H:%M:%S%z",
}
# === API Clients & Logging ===
lidarr = None
slskd = None
config = None
logger = logging.getLogger("soularr")
# === Configuration Constants ===
slskd_api_key = None
lidarr_api_key = None
lidarr_download_dir = None
lidarr_disable_sync = None
slskd_download_dir = None
lidarr_host_url = None
slskd_host_url = None
stalled_timeout = None
remote_queue_timeout = None
delete_searches = None
slskd_url_base = None
ignored_users = []
search_type = None
search_source = None
download_filtering = None
use_extension_whitelist = None
extensions_whitelist = []
rename_download_folders = None
search_sources = []
minimum_match_ratio = None
minimum_search_interval = None
page_size = None
failed_import_denylist = None
failed_import_denylist_file_path = None
use_most_common_tracknum = None
allow_multi_disc = None
accepted_countries = []
skip_region_check = None
accepted_formats = []
use_selected_lidarr_release = None
allowed_filetypes = []
lock_file_path = None
config_file_path = None
current_page_file_path = None
lidarr_sort_dir = None
lidarr_sort_key = None
search_blacklist = []
# === Runtime State & Caches ===
search_cache = {}
folder_cache = {}
broken_user = []
def album_match(lidarr_tracks, slskd_tracks, username, filetype):
counted = []
total_match = 0.0
lidarr_album = lidarr.get_album(lidarr_tracks[0]["albumId"])
lidarr_album_name = lidarr_album["title"]
lidarr_artist_name = lidarr_album["artist"]["artistName"]
for lidarr_track in lidarr_tracks:
lidarr_filename = lidarr_track["title"] + "." + filetype.split(" ")[0]
best_match = 0.0
for slskd_track in slskd_tracks:
slskd_filename = slskd_track["filename"]
# Try to match the ratio with the exact filenames
ratio = difflib.SequenceMatcher(None, lidarr_filename, slskd_filename).ratio()
# If ratio is a bad match try and split off (with " " as the separator) the garbage at the start of the slskd_filename and try again
ratio = check_ratio(" ", ratio, lidarr_filename, slskd_filename)
# Same but with "_" as the separator
ratio = check_ratio("_", ratio, lidarr_filename, slskd_filename)
# Same checks but preappend album name.
ratio = check_ratio("", ratio, lidarr_album_name + " " + lidarr_filename, slskd_filename)
ratio = check_ratio(" ", ratio, lidarr_album_name + " " + lidarr_filename, slskd_filename)
ratio = check_ratio("_", ratio, lidarr_album_name + " " + lidarr_filename, slskd_filename)
if ratio > best_match:
best_match = ratio
if best_match > minimum_match_ratio:
counted.append(lidarr_filename)
total_match += best_match
if len(counted) == len(lidarr_tracks) and username not in ignored_users:
logger.info(f"Found match from user: {username} for {len(counted)} tracks! Track attributes: {filetype}")
logger.info(f"Average sequence match ratio: {total_match / len(counted)}")
logger.info("SUCCESSFUL MATCH")
logger.info("-------------------")
return True
return False
def check_ratio(separator, ratio, lidarr_filename, slskd_filename):
if ratio < minimum_match_ratio:
if separator != "":
lidarr_filename_word_count = len(lidarr_filename.split()) * -1
truncated_slskd_filename = " ".join(slskd_filename.split(separator)[lidarr_filename_word_count:])
ratio = difflib.SequenceMatcher(None, lidarr_filename, truncated_slskd_filename).ratio()
else:
ratio = difflib.SequenceMatcher(None, lidarr_filename, slskd_filename).ratio()
return ratio
return ratio
def album_track_num(directory):
files = directory["files"]
allowed_filetypes_no_attributes = [item.split(" ")[0] for item in allowed_filetypes]
count = 0
index = -1
filetype = ""
for file in files:
if file["filename"].split(".")[-1] in allowed_filetypes_no_attributes:
new_index = allowed_filetypes_no_attributes.index(file["filename"].split(".")[-1])
if index == -1:
index = new_index
filetype = allowed_filetypes_no_attributes[index]
elif new_index != index:
filetype = ""
break
count += 1
return_data = {"count": count, "filetype": filetype}
return return_data
def sanitize_folder_name(folder_name):
valid_characters = re.sub(r'[<>:."/\\|?*]', "", folder_name)
return valid_characters.strip()
def cancel_and_delete(files):
for file in files:
try:
slskd.transfers.cancel_download(username=file["username"], id=file["id"])
except Exception:
logger.warning(f"Failed to cancel download {file['filename']} for {file['username']}", exc_info=True)
delete_dir = file["file_dir"].split("\\")[-1]
os.chdir(slskd_download_dir)
if os.path.exists(delete_dir):
shutil.rmtree(delete_dir)
def release_trackcount_mode(releases):
track_count = {}
for release in releases:
trackcount = release["trackCount"]
if trackcount in track_count:
track_count[trackcount] += 1
else:
track_count[trackcount] = 1
most_common_trackcount = None
max_count = 0
for trackcount, count in track_count.items():
if count > max_count:
max_count = count
most_common_trackcount = trackcount
return most_common_trackcount
def choose_release(artist_name, releases):
if use_selected_lidarr_release:
for release in releases:
if release.get("monitored"):
logger.info(f"Using selected Lidarr release for {artist_name}: {release['format']}, {release['trackCount']} tracks, ID: {release['id']}")
return release
most_common_trackcount = release_trackcount_mode(releases)
for release in releases:
country = release["country"][0] if release["country"] else None
if release["format"][1] == "x" and allow_multi_disc:
format_accepted = release["format"].split("x", 1)[1] in accepted_formats
else:
format_accepted = release["format"] in accepted_formats
if use_most_common_tracknum:
if release["trackCount"] == most_common_trackcount:
track_count_bool = True
else:
track_count_bool = False
else:
track_count_bool = True
if (skip_region_check or country in accepted_countries) and format_accepted and release["status"] == "Official" and track_count_bool:
logger.info(
", ".join(
[
f"Selected release for {artist_name}: {release['status']}",
str(country),
release["format"],
f"Mediums: {release['mediumCount']}",
f"Tracks: {release['trackCount']}",
f"ID: {release['id']}",
]
)
)
return release
if use_most_common_tracknum:
for release in releases:
if release["trackCount"] == most_common_trackcount:
return release
else:
default_release = releases[0]
else:
default_release = releases[0]
return default_release
def verify_filetype(file, allowed_filetype):
current_filetype = file["filename"].split(".")[-1]
bitdepth = None
samplerate = None
bitrate = None
if "bitRate" in file:
bitrate = file["bitRate"]
if "sampleRate" in file:
samplerate = file["sampleRate"]
if "bitDepth" in file:
bitdepth = file["bitDepth"]
# Check if the types match up for the current files type and the current type from the config
if current_filetype == allowed_filetype.split(" ")[0]:
# Check if the current type from the config specifies other attributes than the filetype (bitrate etc)
if " " in allowed_filetype:
selected_attributes = allowed_filetype.split(" ")[1]
# If it is a bitdepth/samplerate pair instead of a simple bitrate
if "/" in selected_attributes:
selected_bitdepth = selected_attributes.split("/")[0]
try:
selected_samplerate = str(int(float(selected_attributes.split("/")[1]) * 1000))
except (ValueError, IndexError):
logger.warning("Invalid samplerate in selected_attributes")
return False
if bitdepth and samplerate:
if str(bitdepth) == str(selected_bitdepth) and str(samplerate) == str(selected_samplerate):
return True
else:
return False
# If it is a bitrate
else:
selected_bitrate = selected_attributes
if bitrate:
if str(bitrate) == str(selected_bitrate):
return True
else:
return False
# If no bitrate or other info then it is a match so return true
else:
return True
else:
return False
def download_filter(allowed_filetype, directory):
"""
Filters the directory listing from SLSKD using the filetype whitelist.
If not using the whitelist it will only return the audio files of the allowed filetype.
This is to prevent downloading m3u,cue,txt,jpg,etc. files that are sometimes stored in
the same folders as the music files.
"""
logging.debug("download_filtering")
if download_filtering:
whitelist = [] # Init an empty list to take just the allowed_filetype
if use_extension_whitelist:
whitelist = copy.deepcopy(extensions_whitelist) # Copy the whitelist to allow us to append the allowed_filetype
whitelist.append(allowed_filetype.split(" ")[0])
unwanted = []
logger.debug(f"Accepted extensions: {whitelist}")
for file in directory["files"]:
for extension in whitelist:
if file["filename"].split(".")[-1].lower() == extension.lower():
break # Jump out and don't add wanted files to the unwanted list
else:
unwanted.append(file["filename"]) # Add to list of files to remove from the wanted list
logger.debug(f"Unwanted file: {file['filename']}")
if len(unwanted) > 0:
temp = []
logger.debug(f"Unwanted Files: {unwanted}")
for file in directory["files"]:
if file["filename"] not in unwanted:
logger.debug(f"Added file to queue: {file['filename']}")
temp.append(file) # Build the new list of files
directory["files"] = temp
for files in temp:
logger.debug(f"File in final list: {files['filename']}")
return directory # Return the modified list
return directory # If we didn't find unwanted files or we aren't filtering just return the original list
def check_for_match(tracks, allowed_filetype, file_dirs, username):
"""
Does the actual match checking on a single disk/album.
"""
logger.debug(f"Current broken users {broken_user}")
if username in broken_user:
return False, {}, ""
for file_dir in file_dirs:
if username not in folder_cache:
logger.debug(f"Add user to cache: {username}")
folder_cache[username] = {}
if file_dir not in folder_cache[username]:
logger.info(f"User: {username} Folder: {file_dir} not in cache. Fetching from SLSKD")
version = slskd.application.version()
version_check = slskd_version_check(version)
if not version_check:
logger.info(f"Error checking slskd version number: {version}. Version check > 0.22.2: {version_check}. This would most likely be fixed by updating your slskd.")
try:
if version_check:
directory = slskd.users.directory(username=username, directory=file_dir)[0]
else:
directory = slskd.users.directory(username=username, directory=file_dir)
except Exception:
logger.exception(f'Error getting directory from user: "{username}"')
broken_user.append(username)
logger.debug(f"Updated broken users {broken_user}")
return False, {}, ""
folder_cache[username][file_dir] = copy.deepcopy(directory)
else:
logger.info(f"User: {username} Folder: {file_dir} in cache. Using cached value")
directory = copy.deepcopy(folder_cache[username][file_dir])
track_num = len(tracks)
tracks_info = album_track_num(directory)
if tracks_info["count"] == track_num and tracks_info["filetype"] != "":
if album_match(tracks, directory["files"], username, allowed_filetype):
return True, directory, file_dir
else:
continue
return False, {}, ""
def is_blacklisted(title: str) -> bool:
blacklist = config.get("Lidarr", "title_blacklist", fallback="").lower().split(",")
for word in blacklist:
if word != "" and word in title.lower():
logger.info(f"Skipping {title} due to blacklisted word: {word}")
return True
return False
def filter_list(albums):
"""
Helper to do all the various filtering in one go and in one place. Same net effect as the previous multi-stage approach
Just neater and easier to work on.
"""
temp_list = copy.deepcopy(albums)
if failed_import_denylist:
import_denylist = load_failed_import_denylist(failed_import_denylist_file_path)
filtered_temp = []
for album in temp_list:
if str(album["id"]) in import_denylist:
logger.info(f"Skipping failed import album: {album['artist']['artistName']} - {album['title']} (ID: {album['id']})")
else:
filtered_temp.append(album)
temp_list = filtered_temp
list_to_download = []
for album in temp_list:
if is_blacklisted(album["title"]):
logger.info(f"Skipping blacklisted album: {album['artist']['artistName']} - {album['title']} (ID: {album['id']}")
continue
else:
list_to_download.append(album)
if len(list_to_download) > 0:
return list_to_download
else:
return None
def search_for_album(album):
album_title = album["title"]
artist_name = album["artist"]["artistName"]
album_id = album["id"]
if len(album_title) == 1: # Need to add some code to wrangle specific artist names in here.. ;)
query = artist_name + " " + album_title
else:
query = artist_name + " " + album_title if config.getboolean("Search Settings", "album_prepend_artist", fallback=False) else album_title
original_query = query
for word in search_blacklist:
if word:
# Case-insensitive replacement
pattern = re.compile(re.escape(word), re.IGNORECASE)
query = pattern.sub("", query)
# Clean up double spaces
query = " ".join(query.split())
if query != original_query:
logger.info(f"Filtered search query: '{original_query}' -> '{query}'")
logger.info(f"Searching for album: {query}")
try:
search = slskd.searches.search_text(
searchText=query,
searchTimeout=config.getint("Search Settings", "search_timeout", fallback=5000),
filterResponses=True,
maximumPeerQueueLength=config.getint("Search Settings", "maximum_peer_queue", fallback=50),
minimumPeerUploadSpeed=config.getint("Search Settings", "minimum_peer_upload_speed", fallback=0),
)
except Exception:
logger.exception(f"Failed to perform search via SLSKD: {query}")
return False
# Add timeout here to increase reliability with Slskd. Sometimes it doesn't update search status fast enough. More of an issue with lots of historical searches in slskd
time.sleep(5)
start_time = time.time()
while True:
if slskd.searches.state(search["id"], False)["state"] != "InProgress": # Added False here as we don't want the search results here. Just the state.
break
time.sleep(1)
if (time.time() - start_time) > config.getint("Search Settings", "search_timeout", fallback=5000):
logger.error("Failed to perform search via SLSKD due to timeout on search results.")
return False
search_results = slskd.searches.search_responses(search["id"]) # We use this API call twice. Let's just cache it locally.
logger.info(f"Search returned {len(search_results)} results")
if delete_searches:
slskd.searches.delete(search["id"])
if not len(search_results) > 0:
return False
if album_id not in search_cache:
search_cache[album_id] = {} # This is so we can check for matches we missed or if a user goes offline during our download
for result in search_results: # Switching to cached version. One less API call
username = result["username"]
if username not in search_cache[album_id]:
# If we don't currently have a cache for a user set one up
search_cache[album_id][username] = {}
logger.info(f"Caching and truncating results for user: {username}")
init_files = result["files"] # init_files short for initial files. Before truncating
# Search the returned files and only cache files that are of the allowed_filetypes
for file in init_files:
file_dir = file["filename"].rsplit("\\", 1)[0] # split dir/filenames on \
for allowed_filetype in allowed_filetypes:
if verify_filetype(file, allowed_filetype): # Check the filename for an allowed type
if allowed_filetype not in search_cache[album_id][username]:
search_cache[album_id][username][allowed_filetype] = [] # Init the cache for this allowed filetype
if file_dir not in search_cache[album_id][username][allowed_filetype]:
search_cache[album_id][username][allowed_filetype].append(file_dir)
return True
def slskd_do_enqueue(username, files, file_dir):
"""
Takes a list of files to download and returns a list of files that were successfully added to the download queue
It also adds to each file the details needed to track that specific file.
"""
downloads = []
try:
enqueue = slskd.transfers.enqueue(username=username, files=files)
except Exception:
logger.debug("Enqueue failed", exc_info=True)
return None
if enqueue:
time.sleep(5)
try:
download_list = slskd.transfers.get_downloads(username=username)
except Exception:
logger.warning(f"Failed to get download status for {username} after enqueue", exc_info=True)
return None
for file in files:
for directory in download_list["directories"]:
if directory["directory"] == file_dir:
for slskd_file in directory["files"]:
if file["filename"] == slskd_file["filename"]:
file_details = {}
file_details["filename"] = file["filename"]
file_details["id"] = slskd_file["id"]
file_details["file_dir"] = file_dir
file_details["username"] = username
file_details["size"] = file["size"]
downloads.append(file_details)
return downloads
else:
return None
def slskd_download_status(downloads):
"""
Takes a list of files and gets the status of each file and packs it into the file object.
"""
ok = True
for file in downloads:
try:
status = slskd.transfers.get_download(file["username"], file["id"])
file["status"] = status
except Exception:
logger.exception(f"Error getting download status of {file['filename']}")
file["status"] = None
ok = False
return ok
def downloads_all_done(downloads):
"""
Checks the status of all the files in an album and returns a flag if all done as well
as returning a list of files with errors to check and how many files are in "Queued, Remotely"
"""
all_done = True
error_list = []
remote_queue = 0
for file in downloads:
if file["status"] is not None:
if not file["status"]["state"] == "Completed, Succeeded":
all_done = False
if file["status"]["state"] in [
"Completed, Cancelled",
"Completed, TimedOut",
"Completed, Errored",
"Completed, Rejected",
"Completed, Aborted",
]:
error_list.append(file)
if file["status"]["state"] == "Queued, Remotely":
remote_queue += 1
if not len(error_list) > 0:
error_list = None
return all_done, error_list, remote_queue
def try_enqueue(all_tracks, results, allowed_filetype):
"""
Single album match and enqueue.
Iterates over all users and enqueues a found match
"""
for username in results:
if allowed_filetype not in results[username]:
continue
logger.debug(f"Parsing result from user: {username}")
file_dirs = results[username][allowed_filetype]
found, directory, file_dir = check_for_match(all_tracks, allowed_filetype, file_dirs, username)
if found:
directory = download_filter(allowed_filetype, directory)
for i in range(0, len(directory["files"])):
directory["files"][i]["filename"] = file_dir + "\\" + directory["files"][i]["filename"]
try:
downloads = slskd_do_enqueue(username=username, files=directory["files"], file_dir=file_dir)
if downloads is not None:
return True, downloads
else:
album = lidarr.get_album(all_tracks[0]["albumId"])
album_name = album["title"]
artist_name = album["artist"]["artistName"]
logger.info(f"Failed to enqueue download to slskd for {artist_name} - {album_name} from {username}")
except Exception as e:
album = lidarr.get_album(all_tracks[0]["albumId"])
album_name = album["title"]
artist_name = album["artist"]["artistName"]
logger.warning(f"Exception enqueueing tracks: {e}")
logger.info(f"Exception enqueueing download to slskd for {artist_name} - {album_name} from {username}")
album = lidarr.get_album(all_tracks[0]["albumId"])
album_name = album["title"]
artist_name = album["artist"]["artistName"]
logger.info(f"Failed to enqueue {artist_name} - {album_name}")
return False, None
def try_multi_enqueue(release, all_tracks, results, allowed_filetype):
"""
This is the multi-disk/media path for locating and enqueueing an album
It does a flat search first. Then it does a split search.
Otherwise it's basically the same as the single album search.
"""
split_release = []
tmp_results = copy.deepcopy(results)
for media in release["media"]:
disk = {}
disk["source"] = None
disk["tracks"] = []
disk["disk_no"] = media["mediumNumber"]
disk["disk_count"] = len(release["media"])
for track in all_tracks:
if track["mediumNumber"] == media["mediumNumber"]:
disk["tracks"].append(track)
split_release.append(disk)
total = len(split_release)
count_found = 0
for disk in split_release:
for username in tmp_results:
if allowed_filetype not in tmp_results[username]:
continue
file_dirs = results[username][allowed_filetype]
found, directory, file_dir = check_for_match(disk["tracks"], allowed_filetype, file_dirs, username)
if found:
directory = download_filter(allowed_filetype, directory)
disk["source"] = (username, directory, file_dir)
count_found += 1
break
else:
return (
False,
None,
) # Only runs if we complete the loop without finding a source for the current disk regardless of how many other disks we located. All or nothing.
if count_found == total:
all_downloads = []
enqueued = 0
for disk in split_release:
username, directory, file_dir = disk["source"]
for i in range(0, len(directory["files"])):
directory["files"][i]["filename"] = file_dir + "\\" + directory["files"][i]["filename"]
try:
downloads = slskd_do_enqueue(username=username, files=directory["files"], file_dir=file_dir)
if downloads is not None:
for file in downloads:
file["disk_no"] = disk["disk_no"]
file["disk_count"] = disk["disk_count"]
all_downloads.extend(downloads)
enqueued += 1
else:
album = lidarr.get_album(all_tracks[0]["albumId"])
album_name = album["title"]
artist_name = album["artist"]["artistName"]
logger.info(f"Failed to enqueue download to slskd for {artist_name} - {album_name} from {username}")
# Delete ALL other downloads in all_downloads list
if len(all_downloads) > 0:
cancel_and_delete(all_downloads)
return False, None
except Exception:
album = lidarr.get_album(all_tracks[0]["albumId"])
album_name = album["title"]
artist_name = album["artist"]["artistName"]
logger.exception("Exception enqueueing tracks")
logger.info(f"Exception enqueueing download to slskd for {artist_name} - {album_name} from {username}")
# Delete all other downloads in all_downloads list
if len(all_downloads) > 0:
cancel_and_delete(all_downloads)
return False, None
if enqueued == total:
return True, all_downloads
else:
# Delete all other downloads
if len(all_downloads) > 0:
cancel_and_delete(all_downloads)
return False, None
else:
return False, None
def find_download(album, grab_list):
"""
This does the main loop over search results and user directories
It has two paths it can take. One is the "single album" path
The other is the multi-media path.
"""
album_id = album["id"]
artist_name = album["artist"]["artistName"]
artist_id = album["artistId"]
results = search_cache[album_id]
for allowed_filetype in allowed_filetypes:
logger.info(f"Checking for Quality: {allowed_filetype}")
releases = lidarr.get_album(album_id)["releases"]
num_releases = len(releases)
for _ in range(0, num_releases):
if len(releases) == 0:
break
release = choose_release(artist_name, releases)
releases.remove(release)
release_id = release["id"]
all_tracks = lidarr.get_tracks(artistId=artist_id, albumId=album_id, albumReleaseId=release_id)
found, downloads = try_enqueue(all_tracks, results, allowed_filetype)
if found:
grab_list[album_id] = {}
grab_list[album_id]["files"] = downloads
grab_list[album_id]["filetype"] = allowed_filetype
grab_list[album_id]["title"] = album["title"]
grab_list[album_id]["artist"] = artist_name
grab_list[album_id]["year"] = album["releaseDate"][0:4]
return True
elif len(release["media"]) > 1:
found, downloads = try_multi_enqueue(release, all_tracks, results, allowed_filetype)
if found:
grab_list[album_id] = {}
grab_list[album_id]["files"] = downloads
grab_list[album_id]["filetype"] = allowed_filetype
grab_list[album_id]["title"] = album["title"]
grab_list[album_id]["artist"] = artist_name
grab_list[album_id]["year"] = album["releaseDate"][0:4]
return True
return False
def search_and_queue(albums):
grab_list = {}
failed_grab = []
failed_search = []
for i, album in enumerate(albums):
search_start = time.time()
if search_for_album(album):
if not find_download(album, grab_list):
failed_grab.append(album)
else:
failed_search.append(album)
if minimum_search_interval > 0 and i < len(albums) - 1:
elapsed = time.time() - search_start
remaining = minimum_search_interval - elapsed
if remaining > 0:
logger.info(f"Search completed in {elapsed:.1f}s, waiting {remaining:.1f}s to meet minimum_search_interval")
time.sleep(remaining)
return grab_list, failed_search, failed_grab
def process_completed_album(album_data, failed_grab):
os.chdir(slskd_download_dir)
if rename_download_folders is True:
import_folder_name = sanitize_folder_name(album_data["artist"] + " - " + album_data["title"] + " (" + album_data["year"] + ")")
else:
import_folder_name = album_data["files"][0]["file_dir"].rstrip("\\/").rsplit("\\", 1)[-1]
import_folder_fullpath = os.path.join(slskd_download_dir, import_folder_name)
lidarr_import_fullpath = os.path.join(lidarr_download_dir, import_folder_name)
album_data["import_folder"] = lidarr_import_fullpath
rm_dirs = []
moved_files_history = []
if not os.path.exists(import_folder_fullpath):
os.mkdir(import_folder_fullpath)
for file in album_data["files"]:
file_folder = file["file_dir"].split("\\")[-1]
filename = file["filename"].split("\\")[-1]
src_folder = os.path.join(slskd_download_dir, file_folder)
if src_folder not in rm_dirs:
rm_dirs.append(src_folder) # Multi disk albums are sometimes in multiple folders. eg. CD01 CD02. So we need to clean up both
src_file = os.path.join(src_folder, filename)
if "disk_no" in file and "disk_count" in file and file["disk_count"] > 1:
filename = f"Disk {file['disk_no']} - {filename}"
dst_file = os.path.join(import_folder_fullpath, filename)
file["import_path"] = dst_file
if os.path.abspath(src_file) == os.path.abspath(dst_file):
continue
try:
shutil.move(src_file, dst_file)
moved_files_history.append((src_file, dst_file))
except Exception:
logger.exception(f"Failed to move: {file['filename']} to temp location for import into Lidarr. Rolling back...")
for src, dst in reversed(moved_files_history):
try:
shutil.move(dst, src)
except Exception:
logger.exception(f"Critical failure during rollback: could not move {dst} back to {src}")
try:
os.rmdir(import_folder_fullpath)
except OSError:
logger.warning(f"Could not remove temp import directory {import_folder_fullpath}")
failed_grab.append(lidarr.get_album(album_data["album_id"]))
return
else: # Only runs if all files are successfully moved
for rm_dir in rm_dirs:
if not rm_dir == import_folder_fullpath:
try:
os.rmdir(rm_dir)
except OSError:
logger.warning(f"Skipping removal of {rm_dir} because it's not empty.")
if lidarr_disable_sync:
logger.info(f"Sync disabled. Skipping Lidarr import of {album_data['artist']} - {album_data['title']}")
return
logger.info(f"Attempting Lidarr import of {album_data['artist']} - {album_data['title']}")
for file in album_data["files"]:
try:
song = music_tag.load_file(file["import_path"])
except NotImplementedError:
continue # Not a supported audio file (e.g. jpg, nfo)
except Exception:
logger.exception(f"Error loading file for tagging: {file['import_path']}")
continue
try:
if "disk_no" in file:
song["discnumber"] = file["disk_no"]
song["totaldiscs"] = file["disk_count"]
song["albumartist"] = album_data["artist"]
song["album"] = album_data["title"]
song.save()
except Exception:
logger.exception(f"Error writing tags for: {file['import_path']}")
command = lidarr.post_command(
name="DownloadedAlbumsScan",
path=album_data["import_folder"],
) # Album all tagged up and in a correctly named folder. This should work more reliably
logger.info(f"Starting Lidarr import for: {album_data['title']} ID: {command['id']}")
while True:
current_task = lidarr.get_command(command["id"])
if current_task["status"] == "completed" or current_task["status"] == "failed":
break
time.sleep(2)
try:
logger.info(f"{current_task['commandName']} {current_task['message']} from: {current_task['body']['path']}")
if "Failed" in current_task["message"]:
folder_path = move_failed_import(current_task["body"]["path"])
failed_grab.append(lidarr.get_album(album_data["album_id"]))
if failed_import_denylist:
add_to_failed_import_denylist(
failed_import_denylist_file_path,
album_data["album_id"],
album_data["artist"],
album_data["title"],
folder_path,
)
except Exception:
logger.exception("Error printing lidarr task message")
logger.error(current_task)
def monitor_downloads(grab_list, failed_grab):
MAX_FILE_RETRIES = 4 # Max requeue attempts per file for hard errors (Errored, Cancelled, etc.)
def delete_album(reason):
cancel_and_delete(grab_list[album_id]["files"])
logger.info(f"{reason} Album: {grab_list[album_id]['title']} Artist: {grab_list[album_id]['artist']}")
del grab_list[album_id]
failed_grab.append(lidarr.get_album(album_id))
def requeue_file(album_id, file):
"""Requeue a single errored file. Returns True on success, False if enqueue failed."""
data_dict = [{"filename": file["filename"], "size": file["size"]}]
logger.info(f"Download error. Requeue file: {file['filename']}")
requeue = slskd_do_enqueue(file["username"], data_dict, file["file_dir"])
if requeue is not None:
file["id"] = requeue[0]["id"]
time.sleep(1)
slskd_download_status(grab_list[album_id]["files"])
return True
return False
def handle_hard_error(album_id, file, problems):
"""
Handle Cancelled/TimedOut/Errored/Aborted files.
Returns True if the album was deleted (caller should stop processing this album).
"""
if len(problems) == len(grab_list[album_id]["files"]):
delete_album("Failed grab of")
return True
file.setdefault("retry", 0)
file["retry"] += 1
if file["retry"] > MAX_FILE_RETRIES:
delete_album("Failed grab of")
return True
if not requeue_file(album_id, file):
delete_album("Failed grab of")
return True
return False
def handle_rejected(album_id, file, problems):
"""
Handle Rejected files. Returns True if the album was deleted or a requeue was
attempted (caller should stop processing this album this iteration).
Rejected files often indicate grab limits; we wait for all other files to reach
a stable state before requeuing.
"""
files = grab_list[album_id]["files"]
if len(problems) == len(files):
delete_album("Failed grab of")
return True
# Only requeue once all non-problem files have settled (no files mid-transfer).
stable_states = ("Completed, Succeeded", "Queued, Remotely", "Queued, Locally")
accounted = sum(1 for f in files if f["status"]["state"] in stable_states) + len(problems)
if accounted < len(files):
return False
grab_list[album_id].setdefault("rejected_retries", 0)
if grab_list[album_id]["rejected_retries"] >= int(len(files) * 1.2):
delete_album("Failed grab of")
return True
if not requeue_file(album_id, file):
delete_album("Failed grab of")
return True
grab_list[album_id]["rejected_retries"] += 1
return True # Requeued one file; wait for next monitoring iteration
while True:
for album_id in list(grab_list.keys()):
if not slskd_download_status(grab_list[album_id]["files"]):
grab_list[album_id]["error_count"] = grab_list[album_id].get("error_count", 0) + 1
continue
album_done, problems, queued = downloads_all_done(grab_list[album_id]["files"])
grab_list[album_id].setdefault("count_start", time.time())
elapsed = time.time() - grab_list[album_id]["count_start"]
if elapsed >= stalled_timeout:
delete_album("Timeout waiting for download of")
continue
if queued == len(grab_list[album_id]["files"]) and elapsed >= remote_queue_timeout:
delete_album("Timeout waiting for download of")
continue
if album_done:
album_data = grab_list[album_id]
album_data["album_id"] = album_id
logger.info(f"Completed download of Album: {album_data['title']} Artist: {album_data['artist']}")
process_completed_album(album_data, failed_grab)
del grab_list[album_id]
continue
if problems:
logger.debug("Files with errors detected.")
for file in problems:
if album_id not in grab_list:
break
logger.debug(f"Checking {file['filename']}")
state = file["status"]["state"]
if state in ("Completed, Cancelled", "Completed, TimedOut", "Completed, Errored", "Completed, Aborted"):
if handle_hard_error(album_id, file, problems):
break
elif state == "Completed, Rejected":
if handle_rejected(album_id, file, problems):
break
else:
logger.error(f"Unexpected file state in problem list: {state}")
if not grab_list:
break
time.sleep(5)
def grab_most_wanted(albums):
"""
This is the "main loop" that calls all the functions to do all the work.
Basic flow per item is as follows:
Perform coarse search
Check search results for a match
enqueue download
After that has happened for all the downloads it then shifts to monitoring the downloads:
Monitor download and perform retries and/or requeues.
When all completed, call lidarr to import