Skip to content

Commit 71b52f5

Browse files
mrussemrusse
committed
refactor of monitor_downloads function
1 parent 64d5c56 commit 71b52f5

1 file changed

Lines changed: 98 additions & 141 deletions

File tree

‎soularr.py‎

Lines changed: 98 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -843,159 +843,116 @@ def process_completed_album(album_data, failed_grab):
843843

844844

845845
def monitor_downloads(grab_list, failed_grab):
846+
MAX_FILE_RETRIES = 4 # Max requeue attempts per file for hard errors (Errored, Cancelled, etc.)
847+
846848
def delete_album(reason):
847849
cancel_and_delete(grab_list[album_id]["files"])
848850
logger.info(f"{reason} Album: {grab_list[album_id]['title']} Artist: {grab_list[album_id]['artist']}")
849851
del grab_list[album_id]
850852
failed_grab.append(lidarr.get_album(album_id))
851853

854+
def requeue_file(album_id, file):
855+
"""Requeue a single errored file. Returns True on success, False if enqueue failed."""
856+
data_dict = [{"filename": file["filename"], "size": file["size"]}]
857+
logger.info(f"Download error. Requeue file: {file['filename']}")
858+
requeue = slskd_do_enqueue(file["username"], data_dict, file["file_dir"])
859+
if requeue is not None:
860+
file["id"] = requeue[0]["id"]
861+
time.sleep(1)
862+
slskd_download_status(grab_list[album_id]["files"])
863+
return True
864+
return False
865+
866+
def handle_hard_error(album_id, file, problems):
867+
"""
868+
Handle Cancelled/TimedOut/Errored/Aborted files.
869+
Returns True if the album was deleted (caller should stop processing this album).
870+
"""
871+
if len(problems) == len(grab_list[album_id]["files"]):
872+
delete_album("Failed grab of")
873+
return True
874+
file.setdefault("retry", 0)
875+
file["retry"] += 1
876+
if file["retry"] > MAX_FILE_RETRIES:
877+
delete_album("Failed grab of")
878+
return True
879+
if not requeue_file(album_id, file):
880+
delete_album("Failed grab of")
881+
return True
882+
return False
883+
884+
def handle_rejected(album_id, file, problems):
885+
"""
886+
Handle Rejected files. Returns True if the album was deleted or a requeue was
887+
attempted (caller should stop processing this album this iteration).
888+
Rejected files often indicate grab limits; we wait for all other files to reach
889+
a stable state before requeueing.
890+
"""
891+
files = grab_list[album_id]["files"]
892+
if len(problems) == len(files):
893+
delete_album("Failed grab of")
894+
return True
895+
# Only requeue once all non-problem files have settled (no files mid-transfer).
896+
stable_states = ("Completed, Succeeded", "Queued, Remotely", "Queued, Locally")
897+
accounted = sum(1 for f in files if f["status"]["state"] in stable_states) + len(problems)
898+
if accounted < len(files):
899+
return False
900+
grab_list[album_id].setdefault("rejected_retries", 0)
901+
if grab_list[album_id]["rejected_retries"] >= int(len(files) * 1.2):
902+
delete_album("Failed grab of")
903+
return True
904+
if not requeue_file(album_id, file):
905+
delete_album("Failed grab of")
906+
return True
907+
grab_list[album_id]["rejected_retries"] += 1
908+
return True # Requeued one file; wait for next monitoring iteration
909+
852910
while True:
853-
total_albums = len(grab_list)
854-
# Deal with the problems.
855-
# "Completed, Cancelled", Abort album as failed
856-
# "Completed, TimedOut", Abort album as failed
857-
# "Completed, Errored", Abort album as failed
858-
# "Completed, Aborted", Abort album as failed
859-
# "Completed, Rejected", Retry. Some users have a max grab count. We need to check if ALL files are Rejected first.
860-
# We're going to need to drop items out of the list. So we might have to resort to enumerating the keys so we don't hit issues.
861-
done_count = 0
862911
for album_id in list(grab_list.keys()):
863-
if slskd_download_status(grab_list[album_id]["files"]):
864-
album_done, problems, queued = downloads_all_done(grab_list[album_id]["files"]) # Lets check to see what status the files have
865-
if "count_start" not in grab_list[album_id]:
866-
grab_list[album_id]["count_start"] = time.time()
867-
if (time.time() - grab_list[album_id]["count_start"]) >= stalled_timeout: # Album is taking too long. Bail out regardless
868-
delete_album("Timeout waiting for download of")
869-
continue
870-
if queued == len(grab_list[album_id]["files"]): # Shorter time out for whole albums in "Queued, Remotely"
871-
if (time.time() - grab_list[album_id]["count_start"]) >= remote_queue_timeout:
872-
delete_album("Timeout waiting for download of")
873-
continue
874-
done_count += album_done
875-
if problems is not None:
876-
logger.debug("We got problems!")
877-
for file in problems:
878-
logger.debug(f"Checking {file['filename']}")
879-
match file["status"]["state"]:
880-
case (
881-
"Completed, Cancelled" | "Completed, TimedOut" | "Completed, Errored" | "Completed, Aborted"
882-
): # Normal errors. We'll retry a few times as sometumes the error is transient
883-
abort = False
884-
if len(problems) == len(grab_list[album_id]["files"]):
885-
delete_album("Failed grab of")
886-
break
887-
for download_file in grab_list[album_id]["files"]:
888-
if file["filename"] == download_file["filename"]:
889-
if "retry" not in download_file:
890-
download_file["retry"] = 0
891-
download_file["retry"] += 1
892-
if download_file["retry"] < 5:
893-
retry = download_file["retry"]
894-
size = file["size"]
895-
data_dict = [{"filename": file["filename"], "size": size}]
896-
logger.info(f"Download error. Requeue file: {file['filename']}")
897-
requeue = slskd_do_enqueue(
898-
file["username"],
899-
data_dict,
900-
file["file_dir"],
901-
)
902-
if requeue is not None:
903-
download_file["id"] = requeue[0]["id"]
904-
download_file["retry"] = retry
905-
time.sleep(1)
906-
_ = slskd_download_status(grab_list[album_id]["files"]) # Refresh the status of the files to prevent issues.
907-
else:
908-
delete_album("Failed grab of")
909-
abort = True # Move to the next album so we don't block or overload a remote user
910-
break
911-
else:
912-
# Delete from album list add to failures
913-
delete_album("Failed grab of")
914-
abort = True # As above.
915-
break
916-
if abort:
917-
break
918-
case "Completed, Rejected":
919-
# Do a measured retry. This is often a soft failure due to grab limits. Check if any files worked then go from there.
920-
# This needs a recode. But it works for now.
921-
# In the recode we need to test to see if we are getting multiple albums from the same user and temper our retries based on
922-
# those other album(s) completing.
923-
# If we aren't in that condition we need to fall back to per file retry counts as files will also be rejected if the file is
924-
# too long or too short based on the share record. This can happen when people re-tag media but don't rescan media.
925-
# Also I've seen cases of single files out of a set being in the "not shared" category.
926-
if len(problems) == len(grab_list[album_id]["files"]):
927-
delete_album("Failed grab of") # They are all rejected. Usually this happens because of misconfigurations. Files appear in search but aren't shared.
928-
break
929-
else:
930-
if "rejected_retries" not in grab_list[album_id]:
931-
grab_list[album_id]["rejected_retries"] = 0
932-
working_count = len(grab_list[album_id]["files"]) - len(problems)
933-
for gfile in grab_list[album_id]["files"]:
934-
if gfile["status"]["state"] in [
935-
"Completed, Succeeded",
936-
"Queued, Remotely",
937-
"Queued, Locally",
938-
]:
939-
working_count -= 1
940-
if working_count == 0:
941-
if grab_list[album_id]["rejected_retries"] < int(len(grab_list[album_id]["files"]) * 1.2): # Little bit of wiggle room here
942-
abort = False
943-
for gfile in grab_list[album_id]["files"]:
944-
if gfile["filename"] == file["filename"]:
945-
size = file["size"]
946-
data_dict = [
947-
{
948-
"filename": file["filename"],
949-
"size": size,
950-
}
951-
]
952-
logger.info(f"Download error. Requeue file: {file['filename']}")
953-
requeue = slskd_do_enqueue(
954-
file["username"],
955-
data_dict,
956-
file["file_dir"],
957-
)
958-
if requeue is not None:
959-
gfile["id"] = requeue[0]["id"]
960-
grab_list[album_id]["rejected_retries"] += 1
961-
_ = slskd_download_status(grab_list[album_id]["files"])
962-
abort = True
963-
break
964-
else:
965-
cancel_and_delete(grab_list[album_id]["files"])
966-
logger.info(f"Failed grab of Album: {grab_list[album_id]['title']} Artist: {grab_list[album_id]['artist']}")
967-
del grab_list[album_id]
968-
failed_grab.append(lidarr.get_album(album_id)) # Not sure if returns an array or not
969-
abort = True
970-
break
971-
if abort:
972-
break
973-
else:
974-
delete_album("Failed grab of")
975-
break
976-
case _:
977-
logger.error(
978-
"Not sure how I got here. This shouldn't be possible for problem files!"
979-
) # This really should be impossible to reach. But is required to round out the case statement.
980-
else:
981-
if album_done:
982-
album_data = grab_list[album_id]
983-
album_data["album_id"] = album_id
984-
logger.info(f"Completed download of Album: {album_data['title']} Artist: {album_data['artist']}")
985-
process_completed_album(album_data, failed_grab)
986-
del grab_list[album_id]
912+
if not slskd_download_status(grab_list[album_id]["files"]):
913+
grab_list[album_id]["error_count"] = grab_list[album_id].get("error_count", 0) + 1
914+
continue
987915

988-
else:
989-
if "error_count" not in grab_list[album_id]:
990-
grab_list[album_id]["error_count"] = 0
991-
grab_list[album_id]["error_count"] += 1
992-
# I dunno. slskd might be broken? Or the user deleted things? I've never seen this so I have no idea what we should do here. It most likely would mean SLSKD is down.
993-
# So we probably want to abort everything because cleanup would be impossible.
916+
album_done, problems, queued = downloads_all_done(grab_list[album_id]["files"])
917+
918+
grab_list[album_id].setdefault("count_start", time.time())
919+
elapsed = time.time() - grab_list[album_id]["count_start"]
994920

995-
if len(grab_list) < 1: # We remove items from the grab list once they are downloaded or aborted. So when there are no grabs left, we are done!
921+
if elapsed >= stalled_timeout:
922+
delete_album("Timeout waiting for download of")
923+
continue
924+
if queued == len(grab_list[album_id]["files"]) and elapsed >= remote_queue_timeout:
925+
delete_album("Timeout waiting for download of")
926+
continue
927+
928+
if album_done:
929+
album_data = grab_list[album_id]
930+
album_data["album_id"] = album_id
931+
logger.info(f"Completed download of Album: {album_data['title']} Artist: {album_data['artist']}")
932+
process_completed_album(album_data, failed_grab)
933+
del grab_list[album_id]
934+
continue
935+
936+
if problems:
937+
logger.debug("Files with errors detected.")
938+
for file in problems:
939+
if album_id not in grab_list:
940+
break
941+
logger.debug(f"Checking {file['filename']}")
942+
state = file["status"]["state"]
943+
if state in ("Completed, Cancelled", "Completed, TimedOut", "Completed, Errored", "Completed, Aborted"):
944+
if handle_hard_error(album_id, file, problems):
945+
break
946+
elif state == "Completed, Rejected":
947+
if handle_rejected(album_id, file, problems):
948+
break
949+
else:
950+
logger.error(f"Unexpected file state in problem list: {state}")
951+
952+
if not grab_list:
996953
break
997954

998-
time.sleep(5) # Wait for things to progress and start the checks again.
955+
time.sleep(5)
999956

1000957

1001958
def grab_most_wanted(albums):

0 commit comments

Comments
 (0)