prevent hitting ARG_MAX limit

Author: chapmanjacobd

library/multidb/allocate_torrents.py

@@ -242,23 +242,25 @@ def allocate_torrents():
                 ssh.connect(d["host"])
                 setattr(ssh, "host", d["host"])
 
-                remote_processes.cmd(
-                    ssh,
-                    "python3",
-                    "-m",
-                    "library",
-                    "torrents-start",
-                    *argparse_utils.forward_arggroups(args, arggroups.torrents_start),
-                    *argparse_utils.forward_arggroups(args, arggroups.qBittorrent),
-                    *argparse_utils.forward_arggroups(
-                        args,
-                        arggroups.qBittorrent_paths,
-                        download_drive=d["mountpoint"],
-                    ),
-                    *torrent_files,
-                    local_files=torrent_files,
-                    # cwd="lb",  # debug
-                )
+                torrent_files_chunks = iterables.chunks(torrent_files, 128)
+                for torrent_files_chunk in torrent_files_chunks:
+                    remote_processes.cmd(
+                        ssh,
+                        "python3",
+                        "-m",
+                        "library",
+                        "torrents-start",
+                        *argparse_utils.forward_arggroups(args, arggroups.torrents_start),
+                        *argparse_utils.forward_arggroups(args, arggroups.qBittorrent),
+                        *argparse_utils.forward_arggroups(
+                            args,
+                            arggroups.qBittorrent_paths,
+                            download_drive=d["mountpoint"],
+                        ),
+                        *torrent_files_chunk,
+                        local_files=torrent_files_chunk,
+                        # cwd="lb",  # debug
+                    )
 
             if args.delete_torrent:
                 for path in torrent_files:

library/playback/torrents_info.py

@@ -166,6 +166,8 @@ def filter_torrents_by_criteria(args, torrents):
         torrents = [t for t in torrents if args.priority(t.priority)]
     if "progress" not in args.defaults:
         torrents = [t for t in torrents if args.progress(t.progress)]
+    if "downloaded" not in args.defaults:
+        torrents = [t for t in torrents if args.downloaded(t.downloaded)]
     if "uploaded" not in args.defaults:
         torrents = [t for t in torrents if args.uploaded(t.uploaded)]
     if "remaining" not in args.defaults:

library/utils/arggroups.py

@@ -2031,7 +2031,9 @@ def qBittorrent_torrents(parent_parser):
         action="append",
         help="""Include torrents constrained by average file sizes (uses the same syntax as fd-find)""",
     )
-    parser.add_argument("--file-count", "--files", action="append", help="Include torrents constrained by total file counts")
+    parser.add_argument(
+        "--file-count", "--files", action="append", help="Include torrents constrained by total file counts"
+    )
     parser.add_argument("--seeders", action="append", help="Include torrents with N seeders")
     parser.add_argument("--leechers", action="append", help="Include torrents with N leechers")
     parser.add_argument("--time-added", action="append", help="Include torrents with N time since added")
@@ -2045,6 +2047,7 @@ def qBittorrent_torrents(parent_parser):
     parser.add_argument("--priority", action="append", help="Include torrents with N priority")
     parser.add_argument("--progress", action="append", help="Include torrents with N%% progress")
     parser.add_argument("--remaining", action="append", help="Include torrents with N bytes remaining")
+    parser.add_argument("--downloaded", action="append", help="Include torrents with N bytes downloaded")
     parser.add_argument("--uploaded", action="append", help="Include torrents with N bytes uploaded")
     parser.add_argument("--ratio", action="append", help="Include torrents with N ratio")
 
@@ -2069,9 +2072,7 @@ def qBittorrent_torrents(parent_parser):
         help="Include uploading torrents",
     )
 
-    parser.add_argument(
-        "--file-counts", action="store_true", help="Include file counts column (a bit slow)"
-    )
+    parser.add_argument("--file-counts", action="store_true", help="Include file counts column (a bit slow)")
     parser.add_argument("--trackers", action="store_true", help="Include tracker column")
     parser.add_argument("--status", "--state", action="store_true", help="Include state column")
     parser.add_argument("--paths", action="store_true", help="Include downloading/seeding path columns")
@@ -2097,6 +2098,7 @@ def qBittorrent_torrents(parent_parser):
 def qBittorrent_torrents_post(args):
     args.sizes = sql_utils.parse_human_to_lambda(nums.human_to_bytes, args.sizes)
     args.remaining = sql_utils.parse_human_to_lambda(nums.human_to_bytes, args.remaining)
+    args.downloaded = sql_utils.parse_human_to_lambda(nums.human_to_bytes, args.downloaded)
     args.uploaded = sql_utils.parse_human_to_lambda(nums.human_to_bytes, args.uploaded)
     args.avg_sizes = sql_utils.parse_human_to_lambda(nums.human_to_bytes, args.avg_sizes)
     args.file_count = sql_utils.parse_human_to_lambda(int, args.file_count)

library/utils/remote_processes.py

@@ -56,7 +56,7 @@ def print_std(s, is_success):
     returncode = stdout.channel.recv_exit_status()
 
     host = getattr(ssh, "host", None)
-    if host:
+    if host:  # for logging purposes
         command = " ".join(["ssh", host, command])
 
     r = subprocess.CompletedProcess(command, returncode, stdout.read().decode(), stderr.read().decode())
@@ -80,6 +80,8 @@ def print_std(s, is_success):
             log.warning("[%s] exited %s", command, r.returncode)
 
         if strict:
+            if error_verbosity <= 1:
+                log.error("[%s] exited %s", command, r.returncode)
             r.check_returncode()
 
     return r