prevent hitting ARG_MAX limit

Author: chapmanjacobd

library/multidb/allocate_torrents.py

@@ -242,23 +242,25 @@ def allocate_torrents():
                 ssh.connect(d["host"])
                 setattr(ssh, "host", d["host"])
 
-                remote_processes.cmd(
-                    ssh,
-                    "python3",
-                    "-m",
-                    "library",
-                    "torrents-start",
-                    *argparse_utils.forward_arggroups(args, arggroups.torrents_start),
-                    *argparse_utils.forward_arggroups(args, arggroups.qBittorrent),
-                    *argparse_utils.forward_arggroups(
-                        args,
-                        arggroups.qBittorrent_paths,
-                        download_drive=d["mountpoint"],
-                    ),
-                    *torrent_files,
-                    local_files=torrent_files,
-                    # cwd="lb",  # debug
-                )
+                torrent_files_chunks = iterables.chunks(torrent_files, 128)
+                for torrent_files_chunk in torrent_files_chunks:
+                    remote_processes.cmd(
+                        ssh,
+                        "python3",
+                        "-m",
+                        "library",
+                        "torrents-start",
+                        *argparse_utils.forward_arggroups(args, arggroups.torrents_start),
+                        *argparse_utils.forward_arggroups(args, arggroups.qBittorrent),
+                        *argparse_utils.forward_arggroups(
+                            args,
+                            arggroups.qBittorrent_paths,
+                            download_drive=d["mountpoint"],
+                        ),
+                        *torrent_files_chunk,
+                        local_files=torrent_files_chunk,
+                        # cwd="lb",  # debug
+                    )
 
             if args.delete_torrent:
                 for path in torrent_files:

library/utils/arggroups.py

@@ -2031,7 +2031,9 @@ def qBittorrent_torrents(parent_parser):
         action="append",
         help="""Include torrents constrained by average file sizes (uses the same syntax as fd-find)""",
     )
-    parser.add_argument("--file-count", "--files", action="append", help="Include torrents constrained by total file counts")
+    parser.add_argument(
+        "--file-count", "--files", action="append", help="Include torrents constrained by total file counts"
+    )
     parser.add_argument("--seeders", action="append", help="Include torrents with N seeders")
     parser.add_argument("--leechers", action="append", help="Include torrents with N leechers")
     parser.add_argument("--time-added", action="append", help="Include torrents with N time since added")
@@ -2069,9 +2071,7 @@ def qBittorrent_torrents(parent_parser):
         help="Include uploading torrents",
     )
 
-    parser.add_argument(
-        "--file-counts", action="store_true", help="Include file counts column (a bit slow)"
-    )
+    parser.add_argument("--file-counts", action="store_true", help="Include file counts column (a bit slow)")
     parser.add_argument("--trackers", action="store_true", help="Include tracker column")
     parser.add_argument("--status", "--state", action="store_true", help="Include state column")
     parser.add_argument("--paths", action="store_true", help="Include downloading/seeding path columns")

library/utils/remote_processes.py

@@ -56,7 +56,7 @@ def print_std(s, is_success):
     returncode = stdout.channel.recv_exit_status()
 
     host = getattr(ssh, "host", None)
-    if host:
+    if host:  # for logging purposes
         command = " ".join(["ssh", host, command])
 
     r = subprocess.CompletedProcess(command, returncode, stdout.read().decode(), stderr.read().decode())
@@ -80,6 +80,8 @@ def print_std(s, is_success):
             log.warning("[%s] exited %s", command, r.returncode)
 
         if strict:
+            if error_verbosity <= 1:
+                log.error("[%s] exited %s", command, r.returncode)
             r.check_returncode()
 
     return r