[Ready for review] make default s3 worker count configurable (#1759)

* make default s3 worker count configurable

* change comment wording

Author: darinyu

metaflow/metaflow_config.py

@@ -81,6 +81,9 @@
 # so setting it to 0 means each operation will be tried once.
 S3_RETRY_COUNT = from_conf("S3_RETRY_COUNT", 7)
 
+# Number of concurrent S3 processes for parallel operations.
+S3_WORKER_COUNT = from_conf("S3_WORKER_COUNT", 64)
+
 # Number of retries on *transient* failures (such as SlowDown errors). Note
 # that if after S3_TRANSIENT_RETRY_COUNT times, all operations haven't been done,
 # it will try up to S3_RETRY_COUNT again so the total number of tries can be up to

metaflow/plugins/datatools/s3/s3op.py

@@ -44,8 +44,9 @@
     TRANSIENT_RETRY_START_LINE,
 )
 import metaflow.tracing as tracing
-
-NUM_WORKERS_DEFAULT = 64
+from metaflow.metaflow_config import (
+    S3_WORKER_COUNT,
+)
 
 DOWNLOAD_FILE_THRESHOLD = 2 * TransferConfig().multipart_threshold
 DOWNLOAD_MAX_CHUNK = 2 * 1024 * 1024 * 1024 - 1
@@ -656,7 +657,7 @@ def common_options(func):
     )
     @click.option(
         "--num-workers",
-        default=NUM_WORKERS_DEFAULT,
+        default=S3_WORKER_COUNT,
         show_default=True,
         help="Number of concurrent connections.",
     )