Merge pull request #3657 from OpenNeuroOrg/fsck-j4

fix(worker): Run export/fsck tasks with J4

Author: nellh

services/datalad/datalad_service/common/s3.py

@@ -182,15 +182,15 @@ def validate_s3_config(dataset_path):
 async def s3_export(dataset_path, target, treeish):
     """Perform an S3 export on a git-annex repo."""
     await run_check(
-        ['git-annex', 'export', treeish, '--to', target],
+        ['git-annex', 'export', '-J4', treeish, '--to', target],
         dataset_path,
     )
 
 
 async def s3_backup_push(dataset_path):
     """Perform an S3 push to the backup remote on a git-annex repo."""
     await run_check(
-        ['git-annex', 'push', '--all', get_s3_backup_remote()],
+        ['git-annex', 'push', '-J4', '--all', get_s3_backup_remote()],
         dataset_path,
         env=backup_remote_env(),
     )

services/datalad/datalad_service/tasks/fsck.py

@@ -37,6 +37,7 @@ def git_annex_fsck_local(dataset_path):
     annex_command = (
         'git-annex',
         'fsck',
+        '-J4',
         '--json',
         '--json-error-messages',
         '--incremental-schedule=45d',
@@ -73,6 +74,7 @@ async def git_annex_fsck_remote(dataset_path, branch, remote='s3-PUBLIC'):
     annex_command = (
         'git-annex',
         'fsck',
+        '-J4',
         f'--branch={branch}',
         f'--from={remote}',
         '--json',