Merge pull request #3659 from OpenNeuroOrg/export-fixes

Fixes for export and drop issues

Author: nellh

helm/openneuro/templates/dataset-worker-stateful-set.yaml

@@ -75,7 +75,7 @@ spec:
             port: 80
       - name: {{ .Release.Name }}-dataset-taskiq-worker
         image: 'openneuro/datalad-service:v{{ .Chart.AppVersion }}'
-        command: ['/srv/.venv/bin/taskiq', 'worker', 'datalad_service.broker:broker', '--tasks-pattern', 'datalad_service/tasks/*.py', '--fs-discover']
+        command: ['/srv/.venv/bin/taskiq', 'worker', 'datalad_service.broker:broker', '--tasks-pattern', 'datalad_service/tasks/*.py', '--fs-discover', '--max-async-tasks', '4']
         resources:
           requests:
             cpu: {{ .Values.workerCpuRequests }}

services/datalad/datalad_service/common/s3.py

@@ -182,15 +182,15 @@ def validate_s3_config(dataset_path):
 async def s3_export(dataset_path, target, treeish):
     """Perform an S3 export on a git-annex repo."""
     await run_check(
-        ['git-annex', 'export', '-J4', treeish, '--to', target],
+        ['git-annex', 'export', '-J2', treeish, '--to', target],
         dataset_path,
     )
 
 
 async def s3_backup_push(dataset_path):
     """Perform an S3 push to the backup remote on a git-annex repo."""
     await run_check(
-        ['git-annex', 'push', '-J4', '--all', get_s3_backup_remote()],
+        ['git-annex', 'push', '-J2', '--all', get_s3_backup_remote()],
         dataset_path,
         env=backup_remote_env(),
     )

services/datalad/datalad_service/tasks/publish.py

@@ -106,7 +106,7 @@ async def export_backup_and_drop(dataset_path):
         for tag in tags:
             logger.info(f'Exporting/dropping tag {dataset_id}@{tag.name}')
             export_ran = False
-            if await find_in_remote(dataset_path, tag.name, get_s3_remote()):
+            if not await find_in_remote(dataset_path, tag.name, get_s3_remote()):
                 export_ran = True
                 await s3_export(dataset_path, get_s3_remote(), tag.name)
             if tag == tags[-1] and export_ran: