chore(helm): Temporary adjustments for data migration

Author: nellh

helm/openneuro/templates/dataset-worker-pv.yaml

@@ -20,3 +20,23 @@ spec:
     fsType: ext4
 ---
 {{- end }}
+{{- range $index, $config := .Values.newWorkerDiskSize }}
+apiVersion: v1
+kind: PersistentVolume
+metadata: 
+  name: datasets-new-{{ $relname }}-dataset-worker-{{ $index }}
+spec:
+  storageClassName: {{ $relname }}-datasets
+  capacity:
+    storage: {{ $config.size }}
+  accessModes:
+    - ReadWriteOnce
+  claimRef:
+    namespace: default
+    name: datasets-new-{{ $relname }}-dataset-worker-new-{{ $index }}
+  csi:
+    driver: pd.csi.storage.gke.io
+    volumeHandle: {{ $config.id }}
+    fsType: ext4
+---
+{{- end }}

helm/openneuro/templates/dataset-worker-stateful-set.yaml

@@ -17,6 +17,14 @@ spec:
         resources:
           requests:
             storage: 10Gi
+    - metadata:
+        name: datasets-new
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 10Gi
   template:
     metadata:
       labels:
@@ -64,6 +72,8 @@ spec:
         volumeMounts:
         - name: datasets
           mountPath: /datasets
+        - name: datasets-new
+          mountPath: /datasets-new
         - name: ssh-key
           mountPath: /datalad-key
           subPath: datalad-key
@@ -75,11 +85,11 @@ spec:
             port: 80
       - name: {{ .Release.Name }}-dataset-taskiq-worker
         image: 'openneuro/datalad-service:v{{ .Chart.AppVersion }}'
-        command: ['/srv/.venv/bin/taskiq', 'worker', 'datalad_service.broker:broker', '--tasks-pattern', 'datalad_service/tasks/*.py', '--fs-discover', '--max-async-tasks', '4']
+        command: ['/srv/.venv/bin/taskiq', 'worker', 'datalad_service.broker:broker', '--tasks-pattern', 'datalad_service/tasks/*.py', '--fs-discover', '--max-async-tasks', '2', '--use-process-pool']
         resources:
           requests:
             cpu: {{ .Values.workerCpuRequests }}
-            memory: 32Gi
+            memory: 96Gi
         envFrom:
         - configMapRef:
             name: {{ .Release.Name }}-configmap

helm/openneuro/values-production.yaml

@@ -60,6 +60,16 @@ workerDiskSize:
   - id: projects/hs-openneuro/zones/us-west1-c/disks/openneuro-prod-datasets-3
     size: 48000Gi
 
+newWorkerDiskSize:
+  - id: projects/hs-openneuro/zones/us-west1-c/disks/openneuro-prod-worker-datasets-0
+    size: 9000Gi
+  - id: projects/hs-openneuro/zones/us-west1-c/disks/openneuro-prod-worker-datasets-1
+    size: 9000Gi
+  - id: projects/hs-openneuro/zones/us-west1-c/disks/openneuro-prod-worker-datasets-2
+    size: 7000Gi
+  - id: projects/hs-openneuro/zones/us-west1-c/disks/openneuro-prod-worker-datasets-3
+    size: 9000Gi
+
 # Disable Redis password for testing
 redis:
   image: