Make attachment downloads asynchronous (through notification)

nginx.conf

@@ -25,6 +25,21 @@ server {
         proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
     }
 
+    # download_all should have an extended timeout limit for large downloads
+    # also proxy to specify 7
+    location /attachment_gw/download_all/ {
+        proxy_read_timeout 300s;
+        client_max_body_size 400M;
+        client_body_buffer_size 400M;
+        client_body_timeout 120;
+        resolver 127.0.0.11 valid=30s;
+        set $backend "http://specify7:8000";
+        proxy_pass $backend;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    }
+
     # proxy everything else to specify 7
     location / {
         client_max_body_size 400M;

specifyweb/attachment_gw/urls.py

@@ -8,6 +8,7 @@
     path('get_token/', views.get_token),
     path('proxy/', views.proxy),
     path('download_all/', views.download_all),
+    path('download_archive/', views.download_archive),
     path('dataset/', views.datasets),
     path('dataset/<int:ds_id>/', views.dataset),
 ]

specifyweb/attachment_gw/views.py

@@ -10,10 +10,11 @@
 from uuid import uuid4
 from xml.etree import ElementTree
 from datetime import datetime
+from django.apps import apps
 
 import requests
 from django.conf import settings
-from django.http import HttpResponse, HttpResponseBadRequest, \
+from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound, \
     StreamingHttpResponse
 from django.db import transaction
 from django.utils.translation import gettext as _
@@ -22,6 +23,9 @@
 
 from specifyweb.middleware.general import require_http_methods
 from specifyweb.specify.views import login_maybe_required, openapi
+from specifyweb.specify import models
+from specifyweb.specify.models_by_table_id import get_model_by_table_id
+from ..notifications.models import Message
 
 from .dataset_views import dataset_view, datasets_view
 logger = logging.getLogger(__name__)
@@ -317,59 +321,114 @@ def download_all(request):
     except ValueError as e:
         return HttpResponseBadRequest(e)
 
-    attachmentLocations = r['attachmentlocations']
-    origFileNames = r['origfilenames']
+    attachment_locations = r['attachmentlocations']
+    orig_filenames = r['origfilenames']
+    archivename = r['archivename']
+
+    # Optional record set parameter
+    # Fetches all the attachment locations instead of using the ones provided by the frontend
+    recordSetId = r.get('recordsetid', None)
+    if recordSetId is not None:
+        attachment_locations = []
+        orig_filenames = []
+        recordset = models.Recordset.objects.get(id=recordSetId)
+        table = get_model_by_table_id(recordset.dbtableid)
+        join_table = apps.get_model(table._meta.app_label, table.__name__ + 'attachment')
+
+        # Reach all attachments
+        recordsetitem_ids = models.Recordsetitem.objects.filter(recordset__id=recordSetId)
+        for rsi in recordsetitem_ids:
+            record_id = rsi.recordid
+            join_records = join_table.objects.filter(**{table.__name__.lower() + '_id': record_id})
+            for join_record in join_records:
+                attachment = join_record.attachment
+                if attachment.attachmentlocation is not None:
+                    attachment_locations.append(attachment.attachmentlocation)
+                    orig_filenames.append(os.path.basename(attachment.origfilename if attachment.origfilename else attachment.attachmentlocation))
 
     filename = 'attachments_%s.zip' % datetime.now().isoformat()
     path = os.path.join(settings.DEPOSITORY_DIR, filename)
 
     try:
-        make_attachment_zip(attachmentLocations, origFileNames, get_collection(request), path)
+        make_attachment_zip(attachment_locations, orig_filenames, get_collection(request), path)
     except Exception as e:
         return HttpResponseBadRequest(e)
 
     if not os.path.exists(path):
+        Message.objects.create(user=request.specify_user, content=json.dumps({
+            'type': 'attachment-download-failed',
+            'archive_name': archivename,
+            'file': filename,
+        }))
         return HttpResponseBadRequest('Attachment archive not found')
 
-    def file_iterator(file_path, chunk_size=512 * 1024):
-        with open(file_path, 'rb') as f:
-            while chunk := f.read(chunk_size):
-                yield chunk
-        os.remove(file_path)
+    Message.objects.create(user=request.specify_user, content=json.dumps({
+        'type': 'attachment-download-ready',
+        'archive_name': archivename,
+        'file': filename,
+        'delete_file': True,
+    }))
 
-    response = StreamingHttpResponse(
-        file_iterator(path),
-        content_type='application/octet-stream')
-    response['Content-Disposition'] = f'attachment; filename="{filename}"'
-    return response
+    return HttpResponse('', status=200)
 
-def make_attachment_zip(attachmentLocations, origFileNames, collection, output_file):
+def make_attachment_zip(attachment_locations, orig_filenames, collection, output_file):
     output_dir = mkdtemp()
     try:
-        fileNameAppearances = {}
-        for i, attachmentLocation in enumerate(attachmentLocations):
+        filename_appearances = {}
+        for i, attachment_location in enumerate(attachment_locations):
             data = {
-                'filename': attachmentLocation,
+                'filename': attachment_location,
                 'coll': collection,
                 'type': 'O',
-                'token': generate_token(get_timestamp(), attachmentLocation)
+                'token': generate_token(get_timestamp(), attachment_location)
             }
             response = requests.get(server_urls['read'], params=data)
             if response.status_code == 200:
-                downloadFileName = origFileNames[i] if i < len(origFileNames) else attachmentLocation
-                fileNameAppearances[downloadFileName] = fileNameAppearances.get(downloadFileName, 0) + 1
-                if fileNameAppearances[downloadFileName] > 1:
-                    downloadOrigName = os.path.splitext(downloadFileName)[0]
-                    downloadExtension = os.path.splitext(downloadFileName)[1]
-                    downloadFileName = f'{downloadOrigName}_{fileNameAppearances[downloadFileName]-1}{downloadExtension}'
-                with open(os.path.join(output_dir, downloadFileName), 'wb') as f:
+                download_filename = orig_filenames[i] if i < len(orig_filenames) else attachment_location
+                filename_appearances[download_filename] = filename_appearances.get(download_filename, 0) + 1
+                if filename_appearances[download_filename] > 1:
+                    download_orig_name = os.path.splitext(download_filename)[0]
+                    download_extension = os.path.splitext(download_filename)[1]
+                    download_filename = f'{download_orig_name}_{filename_appearances[download_filename]-1}{download_extension}'
+                with open(os.path.join(output_dir, download_filename), 'wb') as f:
                     f.write(response.content)
 
         basename = re.sub(r'\.zip$', '', output_file)
         shutil.make_archive(basename, 'zip', output_dir, logger=logger)
     finally:
         shutil.rmtree(output_dir)
 
+@require_POST
+@login_maybe_required
+@never_cache
+def download_archive(request):
+    """
+    Send an attachment archive to the frontend, and delete it from the backend to save storage.
+    """
+    try:
+        r = json.load(request)
+    except ValueError as e:
+        return HttpResponseBadRequest(e)
+
+    filename = r['filename']
+    path = os.path.abspath(os.path.join(settings.DEPOSITORY_DIR, filename))
+
+    if not path.startswith(os.path.abspath(settings.DEPOSITORY_DIR) + os.sep):
+        return HttpResponseBadRequest("Invalid filepath.")
+    if not os.path.exists(path):
+        return HttpResponseNotFound("File does not exist.")
+
+    def file_iterator(file_path, chunk_size=512 * 1024):
+        with open(file_path, 'rb') as f:
+            while chunk := f.read(chunk_size):
+                yield chunk
+        os.remove(file_path)
+    response = StreamingHttpResponse(
+        file_iterator(path),
+        content_type='application/octet-stream')
+    response['Content-Disposition'] = f'attachment; filename="{filename}"'
+    return response
+
 @transaction.atomic()
 @login_maybe_required
 @require_http_methods(['GET', 'POST', 'HEAD'])

specifyweb/frontend/js_src/lib/components/Attachments/DownloadButton.tsx

@@ -0,0 +1,75 @@
+import React from 'react';
+
+import { attachmentsText } from '../../localization/attachments';
+import { commonText } from '../../localization/common';
+import type { RA } from '../../utils/types';
+import { Button } from '../Atoms/Button';
+import { LoadingContext } from '../Core/Contexts';
+import type { SerializedResource } from '../DataModel/helperTypes';
+import type { Attachment } from '../DataModel/types';
+import { Dialog } from '../Molecules/Dialog';
+import { downloadAllAttachments } from './attachments';
+
+export function DownloadAllAttachmentsButton({
+  attachments,
+  disabled = false,
+  archiveName,
+  recordSetId,
+  recordSetRequired,
+}: {
+  readonly attachments: RA<SerializedResource<Attachment>>;
+  readonly disabled?: boolean;
+  readonly archiveName?: string;
+  readonly recordSetId?: number;
+  readonly recordSetRequired?: boolean;
+}): JSX.Element {
+  const loading = React.useContext(LoadingContext);
+
+  const [showCreateRecordSetDialog, setShowCreateRecordSetDialog] =
+    React.useState(false);
+  const createRecordSetDialog = (
+    <Dialog
+      buttons={<Button.DialogClose>{commonText.close()}</Button.DialogClose>}
+      header={attachmentsText.downloadAll()}
+      onClose={(): void => {
+        setShowCreateRecordSetDialog(false);
+      }}
+    >
+      {attachmentsText.createRecordSetToDownloadAll()}
+    </Dialog>
+  );
+
+  const [showDownloadStarted, setShowDownloadStartedDialog] =
+    React.useState(false);
+  const downloadStartedDialog = (
+    <Dialog
+      buttons={commonText.close()}
+      header={attachmentsText.downloadAllStarted()}
+      onClose={(): void => setShowDownloadStartedDialog(false)}
+    >
+      {attachmentsText.downloadAllStartedDescription()}
+    </Dialog>
+  );
+
+  return (
+    <>
+      {showDownloadStarted && downloadStartedDialog}
+      {showCreateRecordSetDialog && createRecordSetDialog}
+      <Button.Info
+        disabled={disabled}
+        title={attachmentsText.downloadAllDescription()}
+        onClick={(): void =>
+          recordSetRequired === true && recordSetId === undefined
+            ? setShowCreateRecordSetDialog(true)
+            : loading(
+                downloadAllAttachments(attachments, archiveName, recordSetId).then(() => {
+                  setShowDownloadStartedDialog(true);
+                })
+              )
+        }
+      >
+        {attachmentsText.downloadAll()}
+      </Button.Info>
+    </>
+  );
+}

specifyweb/frontend/js_src/lib/components/Attachments/RecordSetAttachment.tsx

@@ -9,29 +9,32 @@ import { f } from '../../utils/functools';
 import type { RA } from '../../utils/types';
 import { filterArray } from '../../utils/types';
 import { Button } from '../Atoms/Button';
-import { LoadingContext } from '../Core/Contexts';
 import type { AnySchema } from '../DataModel/helperTypes';
 import type { SpecifyResource } from '../DataModel/legacyTypes';
 import { serializeResource } from '../DataModel/serializers';
 import type { CollectionObjectAttachment } from '../DataModel/types';
 import { Dialog, dialogClassNames } from '../Molecules/Dialog';
 import { defaultAttachmentScale } from '.';
-import { downloadAllAttachments } from './attachments';
+import { DownloadAllAttachmentsButton } from './DownloadButton';
 import { AttachmentGallery } from './Gallery';
 import { getAttachmentRelationship } from './utils';
 
 const haltIncrementSize = 300;
 
 export function RecordSetAttachments<SCHEMA extends AnySchema>({
   records,
+  recordCount,
   onFetch: handleFetch,
   name,
+  recordSetId,
 }: {
   readonly records: RA<SpecifyResource<SCHEMA> | undefined>;
+  readonly recordCount: number;
   readonly onFetch:
     | ((index: number) => Promise<RA<number | undefined> | void>)
     | undefined;
   readonly name: string | undefined;
+  readonly recordSetId: number | undefined;
 }): JSX.Element {
   const fetchedCount = React.useRef<number>(0);
 
@@ -59,8 +62,8 @@ export function RecordSetAttachments<SCHEMA extends AnySchema>({
         )
       );
 
-      const fetchCount = filterArray(records).findIndex(
-        (record) => !record.populated
+      const fetchCount = records.findIndex(
+        (record) => record === undefined || !record.populated
       );
 
       fetchedCount.current = fetchCount === -1 ? records.length : fetchCount;
@@ -85,8 +88,6 @@ export function RecordSetAttachments<SCHEMA extends AnySchema>({
   );
   const attachmentsRef = React.useRef(attachments);
 
-  const loading = React.useContext(LoadingContext);
-
   if (typeof attachments === 'object') attachmentsRef.current = attachments;
 
   /*
@@ -103,9 +104,7 @@ export function RecordSetAttachments<SCHEMA extends AnySchema>({
     'scale'
   );
 
-  const isComplete = fetchedCount.current === records.length;
-  const downloadAllAttachmentsDisabled =
-    !isComplete || attachments?.attachments.length === 0;
+  const isComplete = fetchedCount.current === recordCount;
 
   return (
     <>
@@ -119,20 +118,12 @@ export function RecordSetAttachments<SCHEMA extends AnySchema>({
         <Dialog
           buttons={
             <>
-              <Button.Info
-                disabled={downloadAllAttachmentsDisabled}
-                title={attachmentsText.downloadAllDescription()}
-                onClick={(): void =>
-                  loading(
-                    downloadAllAttachments(
-                      attachmentsRef.current?.attachments ?? [],
-                      name
-                    )
-                  )
-                }
-              >
-                {attachmentsText.downloadAll()}
-              </Button.Info>
+              <DownloadAllAttachmentsButton
+                archiveName={name}
+                attachments={(recordSetId !== undefined && !isComplete) ? [] : attachmentsRef.current?.attachments ?? []}
+                recordSetId={recordSetId}
+                recordSetRequired={!isComplete}
+              />
               <Button.DialogClose>{commonText.close()}</Button.DialogClose>
             </>
           }
@@ -143,10 +134,15 @@ export function RecordSetAttachments<SCHEMA extends AnySchema>({
           header={
             attachmentsRef.current?.attachments === undefined
               ? attachmentsText.attachments()
-              : commonText.countLine({
+              : (isComplete ? 
+                commonText.countLine({
+                  resource: attachmentsText.attachments(),
+                  count: attachmentsRef.current.attachments.length
+                }) : 
+                commonText.countLineOrMore({
                   resource: attachmentsText.attachments(),
-                  count: attachmentsRef.current.attachments.length,
-                })
+                  count: attachmentsRef.current.attachments.length
+                }))
           }
           onClose={handleHideAttachments}
         >
@@ -187,4 +183,4 @@ export function RecordSetAttachments<SCHEMA extends AnySchema>({
       )}
     </>
   );
-}
+}