@tus/s3-store: Upload as multipart upload when incomplete part grows beyond minimal part size (#494)

@tus/s3-store: Upload as multipart upload when incomplete part grows

Author: mitjap

packages/s3-store/index.ts

@@ -249,6 +249,7 @@ export class S3Store extends DataStore {
         // eslint-disable-next-line no-async-promise-executor
         const deferred = new Promise<void>(async (resolve, reject) => {
           try {
+            let incompletePartSize = 0
             // Only the first chunk of each PATCH request can prepend
             // an incomplete part (last chunk) from the previous request.
             if (isFirstChunk) {
@@ -259,6 +260,7 @@ export class S3Store extends DataStore {
               if (incompletePart) {
                 // We found an incomplete part, prepend it to the chunk on disk we were about to upload,
                 // and delete the incomplete part from the bucket. This can be done in parallel.
+                incompletePartSize = incompletePart.length
                 await Promise.all([
                   this.prependIncompletePart(path, incompletePart),
                   this.deleteIncompletePart(metadata.file.id),
@@ -268,7 +270,7 @@ export class S3Store extends DataStore {
 
             const readable = fs.createReadStream(path)
             readable.on('error', reject)
-            if (partSize >= this.minPartSize || isFinalPart) {
+            if (partSize + incompletePartSize >= this.minPartSize || isFinalPart) {
               await this.uploadPart(metadata, readable, partNumber)
             } else {
               await this.uploadIncompletePart(metadata.file.id, readable)

packages/s3-store/test.ts

@@ -104,6 +104,42 @@ describe('S3DataStore', function () {
     }
   })
 
+  it('upload as multipart upload when incomplete part grows beyond minimal part size', async function () {
+    const store = this.datastore
+    const size = 10 * 1024 * 1024 // 10MB
+    const incompleteSize = 2 * 1024 * 1024 // 2MB
+    const getIncompletePart = sinon.spy(store, 'getIncompletePart')
+    const uploadIncompletePart = sinon.spy(store, 'uploadIncompletePart')
+    const uploadPart = sinon.spy(store, 'uploadPart')
+    const upload = new Upload({
+      id: 'incomplete-part-test-' + Uid.rand(),
+      size: size + incompleteSize,
+      offset: 0,
+    })
+
+    let offset = upload.offset
+
+    await store.create(upload)
+    offset = await store.write(
+      Readable.from(Buffer.alloc(incompleteSize)),
+      upload.id,
+      offset
+    )
+    offset = await store.write(
+      Readable.from(Buffer.alloc(incompleteSize)),
+      upload.id,
+      offset
+    )
+
+    assert.equal(getIncompletePart.called, true)
+    assert.equal(uploadIncompletePart.called, true)
+    assert.equal(uploadPart.called, false)
+
+    await store.write(Readable.from(Buffer.alloc(incompleteSize)), upload.id, offset)
+
+    assert.equal(uploadPart.called, true)
+  })
+
   it('should process chunk size of exactly the min size', async function () {
     this.datastore.minPartSize = 1024 * 1024 * 5
     const store = this.datastore