@tus/s3-store: Fix invalid chunk size for uploads with deferred length (#504)

* @tus/s3-store: Fix invalid chunk size for uploads with deferred length
* fix documentation regarding part size limits
* fix units to use binary prefix (KiB, MiB, GiB, TiB)

Author: mitjap

demo/server.js

@@ -29,7 +29,7 @@ const stores = {
     assert.ok(process.env.AWS_REGION, 'environment variable `AWS_REGION` must be set')
 
     return new S3Store({
-      partSize: 8 * 1024 * 1024, // each uploaded part will have ~8MB,
+      partSize: 8 * 1024 * 1024, // each uploaded part will have ~8MiB,
       s3ClientConfig: {
         bucket: process.env.AWS_BUCKET,
         accessKeyId: process.env.AWS_ACCESS_KEY_ID,

packages/s3-store/README.md

@@ -33,7 +33,7 @@ const {Server} = require('@tus/server')
 const {S3Store} = require('@tus/s3-store')
 
 const s3Store = new S3Store({
-  partSize: 8 * 1024 * 1024, // Each uploaded part will have ~8MB,
+  partSize: 8 * 1024 * 1024, // Each uploaded part will have ~8MiB,
   s3ClientConfig: {
     bucket: process.env.AWS_BUCKET,
     region: process.env.AWS_REGION,
@@ -61,7 +61,7 @@ The bucket name.
 
 #### `options.partSize`
 
-The preferred part size for parts send to S3. Can not be lower than 5MB or more than 500MB.
+The preferred part size for parts send to S3. Can not be lower than 5MiB or more than 5GiB.
 The server calculates the optimal part size, which takes this size into account,
 but may increase it to not exceed the S3 10K parts limit.
 

packages/s3-store/index.ts

@@ -18,7 +18,7 @@ function calcOffsetFromParts(parts?: Array<AWS.Part>) {
 }
 
 type Options = {
-  // The preferred part size for parts send to S3. Can not be lower than 5MB or more than 500MB.
+  // The preferred part size for parts send to S3. Can not be lower than 5MiB or more than 5GiB.
   // The server calculates the optimal part size, which takes this size into account,
   // but may increase it to not exceed the S3 10K parts limit.
   partSize?: number
@@ -70,7 +70,8 @@ export class S3Store extends DataStore {
   private client: S3
   private preferredPartSize: number
   public maxMultipartParts = 10_000 as const
-  public minPartSize = 5_242_880 as const // 5MB
+  public minPartSize = 5_242_880 as const // 5MiB
+  public maxUploadSize = 5_497_558_138_880 as const // 5TiB
 
   constructor(options: Options) {
     super()
@@ -262,7 +263,7 @@ export class S3Store extends DataStore {
     currentPartNumber: number,
     offset: number
   ): Promise<number> {
-    const size = metadata.file.size as number
+    const size = metadata.file.size
     const promises: Promise<void>[] = []
     let pendingChunkFilepath: string | null = null
     let bytesUploaded = 0
@@ -422,7 +423,12 @@ export class S3Store extends DataStore {
     this.cache.delete(id)
   }
 
-  private calcOptimalPartSize(size: number): number {
+  private calcOptimalPartSize(size?: number): number {
+    // When upload size is not know we assume largest possible value (`maxUploadSize`)
+    if (size === undefined) {
+      size = this.maxUploadSize
+    }
+
     let optimalPartSize: number
 
     // When upload is smaller or equal to PreferredPartSize, we upload in just one part.

packages/s3-store/test.ts

@@ -21,7 +21,7 @@ describe('S3DataStore', function () {
   })
   beforeEach(function () {
     this.datastore = new S3Store({
-      partSize: 8 * 1024 * 1024, // Each uploaded part will have ~8MB,
+      partSize: 8 * 1024 * 1024, // Each uploaded part will have ~8MiB,
       s3ClientConfig: {
         bucket: process.env.AWS_BUCKET as string,
         credentials: {
@@ -33,6 +33,12 @@ describe('S3DataStore', function () {
     })
   })
 
+  it('calculated part size for deferred lenght should be finite', async function () {
+    const store = this.datastore
+
+    assert.strictEqual(Number.isFinite(store.calcOptimalPartSize(undefined)), true)
+  })
+
   it('should correctly prepend a buffer to a file', async function () {
     const p = path.resolve(fixturesPath, 'foo.txt')
     await fs.writeFile(p, 'world!')
@@ -109,8 +115,8 @@ describe('S3DataStore', function () {
 
   it('upload as multipart upload when incomplete part grows beyond minimal part size', async function () {
     const store = this.datastore
-    const size = 10 * 1024 * 1024 // 10MB
-    const incompleteSize = 2 * 1024 * 1024 // 2MB
+    const size = 10 * 1024 * 1024 // 10MiB
+    const incompleteSize = 2 * 1024 * 1024 // 2MiB
     const getIncompletePart = sinon.spy(store, 'getIncompletePart')
     const uploadIncompletePart = sinon.spy(store, 'uploadIncompletePart')
     const uploadPart = sinon.spy(store, 'uploadPart')