feat: custom chunk size (#2396)

Since we upped the default chunk size, folks uploading directories with
thousands of small files are sometimes finding the worker exceeds
resource limits 🤦‍♂️. This allows them to `encodeDirectory` and then
`storeCar` with a smaller chunk size to allow the directory to upload
successfully.

e.g.

```js
import { NFTStorage } from 'nft.storage'
import { filesFromPaths } from 'files-from-path'

const token = 'API_KEY' // your API key from https://nft.storage/manage

async function main() {
  const storage = new NFTStorage({ token })
  const files = await filesFromPaths(['/path/to/files'])
  
  const { cid, car } = await NFTStorage.encodeDirectory(files)
  console.log(`File CID: ${cid}`)

  console.log('Sending file...')
  await storage.storeCar(car, {
    maxChunkSize: 1024 * 1024 * 10, // 10MB
    onStoredChunk: (size) => console.log(`Stored a chunk of ${size} bytes`)
  })

  console.log('✅ Done')
}
main()
```

Author: Alan Shaw

packages/client/src/lib.js

@@ -169,11 +169,11 @@ class NFTStorage {
   static async storeCar(
     { endpoint, rateLimiter = globalRateLimiter, ...token },
     car,
-    { onStoredChunk, maxRetries, decoders, signal } = {}
+    { onStoredChunk, maxRetries, maxChunkSize, decoders, signal } = {}
   ) {
     const url = new URL('upload/', endpoint)
     const headers = NFTStorage.auth(token)
-    const targetSize = MAX_CHUNK_SIZE
+    const targetSize = maxChunkSize || MAX_CHUNK_SIZE
     const splitter =
       car instanceof Blob
         ? await TreewalkCarSplitter.fromBlob(car, targetSize, { decoders })

packages/client/src/lib/interface.ts

@@ -150,6 +150,10 @@ export interface CarStorerOptions extends RequestOptions {
    * Maximum times to retry a failed upload. Default: 5
    */
   maxRetries?: number
+  /**
+   * Maximum chunk size to upload in bytes. Default: 52,428,800
+   */
+  maxChunkSize?: number
   /**
    * Additional IPLD block decoders. Used to interpret the data in the CAR
    * file and split it into multiple chunks. Note these are only required if

packages/client/test/lib.spec.js

@@ -195,6 +195,29 @@ describe('client', () => {
       assert.equal(cid, expectedCid)
     })
 
+    it('upload CAR with custom chunk size', async function () {
+      let uploadedChunks = 0
+
+      const client = new NFTStorage({ token, endpoint })
+
+      const targetSize = 1024 * 1024 * 20
+      const carReader = await CarReader.fromIterable(
+        await randomCar(targetSize)
+      )
+
+      const roots = await carReader.getRoots()
+      const expectedCid = roots[0]?.toString()
+
+      const cid = await client.storeCar(carReader, {
+        maxChunkSize: 1024 * 1024 * 15,
+        onStoredChunk: () => {
+          uploadedChunks++
+        },
+      })
+      assert.equal(uploadedChunks, 2)
+      assert.equal(cid, expectedCid)
+    })
+
     it('upload CAR with non-default decoder', async () => {
       const client = new NFTStorage({ token, endpoint })
       const block = await encode({