feat: storeDirectory accepts files as AsyncIterable<File> (#1920)

* client storeDirectory accepts an asyncIterable of files

* storeDirectory accepts an AsyncIterable, not just Iterable

Author: gobengo

packages/client/src/lib.js

@@ -24,6 +24,7 @@ import * as Token from './token.js'
 import { fetch, File, Blob, FormData, Blockstore } from './platform.js'
 import { toGatewayURL } from './gateway.js'
 import { BlockstoreCarReader } from './bs-car-reader.js'
+import pipe from 'it-pipe'
 
 const MAX_STORE_RETRIES = 5
 const MAX_CONCURRENT_UPLOADS = 3
@@ -215,7 +216,7 @@ class NFTStorage {
    * `foo/bla/baz.json` is ok but `foo/bar.png`, `bla/baz.json` is not.
    *
    * @param {Service} service
-   * @param {Iterable<File>} files
+   * @param {Iterable<File>|AsyncIterable<File>} files
    * @returns {Promise<CIDString>}
    */
   static async storeDirectory(service, files) {
@@ -455,29 +456,32 @@ class NFTStorage {
    * await client.storeCar(car)
    * ```
    *
-   * @param {Iterable<File>} files
+   * @param {Iterable<File>|AsyncIterable<File>} files
    * @param {object} [options]
    * @param {BlockstoreI} [options.blockstore]
    * @returns {Promise<{ cid: CID, car: CarReader }>}
    */
   static async encodeDirectory(files, { blockstore } = {}) {
-    const input = []
     let size = 0
-    for (const file of files) {
-      input.push(toImportCandidate(file.name, file))
-      size += file.size
-    }
-
+    const input = pipe(
+      isIterable(files) ? toAsyncIterable(files) : files,
+      async function* (files) {
+        for await (const file of files) {
+          yield toImportCandidate(file.name, file)
+          size += file.size
+        }
+      }
+    )
+    const packed = await packCar(input, {
+      blockstore,
+      wrapWithDirectory: true,
+    })
     if (size === 0) {
       throw new Error(
         'Total size of files should exceed 0, make sure to provide some content'
       )
     }
-
-    return packCar(input, {
-      blockstore,
-      wrapWithDirectory: true,
-    })
+    return packed
   }
 
   // Just a sugar so you don't have to pass around endpoint and token around.
@@ -557,7 +561,7 @@ class NFTStorage {
    * Argument can be a [FileList](https://developer.mozilla.org/en-US/docs/Web/API/FileList)
    * instance as well, in which case directory structure will be retained.
    *
-   * @param {Iterable<File>} files
+   * @param {AsyncIterable<File>|Iterable<File>} files
    */
   storeDirectory(files) {
     return NFTStorage.storeDirectory(this, files)
@@ -654,6 +658,29 @@ class NFTStorage {
   }
 }
 
+/**
+ * type guard checking for Iterable
+ * @param {any} x;
+ * @returns {x is Iterable<unknown>}
+ */
+function isIterable(x) {
+  return Symbol.iterator in x
+}
+
+/**
+ * Cast an iterable to an asyncIterable
+ * @template T
+ * @param {Iterable<T>} iterable
+ * @returns {AsyncIterable<T>}
+ */
+export function toAsyncIterable(iterable) {
+  return (async function* () {
+    for (const item of iterable) {
+      yield item
+    }
+  })()
+}
+
 /**
  * @template {import('./lib/interface.js').TokenInput} T
  * @param {T} metadata
@@ -686,7 +713,7 @@ For more context please see ERC-721 specification https://eips.ethereum.org/EIPS
 }
 
 /**
- * @param {Array<{ path: string, content: import('./platform.js').ReadableStream }>} input
+ * @param {import('ipfs-car/pack').ImportCandidateStream|Array<{ path: string, content: import('./platform.js').ReadableStream }>} input
  * @param {object} [options]
  * @param {BlockstoreI} [options.blockstore]
  * @param {boolean} [options.wrapWithDirectory]

packages/client/src/lib/interface.ts

@@ -100,7 +100,7 @@ export interface API {
    * be within the same directory, otherwise error is raised e.g. `foo/bar.png`,
    * `foo/bla/baz.json` is ok but `foo/bar.png`, `bla/baz.json` is not.
    */
-  storeDirectory(service: Service, files: Iterable<File>): Promise<CIDString>
+  storeDirectory(service: Service, files: Iterable<File>|AsyncIterable<File>): Promise<CIDString>
   /**
    * Returns current status of the stored NFT by its CID. Note the NFT must
    * have previously been stored by this account.

packages/client/test/lib.spec.js

@@ -9,6 +9,7 @@ import { pack } from 'ipfs-car/pack'
 import { CarWriter } from '@ipld/car'
 import * as dagJson from '@ipld/dag-json'
 import { randomCar } from './helpers.js'
+import { toAsyncIterable } from '../src/lib.js'
 
 const GATEWAY_LINK = 'nftstorage.link'
 
@@ -222,6 +223,24 @@ describe('client', () => {
       )
     })
 
+    it('upload multiple files as asyncIterable', async () => {
+      const client = new NFTStorage({ token, endpoint })
+      const cid = await client.storeDirectory(
+        toAsyncIterable([
+          new File(['hello world'], 'hello.txt'),
+          new File(
+            [JSON.stringify({ from: 'incognito' }, null, 2)],
+            'metadata.json'
+          ),
+        ])
+      )
+
+      assert.equal(
+        cid,
+        'bafybeigkms36pnnjsa7t2mq2g4mx77s4no2hilirs4wqx3eebbffy2ay3a'
+      )
+    })
+
     it('upload empty files', async () => {
       const client = new NFTStorage({ token, endpoint })
       try {

yarn.lock

@@ -1085,6 +1085,13 @@
     "@babel/template" "^7.16.7"
     "@babel/types" "^7.17.0"
 
+"@babel/helper-get-function-arity@^7.16.7":
+  version "7.16.7"
+  resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.16.7.tgz#ea08ac753117a669f1508ba06ebcc49156387419"
+  integrity sha512-flc+RLSOBXzNzVhcLu6ujeHUrD6tANAOU5ojrRx/as+tbzf8+stUCj7+IfRRoAbEZqj/ahXEMsjhOhgeZsrnTw==
+  dependencies:
+    "@babel/types" "^7.16.7"
+
 "@babel/helper-hoist-variables@^7.16.7":
   version "7.16.7"
   resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.16.7.tgz#86bcb19a77a509c7b77d0e22323ef588fa58c246"
@@ -16390,6 +16397,13 @@ react-debounce-input@=3.2.4:
     lodash.debounce "^4"
     prop-types "^15.7.2"
 
+react-device-detect@^2.2.2:
+  version "2.2.2"
+  resolved "https://registry.yarnpkg.com/react-device-detect/-/react-device-detect-2.2.2.tgz#dbabbce798ec359c83f574c3edb24cf1cca641a5"
+  integrity sha512-zSN1gIAztUekp5qUT/ybHwQ9fmOqVT1psxpSlTn1pe0CO+fnJHKRLOWWac5nKxOxvOpD/w84hk1I+EydrJp7SA==
+  dependencies:
+    ua-parser-js "^1.0.2"
+
 react-docgen-typescript@^2.1.1:
   version "2.2.2"
   resolved "https://registry.yarnpkg.com/react-docgen-typescript/-/react-docgen-typescript-2.2.2.tgz#4611055e569edc071204aadb20e1c93e1ab1659c"
@@ -18925,12 +18939,7 @@ typedoc@^0.22.14:
     minimatch "^5.0.1"
     shiki "^0.10.1"
 
-typescript@4.4.4:
-  version "4.4.4"
-  resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.4.4.tgz#2cd01a1a1f160704d3101fd5a58ff0f9fcb8030c"
-  integrity sha512-DqGhF5IKoBl8WNf8C1gu8q0xZSInh9j1kJJMqT3a94w1JzVaBU4EXOSMrz9yDqMT0xt3selp83fuFMQ0uzv6qA==
-
-typescript@4.5.3:
+typescript@4.4.4, typescript@4.5.3:
   version "4.5.3"
   resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.3.tgz#afaa858e68c7103317d89eb90c5d8906268d353c"
   integrity sha512-eVYaEHALSt+s9LbvgEv4Ef+Tdq7hBiIZgii12xXJnukryt3pMgJf6aKhoCZ3FWQsu6sydEnkg11fYXLzhLBjeQ==
@@ -18940,6 +18949,11 @@ typical@^6.0.1:
   resolved "https://registry.yarnpkg.com/typical/-/typical-6.0.1.tgz#89bd1a6aa5e5e96fa907fb6b7579223bff558a06"
   integrity sha512-+g3NEp7fJLe9DPa1TArHm9QAA7YciZmWnfAqEaFrBihQ7epOv9i99rjtgb6Iz0wh3WuQDjsCTDfgRoGnmHN81A==
 
+ua-parser-js@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-1.0.2.tgz#e2976c34dbfb30b15d2c300b2a53eac87c57a775"
+  integrity sha512-00y/AXhx0/SsnI51fTc0rLRmafiGOM4/O+ny10Ps7f+j/b8p/ZY11ytMgznXkOVo4GQ+KwQG5UQLkLGirsACRg==
+
 ucan-storage@^1.0.0:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/ucan-storage/-/ucan-storage-1.1.3.tgz#6874f887a8c7646f2d521f90d67345ac74532c59"