feat: utils index (#113)

Author: vasco-santos

packages/pack/src/writer.js

@@ -112,12 +112,13 @@ export class PackWriter {
     if (options?.notIndexContaining) {
       // Tee the stream for multiple writers
       const teedStreams = teeMultipleStreams(readable, this.indexWriters.length)
-      for (let i = 0; i < this.indexWriters.length; i++) {
-        // eslint-disable-next-line no-await-in-loop
-        await this.indexWriters[i].addBlobs(
-          streamIndexData(teedStreams[i].getReader())
+      const indexWriters = this.indexWriters
+      // Start processing all writers
+      await Promise.all(
+        teedStreams.map((stream, i) =>
+          indexWriters[i].addBlobs(streamIndexData(stream.getReader()))
         )
-      }
+      )
       return
     }
 
@@ -178,10 +179,14 @@ export class PackWriter {
       bufferedStream(),
       this.indexWriters.length
     )
-    for (let i = 0; i < this.indexWriters.length; i++) {
-      // eslint-disable-next-line no-await-in-loop
-      await this.indexWriters[i].addBlobs(streams[i], { containingMultihash })
-    }
+
+    // Start processing all writers
+    const indexWriters = this.indexWriters
+    await Promise.all(
+      indexWriters.map((indexWriter, i) =>
+        indexWriter.addBlobs(streams[i], { containingMultihash })
+      )
+    )
   }
 }
 

packages/streamer/src/index.js

@@ -85,9 +85,10 @@ export class HashStreamer {
     // 2. Read data for each Pack in batches and stream results
     for (const [encodedLocation, blobRanges] of locationsToRead.entries()) {
       const location = decodeLocation(encodedLocation)
+      const ranges = filterFullyCoveredRanges(blobRanges)
       for await (const { multihash, bytes } of this.packReader.stream(
         location,
-        blobRanges.length > 0 ? blobRanges : undefined // Read full pack if no ranges
+        ranges.length > 0 ? ranges : undefined // Read full pack if no ranges
       )) {
         yield {
           multihash,
@@ -128,6 +129,39 @@ function decodeLocation(encodedLocation) {
   throw new Error(`Invalid location type: ${encodedLocation}`)
 }
 
+/**
+ * Filters out ranges that fully cover other ranges.
+ *
+ * In other words, this removes any range that completely includes at least one other range.
+ *
+ * @param {Array<{
+ *   offset: number;
+ *   length: number;
+ *   multihash: API.MultihashDigest;
+ * }>} ranges - Array of ranges to filter.
+ * @returns {Array<{
+ *   offset: number;
+ *   length: number;
+ *   multihash: API.MultihashDigest;
+ * }>} Filtered array with large covering ranges removed.
+ */
+function filterFullyCoveredRanges(ranges) {
+  return ranges.filter((range, i) => {
+    // Check if this range fully covers any other range (except itself)
+    return !ranges.some((other, j) => {
+      if (i === j) return false
+
+      const rangeStart = range.offset
+      const rangeEnd = range.offset + range.length
+      const otherStart = other.offset
+      const otherEnd = other.offset + other.length
+
+      // Does current range fully cover the other range?
+      return rangeStart <= otherStart && rangeEnd >= otherEnd
+    })
+  })
+}
+
 /**
  * @enum {API.VerifiableBlobType}
  */

packages/utils/README.md

@@ -10,6 +10,93 @@ npm install @hash-stream/utils
 
 ## Usage
 
+### `index/unixfs`
+
+Supports building unixfs like indexes for content that is not at rest stored as content addressable data, but can be served likewise.
+
+#### `writeUnixFsFileLinkIndex`
+
+Creates an index of `FileLink` entries for a given blob using UnixFS layout and writes them to one or more `IndexWriter`s. It returns the multihash of the final chunk in the DAG (the "containing multihash").
+
+```ts
+import {
+  writeUnixFsFileLinkIndex,
+  defaultSettings,
+} from '@hash-stream/utils/index/unixfs'
+import { withMaxChunkSize } from '@ipld/unixfs/file/chunker/fixed'
+
+// Prepare your blob and writers
+const blob = new Blob(['hello world'])
+const indexWriters = [
+  /* your Hashstream IndexWriter instances */
+]
+
+const { containingMultihash } = await writeUnixFsFileLinkIndex(
+  blob,
+  '/file.txt',
+  indexWriters,
+  {
+    notIndexContaining: false,
+    settings: {
+      ...defaultSettings,
+      chunker: withMaxChunkSize(1024 * 1024),
+    },
+  }
+)
+```
+
+**Parameters:**
+
+- `blob` (`BlobLike`) – The file blob to be split into UnixFS chunks.
+- `path` (`string`) – Virtual path to associate with the entries in the index.
+- `indexWriters` (`IndexWriter[]`) – Array of writers that receive streamable index entries.
+- `options` (optional) (`CreateUnixFsFileLikeStreamOptions`):
+  - `notIndexContaining` (`boolean`) – If `true`, skips indexing the containing multihash in an hierarchy.
+  - `settings` (`Partial<UnixFSEncodeSettings>`) – Optional settings passed to the UnixFS writer.
+
+**Returns:** `Promise<{ containingMultihash: MultihashDigest } | undefined>`  
+The multihash of the final block, or `undefined` if no writers were provided.
+
+---
+
+#### `createUnixFsFileLinkStream`
+
+Creates a `ReadableStream` of `FileLink` entries that describe the byte layout and structure of the given blob encoded as UnixFS.
+
+```ts
+import {
+  createUnixFsFileLinkStream,
+  defaultSettings,
+} from '@hash-stream/utils/index/unixfs'
+import { withMaxChunkSize } from '@ipld/unixfs/file/chunker/fixed'
+
+const blob = new Blob(['example content'])
+
+const stream = createUnixFsFileLinkStream(blob, {
+  settings: {
+    ...defaultSettings,
+    chunker: withMaxChunkSize(1024 * 1024),
+  },
+})
+
+// Example: reading the stream
+const reader = stream.getReader()
+while (true) {
+  const { value, done } = await reader.read()
+  if (done) break
+  console.log(value) // FileLink
+}
+```
+
+**Parameters:**
+
+- `blob` (`BlobLike`) – The input blob to be chunked and streamed as UnixFS `FileLink` entries.
+- `options` (optional) (`CreateUnixFsFileLikeStreamOptions`) – Options to control chunking and encoding:
+  - `settings` (`Partial<UnixFSEncodeSettings>`) – Optional settings to configure the UnixFS encoder.
+
+**Returns:** `ReadableStream<FileLink>`  
+A stream of metadata entries (`FileLink`) describing the chunks and layout of the encoded UnixFS file.
+
 ### `trustless-ipfs-gateway`
 
 #### `streamer.asRawUint8Array`

packages/utils/package.json

@@ -26,7 +26,10 @@
   "exports": {
     "./types": "./dist/src/api.js",
     "./trustless-ipfs-gateway": "./dist/src/trustless-ipfs-gateway/index.js",
-    "./trustless-ipfs-gateway/types": "./dist/src/trustless-ipfs-gateway/api.js"
+    "./trustless-ipfs-gateway/types": "./dist/src/trustless-ipfs-gateway/api.js",
+    "./index": "./dist/src/index/index.js",
+    "./index/unixfs": "./dist/src/index/unixfs.js",
+    "./index/types": "./dist/src/index/api.js"
   },
   "typesVersions": {
     "*": {
@@ -38,6 +41,15 @@
       ],
       "trustless-ipfs-gateway/types": [
         "dist/src/trustless-ipfs-gateway/api.d.ts"
+      ],
+      "index": [
+        "dist/src/index/index.d.ts"
+      ],
+      "index/unixfs": [
+        "dist/src/index/unixfs.d.ts"
+      ],
+      "index/types": [
+        "dist/src/index/api.d.ts"
       ]
     }
   },
@@ -48,20 +60,24 @@
     "dist/src/**/*.d.ts.map"
   ],
   "dependencies": {
+    "@hash-stream/pack": "workspace:^",
     "@hash-stream/streamer": "workspace:^",
+    "@vascosantos/unixfs": "^3.0.3",
     "ipfs-core-utils": "^0.18.1",
     "multiformats": "^13.3.2",
     "uint8arrays": "^5.1.0"
   },
   "devDependencies": {
     "@ipld/car": "^5.4.0",
+    "@ipld/dag-pb": "^4.0.0",
     "@hash-stream/eslint-config": "workspace:^",
     "@hash-stream/index": "workspace:^",
     "@hash-stream/pack": "workspace:^",
     "@storacha/one-webcrypto": "^1.0.1",
     "@types/assert": "^1.5.11",
     "@types/mocha": "^10.0.10",
     "@types/node": "^22.13.10",
+    "@web3-storage/upload-client": "^17.1.4",
     "assert": "^2.1.0",
     "c8": "^10.1.3",
     "hundreds": "^0.0.9",

packages/utils/src/index/api.js

@@ -0,0 +1 @@
+export {}

packages/utils/src/index/api.ts

@@ -0,0 +1,9 @@
+import { MultihashDigest } from 'multiformats'
+import { UnixFSEncoderSettingsOptions } from '@web3-storage/upload-client/types'
+
+export interface CreateUnixFsFileLikeStreamOptions
+  extends UnixFSEncoderSettingsOptions {
+  notIndexContaining?: boolean
+}
+
+export type { MultihashDigest }

packages/utils/src/index/index.js

@@ -0,0 +1 @@
+export * as unixfs from './unixfs.js'