feat: allow encoding Data fields before Links in PBNode messages

To allow efficient reading of HAMT directory contents, encode Data
messages before Links.

This would be a breaking change as CIDs would change, so add an
option to allow users to opt-in to this new behaviour.

Refs:
 - ipld/ipld#383
 - ipfs/specs#550

Author: achingbrain

src/index.js

@@ -16,16 +16,18 @@ import { prepare, validate, createNode, createLink, toByteView } from './util.js
 /**
  * @typedef {import('./interface.js').PBLink} PBLink
  * @typedef {import('./interface.js').PBNode} PBNode
+ * @typedef {import('./interface.js').EncodeOptions} EncodeOptions
  */
 
 export const name = 'dag-pb'
 export const code = 0x70
 
 /**
  * @param {PBNode} node
+ * @param {EncodeOptions} [options]
  * @returns {ByteView<PBNode>}
  */
-export function encode (node) {
+export function encode (node, options) {
   validate(node)
 
   const pbn = {}
@@ -48,7 +50,7 @@ export function encode (node) {
     pbn.Data = node.Data
   }
 
-  return encodeNode(pbn)
+  return encodeNode(pbn, options)
 }
 
 /**

src/interface.ts

@@ -31,3 +31,13 @@ export interface RawPBNode {
   Data: Uint8Array
   Links: RawPBLink[]
 }
+
+export interface EncodeOptions {
+  /**
+   * By default the `Data` field in a `PBNode` message is encoded after all
+   * repeated `Links` elements.
+   *
+   * Pass `true` here to encode the `Data` field before any `Links`.
+   */
+  dataFirst?: boolean
+}

src/pb-encode.js

@@ -10,6 +10,10 @@ const maxUInt32 = 2 ** 31
  * @typedef {import('./interface.js').RawPBNode} RawPBNode
  */
 
+/**
+ * @typedef {import('./interface.js').EncodeOptions} EncodeOptions
+ */
+
 // the encoders work backward from the end of the bytes array
 
 /**
@@ -52,22 +56,35 @@ function encodeLink (link, bytes) {
   return bytes.length - i
 }
 
+/**
+ * @param {number} i
+ * @param {RawPBNode} node
+ * @param {Uint8Array} bytes
+ * @returns number
+ */
+function encodeData (i, node, bytes) {
+  i -= node.Data.length
+  bytes.set(node.Data, i)
+  i = encodeVarint(bytes, i, node.Data.length) - 1
+  bytes[i] = 0xa
+
+  return i
+}
+
 /**
  * Encodes a PBNode into a new byte array of precisely the correct size
  *
  * @param {RawPBNode} node
+ * @param {EncodeOptions} [options]
  * @returns {Uint8Array}
  */
-export function encodeNode (node) {
+export function encodeNode (node, options) {
   const size = sizeNode(node)
   const bytes = new Uint8Array(size)
   let i = size
 
-  if (node.Data) {
-    i -= node.Data.length
-    bytes.set(node.Data, i)
-    i = encodeVarint(bytes, i, node.Data.length) - 1
-    bytes[i] = 0xa
+  if (node.Data && options?.dataFirst !== true) {
+    i = encodeData(i, node, bytes)
   }
 
   if (node.Links) {
@@ -79,6 +96,10 @@ export function encodeNode (node) {
     }
   }
 
+  if (node.Data && options?.dataFirst === true) {
+    i = encodeData(i, node, bytes)
+  }
+
   return bytes
 }
 

test/test-basics.spec.js

@@ -85,6 +85,32 @@ describe('Basics', () => {
     assert.containSubset(linkCidsToStrings(node.Links), linkCidsToStrings(links))
   })
 
+  it('encode a node with links, data first', () => {
+    const dataBytes = '0a050001020304'
+    const linksBytes = '122f0a2212207521fe19c374a97759226dc5c0c8e674e73950e81b211f7dd3b6b30883a08a51120968656c6c6f2e747874'
+
+    const node = {
+      Data: Uint8Array.from([0, 1, 2, 3, 4]),
+      Links: [{
+        Name: 'hello.txt', Hash: CID.parse('QmWDtUQj38YLW8v3q4A6LwPn4vYKEbuKWpgSm6bjKW6Xfe')
+      }]
+    }
+    const defaultResult = encode(node)
+    const expectedDefaultBytes = `${linksBytes}${dataBytes}`
+    assert.instanceOf(defaultResult, Uint8Array)
+    assert.deepEqual(bytes.toHex(defaultResult), expectedDefaultBytes)
+
+    const dataFirstResult = encode(node, {
+      dataFirst: true
+    })
+    const expectedDataFirstBytes = `${dataBytes}${linksBytes}`
+    assert.instanceOf(dataFirstResult, Uint8Array)
+    assert.deepEqual(bytes.toHex(dataFirstResult), expectedDataFirstBytes)
+
+    assert.deepEqual(decode(defaultResult), node)
+    assert.deepEqual(decode(dataFirstResult), node)
+  })
+
   it('ignore invalid properties when preparing', () => {
     const prepared = prepare({ foo: 'bar' })
     assert.deepEqual(prepared, { Links: [] })