Merge branch 'main' of github.com:holepunchto/hypercore into use-safetyCatch

# Conflicts:
#	index.js
#	lib/batch.js

Author: HDegroote

index.js

@@ -10,16 +10,18 @@ const Protomux = require('protomux')
 const z32 = require('z32')
 const id = require('hypercore-id-encoding')
 const safetyCatch = require('safety-catch')
+const { createTracer } = require('hypertrace')
 
 const Replicator = require('./lib/replicator')
 const Core = require('./lib/core')
 const BlockEncryption = require('./lib/block-encryption')
 const Info = require('./lib/info')
 const Download = require('./lib/download')
 const Batch = require('./lib/batch')
-const { manifestHash, defaultSignerManifest, createVerifier, createManifest, isCompat } = require('./lib/manifest')
+const { manifestHash, defaultSignerManifest, createManifest, isCompat, sign } = require('./lib/verifier')
 const { ReadStream, WriteStream, ByteStream } = require('./lib/streams')
 const {
+  ASSERTION,
   BAD_ARGUMENT,
   SESSION_CLOSED,
   SESSION_NOT_WRITABLE,
@@ -29,6 +31,10 @@ const {
 const promises = Symbol.for('hypercore.promises')
 const inspect = Symbol.for('nodejs.util.inspect.custom')
 
+// Hypercore actually does not have any notion of max/min block sizes
+// but we enforce 15mb to ensure smooth replication (each block is transmitted atomically)
+const MAX_SUGGESTED_BLOCK_SIZE = 15 * 1024 * 1024
+
 module.exports = class Hypercore extends EventEmitter {
   constructor (storage, key, opts) {
     super()
@@ -49,6 +55,7 @@ module.exports = class Hypercore extends EventEmitter {
 
     this[promises] = true
 
+    this.tracer = createTracer(this)
     this.storage = null
     this.crypto = opts.crypto || hypercoreCrypto
     this.core = null
@@ -133,8 +140,10 @@ module.exports = class Hypercore extends EventEmitter {
       indent + ')'
   }
 
+  static MAX_SUGGESTED_BLOCK_SIZE = MAX_SUGGESTED_BLOCK_SIZE
+
   static key (manifest, { compat } = {}) {
-    return compat ? manifest.signer.publicKey : manifestHash(createManifest(manifest))
+    return compat ? manifest.signers[0].publicKey : manifestHash(createManifest(manifest))
   }
 
   static discoveryKey (key) {
@@ -267,6 +276,8 @@ module.exports = class Hypercore extends EventEmitter {
     this.writable = this._isWritable()
     this.autoClose = o.autoClose
 
+    if (o.core) this.tracer.setParent(o.core.tracer)
+
     if (this.snapshotted && this.core && !this._snapshot) this._updateSnapshot()
   }
 
@@ -377,6 +388,7 @@ module.exports = class Hypercore extends EventEmitter {
       onupdate: this._oncoreupdate.bind(this),
       onconflict: this._oncoreconflict.bind(this)
     })
+    this.tracer.setParent(this.core.tracer)
 
     if (opts.userData) {
       for (const [key, value] of Object.entries(opts.userData)) {
@@ -490,14 +502,14 @@ module.exports = class Hypercore extends EventEmitter {
     }
 
     const manifest = opts.manifest || defaultSignerManifest(keyPair.publicKey)
-    const key = opts.key || (opts.compat !== false ? manifest.signer.publicKey : manifestHash(manifest))
+    const key = opts.key || (opts.compat !== false ? manifest.signers[0].publicKey : manifestHash(manifest))
 
     if (b4a.equals(key, this.key)) {
       throw BAD_ARGUMENT('Clone cannot share verification information')
     }
 
     const signature = opts.signature === undefined
-      ? createVerifier(createManifest(manifest), { compat: isCompat(key, manifest) }).sign(this.core.tree.batch(), keyPair)
+      ? sign(createManifest(manifest), this.core.tree.batch(), keyPair, { compat: isCompat(key, manifest) })
       : opts.signature
 
     const sparse = opts.sparse === false ? false : this.sparse
@@ -809,12 +821,13 @@ module.exports = class Hypercore extends EventEmitter {
     return true
   }
 
-  batch ({ checkout = -1, autoClose = true, session = true, restore = false } = {}) {
-    return new Batch(session ? this.session() : this, checkout, autoClose, restore)
+  batch ({ checkout = -1, autoClose = true, session = true, restore = false, clear = false } = {}) {
+    return new Batch(session ? this.session() : this, checkout, autoClose, restore, clear)
   }
 
   async seek (bytes, opts) {
     if (this.opened === false) await this.opening
+    if (!isValidIndex(bytes)) throw ASSERTION('seek is invalid')
 
     const tree = (opts && opts.tree) || this.core.tree
     const s = tree.seek(bytes, this.padding)
@@ -837,17 +850,20 @@ module.exports = class Hypercore extends EventEmitter {
 
   async has (start, end = start + 1) {
     if (this.opened === false) await this.opening
+    if (!isValidIndex(start) || !isValidIndex(end)) throw ASSERTION('has range is invalid')
 
-    const length = end - start
-    if (length <= 0) return false
-    if (length === 1) return this.core.bitfield.get(start)
+    if (end === start + 1) return this.core.bitfield.get(start)
 
     const i = this.core.bitfield.firstUnset(start)
     return i === -1 || i >= end
   }
 
   async get (index, opts) {
     if (this.opened === false) await this.opening
+    if (!isValidIndex(index)) throw ASSERTION('block index is invalid')
+
+    this.tracer.trace('get', { index })
+
     if (this.closing !== null) throw SESSION_CLOSED()
     if (this._snapshot !== null && index >= this._snapshot.compatLength) throw SNAPSHOT_NOT_AVAILABLE()
 
@@ -878,6 +894,8 @@ module.exports = class Hypercore extends EventEmitter {
       end = start + 1
     }
 
+    if (!isValidIndex(start) || !isValidIndex(end)) throw ASSERTION('clear range is invalid')
+
     const cleared = (opts && opts.diff) ? { blocks: 0 } : null
 
     if (start >= end) return cleared
@@ -910,6 +928,7 @@ module.exports = class Hypercore extends EventEmitter {
       const activeRequests = (opts && opts.activeRequests) || this.activeRequests
 
       const req = this.replicator.addBlock(activeRequests, index)
+      req.snapshot = index < this.length
 
       const timeout = opts && opts.timeout !== undefined ? opts.timeout : this.timeout
       if (timeout) req.context.setTimeout(req, timeout)
@@ -962,6 +981,8 @@ module.exports = class Hypercore extends EventEmitter {
   async _download (range) {
     if (this.opened === false) await this.opening
 
+    this.tracer.trace('download', { range })
+
     const activeRequests = (range && range.activeRequests) || this.activeRequests
 
     return this.replicator.addRange(activeRequests, range)
@@ -1004,6 +1025,7 @@ module.exports = class Hypercore extends EventEmitter {
     if (writable === false) throw SESSION_NOT_WRITABLE()
 
     blocks = Array.isArray(blocks) ? blocks : [blocks]
+    this.tracer.trace('append', { blocks })
 
     const preappend = this.encryption && this._preappend
 
@@ -1014,6 +1036,11 @@ module.exports = class Hypercore extends EventEmitter {
         buffers[i] = this._encode(this.valueEncoding, blocks[i])
       }
     }
+    for (const b of buffers) {
+      if (b.byteLength > MAX_SUGGESTED_BLOCK_SIZE) {
+        throw BAD_ARGUMENT('Appended block exceeds the maximum suggested block size')
+      }
+    }
 
     return this.core.append(buffers, { keyPair, signature, preappend })
   }
@@ -1133,3 +1160,7 @@ function ensureEncryption (core, opts) {
 function createCache (cache) {
   return cache === true ? new Xache({ maxSize: 65536, maxAge: 0 }) : (cache || null)
 }
+
+function isValidIndex (index) {
+  return index === 0 || index > 0
+}

lib/batch.js

@@ -5,7 +5,7 @@ const b4a = require('b4a')
 const safetyCatch = require('safety-catch')
 
 module.exports = class HypercoreBatch extends EventEmitter {
-  constructor (session, checkoutLength, autoClose, restore) {
+  constructor (session, checkoutLength, autoClose, restore, clear) {
     super()
 
     this.session = session
@@ -26,8 +26,10 @@ module.exports = class HypercoreBatch extends EventEmitter {
     this._sessionByteLength = 0
     this._sessionBatch = null
     this._flushing = null
+    this._clear = clear
 
-    this.opening = this.ready().catch(safetyCatch)
+    this.opening = this._open()
+    this.opening.catch(safetyCatch)
   }
 
   get id () {
@@ -70,23 +72,27 @@ module.exports = class HypercoreBatch extends EventEmitter {
     return this.session.manifest
   }
 
-  async ready () {
+  ready () {
+    return this.opening
+  }
+
+  async _open () {
     await this.session.ready()
-    if (this.opened) return
+
+    if (this._clear) this._checkoutLength = this.core.tree.length
 
     if (this._checkoutLength !== -1) {
       const batch = await this.session.core.tree.restoreBatch(this._checkoutLength)
       batch.treeLength = this._checkoutLength
-      if (this.opened) return
       this._sessionLength = batch.length
       this._sessionByteLength = batch.byteLength
       this._sessionBatch = batch
+      if (this._clear) await this.core.clearBatch()
     } else {
       const last = this.restore ? this.session.core.bitfield.findFirst(false, this.session.length) : 0
 
       if (last > this.session.length) {
         const batch = await this.session.core.tree.restoreBatch(last)
-        if (this.opened) return
         this._sessionLength = batch.length
         this._sessionByteLength = batch.byteLength - this.session.padding * batch.length
         this._sessionBatch = batch
@@ -188,9 +194,9 @@ module.exports = class HypercoreBatch extends EventEmitter {
     }
   }
 
-  async restoreBatch (length) {
+  async restoreBatch (length, blocks) {
     if (this.opened === false) await this.opening
-    if (length >= this._sessionLength) return this.createTreeBatch(length)
+    if (length >= this._sessionLength) return this.createTreeBatch(length, blocks)
     return this.session.core.tree.restoreBatch(length)
   }
 

lib/block-store.js

@@ -1,4 +1,5 @@
 const b4a = require('b4a')
+const { WRITE_FAILED } = require('hypercore-errors')
 
 module.exports = class BlockStore {
   constructor (storage, tree) {
@@ -54,7 +55,7 @@ module.exports = class BlockStore {
   _write (offset, data) {
     return new Promise((resolve, reject) => {
       this.storage.write(offset, data, (err) => {
-        if (err) reject(err)
+        if (err) reject(WRITE_FAILED(err.message))
         else resolve(offset + data.byteLength)
       })
     })

lib/caps.js

@@ -25,11 +25,11 @@ exports.replicate = function (isInitiator, key, handshakeHash) {
   return out
 }
 
-exports.treeSignable = function (namespace, hash, length, fork) {
+exports.treeSignable = function (manifestHash, treeHash, length, fork) {
   const state = { start: 0, end: 112, buffer: b4a.allocUnsafe(112) }
-  c.raw.encode(state, TREE)
-  c.raw.encode(state, namespace)
-  c.raw.encode(state, hash)
+  c.fixed32.encode(state, TREE)
+  c.fixed32.encode(state, manifestHash)
+  c.fixed32.encode(state, treeHash)
   c.uint64.encode(state, length)
   c.uint64.encode(state, fork)
   return state.buffer
@@ -38,8 +38,8 @@ exports.treeSignable = function (namespace, hash, length, fork) {
 exports.treeSignableCompat = function (hash, length, fork, noHeader) {
   const end = noHeader ? 48 : 80
   const state = { start: 0, end, buffer: b4a.allocUnsafe(end) }
-  if (!noHeader) c.raw.encode(state, TREE) // ultra legacy mode, kill in future major
-  c.raw.encode(state, hash)
+  if (!noHeader) c.fixed32.encode(state, TREE) // ultra legacy mode, kill in future major
+  c.fixed32.encode(state, hash)
   c.uint64.encode(state, length)
   c.uint64.encode(state, fork)
   return state.buffer