fix: constrain identity bytes to single characters (#338)

Use the char code API to interpret bytes as single characters to instead of using lossy utf-8 conversion.

Fixes: #122

Author: achingbrain

src/bases/identity.ts

@@ -1,9 +1,9 @@
-import { fromString, toString } from '../bytes.ts'
+import { coerce, fromString, toString } from '../bytes.ts'
 import { from } from './base.ts'
 
 export const identity = from({
   prefix: '\x00',
   name: 'identity',
-  encode: (buf) => toString(buf),
-  decode: (str) => fromString(str)
+  encode: (buf) => toString(coerce(buf)),
+  decode: (str) => coerce(fromString(str))
 })

src/bytes.ts

@@ -24,16 +24,16 @@ export function equals (aa: Uint8Array, bb: Uint8Array): boolean {
   return true
 }
 
-/**
- * Normalize binary input to a plain `Uint8Array` backed by an `ArrayBuffer`.
- *
- * Returns the input itself when it is already a plain `Uint8Array` over an
- * `ArrayBuffer`, otherwise a fresh view (or, for `SharedArrayBuffer`-backed
- * input, a copy) over the same bytes.
- *
- * Throws if input is not a recognised binary type.
- */
-export function coerce (o: ArrayBufferView | ArrayBuffer | Uint8Array): Uint8Array<ArrayBuffer> {
+/**
+ * Normalize binary input to a plain `Uint8Array` backed by an `ArrayBuffer`.
+ *
+ * Returns the input itself when it is already a plain `Uint8Array` over an
+ * `ArrayBuffer`, otherwise a fresh view (or, for `SharedArrayBuffer`-backed
+ * input, a copy) over the same bytes.
+ *
+ * Throws if input is not a recognised binary type.
+ */
+export function coerce (o: ArrayBufferView | ArrayBuffer | Uint8Array): Uint8Array<ArrayBuffer> {
   if (o instanceof Uint8Array && o.constructor.name === 'Uint8Array') {
     return toArrayBufferBackedArray(o)
   }
@@ -50,12 +50,48 @@ export function isBinary (o: unknown): o is ArrayBuffer | ArrayBufferView {
   return o instanceof ArrayBuffer || ArrayBuffer.isView(o)
 }
 
+/**
+ * Convert the passed string into a byte array, constraining each character
+ * value to a single byte
+ */
 export function fromString (str: string): Uint8Array<ArrayBuffer> {
-  return new TextEncoder().encode(str)
+  const output = new Uint8Array(str.length)
+
+  for (let i = 0; i < str.length; i++) {
+    output[i] = str.charCodeAt(i)
+  }
+
+  return output
 }
 
+// Based on http://stackoverflow.com/a/22747272/680742, the browser with
+// the lowest limit is Chrome, with 0x10000 args.
+// We go 1 magnitude less, for safety
+const MAX_ARGUMENTS_LENGTH = 0x1000
+
+/**
+ * Convert the passed byte array to a string, interpreting each byte as a single
+ * character
+ */
 export function toString (b: Uint8Array): string {
-  return new TextDecoder().decode(b)
+  const len = b.length
+
+  if (len <= MAX_ARGUMENTS_LENGTH) {
+    // @ts-expect-error cannot ordinarily apply a Uint8Array
+    return String.fromCharCode.apply(String, b) // avoid extra subarray()
+  }
+
+  // Decode in chunks to avoid "call stack size exceeded".
+  let res = ''
+  let i = 0
+  while (i < len) {
+    res += String.fromCharCode.apply(
+      String,
+      // @ts-expect-error cannot ordinarily apply a Uint8Array
+      b.subarray(i, i += MAX_ARGUMENTS_LENGTH)
+    )
+  }
+  return res
 }
 
 function isByteArrayWithArrayBuffer (b?: Uint8Array): b is Uint8Array<ArrayBuffer> {

test/test-multibase.spec.ts

@@ -9,6 +9,7 @@ import * as b36 from '../src/bases/base36.ts'
 import * as b58 from '../src/bases/base58.ts'
 import * as b64 from '../src/bases/base64.ts'
 import * as b8 from '../src/bases/base8.ts'
+import * as id from '../src/bases/identity.ts'
 import * as bytes from '../src/bytes.ts'
 
 const { base16, base32, base58btc, base64 } = { ...b16, ...b32, ...b58, ...b64 }
@@ -64,7 +65,7 @@ describe('multibase', () => {
   const buff = bytes.fromString('test')
   const nonPrintableBuff = Uint8Array.from([239, 250, 254])
 
-  const baseTest = (bases: typeof b2 | typeof b8 | typeof b10 | typeof b16 | typeof b32 | typeof b36 | typeof b58 | typeof b64): void => {
+  const baseTest = (bases: typeof b2 | typeof b8 | typeof b10 | typeof b16 | typeof b32 | typeof b36 | typeof b58 | typeof b64 | typeof id): void => {
     for (const base of Object.values(bases)) {
       if (((base as { name: string })?.name) !== '') {
         it(`encode/decode ${base.name}`, () => {
@@ -118,6 +119,46 @@ describe('multibase', () => {
     baseTest(b64)
   })
 
+  describe('identity', () => {
+    baseTest(id)
+
+    it('should round-trip unprintable characters', () => {
+      const u = new Uint8Array([
+        6, 22, 184, 240, 237, 178,
+        112, 0, 150, 137, 182, 54,
+        220, 1, 217, 221
+      ])
+
+      const s = id.identity.encode(u)
+      const b = id.identity.decode(s)
+
+      assert.equalBytes(b, u)
+    })
+
+    it('should round-trip emojis', () => {
+      const input = '😵‍💫🎉'
+      const u = new TextEncoder().encode(input)
+      const s = id.identity.encode(u)
+      const b = id.identity.decode(s)
+      const output = new TextDecoder().decode(b)
+
+      assert.equalBytes(b, u)
+      assert.equal(output, input)
+    })
+
+    it('should round-trip multi-byte characters', () => {
+      // https://www.kanshudo.com/kanji/%F0%A0%AE%B7
+      const input = '𠮷'
+      const u = new TextEncoder().encode(input)
+      const s = id.identity.encode(u)
+      const b = id.identity.decode(s)
+      const output = new TextDecoder().decode(b)
+
+      assert.equalBytes(b, u)
+      assert.equal(output, input)
+    })
+  })
+
   it('multibase mismatch', () => {
     const b64 = base64.encode(bytes.fromString('test'))
     const msg = `Unable to decode multibase string "${b64}", base32 decoder only supports inputs prefixed with ${base32.prefix}`
@@ -158,7 +199,7 @@ describe('multibase', () => {
     assert.throws(() => base64.decode(b64.substring(0, b64.length - 1)), 'Unexpected end of data')
   })
 
-  it('infers prefix and name corretly', () => {
+  it('infers prefix and name correctly', () => {
     const name = base32.name
 
     // @ts-expect-error - TS catches mismatch