Major Update of Borsh (#65)

* indexed by class name instead of the class itself

* serializer 1.0

* Implemented deserializer

* Fixed indentation

* Added schema validation

* minor improvements

* added more tests

* added more tests

* added more tests

* updated readme

* minor fix to examples

* bump in version

* minor update to README.md

* minor update to README.md

* trigger actions

* Removed unnecesary packages + fixed lint

* simplified buffer

* added base encode/decode

* implemented enums and removed deserializing of classes

* better organized testing

* exported schema

* Added forgotten schemas to schema type

* allowing numbers in BN

* schema now leads serialization order

* bump version

* feat: allow strings in BN

* feat: more tests & checkSchema flag

* fix: made compatible to ES5

* updated readme

* feat: building cjs & esm

* feat: cjs & esm working versions

* removed BN.js & bs58

* simplified tests

* small change in bigint method

* added compatibility with BN

Author: gagdiez

.build_scripts/prepare-package-json.js

@@ -0,0 +1,29 @@
+const fs = require('fs');
+const path = require('path');
+
+const buildDir = './lib';
+function createEsmModulePackageJson() {
+    fs.readdir(buildDir, function (err, dirs) {
+        if (err) {
+            throw err;
+        }
+        dirs.forEach(function (dir) {
+            if (dir === 'esm') {
+                var packageJsonFile = path.join(buildDir, dir, '/package.json');
+                if (!fs.existsSync(packageJsonFile)) {
+                    fs.writeFile(
+                        packageJsonFile,
+                        new Uint8Array(Buffer.from('{"type": "module"}')),
+                        function (err) {
+                            if (err) {
+                                throw err;
+                            }
+                        }
+                    );
+                }
+            }
+        });
+    });
+}
+
+createEsmModulePackageJson();

README.md

@@ -14,37 +14,76 @@ Borsh stands for _Binary Object Representation Serializer for Hashing_. It is me
 safety, speed, and comes with a strict specification.
 
 ## Examples
-### Serializing an object
+
+### (De)serializing a Value
 ```javascript
-const value = new Test({ x: 255, y: 20, z: '123', q: [1, 2, 3] });
-const schema = new Map([[Test, { kind: 'struct', fields: [['x', 'u8'], ['y', 'u64'], ['z', 'string'], ['q', [3]]] }]]);
-const buffer = borsh.serialize(schema, value);
+import * as borsh from 'borsh';
+
+const encodedU16 = borsh.serialize('u16', 2);
+const decodedU16 = borsh.deserialize('u16', encodedU16);
+
+const encodedStr = borsh.serialize('string', 'testing');
+const decodedStr = borsh.deserialize('string', encodedStr);
 ```
 
-### Deserializing an object
+### (De)serializing an Object
 ```javascript
-const newValue = borsh.deserialize(schema, Test, buffer);
+import * as borsh from 'borsh';
+
+const value = {x: 255, y: BigInt(20), z: '123', arr: [1, 2, 3]};
+const schema = { struct: { x: 'u8', y: 'u64', 'z': 'string', 'arr': { array: { type: 'u8' }}}};
+
+const encoded = borsh.serialize(schema, value);
+const decoded = borsh.deserialize(schema, encoded);
 ```
 
-## Type Mappings
-
-| Borsh                 | TypeScript     |
-|-----------------------|----------------|
-| `u8` integer          | `number`       |
-| `u16` integer         | `number`       |
-| `u32` integer         | `number`       |
-| `u64` integer         | `BN`           |
-| `u128` integer        | `BN`           |
-| `u256` integer        | `BN`           |
-| `u512` integer        | `BN`           |
-| `f32` float           | N/A            |
-| `f64` float           | N/A            |
-| fixed-size byte array | `Uint8Array`   |
-| UTF-8 string          | `string`       |
-| option                | `null` or type |
-| map                   | N/A            |
-| set                   | N/A            |
-| structs               | `any`          |
+## API
+The package exposes the following functions:
+- `serialize(schema: Schema, obj: any): Uint8Array` - serializes an object `obj` according to the schema `schema`.
+- `deserialize(schema: Schema, buffer: Uint8Array, class?: Class): any` - deserializes an object according to the schema `schema` from the buffer `buffer`. If the optional parameter `class` is present, the deserialized object will be an of `class`.
+
+## Schemas
+Schemas are used to describe the structure of the data being serialized or deserialized. They are used to
+validate the data and to determine the order of the fields in the serialized data.
+
+> NOTE: You can find examples of valid in the [test](./borsh-ts/test/utils.test.js) folder.
+
+### Basic Types
+Basic types are described by a string. The following types are supported:
+- `u8`, `u16`, `u32`, `u64`, `u128` - unsigned integers of 8, 16, 32, 64, and 128 bits respectively.
+- `i8`, `i16`, `i32`, `i64`, `i128` - signed integers of 8, 16, 32, 64, and 128 bits respectively.
+- `f32`, `f64` - IEEE 754 floating point numbers of 32 and 64 bits respectively.
+- `bool` - boolean value.
+- `string` - UTF-8 string.
+
+### Arrays, Options, Maps, Sets, Enums, and Structs
+More complex objects are described by a JSON object. The following types are supported:
+- `{ array: { type: Schema, len?: number } }` - an array of objects of the same type. The type of the array elements is described by the `type` field. If the field `len` is present, the array is fixed-size and the length of the array is `len`. Otherwise, the array is dynamic-sized and the length of the array is serialized before the elements.
+- `{ option: Schema }` - an optional object. The type of the object is described by the `type` field.
+- `{ map: { key: Schema, value: Schema }}` - a map. The type of the keys and values are described by the `key` and `value` fields respectively.
+- `{ set: Schema }` - a set. The type of the elements is described by the `type` field.
+- `{ enum: [{ className1: { struct: {...} } }, { className2: { struct: {...} } }, ... ] }` - an enum. The variants of the enum are described by the `className1`, `className2`, etc. fields. The variants are structs.
+- `{ struct: { field1: Schema1, field2: Schema2, ... } }` - a struct. The fields of the struct are described by the `field1`, `field2`, etc. fields.
+
+### Type Mappings
+
+| Javascript       | Borsh                             |
+|------------------|-----------------------------------|
+| `number`         | `u8` `u16` `u32` `i8` `i16` `i32` |
+| `bigint`         | `u64` `u128` `i64` `i128`         |
+| `number`         | `f32` `f64`                       |
+| `number`         | `f32` `f64`                       |
+| `boolean`        | `bool`                            |
+| `string`         | UTF-8 string                      |
+| `type[]`         | fixed-size byte array             |
+| `type[]`         | dynamic sized array               |
+| `object`         | enum                              |
+| `Map`            | HashMap                           |
+| `Set`            | HashSet                           |
+| `null` or `type` | Option                            |
+
+
+---
 
 ## Contributing
 
@@ -80,4 +119,4 @@ When publishing to npm use [np](https://github.com/sindresorhus/np).
 This repository is distributed under the terms of both the MIT license and the Apache License (Version 2.0).
 See [LICENSE-MIT](LICENSE-MIT.txt) and [LICENSE-APACHE](LICENSE-APACHE) for details.
 
-[Borsh]:          https://borsh.io
+[Borsh]:          https://borsh.io

borsh-ts/.eslintrc.yml

@@ -11,7 +11,6 @@ rules:
   '@typescript-eslint/no-explicit-any': 1
   '@typescript-eslint/ban-types': 1
   '@typescript-eslint/explicit-function-return-type': 1
-  '@typescript-eslint/no-use-before-define': 1
 
 parserOptions:
   ecmaVersion: 2018

borsh-ts/buffer.ts

@@ -0,0 +1,87 @@
+import { IntegerType } from './types.js';
+
+export class EncodeBuffer {
+    offset: number;
+    buffer_size: number;
+    buffer: ArrayBuffer;
+    view: DataView;
+
+    constructor() {
+        this.offset = 0;
+        this.buffer_size = 256;
+        this.buffer = new ArrayBuffer(this.buffer_size);
+        this.view = new DataView(this.buffer);
+    }
+
+    resize_if_necessary(needed_space: number): void {
+        if (this.buffer_size - this.offset < needed_space) {
+            this.buffer_size = Math.max(this.buffer_size * 2, this.buffer_size + needed_space);
+
+            const new_buffer = new ArrayBuffer(this.buffer_size);
+            new Uint8Array(new_buffer).set(new Uint8Array(this.buffer));
+
+            this.buffer = new_buffer;
+            this.view = new DataView(new_buffer);
+        }
+    }
+
+    get_used_buffer(): Uint8Array {
+        return new Uint8Array(this.buffer).slice(0, this.offset);
+    }
+
+    store_value(value: number, type: IntegerType): void {
+        const bSize = type.substring(1);
+        const size = parseInt(bSize) / 8;
+        this.resize_if_necessary(size);
+
+        const toCall = type[0] === 'f'? `setFloat${bSize}`: type[0] === 'i'? `setInt${bSize}` : `setUint${bSize}`;
+        this.view[toCall](this.offset, value, true);
+        this.offset += size;
+    }
+
+    store_bytes(from: Uint8Array): void {
+        this.resize_if_necessary(from.length);
+        new Uint8Array(this.buffer).set(new Uint8Array(from), this.offset);
+        this.offset += from.length;
+    }
+}
+
+export class DecodeBuffer {
+    offset: number;
+    buffer_size: number;
+    buffer: ArrayBuffer;
+    view: DataView;
+
+    constructor(buf: Uint8Array) {
+        this.offset = 0;
+        this.buffer_size = buf.length;
+        this.buffer = new ArrayBuffer(buf.length);
+        new Uint8Array(this.buffer).set(buf);
+        this.view = new DataView(this.buffer);
+    }
+
+    assert_enough_buffer(size: number): void {
+        if (this.offset + size > this.buffer.byteLength) {
+            throw new Error('Error in schema, the buffer is smaller than expected');
+        }
+    }
+
+    consume_value(type: IntegerType): number {
+        const bSize = type.substring(1);
+        const size = parseInt(bSize) / 8;
+        this.assert_enough_buffer(size);
+
+        const toCall = type[0] === 'f'? `getFloat${bSize}`: type[0] === 'i'? `getInt${bSize}` : `getUint${bSize}`;
+        const ret = this.view[toCall](this.offset, true);
+
+        this.offset += size;
+        return ret;
+    }
+
+    consume_bytes(size: number): ArrayBuffer {
+        this.assert_enough_buffer(size);
+        const ret = this.buffer.slice(this.offset, this.offset + size);
+        this.offset += size;
+        return ret;
+    }
+}

borsh-ts/deserialize.ts

@@ -0,0 +1,125 @@
+import { ArrayType, DecodeTypes, MapType, IntegerType, OptionType, Schema, SetType, StructType, integers, EnumType } from './types.js';
+import { DecodeBuffer } from './buffer.js';
+
+export class BorshDeserializer {
+    buffer: DecodeBuffer;
+
+    constructor(bufferArray: Uint8Array) {
+        this.buffer = new DecodeBuffer(bufferArray);
+    }
+
+    decode(schema: Schema): DecodeTypes {
+        return this.decode_value(schema);
+    }
+
+    decode_value(schema: Schema): DecodeTypes {
+        if (typeof schema === 'string') {
+            if (integers.includes(schema)) return this.decode_integer(schema);
+            if (schema === 'string') return this.decode_string();
+            if (schema === 'bool') return this.decode_boolean();
+        }
+
+        if (typeof schema === 'object') {
+            if ('option' in schema) return this.decode_option(schema as OptionType);
+            if ('enum' in schema) return this.decode_enum(schema as EnumType);
+            if ('array' in schema) return this.decode_array(schema as ArrayType);
+            if ('set' in schema) return this.decode_set(schema as SetType);
+            if ('map' in schema) return this.decode_map(schema as MapType);
+            if ('struct' in schema) return this.decode_struct(schema as StructType);
+        }
+
+        throw new Error(`Unsupported type: ${schema}`);
+    }
+
+    decode_integer(schema: IntegerType): number | bigint {
+        const size: number = parseInt(schema.substring(1));
+
+        if (size <= 32 || schema == 'f64') {
+            return this.buffer.consume_value(schema);
+        }
+        return this.decode_bigint(size, schema.startsWith('i'));
+    }
+
+    decode_bigint(size: number, signed = false): bigint {
+        const buffer_len = size / 8;
+        const buffer = new Uint8Array(this.buffer.consume_bytes(buffer_len));
+        const bits = buffer.reduceRight((r, x) => r + x.toString(16).padStart(2, '0'), '');
+
+        if (signed && buffer[buffer_len - 1]) {
+            return BigInt.asIntN(size, BigInt(`0x${bits}`));
+        }
+        return BigInt(`0x${bits}`);
+    }
+
+    decode_string(): string {
+        const len: number = this.decode_integer('u32') as number;
+        const buffer = new Uint8Array(this.buffer.consume_bytes(len));
+        return String.fromCharCode.apply(null, buffer);
+    }
+
+    decode_boolean(): boolean {
+        return this.buffer.consume_value('u8') > 0;
+    }
+
+    decode_option(schema: OptionType): DecodeTypes {
+        const option = this.buffer.consume_value('u8');
+        if (option === 1) {
+            return this.decode_value(schema.option);
+        }
+        if (option !== 0) {
+            throw new Error(`Invalid option ${option}`);
+        }
+        return null;
+    }
+
+    decode_enum(schema: EnumType): DecodeTypes {
+        const valueIndex = this.buffer.consume_value('u8');
+
+        if (valueIndex > schema.enum.length) {
+            throw new Error(`Enum option ${valueIndex} is not available`);
+        }
+
+        const struct = schema.enum[valueIndex].struct;
+        const key = Object.keys(struct)[0];
+        return { [key]: this.decode_value(struct[key]) };
+    }
+
+    decode_array(schema: ArrayType): Array<DecodeTypes> {
+        const result = [];
+        const len = schema.array.len ? schema.array.len : this.decode_integer('u32') as number;
+
+        for (let i = 0; i < len; ++i) {
+            result.push(this.decode_value(schema.array.type));
+        }
+
+        return result;
+    }
+
+    decode_set(schema: SetType): Set<DecodeTypes> {
+        const len = this.decode_integer('u32') as number;
+        const result = new Set<DecodeTypes>();
+        for (let i = 0; i < len; ++i) {
+            result.add(this.decode_value(schema.set));
+        }
+        return result;
+    }
+
+    decode_map(schema: MapType): Map<DecodeTypes, DecodeTypes> {
+        const len = this.decode_integer('u32') as number;
+        const result = new Map();
+        for (let i = 0; i < len; ++i) {
+            const key = this.decode_value(schema.map.key);
+            const value = this.decode_value(schema.map.value);
+            result.set(key, value);
+        }
+        return result;
+    }
+
+    decode_struct(schema: StructType): object {
+        const result = {};
+        for (const key in schema.struct) {
+            result[key] = this.decode_value(schema.struct[key]);
+        }
+        return result;
+    }
+}