feat!: re-work messaging parts and sharding (#2399)

* remove message-hash package, move it to core

* add js-doc to message hash

* up

* address type changes

* fix lint

Author: weboko

.release-please-manifest.json

@@ -2,7 +2,6 @@
   "packages/utils": "0.0.23",
   "packages/proto": "0.0.10",
   "packages/interfaces": "0.0.30",
-  "packages/message-hash": "0.1.19",
   "packages/enr": "0.0.29",
   "packages/core": "0.0.35",
   "packages/message-encryption": "0.0.33",

.size-limit.cjs

@@ -59,7 +59,7 @@ module.exports = [
   },
   {
     name: "Deterministic Message Hashing",
-    path: "packages/message-hash/bundle/index.js",
+    path: ["packages/core/bundle/index.js"],
     import: "{ messageHash }",
   },
 ];

package-lock.json

@@ -6,7 +6,6 @@
     "packages/proto",
     "packages/interfaces",
     "packages/utils",
-    "packages/message-hash",
     "packages/enr",
     "packages/core",
     "packages/discovery",

package.json

@@ -74,6 +74,7 @@
     "@waku/proto": "0.0.10",
     "@waku/utils": "0.0.23",
     "debug": "^4.3.4",
+    "@noble/hashes": "^1.3.2",
     "it-all": "^3.0.4",
     "it-length-prefixed": "^9.0.4",
     "it-pipe": "^3.0.1",

packages/core/package.json

@@ -20,3 +20,5 @@ export { ConnectionManager } from "./lib/connection_manager/index.js";
 export { StreamManager } from "./lib/stream_manager/index.js";
 
 export { MetadataCodec, wakuMetadata } from "./lib/metadata/index.js";
+
+export { messageHash, messageHashStr } from "./lib/message_hash/index.js";

packages/core/src/index.ts

@@ -22,7 +22,7 @@ export { proto };
 export class DecodedMessage implements IDecodedMessage {
   public constructor(
     public pubsubTopic: string,
-    protected proto: proto.WakuMessage
+    private proto: proto.WakuMessage
   ) {}
 
   public get ephemeral(): boolean {
@@ -37,10 +37,6 @@ export class DecodedMessage implements IDecodedMessage {
     return this.proto.contentTopic;
   }
 
-  public get _rawTimestamp(): bigint | undefined {
-    return this.proto.timestamp;
-  }
-
   public get timestamp(): Date | undefined {
     // In the case we receive a value that is bigger than JS's max number,
     // we catch the error and return undefined.
@@ -63,7 +59,7 @@ export class DecodedMessage implements IDecodedMessage {
   public get version(): number {
     // https://rfc.vac.dev/spec/14/
     // > If omitted, the value SHOULD be interpreted as version 0.
-    return this.proto.version ?? 0;
+    return this.proto.version ?? Version;
   }
 
   public get rateLimitProof(): IRateLimitProof | undefined {
@@ -160,7 +156,7 @@ export class Decoder implements IDecoder<IDecodedMessage> {
   public async fromProtoObj(
     pubsubTopic: string,
     proto: IProtoMessage
-  ): Promise<DecodedMessage | undefined> {
+  ): Promise<IDecodedMessage | undefined> {
     // https://rfc.vac.dev/spec/14/
     // > If omitted, the value SHOULD be interpreted as version 0.
     if (proto.version ?? 0 !== Version) {

packages/core/src/lib/message/version_0.ts

@@ -0,0 +1 @@
+export { messageHash, messageHashStr } from "./message_hash.js";

packages/core/src/lib/message_hash/index.ts

@@ -2,10 +2,10 @@ import type { IDecodedMessage, IProtoMessage } from "@waku/interfaces";
 import { bytesToHex, hexToBytes } from "@waku/utils/bytes";
 import { expect } from "chai";
 
-import { messageHash } from "./index.js";
+import { messageHash, messageHashStr } from "./index.js";
 
 // https://rfc.vac.dev/spec/14/#test-vectors
-describe("RFC Test Vectors", () => {
+describe("Message Hash: RFC Test Vectors", () => {
   it("Waku message hash computation (meta size of 12 bytes)", () => {
     const expectedHash =
       "64cce733fed134e83da02b02c6f689814872b1a0ac97ea56b76095c3c72bfe05";
@@ -98,6 +98,7 @@ describe("RFC Test Vectors", () => {
       "3f11bc950dce0e3ffdcf205ae6414c01130bb5d9f20644869bff80407fa52c8f";
     const pubsubTopic = "/waku/2/default-waku/proto";
     const message: IDecodedMessage = {
+      version: 0,
       payload: new Uint8Array(),
       pubsubTopic,
       contentTopic: "/waku/2/default-content/proto",
@@ -110,3 +111,71 @@ describe("RFC Test Vectors", () => {
     expect(bytesToHex(hash)).to.equal(expectedHash);
   });
 });
+
+describe("messageHash and messageHashStr", () => {
+  const pubsubTopic = "/waku/2/default-waku/proto";
+  const testMessage: IProtoMessage = {
+    payload: hexToBytes("0x010203045445535405060708"),
+    contentTopic: "/waku/2/default-content/proto",
+    meta: hexToBytes("0x73757065722d736563726574"),
+    timestamp: BigInt("0x175789bfa23f8400"),
+    ephemeral: undefined,
+    rateLimitProof: undefined,
+    version: undefined
+  };
+
+  it("messageHash returns a Uint8Array", () => {
+    const hash = messageHash(pubsubTopic, testMessage);
+    expect(hash).to.be.instanceOf(Uint8Array);
+    expect(hash.length).to.equal(32); // SHA-256 hash is 32 bytes
+  });
+
+  it("messageHashStr returns a hex string", () => {
+    const hashStr = messageHashStr(pubsubTopic, testMessage);
+    expect(typeof hashStr).to.equal("string");
+    expect(hashStr.length).to.equal(64); // SHA-256 hash is 32 bytes = 64 hex chars
+    expect(hashStr).to.match(/^[0-9a-f]+$/); // Should be a valid hex string
+  });
+
+  it("messageHashStr returns the same value as bytesToHex(messageHash)", () => {
+    const hash = messageHash(pubsubTopic, testMessage);
+    const hashStrFromBytes = bytesToHex(hash);
+    const hashStr = messageHashStr(pubsubTopic, testMessage);
+    expect(hashStr).to.equal(hashStrFromBytes);
+  });
+
+  it("messageHashStr works with IDecodedMessage", () => {
+    const decodedMessage: IDecodedMessage = {
+      version: 0,
+      payload: new Uint8Array([1, 2, 3, 4]),
+      pubsubTopic,
+      contentTopic: "/waku/2/default-content/proto",
+      meta: new Uint8Array([5, 6, 7, 8]),
+      timestamp: new Date("2024-04-30T10:54:14.978Z"),
+      ephemeral: undefined,
+      rateLimitProof: undefined
+    };
+
+    const hashStr = messageHashStr(pubsubTopic, decodedMessage);
+    expect(typeof hashStr).to.equal("string");
+    expect(hashStr.length).to.equal(64);
+  });
+
+  it("messageHashStr produces consistent results for the same input", () => {
+    const hashStr1 = messageHashStr(pubsubTopic, testMessage);
+    const hashStr2 = messageHashStr(pubsubTopic, testMessage);
+    expect(hashStr1).to.equal(hashStr2);
+  });
+
+  it("messageHashStr produces different results for different inputs", () => {
+    const hashStr1 = messageHashStr(pubsubTopic, testMessage);
+
+    const differentMessage = {
+      ...testMessage,
+      payload: hexToBytes("0x0102030454455354050607080A") // Different payload
+    };
+
+    const hashStr2 = messageHashStr(pubsubTopic, differentMessage);
+    expect(hashStr1).to.not.equal(hashStr2);
+  });
+});

packages/message-hash/src/index.spec.ts …rc/lib/message_hash/message_hash.spec.tspackages/message-hash/src/index.spec.ts renamed to packages/core/src/lib/message_hash/message_hash.spec.ts packages/message-hash/src/index.spec.ts renamed to packages/core/src/lib/message_hash/message_hash.spec.ts

@@ -11,6 +11,27 @@ import {
 /**
  * Deterministic Message Hashing as defined in
  * [14/WAKU2-MESSAGE](https://rfc.vac.dev/spec/14/#deterministic-message-hashing)
+ *
+ * Computes a SHA-256 hash of the concatenation of pubsub topic, payload, content topic, meta, and timestamp.
+ *
+ * @param pubsubTopic - The pubsub topic string
+ * @param message - The message to be hashed
+ * @returns A Uint8Array containing the SHA-256 hash
+ *
+ * @example
+ * ```typescript
+ * import { messageHash } from "@waku/core";
+ *
+ * const pubsubTopic = "/waku/2/default-waku/proto";
+ * const message = {
+ *   payload: new Uint8Array([1, 2, 3, 4]),
+ *   contentTopic: "/waku/2/default-content/proto",
+ *   meta: new Uint8Array([5, 6, 7, 8]),
+ *   timestamp: new Date()
+ * };
+ *
+ * const hash = messageHash(pubsubTopic, message);
+ * ```
  */
 export function messageHash(
   pubsubTopic: string,
@@ -51,6 +72,30 @@ function tryConvertTimestampToBytes(
   return numberToBytes(bigIntTimestamp);
 }
 
+/**
+ * Computes a deterministic message hash and returns it as a hexadecimal string.
+ * This is a convenience wrapper around messageHash that converts the result to a hex string.
+ *
+ * @param pubsubTopic - The pubsub topic string
+ * @param message - The message to be hashed
+ * @returns A string containing the hex representation of the SHA-256 hash
+ *
+ * @example
+ * ```typescript
+ * import { messageHashStr } from "@waku/core";
+ *
+ * const pubsubTopic = "/waku/2/default-waku/proto";
+ * const message = {
+ *   payload: new Uint8Array([1, 2, 3, 4]),
+ *   contentTopic: "/waku/2/default-content/proto",
+ *   meta: new Uint8Array([5, 6, 7, 8]),
+ *   timestamp: new Date()
+ * };
+ *
+ * const hashString = messageHashStr(pubsubTopic, message);
+ * console.log(hashString); // e.g. "a1b2c3d4..."
+ * ```
+ */
 export function messageHashStr(
   pubsubTopic: string,
   message: IProtoMessage | IDecodedMessage