Fix comlink serialization by converting Buffer to Uint8Array at worker boundary

Fix comlink serialization by returning Uint8Array from aesUtils

aesUtils now returns plain Uint8Array instead of Buffer, fixing
'Unserializable return value' errors on Linux CI where Buffer
objects don't serialize correctly through comlink's structured clone.

Author: mondoreale

packages/sdk/src/encryption/EncryptionService.ts

@@ -36,8 +36,11 @@ export class EncryptionService {
      * Note: The input data buffer is transferred to the worker and becomes unusable after this call.
      */
     async encryptWithAES(data: Uint8Array, cipherKey: Uint8Array): Promise<Uint8Array> {
+        // Ensure we have plain Uint8Array instances for worker communication (not Buffer subclass)
+        const dataArray = new Uint8Array(data)
+        const keyArray = new Uint8Array(cipherKey)
         const result = await this.getWorkerApi().encrypt(
-            transfer({ data, cipherKey }, [data.buffer])
+            transfer({ data: dataArray, cipherKey: keyArray }, [dataArray.buffer])
         )
         if (result.type === 'error') {
             throw new Error(`AES encryption failed: ${result.message}`)
@@ -49,10 +52,11 @@ export class EncryptionService {
      * Encrypt the next group key using the current group key.
      */
     async encryptNextGroupKey(currentKey: GroupKey, nextKey: GroupKey): Promise<EncryptedGroupKey> {
+        // Convert Buffer to Uint8Array for worker communication
         const result = await this.getWorkerApi().encryptGroupKey({
             nextGroupKeyId: nextKey.id,
-            nextGroupKeyData: nextKey.data,
-            currentGroupKeyData: currentKey.data
+            nextGroupKeyData: new Uint8Array(nextKey.data),
+            currentGroupKeyData: new Uint8Array(currentKey.data)
         })
         if (result.type === 'error') {
             throw new Error(`Group key encryption failed: ${result.message}`)
@@ -67,10 +71,11 @@ export class EncryptionService {
      * Decrypt an encrypted group key using the current group key.
      */
     async decryptNextGroupKey(currentKey: GroupKey, encryptedKey: EncryptedGroupKey): Promise<GroupKey> {
+        // Convert Buffer to Uint8Array for worker communication
         const result = await this.getWorkerApi().decryptGroupKey({
             encryptedGroupKeyId: encryptedKey.id,
-            encryptedGroupKeyData: encryptedKey.data,
-            currentGroupKeyData: currentKey.data
+            encryptedGroupKeyData: new Uint8Array(encryptedKey.data),
+            currentGroupKeyData: new Uint8Array(currentKey.data)
         })
         if (result.type === 'error') {
             throw new Error(`Group key decryption failed: ${result.message}`)
@@ -88,12 +93,13 @@ export class EncryptionService {
         groupKey: GroupKey,
         encryptedNewGroupKey?: EncryptedGroupKey
     ): Promise<[Uint8Array, GroupKey?]> {
+        // Convert Buffer to Uint8Array for worker communication
         const request = {
             content,
-            groupKeyData: groupKey.data,
+            groupKeyData: new Uint8Array(groupKey.data),
             newGroupKey: encryptedNewGroupKey ? {
                 id: encryptedNewGroupKey.id,
-                data: encryptedNewGroupKey.data
+                data: new Uint8Array(encryptedNewGroupKey.data)
             } : undefined
         }
         const result = await this.getWorkerApi().decryptStreamMessage(

packages/sdk/src/encryption/EncryptionUtil.ts

@@ -118,7 +118,7 @@ export class EncryptionUtil {
         const wrappingAESKey = await this.deriveAESWrapperKey(sharedSecret, kdfSalt)
 
         // Encrypt plaintext with the AES wrapping key
-        const aesEncryptedPlaintext = encryptWithAES(plaintextBuffer, Buffer.from(wrappingAESKey))
+        const aesEncryptedPlaintext = encryptWithAES(plaintextBuffer, wrappingAESKey)
 
         // Concatenate the deliverables into a binary package
         return Buffer.concat([kemCipher, kdfSalt, aesEncryptedPlaintext])
@@ -140,6 +140,6 @@ export class EncryptionUtil {
         const wrappingAESKey = await this.deriveAESWrapperKey(sharedSecret, kdfSalt)
 
         // Decrypt the aesEncryptedPlaintext
-        return decryptWithAES(aesEncryptedPlaintext, Buffer.from(wrappingAESKey))
+        return Buffer.from(decryptWithAES(aesEncryptedPlaintext, wrappingAESKey))
     }
 }

packages/sdk/src/encryption/EncryptionWorker.ts

@@ -21,8 +21,8 @@ import {
 const workerApi = {
     encrypt: async (request: AESEncryptRequest): Promise<AESEncryptResult> => {
         try {
-            const result = encryptWithAES(request.data, request.cipherKey)
-            return transfer({ type: 'success', data: result }, [result.buffer])
+            const data = encryptWithAES(request.data, request.cipherKey)
+            return transfer({ type: 'success', data }, [data.buffer])
         } catch (err) {
             return { type: 'error', message: String(err) }
         }

packages/sdk/src/encryption/aesUtils.ts

@@ -7,22 +7,36 @@ import { createCipheriv, createDecipheriv } from '@streamr/utils'
 
 export const INITIALIZATION_VECTOR_LENGTH = 16
 
+/**
+ * Concatenate multiple Uint8Arrays into a single Uint8Array.
+ */
+function concatBytes(...arrays: Uint8Array[]): Uint8Array {
+    const totalLength = arrays.reduce((sum, arr) => sum + arr.length, 0)
+    const result = new Uint8Array(totalLength)
+    let offset = 0
+    for (const arr of arrays) {
+        result.set(arr, offset)
+        offset += arr.length
+    }
+    return result
+}
+
 /**
  * Encrypt data using AES-256-CTR.
  * Returns IV prepended to ciphertext.
  */
 export function encryptWithAES(data: Uint8Array, cipherKey: Uint8Array): Uint8Array {
     const iv = randomBytes(INITIALIZATION_VECTOR_LENGTH) // always need a fresh IV when using CTR mode
     const cipher = createCipheriv('aes-256-ctr', cipherKey, iv)
-    return Buffer.concat([iv, cipher.update(data), cipher.final()])
+    return concatBytes(iv, cipher.update(data), cipher.final())
 }
 
 /**
  * Decrypt AES-256-CTR encrypted data.
  * Expects IV prepended to ciphertext.
  */
-export function decryptWithAES(cipher: Uint8Array, cipherKey: Uint8Array): Buffer {
+export function decryptWithAES(cipher: Uint8Array, cipherKey: Uint8Array): Uint8Array {
     const iv = cipher.slice(0, INITIALIZATION_VECTOR_LENGTH)
     const decipher = createDecipheriv('aes-256-ctr', cipherKey, iv)
-    return Buffer.concat([decipher.update(cipher.slice(INITIALIZATION_VECTOR_LENGTH)), decipher.final()])
+    return concatBytes(decipher.update(cipher.slice(INITIALIZATION_VECTOR_LENGTH)), decipher.final())
 }

packages/sdk/test/unit/EncryptionUtil.test.ts

@@ -17,7 +17,7 @@ describe('EncryptionUtil', () => {
         it('returns the initial plaintext after decrypting the ciphertext', async () => {
             const key = await RSAKeyPair.create(512)
             const ciphertext = await EncryptionUtil.encryptForPublicKey(plaintext, key.getPublicKey(), AsymmetricEncryptionType.RSA)
-            expect(await EncryptionUtil.decryptWithPrivateKey(ciphertext, key.getPrivateKey(), AsymmetricEncryptionType.RSA)).toStrictEqual(plaintext)
+            expect(await EncryptionUtil.decryptWithPrivateKey(ciphertext, key.getPrivateKey(), AsymmetricEncryptionType.RSA)).toEqualBinary(plaintext)
         })
 
         it('produces different ciphertexts upon multiple encrypt() calls', async () => {
@@ -40,7 +40,7 @@ describe('EncryptionUtil', () => {
             const ciphertext = await EncryptionUtil.encryptForPublicKey(plaintext, key.getPublicKey(), AsymmetricEncryptionType.ML_KEM)
             expect(await EncryptionUtil.decryptWithPrivateKey(
                 ciphertext, key.getPrivateKey(), AsymmetricEncryptionType.ML_KEM
-            )).toStrictEqual(plaintext)
+            )).toEqualBinary(plaintext)
         })
 
         it('produces different ciphertexts upon multiple encrypt() calls', async () => {

packages/sdk/test/unit/aesUtils.test.ts

@@ -20,7 +20,7 @@ describe('aesUtils', () => {
         it('returns the initial plaintext after decrypting the ciphertext', () => {
             const key = GroupKey.generate()
             const ciphertext = encryptWithAES(plaintext, key.data)
-            expect(decryptWithAES(ciphertext, key.data)).toStrictEqual(plaintext)
+            expect(decryptWithAES(ciphertext, key.data)).toEqualBinary(plaintext)
         })
 
         it('preserves size (plaintext + iv)', () => {