fix(deps): bencode library

Author: roshii

.github/workflows/unittests.yml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       matrix:
         os: [macos-13, ubuntu-latest]
-        python-version: ["3.8", "3.12"]
+        python-version: ["3.9", "3.12"]
         bitcoind-version: ["29.2"]
 
     steps:

pyproject.toml

@@ -7,7 +7,7 @@ name = "joinmarket"
 version = "0.9.12dev"
 description = "Joinmarket client library for Bitcoin coinjoins"
 readme = "README.md"
-requires-python = ">=3.8,<3.13"
+requires-python = ">=3.9,<3.13"
 license = {file = "LICENSE"}
 dependencies = [
     "chromalog==1.0.5",
@@ -24,7 +24,7 @@ jmbitcoin = [
 jmclient = [
     "argon2_cffi==21.3.0",
     "autobahn==20.12.3",
-    "bencoder.pyx==3.0.1",
+    "fastbencode==0.3.5",
     "klein==20.6.0",
     "mnemonic==0.20",
     "pyjwt==2.4.0",

src/jmclient/storage.py

@@ -1,17 +1,30 @@
+import atexit
 import os
 import shutil
-import atexit
-import bencoder
 from hashlib import sha256
+from typing import Any
+
 from argon2 import low_level
-from jmbase import aes_cbc_encrypt, aes_cbc_decrypt
+from fastbencode import bdecode, bencode_utf8
+
+from jmbase import aes_cbc_decrypt, aes_cbc_encrypt
+
 from .support import get_random_bytes
 
 
 class Argon2Hash(object):
-    def __init__(self, password, salt=None, hash_len=32, salt_len=16,
-                 time_cost=500, memory_cost=1000, parallelism=4,
-                 argon2_type=low_level.Type.I, version=19):
+    def __init__(
+        self,
+        password,
+        salt=None,
+        hash_len=32,
+        salt_len=16,
+        time_cost=500,
+        memory_cost=1000,
+        parallelism=4,
+        argon2_type=low_level.Type.I,
+        version=19,
+    ):
         """
         args:
           password: password as bytes
@@ -30,11 +43,12 @@ def __init__(self, password, salt=None, hash_len=32, salt_len=16,
             'parallelism': parallelism,
             'hash_len': hash_len,
             'type': argon2_type,
-            'version': version
+            'version': version,
         }
         self.salt = salt if salt is not None else get_random_bytes(salt_len)
-        self.hash = low_level.hash_secret_raw(password, self.salt,
-                                              **self.settings)
+        self.hash = low_level.hash_secret_raw(
+            password, self.salt, **self.settings
+        )
 
 
 class StorageError(Exception):
@@ -62,8 +76,9 @@ class Storage(object):
 
     KDF: argon2, ENC: AES-256-CBC
     """
+
     MAGIC_UNENC = b'JMWALLET'
-    MAGIC_ENC =   b'JMENCWLT'
+    MAGIC_ENC = b'JMENCWLT'
     MAGIC_DETECT_ENC = b'JMWALLET'
 
     ENC_KEY_BYTES = 32  # AES-256
@@ -116,7 +131,10 @@ def was_changed(self):
         return self._data_checksum != self._get_data_checksum()
 
     def check_password(self, password):
-        return self._hash.hash == self._hash_password(password, self._hash.salt).hash
+        return (
+            self._hash.hash
+            == self._hash_password(password, self._hash.salt).hash
+        )
 
     def change_password(self, password):
         if self.read_only:
@@ -128,7 +146,7 @@ def save(self):
         """
         Write file to disk if data was modified
         """
-        #if not self.was_changed():
+        # if not self.was_changed():
         #    return
         if self.read_only:
             raise StorageError("Read-only storage cannot be saved.")
@@ -149,7 +167,7 @@ def _get_file_magic(cls, path):
             return fh.read(len(cls.MAGIC_ENC))
 
     def _get_data_checksum(self):
-        if self.data is None:  #pragma: no cover
+        if self.data is None:  # pragma: no cover
             return None
         return sha256(self._serialize(self.data)).digest()
 
@@ -167,7 +185,8 @@ def _set_hash(self, password):
             self._hash = self._hash_password(password)
 
     def _save_file(self):
-        assert self.read_only == False
+        if self.read_only:
+            raise StorageError("Read-only storage cannot be saved.")
         data = self._serialize(self.data)
         enc_data = self._encrypt_file(data)
 
@@ -181,13 +200,17 @@ def _load_file(self, password):
         magic = data[:8]
 
         if magic not in (self.MAGIC_ENC, self.MAGIC_UNENC):
-            raise StorageError("File does not appear to be a joinmarket wallet.")
+            raise StorageError(
+                "File does not appear to be a joinmarket wallet."
+            )
 
         data = data[8:]
 
         if magic == self.MAGIC_ENC:
             if password is None:
-                raise RetryableStorageError("Password required to open wallet.")
+                raise RetryableStorageError(
+                    "Password required to open wallet."
+                )
             data = self._decrypt_file(password, data)
         else:
             assert magic == self.MAGIC_UNENC
@@ -211,7 +234,7 @@ def _write_file(self, data):
             shutil.copystat(self.path, tmpfile)
             fh.write(data)
 
-        #FIXME: behaviour with symlinks might be weird
+        # FIXME: behaviour with symlinks might be weird
         shutil.move(tmpfile, self.path)
 
     def _read_file(self):
@@ -223,12 +246,12 @@ def get_location(self):
         return self.path
 
     @staticmethod
-    def _serialize(data):
-        return bencoder.bencode(data)
+    def _serialize(data: Any) -> bytes:
+        return bencode_utf8(data)
 
     @staticmethod
-    def _deserialize(data):
-        return bencoder.bdecode(data)
+    def _deserialize(data: bytes) -> Any:
+        return bdecode(data)
 
     def _encrypt_file(self, data):
         if not self.is_encrypted():
@@ -237,7 +260,7 @@ def _encrypt_file(self, data):
         iv = get_random_bytes(16)
         container = {
             b'enc': {b'salt': self._hash.salt, b'iv': iv},
-            b'data': self._encrypt(data, iv)
+            b'data': self._encrypt(data, iv),
         }
         return self._serialize(container)
 
@@ -253,8 +276,9 @@ def _decrypt_file(self, password, data):
         return self._decrypt(container[b'data'], container[b'enc'][b'iv'])
 
     def _encrypt(self, data: bytes, iv: bytes) -> bytes:
-        return aes_cbc_encrypt(self._hash.hash,
-                               self.MAGIC_DETECT_ENC + data, iv)
+        return aes_cbc_encrypt(
+            self._hash.hash, self.MAGIC_DETECT_ENC + data, iv
+        )
 
     def _decrypt(self, data: bytes, iv: bytes) -> bytes:
         try:
@@ -265,12 +289,16 @@ def _decrypt(self, data: bytes, iv: bytes) -> bytes:
 
         if not dec_data.startswith(self.MAGIC_DETECT_ENC):
             raise StoragePasswordError("Wrong password.")
-        return dec_data[len(self.MAGIC_DETECT_ENC):]
+        return dec_data[len(self.MAGIC_DETECT_ENC) :]
 
     @classmethod
     def _hash_password(cls, password, salt=None):
-        return Argon2Hash(password, salt,
-                          hash_len=cls.ENC_KEY_BYTES, salt_len=cls.SALT_LENGTH)
+        return Argon2Hash(
+            password,
+            salt,
+            hash_len=cls.ENC_KEY_BYTES,
+            salt_len=cls.SALT_LENGTH,
+        )
 
     @staticmethod
     def _get_lock_filename(path: str) -> str:
@@ -296,8 +324,9 @@ def verify_lock(cls, path: str):
             raise RetryableStorageError(
                 "File is currently in use (locked by pid {}). "
                 "If this is a leftover from a crashed instance "
-                "you need to remove the lock file `{}` manually.".
-                format(locked_by_pid, cls._get_lock_filename(path))
+                "you need to remove the lock file `{}` manually.".format(
+                    locked_by_pid, cls._get_lock_filename(path)
+                )
             )
 
     def _create_lock(self):

test/jmclient/test_storage.py

@@ -1,4 +1,3 @@
-
 from jmclient import storage
 import pytest
 
@@ -81,6 +80,7 @@ def test_storage_invalid():
         MockStorage(b'garbagefile', __file__, b'password')
         pytest.fail("Non-wallet file, encrypted")
 
+
 def test_storage_readonly():
     s = MockStorage(None, 'nonexistant', b'password', create=True)
     s = MockStorage(s.file_data, __file__, b'password', read_only=True)
@@ -136,3 +136,30 @@ def test_storage_lock(tmpdir):
         s._create_lock()
         pytest.fail("It should not be possible to re-create a lock")
 
+
+testdata = {
+    b"bytes_key": b"bytes_value",
+    b"int_key": 42,
+    b"list_key": [b"a", b"b", b"c", 1, 2, 3],
+    b"dict_key": {b"nested": b"data", b"number": 999},
+}
+
+
+@pytest.mark.parametrize(
+    "test_data, expected_out",
+    [
+        (testdata, testdata),
+        ({b"dict_key": {b"utf": "value"}}, {b"dict_key": {b"utf": b"value"}}),
+    ],
+)
+def test_bencode_roundtrip_consistency(test_data, expected_out):
+    s = MockStorage(None, "nonexistant", None, create=True)
+
+    serialized1 = s._serialize(test_data)
+    deserialized1 = s._deserialize(serialized1)
+
+    serialized2 = s._serialize(deserialized1)
+    deserialized2 = s._deserialize(serialized2)
+
+    assert serialized1 == serialized2
+    assert deserialized1 == deserialized2 == expected_out