stored_dict: convert objects to simple python types before passing them to db

Author: ecdsa

electrum/json_db.py

@@ -33,7 +33,7 @@
 
 from .util import WalletFileException, profiler, sticky_property
 from .logging import Logger
-from .stored_dict import _FLEX_KEY, BaseDB, json_default
+from .stored_dict import _FLEX_KEY, BaseDB
 from .storage import FileStorage
 
 
@@ -174,7 +174,6 @@ def replace(self, path, key, value):
     @modifier
     def put(self, path, key, value):
         d = self._subdict(path)
-        value = json.loads(json.dumps(value, default=json_default)) # default() is applied recursively
         is_new = key not in d
         d[key] = value
         self.db_add(path, key, value) if is_new else self.db_replace(path, key, value)
@@ -293,7 +292,7 @@ def modified(self):
 
     @locked
     def add_patch(self, patch):
-        self.pending_changes.append(json.dumps(patch, default=json_default))
+        self.pending_changes.append(json.dumps(patch))
         self.set_modified(True)
 
     def db_add(self, path, key: _FLEX_KEY, value) -> None:
@@ -317,7 +316,6 @@ def dump(self, *, human_readable: bool = True) -> str:
             self.json_data,
             indent=4 if human_readable else None,
             sort_keys=bool(human_readable),
-            default=json_default,
         )
 
     @contextmanager

electrum/level_db.py

@@ -7,7 +7,7 @@
 import plyvel
 from contextlib import contextmanager
 
-from .stored_dict import BaseDB, _FLEX_KEY, json_default
+from .stored_dict import BaseDB, _FLEX_KEY
 
 # Todo:
 # - simplify path: first element is unused
@@ -24,7 +24,7 @@ class JsonCodec:
     """Default value codec: JSON (utf-8)."""
     @staticmethod
     def dumps(value: Any) -> bytes:
-        return json.dumps(value, separators=(",", ":"), ensure_ascii=False, default=json_default).encode("utf-8")
+        return json.dumps(value, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
 
     @staticmethod
     def loads(data: bytes) -> Any:
@@ -60,6 +60,7 @@ def __init__(
         self.codec = JsonCodec
         if init_db:
             self.init_db()
+            self._debug()
 
     def is_encrypted(self):
         return False
@@ -224,7 +225,11 @@ def get(self, path, key: _FLEX_KEY) -> Any:
         raw = db.get(self._full_key(path, key))
         if raw is None:
             raise KeyError((path, key, self._full_key(path, key)))
-        return self.codec.loads(raw) # json to python
+        r = self.codec.loads(raw)
+        # internally we store tuples as non-empty lists
+        if type(r) is list and len(r) > 0:
+            r = tuple(r)
+        return r
 
     @contextmanager
     def write_batch(self):
@@ -278,13 +283,7 @@ def put(self, path, key: _FLEX_KEY, value: Any) -> None:
     @locked
     def replace(self, path, key: _FLEX_KEY, value: Any) -> None:
         # called by StoredObject in setattr
-        db = self.db
-        if db is None:
-            raise RuntimeError("DB is closed")
-        fullkey = self._full_key(path[:-1], path[-1])
-        d = self.codec.loads(db.get(fullkey))
-        d[key] = value
-        db.put(fullkey, self.codec.dumps(d))
+        self.put(path, key, value)
 
     @locked
     def contains(self, path, key: object) -> bool:

electrum/stored_dict.py

@@ -89,23 +89,37 @@ class StorageEncryptionVersion(IntEnum):
 
 
 def json_default(obj):
+    """Convert object to simple python types.
+    The result must be json serializable.
+    """
     if isinstance(obj, (set, frozenset)):
-        return list(obj)
+        return [json_default(x) for x in list(obj)]
     if isinstance(obj, bytes):
         return obj.hex()
     if hasattr(obj, 'as_str') and callable(obj.as_str):
         return obj.as_str()
     if hasattr(obj, 'as_dict') and callable(obj.as_dict):
-        return obj.as_dict()
+        obj = obj.as_dict()
+        return dict([(key_to_str(k), json_default(v)) for k, v in obj.items()])
     if hasattr(obj, 'as_tuple') and callable(obj.as_tuple):
-        return obj.as_tuple()
+        obj = obj.as_tuple()
+        return tuple([json_default(x) for x in list(obj)])
+    if hasattr(obj, 'to_json') and callable(obj.to_json): # TxOutpoint
+        return obj.to_json()
+    if isinstance(obj, dict):
+        return dict([(key_to_str(k), json_default(v)) for k, v in obj.items()])
+    if isinstance(obj, list):
+        return [json_default(x) for x in obj]
+    if isinstance(obj, tuple):
+        return tuple([json_default(x) for x in list(obj)])
     if isinstance(obj, StoredDict):
-        return dict(obj)
+        raise Exception('trying to convert StoredDict')
     if isinstance(obj, StoredList):
-        return list(obj)
+        raise Exception('trying to convert StoredList')
     return obj
 
 
+
 class BaseDB(Logger):
 
     def __init__(self, path):
@@ -136,6 +150,7 @@ def _convert_dict_key(path: List[str], key: str) -> _FLEX_KEY:
     assert isinstance(key, _FLEX_KEY), f"unexpected type for {key=!r} at {path=}"
     return key
 
+
 def _convert_dict_value(path: List[str], v) -> Any:
     assert all(isinstance(x, _FLEX_KEY) for x in path)
     n = len(path)
@@ -145,8 +160,13 @@ def _convert_dict_value(path: List[str], v) -> Any:
             if reg:
                 _type, constructor = reg
                 if _type == dict:
+                    if isinstance(v, StoredDict):
+                        v = v._dump()
+                        #v = {k:_convert_dict_value(path + [k], x) for k, x in v.items()}
                     v = constructor(**v)
                 elif _type == tuple:
+                    if isinstance(v, StoredList):
+                        v = v._dump()
                     v = constructor(*v)
                 else:
                     v = constructor(v)
@@ -207,7 +227,7 @@ def __setattr__(self, key: str, value):
         assert isinstance(key, str), repr(key)
         if not key.startswith('_') and self.path:
             if value != getattr(self, key):
-                self._db.replace(self.path, key, value)
+                self._db.replace(self.path, key, json_default(value))
         object.__setattr__(self, key, value)
 
     def as_dict(self):
@@ -218,7 +238,6 @@ def as_dict(self):
         return d
 
 
-
 class StoredDict(BaseStoredObject):
     """
     dict-like object that queries the DB
@@ -255,14 +274,15 @@ def _dump(self) -> dict:
     def __getitem__(self, key: _FLEX_KEY) -> Any:
         key = key_to_str(key)  # fixme: assumes that db keys are str
         value = self._db.get(self.path, key)
+        value = self._to_stored_dict_or_list(key, value)
         if not self.should_convert():
-            return self._to_stored_dict_or_list(key, value)
+            return value
         value = _convert_dict_value(self.path + [key], value)
         # set db for StoredObject, because it is not set in the constructor
         if isinstance(value, StoredObject):
             value.set_db(self._db)
             value.set_parent(key=key, parent=self)
-        return self._to_stored_dict_or_list(key, value)
+        return value
 
     def __setitem__(self, key: _FLEX_KEY, value: Any) -> None:
         key = key_to_str(key)
@@ -273,6 +293,8 @@ def __setitem__(self, key: _FLEX_KEY, value: Any) -> None:
         if isinstance(value, StoredDict):
             value = value._dump()
             assert isinstance(value, dict)
+        # convert here
+        value = json_default(value)
         self._db.put(self.path, key, value)
 
     def __delitem__(self, key: _FLEX_KEY) -> None:
@@ -365,9 +387,9 @@ def should_convert(self):
     def _get_list_item(self, key: int):
         key = int(key)
         value = self._db.get(self.path, key)
+        value = self._to_stored_dict_or_list(key, value)
         if self.should_convert():
             value = _convert_dict_value(self.path + [key], value)
-        value = self._to_stored_dict_or_list(key, value)
         return value
 
     def __getitem__(self, s: slice) -> Any:
@@ -390,17 +412,20 @@ def __iter__(self) -> Iterator[str]:
         for i in range(self._db.list_len(self.path)):
             yield self._get_list_item(i)
 
-    def append(self, item):
-        self._db.list_append(self.path, item)
+    def append(self, value):
+        value = json_default(value)
+        self._db.list_append(self.path, value)
 
     def clear(self):
         self._db.list_clear(self.path)
         assert len(self) == 0
 
     def index(self, item) -> int:
+        item = json_default(item)
         return self._db.list_index(self.path, item)
 
     def remove(self, item):
+        item = json_default(item)
         self._db.list_remove(self.path, item)
 
     def _dump(self) -> list: