make StoredDict database agnostic

  - WalletDB no longer inherits from JsonDB, it uses a StoredDict
  - JsonDB inherits from BaseDB
  - FileStorage is only seen by JsonDB
  - calling JSonDB constructor creates the storage

Author: ecdsa

electrum/__init__.py

@@ -17,7 +17,7 @@ class GuiImportError(ImportError):
 from .version import ELECTRUM_VERSION
 from .util import format_satoshis
 from .wallet import Wallet
-from .storage import WalletStorage
+from .stored_dict import WalletStorage
 from .coinchooser import COIN_CHOOSERS
 from .network import Network, pick_random_server
 from .interface import Interface

electrum/daemon.py

@@ -47,7 +47,7 @@
     log_exceptions, randrange, OldTaskGroup, UserFacingException, JsonRPCError, os_chmod
 )
 from .wallet import Wallet, Abstract_Wallet
-from .storage import WalletStorage
+from .stored_dict import WalletStorage
 from .wallet_db import WalletDB, WalletUnfinished
 from .commands import known_commands, Commands
 from .simple_config import SimpleConfig
@@ -551,8 +551,7 @@ def _load_wallet(
             if not password:
                 raise InvalidPassword('No password given')
             storage.decrypt(password)
-        # read data, pass it to db
-        db = WalletDB(storage.read(), storage=storage, upgrade=upgrade)
+        db = WalletDB(storage, upgrade=upgrade)
         if db.get_action():
             raise WalletUnfinished(db)
         wallet = Wallet(db, config=config)

electrum/gui/qml/qedaemon.py

@@ -13,7 +13,7 @@
 from electrum.lnchannel import ChannelState
 from electrum.bitcoin import is_address
 from electrum.bitcoin import verify_usermessage_with_address
-from electrum.storage import StorageReadWriteError, WalletStorage
+from electrum.stored_dict import StorageReadWriteError, WalletStorage
 
 from .auth import AuthMixin, auth_protect
 from .qefx import QEFX

electrum/gui/qml/qewallet.py

@@ -751,7 +751,8 @@ def setPassword(self, password):
 
         try:
             self._logger.info('setting new password')
-            self.wallet.update_password(current_password, password, encrypt_storage=True)
+            encrypt_storage = self.wallet.storage.supports_file_encryption()
+            self.wallet.update_password(current_password, password, encrypt_storage=encrypt_storage)
             # restore the invariant that all loaded wallets in qml must be unlocked:
             self.wallet.unlock(password)
             return True

electrum/gui/qt/__init__.py

@@ -507,7 +507,7 @@ def _start_wizard_to_select_or_create_wallet(self, path) -> Optional[Abstract_Wa
                 self.logger.info('wizard dialog cancelled by user')
                 return
             db.put('x3', wizard.get_wizard_data()['x3'])
-            db.write_and_force_consolidation()  # TODO API for db is a bit weird: there should be a close method
+            db.storage.write()  # TODO API for db is a bit weird: there should be a close method
 
         wallet = self.daemon.load_wallet(wallet_file, password, upgrade=True)
         return wallet

electrum/gui/qt/main_window.py

@@ -1748,7 +1748,7 @@ def save_notes_text(self):
 
     def update_console(self):
         console = self.console
-        console.history = self.wallet.db.get_stored_item("qt-console-history", [])
+        console.history = self.wallet.db.get_list("qt-console-history")
         console.history_index = len(console.history)
 
         console.updateNamespace({
@@ -1950,8 +1950,12 @@ def on_password(hw_dev_pw):
         self.update_lock_menu()
 
     def _update_wallet_password(self, *, old_password, new_password, xpub_encrypt=False):
+        encrypt_storage = self.wallet.storage.supports_file_encryption()
         try:
-            self.wallet.update_password(old_password, new_password, encrypt_storage=True, xpub_encrypt=xpub_encrypt)
+            self.wallet.update_password(
+                old_password, new_password,
+                encrypt_storage=encrypt_storage,
+                xpub_encrypt=xpub_encrypt)
         except InvalidPassword as e:
             self.show_error(str(e))
             return

electrum/gui/qt/wizard/wallet.py

@@ -327,7 +327,7 @@ def on_filename(filename_or_path):
                 msg = ""
             self.valid = temp_storage is not None
             user_needs_to_enter_password = False
-            if temp_storage:
+            if temp_storage is not None:
                 if not temp_storage.file_exists():
                     msg = _("This file does not exist.") + '\n' \
                           + _("Press 'Next' to create this wallet, or choose another file.")

electrum/gui/text.py

@@ -21,7 +21,7 @@
 from electrum.transaction import PartialTxOutput
 from electrum.wallet import Wallet, Abstract_Wallet
 from electrum.wallet_db import WalletDB
-from electrum.storage import WalletStorage
+from electrum.stored_dict import WalletStorage
 from electrum.network import NetworkParameters, TxBroadcastError, BestEffortRequestFailed, ProxySettings
 from electrum.interface import ServerAddr
 from electrum.invoices import Invoice

electrum/json_db.py

@@ -30,15 +30,12 @@
 import jsonpatch
 import jsonpointer
 
-from . import util
-from .util import WalletFileException, profiler, sticky_property
+from .util import WalletFileException, profiler, sticky_property, MyEncoder
 from .logging import Logger
-from .stored_dict import StoredDict, _FLEX_KEY, registered_names, registered_keys, _convert_dict_key, _convert_dict_value
+from .stored_dict import _FLEX_KEY, BaseDB, _convert_dict_key, _convert_dict_value
+from .storage import FileStorage
 
 
-if TYPE_CHECKING:
-    from .storage import WalletStorage
-
 
 # We monkeypatch exceptions in the jsonpatch package to ensure they do not contain secrets from the DB.
 # We often log exceptions and offer to send them to the crash reporter, so they must not contain secrets.
@@ -84,36 +81,155 @@ def wrapper(self, *args, **kwargs):
 
 
 
-
-class JsonDB(Logger):
+class JsonDB(BaseDB):
 
     def __init__(
-        self,
-        s: str,
-        *,
-        storage: Optional['WalletStorage'] = None,
-        encoder=None,
-        upgrader=None,
+            self,
+            path: Optional[str],
+            *,
+            allow_partial_writes = True,
+            init_db = True,
+            encoder = MyEncoder,
+            upgrader = None,
     ):
-        Logger.__init__(self)
+        BaseDB.__init__(self, path)
+        self._is_closed = True
         self.lock = threading.RLock()
-        self.storage = storage
         self.encoder = encoder
+        self.upgrader = upgrader
         self.pending_changes = []  # type: List[str]
         self._modified = False
-        # load data
-        data = self.load_data(s)
-        if upgrader:
-            data, was_upgraded = upgrader(data)
+        if self.path:
+            self.storage = FileStorage(path, allow_partial_writes=allow_partial_writes)
+            if init_db and not self.is_encrypted():
+                # open DB if file is not encrypted
+                # otherwise, this will be called in self.decrypt
+                self.init_db()
+        else:
+            self.storage = None
+            self.set_data('{}')
+            self._is_closed = False
+
+    def set_data(self, json_str):
+        data = self.load_data(json_str)
+        if self.upgrader:
+            data, was_upgraded = self.upgrader(data)
             self._modified |= was_upgraded
-        # convert json to python objects
-        data = self._convert_dict([], data)
-        # convert dict to StoredDict
-        self.data = StoredDict(data, self)
-        self.data.set_parent(key='', parent=None)
+        self.json_data = self._convert_dict([], data)
+
+    def init_db(self):
+        if self.storage.is_encrypted():
+            assert self.storage.is_past_initial_decryption()
+        json_str = self.storage.read()
+        self.set_data(json_str)
         # write file in case there was a db upgrade
-        if self.storage and self.storage.file_exists():
-            self.write_and_force_consolidation()
+        self.write_and_force_consolidation()
+        self._is_closed = False
+
+    def decrypt(self, password: str):
+        self.storage.decrypt(password)
+        json_str = self.storage.read()
+        self.json_data = self.load_data(json_str)
+        self._is_closed = False
+
+    def check_password(self, password):
+        self.storage.check_password(password)
+
+    def supports_file_encryption(self):
+        return bool(self.storage)
+
+    def get_encryption_version(self):
+        return self.storage.get_encryption_version()
+
+    def is_encrypted(self):
+        return self.storage and self.storage.is_encrypted()
+
+    def is_encrypted_with_user_pw(self) -> bool:
+        return self.storage and self.storage.is_encrypted_with_user_pw()
+
+    def is_encrypted_with_hw_device(self) -> bool:
+        return self.storage and self.storage.is_encrypted_with_hw_device()
+
+    def set_password(self, password: str, enc_version=None):
+        self.storage.set_password(password, enc_version=enc_version)
+
+    def file_exists(self):
+        return self.storage and self.storage.file_exists()
+
+    def _subdict(self, path):
+        d = self.json_data
+        for k in path[1:]:
+            d = d[k]
+        return d
+
+    def iter_keys(self, path):
+        d = self._subdict(path)
+        return d.__iter__()
+
+    def dict_len(self, path):
+        d = self._subdict(path)
+        return len(d)
+
+    def contains(self, path, key):
+        d = self._subdict(path)
+        return key in d
+
+    def replace(self, path, key, value):
+        # called by setattr
+        self.db_replace(path, key, value)
+
+    @modifier
+    def put(self, path, key, value):
+        d = self._subdict(path)
+        is_new = key not in d
+        d[key] = value
+        self.db_add(path, key, value) if is_new else self.db_replace(path, key, value)
+
+    @modifier
+    def clear(self, path):
+        d = self._subdict(path)
+        d.clear()
+
+    def get(self, hint, key):
+        return hint[key]
+
+    def get_hint(self, path):
+        return self._subdict(path)
+
+    @modifier
+    def remove(self, path, key):
+        d = self._subdict(path)
+        d.pop(key)
+        self.db_remove(path, key)
+
+    @modifier
+    def list_append(self, path, item):
+        _list = self._subdict(path)
+        n = len(_list)
+        _list.append(item)
+        self.db_add(path, str(n), item)
+
+    def list_index(self, path, item):
+        _list = self._subdict(path)
+        return _list.index(item)
+
+    def list_len(self, path):
+        _list = self._subdict(path)
+        return len(_list)
+
+    @modifier
+    def list_clear(self, path):
+        _list = self._subdict(path)
+        _list.clear()
+        self.db_remove(path[:-1], path[-1])
+        self.db_add(path[:-1], path[-1], [])
+
+    @modifier
+    def list_remove(self, path, item):
+        _list = self._subdict(path)
+        n = _list.index(item)
+        _list.remove(item)
+        self.db_remove(path, str(n)) # fixme: keys
 
     def load_data(self, s: str) -> Dict[str, Any]:
         if s == '':
@@ -185,71 +301,30 @@ def add_patch(self, patch):
         self.pending_changes.append(json.dumps(patch, cls=self.encoder))
         self.set_modified(True)
 
-    def add(self, path, key: _FLEX_KEY, value) -> None:
+    def db_add(self, path, key: _FLEX_KEY, value) -> None:
         assert isinstance(key, _FLEX_KEY), repr(key)
         self.add_patch({'op': 'add', 'path': key_path(path, key), 'value': value})
 
-    def replace(self, path, key: _FLEX_KEY, value) -> None:
+    def db_replace(self, path, key: _FLEX_KEY, value) -> None:
         assert isinstance(key, _FLEX_KEY), repr(key)
         self.add_patch({'op': 'replace', 'path': key_path(path, key), 'value': value})
 
-    def remove(self, path, key: _FLEX_KEY) -> None:
+    def db_remove(self, path, key: _FLEX_KEY) -> None:
         assert isinstance(key, _FLEX_KEY), repr(key)
         self.add_patch({'op': 'remove', 'path': key_path(path, key)})
 
-    @locked
-    def get(self, key, default=None):
-        v = self.data.get(key)
-        if v is None:
-            v = default
-        return v
-
-    @modifier
-    def put(self, key, value):
-        try:
-            json.dumps(key, cls=self.encoder)
-            json.dumps(value, cls=self.encoder)
-        except Exception:
-            self.logger.info(f"json error: cannot save {repr(key)} ({repr(value)})")
-            return False
-        if value is not None:
-            if self.data.get(key) != value:
-                self.data[key] = copy.deepcopy(value)
-                return True
-        elif key in self.data:
-            self.data.pop(key)
-            return True
-        return False
-
-    @locked
-    def get_dict(self, name) -> dict:
-        # Warning: interacts un-intuitively with 'put': certain parts
-        # of 'data' will have pointers saved as separate variables.
-        if name not in self.data:
-            self.data[name] = {}
-        return self.data[name]
-
-    @locked
-    def get_stored_item(self, key, default) -> dict:
-        if key not in self.data:
-            self.data[key] = default
-        return self.data[key]
-
     @locked
     def dump(self, *, human_readable: bool = True) -> str:
         """Serializes the DB as a string.
         'human_readable': makes the json indented and sorted, but this is ~2x slower
         """
         return json.dumps(
-            self.data,
+            self.json_data,
             indent=4 if human_readable else None,
             sort_keys=bool(human_readable),
             cls=self.encoder,
         )
 
-    def _should_convert_to_stored_dict(self, key) -> bool:
-        return True
-
     def _convert_dict_key(self, path: List[str], key: str) -> _FLEX_KEY:
         return _convert_dict_key(path, key)
 
@@ -272,11 +347,20 @@ def _convert_dict(self, path: List[str], data: dict):
 
     @locked
     def write(self):
+        if not self.storage:
+            return
         if self.storage.should_do_full_write_next():
             self.write_and_force_consolidation()
         else:
             self._append_pending_changes()
 
+    def close(self):
+        # do not call write
+        self._is_closed = True
+
+    def is_closed(self):
+        return self._is_closed
+
     @locked
     def _append_pending_changes(self):
         if threading.current_thread().daemon: