add support for LevelDB

.github/workflows/regtest.yml

@@ -53,7 +53,7 @@ jobs:
       - name: Install OS deps
         run: |
           sudo apt-get update
-          sudo apt-get -y install curl jq bc automake libtool
+          sudo apt-get -y install curl jq bc automake libtool libleveldb-dev
       - name: Install Electrum and ElectrumX
         # installs e-x some commits after 1.18.0 tag
         # uses --ignore-installed to ignore installed system installed attrs

.github/workflows/security-review.yml

@@ -76,7 +76,7 @@ jobs:
       - name: Install Qt/QML runtime deps
         run: |
           sudo apt-get update
-          sudo apt-get -y install libgl1 libegl1 libxkbcommon0 libdbus-1-3
+          sudo apt-get -y install libgl1 libegl1 libxkbcommon0 libdbus-1-3 libleveldb-dev
 
       - name: Install Python dependencies
         run: |

.github/workflows/tests.yml

@@ -101,7 +101,7 @@ jobs:
       - name: Install Qt/QML runtime deps
         run: |
           sudo apt-get update
-          sudo apt-get -y install libgl1 libegl1 libxkbcommon0 libdbus-1-3
+          sudo apt-get -y install libgl1 libegl1 libxkbcommon0 libdbus-1-3 libleveldb-dev
       - name: Install Python dependencies
         run: |
           pip install -r contrib/requirements/requirements-ci.txt

contrib/requirements/requirements-ci.txt

@@ -1,3 +1,4 @@
 pytest
 coverage
 coveralls
+plyvel

electrum/__init__.py

@@ -17,7 +17,7 @@ class GuiImportError(ImportError):
 from .version import ELECTRUM_VERSION
 from .util import format_satoshis
 from .wallet import Wallet
-from .storage import WalletStorage
+from .stored_dict import WalletStorage
 from .coinchooser import COIN_CHOOSERS
 from .network import Network, pick_random_server
 from .interface import Interface

electrum/commands.py

@@ -298,29 +298,32 @@ async def close_wallet(self, wallet_path=None):
         return await self.daemon._stop_wallet(wallet_path)
 
     @command('')
-    async def create(self, passphrase=None, password=None, encrypt_file=True, seed_type=None, wallet_path=None):
+    async def create(self, passphrase=None, password=None, encrypt_file=True, seed_type=None, wallet_path=None, use_levelDB=False):
         """Create a new wallet.
         If you want to be prompted for an argument, type '?' or ':' (concealed)
 
         arg:str:passphrase:Seed extension
         arg:str:seed_type:The type of wallet to create, e.g. 'standard' or 'segwit'
         arg:bool:encrypt_file:Whether the file on disk should be encrypted with the provided password
+        arg:bool:use_levelDB:Create levelDB storage. Note that LevelDB storage does not support file encryption. The password will only encrypt the keystore.
         """
         d = create_new_wallet(
             path=wallet_path,
             passphrase=passphrase,
             password=password,
             encrypt_file=encrypt_file,
             seed_type=seed_type,
+            use_levelDB=use_levelDB,
             config=self.config)
+        d['wallet'].storage.close()
         return {
             'seed': d['seed'],
             'path': d['wallet'].storage.get_path(),
             'msg': d['msg'],
         }
 
     @command('')
-    async def restore(self, text, passphrase=None, password=None, encrypt_file=True, wallet_path=None):
+    async def restore(self, text, passphrase=None, password=None, encrypt_file=True, wallet_path=None, use_levelDB=False):
         """Restore a wallet from text. Text can be a seed phrase, a master
         public key, a master private key, a list of bitcoin addresses
         or bitcoin private keys.
@@ -329,6 +332,7 @@ async def restore(self, text, passphrase=None, password=None, encrypt_file=True,
         arg:str:text:seed phrase
         arg:str:passphrase:Seed extension
         arg:bool:encrypt_file:Whether the file on disk should be encrypted with the provided password
+        arg:bool:use_levelDB:Create levelDB storage. Note that LevelDB storage does not support file encryption. The password will only encrypt the keystore.
         """
         # TODO create a separate command that blocks until wallet is synced
         d = restore_wallet_from_text(
@@ -337,7 +341,9 @@ async def restore(self, text, passphrase=None, password=None, encrypt_file=True,
             passphrase=passphrase,
             password=password,
             encrypt_file=encrypt_file,
+            use_levelDB=use_levelDB,
             config=self.config)
+        d['wallet'].storage.close()
         return {
             'path': d['wallet'].storage.get_path(),
             'msg': d['msg'],
@@ -356,7 +362,7 @@ async def password(self, password=None, new_password=None, encrypt_file=None, wa
         if encrypt_file is None:
             if not password and new_password:
                 # currently no password, setting one now: we encrypt by default
-                encrypt_file = True
+                encrypt_file = wallet.storage.supports_file_encryption()
             else:
                 encrypt_file = wallet.storage.is_encrypted()
         wallet.update_password(password, new_password, encrypt_storage=encrypt_file)

electrum/daemon.py

@@ -47,7 +47,7 @@
     log_exceptions, randrange, OldTaskGroup, UserFacingException, JsonRPCError, os_chmod
 )
 from .wallet import Wallet, Abstract_Wallet
-from .storage import WalletStorage
+from .stored_dict import WalletStorage
 from .wallet_db import WalletDB, WalletUnfinished
 from .commands import known_commands, Commands
 from .simple_config import SimpleConfig
@@ -551,8 +551,7 @@ def _load_wallet(
             if not password:
                 raise InvalidPassword('No password given')
             storage.decrypt(password)
-        # read data, pass it to db
-        db = WalletDB(storage.read(), storage=storage, upgrade=upgrade)
+        db = WalletDB(storage, upgrade=upgrade)
         if db.get_action():
             raise WalletUnfinished(db)
         wallet = Wallet(db, config=config)

electrum/gui/qml/qedaemon.py

@@ -13,7 +13,7 @@
 from electrum.lnchannel import ChannelState
 from electrum.bitcoin import is_address
 from electrum.bitcoin import verify_usermessage_with_address
-from electrum.storage import StorageReadWriteError, WalletStorage
+from electrum.stored_dict import StorageReadWriteError, WalletStorage
 
 from .auth import AuthMixin, auth_protect
 from .qefx import QEFX

electrum/gui/qml/qewallet.py

@@ -751,7 +751,8 @@ def setPassword(self, password):
 
         try:
             self._logger.info('setting new password')
-            self.wallet.update_password(current_password, password, encrypt_storage=True)
+            encrypt_storage = self.wallet.storage.supports_file_encryption()
+            self.wallet.update_password(current_password, password, encrypt_storage=encrypt_storage)
             # restore the invariant that all loaded wallets in qml must be unlocked:
             self.wallet.unlock(password)
             return True

electrum/gui/qt/__init__.py

@@ -507,7 +507,7 @@ def _start_wizard_to_select_or_create_wallet(self, path) -> Optional[Abstract_Wa
                 self.logger.info('wizard dialog cancelled by user')
                 return
             db.put('x3', wizard.get_wizard_data()['x3'])
-            db.write_and_force_consolidation()  # TODO API for db is a bit weird: there should be a close method
+            db.storage.write()  # TODO API for db is a bit weird: there should be a close method
 
         wallet = self.daemon.load_wallet(wallet_file, password, upgrade=True)
         return wallet

electrum/gui/qt/history_list.py

@@ -142,6 +142,8 @@ def get_data_for_role(self, index: QModelIndex, role: Qt.ItemDataRole) -> QVaria
         assert index.isValid()
         col = index.column()
         window = self.model.window
+        if not window.isVisible():
+            return
         tx_item = self.get_data()
         is_lightning = tx_item.get('lightning', False)
         if not is_lightning and 'txid' not in tx_item:

electrum/gui/qt/main_window.py

@@ -1748,7 +1748,7 @@ def save_notes_text(self):
 
     def update_console(self):
         console = self.console
-        console.history = self.wallet.db.get_stored_item("qt-console-history", [])
+        console.history = self.wallet.db.get_list("qt-console-history")
         console.history_index = len(console.history)
 
         console.updateNamespace({
@@ -1950,8 +1950,12 @@ def on_password(hw_dev_pw):
         self.update_lock_menu()
 
     def _update_wallet_password(self, *, old_password, new_password, xpub_encrypt=False):
+        encrypt_storage = self.wallet.storage.supports_file_encryption()
         try:
-            self.wallet.update_password(old_password, new_password, encrypt_storage=True, xpub_encrypt=xpub_encrypt)
+            self.wallet.update_password(
+                old_password, new_password,
+                encrypt_storage=encrypt_storage,
+                xpub_encrypt=xpub_encrypt)
         except InvalidPassword as e:
             self.show_error(str(e))
             return

electrum/gui/qt/wizard/wallet.py

@@ -248,9 +248,6 @@ def __init__(self, parent, wizard):
 
         path = wizard._path
 
-        if os.path.isdir(path):
-            raise Exception("wallet path cannot point to a directory")
-
         self.wallet_exists = False
         self.wallet_is_open = False
         self.wallet_needs_hw_unlock = False
@@ -327,7 +324,7 @@ def on_filename(filename_or_path):
                 msg = ""
             self.valid = temp_storage is not None
             user_needs_to_enter_password = False
-            if temp_storage:
+            if temp_storage is not None:
                 if not temp_storage.file_exists():
                     msg = _("This file does not exist.") + '\n' \
                           + _("Press 'Next' to create this wallet, or choose another file.")

electrum/gui/text.py

@@ -21,7 +21,7 @@
 from electrum.transaction import PartialTxOutput
 from electrum.wallet import Wallet, Abstract_Wallet
 from electrum.wallet_db import WalletDB
-from electrum.storage import WalletStorage
+from electrum.stored_dict import WalletStorage
 from electrum.network import NetworkParameters, TxBroadcastError, BestEffortRequestFailed, ProxySettings
 from electrum.interface import ServerAddr
 from electrum.invoices import Invoice

electrum/invoices.py

@@ -236,7 +236,7 @@ def get_id(self) -> str:
         else:  # on-chain
             return get_id_from_onchain_outputs(outputs=self.get_outputs(), timestamp=self.time)
 
-    def as_dict(self, status):
+    def export(self, status):
         d = {
             'is_lightning': self.is_lightning(),
             'amount_BTC': format_satoshis(self.get_amount_sat()),
@@ -298,7 +298,7 @@ def can_be_paid_onchain(self) -> bool:
             return True
 
     def to_debug_json(self) -> Dict[str, Any]:
-        d = self.to_json()
+        d = self.as_dict()
         d["lnaddr"] = self._lnaddr.to_debug_json()
         return d