test: Recover state from real main net blocks

Add a test of `GlobalState::bootstrap_from_directory` using real data
from main net. The test uses the `blk0.dat` file from a node that has
been online since genesis, thus containing reorganizations that some
previously untested branches of `GlobalState::bootstrap_from_directory`.

This test also serves as a check that the first 113 main net blocks are
still considered valid by future versions of the software.

Author: Sword-Smith

.gitignore

@@ -7,8 +7,9 @@ target/
 /makefile-target
 /makefile-target-opt-level3
 
-# git shouldn't see the proof files. Bc big.
+# git shouldn't see the proof files, or block files. Bc big.
 /test_data/*.proof
+/test_data/blk*.dat
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html

src/models/blockchain/block/mod.rs

@@ -1156,7 +1156,7 @@ pub(crate) mod tests {
 
     #[cfg(test)]
     impl Block {
-        fn with_difficulty(mut self, difficulty: Difficulty) -> Self {
+        pub(crate) fn with_difficulty(mut self, difficulty: Difficulty) -> Self {
             self.kernel.header.difficulty = difficulty;
             self.unset_digest();
             self

src/models/state/mod.rs

@@ -3993,7 +3993,13 @@ mod tests {
     }
 
     mod bootstrap_from_raw_block_files {
+        use std::fs::File;
+        use std::io::Write;
+
         use super::*;
+        use crate::tests::shared::mock_genesis_global_state_with_block;
+        use crate::tests::shared::test_helper_data_dir;
+        use crate::tests::shared::try_fetch_file_from_server;
 
         async fn state_with_three_big_mocked_blocks(network: Network) -> GlobalStateLock {
             // Ensure more than one file is used to store blocks.
@@ -4092,6 +4098,75 @@ mod tests {
                 "Restored wallet state must agree with original state"
             );
         }
+
+        #[traced_test]
+        #[apply(shared_tokio_runtime)]
+        async fn can_restore_from_real_mainnet_data_with_reorganizations() {
+            let expected_blk_files = ["blk0.dat"];
+            let network = Network::Main;
+
+            // We need to override difficulty since it's different when the
+            // test flag is set. We need the real main net block.
+            let mainnets_real_genesis_block =
+                Block::genesis(network).with_difficulty(network.genesis_difficulty());
+
+            // Use at least four MPs per UTXO, otherwise they get unsynced.
+            let cli = cli_args::Args {
+                network,
+                number_of_mps_per_utxo: 4,
+                ..Default::default()
+            };
+            let peer_count = 0;
+
+            let mut state = mock_genesis_global_state_with_block(
+                peer_count,
+                WalletEntropy::devnet_wallet(),
+                cli,
+                mainnets_real_genesis_block,
+            )
+            .await;
+            let mut state = state.lock_guard_mut().await;
+
+            // Are the required blk files present on disk? If not, fetch them
+            // from a server.
+            let test_data_dir = test_helper_data_dir();
+            for blk_file_name in expected_blk_files {
+                let mut path = test_data_dir.clone();
+                path.push(blk_file_name);
+                if File::open(&path).is_err() {
+                    // Try fetching file from server and write it to disk.
+                    let (file, _server) = try_fetch_file_from_server(blk_file_name.to_owned())
+                        .unwrap_or_else(|| {
+                            panic!("File {blk_file_name} must be available from a server")
+                        });
+                    let mut f = File::create_new(&path).unwrap();
+                    f.write_all(&file).unwrap();
+                }
+            }
+
+            let validate_blocks = true;
+            state
+                .bootstrap_from_directory(&test_data_dir, validate_blocks)
+                .await
+                .unwrap();
+            let restored_block_height = state.chain.light_state().header().height;
+            assert_eq!(
+                BlockHeight::new(bfe!(113)),
+                restored_block_height,
+                "Expected block height not reached in state-recovery"
+            );
+
+            let wallet_status = state.get_wallet_status_for_tip().await;
+            let balance = state.wallet_state.confirmed_available_balance(
+                &wallet_status,
+                network.launch_date() + Timestamp::months(7),
+            );
+            assert_eq!(
+                NativeCurrencyAmount::coins(20),
+                balance,
+                "Expected balance must be available after state-recovery"
+            );
+        }
     }
 
     // note: removed test have_to_specify_change_policy()

src/tests/shared.rs

@@ -106,6 +106,7 @@ use crate::models::state::wallet::address::generation_address::GenerationReceivi
 use crate::models::state::wallet::expected_utxo::ExpectedUtxo;
 use crate::models::state::wallet::expected_utxo::UtxoNotifier;
 use crate::models::state::wallet::transaction_output::TxOutputList;
+use crate::models::state::wallet::wallet_configuration::WalletConfiguration;
 use crate::models::state::wallet::wallet_entropy::WalletEntropy;
 use crate::models::state::wallet::wallet_state::WalletState;
 use crate::models::state::GlobalState;
@@ -188,17 +189,15 @@ pub(crate) fn get_dummy_peer_connection_data_genesis(
     (handshake, socket_address)
 }
 
-/// Get a global state object for unit test purposes. This global state
-/// populated with state from the genesis block, e.g. in the archival mutator
-/// set and the wallet.
-///
+/// Get a global state object for unit test purposes. This global state is
+/// populated with state from a caller-defined genesis block.
 /// All contained peers represent outgoing connections.
-pub(crate) async fn mock_genesis_global_state(
+pub(crate) async fn mock_genesis_global_state_with_block(
     peer_count: u8,
     wallet: WalletEntropy,
     cli: cli_args::Args,
+    genesis_block: Block,
 ) -> GlobalStateLock {
-    let genesis_block = Block::genesis(cli.network);
     let data_dir: DataDirectory = unit_test_data_directory(cli.network).unwrap();
     let archival_state = ArchivalState::new(data_dir.clone(), genesis_block.clone()).await;
 
@@ -228,7 +227,10 @@ pub(crate) async fn mock_genesis_global_state(
         genesis_block.hash(),
     );
 
-    let wallet_state = WalletState::new_from_wallet_entropy(&data_dir, wallet, &cli).await;
+    let configuration = WalletConfiguration::new(&data_dir).absorb_options(&cli);
+    let wallet_state = WalletState::try_new(configuration, wallet, &genesis_block)
+        .await
+        .unwrap();
 
     // dummy channel
     let (rpc_to_main_tx, mut rpc_to_main_rx) = tokio::sync::mpsc::channel::<RPCServerToMain>(5);
@@ -243,6 +245,20 @@ pub(crate) async fn mock_genesis_global_state(
     GlobalStateLock::from_global_state(global_state, rpc_to_main_tx)
 }
 
+/// Get a global state object for unit test purposes. This global state is
+/// populated with state from the genesis block, e.g. in the archival mutator
+/// set and the wallet.
+///
+/// All contained peers represent outgoing connections.
+pub(crate) async fn mock_genesis_global_state(
+    peer_count: u8,
+    wallet: WalletEntropy,
+    cli: cli_args::Args,
+) -> GlobalStateLock {
+    let genesis_block = Block::genesis(cli.network);
+    mock_genesis_global_state_with_block(peer_count, wallet, cli, genesis_block).await
+}
+
 /// A state with a premine UTXO and self-mined blocks. Both composing and
 /// guessing was done by the returned entity. Tip has height of
 /// `num_blocks_mined`.