On detecting a deadlock, we are shutting down the Node Software now if the feature "shutdown-on-deadlock" is defined, and is activated as a default feature for now.

Shutdowns are now timed to take a maximum of 60 seconds, instead of 300 seconds.
DMDcoin#231
Code Infrastructure changes: ShutdownManager is now hosted as an Arc.

Author: SurfingNerd

Cargo.toml

@@ -99,7 +99,7 @@ lazy_static = "1.2.0"
 winapi = { version = "0.3.4", features = ["winsock2", "winuser", "shellapi"] }
 
 [features]
-default = ["accounts"]
+default = ["shutdown-on-deadlock", "accounts"]
 accounts = ["ethcore-accounts", "parity-rpc/accounts"]
 miner-debug = ["ethcore/miner-debug"]
 json-tests = ["ethcore/json-tests"]
@@ -118,6 +118,7 @@ deadlock_detection = ["parking_lot/deadlock_detection"]
 # and `massif-visualizer` for visualization
 memory_profiling = []
 secretstore = []
+shutdown-on-deadlock = ["deadlock_detection"]
 
 [lib]
 path = "bin/oe/lib.rs"

bin/oe/blockchain.rs

@@ -214,7 +214,7 @@ fn execute_import(cmd: ImportBlockchain) -> Result<(), String> {
         // TODO [ToDr] don't use test miner here
         // (actually don't require miner at all)
         Arc::new(Miner::new_for_tests(&spec, None)),
-        ShutdownManager::null(),
+        Arc::new(ShutdownManager::null()),
     )
     .map_err(|e| format!("Client service error: {:?}", e))?;
 
@@ -282,7 +282,7 @@ fn start_client(
     require_fat_db: bool,
     max_round_blocks_to_import: usize,
     shutdown_on_missing_block_import: Option<u64>,
-    shutdown: ShutdownManager,
+    shutdown: Arc<ShutdownManager>,
 ) -> Result<ClientService, String> {
     // load spec file
     let spec = spec.spec(&dirs.cache)?;
@@ -376,7 +376,7 @@ fn execute_export(cmd: ExportBlockchain) -> Result<(), String> {
         false,
         cmd.max_round_blocks_to_import,
         None,
-        ShutdownManager::null(),
+        Arc::new(ShutdownManager::null()),
     )?;
     let client = service.client();
 
@@ -407,7 +407,7 @@ fn execute_export_state(cmd: ExportState) -> Result<(), String> {
         true,
         cmd.max_round_blocks_to_import,
         None,
-        ShutdownManager::null(),
+        Arc::new(ShutdownManager::null()),
     )?;
 
     let client = service.client();
@@ -527,7 +527,7 @@ fn execute_reset(cmd: ResetBlockchain) -> Result<(), String> {
         false,
         0,
         None,
-        ShutdownManager::null(),
+        Arc::new(ShutdownManager::null()),
     )?;
 
     let client = service.client();

bin/oe/lib.rs

@@ -146,7 +146,18 @@ fn print_hash_of(maybe_file: Option<String>) -> Result<String, String> {
 }
 
 #[cfg(feature = "deadlock_detection")]
-fn run_deadlock_detection_thread() {
+#[cfg(feature = "shutdown-on-deadlock")]
+fn on_deadlock_detected(shutdown: &Arc<ShutdownManager>) {
+    warn!("Deadlock detected, trying to shutdown the node software");
+    shutdown.demand_shutdown();
+}
+
+#[cfg(feature = "deadlock_detection")]
+#[cfg(not(feature = "shutdown-on-deadlock"))]
+fn on_deadlock_detected(_: &Arc<ShutdownManager>) {}
+
+#[cfg(feature = "deadlock_detection")]
+fn run_deadlock_detection_thread(shutdown: Arc<ShutdownManager>) {
     use ansi_term::Style;
     use parking_lot::deadlock;
     use std::{thread, time::Duration};
@@ -176,6 +187,7 @@ fn run_deadlock_detection_thread() {
                     warn!("{:#?}", t.backtrace());
                 }
             }
+            on_deadlock_detected(&shutdown);
         }
     });
 
@@ -206,12 +218,14 @@ fn execute(
     logger: Arc<RotatingLogger>,
     shutdown: ShutdownManager,
 ) -> Result<ExecutionAction, String> {
+    let shutdown_arc = Arc::new(shutdown);
+
     #[cfg(feature = "deadlock_detection")]
-    run_deadlock_detection_thread();
+    run_deadlock_detection_thread(shutdown_arc.clone());
 
     match command.cmd {
         Cmd::Run(run_cmd) => {
-            let outcome = run::execute(run_cmd, logger, shutdown)?;
+            let outcome = run::execute(run_cmd, logger, shutdown_arc)?;
             Ok(ExecutionAction::Running(outcome))
         }
         Cmd::Version => Ok(ExecutionAction::Instant(Some(Args::print_version()))),

bin/oe/run.rs

@@ -177,7 +177,7 @@ impl ChainSyncing for SyncProviderWrapper {
 pub fn execute(
     cmd: RunCmd,
     logger: Arc<RotatingLogger>,
-    shutdown: ShutdownManager,
+    shutdown: Arc<ShutdownManager>,
 ) -> Result<RunningClient, String> {
     // load spec
     let spec = cmd.spec.spec(&cmd.dirs.cache)?;
@@ -755,8 +755,8 @@ fn print_running_environment(data_dir: &str, dirs: &Directories, db_dirs: &Datab
 
 fn wait_for_drop<T>(w: Weak<T>) {
     const SLEEP_DURATION: Duration = Duration::from_secs(1);
-    const WARN_TIMEOUT: Duration = Duration::from_secs(60);
-    const MAX_TIMEOUT: Duration = Duration::from_secs(300);
+    const WARN_TIMEOUT: Duration = Duration::from_secs(30);
+    const MAX_TIMEOUT: Duration = Duration::from_secs(60);
 
     let instant = Instant::now();
     let mut warned = false;

bin/oe/snapshot.rs

@@ -241,7 +241,7 @@ impl SnapshotCommand {
             // TODO [ToDr] don't use test miner here
             // (actually don't require miner at all)
             Arc::new(Miner::new_for_tests(&spec, None)),
-            ShutdownManager::null(),
+            Arc::new(ShutdownManager::null()),
         )
         .map_err(|e| format!("Client service error: {:?}", e))?;
 

crates/ethcore/service/src/service.rs

@@ -58,7 +58,7 @@ impl ClientService {
         restoration_db_handler: Box<dyn BlockChainDBHandler>,
         _ipc_path: &Path,
         miner: Arc<Miner>,
-        shutdown: ShutdownManager,
+        shutdown: Arc<ShutdownManager>,
     ) -> Result<ClientService, Error> {
         let io_service = IoService::<ClientIoMessage>::start("Client", 4)?;
 

crates/ethcore/src/client/client.rs

@@ -304,7 +304,7 @@ pub struct Client {
 
     importer: Importer,
 
-    shutdown: ShutdownManager,
+    shutdown: Arc<ShutdownManager>,
 
     statistics: ClientStatistics,
 }
@@ -994,7 +994,7 @@ impl Client {
         db: Arc<dyn BlockChainDB>,
         miner: Arc<Miner>,
         message_channel: IoChannel<ClientIoMessage>,
-        shutdown: ShutdownManager,
+        shutdown: Arc<ShutdownManager>,
     ) -> Result<Arc<Client>, crate::error::Error> {
         let trie_spec = match config.fat_db {
             true => TrieSpec::Fat,