fix: Only cancel background work if primary RocksDB instance is dropped (#2918)

This effectively reverts
https://github.com/FuelLabs/fuel-core/pull/2730/files which causes a lot
of "Shutdown in progress: stalled writes" errors when multiple rocksdb
instances are being dropped in a running node.

---------

Co-authored-by: green <xgreenx9999@gmail.com>
Co-authored-by: Aaryamann Challani <43716372+rymnc@users.noreply.github.com>

Author: 3 people

.changes/fixed/2918.md

@@ -0,0 +1 @@
+Only cancel background work if primary RocksDB instance is dropped

Cargo.toml

@@ -104,7 +104,7 @@ fuel-vm-private = { version = "0.60.0", package = "fuel-vm", default-features =
 
 # Common dependencies
 anyhow = "1.0"
-async-graphql = { version = "7.0.11", features = [
+async-graphql = { version = "7.0.15", features = [
   "graphiql",
   "tracing",
 ], default-features = false }

crates/fuel-core/src/state/rocks_db.rs

@@ -12,6 +12,7 @@ use super::rocks_db_key_iterator::{
     ExtractItem,
     RocksDBKeyIterator,
 };
+use core::ops::Deref;
 use fuel_core_metrics::core_metrics::DatabaseMetrics;
 use fuel_core_storage::{
     iter::{
@@ -72,6 +73,23 @@ use std::{
 };
 use tempfile::TempDir;
 
+#[derive(Debug)]
+struct PrimaryInstance(DBWithThreadMode<MultiThreaded>);
+
+impl Deref for PrimaryInstance {
+    type Target = DBWithThreadMode<MultiThreaded>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl Drop for PrimaryInstance {
+    fn drop(&mut self) {
+        self.cancel_all_background_work(true);
+    }
+}
+
 type DB = DBWithThreadMode<MultiThreaded>;
 
 type DropFn = Box<dyn FnOnce() + Send + Sync>;
@@ -135,7 +153,7 @@ impl DatabaseConfig {
 
 pub struct RocksDb<Description> {
     read_options: ReadOptions,
-    db: Arc<DB>,
+    db: Arc<PrimaryInstance>,
     block_opts: Arc<BlockBasedOptions>,
     create_family: Option<Arc<Mutex<BTreeMap<String, Options>>>>,
     snapshot: Option<rocksdb::SnapshotWithThreadMode<'static, DB>>,
@@ -150,7 +168,6 @@ impl<Description> Drop for RocksDb<Description> {
         // Drop the snapshot before the db.
         // Dropping the snapshot after the db will cause a sigsegv.
         self.snapshot = None;
-        self.db.cancel_all_background_work(true);
     }
 }
 
@@ -402,7 +419,7 @@ where
             }
             ColumnsPolicy::Lazy => Some(Arc::new(Mutex::new(cf_descriptors_to_create))),
         };
-        let db = Arc::new(db);
+        let db = Arc::new(PrimaryInstance(db));
 
         let rocks_db = RocksDb {
             read_options: Self::generate_read_options(&None),