Experiment: verify secondaries in crash test by recreating expected state as of sequence number

db_stress_tool/db_stress_shared_state.h

@@ -10,6 +10,8 @@
 #ifdef GFLAGS
 #pragma once
 
+#include <iostream>
+
 #include "db_stress_tool/db_stress_stat.h"
 #include "db_stress_tool/expected_state.h"
 // SyncPoint is not supported in Released Windows Mode.
@@ -108,9 +110,11 @@ class SharedState {
     }
     if (status.ok()) {
       if (FLAGS_expected_values_dir.empty()) {
+        std::cout << "Using AnonExpectedStateManager" << std::endl;
         expected_state_manager_.reset(
             new AnonExpectedStateManager(FLAGS_max_key, FLAGS_column_families));
       } else {
+        std::cout << "Using FileExpectedStateManager" << std::endl;
         expected_state_manager_.reset(new FileExpectedStateManager(
             FLAGS_max_key, FLAGS_column_families, FLAGS_expected_values_dir));
       }
@@ -166,6 +170,8 @@ class SharedState {
 
   port::Mutex* GetMutex() { return &mu_; }
 
+  port::Mutex* GetSecondaryMutex() { return &secondary_mu_; }
+
   port::CondVar* GetCondVar() { return &cv_; }
 
   StressTest* GetStressTest() const { return stress_test_; }
@@ -261,6 +267,10 @@ class SharedState {
 
   Status Restore(DB* db) { return expected_state_manager_->Restore(db); }
 
+  Status SetSecondaryExpectedState(DB* db) {
+    return expected_state_manager_->SetSecondaryExpectedState(db);
+  }
+
   // Requires external locking covering all keys in `cf`.
   void ClearColumnFamily(int cf) {
     return expected_state_manager_->ClearColumnFamily(cf);
@@ -291,6 +301,10 @@ class SharedState {
     return expected_state_manager_->Get(cf, key);
   }
 
+  ExpectedValue GetSecondary(int cf, int64_t key) {
+    return expected_state_manager_->GetSecondary(cf, key);
+  }
+
   // Prepare a Delete that will be started but not finish yet
   // This is useful for crash-recovery testing when the process may crash
   // before updating the corresponding expected value
@@ -411,6 +425,7 @@ class SharedState {
   }
 
   port::Mutex mu_;
+  port::Mutex secondary_mu_;
   port::CondVar cv_;
   port::Mutex persist_seqno_mu_;
   const uint32_t seed_;

db_stress_tool/db_stress_stat.h

@@ -209,6 +209,7 @@ class Stats {
             num_compact_files_succeed_);
     fprintf(stdout, "%-12s: %ld CompactFiles() did not succeed\n", "",
             num_compact_files_failed_);
+    fprintf(stdout, "%-12s: %.2f elapsed\n", "", elapsed);
 
     if (FLAGS_histogram) {
       fprintf(stdout, "Microseconds per op:\n%s\n", hist_.ToString().c_str());

db_stress_tool/db_stress_test_base.cc

@@ -9,6 +9,7 @@
 //
 
 #include <ios>
+#include <iostream>
 #include <thread>
 
 #include "db_stress_tool/db_stress_listener.h"
@@ -3697,6 +3698,7 @@ void StressTest::Open(SharedState* shared, bool reopen) {
       const std::string& secondary_path = FLAGS_secondaries_base;
       s = DB::OpenAsSecondary(tmp_opts, FLAGS_db, secondary_path,
                               cf_descriptors, &cmp_cfhs_, &cmp_db_);
+      std::cout << "Opening secondary db" << std::endl;
       assert(s.ok());
       assert(cmp_cfhs_.size() == static_cast<size_t>(FLAGS_column_families));
     }