Fixed initialization of signed_num in new views; fixed default ports

Another contributor to the race condition in handle_verify_request is
the fact that the new-view initialization of the SST leaves all of the
delivery-counter fields (like persisted_num) at their default values,
assuming the first delivery predicate will update them, but
handle_verify_request can read from a remote node's signed_num before
that node has written to it. It's safer to initialize signed_num and the
corresponding signature to its correct value from the last view.
verified_num is written before it's read, like persisted_num, so it
doesn't need to be initialized.

Separately, the default values for the Derecho ports cause problems when
trying to test locally on Linux machines because they overlap with the
Linux ephemeral port range (32768 and higher). Tests keep failing
because a process accidentally connects to itself, or to another node's
system-assigned outgoing port for a different connection. It's better to
use lower values for these ports so they won't overlap with the
ephemeral range, and these values are still high enough to be unused by
any other service.

Author: etremel

include/derecho/conf/conf.hpp

@@ -83,16 +83,16 @@ class Conf {
     std::map<const std::string, std::string> config = {
             // [DERECHO]
             {DERECHO_CONTACT_IP, "127.0.0.1"},
-            {DERECHO_CONTACT_PORT, "23580"},
+            {DERECHO_CONTACT_PORT, "14480"},
             {DERECHO_RESTART_LEADERS, "127.0.0.1"},
-            {DERECHO_RESTART_LEADER_PORTS, "23580"},
+            {DERECHO_RESTART_LEADER_PORTS, "14480"},
             {DERECHO_LOCAL_ID, "0"},
             {DERECHO_LOCAL_IP, "127.0.0.1"},
-            {DERECHO_GMS_PORT, "23580"},
-            {DERECHO_STATE_TRANSFER_PORT, "28366"},
-            {DERECHO_SST_PORT, "37683"},
-            {DERECHO_RDMC_PORT, "31675"},
-            {DERECHO_EXTERNAL_PORT, "32645"},
+            {DERECHO_GMS_PORT, "14480"},
+            {DERECHO_STATE_TRANSFER_PORT, "14560"},
+            {DERECHO_SST_PORT, "14660"},
+            {DERECHO_RDMC_PORT, "14720"},
+            {DERECHO_EXTERNAL_PORT, "14880"},
             {SUBGROUP_DEFAULT_RDMC_SEND_ALGORITHM, "binomial_send"},
             {DERECHO_P2P_LOOP_BUSY_WAIT_BEFORE_SLEEP_MS, "250"},
             {DERECHO_SST_POLL_CQ_TIMEOUT_MS, "2000"},
@@ -126,7 +126,7 @@ class Conf {
             {LOGGER_DEFAULT_LOG_NAME, "derecho_debug"},
             {LOGGER_DEFAULT_LOG_LEVEL, "debug"},
             {LOGGER_LOG_TO_TERMINAL, "true"},
-            {LOGGER_LOG_FILE_DEPTH, "3"}};
+            {LOGGER_LOG_FILE_DEPTH, "10"}};
 
 public:
     // the option for parsing command line with getopt(not GetPot!!!)

include/derecho/core/detail/derecho_sst.hpp

@@ -389,7 +389,7 @@ void set(volatile char* string_array, const std::string& value);
 
 void increment(volatile int& member);
 
-bool equals(const volatile char& string_array, const std::string& value);
+bool equals(const volatile char* string_array, const std::string& value);
 
 }  // namespace gmssst
 

src/conf/derecho-sample.cfg

@@ -2,21 +2,21 @@
 # contact ip - the active leader's ip address
 contact_ip = 127.0.0.1
 # contact port - the active leader's gms port
-contact_port = 23580
+contact_port = 14480
 # list of leaders to contact during a restart in priority order
 restart_leaders = 127.0.0.1,127.0.0.1
 # list of GMS ports of the restart leaders, in the same order
-restart_leader_ports = 23580,23581
+restart_leader_ports = 14480,14481
 # derecho gms port
-gms_port = 23580
+gms_port = 14480
 # derecho state-transfer port
-state_transfer_port = 28366
+state_transfer_port = 14560
 # sst tcp port
-sst_port = 37683
+sst_port = 14660
 # rdmc tcp port
-rdmc_port = 31675
+rdmc_port = 14720
 # externel tcp port listening to external clients
-external_port = 32645
+external_port = 14880
 # Maximum possible node ID value
 # Node IDs are 32-bit integers, but all Derecho systems will have
 # many fewer nodes than this. Derecho will pre-allocate space for a
@@ -145,8 +145,8 @@ persistence_log_level = info
 # Whether logs should be printed to the terminal as well as saved to files (default is true)
 log_to_terminal = true
 # The number of older log files to save. Log files are rotated automatically
-# when the current one reaches 1MB in size. Default is 3.
-log_file_depth = 3
+# when the current one reaches 1MB in size. Default is 10.
+log_file_depth = 10
 
 # optional layout configurations
 [LAYOUT]

src/conf/derecho_node-sample.cfg

@@ -6,15 +6,15 @@ local_ip = 127.0.0.1
 # These ports are optional: nodes will use the values from the group derecho.cfg by default,
 # but if the port options are specified here they will override the defaults.
 # derecho gms port
-gms_port = 23580
+gms_port = 14481
 # derecho state-transfer port
-state_transfer_port = 28366
+state_transfer_port = 14561
 # sst tcp port
-sst_port = 37683
+sst_port = 14661
 # rdmc tcp port
-rdmc_port = 31675
+rdmc_port = 14721
 # externel tcp port listening to external clients
-external_port = 32645
+external_port = 14881
 
 
 # RDMA section contains configurations of the following

src/core/derecho_sst.cpp

@@ -33,7 +33,7 @@ void DerechoSST::init_local_row_from_previous(const DerechoSST& old_sst, const i
     memcpy(const_cast<uint16_t*>(joiner_external_ports[local_row]),
            const_cast<const uint16_t*>(old_sst.joiner_external_ports[row] + num_changes_installed),
            (old_sst.joiner_external_ports.size() - num_changes_installed) * sizeof(uint16_t));
-    //TODO: Copy over the last committed signature here? Or will the new view start with no signatures?
+    // Initialize these flags to false
     for(size_t i = 0; i < suspected.size(); ++i) {
         suspected[local_row][i] = false;
     }
@@ -43,11 +43,20 @@ void DerechoSST::init_local_row_from_previous(const DerechoSST& old_sst, const i
     for(size_t i = 0; i < global_min.size(); ++i) {
         global_min[local_row][i] = 0;
     }
+    // Initialize these counters with their previous values, except num_installed gets incremented
     num_changes[local_row] = old_sst.num_changes[row];
     num_committed[local_row] = old_sst.num_committed[row];
     num_acked[local_row] = old_sst.num_acked[row];
     num_installed[local_row] = old_sst.num_installed[row] + num_changes_installed;
     wedged[local_row] = false;
+    // Copy over the previous view's last known signature and signed_num array
+    // Unlike seq_num and persisted_num, these may get read by other nodes before they are updated in the new view
+    memcpy(const_cast<persistent::version_t*>(signed_num[local_row]),
+           const_cast<const persistent::version_t*>(old_sst.signed_num[row]),
+           old_sst.signed_num.size() * sizeof(persistent::version_t));
+    memcpy(const_cast<uint8_t*>(signatures[local_row]),
+           const_cast<const uint8_t*>(old_sst.signatures[row]),
+           old_sst.signatures.size() * sizeof(uint8_t));
 }
 
 void DerechoSST::init_local_change_proposals(const int other_row) {

src/core/git_version.cpp

@@ -13,8 +13,8 @@ namespace derecho {
 const int MAJOR_VERSION = 2;
 const int MINOR_VERSION = 4;
 const int PATCH_VERSION = 1;
-const int COMMITS_AHEAD_OF_VERSION = 10;
+const int COMMITS_AHEAD_OF_VERSION = 11;
 const char* VERSION_STRING = "2.4.1";
-const char* VERSION_STRING_PLUS_COMMITS = "2.4.1+10";
+const char* VERSION_STRING_PLUS_COMMITS = "2.4.1+11";
 
 }

src/core/view_manager.cpp

@@ -1484,7 +1484,7 @@ void ViewManager::deliver_ragged_trim(DerechoSST& gmsSST) {
                               gmsSST.persisted_num[member_row][subgroup_id])
                                       .second
                               < last_delivered_seq_num) {
-                    dbg_debug(vm_logger, "Waiting for node {} to finish persisting update {}", shard_member, last_delivered_seq_num);
+                    dbg_trace(vm_logger, "Waiting for node {} to finish persisting update {}", shard_member, last_delivered_seq_num);
                     return false;
                 }
             }
@@ -1639,7 +1639,6 @@ void ViewManager::finish_view_change(DerechoSST& gmsSST) {
     // This will block until everyone responds to SST/RDMC initial handshakes
     transition_multicast_group(next_subgroup_settings, new_num_received_size, new_slot_size, new_index_field_size);
     dbg_debug(vm_logger, "Done setting up SST and MulticastGroup for view {}; about to do a sync_with_members()", next_view->vid);
-    dbg_trace(vm_logger, "My row in new SST initialized to: {}", next_view->gmsSST->to_string());
 
     // New members can now proceed to view_manager.finish_setup(), which will call put() and sync()
     next_view->gmsSST->push_row_except_slots();
@@ -1650,6 +1649,7 @@ void ViewManager::finish_view_change(DerechoSST& gmsSST) {
         old_views_cv.notify_all();
     }
     curr_view = std::move(next_view);
+    dbg_trace(vm_logger, "New SST in view {}: {}", curr_view->vid, curr_view->gmsSST->to_string());
 
     if(any_persistent_objects) {
         // Write the new view to disk before using it