Millisecond-resolution close times (MS_CLOSE_TIME, vnext)

common.mk

@@ -46,3 +46,7 @@ endif # ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION
 if ENABLE_FASTDEV_UNSAFE_FOR_PRODUCTION
 AM_CPPFLAGS += -DENABLE_FASTDEV_UNSAFE_FOR_PRODUCTION
 endif # ENABLE_FASTDEV_UNSAFE_FOR_PRODUCTION
+
+if MS_CLOSE_TIME
+AM_CPPFLAGS += -DMS_CLOSE_TIME
+endif # MS_CLOSE_TIME

configure.ac

@@ -588,6 +588,11 @@ AS_IF([test "x$enable_fastdev_unsafe_for_production" = "xyes"], [
 AM_CONDITIONAL(ENABLE_NEXT_PROTOCOL_VERSION_UNSAFE_FOR_PRODUCTION,
     [test x$enable_next_protocol_version_unsafe_for_production = xyes])
 
+# Millisecond-resolution closeTime (new StellarValue arms). Enabled together
+# with the next-protocol build until productionized.
+AM_CONDITIONAL(MS_CLOSE_TIME,
+    [test x$enable_next_protocol_version_unsafe_for_production = xyes])
+
 AC_PATH_PROG(CARGO, cargo)
 if test x"$CARGO" = x; then
   AC_MSG_ERROR([cannot find cargo, needed for rust code])

docs/stellar-core_example.cfg

@@ -561,7 +561,8 @@ ARTIFICIALLY_GENERATE_LOAD_FOR_TESTING=false
 #   incompatible with those of anyone else.
 ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=false
 
-# ARTIFICIALLY_SET_CLOSE_TIME_FOR_TESTING (in seconds), defaults to no override
+# ARTIFICIALLY_SET_CLOSE_TIME_FOR_TESTING (in milliseconds), defaults to no
+# override
 # Overrides the close time to the specified value but does not change checkpoint
 # frequency - this may cause network instability.
 # Do not use in production.

src/Makefile.am

@@ -113,6 +113,9 @@ BUILT_SOURCES = $(SRC_X_FILES:.x=.h) $(GENERATED_VERSION_SOURCES) $(TEST_FILES)
 
 $(SRC_X_FILES:.x=.h): $(XDRC)
 XDR_FEATURE_FLAGS =
+if MS_CLOSE_TIME
+XDR_FEATURE_FLAGS += -DMS_CLOSE_TIME
+endif # MS_CLOSE_TIME
 
 SUFFIXES = .x .h .rs
 .x.h:

src/bucket/test/BucketTestUtils.cpp

@@ -12,6 +12,7 @@
 #include "ledger/LedgerManager.h"
 #include "ledger/LedgerTxn.h"
 #include "main/Application.h"
+#include "test/TxTests.h"
 #include "test/test.h"
 #include "util/ProtocolVersion.h"
 #include "xdr/Stellar-ledger.h"
@@ -86,9 +87,10 @@ closeLedger(Application& app, std::optional<SecretKey> skToSignValue,
     CLOG_INFO(Bucket, "Artificially closing ledger {} with lcl={}, buckets={}",
               ledgerNum, hexAbbrev(lcl.hash),
               hexAbbrev(app.getBucketManager().getLiveBucketList().getHash()));
-    app.getHerder().externalizeValue(TxSetXDRFrame::makeEmpty(lcl), ledgerNum,
-                                     lcl.header.scpValue.closeTime, upgrades,
-                                     skToSignValue);
+    app.getHerder().externalizeValue(
+        TxSetXDRFrame::makeEmpty(lcl), ledgerNum,
+        txtest::makeConsensusTime(lcl.header.scpValue.closeTime), upgrades,
+        skToSignValue);
     while (lm.getLastClosedLedgerNum() < ledgerNum)
     {
         app.getClock().crank(true);

src/catchup/ApplyCheckpointWork.cpp

@@ -12,6 +12,7 @@
 #include "history/HistoryUtils.h"
 #include "historywork/Progress.h"
 #include "ledger/CheckpointRange.h"
+#include "ledger/LedgerHeaderUtils.h"
 #include "ledger/LedgerManager.h"
 #include "main/Application.h"
 #include "util/GlobalChecks.h"
@@ -277,14 +278,51 @@ ApplyCheckpointWork::getNextLedgerCloseData()
 
     // Empty-tx-set values should have the empty-tx-set hash (and vice versa)
     if ((header.scpValue.txSetHash == Herder::EMPTY_TX_SET_HASH) !=
-        (header.scpValue.ext.v() == STELLAR_VALUE_EMPTY_TX_SET))
+        isEmptyTxSetStellarValue(header.scpValue))
     {
         throw std::runtime_error(fmt::format(
             FMT_STRING("ledger header for {:d} has mismatched empty-tx-set "
                        "hash and StellarValue type {:d}"),
             header.ledgerSeq, static_cast<int32_t>(header.scpValue.ext.v())));
     }
 
+    // Check that we use the correct time format in the ledger header.
+    if (!hasValidCloseTime(header.scpValue))
+    {
+        throw std::runtime_error(fmt::format(
+            FMT_STRING("ledger header for {:d} has inconsistent close time "
+                       "fields: closeTime {:d}, consensus close time {}"),
+            header.ledgerSeq, header.scpValue.closeTime,
+            getConsensusTime(header.scpValue).toString()));
+    }
+
+    bool const hasMsCloseTime = isMsCloseTimeStellarValue(header.scpValue);
+    bool const protocolRequiresMsCloseTime =
+        protocolHasMsCloseTime(lclHeader.header.ledgerVersion);
+    if (hasMsCloseTime != protocolRequiresMsCloseTime)
+    {
+        throw std::runtime_error(fmt::format(
+            FMT_STRING("ledger header for {:d} has a {} close time "
+                       "(StellarValue type {:d}) but protocol {:d} requires a "
+                       "{} close time"),
+            header.ledgerSeq, hasMsCloseTime ? "millisecond" : "whole-second",
+            static_cast<int32_t>(header.scpValue.ext.v()),
+            lclHeader.header.ledgerVersion,
+            protocolRequiresMsCloseTime ? "millisecond" : "whole-second"));
+    }
+
+    // Close times must strictly increase from ledger to ledger
+    auto const previousCloseTime = getConsensusTime(lclHeader.header.scpValue);
+    auto const nextCloseTime = getConsensusTime(header.scpValue);
+    if (nextCloseTime <= previousCloseTime)
+    {
+        throw std::runtime_error(fmt::format(
+            FMT_STRING("ledger header for {:d} has a non-advancing close "
+                       "time {} (previous ledger closed at {})"),
+            header.ledgerSeq, nextCloseTime.toString(),
+            previousCloseTime.toString()));
+    }
+
     // We've verified the ledgerHeader (in the "trusted part of history"
     // sense) in CATCHUP_VERIFY phase; we now need to check that the
     // txhash we're about to apply is the one denoted by that ledger

src/catchup/CatchupWork.cpp

@@ -19,6 +19,7 @@
 #include "historywork/DownloadVerifyTxResultsWork.h"
 #include "historywork/GetAndUnzipRemoteFileWork.h"
 #include "historywork/GetHistoryArchiveStateWork.h"
+#include "ledger/LedgerHeaderUtils.h"
 #include "ledger/LedgerManager.h"
 #include "main/Application.h"
 #include "main/PersistentState.h"
@@ -69,7 +70,7 @@ setHerderStateTo(FileTransferInfo const& ft, uint32_t ledger, Application& app)
     app.getHerder().setTrackingSCPState(ledger, entry->header.scpValue,
                                         /* isTrackingNetwork */ false);
     CLOG_INFO(History, "Herder state is set! tracking={}, closeTime={}", ledger,
-              entry->header.scpValue.closeTime);
+              getConsensusTime(entry->header.scpValue).toString());
     return true;
 }
 

src/herder/Herder.h

@@ -28,6 +28,7 @@ struct EmptyTxSet
 };
 using TxSetResult = std::variant<TxSetXDRFrameConstPtr, EmptyTxSet>;
 class Application;
+class ConsensusTime;
 class XDROutputFileStream;
 
 /*
@@ -176,7 +177,7 @@ class Herder
 
     virtual void
     externalizeValue(TxSetXDRFrameConstPtr txSet, uint32_t ledgerSeq,
-                     uint64_t closeTime,
+                     ConsensusTime closeTime,
                      xdr::xvector<UpgradeType, 6> const& upgrades,
                      std::optional<SecretKey> skToSignValue = std::nullopt) = 0;
 
@@ -228,7 +229,7 @@ class Herder
 
     // helper function to craft an SCPValue
     virtual StellarValue
-    makeStellarValue(Hash const& txSetHash, uint64_t closeTime,
+    makeStellarValue(Hash const& txSetHash, ConsensusTime closeTime,
                      xdr::xvector<UpgradeType, 6> const& upgrades,
                      SecretKey const& s) = 0;