diff --git a/fdbclient/ServerKnobs.cpp b/fdbclient/ServerKnobs.cpp index 7426ffc8ef4..179f48c1120 100644 --- a/fdbclient/ServerKnobs.cpp +++ b/fdbclient/ServerKnobs.cpp @@ -929,6 +929,8 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi init( SPRING_BYTES_TLOG, 400e6 ); if( smallTlogTarget ) SPRING_BYTES_TLOG = 200e3; init( TARGET_BYTES_PER_TLOG_BATCH, 1400e6 ); if( smallTlogTarget ) TARGET_BYTES_PER_TLOG_BATCH = 1400e3; init( SPRING_BYTES_TLOG_BATCH, 300e6 ); if( smallTlogTarget ) SPRING_BYTES_TLOG_BATCH = 150e3; + // Match MIN_AVAILABLE_SPACE by default; buggified simulations exercise the earlier ramp. + init( TLOG_THROTTLE_START_AVAILABLE_SPACE_RATIO, 0.05 ); if( randomize && isSimulated && BUGGIFY ) TLOG_THROTTLE_START_AVAILABLE_SPACE_RATIO = 0.20; init( TLOG_SPILL_THRESHOLD, 1500e6 ); if( smallTlogTarget ) TLOG_SPILL_THRESHOLD = 1500e3; if( randomize && BUGGIFY ) TLOG_SPILL_THRESHOLD = 0; init( REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT, 20e6 ); if( (randomize && BUGGIFY) || smallTlogTarget ) REFERENCE_SPILL_UPDATE_STORAGE_BYTE_LIMIT = 1e6; init( TLOG_HARD_LIMIT_BYTES, 3000e6 ); if( smallTlogTarget ) TLOG_HARD_LIMIT_BYTES = 30e6; diff --git a/fdbclient/include/fdbclient/ServerKnobs.h b/fdbclient/include/fdbclient/ServerKnobs.h index 0eddcde938f..6a9584b3cec 100644 --- a/fdbclient/include/fdbclient/ServerKnobs.h +++ b/fdbclient/include/fdbclient/ServerKnobs.h @@ -956,6 +956,7 @@ class SWIFT_CXX_IMMORTAL_SINGLETON_TYPE ServerKnobs : public KnobsImpl + #include "fdbclient/ClientKnobs.h" #include "fdbserver/core/Knobs.h" #include "fdbserver/core/ServerDBInfo.h" @@ -978,17 +980,31 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) { limitReason_t tlogLimitReason = limitReason_t::log_server_write_queue; - int64_t minFreeSpace = std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE, - (int64_t)(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * tl.getSmoothTotalSpace())); - - worstFreeSpaceTLog = - std::min(worstFreeSpaceTLog, std::max((int64_t)tl.getSmoothFreeSpace() - minFreeSpace, (int64_t)0)); - - int64_t springBytes = std::max( - 1, std::min(limits->logSpringBytes, (tl.getSmoothFreeSpace() - minFreeSpace) * 0.2)); - int64_t targetBytes = - std::max(1, std::min(limits->logTargetBytes, (int64_t)tl.getSmoothFreeSpace() - minFreeSpace)); - if (targetBytes != limits->logTargetBytes) { + const auto smoothTotalSpace = tl.getSmoothTotalSpace(); + const auto smoothFreeSpace = tl.getSmoothFreeSpace(); + const auto minFreeSpaceByRatio = + static_cast(SERVER_KNOBS->MIN_AVAILABLE_SPACE_RATIO * smoothTotalSpace); + const auto minFreeSpace = std::max(SERVER_KNOBS->MIN_AVAILABLE_SPACE, minFreeSpaceByRatio); + // Start shrinking the tlog queue budget once available space drops below the configured ratio, + // then ramp down linearly until reaching minFreeSpace. + const auto throttleStartSpaceByRatio = + static_cast(SERVER_KNOBS->TLOG_THROTTLE_START_AVAILABLE_SPACE_RATIO * smoothTotalSpace); + const auto throttleStartSpace = std::max(minFreeSpace + 1, throttleStartSpaceByRatio); + const auto availableAboveMin = smoothFreeSpace - minFreeSpace; + const auto availableAboveMinBytes = static_cast(availableAboveMin); + const auto throttleWindow = static_cast(throttleStartSpace - minFreeSpace); + const auto diskBudgetRatio = std::clamp(availableAboveMin / throttleWindow, 0.0, 1.0); + + worstFreeSpaceTLog = std::min(worstFreeSpaceTLog, std::max(availableAboveMinBytes, int64_t{ 0 })); + + const auto scaledSpringBytes = static_cast(limits->logSpringBytes * diskBudgetRatio); + const auto springBytes = + std::max(int64_t{ 1 }, std::min(std::max(int64_t{ 1 }, scaledSpringBytes), availableAboveMinBytes)); + const auto scaledTargetBytes = static_cast(limits->logTargetBytes * diskBudgetRatio); + const auto targetBytes = + std::max(int64_t{ 1 }, std::min(std::max(int64_t{ 1 }, scaledTargetBytes), availableAboveMinBytes)); + if (diskBudgetRatio < 1.0) { + CODE_PROBE(true, "Ratekeeper tlog disk budget ratio below one"); if (minFreeSpace == SERVER_KNOBS->MIN_AVAILABLE_SPACE) { tlogLimitReason = limitReason_t::log_server_min_free_space; } else { @@ -998,9 +1014,11 @@ void Ratekeeper::updateRate(RatekeeperLimits* limits) { TraceEvent("RatekeeperLimitReasonDetails") .detail("TLogID", tl.id) .detail("Reason", tlogLimitReason) - .detail("TLSmoothFreeSpace", tl.getSmoothFreeSpace()) - .detail("TLSmoothTotalSpace", tl.getSmoothTotalSpace()) + .detail("TLSmoothFreeSpace", smoothFreeSpace) + .detail("TLSmoothTotalSpace", smoothTotalSpace) .detail("LimitsLogTargetBytes", limits->logTargetBytes) + .detail("ThrottleStartSpace", throttleStartSpace) + .detail("DiskBudgetRatio", diskBudgetRatio) .detail("TargetBytes", targetBytes) .detail("MinFreeSpace", minFreeSpace); }