From de9d2852749d4f2b6e21f31590b16015c7fc4d14 Mon Sep 17 00:00:00 2001 From: Trevor Clinkenbeard Date: Wed, 13 May 2026 01:18:36 -0700 Subject: [PATCH 1/2] Bound status fetch latency to overall timeout budget (cherry picked from commit 79de02031e115e4c2644870be0da85f95d0e7777) --- fdbclient/StatusClient.actor.cpp | 38 +++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/fdbclient/StatusClient.actor.cpp b/fdbclient/StatusClient.actor.cpp index a6ca87c549a..7330cf1c1d9 100644 --- a/fdbclient/StatusClient.actor.cpp +++ b/fdbclient/StatusClient.actor.cpp @@ -422,9 +422,10 @@ ACTOR Future clientStatusFetcher(Reference> clusterStatusFetcher(ClusterInterface cI, StatusArray* messages, - std::string statusField) { + std::string statusField, + double timeoutSeconds) { state StatusRequest req(statusField); - state Future clusterTimeout = delay(CLIENT_KNOBS->STATUS_TIMEOUT); + state Future clusterTimeout = delay(timeoutSeconds); state Optional oStatusObj; wait(delay(0.0)); // make sure the cluster controller is marked as not failed @@ -462,6 +463,27 @@ ACTOR Future> clusterStatusFetcher(ClusterInterface cI, return oStatusObj; } +TEST_CASE("/fdbclient/status/overallTimeoutBudget") { + state ClusterInterface clusterInterface; + state StatusArray messages; + state double overallTimeout = 0.2; + state double coordinatorProbeDuration = 0.1; + state double startTime = now(); + state double statusDeadline = startTime + overallTimeout; + + wait(delay(coordinatorProbeDuration)); + state Optional status = wait( + clusterStatusFetcher(clusterInterface, &messages, "", std::max(0.0, statusDeadline - now()))); + + double elapsed = now() - startTime; + ASSERT(!status.present()); + ASSERT_EQ(messages.size(), 1); + ASSERT_EQ(messages.front().get_obj().at("name").get_str(), "status_incomplete_timeout"); + ASSERT_GE(elapsed, overallTimeout); + ASSERT_LT(elapsed, overallTimeout + 0.05); + return Void(); +} + // Create and return a database_status section. // Will not throw, will not return an empty section. StatusObject getClientDatabaseStatus(StatusObjectReader client, StatusObjectReader cluster) { @@ -516,6 +538,9 @@ ACTOR Future statusFetcherImpl(Reference if (!g_network) throw network_not_setup(); + // Keep the overall status request within STATUS_TIMEOUT, including the client-side coordinator probe that runs + // before we ask the cluster controller for its status payload. + state double statusDeadline = now() + CLIENT_KNOBS->STATUS_TIMEOUT; state StatusObject statusObj; state StatusObject statusObjClient; state StatusArray clientMessages; @@ -549,12 +574,15 @@ ACTOR Future statusFetcherImpl(Reference if (quorum_reachable) { try { - state Future interfaceTimeout = delay(2.0); + state Future interfaceTimeout = delay(std::min(2.0, std::max(0.0, statusDeadline - now()))); loop { if (clusterInterface->get().present()) { - Optional _statusObjCluster = - wait(clusterStatusFetcher(clusterInterface->get().get(), &clientMessages, statusField)); + Optional _statusObjCluster = wait(clusterStatusFetcher(clusterInterface->get().get(), + &clientMessages, + statusField, + std::max(0.0, + statusDeadline - now()))); if (_statusObjCluster.present()) { statusObjCluster = _statusObjCluster.get(); // TODO: this is a temporary fix, getting the number of available coordinators should move to From 3d82058d64e2ebde6d59efffce53e8f47deff42a Mon Sep 17 00:00:00 2001 From: Han Xu Date: Tue, 21 Jul 2026 11:15:59 +0800 Subject: [PATCH 2/2] fix(status): include client probe in timeout budget Propagate the overall status deadline through the client coordinator probe and add coverage for the client phase. --- fdbclient/StatusClient.actor.cpp | 48 +++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/fdbclient/StatusClient.actor.cpp b/fdbclient/StatusClient.actor.cpp index 7330cf1c1d9..4851cebb60f 100644 --- a/fdbclient/StatusClient.actor.cpp +++ b/fdbclient/StatusClient.actor.cpp @@ -19,6 +19,7 @@ */ #include "flow/flow.h" +#include "fdbclient/ClusterConnectionMemoryRecord.h" #include "fdbclient/CoordinationInterface.h" #include "fdbclient/MonitorLeader.h" #include "fdbclient/ClusterInterface.h" @@ -305,7 +306,8 @@ void JSONDoc::mergeValueInto(json_spirit::mValue& dst, const json_spirit::mValue // Will not throw, will just return non-present Optional if error ACTOR Future> clientCoordinatorsStatusFetcher(Reference connRecord, bool* quorum_reachable, - int* coordinatorsFaultTolerance) { + int* coordinatorsFaultTolerance, + double statusDeadline) { try { state ClientCoordinators coord(connRecord); state StatusObject statusObj; @@ -340,7 +342,7 @@ ACTOR Future> clientCoordinatorsStatusFetcher(Reference> clientCoordinatorsStatusFetcher(Reference clientStatusFetcher(Reference connRecord, StatusArray* messages, bool* quorum_reachable, - int* coordinatorsFaultTolerance) { + int* coordinatorsFaultTolerance, + double statusDeadline) { state StatusObject statusObj; state Optional coordsStatusObj = - wait(clientCoordinatorsStatusFetcher(connRecord, quorum_reachable, coordinatorsFaultTolerance)); + wait(clientCoordinatorsStatusFetcher(connRecord, quorum_reachable, coordinatorsFaultTolerance, statusDeadline)); state bool contentsUpToDate = wait(connRecord->upToDate()); if (coordsStatusObj.present()) { @@ -472,8 +475,8 @@ TEST_CASE("/fdbclient/status/overallTimeoutBudget") { state double statusDeadline = startTime + overallTimeout; wait(delay(coordinatorProbeDuration)); - state Optional status = wait( - clusterStatusFetcher(clusterInterface, &messages, "", std::max(0.0, statusDeadline - now()))); + state Optional status = + wait(clusterStatusFetcher(clusterInterface, &messages, "", std::max(0.0, statusDeadline - now()))); double elapsed = now() - startTime; ASSERT(!status.present()); @@ -484,6 +487,25 @@ TEST_CASE("/fdbclient/status/overallTimeoutBudget") { return Void(); } +TEST_CASE("/fdbclient/status/clientCoordinatorTimeoutBudget") { + state Reference connRecord = + makeReference(ClusterConnectionString("test:test@127.0.0.1:1")); + state bool quorumReachable = false; + state int coordinatorsFaultTolerance = 0; + state double timeout = 0.2; + state double startTime = now(); + + state Optional status = wait(clientCoordinatorsStatusFetcher( + connRecord, &quorumReachable, &coordinatorsFaultTolerance, startTime + timeout)); + + double elapsed = now() - startTime; + ASSERT(status.present()); + ASSERT(!quorumReachable); + ASSERT_GE(elapsed, timeout); + ASSERT_LT(elapsed, timeout + 0.05); + return Void(); +} + // Create and return a database_status section. // Will not throw, will not return an empty section. StatusObject getClientDatabaseStatus(StatusObjectReader client, StatusObjectReader cluster) { @@ -552,8 +574,8 @@ ACTOR Future statusFetcherImpl(Reference try { state int64_t clientTime = g_network->timer(); - StatusObject _statusObjClient = - wait(clientStatusFetcher(connRecord, &clientMessages, &quorum_reachable, &coordinatorsFaultTolerance)); + StatusObject _statusObjClient = wait(clientStatusFetcher( + connRecord, &clientMessages, &quorum_reachable, &coordinatorsFaultTolerance, statusDeadline)); statusObjClient = _statusObjClient; if (clientTime != -1) @@ -578,11 +600,11 @@ ACTOR Future statusFetcherImpl(Reference loop { if (clusterInterface->get().present()) { - Optional _statusObjCluster = wait(clusterStatusFetcher(clusterInterface->get().get(), - &clientMessages, - statusField, - std::max(0.0, - statusDeadline - now()))); + Optional _statusObjCluster = + wait(clusterStatusFetcher(clusterInterface->get().get(), + &clientMessages, + statusField, + std::max(0.0, statusDeadline - now()))); if (_statusObjCluster.present()) { statusObjCluster = _statusObjCluster.get(); // TODO: this is a temporary fix, getting the number of available coordinators should move to