Mitigate rocksdb external timeout in AtomicBackupToDBCorrectness test (apple#12187)

* fix rocksdb external timeout

* address comments

Author: kakaiu

fdbclient/ServerKnobs.cpp

@@ -559,7 +559,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( ROCKSDB_SINGLEKEY_DELETES_MAX,                         200 ); // Max rocksdb::delete calls in a transaction
 	init( ROCKSDB_ENABLE_CLEAR_RANGE_EAGER_READS,              false );
 	init( ROCKSDB_FORCE_DELETERANGE_FOR_CLEARRANGE,            false );
-	init( ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT,                    0 ); if( isSimulated ) ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT = deterministicRandom()->randomInt(1000, 10000); // Default: 0 (disabled).
+	init( ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT,                    0 ); if( isSimulated ) ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT = deterministicRandom()->randomInt(200, 500); // Default: 0 (disabled).
 	// ROCKSDB_STATS_LEVEL=1 indicates rocksdb::StatsLevel::kExceptHistogramOrTimers
 	// Refer StatsLevel: https://github.com/facebook/rocksdb/blob/main/include/rocksdb/statistics.h#L594
 	init( ROCKSDB_STATS_LEVEL,                                     1 );
@@ -627,7 +627,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init( SHARDED_ROCKSDB_VALIDATE_MAPPING_RATIO,               0.01 ); if (isSimulated) SHARDED_ROCKSDB_VALIDATE_MAPPING_RATIO = deterministicRandom()->random01();
 	init( SHARD_METADATA_SCAN_BYTES_LIMIT,                  10485760 ); // 10MB
 	init( ROCKSDB_MAX_MANIFEST_FILE_SIZE,                  100 << 20 ); if (isSimulated) ROCKSDB_MAX_MANIFEST_FILE_SIZE = 500 << 20; // 500MB in simulation
-	init( SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS,          500 ); if (isSimulated) SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS = 50;
+	init( SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS,          500 ); if (isSimulated) SHARDED_ROCKSDB_MEMTABLE_MAX_RANGE_DELETIONS = 5;
 	init( SHARDED_ROCKSDB_AVERAGE_FILE_SIZE,                 8 << 20 ); // 8MB
 	init( SHARDED_ROCKSDB_COMPACTION_PERIOD, isSimulated? 3600 : 2592000 ); // 30d
 	init( SHARDED_ROCKSDB_COMPACTION_ACTOR_DELAY,               3600 ); // 1h

fdbserver/storageserver.actor.cpp

@@ -599,7 +599,9 @@ struct StorageServerDisk {
 	                                 Version newStorageVersion,
 	                                 int64_t& bytesLeft,
 	                                 UnlimitedCommitBytes unlimitedCommitBytes,
-	                                 int64_t& clearRangesLeft);
+	                                 int64_t& clearRangesLeft,
+	                                 const UID& ssId,
+	                                 bool verbose = false);
 	void makeVersionDurable(Version version);
 	void makeAccumulativeChecksumDurable(const AccumulativeChecksumState& acsState);
 	void clearAccumulativeChecksumState(const AccumulativeChecksumState& acsState);
@@ -12902,7 +12904,8 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 		state Version newOldestVersion = data->storageVersion();
 		state Version desiredVersion = data->desiredOldestVersion.get();
 		state int64_t bytesLeft = SERVER_KNOBS->STORAGE_COMMIT_BYTES;
-		state int64_t clearRangesLeft = data->storage.getKeyValueStoreType() == KeyValueStoreType::SSD_ROCKSDB_V1
+		state int64_t clearRangesLeft = (data->storage.getKeyValueStoreType() == KeyValueStoreType::SSD_ROCKSDB_V1 ||
+		                                 data->storage.getKeyValueStoreType() == KeyValueStoreType::SSD_SHARDED_ROCKSDB)
 		                                    ? (SERVER_KNOBS->ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT > 0
 		                                           ? SERVER_KNOBS->ROCKSDB_CLEARRANGES_LIMIT_PER_COMMIT
 		                                           : INT_MAX)
@@ -12981,12 +12984,22 @@ ACTOR Future<Void> updateStorage(StorageServer* data) {
 			}
 		}
 
+		// When unlimitedCommitBytes is set to true, clearRangesLeft will be ignored.
+		// Make sure unlimitedCommitBytes is set to True only when storage engine is sharded rocksdb.
+		ASSERT(data->shardAware || unlimitedCommitBytes == UnlimitedCommitBytes::False);
+
 		// Write mutations to storage until we reach the desiredVersion or have written too much (bytesleft)
 		// or until we reach clearRanges limit, in case of rocksdb.
 		state double beforeStorageUpdates = now();
 		loop {
-			state bool done = data->storage.makeVersionMutationsDurable(
-			    newOldestVersion, desiredVersion, bytesLeft, unlimitedCommitBytes, clearRangesLeft);
+			state bool done = data->storage.makeVersionMutationsDurable(newOldestVersion,
+			                                                            desiredVersion,
+			                                                            bytesLeft,
+			                                                            unlimitedCommitBytes,
+			                                                            clearRangesLeft,
+			                                                            data->thisServerID,
+			                                                            data->storage.getKeyValueStoreType() ==
+			                                                                KeyValueStoreType::SSD_ROCKSDB_V1);
 			if (data->tenantMap.getLatestVersion() < newOldestVersion) {
 				data->tenantMap.createNewVersion(newOldestVersion);
 			}
@@ -13574,10 +13587,21 @@ bool StorageServerDisk::makeVersionMutationsDurable(Version& prevStorageVersion,
                                                     Version newStorageVersion,
                                                     int64_t& bytesLeft,
                                                     UnlimitedCommitBytes unlimitedCommitBytes,
-                                                    int64_t& clearRangesLeft) {
-	if ((!unlimitedCommitBytes && bytesLeft <= 0) || clearRangesLeft <= 0)
+                                                    int64_t& clearRangesLeft,
+                                                    const UID& ssId,
+                                                    bool verbose) {
+	if (!unlimitedCommitBytes && (bytesLeft <= 0 || clearRangesLeft <= 0))
 		return true;
 
+	if (clearRangesLeft <= 0 && verbose) {
+		TraceEvent(SevInfo, "MakeVersionMutationsDurableClearRangesLeftZero", ssId)
+		    .suppressFor(5.0)
+		    .detail("PrevStorageVersion", prevStorageVersion)
+		    .detail("NewStorageVersion", newStorageVersion)
+		    .detail("BytesLeft", bytesLeft)
+		    .detail("ClearRangesLeft", clearRangesLeft)
+		    .detail("UnlimitedCommitBytes", unlimitedCommitBytes);
+	}
 	// Apply mutations from the mutationLog
 	auto u = data->getMutationLog().upper_bound(prevStorageVersion);
 	if (u != data->getMutationLog().end() && u->first <= newStorageVersion) {

tests/fast/AtomicBackupToDBCorrectness.toml

@@ -10,8 +10,8 @@ simBackupAgents = 'BackupToDB'
 
     [[test.workload]]
     testName = 'AtomicOps'
-    nodeCount = 30000
-    transactionsPerSecond = 2500.0
+    nodeCount = 10000
+    transactionsPerSecond = 750.0
     testDuration = 30.0
 
     [[test.workload]]