Skip to content

Commit f3839df

Browse files
NaluTripicianNaluTripician
committed
Merge branch 'users/nalutripician/ppaf_with_0s_requestTimeout-fix' of https://github.com/Azure/azure-cosmos-dotnet-v3 into users/nalutripician/ppaf_with_0s_requestTimeout-fix
2 parents 1b2e275 + e11fa3f commit f3839df

12 files changed

Lines changed: 387 additions & 11 deletions

File tree

‎Directory.Build.props‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<ClientOfficialVersion>3.53.1</ClientOfficialVersion>
44
<ClientPreviewVersion>3.54.0</ClientPreviewVersion>
55
<ClientPreviewSuffixVersion>preview.1</ClientPreviewSuffixVersion>
6-
<DirectVersion>3.40.3</DirectVersion>
6+
<DirectVersion>3.41.0</DirectVersion>
77
<FaultInjectionVersion>1.0.0</FaultInjectionVersion>
88
<FaultInjectionSuffixVersion>beta.0</FaultInjectionSuffixVersion>
99
<EncryptionOfficialVersion>2.0.5</EncryptionOfficialVersion>

‎Microsoft.Azure.Cosmos/src/DocumentClient.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6778,6 +6778,8 @@ private void InitializeDirectConnectivity(IStoreClientFactory storeClientFactory
67786778

67796779
private void CreateStoreModel(bool subscribeRntbdStatus)
67806780
{
6781+
AccountConfigurationProperties accountConfigurationProperties = new (EnableNRegionSynchronousCommit: this.accountServiceConfiguration.AccountProperties.EnableNRegionSynchronousCommit);
6782+
67816783
//EnableReadRequestsFallback, if not explicity set on the connection policy,
67826784
//is false if the account's consistency is bounded staleness,
67836785
//and true otherwise.
@@ -6792,7 +6794,8 @@ private void CreateStoreModel(bool subscribeRntbdStatus)
67926794
this.UseMultipleWriteLocations && (this.accountServiceConfiguration.DefaultConsistencyLevel != Documents.ConsistencyLevel.Strong),
67936795
true,
67946796
enableReplicaValidation: this.isReplicaAddressValidationEnabled,
6795-
sessionRetryOptions: this.ConnectionPolicy.SessionRetryOptions);
6797+
sessionRetryOptions: this.ConnectionPolicy.SessionRetryOptions,
6798+
accountConfigurationProperties: accountConfigurationProperties);
67966799

67976800
if (subscribeRntbdStatus)
67986801
{

‎Microsoft.Azure.Cosmos/src/Resource/Settings/AccountProperties.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -269,5 +269,7 @@ private IDictionary<string, object> QueryStringToDictConverter()
269269
[JsonExtensionData]
270270
internal IDictionary<string, JToken> AdditionalProperties { get; set; }
271271

272+
[JsonProperty(PropertyName = Constants.Properties.EnableNRegionSynchronousCommit, NullValueHandling = NullValueHandling.Ignore)]
273+
internal bool EnableNRegionSynchronousCommit { get; set; }
272274
}
273275
}

‎Microsoft.Azure.Cosmos/src/Routing/AvailabilityStrategy/CrossRegionHedgingAvailabilityStrategy.cs‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ internal class CrossRegionHedgingAvailabilityStrategy : AvailabilityStrategyInte
2525
{
2626
private const string HedgeContext = "Hedge Context";
2727
private const string HedgeConfig = "Hedge Config";
28+
private const string ResponseRegion = "Response Region";
2829

2930
/// <summary>
3031
/// Latency threshold which activates the first region hedging
@@ -206,6 +207,10 @@ internal override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
206207
((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum(
207208
HedgeContext,
208209
hedgeRegions.Take(requestNumber + 1));
210+
// Note that the target region can be seperate than the actual region that serviced the request depending on the scenario
211+
((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum(
212+
ResponseRegion,
213+
hedgeResponse.TargetRegionName);
209214
return hedgeResponse.ResponseMessage;
210215
}
211216
}
@@ -234,6 +239,9 @@ internal override async Task<ResponseMessage> ExecuteAvailabilityStrategyAsync(
234239
((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum(
235240
HedgeContext,
236241
hedgeRegions);
242+
((CosmosTraceDiagnostics)hedgeResponse.ResponseMessage.Diagnostics).Value.AddOrUpdateDatum(
243+
ResponseRegion,
244+
hedgeResponse.TargetRegionName);
237245
return hedgeResponse.ResponseMessage;
238246
}
239247
}
@@ -278,6 +286,7 @@ private async Task<HedgingResponse> CloneAndSendAsync(
278286
return await this.RequestSenderAndResultCheckAsync(
279287
sender,
280288
clonedRequest,
289+
hedgeRegions.ElementAt(requestNumber),
281290
cancellationToken,
282291
cancellationTokenSource,
283292
trace);
@@ -287,6 +296,7 @@ private async Task<HedgingResponse> CloneAndSendAsync(
287296
private async Task<HedgingResponse> RequestSenderAndResultCheckAsync(
288297
Func<RequestMessage, CancellationToken, Task<ResponseMessage>> sender,
289298
RequestMessage request,
299+
string targetRegionName,
290300
CancellationToken cancellationToken,
291301
CancellationTokenSource cancellationTokenSource,
292302
ITrace trace)
@@ -301,10 +311,10 @@ private async Task<HedgingResponse> RequestSenderAndResultCheckAsync(
301311
cancellationTokenSource.Cancel();
302312
}
303313

304-
return new HedgingResponse(true, response);
314+
return new HedgingResponse(true, response, targetRegionName);
305315
}
306316

307-
return new HedgingResponse(false, response);
317+
return new HedgingResponse(false, response, targetRegionName);
308318
}
309319
catch (OperationCanceledException oce) when (cancellationTokenSource.IsCancellationRequested)
310320
{
@@ -346,11 +356,13 @@ private sealed class HedgingResponse
346356
{
347357
public readonly bool IsNonTransient;
348358
public readonly ResponseMessage ResponseMessage;
359+
public readonly string TargetRegionName;
349360

350-
public HedgingResponse(bool isNonTransient, ResponseMessage responseMessage)
361+
public HedgingResponse(bool isNonTransient, ResponseMessage responseMessage, string targetRegionName)
351362
{
352363
this.IsNonTransient = isNonTransient;
353364
this.ResponseMessage = responseMessage;
365+
this.TargetRegionName = targetRegionName;
354366
}
355367
}
356368
}

‎Microsoft.Azure.Cosmos/src/ThinClientTransportSerializer.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,8 @@ public static async Task<Stream> SerializeProxyRequestAsync(
113113
activityId,
114114
bufferProvider.Provider,
115115
accountName,
116-
out _,
116+
accountName,
117+
out _,
117118
out _);
118119

119120
int length = serializedRequest.RequestSize;

‎Microsoft.Azure.Cosmos/src/Tracing/TraceWriter.TraceJsonWriter.cs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -409,7 +409,10 @@ public void Visit(StoreResult storeResult)
409409
this.WriteStringValueOrNull(storeResult.PartitionKeyRangeId);
410410

411411
this.jsonWriter.WriteFieldName(nameof(storeResult.GlobalCommittedLSN));
412-
this.jsonWriter.WriteNumberValue(storeResult.GlobalCommittedLSN);
412+
this.jsonWriter.WriteNumberValue(storeResult.GlobalCommittedLSN);
413+
414+
this.jsonWriter.WriteFieldName(nameof(storeResult.GlobalNRegionCommittedGLSN));
415+
this.jsonWriter.WriteNumberValue(storeResult.GlobalNRegionCommittedGLSN);
413416

414417
this.jsonWriter.WriteFieldName(nameof(storeResult.ItemLSN));
415418
this.jsonWriter.WriteNumberValue(storeResult.ItemLSN);
@@ -453,7 +456,7 @@ public void Visit(StoreResult storeResult)
453456

454457
this.jsonWriter.WriteFieldName("TransportException");
455458
TransportException transportException = storeResult.Exception?.InnerException as TransportException;
456-
this.WriteStringValueOrNull(transportException?.Message);
459+
this.WriteStringValueOrNull(transportException?.Message);
457460

458461
this.jsonWriter.WriteObjectEnd();
459462
}

‎Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosAvailabilityStrategyTests.cs‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,71 @@ public async Task AvailabilityStrategyRequestOptionsTriggerTest(bool isPreferred
340340
}
341341
}
342342

343+
[TestMethod]
344+
[DataRow(false, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest with preferred regions.")]
345+
[DataRow(true, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest w/o preferred regions.")]
346+
[TestCategory("MultiRegion")]
347+
public async Task AvailabilityStrategyResponseRegionDiagnosticsTest(bool isPreferredLocationsEmpty)
348+
{
349+
FaultInjectionRule responseDelay = new FaultInjectionRuleBuilder(
350+
id: "responseDely",
351+
condition:
352+
new FaultInjectionConditionBuilder()
353+
.WithRegion(region1)
354+
.WithOperationType(FaultInjectionOperationType.ReadItem)
355+
.Build(),
356+
result:
357+
FaultInjectionResultBuilder.GetResultBuilder(FaultInjectionServerErrorType.ResponseDelay)
358+
.WithDelay(TimeSpan.FromMilliseconds(4000))
359+
.Build())
360+
.WithDuration(TimeSpan.FromMinutes(90))
361+
.Build();
362+
363+
List<FaultInjectionRule> rules = new List<FaultInjectionRule>() { responseDelay };
364+
FaultInjector faultInjector = new FaultInjector(rules);
365+
366+
responseDelay.Disable();
367+
368+
CosmosClientOptions clientOptions = new CosmosClientOptions()
369+
{
370+
ConnectionMode = ConnectionMode.Direct,
371+
ApplicationPreferredRegions = isPreferredLocationsEmpty ? new List<string>() : new List<string>() { region1, region2 },
372+
Serializer = this.cosmosSystemTextJsonSerializer
373+
};
374+
375+
using (CosmosClient faultInjectionClient = new CosmosClient(
376+
connectionString: this.connectionString,
377+
clientOptions: faultInjector.GetFaultInjectionClientOptions(clientOptions)))
378+
{
379+
Database database = faultInjectionClient.GetDatabase(MultiRegionSetupHelpers.dbName);
380+
Container container = database.GetContainer(MultiRegionSetupHelpers.containerName);
381+
382+
//warm up connections read
383+
ItemResponse<CosmosIntegrationTestObject> _ = await container.ReadItemAsync<CosmosIntegrationTestObject>("testId", new PartitionKey("pk"));
384+
385+
responseDelay.Enable();
386+
387+
ItemRequestOptions requestOptions = new ItemRequestOptions
388+
{
389+
AvailabilityStrategy = new CrossRegionHedgingAvailabilityStrategy(
390+
threshold: TimeSpan.FromMilliseconds(100),
391+
thresholdStep: TimeSpan.FromMilliseconds(50))
392+
};
393+
ItemResponse<CosmosIntegrationTestObject> ir = await container.ReadItemAsync<CosmosIntegrationTestObject>(
394+
"testId",
395+
new PartitionKey("pk"),
396+
requestOptions);
397+
398+
CosmosTraceDiagnostics traceDiagnostic = ir.Diagnostics as CosmosTraceDiagnostics;
399+
Assert.IsNotNull(traceDiagnostic);
400+
Assert.IsTrue(traceDiagnostic.ToString()
401+
.Contains($"\"Hedge Context\":[\"{region1}\",\"{region2}\""));
402+
traceDiagnostic.Value.Data.TryGetValue("Response Region", out object responseRegionObj);
403+
Assert.IsNotNull(responseRegionObj);
404+
Assert.AreEqual(region2, responseRegionObj as string);
405+
}
406+
}
407+
343408
[TestMethod]
344409
[DataRow(false, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest with preferred regions.")]
345410
[DataRow(true, DisplayName = "ValidateAvailabilityStrategyNoTriggerTest w/o preferred regions.")]

‎Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/DocumentClientUnitTests.cs‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,18 @@ namespace Microsoft.Azure.Cosmos.SDK.EmulatorTests
88
using System.Globalization;
99
using System.Net;
1010
using System.Net.Http;
11+
using System.Net.Security;
1112
using System.Threading;
1213
using System.Threading.Tasks;
14+
using Microsoft.Azure.Cosmos.Common;
1315
using Microsoft.Azure.Cosmos.Internal;
1416
using Microsoft.Azure.Cosmos.Linq;
1517
using Microsoft.Azure.Cosmos.Query.Core.QueryPlan;
18+
using Microsoft.Azure.Cosmos.Tests;
1619
using Microsoft.Azure.Cosmos.Tracing;
1720
using Microsoft.Azure.Cosmos.Utils;
1821
using Microsoft.Azure.Documents;
22+
using Microsoft.Azure.Documents.Client;
1923
using Microsoft.VisualStudio.TestTools.UnitTesting;
2024
using Moq;
2125

@@ -272,6 +276,112 @@ private void TestRetryOnThrottled(int? numberOfRetries)
272276
Assert.IsTrue(throttled);
273277
}
274278

279+
[TestMethod]
280+
[DataRow(false, DisplayName = "NRegion Synchronous commit is disabled for the account")]
281+
[DataRow(true, DisplayName = "NRegion Synchronous commit is enabled for the account")]
282+
public void EnableNRegionSynchronousCommit_PassedToStoreClient(bool nRegionCommitEnabled)
283+
{
284+
285+
StoreClient storeClient = new StoreClient(
286+
new Mock<IAddressResolver>().Object,
287+
new SessionContainer(string.Empty),
288+
new Mock<IServiceConfigurationReader>().Object,
289+
new Mock<IAuthorizationTokenProvider>().Object,
290+
Protocol.Tcp,
291+
new Mock<TransportClient>().Object);
292+
// Arrange
293+
Mock<IStoreClientFactory> mockStoreClientFactory = new Mock<IStoreClientFactory>();
294+
mockStoreClientFactory.Setup(f => f.CreateStoreClient(
295+
It.IsAny<IAddressResolver>(),
296+
It.IsAny<ISessionContainer>(),
297+
It.IsAny<IServiceConfigurationReader>(),
298+
It.IsAny<IAuthorizationTokenProvider>(),
299+
It.IsAny<bool>(),
300+
It.IsAny<bool>(),
301+
It.IsAny<bool>(),
302+
It.IsAny<bool>(),
303+
It.IsAny<bool>(),
304+
It.IsAny<bool>(),
305+
It.IsAny<AccountConfigurationProperties>(),
306+
It.IsAny<ISessionRetryOptions>()
307+
)).Returns(storeClient);
308+
309+
DocumentClient documentClient = new DocumentClient(
310+
new Uri("https://localhost:8081"),
311+
new Mock<AuthorizationTokenProvider>().Object,
312+
new EventHandler<SendingRequestEventArgs>((s, e) => { }),
313+
new ConnectionPolicy(),
314+
null, // desiredConsistencyLevel
315+
null, // serializerSettings
316+
ApiType.None,
317+
new EventHandler<ReceivedResponseEventArgs>((s, e) => { }),
318+
null, // handler
319+
new Mock<ISessionContainer>().Object,
320+
null, // enableCpuMonitor
321+
new Func<TransportClient, TransportClient>(tc => tc),
322+
mockStoreClientFactory.Object,
323+
false, // isLocalQuorumConsistency
324+
"testClientId",
325+
new RemoteCertificateValidationCallback((sender, certificate, chain, sslPolicyErrors) => true),
326+
new Mock<CosmosClientTelemetryOptions>().Object,
327+
new Mock<IChaosInterceptorFactory>().Object,
328+
true // enableAsyncCacheExceptionNoSharing
329+
);
330+
331+
AccountProperties accountProperties = new AccountProperties
332+
{
333+
// Set the property to true for test
334+
EnableNRegionSynchronousCommit = nRegionCommitEnabled,
335+
};
336+
337+
AccountConsistency ac = new AccountConsistency();
338+
ac.DefaultConsistencyLevel = (Cosmos.ConsistencyLevel) ConsistencyLevel.Session;
339+
accountProperties.Consistency = ac;
340+
341+
Func<Task<AccountProperties>> getDatabaseAccountFn = () =>
342+
// When called with any Uri, return the expected AccountProperties
343+
Task.FromResult(accountProperties);
344+
345+
CosmosAccountServiceConfiguration accountServiceConfiguration = new CosmosAccountServiceConfiguration(
346+
getDatabaseAccountFn);
347+
348+
typeof(CosmosAccountServiceConfiguration)
349+
.GetProperty("AccountProperties", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)
350+
.SetValue(accountServiceConfiguration, accountProperties);
351+
352+
//Inject the accountServiceConfiguration into the DocumentClient via reflection.
353+
typeof(DocumentClient)
354+
.GetProperty("accountServiceConfiguration", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)
355+
.SetValue(documentClient, accountServiceConfiguration);
356+
357+
358+
typeof(DocumentClient)
359+
.GetField("storeClientFactory", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)
360+
.SetValue(documentClient, mockStoreClientFactory.Object);
361+
362+
// Act: Call the private method via reflection
363+
typeof(DocumentClient)
364+
.GetMethod("CreateStoreModel", System.Reflection.BindingFlags.Instance | System.Reflection.BindingFlags.NonPublic)
365+
.Invoke(documentClient, new object[] { true });
366+
367+
// Assert: Verify the correct value was passed
368+
mockStoreClientFactory.Verify(f =>
369+
f.CreateStoreClient(
370+
It.IsAny<IAddressResolver>(),
371+
It.IsAny<ISessionContainer>(),
372+
It.IsAny<IServiceConfigurationReader>(),
373+
It.IsAny<IAuthorizationTokenProvider>(),
374+
It.IsAny<bool>(),
375+
It.IsAny<bool>(),
376+
It.IsAny<bool>(),
377+
It.IsAny<bool>(),
378+
It.IsAny<bool>(),
379+
It.IsAny<bool>(),
380+
It.Is<AccountConfigurationProperties>(config => config.EnableNRegionSynchronousCommit == accountProperties.EnableNRegionSynchronousCommit),
381+
It.IsAny<ISessionRetryOptions>()),
382+
Times.Once,
383+
"EnableNRegionSynchronousCommit was not passed correctly to AccountConfigurationProperties and StoreClient.");
384+
}
275385

276386
private DocumentClientException CreateTooManyRequestException(int retryAfterInMilliseconds)
277387
{

‎Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/BaselineTest/TestBaseline/TraceWriterBaselineTests.TraceData.xml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,8 @@
272272
Exception
273273
)
274274
]]></Text>
275-
<Json><![CDATA[{
275+
<Json>
276+
<![CDATA[{
276277
"Summary": {
277278
"DirectCalls": {
278279
"(0, 0)": 1
@@ -330,6 +331,7 @@
330331
"LSN": 1337,
331332
"PartitionKeyRangeId": "42",
332333
"GlobalCommittedLSN": 1234,
334+
"GlobalNRegionCommittedGLSN": -1,
333335
"ItemLSN": 15,
334336
"UsingLocalLSN": true,
335337
"QuorumAckedLSN": 23,
@@ -527,6 +529,7 @@
527529
"LSN": 0,
528530
"PartitionKeyRangeId": null,
529531
"GlobalCommittedLSN": 0,
532+
"GlobalNRegionCommittedGLSN": 0,
530533
"ItemLSN": 0,
531534
"UsingLocalLSN": false,
532535
"QuorumAckedLSN": 0,

‎Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosJsonSerializerUnitTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ public void ValidatePropertySerialization()
5555
string id = "testId";
5656
this.TestProperty<AccountProperties>(
5757
id,
58-
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false,""enablePerPartitionFailoverBehavior"":null}}");
58+
$@"{{""id"":""{id}"",""writableLocations"":[],""readableLocations"":[],""userConsistencyPolicy"":null,""addresses"":null,""userReplicationPolicy"":null,""systemReplicationPolicy"":null,""readPolicy"":null,""queryEngineConfiguration"":null,""enableMultipleWriteLocations"":false,""enablePerPartitionFailoverBehavior"":null,""enableNRegionSynchronousCommit"":false}}");
5959

6060
this.TestProperty<DatabaseProperties>(
6161
id,

0 commit comments

Comments
 (0)