Code changes to introduce a new aggressive timeout http policy for PPAF.

Author: kundadebdatta

Microsoft.Azure.Cosmos/src/GatewayStoreClient.cs

@@ -22,6 +22,7 @@ namespace Microsoft.Azure.Cosmos
 
     internal class GatewayStoreClient : TransportClient
     {
+        private readonly bool isPartitionLevelFailoverEnabled;
         private readonly ICommunicationEventSource eventSource;
         protected readonly CosmosHttpClient httpClient;
         protected readonly JsonSerializerSettings SerializerSettings;
@@ -31,11 +32,13 @@ internal class GatewayStoreClient : TransportClient
         public GatewayStoreClient(
             CosmosHttpClient httpClient,
             ICommunicationEventSource eventSource,
-            JsonSerializerSettings serializerSettings = null)
+            JsonSerializerSettings serializerSettings = null,
+            bool isPartitionLevelFailoverEnabled = false)
         {
             this.httpClient = httpClient;
             this.SerializerSettings = serializerSettings;
             this.eventSource = eventSource;
+            this.isPartitionLevelFailoverEnabled = isPartitionLevelFailoverEnabled;
         }
 
         public async Task<DocumentServiceResponse> InvokeAsync(
@@ -375,7 +378,9 @@ private Task<HttpResponseMessage> InvokeClientAsync(
             return this.httpClient.SendHttpAsync(
                 () => this.PrepareRequestMessageAsync(request, physicalAddress),
                 resourceType,
-                HttpTimeoutPolicy.GetTimeoutPolicy(request),
+                HttpTimeoutPolicy.GetTimeoutPolicy(
+                    request,
+                    this.isPartitionLevelFailoverEnabled),
                 request.RequestContext.ClientRequestStatistics,
                 cancellationToken,
                 request);

Microsoft.Azure.Cosmos/src/GatewayStoreModel.cs

@@ -58,7 +58,8 @@ public GatewayStoreModel(
             this.gatewayStoreClient = new GatewayStoreClient(
                 httpClient,
                 this.eventSource,
-                serializerSettings);
+                serializerSettings,
+                isPartitionLevelFailoverEnabled);
 
             this.globalPartitionEndpointManager.SetBackgroundConnectionPeriodicRefreshTask(
                 this.MarkEndpointsToHealthyAsync);

Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicy.cs

@@ -20,7 +20,8 @@ internal abstract class HttpTimeoutPolicy
         public virtual bool ShouldThrow503OnTimeout => false;
 
         public static HttpTimeoutPolicy GetTimeoutPolicy(
-           DocumentServiceRequest documentServiceRequest)
+           DocumentServiceRequest documentServiceRequest,
+           bool isPartitionLevelFailoverEnabled = false)
         {
             //Query Plan Requests
             if (documentServiceRequest.ResourceType == ResourceType.Document
@@ -45,7 +46,9 @@ public static HttpTimeoutPolicy GetTimeoutPolicy(
             //Data Plane Read
             if (!HttpTimeoutPolicy.IsMetaData(documentServiceRequest) && documentServiceRequest.IsReadOnlyRequest)
             {
-                return HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout;
+                return isPartitionLevelFailoverEnabled
+                    ? HttpTimeoutPolicyForPartitionFailover.InstanceShouldThrow503OnTimeout
+                    : HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout;
             }
 
             //Meta Data Read

Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyForPartitionFailover.cs

@@ -0,0 +1,52 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+namespace Microsoft.Azure.Cosmos
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Net.Http;
+
+    internal sealed class HttpTimeoutPolicyForPartitionFailover : HttpTimeoutPolicy
+    {
+        public static readonly HttpTimeoutPolicy Instance = new HttpTimeoutPolicyForPartitionFailover(false);
+        public static readonly HttpTimeoutPolicy InstanceShouldThrow503OnTimeout = new HttpTimeoutPolicyForPartitionFailover(true);
+        public bool shouldThrow503OnTimeout;
+        private static readonly string Name = nameof(HttpTimeoutPolicyDefault);
+
+        private HttpTimeoutPolicyForPartitionFailover(bool shouldThrow503OnTimeout)
+        {
+            this.shouldThrow503OnTimeout = shouldThrow503OnTimeout;
+        }
+
+        private readonly IReadOnlyList<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> TimeoutsAndDelays = new List<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)>()
+        {
+            (TimeSpan.FromSeconds(.5), TimeSpan.Zero),
+            (TimeSpan.FromSeconds(.5), TimeSpan.Zero),
+            (TimeSpan.FromSeconds(1), TimeSpan.Zero),
+        };
+
+        public override string TimeoutPolicyName => HttpTimeoutPolicyForPartitionFailover.Name;
+
+        public override int TotalRetryCount => this.TimeoutsAndDelays.Count;
+
+        public override IEnumerator<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> GetTimeoutEnumerator()
+        {
+            return this.TimeoutsAndDelays.GetEnumerator();
+        }
+
+        // Assume that it is not safe to retry unless it is a get method.
+        // Create and other operations could have succeeded even though a timeout occurred.
+        public override bool IsSafeToRetry(HttpMethod httpMethod)
+        {
+            return httpMethod == HttpMethod.Get;
+        }
+
+        public override bool ShouldRetryBasedOnResponse(HttpMethod requestHttpMethod, HttpResponseMessage responseMessage)
+        {
+            return false;
+        }
+
+        public override bool ShouldThrow503OnTimeout => this.shouldThrow503OnTimeout;
+    }
+}