[Internal] ThinProxy Integration: Adds New HttpTimeoutPolicy and Fix Cross Regional Retry Logic

Microsoft.Azure.Cosmos/src/ClientRetryPolicy.cs

@@ -226,7 +226,11 @@ public void OnBeforeSendRequest(DocumentServiceRequest request)
 
             // Resolve the endpoint for the request and pin the resolution to the resolved endpoint
             // This enables marking the endpoint unavailability on endpoint failover/unreachability
-            this.locationEndpoint = this.globalEndpointManager.ResolveServiceEndpoint(request);
+            this.locationEndpoint = ConfigurationManager.IsThinClientEnabled(defaultValue: false)
+                && !LocationCache.IsMetaData(request)
+                ? this.globalEndpointManager.ResolveThinClientEndpoint(request)
+                : this.globalEndpointManager.ResolveServiceEndpoint(request);
+
             request.RequestContext.RouteToLocation(this.locationEndpoint);
         }
 

Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicy.cs

@@ -21,7 +21,8 @@ internal abstract class HttpTimeoutPolicy
 
         public static HttpTimeoutPolicy GetTimeoutPolicy(
            DocumentServiceRequest documentServiceRequest,
-           bool isPartitionLevelFailoverEnabled = false)
+           bool isPartitionLevelFailoverEnabled = false,
+           bool isThinClientEnabled = false)
         {
             //Query Plan Requests
             if (documentServiceRequest.ResourceType == ResourceType.Document
@@ -43,12 +44,22 @@ public static HttpTimeoutPolicy GetTimeoutPolicy(
                 return HttpTimeoutPolicyControlPlaneRetriableHotPath.InstanceShouldThrow503OnTimeout;
             }
 
-            //Data Plane Read
-            if (!HttpTimeoutPolicy.IsMetaData(documentServiceRequest) && documentServiceRequest.IsReadOnlyRequest)
+            //Data Plane Operations
+            if (!HttpTimeoutPolicy.IsMetaData(documentServiceRequest))
             {
-                return isPartitionLevelFailoverEnabled
-                    ? HttpTimeoutPolicyForPartitionFailover.InstanceShouldThrow503OnTimeout
-                    : HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout;
+                if (isThinClientEnabled)
+                {
+                    return documentServiceRequest.IsReadOnlyRequest
+                        ? HttpTimeoutPolicyForThinClient.InstanceShouldRetryAndThrow503OnTimeout
+                        : HttpTimeoutPolicyForThinClient.InstanceShouldNotRetryAndThrow503OnTimeout;
+                }
+                // Data Plane Reads.
+                else if (documentServiceRequest.IsReadOnlyRequest)
+                {
+                    return isPartitionLevelFailoverEnabled
+                        ? HttpTimeoutPolicyForPartitionFailover.InstanceShouldThrow503OnTimeout
+                        : HttpTimeoutPolicyDefault.InstanceShouldThrow503OnTimeout;
+                }
             }
 
             //Meta Data Read

Microsoft.Azure.Cosmos/src/HttpClient/HttpTimeoutPolicyForThinClient.cs

@@ -0,0 +1,71 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+namespace Microsoft.Azure.Cosmos
+{
+    using System;
+    using System.Collections.Generic;
+    using System.Net.Http;
+
+    internal sealed class HttpTimeoutPolicyForThinClient : HttpTimeoutPolicy
+    {
+        public bool shouldRetry;
+        public bool shouldThrow503OnTimeout;
+        private static readonly string Name = nameof(HttpTimeoutPolicyForThinClient);
+        public static readonly HttpTimeoutPolicy InstanceShouldRetryAndThrow503OnTimeout = new HttpTimeoutPolicyForThinClient(true, true);
+        public static readonly HttpTimeoutPolicy InstanceShouldNotRetryAndThrow503OnTimeout = new HttpTimeoutPolicyForThinClient(true, false);
+
+        private HttpTimeoutPolicyForThinClient(
+            bool shouldThrow503OnTimeout,
+            bool shouldRetry)
+        {
+            this.shouldThrow503OnTimeout = shouldThrow503OnTimeout;
+            this.shouldRetry = shouldRetry;
+        }
+
+        private readonly IReadOnlyList<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> TimeoutsAndDelays = new List<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)>()
+        {
+            (TimeSpan.FromSeconds(.5), TimeSpan.Zero),
+            (TimeSpan.FromSeconds(1), TimeSpan.Zero),
+            (TimeSpan.FromSeconds(1.5), TimeSpan.Zero),
+        };
+
+        public override string TimeoutPolicyName => HttpTimeoutPolicyForThinClient.Name;
+
+        public override int TotalRetryCount => this.TimeoutsAndDelays.Count;
+
+        public override IEnumerator<(TimeSpan requestTimeout, TimeSpan delayForNextRequest)> GetTimeoutEnumerator()
+        {
+            return this.TimeoutsAndDelays.GetEnumerator();
+        }
+
+        // The hot path should always be safe to retires since it should be retrieving meta data 
+        // information that is not idempotent.
+        public override bool IsSafeToRetry(HttpMethod httpMethod)
+        {
+            return this.shouldRetry;
+        }
+
+        public override bool ShouldRetryBasedOnResponse(HttpMethod requestHttpMethod, HttpResponseMessage responseMessage)
+        { 
+            if (responseMessage == null)
+            {
+                return false;
+            }
+
+            if (responseMessage.StatusCode != System.Net.HttpStatusCode.RequestTimeout)
+            {
+                return false;
+            }
+
+            if (!this.IsSafeToRetry(requestHttpMethod))
+            {
+                return false;
+            }
+
+            return true;
+        }
+
+        public override bool ShouldThrow503OnTimeout => this.shouldThrow503OnTimeout;
+    }
+}

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -100,6 +100,10 @@ public GlobalEndpointManager(
 
         public ReadOnlyCollection<Uri> WriteEndpoints => this.locationCache.WriteEndpoints;
 
+        public ReadOnlyCollection<Uri> ThinClientReadEndpoints => this.locationCache.ThinClientReadEndpoints;
+
+        public ReadOnlyCollection<Uri> ThinClientWriteEndpoints => this.locationCache.ThinClientWriteEndpoints;
+
         public int PreferredLocationCount
         {
             get

Microsoft.Azure.Cosmos/src/Routing/IGlobalEndpointManager.cs

@@ -17,6 +17,10 @@ internal interface IGlobalEndpointManager : IDisposable
 
         ReadOnlyCollection<Uri> WriteEndpoints { get; }
 
+        ReadOnlyCollection<Uri> ThinClientReadEndpoints { get; }
+
+        ReadOnlyCollection<Uri> ThinClientWriteEndpoints { get; }
+
         int PreferredLocationCount { get; }
 
         Uri ResolveServiceEndpoint(DocumentServiceRequest request);

Microsoft.Azure.Cosmos/src/Routing/LocationCache.cs

@@ -124,6 +124,16 @@ public ReadOnlyCollection<Uri> WriteEndpoints
                 return this.locationInfo.WriteEndpoints;
             }
         }
+
+        /// <summary>
+        /// Gets the list of thin client read endpoints.
+        /// </summary>
+        public ReadOnlyCollection<Uri> ThinClientReadEndpoints => this.locationInfo.ThinClientReadEndpoints;
+
+        /// <summary>
+        /// Gets the list of thin client write endpoints.
+        /// </summary>
+        public ReadOnlyCollection<Uri> ThinClientWriteEndpoints => this.locationInfo.ThinClientWriteEndpoints;
 
         public ReadOnlyCollection<string> EffectivePreferredLocations => this.locationInfo.EffectivePreferredLocations;
 
@@ -216,7 +226,7 @@ public void OnLocationPreferenceChanged(ReadOnlyCollection<string> preferredLoca
                 preferenceList: preferredLocations);
         }
 
-        public bool IsMetaData(DocumentServiceRequest request)
+        public static bool IsMetaData(DocumentServiceRequest request)
         {
             return (request.OperationType != Documents.OperationType.ExecuteJavaScript && request.ResourceType == ResourceType.StoredProcedure) ||
                 request.ResourceType != ResourceType.Document;
@@ -225,7 +235,7 @@ public bool IsMetaData(DocumentServiceRequest request)
         public bool IsMultimasterMetadataWriteRequest(DocumentServiceRequest request)
         {
             return !request.IsReadOnlyRequest && this.locationInfo.AvailableWriteLocations.Count > 1
-                && this.IsMetaData(request) 
+                && LocationCache.IsMetaData(request) 
                 && this.CanUseMultipleWriteLocations();
 
         }
@@ -896,6 +906,11 @@ internal bool CanUseMultipleWriteLocations()
 
         internal Uri ResolveThinClientEndpoint(DocumentServiceRequest request, bool isReadRequest)
         {
+            if (request.RequestContext != null && request.RequestContext.LocationEndpointToRoute != null)
+            {
+                return request.RequestContext.LocationEndpointToRoute;
+            }
+
             DatabaseAccountLocationsInfo snapshot = this.locationInfo;
             ReadOnlyCollection<Uri> endpoints = isReadRequest
                 ? snapshot.ThinClientReadEndpoints

Microsoft.Azure.Cosmos/src/ThinClientStoreClient.cs

@@ -151,7 +151,7 @@ private Task<HttpResponseMessage> InvokeClientAsync(
             return base.httpClient.SendHttpAsync(
                 () => this.PrepareRequestForProxyAsync(request, physicalAddress, thinClientEndpoint, globalDatabaseAccountName, clientCollectionCache),
                 resourceType,
-                HttpTimeoutPolicy.GetTimeoutPolicy(request),
+                HttpTimeoutPolicy.GetTimeoutPolicy(request, isThinClientEnabled: true),
                 request.RequestContext.ClientRequestStatistics,
                 cancellationToken,
                 request);

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/PartitionKeyRangeFailoverTests/MockSetupsHelper.cs

@@ -36,7 +36,7 @@ public static void SetupStrongAccountProperties(
                 accountName,
                 writeRegions,
                 readRegions,
-                shouldEnablePPAF);
+                shouldEnablePPAF: shouldEnablePPAF);
 
             Uri endpointUri = new Uri(endpoint);
             mockHttpClientHandler.Setup(x => x.SendAsync(
@@ -109,6 +109,7 @@ public static HttpResponseMessage CreateStrongAccount(
             string accountName,
             IList<AccountRegion> writeRegions,
             IList<AccountRegion> readRegions,
+            bool shouldEnableThinClient = false,
             bool? shouldEnablePPAF = null)
         {
             AccountProperties accountProperties = new AccountProperties()
@@ -137,9 +138,29 @@ public static HttpResponseMessage CreateStrongAccount(
                     AsyncReplication = false,
                     MinReplicaSetSize = 3,
                     MaxReplicaSetSize = 4
-                }
+                },
             };
 
+            if (shouldEnableThinClient)
+            {
+                accountProperties.AdditionalProperties = new Dictionary<string, JToken>
+                {
+                    {
+                        "thinClientWritableLocations",
+                        JArray.Parse(@"[
+                            { 'name': 'East US', 'databaseAccountEndpoint': 'https://thinclientwrite-eastus.documents.azure.com:10650/' }
+                        ]")
+                    },
+                    {
+                        "thinClientReadableLocations",
+                        JArray.Parse(@"[
+                            { 'name': 'East US', 'databaseAccountEndpoint': 'https://thinclientread-eastus.documents.azure.com:10650/' },
+                            { 'name': 'West US', 'databaseAccountEndpoint': 'https://thinclientread-westus.documents.azure.com:10650/' }
+                        ]")
+                    }
+                };
+            }
+
             return new HttpResponseMessage()
             {
                 StatusCode = HttpStatusCode.OK,