Merge branch 'master' into users/nalutripician/issue-5095-contacted-regions-hub-fallback

Author: NaluTripician

Microsoft.Azure.Cosmos.Samples/Usage/HttpClientFactory/Program.cs

@@ -193,7 +193,10 @@ private static void ShareHttpHandlers(
             {
                 PooledConnectionLifetime = TimeSpan.FromMinutes(10), // Customize this value based on desired DNS refresh timer
                 MaxConnectionsPerServer = 20, // Customize the maximum number of allowed connections
-                EnableMultipleHttp2Connections = true // Recommended for thin client (HTTP/2) mode to open additional connections when stream limits are reached
+                EnableMultipleHttp2Connections = true, // Recommended for thin client (HTTP/2) mode to open additional connections when stream limits are reached
+                KeepAlivePingDelay = TimeSpan.FromSeconds(1), // Send HTTP/2 PING after 1s of inactivity to detect broken connections
+                KeepAlivePingTimeout = TimeSpan.FromSeconds(2), // Mark connection dead if no PONG within 2s
+                KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always 
             };
 
             CosmosClientOptions cosmosClientOptions = new CosmosClientOptions()

Microsoft.Azure.Cosmos/src/ChangeFeedProcessor/LeaseManagement/DocumentServiceLeaseManagerCosmos.cs

@@ -31,6 +31,8 @@ internal sealed class DocumentServiceLeaseManagerCosmos : DocumentServiceLeaseMa
         private readonly AsyncLazy<TryCatch<string>> lazyContainerRid;
         private PartitionKeyRangeCache partitionKeyRangeCache;
 
+        internal static bool IsChangeFeedLeaseIdAsPartitionKeyEnabled = ConfigurationManager.IsChangeFeedLeaseIdAsPartitionKeyEnabled();
+
         public DocumentServiceLeaseManagerCosmos(
             ContainerInternal monitoredContainer,
             ContainerInternal leaseContainer,
@@ -128,7 +130,9 @@ public override Task<DocumentServiceLease> CreateLeaseIfNotExistAsync(
                 Mode = this.GetChangeFeedMode()
             };
 
-            this.requestOptionsFactory.AddPartitionKeyIfNeeded((string pk) => documentServiceLease.LeasePartitionKey = pk, Guid.NewGuid().ToString());
+            this.requestOptionsFactory.AddPartitionKeyIfNeeded(
+                (string pk) => documentServiceLease.LeasePartitionKey = pk,
+                DocumentServiceLeaseManagerCosmos.GetLeasePartitionKeyValue(documentServiceLease.LeaseId));
 
             return this.TryCreateDocumentServiceLeaseAsync(documentServiceLease);
         }
@@ -153,11 +157,20 @@ public override Task<DocumentServiceLease> CreateLeaseIfNotExistAsync(
                 Mode = this.GetChangeFeedMode()
             };
 
-            this.requestOptionsFactory.AddPartitionKeyIfNeeded((string pk) => documentServiceLease.LeasePartitionKey = pk, Guid.NewGuid().ToString());
+            this.requestOptionsFactory.AddPartitionKeyIfNeeded(
+                (string pk) => documentServiceLease.LeasePartitionKey = pk,
+                DocumentServiceLeaseManagerCosmos.GetLeasePartitionKeyValue(documentServiceLease.LeaseId));
 
             return this.TryCreateDocumentServiceLeaseAsync(documentServiceLease);
         }
 
+        private static string GetLeasePartitionKeyValue(string leaseId)
+        {
+            return DocumentServiceLeaseManagerCosmos.IsChangeFeedLeaseIdAsPartitionKeyEnabled
+                ? leaseId
+                : Guid.NewGuid().ToString();
+        }
+
         private string GetChangeFeedMode()
         {
             return this.options.Mode == ChangeFeedMode.AllVersionsAndDeletes

Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs

@@ -244,11 +244,19 @@ public string ApplicationName
         /// </summary>
         /// <remarks>
         /// This setting is only applicable in Gateway mode.
-        /// The SDK sets EnableMultipleHttp2Connections = true on the underlying SocketsHttpHandler,
-        /// allowing additional HTTP/2 TCP connections to be opened when the maximum concurrent streams
-        /// limit on an existing connection is reached. This property controls the upper bound on the
-        /// total number of connections per server endpoint.
-        /// When using a custom <see cref="HttpClientFactory"/>, set EnableMultipleHttp2Connections
+        /// The SDK sets the following on the underlying SocketsHttpHandler:
+        /// <list type="bullet">
+        /// <item><description>EnableMultipleHttp2Connections = true — allows additional HTTP/2 TCP connections
+        /// to be opened when the maximum concurrent streams limit on an existing connection is reached.</description></item>
+        /// <item><description>KeepAlivePingDelay = 1 second — sends HTTP/2 PING frames after 1 second
+        /// of inactivity to detect broken connections in the pool.</description></item>
+        /// <item><description>KeepAlivePingTimeout = 2 seconds — marks a connection as dead if no PONG
+        /// response is received within 2 seconds.</description></item>
+        /// <item><description>KeepAlivePingPolicy = Always — sends pings even for idle connections, which
+        /// is critical for detecting broken connections that remain in the pool.</description></item>
+        /// </list>
+        /// This property controls the upper bound on the total number of connections per server endpoint.
+        /// When using a custom <see cref="HttpClientFactory"/>, configure these properties
         /// directly on your SocketsHttpHandler for equivalent behavior.
         /// </remarks>
         /// <example>
@@ -268,7 +276,10 @@ public string ApplicationName
         /// SocketsHttpHandler handler = new SocketsHttpHandler
         /// {
         ///     MaxConnectionsPerServer = 100,
-        ///     EnableMultipleHttp2Connections = true
+        ///     EnableMultipleHttp2Connections = true,
+        ///     KeepAlivePingDelay = TimeSpan.FromSeconds(1),
+        ///     KeepAlivePingTimeout = TimeSpan.FromSeconds(2),
+        ///     KeepAlivePingPolicy = HttpKeepAlivePingPolicy.Always
         /// };
         /// CosmosClientOptions options = new CosmosClientOptions()
         /// {
@@ -306,6 +317,24 @@ public int GatewayModeMaxConnectionLimit
         /// <seealso cref="CosmosClientBuilder.WithRequestTimeout(TimeSpan)"/>
         public TimeSpan RequestTimeout { get; set; }
 
+        /// <summary>
+        /// Gets or sets the request timeout for inference service operations (e.g., semantic reranking).
+        /// The number specifies the time to wait for a response from the inference service before the request is cancelled.
+        /// This is a single-attempt timeout with no retries.
+        /// </summary>
+        /// <value>Default value is 5 seconds.</value>
+        /// <remarks>
+        /// This timeout is specific to inference service operations and is separate from the standard <see cref="RequestTimeout"/>.
+        /// If the request does not complete within the specified duration, a <see cref="CosmosException"/> with status 408 (Request Timeout) is thrown.
+        /// No retries are attempted on timeout.
+        /// </remarks>
+#if PREVIEW
+        public
+#else
+        internal
+#endif
+        TimeSpan InferenceRequestTimeout { get; set; } = InferenceService.DefaultInferenceRequestTimeout;
+
         /// <summary>
         /// The SDK does a background refresh based on the time interval set to refresh the token credentials.
         /// This avoids latency issues because the old token is used until the new token is retrieved.

Microsoft.Azure.Cosmos/src/DocumentClientEventSource.cs

@@ -34,6 +34,51 @@ public class Keywords
             public const EventKeywords HttpRequestAndResponse = (EventKeywords)1;
         }
 
+        // Ordered list of headers extracted and forwarded positionally to Event(1).
+        // IMPORTANT: The order of entries here matches the field order declared by [Event(1)]
+        // and the `{index}` placeholders in its Message template. Do not reorder, insert, or
+        // remove entries without updating Event(1) and the redaction logic in Request(...).
+        // The Authorization slot is populated from the raw header and then overwritten with
+        // "REDACTED" in the [NonEvent] wrapper before being emitted to ETW.
+        private static readonly string[] RequestHeaderKeysToExtract =
+        {
+            HttpConstants.HttpHeaders.Accept,
+            HttpConstants.HttpHeaders.Authorization, // SECURITY: redacted to "REDACTED" in Request(...) before ETW emission. Do not remove.
+            HttpConstants.HttpHeaders.ConsistencyLevel,
+            HttpConstants.HttpHeaders.ContentType,
+            HttpConstants.HttpHeaders.ContentEncoding,
+            HttpConstants.HttpHeaders.ContentLength,
+            HttpConstants.HttpHeaders.ContentLocation,
+            HttpConstants.HttpHeaders.Continuation,
+            HttpConstants.HttpHeaders.EmitVerboseTracesInQuery,
+            HttpConstants.HttpHeaders.EnableScanInQuery,
+            HttpConstants.HttpHeaders.ETag,
+            HttpConstants.HttpHeaders.HttpDate,
+            HttpConstants.HttpHeaders.IfMatch,
+            HttpConstants.HttpHeaders.IfNoneMatch,
+            HttpConstants.HttpHeaders.IndexingDirective,
+            HttpConstants.HttpHeaders.KeepAlive,
+            HttpConstants.HttpHeaders.OfferType,
+            HttpConstants.HttpHeaders.PageSize,
+            HttpConstants.HttpHeaders.PreTriggerExclude,
+            HttpConstants.HttpHeaders.PreTriggerInclude,
+            HttpConstants.HttpHeaders.PostTriggerExclude,
+            HttpConstants.HttpHeaders.PostTriggerInclude,
+            HttpConstants.HttpHeaders.ProfileRequest,
+            HttpConstants.HttpHeaders.ResourceTokenExpiry,
+            HttpConstants.HttpHeaders.SessionToken,
+            HttpConstants.HttpHeaders.SetCookie,
+            HttpConstants.HttpHeaders.Slug,
+            HttpConstants.HttpHeaders.UserAgent,
+            HttpConstants.HttpHeaders.XDate
+        };
+
+        // Index of the Authorization header in RequestHeaderKeysToExtract, computed once at class
+        // initialization so the per-request redaction path in Request(...) does not pay an O(n)
+        // Array.IndexOf lookup on every HTTP call.
+        private static readonly int AuthorizationHeaderIndex
+            = Array.IndexOf(RequestHeaderKeysToExtract, HttpConstants.HttpHeaders.Authorization);
+
         [NonEvent]
         private unsafe void WriteEventCoreWithActivityId(Guid activityId, int eventId, int eventDataCount, EventSource.EventData* dataDesc)
         {
@@ -270,45 +315,48 @@ private unsafe void Request(
             }
         }
 
+        // SECURITY: The Authorization header contains a live credential (master-key HMAC
+        // signature, resource token, or AAD Bearer access token). It must never be written
+        // to the ETW event payload where any listener subscribing to the "DocumentDBClient"
+        // EventSource at Verbose level (e.g., Geneva MonitoringAgent, PerfView, dotnet-trace)
+        // would capture it. Replace the value with a fixed placeholder while preserving the
+        // 33-field ETW manifest so existing consumers remain compatible. The IsNullOrEmpty
+        // guard preserves the existing "" semantics for requests that never had an
+        // Authorization header attached (e.g. internal plumbing); we only overwrite when a
+        // real value is present. Factored out as an internal helper so the redaction logic
+        // can be unit-tested directly without relying on ETW listener wiring (which is
+        // inherently racy under parallel test execution).
+        internal static void RedactSensitiveHeaderValues(string[] headerValues)
+        {
+            if (headerValues == null)
+            {
+                return;
+            }
+
+            // AuthorizationHeaderIndex is computed once at class initialization (see field
+            // above); the Debug.Assert catches any future refactor that removes Authorization
+            // from RequestHeaderKeysToExtract.
+            System.Diagnostics.Debug.Assert(
+                AuthorizationHeaderIndex >= 0,
+                "Authorization must be present in RequestHeaderKeysToExtract so the redaction below takes effect.");
+
+            if (AuthorizationHeaderIndex >= 0
+                && headerValues.Length > AuthorizationHeaderIndex
+                && !string.IsNullOrEmpty(headerValues[AuthorizationHeaderIndex]))
+            {
+                headerValues[AuthorizationHeaderIndex] = "REDACTED";
+            }
+        }
+
         [NonEvent]
         public void Request(Guid activityId, Guid localId, string uri, string resourceType, HttpRequestHeaders requestHeaders)
         {
             if (this.IsEnabled(EventLevel.Verbose, Keywords.HttpRequestAndResponse))
             {
-                string[] keysToExtract =
-                {
-                    HttpConstants.HttpHeaders.Accept,
-                    HttpConstants.HttpHeaders.Authorization,
-                    HttpConstants.HttpHeaders.ConsistencyLevel,
-                    HttpConstants.HttpHeaders.ContentType,
-                    HttpConstants.HttpHeaders.ContentEncoding,
-                    HttpConstants.HttpHeaders.ContentLength,
-                    HttpConstants.HttpHeaders.ContentLocation,
-                    HttpConstants.HttpHeaders.Continuation,
-                    HttpConstants.HttpHeaders.EmitVerboseTracesInQuery,
-                    HttpConstants.HttpHeaders.EnableScanInQuery,
-                    HttpConstants.HttpHeaders.ETag,
-                    HttpConstants.HttpHeaders.HttpDate,
-                    HttpConstants.HttpHeaders.IfMatch,
-                    HttpConstants.HttpHeaders.IfNoneMatch,
-                    HttpConstants.HttpHeaders.IndexingDirective,
-                    HttpConstants.HttpHeaders.KeepAlive,
-                    HttpConstants.HttpHeaders.OfferType,
-                    HttpConstants.HttpHeaders.PageSize,
-                    HttpConstants.HttpHeaders.PreTriggerExclude,
-                    HttpConstants.HttpHeaders.PreTriggerInclude,
-                    HttpConstants.HttpHeaders.PostTriggerExclude,
-                    HttpConstants.HttpHeaders.PostTriggerInclude,
-                    HttpConstants.HttpHeaders.ProfileRequest,
-                    HttpConstants.HttpHeaders.ResourceTokenExpiry,
-                    HttpConstants.HttpHeaders.SessionToken,
-                    HttpConstants.HttpHeaders.SetCookie,
-                    HttpConstants.HttpHeaders.Slug,
-                    HttpConstants.HttpHeaders.UserAgent,
-                    HttpConstants.HttpHeaders.XDate
-                };
+                string[] headerValues = Helpers.ExtractValuesFromHTTPHeaders(requestHeaders, RequestHeaderKeysToExtract);
+
+                RedactSensitiveHeaderValues(headerValues);
 
-                string[] headerValues = Helpers.ExtractValuesFromHTTPHeaders(requestHeaders, keysToExtract);
                 this.Request(activityId, localId, uri, resourceType, headerValues[0], headerValues[1], headerValues[2], headerValues[3], headerValues[4],
                     headerValues[5], headerValues[6], headerValues[7], headerValues[8], headerValues[9], headerValues[10], headerValues[11], headerValues[12],
                     headerValues[13], headerValues[14], headerValues[15], headerValues[16], headerValues[17], headerValues[18], headerValues[19], headerValues[20],

Microsoft.Azure.Cosmos/src/Fluent/CosmosClientBuilder.cs

@@ -387,6 +387,26 @@ public CosmosClientBuilder WithRequestTimeout(TimeSpan requestTimeout)
             return this;
         }
 
+        /// <summary>
+        /// Sets the request timeout for inference service operations (e.g., semantic reranking).
+        /// This is a single-attempt timeout with no retries; if the request does not complete
+        /// within the specified duration, a <see cref="CosmosException"/> with status 408 (Request Timeout) is thrown.
+        /// </summary>
+        /// <param name="inferenceRequestTimeout">A time to use as timeout for inference operations.</param>
+        /// <value>Default value is 5 seconds.</value>
+        /// <returns>The current <see cref="CosmosClientBuilder"/>.</returns>
+        /// <seealso cref="CosmosClientOptions.InferenceRequestTimeout"/>
+#if PREVIEW
+        public
+#else
+        internal
+#endif
+        CosmosClientBuilder WithInferenceRequestTimeout(TimeSpan inferenceRequestTimeout)
+        {
+            this.clientOptions.InferenceRequestTimeout = inferenceRequestTimeout;
+            return this;
+        }
+
         /// <summary>
         /// Sets the connection mode to Direct. This is used by the client when connecting to the Azure Cosmos DB service.
         /// </summary>

Microsoft.Azure.Cosmos/src/HttpClient/CosmosHttpClientCore.cs

@@ -183,6 +183,42 @@ public static HttpMessageHandler CreateSocketsHttpHandlerHelper(
                 DefaultTrace.TraceWarning("Failed to set EnableMultipleHttp2Connections on SocketsHttpHandler: {0}", ex.Message);
             }
 
+            // Enable HTTP/2 PING keep-alive to detect broken connections.
+            // Without this, a broken HTTP/2 connection (e.g. after a network blip or load balancer
+            // reset) can remain in the pool indefinitely, causing persistent request failures
+            // that only resolve after application restart.
+            // KeepAlivePingDelay/Timeout/Policy are available on SocketsHttpHandler in .NET 5.0+.
+            try
+            {
+                int pingDelayInSeconds = ConfigurationManager.GetEnvironmentVariable<int>(
+                    ConfigurationManager.Http2KeepAlivePingDelayInSeconds,
+                    defaultValue: 1);
+
+                int pingTimeoutInSeconds = ConfigurationManager.GetEnvironmentVariable<int>(
+                    ConfigurationManager.Http2KeepAlivePingTimeoutInSeconds,
+                    defaultValue: 2);
+
+                PropertyInfo keepAlivePingDelayInfo = socketHandlerType.GetProperty("KeepAlivePingDelay");
+                keepAlivePingDelayInfo?.SetValue(socketHttpHandler, TimeSpan.FromSeconds(pingDelayInSeconds));
+
+                PropertyInfo keepAlivePingTimeoutInfo = socketHandlerType.GetProperty("KeepAlivePingTimeout");
+                keepAlivePingTimeoutInfo?.SetValue(socketHttpHandler, TimeSpan.FromSeconds(pingTimeoutInSeconds));
+
+                // HttpKeepAlivePingPolicy.Always = 1: send pings even for idle connections,
+                // which is critical for detecting broken connections lingering in the pool.
+                PropertyInfo keepAlivePingPolicyInfo = socketHandlerType.GetProperty("KeepAlivePingPolicy");
+                if (keepAlivePingPolicyInfo != null)
+                {
+                    Type pingPolicyType = keepAlivePingPolicyInfo.PropertyType;
+                    object alwaysValue = Enum.ToObject(pingPolicyType, 1);
+                    keepAlivePingPolicyInfo.SetValue(socketHttpHandler, alwaysValue);
+                }
+            }
+            catch (Exception ex)
+            {
+                DefaultTrace.TraceWarning("Failed to configure HTTP/2 keep-alive ping on SocketsHttpHandler: {0}", ex.Message);
+            }
+
             if (serverCertificateCustomValidationCallback != null)
             {
                 //Get SslOptions Property