[Per Partition Automatic Failover] Remove Environment Variable to Set PPAF at the SDK Layer and Add Support for Internal Client Options

Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs

@@ -338,6 +338,16 @@ public bool EnablePartitionLevelCircuitBreaker
             set;
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether to disable Per Partition Automatic Failover (PPAF) explicitly.
+        /// When set to true, this will be used to disable PPAF irrespective of the account settings.
+        /// </summary>
+        public bool DisablePartitionLevelFailover
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Gets or sets the certificate validation callback.
         /// </summary>

Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs

@@ -766,12 +766,19 @@ bool EnableRemoteRegionPreferredForSessionRetry
         }
 
         /// <summary>
-        /// Enable partition level circuit breaker (aka PPCB). For compute gateway use case, by default per partition automatic failover will be disabled, so does the PPCB.
-        /// If compute gateway chooses to enable PPAF, then the .NET SDK will enable PPCB by default, which will improve the read availability and latency. This would mean
+        /// Enable partition level circuit breaker (aka PPCB). For compute gateway use case, by default per partition automatic failover will be disabled, so does the PPCB.
+        /// If compute gateway chooses to enable PPAF, then the .NET SDK will enable PPCB by default, which will improve the read availability and latency. This would mean
         /// when PPAF is enabled, the SDK will automatically enable PPCB as well.
         /// </summary>
         internal bool EnablePartitionLevelCircuitBreaker { get; set; } = ConfigurationManager.IsPartitionLevelCircuitBreakerEnabled(defaultValue: false);
 
+        /// <summary>
+        /// Internal option to disable Per Partition Automatic Failover (PPAF) explicitly.
+        /// When set to true, this will be used to disable PPAF irrespective of the account settings.
+        /// The default value for this parameter is 'false'.
+        /// </summary>
+        internal bool DisablePartitionLevelFailover { get; set; } = false;
+
         /// <summary>
         /// Quorum Read allowed with eventual consistency account or consistent prefix account.
         /// </summary>
@@ -1030,6 +1037,7 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId)
                 MaxTcpConnectionsPerEndpoint = this.MaxTcpConnectionsPerEndpoint,
                 EnableEndpointDiscovery = !this.LimitToEndpoint,
                 EnablePartitionLevelCircuitBreaker = this.EnablePartitionLevelCircuitBreaker,
+                DisablePartitionLevelFailover = this.DisablePartitionLevelFailover,
                 PortReuseMode = this.portReuseMode,
                 EnableTcpConnectionEndpointRediscovery = this.EnableTcpConnectionEndpointRediscovery,
                 EnableAdvancedReplicaSelectionForTcp = this.EnableAdvancedReplicaSelectionForTcp,

Microsoft.Azure.Cosmos/src/DocumentClient.cs

@@ -1056,13 +1056,11 @@ private async Task<bool> GetInitializationTaskAsync(IStoreClientFactory storeCli
                 this.EnsureValidOverwrite(this.desiredConsistencyLevel.Value);
             }
 
-            bool isPPafEnabled = ConfigurationManager.IsPartitionLevelFailoverEnabled(defaultValue: false);
-            if (this.accountServiceConfiguration != null && this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.HasValue)
-            {
-                isPPafEnabled = this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.Value;
-            }
-
-            this.ConnectionPolicy.EnablePartitionLevelFailover = isPPafEnabled;
+            // Apply the DisablePartitionLevelFailover setting to override PPAF if explicitly disabled
+            this.ConnectionPolicy.EnablePartitionLevelFailover = !this.ConnectionPolicy.DisablePartitionLevelFailover &&
+                this.accountServiceConfiguration != null &&
+                this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.HasValue &&
+                this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.Value;
             this.ConnectionPolicy.EnablePartitionLevelCircuitBreaker |= this.ConnectionPolicy.EnablePartitionLevelFailover;
             this.ConnectionPolicy.UserAgentContainer.AppendFeatures(this.GetUserAgentFeatures());
             this.InitializePartitionLevelFailoverWithDefaultHedging();

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -602,13 +602,10 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account
                 return;
             }
 
-            bool isPPafEnabled = ConfigurationManager.IsPartitionLevelFailoverEnabled(defaultValue: false);
-            if (databaseAccount.EnablePartitionLevelFailover.HasValue)
-            {
-                isPPafEnabled = databaseAccount.EnablePartitionLevelFailover.Value;
+            if (!this.connectionPolicy.DisablePartitionLevelFailover && databaseAccount.EnablePartitionLevelFailover.HasValue)
+            {
+                this.connectionPolicy.EnablePartitionLevelFailover = databaseAccount.EnablePartitionLevelFailover.Value;
             }
-
-            this.connectionPolicy.EnablePartitionLevelFailover = isPPafEnabled;
             GlobalEndpointManager.ParseThinClientLocationsFromAdditionalProperties(databaseAccount);
 
             this.locationCache.OnDatabaseAccountRead(databaseAccount);

Microsoft.Azure.Cosmos/src/Routing/GlobalPartitionEndpointManagerCore.cs

@@ -608,7 +608,7 @@ public PartitionKeyRangeFailoverInfo(
                 this.ConsecutiveWriteRequestFailureCount = 0;
                 this.ReadRequestFailureCounterThreshold = ConfigurationManager.GetCircuitBreakerConsecutiveFailureCountForReads(10);
                 this.WriteRequestFailureCounterThreshold = ConfigurationManager.GetCircuitBreakerConsecutiveFailureCountForWrites(5);
-                this.TimeoutCounterResetWindowInMinutes = TimeSpan.FromMinutes(1);
+                this.TimeoutCounterResetWindowInMinutes = TimeSpan.FromMinutes(ConfigurationManager.GetCircuitBreakerTimeoutCounterResetWindowInMinutes(5));
                 this.FirstRequestFailureTime = DateTime.UtcNow;
                 this.LastRequestFailureTime = DateTime.UtcNow;
             }

Microsoft.Azure.Cosmos/src/Util/ConfigurationManager.cs

@@ -16,11 +16,10 @@ internal static class ConfigurationManager
         internal static readonly string ReplicaConnectivityValidationEnabled = "AZURE_COSMOS_REPLICA_VALIDATION_ENABLED";
 
         /// <summary>
-        /// A read-only string containing the environment variable name for enabling per partition automatic failover.
-        /// This will eventually be removed once per partition automatic failover is enabled by default for both preview
-        /// and GA.
+        /// A read-only string containing the environment variable name for capturing the PPCB timeout counter reset window time
+        /// in minutes. The default value for this window is 5 minutes.
         /// </summary>
-        internal static readonly string PartitionLevelFailoverEnabled = "AZURE_COSMOS_PARTITION_LEVEL_FAILOVER_ENABLED";
+        internal static readonly string CircuitBreakerTimeoutCounterResetWindowInMinutes = "AZURE_COSMOS_PPCB_TIMEOUT_COUNTER_RESET_WINDOW_IN_MINUTES";
 
         /// <summary>
         /// A read-only string containing the environment variable name for enabling per partition circuit breaker. The default value
@@ -160,19 +159,19 @@ public static bool IsReplicaAddressValidationEnabled(
         }
 
         /// <summary>
-        /// Gets the boolean value of the partition level failover environment variable. Note that, partition level failover
-        /// is disabled by default for both preview and GA releases. The user can set the  respective environment variable
-        /// 'AZURE_COSMOS_PARTITION_LEVEL_FAILOVER_ENABLED' to override the value for both preview and GA. The method will
-        /// eventually be removed, once partition level failover is enabled by default for  both preview and GA.
+        /// Gets the PPCB timeout counter reset window in minutes.
+        /// The default value for this window is 5 minutes. The user can set the respective
+        /// environment variable 'AZURE_COSMOS_PPCB_TIMEOUT_COUNTER_RESET_WINDOW_IN_MINUTES'
+        /// to override the value.
         /// </summary>
-        /// <param name="defaultValue">A boolean field containing the default value for partition level failover.</param>
-        /// <returns>A boolean flag indicating if partition level failover is enabled.</returns>
-        public static bool IsPartitionLevelFailoverEnabled(
-            bool defaultValue)
+        /// <param name="defaultValue">An integer containing the default value for the timeout counter reset window in minutes.</param>
+        /// <returns>An integer representing the timeout counter reset window in minutes.</returns>
+        public static int GetCircuitBreakerTimeoutCounterResetWindowInMinutes(
+            int defaultValue)
         {
             return ConfigurationManager
                     .GetEnvironmentVariable(
-                        variable: ConfigurationManager.PartitionLevelFailoverEnabled,
+                        variable: ConfigurationManager.CircuitBreakerTimeoutCounterResetWindowInMinutes,
                         defaultValue: defaultValue);
         }
 

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ConfigurationManagerTests.cs

@@ -0,0 +1,49 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.Tests
+{
+    using System;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class ConfigurationManagerTests
+    {
+        [TestMethod]
+        public void GetCircuitBreakerTimeoutCounterResetWindowInMinutes_DefaultValue()
+        {
+            // Test that the default value is returned when environment variable is not set
+            int result = ConfigurationManager.GetCircuitBreakerTimeoutCounterResetWindowInMinutes(5);
+            Assert.AreEqual(5, result);
+        }
+
+        [TestMethod]
+        public void GetCircuitBreakerTimeoutCounterResetWindowInMinutes_CustomDefaultValue()
+        {
+            // Test that custom default values are respected
+            int result = ConfigurationManager.GetCircuitBreakerTimeoutCounterResetWindowInMinutes(10);
+            Assert.AreEqual(10, result);
+        }
+
+        [TestMethod]
+        public void GetCircuitBreakerTimeoutCounterResetWindowInMinutes_EnvironmentVariableOverride()
+        {
+            // Test that environment variable overrides the default value
+            const string envVarName = "AZURE_COSMOS_PPCB_TIMEOUT_COUNTER_RESET_WINDOW_IN_MINUTES";
+            const string testValue = "15";
+
+            try
+            {
+                Environment.SetEnvironmentVariable(envVarName, testValue);
+                int result = ConfigurationManager.GetCircuitBreakerTimeoutCounterResetWindowInMinutes(5);
+                Assert.AreEqual(15, result);
+            }
+            finally
+            {
+                // Clean up environment variable
+                Environment.SetEnvironmentVariable(envVarName, null);
+            }
+        }
+    }
+}