[Per Partition Automatic Failover] Remove Environment Variable to Set PPAF at the SDK Layer and Add Support for Internal Client Options

Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs

@@ -338,6 +338,16 @@ public bool EnablePartitionLevelCircuitBreaker
             set;
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether to disable Per Partition Automatic Failover (PPAF) explicitly.
+        /// When set to true, this will be used to disable PPAF irrespective of the account settings.
+        /// </summary>
+        public bool DisablePartitionLevelFailover
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Gets or sets the certificate validation callback.
         /// </summary>

Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs

@@ -766,12 +766,19 @@ bool EnableRemoteRegionPreferredForSessionRetry
         }
 
         /// <summary>
-        /// Enable partition level circuit breaker (aka PPCB). For compute gateway use case, by default per partition automatic failover will be disabled, so does the PPCB.
-        /// If compute gateway chooses to enable PPAF, then the .NET SDK will enable PPCB by default, which will improve the read availability and latency. This would mean
+        /// Enable partition level circuit breaker (aka PPCB). For compute gateway use case, by default per partition automatic failover will be disabled, so does the PPCB.
+        /// If compute gateway chooses to enable PPAF, then the .NET SDK will enable PPCB by default, which will improve the read availability and latency. This would mean
         /// when PPAF is enabled, the SDK will automatically enable PPCB as well.
         /// </summary>
         internal bool EnablePartitionLevelCircuitBreaker { get; set; } = ConfigurationManager.IsPartitionLevelCircuitBreakerEnabled(defaultValue: false);
 
+        /// <summary>
+        /// Internal option to disable Per Partition Automatic Failover (PPAF) explicitly.
+        /// When set to true, this will be used to disable PPAF irrespective of the account settings.
+        /// The default value for this parameter is 'false'.
+        /// </summary>
+        internal bool DisablePartitionLevelFailover { get; set; } = false;
+
         /// <summary>
         /// Quorum Read allowed with eventual consistency account or consistent prefix account.
         /// </summary>
@@ -1030,6 +1037,7 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId)
                 MaxTcpConnectionsPerEndpoint = this.MaxTcpConnectionsPerEndpoint,
                 EnableEndpointDiscovery = !this.LimitToEndpoint,
                 EnablePartitionLevelCircuitBreaker = this.EnablePartitionLevelCircuitBreaker,
+                DisablePartitionLevelFailover = this.DisablePartitionLevelFailover,
                 PortReuseMode = this.portReuseMode,
                 EnableTcpConnectionEndpointRediscovery = this.EnableTcpConnectionEndpointRediscovery,
                 EnableAdvancedReplicaSelectionForTcp = this.EnableAdvancedReplicaSelectionForTcp,

Microsoft.Azure.Cosmos/src/DocumentClient.cs

@@ -1056,10 +1056,16 @@ private async Task<bool> GetInitializationTaskAsync(IStoreClientFactory storeCli
                 this.EnsureValidOverwrite(this.desiredConsistencyLevel.Value);
             }
 
-            bool isPPafEnabled = ConfigurationManager.IsPartitionLevelFailoverEnabled(defaultValue: false);
-            if (this.accountServiceConfiguration != null && this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.HasValue)
-            {
-                isPPafEnabled = this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.Value;
+            bool isPPafEnabled = false;
+            if (this.accountServiceConfiguration != null && this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.HasValue)
+            {
+                isPPafEnabled = this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.Value;
+            }
+
+            // Apply the DisablePartitionLevelFailover setting to override PPAF if explicitly disabled
+            if (this.ConnectionPolicy.DisablePartitionLevelFailover)
+            {
+                isPPafEnabled = false;
             }
 
             this.ConnectionPolicy.EnablePartitionLevelFailover = isPPafEnabled;

Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs

@@ -602,10 +602,16 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account
                 return;
             }
 
-            bool isPPafEnabled = ConfigurationManager.IsPartitionLevelFailoverEnabled(defaultValue: false);
-            if (databaseAccount.EnablePartitionLevelFailover.HasValue)
-            {
-                isPPafEnabled = databaseAccount.EnablePartitionLevelFailover.Value;
+            bool isPPafEnabled = false;
+            if (databaseAccount.EnablePartitionLevelFailover.HasValue)
+            {
+                isPPafEnabled = databaseAccount.EnablePartitionLevelFailover.Value;
+            }
+
+            // Apply the DisablePartitionLevelFailover setting to override PPAF if explicitly disabled
+            if (this.connectionPolicy.DisablePartitionLevelFailover)
+            {
+                isPPafEnabled = false;
             }
 
             this.connectionPolicy.EnablePartitionLevelFailover = isPPafEnabled;

Microsoft.Azure.Cosmos/src/Routing/GlobalPartitionEndpointManagerCore.cs

@@ -608,7 +608,7 @@ public PartitionKeyRangeFailoverInfo(
                 this.ConsecutiveWriteRequestFailureCount = 0;
                 this.ReadRequestFailureCounterThreshold = ConfigurationManager.GetCircuitBreakerConsecutiveFailureCountForReads(10);
                 this.WriteRequestFailureCounterThreshold = ConfigurationManager.GetCircuitBreakerConsecutiveFailureCountForWrites(5);
-                this.TimeoutCounterResetWindowInMinutes = TimeSpan.FromMinutes(1);
+                this.TimeoutCounterResetWindowInMinutes = TimeSpan.FromMinutes(ConfigurationManager.GetCircuitBreakerTimeoutCounterResetWindowInMinutes(5));
                 this.FirstRequestFailureTime = DateTime.UtcNow;
                 this.LastRequestFailureTime = DateTime.UtcNow;
             }

Microsoft.Azure.Cosmos/src/Util/ConfigurationManager.cs

@@ -16,11 +16,10 @@ internal static class ConfigurationManager
         internal static readonly string ReplicaConnectivityValidationEnabled = "AZURE_COSMOS_REPLICA_VALIDATION_ENABLED";
 
         /// <summary>
-        /// A read-only string containing the environment variable name for enabling per partition automatic failover.
-        /// This will eventually be removed once per partition automatic failover is enabled by default for both preview
-        /// and GA.
+        /// A read-only string containing the environment variable name for capturing the PPCB timeout counter reset window time
+        /// in minutes. The default value for this window is 5 minutes.
         /// </summary>
-        internal static readonly string PartitionLevelFailoverEnabled = "AZURE_COSMOS_PARTITION_LEVEL_FAILOVER_ENABLED";
+        internal static readonly string CircuitBreakerTimeoutCounterResetWindowInMinutes = "AZURE_COSMOS_PPCB_TIMEOUT_COUNTER_RESET_WINDOW_IN_MINUTES";
 
         /// <summary>
         /// A read-only string containing the environment variable name for enabling per partition circuit breaker. The default value
@@ -160,19 +159,19 @@ public static bool IsReplicaAddressValidationEnabled(
         }
 
         /// <summary>
-        /// Gets the boolean value of the partition level failover environment variable. Note that, partition level failover
-        /// is disabled by default for both preview and GA releases. The user can set the  respective environment variable
-        /// 'AZURE_COSMOS_PARTITION_LEVEL_FAILOVER_ENABLED' to override the value for both preview and GA. The method will
-        /// eventually be removed, once partition level failover is enabled by default for  both preview and GA.
+        /// Gets the PPCB timeout counter reset window in minutes.
+        /// The default value for this window is 5 minutes. The user can set the respective
+        /// environment variable 'AZURE_COSMOS_PPCB_TIMEOUT_COUNTER_RESET_WINDOW_IN_MINUTES'
+        /// to override the value.
         /// </summary>
-        /// <param name="defaultValue">A boolean field containing the default value for partition level failover.</param>
-        /// <returns>A boolean flag indicating if partition level failover is enabled.</returns>
-        public static bool IsPartitionLevelFailoverEnabled(
-            bool defaultValue)
+        /// <param name="defaultValue">An integer containing the default value for the timeout counter reset window in minutes.</param>
+        /// <returns>An integer representing the timeout counter reset window in minutes.</returns>
+        public static int GetCircuitBreakerTimeoutCounterResetWindowInMinutes(
+            int defaultValue)
         {
             return ConfigurationManager
                     .GetEnvironmentVariable(
-                        variable: ConfigurationManager.PartitionLevelFailoverEnabled,
+                        variable: ConfigurationManager.CircuitBreakerTimeoutCounterResetWindowInMinutes,
                         defaultValue: defaultValue);
         }
 

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/ConfigurationManagerTests.cs

@@ -0,0 +1,72 @@
+//------------------------------------------------------------
+// Copyright (c) Microsoft Corporation.  All rights reserved.
+//------------------------------------------------------------
+
+namespace Microsoft.Azure.Cosmos.Tests
+{
+    using System;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class ConfigurationManagerTests
+    {
+        [TestMethod]
+        public void GetCircuitBreakerTimeoutCounterResetWindowInMinutes_DefaultValue()
+        {
+            // Test that the default value is returned when environment variable is not set
+            int result = ConfigurationManager.GetCircuitBreakerTimeoutCounterResetWindowInMinutes(5);
+            Assert.AreEqual(5, result);
+        }
+
+        [TestMethod]
+        public void GetCircuitBreakerTimeoutCounterResetWindowInMinutes_CustomDefaultValue()
+        {
+            // Test that custom default values are respected
+            int result = ConfigurationManager.GetCircuitBreakerTimeoutCounterResetWindowInMinutes(10);
+            Assert.AreEqual(10, result);
+        }
+
+        [TestMethod]
+        public void GetCircuitBreakerTimeoutCounterResetWindowInMinutes_EnvironmentVariableOverride()
+        {
+            // Test that environment variable overrides the default value
+            const string envVarName = "AZURE_COSMOS_PPCB_TIMEOUT_COUNTER_RESET_WINDOW_IN_MINUTES";
+            const string testValue = "15";
+
+            try
+            {
+                Environment.SetEnvironmentVariable(envVarName, testValue);
+                int result = ConfigurationManager.GetCircuitBreakerTimeoutCounterResetWindowInMinutes(5);
+                Assert.AreEqual(15, result);
+            }
+            finally
+            {
+                // Clean up environment variable
+                Environment.SetEnvironmentVariable(envVarName, null);
+            }
+        }
+
+        [TestMethod]
+        public void PartitionLevelFailoverEnvironmentVariableRemoved()
+        {
+            // Verify that the deprecated environment variable constant is no longer available
+            // This test ensures the AZURE_COSMOS_PARTITION_LEVEL_FAILOVER_ENABLED constant was removed
+            Type configManagerType = typeof(ConfigurationManager);
+            var field = configManagerType.GetField("PartitionLevelFailoverEnabled", 
+                System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Public);
+
+            Assert.IsNull(field, "PartitionLevelFailoverEnabled constant should have been removed");
+        }
+
+        [TestMethod]
+        public void IsPartitionLevelFailoverEnabledMethodRemoved()
+        {
+            // Verify that the deprecated IsPartitionLevelFailoverEnabled method is no longer available
+            Type configManagerType = typeof(ConfigurationManager);
+            var method = configManagerType.GetMethod("IsPartitionLevelFailoverEnabled", 
+                System.Reflection.BindingFlags.Static | System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Public);
+
+            Assert.IsNull(method, "IsPartitionLevelFailoverEnabled method should have been removed");
+        }
+    }
+}

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/CosmosClientOptionsUnitTests.cs

@@ -225,17 +225,17 @@ public void VerifyCosmosConfigurationPropertiesGetUpdated()
             CollectionAssert.AreEqual(regionalEndpoints.ToArray(), policy.AccountInitializationCustomEndpoints.ToArray());
         }
 
-        /// <summary>
-        /// Test to validate that when the partition level failover is enabled with the preferred regions list is missing, then the client
-        /// initialization should succeed. This should hold true for both environment variable and CosmosClientOptions.
-        /// </summary>
-        [TestMethod]
-        [Owner("dkunda")]
-        public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredRegionsNotSet_ShouldInitializeCosmosClientSuccessfully()
-        {
-            try
-            {
-                Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelFailoverEnabled, "True");
+        /// <summary>
+        /// Test to validate that when the partition level failover is enabled, client
+        /// initialization should succeed. Environment variable support was removed.
+        /// </summary>
+        [TestMethod]
+        [Owner("dkunda")]
+        public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredRegionsNotSet_ShouldInitializeCosmosClientSuccessfully()
+        {
+            try
+            {
+                // Note: Environment variable for PPAF was removed. This test now verifies the new approach.
 
                 string endpoint = AccountEndpoint;
                 string key = MockCosmosUtil.RandomInvalidCorrectlyFormatedAuthKey;
@@ -273,26 +273,26 @@ public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredReg
 
                 CosmosClient cosmosClient = cosmosClientBuilder.Build();
 
-                Assert.IsNotNull(cosmosClient,
-                    message: "ApplicationPreferredRegions or ApplicationRegion is no longer mandatory fields, hence the client initialization should succeed.");
-            }
-            finally
-            {
-                Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelFailoverEnabled, null);
+                Assert.IsNotNull(cosmosClient,
+                    message: "ApplicationPreferredRegions or ApplicationRegion is no longer mandatory fields, hence the client initialization should succeed.");
+            }
+            finally
+            {
+                // Cleanup - no environment variable to reset since PPAF env var was removed
             }
         }
 
-        /// <summary>
-        /// Test to validate that when the partition level failover is enabled with the preferred regions list is provided, then the client
-        /// initialization should be successful. This holds true for both environment variable and CosmosClientOptions.
-        /// </summary>
-        [TestMethod]
-        [Owner("dkunda")]
-        public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredRegionsSet_ShouldInitializeSuccessfully()
-        {
-            try
-            {
-                Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelFailoverEnabled, "True");
+        /// <summary>
+        /// Test to validate that when the partition level failover is enabled with the preferred regions list is provided, then the client
+        /// initialization should be successful. Environment variable support was removed.
+        /// </summary>
+        [TestMethod]
+        [Owner("dkunda")]
+        public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredRegionsSet_ShouldInitializeSuccessfully()
+        {
+            try
+            {
+                // Note: Environment variable for PPAF was removed. This test now verifies the new approach.
 
                 string endpoint = AccountEndpoint;
                 string key = MockCosmosUtil.RandomInvalidCorrectlyFormatedAuthKey;
@@ -355,12 +355,12 @@ public void CosmosClientOptions_WhenPartitionLevelFailoverEnabledAndPreferredReg
                 Assert.AreEqual(cosmosSerializerOptions.Indented, clientOptions.SerializerOptions.Indented);
                 Assert.IsFalse(clientOptions.AllowBulkExecution);
                 Assert.AreEqual(consistencyLevel, clientOptions.ConsistencyLevel);
-                Assert.IsNotNull(clientOptions.ApplicationPreferredRegions);
-                Assert.IsNotNull(clientOptions.AccountInitializationCustomEndpoints);
-            }
-            finally
-            {
-                Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelFailoverEnabled, null);
+                Assert.IsNotNull(clientOptions.ApplicationPreferredRegions);
+                Assert.IsNotNull(clientOptions.AccountInitializationCustomEndpoints);
+            }
+            finally
+            {
+                // Cleanup - no environment variable to reset since PPAF env var was removed
             }
         }
 
@@ -1212,8 +1212,26 @@ public void TestServerCertificatesValidationWithDisableSSLFlagTrue(string connSt
             RemoteCertificateValidationCallback? httpClientRemoreCertValidationCallback = socketsHttpHandler.SslOptions.RemoteCertificateValidationCallback;
             Assert.IsNotNull(httpClientRemoreCertValidationCallback);
 #nullable disable
-        }
-
+        }
+
+        [TestMethod]
+        public void DisablePartitionLevelFailoverOptionTest()
+        {
+            // Test that the DisablePartitionLevelFailover option defaults to false
+            CosmosClientOptions clientOptions = new CosmosClientOptions();
+            Assert.IsFalse(clientOptions.DisablePartitionLevelFailover);
+
+            // Test that setting DisablePartitionLevelFailover to true is reflected in the connection policy
+            clientOptions.DisablePartitionLevelFailover = true;
+            ConnectionPolicy policy = clientOptions.GetConnectionPolicy(clientId: 0);
+            Assert.IsTrue(policy.DisablePartitionLevelFailover);
+
+            // Test that setting DisablePartitionLevelFailover to false is reflected in the connection policy
+            clientOptions.DisablePartitionLevelFailover = false;
+            policy = clientOptions.GetConnectionPolicy(clientId: 0);
+            Assert.IsFalse(policy.DisablePartitionLevelFailover);
+        }
+
         private class TestWebProxy : IWebProxy
         {
             public ICredentials Credentials { get; set; }

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/PartitionKeyRangeFailoverTests/GlobalPartitionEndpointManagerTests.cs

@@ -398,10 +398,8 @@ public async Task TestPPAFClientAndServerEnablementCombinationScenariosAsync(
             bool ppafEnabledFromClient,
             bool? ppafEnabledFromService)
         {
-            if (ppafEnabledFromClient)
-            {
-                Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelFailoverEnabled, "True");
-            }
+            // Note: Environment variable for PPAF was removed as per acceptance criteria.
+            // PPAF is now controlled exclusively by account metadata and client options.
 
             try
             {
@@ -495,8 +493,7 @@ public async Task TestPPAFClientAndServerEnablementCombinationScenariosAsync(
             }
             finally
             {
-                // Reset the environment variable to avoid affecting other tests.
-                Environment.SetEnvironmentVariable(ConfigurationManager.PartitionLevelFailoverEnabled, null);
+                // Cleanup - no environment variable to reset since PPAF env var was removed
             }
         }