[Internal] Per Partition Automatic Failover: Removes Remove Environment Variable to Set PPAF at the SDK Layer and Add Support for Internal Client Options (#5284)

NaluTripician · web-flow · commit f869cce9bedf · 2025-07-16T17:30:53.000-07:00
# Pull Request Template ## Description - **Remove Dependency on Environment Variable:** Today, the PPAF enablement in the .NET SDK is done completely through the Get Account metadata response. However, we still kept the environment variable `AZURE_COSMOS_PARTITION_LEVEL_FAILOVER_ENABLED` at a deprecated state to toggle the behavior, if the cosmos account doesn't have the flag enabled. As a part of this task, we will remove the environment variable completely from the .NET SDK ecosystem. - **Create a new Environment Variable to externalize circuit breaker timeout counter reset window:** Goal is to create a new environment variable `AZURE_COSMOS_PPCB_TIMEOUT_COUNTER_RESET_WINDOW_IN_MINUTES` to externalize the PPCB timeout counter reset window. The default value for this would be `5` minutes. - **Add New Internal Client Options to disable PPAF:** Add a new **internal** cosmos client options to disable PPAF explicitly. Once set, this will be used to disable PPAF irrespective of the account settings. ## Type of change Please delete options that are not relevant. - [] New feature (non-breaking change which adds functionality) ## Closing issues To automatically close an issue: closes #5277
diff --git a/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs b/Microsoft.Azure.Cosmos/src/ConnectionPolicy.cs
@@ -338,6 +338,12 @@ public bool EnablePartitionLevelCircuitBreaker
             set;
         }
 
+        internal bool DisablePartitionLevelFailoverClientLevelOverride
+        {
+            get;
+            set;
+        }
+
         /// <summary>
         /// Gets or sets the certificate validation callback.
         /// </summary>
diff --git a/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs b/Microsoft.Azure.Cosmos/src/CosmosClientOptions.cs
@@ -770,7 +770,15 @@ bool EnableRemoteRegionPreferredForSessionRetry
         /// If compute gateway chooses to enable PPAF, then the .NET SDK will enable PPCB by default, which will improve the read availability and latency. This would mean
         /// when PPAF is enabled, the SDK will automatically enable PPCB as well.
         /// </summary>
-        internal bool EnablePartitionLevelCircuitBreaker { get; set; } = ConfigurationManager.IsPartitionLevelCircuitBreakerEnabled(defaultValue: false);
+        internal bool EnablePartitionLevelCircuitBreaker { get; set; } = ConfigurationManager.IsPartitionLevelCircuitBreakerEnabled(defaultValue: false);
+
+        /// <summary>
+        /// Flag from gateway to disable partition level failover. Normally, the SDK will enable partition level failover based on the account settings. 
+        /// This flag will be used internally by the compute gateway as by default it will disable partition level failover.
+        /// 
+        /// The default value for this parameter is 'false'.
+        /// </summary>
+        internal bool DisablePartitionLevelFailover { get; set; } = false;
 
         /// <summary>
         /// Quorum Read allowed with eventual consistency account or consistent prefix account.
@@ -1029,7 +1037,8 @@ internal virtual ConnectionPolicy GetConnectionPolicy(int clientId)
                 MaxRequestsPerTcpConnection = this.MaxRequestsPerTcpConnection,
                 MaxTcpConnectionsPerEndpoint = this.MaxTcpConnectionsPerEndpoint,
                 EnableEndpointDiscovery = !this.LimitToEndpoint,
-                EnablePartitionLevelCircuitBreaker = this.EnablePartitionLevelCircuitBreaker,
+                EnablePartitionLevelCircuitBreaker = this.EnablePartitionLevelCircuitBreaker,
+                DisablePartitionLevelFailoverClientLevelOverride = this.DisablePartitionLevelFailover,
                 PortReuseMode = this.portReuseMode,
                 EnableTcpConnectionEndpointRediscovery = this.EnableTcpConnectionEndpointRediscovery,
                 EnableAdvancedReplicaSelectionForTcp = this.EnableAdvancedReplicaSelectionForTcp,
diff --git a/Microsoft.Azure.Cosmos/src/DocumentClient.cs b/Microsoft.Azure.Cosmos/src/DocumentClient.cs
@@ -1056,13 +1056,12 @@ private async Task<bool> GetInitializationTaskAsync(IStoreClientFactory storeCli
                 this.EnsureValidOverwrite(this.desiredConsistencyLevel.Value);
             }
 
-            bool isPPafEnabled = ConfigurationManager.IsPartitionLevelFailoverEnabled(defaultValue: false);
-            if (this.accountServiceConfiguration != null && this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.HasValue)
+            if (!this.ConnectionPolicy.DisablePartitionLevelFailoverClientLevelOverride
+                && this.accountServiceConfiguration != null && this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.HasValue)
             {
-                isPPafEnabled = this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.Value;
+                this.ConnectionPolicy.EnablePartitionLevelFailover = this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.Value;
             }
 
-            this.ConnectionPolicy.EnablePartitionLevelFailover = isPPafEnabled;
             this.ConnectionPolicy.EnablePartitionLevelCircuitBreaker |= this.ConnectionPolicy.EnablePartitionLevelFailover;
             this.ConnectionPolicy.UserAgentContainer.AppendFeatures(this.GetUserAgentFeatures());
             this.InitializePartitionLevelFailoverWithDefaultHedging();
diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalEndpointManager.cs
@@ -602,13 +602,11 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account
                 return;
             }
 
-            bool isPPafEnabled = ConfigurationManager.IsPartitionLevelFailoverEnabled(defaultValue: false);
-            if (databaseAccount.EnablePartitionLevelFailover.HasValue)
+            if (!this.connectionPolicy.DisablePartitionLevelFailoverClientLevelOverride && databaseAccount.EnablePartitionLevelFailover.HasValue)
             {
-                isPPafEnabled = databaseAccount.EnablePartitionLevelFailover.Value;
+                this.connectionPolicy.EnablePartitionLevelFailover = databaseAccount.EnablePartitionLevelFailover.Value;
             }
 
-            this.connectionPolicy.EnablePartitionLevelFailover = isPPafEnabled;
             GlobalEndpointManager.ParseThinClientLocationsFromAdditionalProperties(databaseAccount);
 
             this.locationCache.OnDatabaseAccountRead(databaseAccount);
diff --git a/Microsoft.Azure.Cosmos/src/Routing/GlobalPartitionEndpointManagerCore.cs b/Microsoft.Azure.Cosmos/src/Routing/GlobalPartitionEndpointManagerCore.cs
@@ -608,7 +608,7 @@ public PartitionKeyRangeFailoverInfo(
                 this.ConsecutiveWriteRequestFailureCount = 0;
                 this.ReadRequestFailureCounterThreshold = ConfigurationManager.GetCircuitBreakerConsecutiveFailureCountForReads(10);
                 this.WriteRequestFailureCounterThreshold = ConfigurationManager.GetCircuitBreakerConsecutiveFailureCountForWrites(5);
-                this.TimeoutCounterResetWindowInMinutes = TimeSpan.FromMinutes(1);
+                this.TimeoutCounterResetWindowInMinutes = TimeSpan.FromMinutes(ConfigurationManager.GetCircuitBreakerTimeoutCounterResetWindowInMinutes(5));
                 this.FirstRequestFailureTime = DateTime.UtcNow;
                 this.LastRequestFailureTime = DateTime.UtcNow;
             }
diff --git a/Microsoft.Azure.Cosmos/src/Util/ConfigurationManager.cs b/Microsoft.Azure.Cosmos/src/Util/ConfigurationManager.cs
@@ -60,7 +60,13 @@ internal static class ConfigurationManager
         /// A read-only string containing the environment variable name for capturing the consecutive failure count for writes, before triggering per partition
         /// circuit breaker flow. The default value for this interval is 10 consecutive requests within 1 min window.
         /// </summary>
-        internal static readonly string CircuitBreakerConsecutiveFailureCountForWrites = "AZURE_COSMOS_PPCB_CONSECUTIVE_FAILURE_COUNT_FOR_WRITES";
+        internal static readonly string CircuitBreakerConsecutiveFailureCountForWrites = "AZURE_COSMOS_PPCB_CONSECUTIVE_FAILURE_COUNT_FOR_WRITES";
+
+        /// <summary>
+        /// A read-only string containing the environment variable name for capturing the consecutive failure count for writes, before triggering per partition
+        /// circuit breaker flow. The default value for this interval is 5 consecutive requests within 1 min window.
+        /// </summary>
+        internal static readonly string CircuitBreakerTimeoutCounterResetWindowInMinutes = "AZURE_COSMOS_PPCB_TIMEOUT_COUNTER_RESET_WINDOW_IN_MINUTES";
 
         /// <summary>
         /// Environment variable name for overriding optimistic direct execution of queries.
@@ -159,23 +165,6 @@ public static bool IsReplicaAddressValidationEnabled(
                         defaultValue: true);
         }
 
-        /// <summary>
-        /// Gets the boolean value of the partition level failover environment variable. Note that, partition level failover
-        /// is disabled by default for both preview and GA releases. The user can set the  respective environment variable
-        /// 'AZURE_COSMOS_PARTITION_LEVEL_FAILOVER_ENABLED' to override the value for both preview and GA. The method will
-        /// eventually be removed, once partition level failover is enabled by default for  both preview and GA.
-        /// </summary>
-        /// <param name="defaultValue">A boolean field containing the default value for partition level failover.</param>
-        /// <returns>A boolean flag indicating if partition level failover is enabled.</returns>
-        public static bool IsPartitionLevelFailoverEnabled(
-            bool defaultValue)
-        {
-            return ConfigurationManager
-                    .GetEnvironmentVariable(
-                        variable: ConfigurationManager.PartitionLevelFailoverEnabled,
-                        defaultValue: defaultValue);
-        }
-
         /// <summary>
         /// Gets the boolean value indicating whether the thin client mode is enabled based on the environment variable override.
         /// </summary>
@@ -284,7 +273,23 @@ public static int GetCircuitBreakerConsecutiveFailureCountForWrites(
                     .GetEnvironmentVariable(
                         variable: ConfigurationManager.CircuitBreakerConsecutiveFailureCountForWrites,
                         defaultValue: defaultValue);
-        }
+        }
+
+        /// <summary>
+        /// Gets the consecutive faulure count for writes (applicable for multi master accounts) before triggering
+        /// the per partition circuit breaker flow. The default value for this interval is 5 consecutive requests within a 1-minute window.
+        /// The user can set the respective environment variable 'AZURE_COSMOS_PPCB_TIMEOUT_COUNTER_RESET_WINDOW_IN_MINUTES' to override the value.
+        /// </summary>
+        /// <param name="defaultValue">An integer containing the default value for the consecutive failure count.</param>
+        /// <returns>An double representing the timeout counter reset window in minutes.</returns>
+        public static double GetCircuitBreakerTimeoutCounterResetWindowInMinutes(
+            double defaultValue)
+        {
+            return ConfigurationManager
+                    .GetEnvironmentVariable(
+                        variable: ConfigurationManager.CircuitBreakerTimeoutCounterResetWindowInMinutes,
+                        defaultValue: defaultValue);
+        }
 
         /// <summary>
         /// Gets the boolean value indicating whether optimistic direct execution is enabled based on the environment variable override.
diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemIntegrationTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/CosmosItemIntegrationTests.cs
diff --git a/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/PartitionKeyRangeFailoverTests/GlobalPartitionEndpointManagerTests.cs b/Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/PartitionKeyRangeFailoverTests/GlobalPartitionEndpointManagerTests.cs

Original file line number	Diff line number	Diff line change
`@@ -1056,13 +1056,12 @@ private async Task<bool> GetInitializationTaskAsync(IStoreClientFactory storeCli`
`1056`	`1056`	`this.EnsureValidOverwrite(this.desiredConsistencyLevel.Value);`
`1057`	`1057`	`}`
`1058`	`1058`
`1059`		`- bool isPPafEnabled = ConfigurationManager.IsPartitionLevelFailoverEnabled(defaultValue: false);`
`1060`		`- if (this.accountServiceConfiguration != null && this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.HasValue)`
	`1059`	`+ if (!this.ConnectionPolicy.DisablePartitionLevelFailoverClientLevelOverride`
	`1060`	`+ && this.accountServiceConfiguration != null && this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.HasValue)`
`1061`	`1061`	`{`
`1062`		`- isPPafEnabled = this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.Value;`
	`1062`	`+ this.ConnectionPolicy.EnablePartitionLevelFailover = this.accountServiceConfiguration.AccountProperties.EnablePartitionLevelFailover.Value;`
`1063`	`1063`	`}`
`1064`	`1064`
`1065`		`- this.ConnectionPolicy.EnablePartitionLevelFailover = isPPafEnabled;`
`1066`	`1065`	`this.ConnectionPolicy.EnablePartitionLevelCircuitBreaker \|= this.ConnectionPolicy.EnablePartitionLevelFailover;`
`1067`	`1066`	`this.ConnectionPolicy.UserAgentContainer.AppendFeatures(this.GetUserAgentFeatures());`
`1068`	`1067`	`this.InitializePartitionLevelFailoverWithDefaultHedging();`
Original file line number	Diff line number	Diff line change
`@@ -602,13 +602,11 @@ public virtual void InitializeAccountPropertiesAndStartBackgroundRefresh(Account`
`602`	`602`	`return;`
`603`	`603`	`}`
`604`	`604`
`605`		`- bool isPPafEnabled = ConfigurationManager.IsPartitionLevelFailoverEnabled(defaultValue: false);`
`606`		`- if (databaseAccount.EnablePartitionLevelFailover.HasValue)`
	`605`	`+ if (!this.connectionPolicy.DisablePartitionLevelFailoverClientLevelOverride && databaseAccount.EnablePartitionLevelFailover.HasValue)`
`607`	`606`	`{`
`608`		`- isPPafEnabled = databaseAccount.EnablePartitionLevelFailover.Value;`
	`607`	`+ this.connectionPolicy.EnablePartitionLevelFailover = databaseAccount.EnablePartitionLevelFailover.Value;`
`609`	`608`	`}`
`610`	`609`
`611`		`- this.connectionPolicy.EnablePartitionLevelFailover = isPPafEnabled;`
`612`	`610`	`GlobalEndpointManager.ParseThinClientLocationsFromAdditionalProperties(databaseAccount);`
`613`	`611`
`614`	`612`	`this.locationCache.OnDatabaseAccountRead(databaseAccount);`
Original file line number	Diff line number	Diff line change
`@@ -608,7 +608,7 @@ public PartitionKeyRangeFailoverInfo(`
`608`	`608`	`this.ConsecutiveWriteRequestFailureCount = 0;`
`609`	`609`	`this.ReadRequestFailureCounterThreshold = ConfigurationManager.GetCircuitBreakerConsecutiveFailureCountForReads(10);`
`610`	`610`	`this.WriteRequestFailureCounterThreshold = ConfigurationManager.GetCircuitBreakerConsecutiveFailureCountForWrites(5);`
`611`		`- this.TimeoutCounterResetWindowInMinutes = TimeSpan.FromMinutes(1);`
	`611`	`+ this.TimeoutCounterResetWindowInMinutes = TimeSpan.FromMinutes(ConfigurationManager.GetCircuitBreakerTimeoutCounterResetWindowInMinutes(5));`
`612`	`612`	`this.FirstRequestFailureTime = DateTime.UtcNow;`
`613`	`613`	`this.LastRequestFailureTime = DateTime.UtcNow;`
`614`	`614`	`}`