Add backup endpoint fallback for Cosmos client

Add with_backup_endpoints() API to support fallback endpoints when
the primary global endpoint is unavailable. Endpoints are tried in
order during initialization and steady-state metadata refreshes.

Driver crate:
- AccountReference: add backup_endpoints field, builder method, accessor
- CosmosDriver: fallback in probe_http_version and refresh_account_properties

SDK crate:
- CosmosClientBuilder::with_backup_endpoints() public API
- GlobalEndpointManager: fallback in get_database_account()
- Integration tests for boot-via-backup and post-boot operations

Fixes Azure#4099

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: tvaron3

sdk/cosmos/azure_data_cosmos/CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features Added
 
+- Added `CosmosClientBuilder::with_backup_endpoints()` for specifying fallback endpoints when the primary global endpoint is unavailable during initialization and account metadata refreshes. ([#4099](https://github.com/Azure/azure-sdk-for-rust/issues/4099))
 - Added `CosmosClientBuilder::with_proxy_allowed(bool)` for explicit opt-in to HTTP proxy usage with documented support limitations. ([#4062](https://github.com/Azure/azure-sdk-for-rust/pull/4062))
 - Added `CustomResponseBuilder` and `FaultInjectionRule::hit_count()` APIs for fault injection, enabling ergonomic construction of synthetic HTTP responses and test verification of rule activation counts. ([#3888](https://github.com/Azure/azure-sdk-for-rust/pull/3888))
 

sdk/cosmos/azure_data_cosmos/src/clients/cosmos_client_builder.rs

@@ -87,6 +87,8 @@ pub struct CosmosClientBuilder {
     /// Fault injection builder for testing error handling
     #[cfg(feature = "fault_injection")]
     fault_injection_builder: Option<crate::fault_injection::FaultInjectionClientBuilder>,
+    /// Fallback endpoints tried when the primary endpoint is unavailable.
+    backup_endpoints: Vec<azure_core::http::Url>,
 }
 
 impl CosmosClientBuilder {
@@ -162,6 +164,22 @@ impl CosmosClientBuilder {
         self
     }
 
+    /// Sets backup endpoints that the client will try when the primary global
+    /// endpoint is unavailable.
+    ///
+    /// During initialization and periodic account metadata refreshes, if the
+    /// primary endpoint fails, the SDK will try each backup endpoint in order
+    /// until one succeeds. A successful connection allows normal service
+    /// discovery to proceed.
+    ///
+    /// # Arguments
+    ///
+    /// * `endpoints` - Ordered list of fallback endpoint URLs.
+    pub fn with_backup_endpoints(mut self, endpoints: Vec<crate::CosmosAccountEndpoint>) -> Self {
+        self.backup_endpoints = endpoints.into_iter().map(|e| e.into_url()).collect();
+        self
+    }
+
     /// Builds the [`CosmosClient`] with the specified account reference and region selection strategy.
     ///
     /// The account reference bundles an endpoint and credential. You can create one using
@@ -317,6 +335,7 @@ impl CosmosClientBuilder {
             preferred_regions,
             Vec::new(),
             pipeline_core.clone(),
+            self.backup_endpoints.clone(),
         );
 
         // Enable per-partition circuit breaker based on the
@@ -364,7 +383,8 @@ impl CosmosClientBuilder {
         // TODO: Each CosmosClient currently creates its own CosmosDriverRuntime. The runtime
         // should be shared across clients targeting the same account to avoid duplicate
         // background tasks and connection pools. See https://github.com/Azure/azure-sdk-for-rust/issues/3908
-        let driver_account = build_driver_account(endpoint, driver_credential);
+        let driver_account =
+            build_driver_account(endpoint, driver_credential, self.backup_endpoints);
         #[allow(unused_mut)]
         let mut driver_runtime_builder = CosmosDriverRuntimeBuilder::new();
         #[cfg(feature = "allow_invalid_certificates")]
@@ -401,15 +421,21 @@ impl CosmosClientBuilder {
 fn build_driver_account(
     endpoint: azure_core::http::Url,
     credential: CosmosCredential,
+    backup_endpoints: Vec<azure_core::http::Url>,
 ) -> azure_data_cosmos_driver::models::AccountReference {
-    match credential {
+    let base = match credential {
         CosmosCredential::TokenCredential(tc) => {
             azure_data_cosmos_driver::models::AccountReference::with_credential(endpoint, tc)
         }
         #[cfg(feature = "key_auth")]
         CosmosCredential::MasterKey(key) => {
             azure_data_cosmos_driver::models::AccountReference::with_master_key(endpoint, key)
         }
+    };
+    if backup_endpoints.is_empty() {
+        base
+    } else {
+        base.with_backup_endpoints(backup_endpoints)
     }
 }
 

sdk/cosmos/azure_data_cosmos/src/retry_policies/client_retry_policy.rs

@@ -766,6 +766,7 @@ mod tests {
             vec![Region::from("West US"), Region::from("East US")],
             vec![],
             pipeline,
+            vec![],
         )
     }
 
@@ -784,6 +785,7 @@ mod tests {
             vec![],
             vec![],
             pipeline,
+            vec![],
         )
     }
 
@@ -802,6 +804,7 @@ mod tests {
             vec![Region::EAST_ASIA, Region::WEST_US, Region::NORTH_CENTRAL_US],
             vec![],
             pipeline,
+            vec![],
         )
     }
 
@@ -891,6 +894,7 @@ mod tests {
             vec![Region::from("West US"), Region::from("East US")],
             vec![],
             pipeline,
+            vec![],
         );
 
         let west = AccountRegion {

sdk/cosmos/azure_data_cosmos/src/retry_policies/metadata_request_retry_policy.rs

@@ -291,6 +291,7 @@ mod tests {
             vec![Region::from("West US"), Region::from("East US")],
             vec![],
             pipeline,
+            vec![],
         )
     }
 
@@ -309,6 +310,7 @@ mod tests {
             vec![],
             vec![],
             pipeline,
+            vec![],
         )
     }
 
@@ -327,6 +329,7 @@ mod tests {
             vec![Region::EAST_ASIA, Region::WEST_US, Region::NORTH_CENTRAL_US],
             vec![],
             pipeline,
+            vec![],
         )
     }
 

sdk/cosmos/azure_data_cosmos/src/routing/global_endpoint_manager.rs

@@ -35,6 +35,9 @@ pub(crate) struct GlobalEndpointManager {
     /// The primary default endpoint URL for the Cosmos DB account
     default_endpoint: Url,
 
+    /// Fallback endpoints tried when the primary endpoint is unavailable
+    backup_endpoints: Vec<Url>,
+
     /// Thread-safe cache of location information including read/write endpoints and availability status
     location_cache: Mutex<LocationCache>,
 
@@ -64,6 +67,7 @@ impl Debug for GlobalEndpointManager {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         f.debug_struct("GlobalEndpointManager")
             .field("default_endpoint", &self.default_endpoint)
+            .field("backup_endpoints", &self.backup_endpoints)
             .field("location_cache", &self.location_cache)
             .field("pipeline", &self.pipeline)
             .field("account_properties_cache", &self.account_properties_cache)
@@ -87,6 +91,7 @@ impl GlobalEndpointManager {
     /// * `preferred_locations` - Ordered list of preferred Azure regions for request routing
     /// * `excluded_regions` - List of regions to exclude from routing
     /// * `pipeline` - HTTP pipeline for making service requests
+    /// * `backup_endpoints` - Ordered fallback endpoint URLs tried when the primary endpoint is unavailable
     ///
     /// # Returns
     /// A new `GlobalEndpointManager` instance ready for request routing
@@ -95,6 +100,7 @@ impl GlobalEndpointManager {
         preferred_locations: Vec<Region>,
         excluded_regions: Vec<Region>,
         pipeline: Pipeline,
+        backup_endpoints: Vec<Url>,
     ) -> Arc<Self> {
         let location_cache = Mutex::new(LocationCache::new(
             default_endpoint.clone(),
@@ -108,6 +114,7 @@ impl GlobalEndpointManager {
 
         let instance = Arc::new(Self {
             default_endpoint,
+            backup_endpoints,
             location_cache,
             pipeline,
             account_properties_cache,
@@ -442,14 +449,65 @@ impl GlobalEndpointManager {
     /// # Returns
     /// `Ok(Response<AccountProperties>)` with account metadata, or `Err` if request failed
     pub async fn get_database_account(&self) -> azure_core::Result<Response<AccountProperties>> {
+        match self.get_database_account_from_endpoint(None).await {
+            Ok(response) => Ok(response),
+            Err(primary_error) if !self.backup_endpoints.is_empty() => {
+                tracing::warn!(
+                    endpoint = %self.default_endpoint,
+                    error = %primary_error,
+                    "primary endpoint account fetch failed; trying backup endpoints"
+                );
+
+                for backup_url in &self.backup_endpoints {
+                    match self
+                        .get_database_account_from_endpoint(Some(backup_url.clone()))
+                        .await
+                    {
+                        Ok(response) => {
+                            tracing::info!(
+                                backup_endpoint = %backup_url,
+                                "backup endpoint account fetch succeeded"
+                            );
+                            return Ok(response);
+                        }
+                        Err(e) => {
+                            tracing::warn!(
+                                backup_endpoint = %backup_url,
+                                error = %e,
+                                "backup endpoint account fetch failed; trying next"
+                            );
+                        }
+                    }
+                }
+
+                tracing::error!(
+                    endpoint = %self.default_endpoint,
+                    backup_count = self.backup_endpoints.len(),
+                    "all endpoints exhausted during account properties fetch"
+                );
+                Err(primary_error)
+            }
+            Err(error) => Err(error),
+        }
+    }
+
+    /// Fetches database account properties from a specific endpoint.
+    ///
+    /// If `endpoint_override` is `None`, uses the endpoint resolved from the
+    /// location cache. Otherwise uses the provided endpoint directly.
+    async fn get_database_account_from_endpoint(
+        &self,
+        endpoint_override: Option<Url>,
+    ) -> azure_core::Result<Response<AccountProperties>> {
         let resource_link = ResourceLink::root(ResourceType::DatabaseAccount);
         let builder = CosmosRequest::builder(OperationType::Read, resource_link.clone());
         let mut cosmos_request = builder.build()?;
-        let endpoint = self
-            .location_cache
-            .lock()
-            .unwrap()
-            .resolve_service_endpoint(&cosmos_request);
+        let endpoint = endpoint_override.unwrap_or_else(|| {
+            self.location_cache
+                .lock()
+                .unwrap()
+                .resolve_service_endpoint(&cosmos_request)
+        });
         cosmos_request.request_context.location_endpoint_to_route = Some(endpoint);
         let ctx_owned = Context::default().with_value(resource_link);
         self.pipeline
@@ -575,6 +633,7 @@ mod tests {
             vec![Region::from("West US"), Region::from("East US")],
             vec![],
             create_test_pipeline(),
+            vec![],
         )
     }
 

sdk/cosmos/azure_data_cosmos/src/routing/global_partition_endpoint_manager.rs

@@ -1136,6 +1136,7 @@ mod tests {
             vec![Region::from("West US")],
             vec![],
             create_test_pipeline(),
+            vec![],
         )
     }
 
@@ -1145,6 +1146,7 @@ mod tests {
             vec![Region::from("West US"), Region::from("East US")],
             vec![],
             create_test_pipeline(),
+            vec![],
         );
 
         let west = AccountRegion {
@@ -1170,6 +1172,7 @@ mod tests {
             ],
             vec![],
             create_test_pipeline(),
+            vec![],
         );
 
         let west = AccountRegion {
@@ -1202,6 +1205,7 @@ mod tests {
             vec![Region::from("West US"), Region::from("East US")],
             vec![],
             create_test_pipeline(),
+            vec![],
         );
 
         let west = AccountRegion {

sdk/cosmos/azure_data_cosmos/tests/emulator_tests/cosmos_backup_endpoints.rs

@@ -0,0 +1,99 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+//! Integration tests for backup endpoint fallback.
+//! Verifies that the SDK can boot using a backup endpoint when the global endpoint is unavailable.
+
+#![cfg(feature = "key_auth")]
+
+use super::framework;
+
+use azure_data_cosmos::{
+    ConnectionString, CosmosAccountEndpoint, CosmosAccountReference, CosmosClient, RoutingStrategy,
+};
+use framework::{TestClient, CONNECTION_STRING_ENV_VAR, HUB_REGION};
+use std::error::Error;
+
+/// Tests that the SDK can initialize when the global endpoint is unreachable
+/// but a valid backup endpoint is provided.
+///
+/// The test uses a RFC 5737 TEST-NET address (guaranteed non-routable) as the
+/// global endpoint and sets the real account endpoint as a backup. The client
+/// should fail to reach the global endpoint, fall back to the backup endpoint,
+/// and initialize successfully.
+#[tokio::test]
+async fn client_boots_via_backup_when_global_endpoint_unreachable() -> Result<(), Box<dyn Error>> {
+    TestClient::run(async |_run_context| {
+        let Ok(env_var) = std::env::var(CONNECTION_STRING_ENV_VAR) else {
+            eprintln!("Skipping: no connection string");
+            return Ok(());
+        };
+
+        let connection_string: ConnectionString = env_var.parse()?;
+        let real_endpoint: CosmosAccountEndpoint = connection_string.account_endpoint.parse()?;
+
+        // RFC 5737 TEST-NET address — guaranteed non-routable for deterministic failure.
+        let fake_endpoint: CosmosAccountEndpoint = "https://192.0.2.1:443/".parse()?;
+
+        let client = CosmosClient::builder()
+            .with_backup_endpoints(vec![real_endpoint])
+            .build(
+                CosmosAccountReference::with_master_key(
+                    fake_endpoint,
+                    connection_string.account_key.clone(),
+                ),
+                RoutingStrategy::ProximityTo(HUB_REGION),
+            )
+            .await;
+
+        assert!(
+            client.is_ok(),
+            "client should initialize via backup endpoint, but got: {:?}",
+            client.err()
+        );
+
+        Ok(())
+    })
+    .await
+}
+
+/// Tests that the SDK can list databases after initializing via a backup endpoint.
+#[tokio::test]
+async fn operations_work_after_backup_endpoint_boot() -> Result<(), Box<dyn Error>> {
+    TestClient::run(async |_run_context| {
+        let Ok(env_var) = std::env::var(CONNECTION_STRING_ENV_VAR) else {
+            eprintln!("Skipping: no connection string");
+            return Ok(());
+        };
+
+        let connection_string: ConnectionString = env_var.parse()?;
+        let real_endpoint: CosmosAccountEndpoint = connection_string.account_endpoint.parse()?;
+
+        let fake_endpoint: CosmosAccountEndpoint = "https://192.0.2.1:443/".parse()?;
+
+        let client = CosmosClient::builder()
+            .with_backup_endpoints(vec![real_endpoint])
+            .build(
+                CosmosAccountReference::with_master_key(
+                    fake_endpoint,
+                    connection_string.account_key.clone(),
+                ),
+                RoutingStrategy::ProximityTo(HUB_REGION),
+            )
+            .await?;
+
+        // Verify the client can list databases (a basic read operation)
+        use futures::TryStreamExt;
+        let query = azure_data_cosmos::Query::from("SELECT * FROM root r");
+        let mut pager = client.query_databases(query, None)?;
+        let page = pager.try_next().await;
+        assert!(
+            page.is_ok(),
+            "should be able to list databases after backup boot: {:?}",
+            page.err()
+        );
+
+        Ok(())
+    })
+    .await
+}

sdk/cosmos/azure_data_cosmos/tests/emulator_tests/mod.rs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
+mod cosmos_backup_endpoints;
 mod cosmos_batch;
 mod cosmos_containers;
 mod cosmos_databases;