Skip to content
Open
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions Microsoft.Azure.Cosmos/src/Handler/RequestInvokerHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -509,6 +509,9 @@ private async Task ValidateAndSetConsistencyLevelAsync(RequestMessage requestMes

/// <summary>
/// Validate and set the ReadConsistencyStrategy header.
/// When the strategy is LastCommittedWriteRegion and the operation is a read,
/// also set the hub region processing header so the backend routes the request
/// to the hub (write) region.
/// </summary>
private Task ValidateAndSetReadConsistencyStrategyAsync(RequestMessage requestMessage)
{
Expand All @@ -529,6 +532,14 @@ private Task ValidateAndSetReadConsistencyStrategyAsync(RequestMessage requestMe
requestMessage.Headers.Set(
HttpConstants.HttpHeaders.ReadConsistencyStrategy,
readConsistencyStrategy.Value.ToString());

if (readConsistencyStrategy.Value == Cosmos.ReadConsistencyStrategy.LastCommittedWriteRegion
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Recommendation — Design: Consider interaction with CrossRegionHedgingAvailabilityStrategy

When CrossRegionHedgingAvailabilityStrategy and LastCommittedWriteRegion are both active, the hedging strategy clones the request (including the ShouldProcessOnlyInHubRegion header) and sends copies to multiple regions in parallel. Non-hub regions will return 403/3, which IsFinalResult() does NOT treat as final — so each hedged copy retries internally, generating cascading 403/3 traffic.

Concrete scenario: 3-region account with hedging enabled + LastCommittedWriteRegion read → 3 parallel requests → 2 get 403/3 and retry → generates ~5+ backend requests for one logical read, with wasted RUs and added latency.

Why it matters: The combination is semantically contradictory — hedging spreads across regions, but hub-only routing rejects all non-hub regions. This isn't a correctness bug (it eventually converges), but it's wasteful and could confuse diagnostics.

Suggestion: Either:

  1. Add a check in ShouldHedge to skip hedging when ShouldProcessOnlyInHubRegion is set, or
  2. Document this as an unsupported combination, or
  3. At minimum, add a code comment noting this interaction for future maintainers.

⚠️ AI-generated review — may be incorrect. Agree? → resolve the conversation. Disagree? → reply with your reasoning.

&& OperationTypeExtensions.IsReadOperation(requestMessage.OperationType))
{
requestMessage.Headers.Set(
HttpConstants.HttpHeaders.ShouldProcessOnlyInHubRegion,
bool.TrueString);
}
}

return Task.CompletedTask;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,13 @@ enum ReadConsistencyStrategy
/// Quorum read with GCLSN barrier - returns the latest version across all regions.
/// Only valid for accounts configured with Strong consistency.
/// </summary>
GlobalStrong = 4
GlobalStrong = 4,

/// <summary>
Comment thread
aavasthy marked this conversation as resolved.
/// When set, the SDK automatically adds the hub region processing header so that
/// the backend directs the request to the hub region. Non-hub regions respond with
/// 403/3 (WriteForbidden), which the SDK retries transparently.
/// </summary>
LastCommittedWriteRegion = 5
Comment thread
aavasthy marked this conversation as resolved.
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4422,6 +4422,328 @@ public async Task ReadItemAsync_ShouldAddHubHeader_OnRetryAfter_404_1002()
Assert.AreEqual(2, return404Count, "Both requests should have returned 404/1002");
}

/// <summary>
/// Verifies that when ReadConsistencyStrategy.LastCommittedWriteRegion is set on
/// ItemRequestOptions, the hub region header and ReadConsistencyStrategy header are
/// present from the very first request and persist through all 403/3 retries.
/// Only LastCommittedWriteRegion triggers the hub header on reads.
/// Parameterized over forbidden count to cover single and multiple 403/3 retries.
/// </summary>
[TestMethod]
[Owner("aavasthy")]
[DataRow(1, DisplayName = "LastCommittedWriteRegion request-level: single 403/3 → success")]
[DataRow(3, DisplayName = "LastCommittedWriteRegion request-level: multiple 403/3 → success")]
[Description("Request-level LastCommittedWriteRegion sets hub header on reads with 403/3 retry flow.")]
public async Task ReadItemAsync_WithLastCommittedWriteRegion_RequestLevel_403_3_ThenSuccess(int forbiddenCount)
Comment thread
aavasthy marked this conversation as resolved.
{
int docReadRequestCount = 0;
int return403Count = 0;
List<bool> hubHeaderPerRequest = new List<bool>();
List<string> rcsHeaderPerRequest = new List<string>();

HttpClientHandlerHelper httpHandler = new HttpClientHandlerHelper
{
RequestCallBack = (request, cancellationToken) =>
{
if (request.Method == HttpMethod.Get
&& request.RequestUri != null
&& request.RequestUri.AbsolutePath.Contains("/docs/"))
{
docReadRequestCount++;

bool hasHubHeader = request.Headers.TryGetValues(HubRegionHeader, out IEnumerable<string> values)
Comment thread
aavasthy marked this conversation as resolved.
Outdated
&& values.Any();
hubHeaderPerRequest.Add(hasHubHeader);

string rcsValue = null;
if (request.Headers.TryGetValues("x-ms-cosmos-read-consistency-strategy", out IEnumerable<string> rcsValues))
{
rcsValue = rcsValues.FirstOrDefault();
}
rcsHeaderPerRequest.Add(rcsValue);

if (return403Count < forbiddenCount)
{
return403Count++;

HttpResponseMessage forbiddenResponse = new HttpResponseMessage(HttpStatusCode.Forbidden)
{
Content = new StringContent(
JsonConvert.SerializeObject(new { code = "Forbidden", message = "Simulated 403/3 WriteForbidden - not hub region" }),
Encoding.UTF8,
"application/json")
};
forbiddenResponse.Headers.Add("x-ms-substatus", ((int)SubStatusCodes.WriteForbidden).ToString());
forbiddenResponse.Headers.Add("x-ms-activity-id", Guid.NewGuid().ToString());
forbiddenResponse.Headers.Add("x-ms-request-charge", "1.0");

return Task.FromResult(forbiddenResponse);
}
}

return Task.FromResult<HttpResponseMessage>(null);
}
};

CosmosClientOptions clientOptions = new CosmosClientOptions
{
ConnectionMode = ConnectionMode.Gateway,
ConsistencyLevel = Cosmos.ConsistencyLevel.Session,
HttpClientFactory = () => new HttpClient(httpHandler),
};

using CosmosClient customClient = TestCommon.CreateCosmosClient(clientOptions);
Container customContainer = customClient.GetContainer(this.database.Id, this.Container.Id);

ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();
await this.Container.CreateItemAsync(testItem, new Cosmos.PartitionKey(testItem.pk));

ItemResponse<ToDoActivity> response = await customContainer.ReadItemAsync<ToDoActivity>(
testItem.id,
new Cosmos.PartitionKey(testItem.pk),
new ItemRequestOptions { ReadConsistencyStrategy = Cosmos.ReadConsistencyStrategy.LastCommittedWriteRegion });

Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
Assert.IsNotNull(response.Resource);
Assert.AreEqual(testItem.id, response.Resource.id);

Assert.AreEqual(forbiddenCount, return403Count, $"Should have returned 403/3 exactly {forbiddenCount} time(s).");
Assert.IsTrue(docReadRequestCount >= forbiddenCount + 1,
$"Expected at least {forbiddenCount + 1} requests ({forbiddenCount}x 403/3 + 1x success), got {docReadRequestCount}.");

for (int i = 0; i < hubHeaderPerRequest.Count; i++)
{
Assert.IsTrue(hubHeaderPerRequest[i],
$"Hub region header MUST be present on request #{i + 1} for LastCommittedWriteRegion read.");
Assert.AreEqual("LastCommittedWriteRegion", rcsHeaderPerRequest[i],
$"ReadConsistencyStrategy header MUST be 'LastCommittedWriteRegion' on request #{i + 1}.");
}
}

/// <summary>
/// Verifies that when ReadConsistencyStrategy.LastCommittedWriteRegion is set at the
/// CosmosClientOptions level, the hub region header is present on the first read request
/// and the 403/3 retry flow succeeds.
/// </summary>
[TestMethod]
[Owner("aavasthy")]
[Description("Client-level LastCommittedWriteRegion sets hub header on reads with 403/3 then success.")]
public async Task ReadItemAsync_WithLastCommittedWriteRegion_ClientLevel_403_3_ThenSuccess()
{
int docReadRequestCount = 0;
int return403Count = 0;
const int maxReturn403 = 1;
List<bool> hubHeaderPerRequest = new List<bool>();
List<string> rcsHeaderPerRequest = new List<string>();

HttpClientHandlerHelper httpHandler = new HttpClientHandlerHelper
{
RequestCallBack = (request, cancellationToken) =>
{
if (request.Method == HttpMethod.Get
&& request.RequestUri != null
&& request.RequestUri.AbsolutePath.Contains("/docs/"))
{
docReadRequestCount++;

bool hasHubHeader = request.Headers.TryGetValues(HubRegionHeader, out IEnumerable<string> values)
&& values.Any();
hubHeaderPerRequest.Add(hasHubHeader);

string rcsValue = null;
if (request.Headers.TryGetValues("x-ms-cosmos-read-consistency-strategy", out IEnumerable<string> rcsValues))
{
rcsValue = rcsValues.FirstOrDefault();
}
rcsHeaderPerRequest.Add(rcsValue);

if (return403Count < maxReturn403)
{
return403Count++;

HttpResponseMessage forbiddenResponse = new HttpResponseMessage(HttpStatusCode.Forbidden)
{
Content = new StringContent(
JsonConvert.SerializeObject(new { code = "Forbidden", message = "Simulated 403/3" }),
Encoding.UTF8,
"application/json")
};
forbiddenResponse.Headers.Add("x-ms-substatus", ((int)SubStatusCodes.WriteForbidden).ToString());
forbiddenResponse.Headers.Add("x-ms-activity-id", Guid.NewGuid().ToString());
forbiddenResponse.Headers.Add("x-ms-request-charge", "1.0");

return Task.FromResult(forbiddenResponse);
}
}

return Task.FromResult<HttpResponseMessage>(null);
}
};

CosmosClientOptions clientOptions = new CosmosClientOptions
{
ConnectionMode = ConnectionMode.Gateway,
ConsistencyLevel = Cosmos.ConsistencyLevel.Session,
HttpClientFactory = () => new HttpClient(httpHandler),
ReadConsistencyStrategy = Cosmos.ReadConsistencyStrategy.LastCommittedWriteRegion
};

using CosmosClient customClient = TestCommon.CreateCosmosClient(clientOptions);
Container customContainer = customClient.GetContainer(this.database.Id, this.Container.Id);

ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();
await this.Container.CreateItemAsync(testItem, new Cosmos.PartitionKey(testItem.pk));

// Read without per-request options — client-level strategy should take effect
ItemResponse<ToDoActivity> response = await customContainer.ReadItemAsync<ToDoActivity>(
testItem.id,
new Cosmos.PartitionKey(testItem.pk));

Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
Assert.AreEqual(testItem.id, response.Resource.id);
Assert.AreEqual(maxReturn403, return403Count, "Should have returned 403/3 exactly once.");
Assert.IsTrue(docReadRequestCount >= 2, $"Expected at least 2 requests, got {docReadRequestCount}.");

for (int i = 0; i < hubHeaderPerRequest.Count; i++)
{
Assert.IsTrue(hubHeaderPerRequest[i],
$"Hub region header MUST be present on request #{i + 1} for client-level LastCommittedWriteRegion.");
Assert.AreEqual("LastCommittedWriteRegion", rcsHeaderPerRequest[i],
$"ReadConsistencyStrategy header MUST be 'LastCommittedWriteRegion' on request #{i + 1}.");
}
}

/// <summary>
/// Verifies that non-LastCommittedWriteRegion strategies (e.g. Session, Eventual)
/// set the ReadConsistencyStrategy header but do NOT set the hub region header.
/// This is the key behavioral distinction: only LastCommittedWriteRegion triggers
/// hub region routing.
/// </summary>
[TestMethod]
[Owner("aavasthy")]
[DataRow("Session", DisplayName = "Session strategy: RCS header present, no hub header")]
[DataRow("Eventual", DisplayName = "Eventual strategy: RCS header present, no hub header")]
[DataRow("LatestCommitted", DisplayName = "LatestCommitted strategy: RCS header present, no hub header")]
[Description("Non-LastCommittedWriteRegion strategies set RCS header but NOT hub region header.")]
public async Task ReadItemAsync_WithNonLastCommittedWriteRegionStrategy_NoHubHeader(string strategyName)
{
Cosmos.ReadConsistencyStrategy strategy =
(Cosmos.ReadConsistencyStrategy)Enum.Parse(typeof(Cosmos.ReadConsistencyStrategy), strategyName);

bool hubHeaderOnFirstRequest = false;
string rcsHeaderValue = null;
bool interceptedFirstRequest = false;

HttpClientHandlerHelper httpHandler = new HttpClientHandlerHelper
{
RequestCallBack = (request, cancellationToken) =>
{
if (!interceptedFirstRequest
&& request.Method == HttpMethod.Get
&& request.RequestUri != null
&& request.RequestUri.AbsolutePath.Contains("/docs/"))
{
interceptedFirstRequest = true;

hubHeaderOnFirstRequest = request.Headers.TryGetValues(HubRegionHeader, out IEnumerable<string> values)
&& values.Any();

if (request.Headers.TryGetValues("x-ms-cosmos-read-consistency-strategy", out IEnumerable<string> rcsValues))
{
rcsHeaderValue = rcsValues.FirstOrDefault();
}
}

return Task.FromResult<HttpResponseMessage>(null);
}
};

CosmosClientOptions clientOptions = new CosmosClientOptions
{
ConnectionMode = ConnectionMode.Gateway,
ConsistencyLevel = Cosmos.ConsistencyLevel.Session,
HttpClientFactory = () => new HttpClient(httpHandler),
};

using CosmosClient customClient = TestCommon.CreateCosmosClient(clientOptions);
Container customContainer = customClient.GetContainer(this.database.Id, this.Container.Id);

ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();
await this.Container.CreateItemAsync(testItem, new Cosmos.PartitionKey(testItem.pk));

ItemResponse<ToDoActivity> response = await customContainer.ReadItemAsync<ToDoActivity>(
testItem.id,
new Cosmos.PartitionKey(testItem.pk),
new ItemRequestOptions { ReadConsistencyStrategy = strategy });

Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
Assert.IsTrue(interceptedFirstRequest, "Should have intercepted at least one doc read request.");
Assert.IsFalse(hubHeaderOnFirstRequest,
$"Hub region header should NOT be present for {strategyName} strategy — only LastCommittedWriteRegion triggers it.");
Assert.AreEqual(strategyName, rcsHeaderValue,
$"ReadConsistencyStrategy header should be '{strategyName}'.");
}

/// <summary>
/// Verifies that when ReadConsistencyStrategy is NOT set at all, neither the hub header
/// nor the ReadConsistencyStrategy header is present on the request.
/// </summary>
[TestMethod]
[Owner("aavasthy")]
[Description("Without ReadConsistencyStrategy, neither hub header nor RCS header is present.")]
public async Task ReadItemAsync_WithoutReadConsistencyStrategy_NoHeaders()
{
bool hubHeaderOnFirstRequest = false;
bool rcsHeaderOnFirstRequest = false;
bool interceptedFirstRequest = false;

HttpClientHandlerHelper httpHandler = new HttpClientHandlerHelper
{
RequestCallBack = (request, cancellationToken) =>
{
if (!interceptedFirstRequest
&& request.Method == HttpMethod.Get
&& request.RequestUri != null
&& request.RequestUri.AbsolutePath.Contains("/docs/"))
{
interceptedFirstRequest = true;

hubHeaderOnFirstRequest = request.Headers.TryGetValues(HubRegionHeader, out IEnumerable<string> values)
&& values.Any();

rcsHeaderOnFirstRequest = request.Headers.TryGetValues("x-ms-cosmos-read-consistency-strategy", out IEnumerable<string> rcsValues)
&& rcsValues.Any();
}

return Task.FromResult<HttpResponseMessage>(null);
}
};

CosmosClientOptions clientOptions = new CosmosClientOptions
{
ConnectionMode = ConnectionMode.Gateway,
ConsistencyLevel = Cosmos.ConsistencyLevel.Session,
HttpClientFactory = () => new HttpClient(httpHandler),
};

using CosmosClient customClient = TestCommon.CreateCosmosClient(clientOptions);
Container customContainer = customClient.GetContainer(this.database.Id, this.Container.Id);

ToDoActivity testItem = ToDoActivity.CreateRandomToDoActivity();
await this.Container.CreateItemAsync(testItem, new Cosmos.PartitionKey(testItem.pk));

ItemResponse<ToDoActivity> response = await customContainer.ReadItemAsync<ToDoActivity>(
testItem.id,
new Cosmos.PartitionKey(testItem.pk));

Assert.AreEqual(HttpStatusCode.OK, response.StatusCode);
Assert.IsTrue(interceptedFirstRequest, "Should have intercepted at least one doc read request.");
Assert.IsFalse(hubHeaderOnFirstRequest,
"Hub region header should NOT be present when ReadConsistencyStrategy is not set.");
Assert.IsFalse(rcsHeaderOnFirstRequest,
"ReadConsistencyStrategy header should NOT be present when ReadConsistencyStrategy is not set.");
}

private async Task<T> AutoGenerateIdPatternTest<T>(Cosmos.PartitionKey pk, T itemWithoutId)
{
string autoId = Guid.NewGuid().ToString();
Expand Down
Loading
Loading