diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Reindex/ReindexJobWorkerTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Reindex/ReindexJobWorkerTests.cs index 8a79d4f5bf..ec6d4649c1 100644 --- a/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Reindex/ReindexJobWorkerTests.cs +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Features/Operations/Reindex/ReindexJobWorkerTests.cs @@ -9,8 +9,10 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Options; +using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Operations; using Microsoft.Health.Fhir.Core.Features.Operations.Reindex; using Microsoft.Health.Fhir.Core.Features.Operations.Reindex.Models; @@ -59,6 +61,7 @@ public ReindexJobWorkerTests() Options.Create(_reindexJobConfiguration), _reindexJobTask.CreateMockScopeProvider(), searchParameterOperations, + Substitute.For>(), NullLogger.Instance); _reindexJobWorker.Handle(new Messages.Search.SearchParametersInitializedNotification(), CancellationToken.None); diff --git a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexJobWorker.cs b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexJobWorker.cs index 9f46990f61..fc053624b9 100644 --- a/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexJobWorker.cs +++ b/src/Microsoft.Health.Fhir.Core/Features/Operations/Reindex/ReindexJobWorker.cs @@ -12,8 +12,11 @@ using MediatR; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; +using Microsoft.Extensions.Primitives; +using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Extensions.DependencyInjection; using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Context; using Microsoft.Health.Fhir.Core.Features.Operations.Reindex.Models; using Microsoft.Health.Fhir.Core.Features.Search.Parameters; using Microsoft.Health.Fhir.Core.Messages.Search; @@ -29,6 +32,7 @@ public class ReindexJobWorker : INotificationHandler _reindexJobTaskFactory; private readonly ISearchParameterOperations _searchParameterOperations; + private readonly RequestContextAccessor _contextAccessor; private readonly ILogger _logger; private bool _searchParametersInitialized = false; @@ -37,19 +41,15 @@ public ReindexJobWorker( IOptions reindexJobConfiguration, IScopeProvider reindexJobTaskFactory, ISearchParameterOperations searchParameterOperations, + RequestContextAccessor contextAccessor, ILogger logger) { - EnsureArg.IsNotNull(fhirOperationDataStoreFactory, nameof(fhirOperationDataStoreFactory)); - EnsureArg.IsNotNull(reindexJobConfiguration?.Value, nameof(reindexJobConfiguration)); - EnsureArg.IsNotNull(reindexJobTaskFactory, nameof(reindexJobTaskFactory)); - EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); - EnsureArg.IsNotNull(logger, nameof(logger)); - - _fhirOperationDataStoreFactory = fhirOperationDataStoreFactory; - _reindexJobConfiguration = reindexJobConfiguration.Value; - _reindexJobTaskFactory = reindexJobTaskFactory; - _searchParameterOperations = searchParameterOperations; - _logger = logger; + _fhirOperationDataStoreFactory = EnsureArg.IsNotNull(fhirOperationDataStoreFactory, nameof(fhirOperationDataStoreFactory)); + _reindexJobConfiguration = EnsureArg.IsNotNull(reindexJobConfiguration?.Value, nameof(reindexJobConfiguration)); + _reindexJobTaskFactory = EnsureArg.IsNotNull(reindexJobTaskFactory, nameof(reindexJobTaskFactory)); + _searchParameterOperations = EnsureArg.IsNotNull(searchParameterOperations, nameof(searchParameterOperations)); + _contextAccessor = EnsureArg.IsNotNull(contextAccessor, nameof(contextAccessor)); + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); } public async Task ExecuteAsync(CancellationToken cancellationToken) @@ -60,6 +60,22 @@ public async Task ExecuteAsync(CancellationToken cancellationToken) { if (_searchParametersInitialized) { + var originalRequestContext = _contextAccessor.RequestContext; + + // Create a background task context to trigger the correct retry policy. + var fhirRequestContext = new FhirRequestContext( + method: nameof(ReindexJobWorker), + uriString: nameof(ReindexJobWorker), + baseUriString: nameof(ReindexJobWorker), + correlationId: Guid.NewGuid().ToString(), + requestHeaders: new Dictionary(), + responseHeaders: new Dictionary()) + { + IsBackgroundTask = true, + }; + + _contextAccessor.RequestContext = fhirRequestContext; + // Check for any changes to Search Parameters try { @@ -124,6 +140,8 @@ public async Task ExecuteAsync(CancellationToken cancellationToken) // The job failed. _logger.LogError(ex, "Error polling Reindex jobs."); } + + _contextAccessor.RequestContext = originalRequestContext; } try diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs index 5958d6ee22..d8de268713 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/CosmosFhirDataStoreTests.cs @@ -76,7 +76,7 @@ public CosmosFhirDataStoreTests() _cosmosDataStoreConfiguration, Substitute.For>(), _cosmosQueryFactory, - new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration, requestContextAccessor), + new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration, requestContextAccessor, NullLogger.Instance), NullLogger.Instance, Options.Create(new CoreFeatureConfiguration()), _bundleOrchestrator, diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/FhirCosmosClientInitializerTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/FhirCosmosClientInitializerTests.cs index 503b8af40b..0364c5b1c4 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/FhirCosmosClientInitializerTests.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/FhirCosmosClientInitializerTests.cs @@ -40,7 +40,7 @@ public FhirCosmosClientInitializerTests() _initializer = new FhirCosmosClientInitializer( clientTestProvider, () => new[] { new TestRequestHandler() }, - new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration, Substitute.For>()), + new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration, Substitute.For>(), NullLogger.Instance), Substitute.For(), NullLogger.Instance); diff --git a/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Queues/CosmosQueueClientTests.cs b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Queues/CosmosQueueClientTests.cs new file mode 100644 index 0000000000..51a0822d39 --- /dev/null +++ b/src/Microsoft.Health.Fhir.CosmosDb.UnitTests/Features/Storage/Queues/CosmosQueueClientTests.cs @@ -0,0 +1,140 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Diagnostics; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Health.Abstractions.Exceptions; +using Microsoft.Health.Core.Features.Context; +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Core.Features.Context; +using Microsoft.Health.Fhir.CosmosDb.Core.Configs; +using Microsoft.Health.Fhir.CosmosDb.Core.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Features.Queries; +using Microsoft.Health.Fhir.CosmosDb.Features.Storage; +using Microsoft.Health.Fhir.CosmosDb.Features.Storage.Queues; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.CosmosDb.UnitTests.Features.Storage.Queues; + +[Trait(Traits.OwningTeam, OwningTeam.Fhir)] +[Trait(Traits.Category, Categories.DataSourceValidation)] +public class CosmosQueueClientTests +{ + private readonly ICosmosQueryFactory _cosmosQueryFactory; + private readonly ICosmosDbDistributedLockFactory _distributedLockFactory; + private readonly CosmosDataStoreConfiguration _cosmosDataStoreConfiguration = new CosmosDataStoreConfiguration(); + private readonly RequestContextAccessor _requestContextAccessor; + private readonly RetryExceptionPolicyFactory _retryPolicyFactory; + private readonly CosmosQueueClient _cosmosQueueClient; + + public CosmosQueueClientTests() + { + _cosmosQueryFactory = Substitute.For(); + _distributedLockFactory = Substitute.For(); + _requestContextAccessor = Substitute.For>(); + _retryPolicyFactory = new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration, _requestContextAccessor, NullLogger.Instance); + + _cosmosQueueClient = new CosmosQueueClient( + Substitute.For>>(), + _cosmosQueryFactory, + _distributedLockFactory, + _retryPolicyFactory); + } + + [Theory] + [InlineData(HttpStatusCode.ServiceUnavailable)] + [InlineData(HttpStatusCode.TooManyRequests)] + [InlineData(HttpStatusCode.Gone)] + [InlineData((HttpStatusCode)449)] + [InlineData(HttpStatusCode.RequestTimeout)] + public async Task GivenADequeueJobOperation_WhenExceptionOccurs_RetryWillHappen(HttpStatusCode statusCode) + { + // Arrange + ICosmosQuery cosmosQuery = Substitute.For>(); + _cosmosQueryFactory.Create(Arg.Any(), Arg.Any()) + .ReturnsForAnyArgs(cosmosQuery); + + int callCount = 0; + cosmosQuery.ExecuteNextAsync(Arg.Any()).ReturnsForAnyArgs(_ => + { + if (callCount++ == 0) + { + throw new TestCosmosException(statusCode); + } + + return Task.FromResult(Substitute.For>()); + }); + + // Act + await _cosmosQueueClient.DequeueAsync(0, "testworker", 10, CancellationToken.None); + + // Assert + Assert.Equal(2, callCount); + await cosmosQuery.ReceivedWithAnyArgs(2).ExecuteNextAsync(Arg.Any()); + } + + [Theory] + [InlineData(typeof(CosmosException))] + [InlineData(typeof(RequestRateExceededException))] + public async Task GivenADequeueJobOperation_WhenExceptionWithRetryAfterIsProvided_PolicyRespectsRetryAfter(Type exceptionType) + { + // Arrange + ICosmosQuery cosmosQuery = Substitute.For>(); + _cosmosQueryFactory.Create(Arg.Any(), Arg.Any()) + .ReturnsForAnyArgs(cosmosQuery); + var retryAfter = TimeSpan.FromSeconds(2); + int callCount = 0; + + cosmosQuery.ExecuteNextAsync(Arg.Any()).ReturnsForAnyArgs(_ => + { + if (callCount++ == 0) + { + throw exceptionType == typeof(CosmosException) + ? new TestCosmosException(HttpStatusCode.TooManyRequests, retryAfter) + : new RequestRateExceededException(retryAfter); + } + + return Task.FromResult(Substitute.For>()); + }); + + var stopwatch = Stopwatch.StartNew(); + + // Act + await _cosmosQueueClient.DequeueAsync(0, "testworker", 10, CancellationToken.None); + + stopwatch.Stop(); + + // Assert + Assert.Equal(2, callCount); + await cosmosQuery.ReceivedWithAnyArgs(2).ExecuteNextAsync(Arg.Any()); + + // Allowing small imprecision due to timer resolution + var actualElapsedSeconds = stopwatch.Elapsed.TotalSeconds; + Assert.True( + Math.Abs(actualElapsedSeconds - retryAfter.TotalSeconds) <= 0.5, + $"Expected retry after {retryAfter.TotalSeconds} seconds, but actual elapsed time was {actualElapsedSeconds} seconds."); + } + + public class TestCosmosException : CosmosException + { + private readonly TimeSpan? _retryAfter; + + public TestCosmosException(HttpStatusCode statusCode, TimeSpan? retryAfter = null) + : base("Test exception message", statusCode, 0, "test-activity-id", 0.0) + { + _retryAfter = retryAfter; + } + + public override TimeSpan? RetryAfter => _retryAfter; + } +} diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/CosmosQueueClient.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/CosmosQueueClient.cs index 379b8168a0..725c7ed0f6 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/CosmosQueueClient.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/Queues/CosmosQueueClient.cs @@ -12,6 +12,7 @@ using System.Threading.Tasks; using EnsureThat; using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.Logging; using Microsoft.Health.Abstractions.Exceptions; using Microsoft.Health.Core; using Microsoft.Health.Core.Extensions; @@ -29,20 +30,21 @@ public class CosmosQueueClient : IQueueClient private readonly Func> _containerFactory; private readonly ICosmosQueryFactory _queryFactory; private readonly ICosmosDbDistributedLockFactory _distributedLockFactory; - private static readonly AsyncPolicy _retryPolicy = Policy - .Handle(ex => ex.StatusCode == HttpStatusCode.PreconditionFailed) - .Or(ex => ex.StatusCode == HttpStatusCode.TooManyRequests) - .Or() - .WaitAndRetryAsync(5, _ => TimeSpan.FromMilliseconds(RandomNumberGenerator.GetInt32(100, 1000))); + private readonly RetryExceptionPolicyFactory _retryExceptionPolicyFactory; + private readonly AsyncPolicy _retryPolicy; public CosmosQueueClient( Func> containerFactory, ICosmosQueryFactory queryFactory, - ICosmosDbDistributedLockFactory distributedLockFactory) + ICosmosDbDistributedLockFactory distributedLockFactory, + RetryExceptionPolicyFactory retryExceptionPolicyFactory) { _containerFactory = EnsureArg.IsNotNull(containerFactory, nameof(containerFactory)); _queryFactory = EnsureArg.IsNotNull(queryFactory, nameof(queryFactory)); _distributedLockFactory = EnsureArg.IsNotNull(distributedLockFactory, nameof(distributedLockFactory)); + _retryExceptionPolicyFactory = EnsureArg.IsNotNull(retryExceptionPolicyFactory, nameof(retryExceptionPolicyFactory)); + + _retryPolicy = _retryExceptionPolicyFactory.BackgroundWorkerRetryPolicy; } public bool IsInitialized() => true; diff --git a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/RetryExceptionPolicyFactory.cs b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/RetryExceptionPolicyFactory.cs index 2850384550..bde69df1d9 100644 --- a/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/RetryExceptionPolicyFactory.cs +++ b/src/Microsoft.Health.Fhir.CosmosDb/Features/Storage/RetryExceptionPolicyFactory.cs @@ -4,10 +4,13 @@ // ------------------------------------------------------------------------------------------------- using System; +using System.Net; using System.Runtime.ExceptionServices; +using System.Security.Cryptography; using System.Threading.Tasks; using EnsureThat; using Microsoft.Azure.Cosmos; +using Microsoft.Extensions.Logging; using Microsoft.Health.Abstractions.Exceptions; using Microsoft.Health.Core.Features.Context; using Microsoft.Health.Fhir.Core.Extensions; @@ -22,15 +25,23 @@ public class RetryExceptionPolicyFactory { private const string RetryEndTimeContextKey = "RetryEndTime"; private readonly RequestContextAccessor _requestContextAccessor; + private readonly ILogger _logger; private readonly AsyncPolicy _sdkOnlyRetryPolicy; private readonly AsyncPolicy _bundleActionRetryPolicy; private readonly AsyncPolicy _backgroundJobRetryPolicy; - public RetryExceptionPolicyFactory(CosmosDataStoreConfiguration configuration, RequestContextAccessor requestContextAccessor) + private const int _exponentialBackoffBaseDelayMs = 100; + private const int _exponentialMaxJitterMs = 300; + private const int _exponentialMaxDelayMs = 60 * 1000; + + private const int _nonExponentialBaseDelayMs = 500; + private const int _nonExponentialMaxJitterMs = 300; + + public RetryExceptionPolicyFactory(CosmosDataStoreConfiguration configuration, RequestContextAccessor requestContextAccessor, ILogger logger) { - _requestContextAccessor = requestContextAccessor; + _requestContextAccessor = EnsureArg.IsNotNull(requestContextAccessor, nameof(requestContextAccessor)); EnsureArg.IsNotNull(configuration, nameof(configuration)); - EnsureArg.IsNotNull(requestContextAccessor, nameof(requestContextAccessor)); + _logger = EnsureArg.IsNotNull(logger, nameof(logger)); _sdkOnlyRetryPolicy = Policy.NoOpAsync(); @@ -38,7 +49,7 @@ public RetryExceptionPolicyFactory(CosmosDataStoreConfiguration configuration, R ? CreateExtendedRetryPolicy(configuration.IndividualBatchActionRetryOptions.MaxNumberOfRetries / configuration.RetryOptions.MaxNumberOfRetries, configuration.IndividualBatchActionRetryOptions.MaxWaitTimeInSeconds) : Policy.NoOpAsync(); - _backgroundJobRetryPolicy = CreateExtendedRetryPolicy(100, -1); + _backgroundJobRetryPolicy = CreateExtendedRetryPolicy(30, -1, true); } public AsyncPolicy RetryPolicy @@ -54,16 +65,61 @@ public AsyncPolicy RetryPolicy } } - private static AsyncRetryPolicy CreateExtendedRetryPolicy(int maxRetries, int maxWaitTimeInSeconds) + public AsyncPolicy BackgroundWorkerRetryPolicy => _backgroundJobRetryPolicy; + + private AsyncRetryPolicy CreateExtendedRetryPolicy(int maxRetries, int maxWaitTimeInSeconds, bool useExponentialRetry = false) { + // Define a sleep duration provider based on the retry strategy + TimeSpan SleepDurationProvider(int retryAttempt, Exception exception) + { + // Respect x-ms-retry-after-ms from RequestRateExceededException + if (exception.AsRequestRateExceeded()?.RetryAfter is TimeSpan retryAfter) + { + return retryAfter; + } + + // Respect x-ms-retry-after-ms from CosmosException + if (exception is CosmosException cosmosException && cosmosException.StatusCode == HttpStatusCode.TooManyRequests && cosmosException.RetryAfter.HasValue) + { + return cosmosException.RetryAfter.Value; + } + + // Exponential backoff is used for background jobs. Given current values, exponential backoff is used for the first 10 retries. After that a fixed wait time of_exponentialMaxDelayMs (60 seconds) is used. + // Jitter is multiplied by the retry attempt to increase the randomness of the retry interval for longer retry delays (especially retry > 10). + if (useExponentialRetry) + { + // Calculate exponential backoff with a cap of 60 seconds + var backoff = Math.Min(Math.Pow(2, retryAttempt) * _exponentialBackoffBaseDelayMs, _exponentialMaxDelayMs); + + var jitter = RandomNumberGenerator.GetInt32(0, _exponentialMaxJitterMs) * retryAttempt; // Add jitter in milliseconds + return TimeSpan.FromMilliseconds(backoff + jitter); + } + + // Default logic: 500ms + retryAttempt * jitter + var defaultJitter = RandomNumberGenerator.GetInt32(0, _nonExponentialMaxJitterMs) * retryAttempt; // Jitter scaled by retry attempt + return TimeSpan.FromMilliseconds(_nonExponentialBaseDelayMs + defaultJitter); + } + + // Retry recommendations for Cosmos DB: https://learn.microsoft.com/azure/cosmos-db/nosql/conceptual-resilient-sdk-applications#should-my-application-retry-on-errors return Policy.Handle() .Or(e => e.IsRequestRateExceeded()) - .Or(e => (e.StatusCode == System.Net.HttpStatusCode.ServiceUnavailable || e.StatusCode == System.Net.HttpStatusCode.RequestTimeout)) + .Or(e => + e.StatusCode == HttpStatusCode.ServiceUnavailable || + e.StatusCode == HttpStatusCode.TooManyRequests || + e.StatusCode == HttpStatusCode.Gone || + e.StatusCode == (HttpStatusCode)449 || // "Retry with" status code + e.StatusCode == HttpStatusCode.RequestTimeout) .WaitAndRetryAsync( retryCount: maxRetries, - sleepDurationProvider: (_, e, _) => e.AsRequestRateExceeded()?.RetryAfter ?? TimeSpan.FromSeconds(2), + sleepDurationProvider: (retryAttempt, exception, context) => SleepDurationProvider(retryAttempt, exception), onRetryAsync: (e, _, _, ctx) => { + if (e is CosmosException cosmosException && cosmosException.StatusCode == HttpStatusCode.ServiceUnavailable) + { + var diagnostics = cosmosException.Diagnostics?.ToString() ?? "empty"; + _logger.LogWarning(cosmosException, "Received a ServiceUnavailable response from Cosmos DB. Retrying. Diagnostics: {CosmosDiagnostics}", diagnostics); + } + if (maxWaitTimeInSeconds == -1) { // no timeout diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs index 7e710831cf..e7f150caec 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Features/Operations/Reindex/ReindexJobTests.cs @@ -899,6 +899,7 @@ private async Task SetUpForReindexing(CreateReindexReques Options.Create(_jobConfiguration), InitializeReindexJobTask().CreateMockScopeProvider(), _searchParameterOperations, + Substitute.For>(), NullLogger.Instance); await _reindexJobWorker.Handle(new SearchParametersInitializedNotification(), CancellationToken.None); diff --git a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs index 8cd2204e52..aa7256ace8 100644 --- a/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs +++ b/test/Microsoft.Health.Fhir.Shared.Tests.Integration/Persistence/CosmosDbFhirStorageTestsFixture.cs @@ -147,7 +147,7 @@ public virtual async Task InitializeAsync() var responseProcessor = new CosmosResponseProcessor(_fhirRequestContextAccessor, mediator, Substitute.For(), NullLogger.Instance); var handler = new FhirCosmosResponseHandler(() => new NonDisposingScope(_container), _cosmosDataStoreConfiguration, _fhirRequestContextAccessor, responseProcessor); - var retryExceptionPolicyFactory = new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration, _fhirRequestContextAccessor); + var retryExceptionPolicyFactory = new RetryExceptionPolicyFactory(_cosmosDataStoreConfiguration, _fhirRequestContextAccessor, NullLogger.Instance); var documentClientInitializer = new FhirCosmosClientInitializer( testProvider, () => new[] { handler }, @@ -232,7 +232,8 @@ public virtual async Task InitializeAsync() _queueClient = new CosmosQueueClient( () => _container.CreateMockScope(), new CosmosQueryFactory(Substitute.For(), Substitute.For()), - new CosmosDbDistributedLockFactory(() => _container.CreateMockScope(), NullLogger.Instance)); + new CosmosDbDistributedLockFactory(() => _container.CreateMockScope(), NullLogger.Instance), + retryExceptionPolicyFactory); _cosmosFhirOperationDataStore = new CosmosFhirOperationDataStore( _queueClient,