diff --git a/src/Microsoft.Health.Fhir.Api/AssemblyInfo.cs b/src/Microsoft.Health.Fhir.Api/AssemblyInfo.cs index 5034727b60..2bc932e5f7 100644 --- a/src/Microsoft.Health.Fhir.Api/AssemblyInfo.cs +++ b/src/Microsoft.Health.Fhir.Api/AssemblyInfo.cs @@ -19,3 +19,4 @@ [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Stu3.Api")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Stu3.Api.UnitTests")] [assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Stu3.Client")] +[assembly: InternalsVisibleTo("Microsoft.Health.Fhir.Shared.Api.UnitTests")] diff --git a/src/Microsoft.Health.Fhir.Api/Features/BackgroundJobService/HostingBackgroundService.cs b/src/Microsoft.Health.Fhir.Api/Features/BackgroundJobService/HostingBackgroundService.cs index 45f255eac8..2cca05b170 100644 --- a/src/Microsoft.Health.Fhir.Api/Features/BackgroundJobService/HostingBackgroundService.cs +++ b/src/Microsoft.Health.Fhir.Api/Features/BackgroundJobService/HostingBackgroundService.cs @@ -5,6 +5,8 @@ using System; using System.Collections.Generic; +using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using EnsureThat; @@ -72,10 +74,12 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) await Task.Delay(TimeSpan.FromSeconds(1), cancellationTokenSource.Token); } - foreach (var operation in _operationsConfiguration.HostingBackgroundServiceQueues) + var enabledQueueConfigs = GetEnabledQueueConfigs(); + + foreach (var queueConfig in enabledQueueConfigs) { - short runningJobCount = operation.MaxRunningTaskCount ?? _hostingConfiguration?.MaxRunningTaskCount ?? Constants.DefaultMaxRunningJobCount; - jobQueues.Add(jobHostingValue.ExecuteAsync((byte)operation.Queue, runningJobCount, Environment.MachineName, cancellationTokenSource)); + short runningJobCount = queueConfig.MaxRunningTaskCount ?? _hostingConfiguration?.MaxRunningTaskCount ?? Constants.DefaultMaxRunningJobCount; + jobQueues.Add(jobHostingValue.ExecuteAsync((byte)queueConfig.Queue, runningJobCount, Environment.MachineName, cancellationTokenSource)); } await Task.WhenAll(jobQueues); @@ -95,5 +99,14 @@ public Task Handle(SearchParametersInitializedNotification notification, Cancell _storageReady = true; return Task.CompletedTask; } + + internal virtual IEnumerable GetEnabledQueueConfigs() + { + return _operationsConfiguration.GetType() + .GetProperties(BindingFlags.Instance | BindingFlags.Public) + .Where(p => typeof(HostingBackgroundServiceQueueItem).IsAssignableFrom(p.PropertyType)) + .Select(p => p.GetValue(_operationsConfiguration) as HostingBackgroundServiceQueueItem) + .Where(q => q != null && q.Enabled); + } } } diff --git a/src/Microsoft.Health.Fhir.Api/Features/Operations/Import/InitialImportLockMiddleware.cs b/src/Microsoft.Health.Fhir.Api/Features/Operations/Import/InitialImportLockMiddleware.cs index ed99f2f169..5db8025b3a 100644 --- a/src/Microsoft.Health.Fhir.Api/Features/Operations/Import/InitialImportLockMiddleware.cs +++ b/src/Microsoft.Health.Fhir.Api/Features/Operations/Import/InitialImportLockMiddleware.cs @@ -20,7 +20,7 @@ namespace Microsoft.Health.Fhir.Api.Features.Operations.Import public sealed class InitialImportLockMiddleware { private RequestDelegate _next; - private ImportTaskConfiguration _importTaskConfiguration; + private ImportJobConfiguration _importTaskConfiguration; private readonly HashSet<(string method, string pathRegex)> _excludedEndpoints; private readonly HashSet<(string method, string pathRegex)> _filteredEndpoints; @@ -30,7 +30,7 @@ public sealed class InitialImportLockMiddleware public InitialImportLockMiddleware( RequestDelegate next, - IOptions importTaskConfiguration) + IOptions importTaskConfiguration) { _next = EnsureArg.IsNotNull(next, nameof(next)); _importTaskConfiguration = EnsureArg.IsNotNull(importTaskConfiguration?.Value, nameof(importTaskConfiguration)); diff --git a/src/Microsoft.Health.Fhir.Core/Configs/BulkDeleteJobConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/BulkDeleteJobConfiguration.cs new file mode 100644 index 0000000000..c45c01df94 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core/Configs/BulkDeleteJobConfiguration.cs @@ -0,0 +1,17 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using Microsoft.Health.Fhir.Core.Features.Operations; + +namespace Microsoft.Health.Fhir.Core.Configs +{ + public class BulkDeleteJobConfiguration : HostingBackgroundServiceQueueItem + { + public BulkDeleteJobConfiguration() + { + Queue = QueueType.BulkDelete; + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs index 215005930c..b46070cf6e 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/ExportJobConfiguration.cs @@ -5,15 +5,16 @@ using System; using System.Collections.Generic; +using Microsoft.Health.Fhir.Core.Features.Operations; namespace Microsoft.Health.Fhir.Core.Configs { - public class ExportJobConfiguration + public class ExportJobConfiguration : HostingBackgroundServiceQueueItem { - /// - /// Determines whether export is enabled or not. - /// - public bool Enabled { get; set; } + public ExportJobConfiguration() + { + Queue = QueueType.Export; + } /// /// Determines the storage account connection that will be used to export data to. diff --git a/src/Microsoft.Health.Fhir.Core/Configs/HostingBackgroundServiceQueueItem.cs b/src/Microsoft.Health.Fhir.Core/Configs/HostingBackgroundServiceQueueItem.cs index a3be8815f3..0f28b85bc0 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/HostingBackgroundServiceQueueItem.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/HostingBackgroundServiceQueueItem.cs @@ -9,6 +9,11 @@ namespace Microsoft.Health.Fhir.Core.Configs; public class HostingBackgroundServiceQueueItem { + /// + /// Determines whether job and related queue is enabled or not. + /// + public bool Enabled { get; set; } + /// /// Gets or sets the queue type. /// @@ -18,7 +23,4 @@ public class HostingBackgroundServiceQueueItem /// Gets or sets the max running task count at the same time for this queue type. /// public short? MaxRunningTaskCount { get; set; } - - // TODO: This is not honored. Make sure that it is not used in PaaS and remove. - public bool UpdateProgressOnHeartbeat { get; set; } } diff --git a/src/Microsoft.Health.Fhir.Core/Configs/ImportTaskConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/ImportJobConfiguration.cs similarity index 86% rename from src/Microsoft.Health.Fhir.Core/Configs/ImportTaskConfiguration.cs rename to src/Microsoft.Health.Fhir.Core/Configs/ImportJobConfiguration.cs index fde1af0d93..b1b0f874df 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/ImportTaskConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/ImportJobConfiguration.cs @@ -3,19 +3,21 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- +using Microsoft.Health.Fhir.Core.Features.Operations; + namespace Microsoft.Health.Fhir.Core.Configs { - public class ImportTaskConfiguration // This class name is inconistent with others which use Job instrad of Task. + public class ImportJobConfiguration : HostingBackgroundServiceQueueItem { private const int DefaultTransactionSize = 1000; private const int DefaultSqlIndexRebuildThreads = 3; private const int DefaultInfinitySqlTimeoutSec = 0; private const int DefaultPollingPeriodSec = 60; - /// - /// Determines whether bulk import is enabled or not. - /// - public bool Enabled { get; set; } + public ImportJobConfiguration() + { + Queue = QueueType.Import; + } /// /// Initial import mode diff --git a/src/Microsoft.Health.Fhir.Core/Configs/OperationsConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/OperationsConfiguration.cs index 11b95b41e3..ffd93c7818 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/OperationsConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/OperationsConfiguration.cs @@ -3,14 +3,12 @@ // Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. // ------------------------------------------------------------------------------------------------- -using System.Collections.Generic; +using Microsoft.Health.Fhir.Core.Features.Operations; namespace Microsoft.Health.Fhir.Core.Configs { public class OperationsConfiguration { - public IList HostingBackgroundServiceQueues { get; } = new List(); - public ExportJobConfiguration Export { get; set; } = new ExportJobConfiguration(); public ReindexJobConfiguration Reindex { get; set; } = new ReindexJobConfiguration(); @@ -21,6 +19,8 @@ public class OperationsConfiguration public IntegrationDataStoreConfiguration IntegrationDataStore { get; set; } = new IntegrationDataStoreConfiguration(); - public ImportTaskConfiguration Import { get; set; } = new ImportTaskConfiguration(); + public ImportJobConfiguration Import { get; set; } = new ImportJobConfiguration(); + + public BulkDeleteJobConfiguration BulkDelete { get; set; } = new BulkDeleteJobConfiguration(); } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ImportControllerTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ImportControllerTests.cs index a5b5867170..17993847d0 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ImportControllerTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Controllers/ImportControllerTests.cs @@ -57,7 +57,7 @@ public class ImportControllerTests [MemberData(nameof(ValidBody), MemberType = typeof(ImportControllerTests))] public async Task GivenAnBulkImportRequest_WhenDisabled_ThenRequestNotValidExceptionShouldBeThrown(ImportRequest body) { - var bulkImportController = GetController(new ImportTaskConfiguration() { Enabled = false }); + var bulkImportController = GetController(new ImportJobConfiguration() { Enabled = false }); body.Mode = ImportMode.InitialLoad.ToString(); await Assert.ThrowsAsync(() => bulkImportController.Import(body.ToParameters())); @@ -67,7 +67,7 @@ public async Task GivenAnBulkImportRequest_WhenDisabled_ThenRequestNotValidExcep [MemberData(nameof(InvalidBody), MemberType = typeof(ImportControllerTests))] public async Task GivenAnBulkImportRequest_WhenRequestConfigurationNotValid_ThenRequestNotValidExceptionShouldBeThrown(ImportRequest body) { - var bulkImportController = GetController(new ImportTaskConfiguration() { Enabled = true }); + var bulkImportController = GetController(new ImportJobConfiguration() { Enabled = true }); body.Mode = ImportMode.InitialLoad.ToString(); await Assert.ThrowsAsync(() => bulkImportController.Import(body.ToParameters())); @@ -77,7 +77,7 @@ public async Task GivenAnBulkImportRequest_WhenRequestConfigurationNotValid_Then public async Task GivenAnBulkImportRequest_WhenRequestWithNullParameters_ThenRequestNotValidExceptionShouldBeThrown() { Parameters parameters = null; - var bulkImportController = GetController(new ImportTaskConfiguration() { Enabled = true }); + var bulkImportController = GetController(new ImportJobConfiguration() { Enabled = true }); await Assert.ThrowsAsync(() => bulkImportController.Import(parameters)); } @@ -85,7 +85,7 @@ public async Task GivenAnBulkImportRequest_WhenRequestWithNullParameters_ThenReq public async Task GivenAnBulkImportRequest_WhenRequestWithDuplicateFiles_ThenRequestNotValidExceptionShouldBeThrown() { var requestWithDuplicateUrls = GetDuplicateFileImportRequest(); - var bulkImportController = GetController(new ImportTaskConfiguration() { Enabled = true }); + var bulkImportController = GetController(new ImportJobConfiguration() { Enabled = true }); var controllerException = await Assert.ThrowsAsync( () => bulkImportController.Import(requestWithDuplicateUrls.ToParameters())); @@ -93,7 +93,7 @@ public async Task GivenAnBulkImportRequest_WhenRequestWithDuplicateFiles_ThenReq Assert.Contains(requestWithDuplicateUrls.Input[0].Url.ToString(), controllerException.Message); } - private ImportController GetController(ImportTaskConfiguration bulkImportConfig) + private ImportController GetController(ImportJobConfiguration bulkImportConfig) { var operationConfig = new OperationsConfiguration() { diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/BackgroundJobService/HostingBackgroundServiceTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/BackgroundJobService/HostingBackgroundServiceTests.cs new file mode 100644 index 0000000000..8637a1dcce --- /dev/null +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/BackgroundJobService/HostingBackgroundServiceTests.cs @@ -0,0 +1,57 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Linq; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Microsoft.Health.Extensions.DependencyInjection; +using Microsoft.Health.Fhir.Api.Features.BackgroundJobService; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.JobManagement; +using Microsoft.Health.Test.Utilities; +using NSubstitute; +using Xunit; + +namespace Microsoft.Health.Fhir.Api.UnitTests.Features.BackgroundJobService; + +[Trait(Traits.OwningTeam, OwningTeam.Fhir)] +[Trait(Traits.Category, Categories.Operations)] +public sealed class HostingBackgroundServiceTests +{ + [Fact] + public void GivenOperationsConfigurationWithMixedEnabledQueues_WhenGetEnabledQueueConfigsIsCalled_ThenOnlyEnabledQueuesAreReturned() + { + // Arrange + var exportConfig = new ExportJobConfiguration { Enabled = true }; + var importConfig = new ImportJobConfiguration { Enabled = false }; + var bulkDeleteConfig = new BulkDeleteJobConfiguration { Enabled = true }; + + var operationsConfig = new OperationsConfiguration + { + Export = exportConfig, + Import = importConfig, + BulkDelete = bulkDeleteConfig, + }; + + var jobHostingFactory = Substitute.For>(); + var hostingConfig = Options.Create(new TaskHostingConfiguration()); + var operationsConfigOptions = Options.Create(operationsConfig); + var logger = Substitute.For>(); + + var service = new HostingBackgroundService(jobHostingFactory, hostingConfig, operationsConfigOptions, logger); + + // Act + var enabledQueues = service.GetEnabledQueueConfigs().ToList(); + + // Assert + Assert.Contains(enabledQueues, q => q == exportConfig); + Assert.Contains(enabledQueues, q => q == bulkDeleteConfig); + Assert.DoesNotContain(enabledQueues, q => q == importConfig); + Assert.Equal(2, enabledQueues.Count); + } +} diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Operations/Import/InitialImportLockMiddlewareTests.cs b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Operations/Import/InitialImportLockMiddlewareTests.cs index 19e998ece1..2a11bdf90d 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Operations/Import/InitialImportLockMiddlewareTests.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Features/Operations/Import/InitialImportLockMiddlewareTests.cs @@ -21,7 +21,7 @@ public class InitialImportLockMiddlewareTests [Fact] public async Task GivenPostResourceRequest_WhenInitialImportModeEnabled_Then423ShouldBeReturned() { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = true, InitialImportMode = true }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = true, InitialImportMode = true }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = "/Patient"; httpContext.Request.Method = HttpMethods.Post.ToString(); @@ -33,7 +33,7 @@ public async Task GivenPostResourceRequest_WhenInitialImportModeEnabled_Then423S [Fact] public async Task GivenCustomErrorRequest_WhenInitialImportModeEnabled_Then423ShouldNotBeReturned() { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = true, InitialImportMode = true }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = true, InitialImportMode = true }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = "/CustomError"; httpContext.Request.Method = HttpMethods.Post.ToString(); @@ -45,7 +45,7 @@ public async Task GivenCustomErrorRequest_WhenInitialImportModeEnabled_Then423Sh [Fact] public async Task GivenGetResourceRequest_WhenInitialImportModeEnabled_Then200ShouldBeReturned() { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = false, InitialImportMode = true }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = false, InitialImportMode = true }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = "/patient"; httpContext.Request.Method = HttpMethods.Get.ToString(); @@ -57,7 +57,7 @@ public async Task GivenGetResourceRequest_WhenInitialImportModeEnabled_Then200Sh [Fact] public async Task GivenStartImportRequest_WhenInitialImportModeEnabled_Then200ShouldBeReturned() { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = false, InitialImportMode = true }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = false, InitialImportMode = true }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = "/$import"; httpContext.Request.Method = HttpMethods.Post.ToString(); @@ -69,7 +69,7 @@ public async Task GivenStartImportRequest_WhenInitialImportModeEnabled_Then200Sh [Fact] public async Task GivenImportRequestWithPrefix_WhenInitialImportModeEnabled_Then200ShouldBeReturned() { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = false, InitialImportMode = true }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = false, InitialImportMode = true }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = "/prefix/$import"; httpContext.Request.Method = HttpMethods.Post.ToString(); @@ -81,7 +81,7 @@ public async Task GivenImportRequestWithPrefix_WhenInitialImportModeEnabled_Then [Fact] public async Task GivenCancelImportRequest_WhenInitialImportModeEnabled_Then200ShouldBeReturned() { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = false, InitialImportMode = true }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = false, InitialImportMode = true }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = "/_operations/import/abc"; httpContext.Request.Method = HttpMethods.Delete.ToString(); @@ -93,7 +93,7 @@ public async Task GivenCancelImportRequest_WhenInitialImportModeEnabled_Then200S [Fact] public async Task GivenCancelImportRequestWithPrefix_WhenInitialImportModeEnabled_Then200ShouldBeReturned() { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = false, InitialImportMode = true }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = false, InitialImportMode = true }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = "/prefix/_operations/import/abc"; httpContext.Request.Method = HttpMethods.Delete.ToString(); @@ -105,7 +105,7 @@ public async Task GivenCancelImportRequestWithPrefix_WhenInitialImportModeEnable [Fact] public async Task GivenPostResourceRequest_WhenImportNotEnabled_Then200ShouldBeReturned() { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = false, InitialImportMode = true }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = false, InitialImportMode = true }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = "/Patient"; httpContext.Request.Method = HttpMethods.Post.ToString(); @@ -117,7 +117,7 @@ public async Task GivenPostResourceRequest_WhenImportNotEnabled_Then200ShouldBeR [Fact] public async Task GivenPostResourceRequest_WhenInitialImportModeNotEnabled_Then200ShouldBeReturned() { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = true, InitialImportMode = false }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = true, InitialImportMode = false }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = "/Patient"; httpContext.Request.Method = HttpMethods.Post.ToString(); @@ -132,7 +132,7 @@ public async Task GivenPostResourceRequest_WhenInitialImportModeNotEnabled_Then2 [InlineData("/Observation", "Delete")] public async Task GivenLockedRequests_WhenInitialImportModeEnabled_Then423ShouldBeReturned(string path, string method) { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = true, InitialImportMode = true }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = true, InitialImportMode = true }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = path; httpContext.Request.Method = method; @@ -153,7 +153,7 @@ public async Task GivenLockedRequests_WhenInitialImportModeEnabled_Then423Should [InlineData("/_operations/import/123", "Delete")] public async Task GivenAllowedRequests_WhenInitialImportModeEnabled_Then200ShouldBeReturned(string path, string method) { - InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportTaskConfiguration() { Enabled = true, InitialImportMode = true }); + InitialImportLockMiddleware middleware = CreateInitialImportLockMiddleware(new ImportJobConfiguration() { Enabled = true, InitialImportMode = true }); HttpContext httpContext = new DefaultHttpContext(); httpContext.Request.Path = path; httpContext.Request.Method = method; @@ -162,7 +162,7 @@ public async Task GivenAllowedRequests_WhenInitialImportModeEnabled_Then200Shoul Assert.Equal(200, httpContext.Response.StatusCode); } - private InitialImportLockMiddleware CreateInitialImportLockMiddleware(ImportTaskConfiguration importJobConfiguration) + private InitialImportLockMiddleware CreateInitialImportLockMiddleware(ImportJobConfiguration importJobConfiguration) { return new InitialImportLockMiddleware( async x => diff --git a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems index 44439bcb44..53898670f0 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems +++ b/src/Microsoft.Health.Fhir.Shared.Api.UnitTests/Microsoft.Health.Fhir.Shared.Api.UnitTests.projitems @@ -27,6 +27,7 @@ + @@ -82,4 +83,7 @@ Never + + + \ No newline at end of file diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ImportController.cs b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ImportController.cs index da3fdcf8f2..bf7e2e3f31 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ImportController.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Controllers/ImportController.cs @@ -57,7 +57,7 @@ public class ImportController : Controller private readonly IUrlResolver _urlResolver; private readonly FeatureConfiguration _features; private readonly ILogger _logger; - private readonly ImportTaskConfiguration _importConfig; + private readonly ImportJobConfiguration _importConfig; public ImportController( IMediator mediator, diff --git a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json index ad45c140e7..ae6ae6a52a 100644 --- a/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json +++ b/src/Microsoft.Health.Fhir.Shared.Web/appsettings.json @@ -49,20 +49,6 @@ "AllowCredentials": false }, "Operations": { - "HostingBackgroundServiceQueues": [ - { - "Queue": "Import", - "UpdateProgressOnHeartbeat": true - }, - { - "Queue": "Export", - "UpdateProgressOnHeartbeat": false - }, - { - "Queue": "BulkDelete", - "UpdateProgressOnHeartbeat": false - } - ], "Export": { "Enabled": true, "StorageAccountConnection": null, @@ -104,7 +90,10 @@ "CacheDurationInSeconds": 14400 }, "Import": { - "Enabled": false + "Enabled": false, + }, + "BulkDelete": { + "Enabled": true, }, "IntegrationDataStore": { "StorageAccountConnection": null, diff --git a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportOrchestratorJobTests.cs b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportOrchestratorJobTests.cs index c8702c2e4e..93e6dd1d35 100644 --- a/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportOrchestratorJobTests.cs +++ b/src/Microsoft.Health.Fhir.SqlServer.UnitTests/Features/Operations/Import/ImportOrchestratorJobTests.cs @@ -74,7 +74,7 @@ public async Task GivenAnOrchestratorJobAndWrongEtag_WhenOrchestratorJobStart_Th contextAccessor, integrationDataStoreClient, testQueueClient, - Options.Create(new ImportTaskConfiguration()), + Options.Create(new ImportJobConfiguration()), loggerFactory, auditLogger); @@ -167,7 +167,7 @@ public async Task GivenAnOrchestratorJob_WhenIntegrationExceptionThrown_ThenJobS contextAccessor, integrationDataStoreClient, testQueueClient, - Options.Create(new ImportTaskConfiguration()), + Options.Create(new ImportJobConfiguration()), loggerFactory, auditLogger); diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Import/ImportOrchestratorJob.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Import/ImportOrchestratorJob.cs index 7c5af083bd..b748251dcc 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Import/ImportOrchestratorJob.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Import/ImportOrchestratorJob.cs @@ -45,7 +45,7 @@ public class ImportOrchestratorJob : IJob private readonly IMediator _mediator; private readonly RequestContextAccessor _contextAccessor; private readonly IQueueClient _queueClient; - private ImportTaskConfiguration _importConfiguration; + private ImportJobConfiguration _importConfiguration; private ILogger _logger; private IIntegrationDataStoreClient _integrationDataStoreClient; private readonly IAuditLogger _auditLogger; @@ -59,7 +59,7 @@ public ImportOrchestratorJob( RequestContextAccessor contextAccessor, IIntegrationDataStoreClient integrationDataStoreClient, IQueueClient queueClient, - IOptions importConfiguration, + IOptions importConfiguration, ILoggerFactory loggerFactory, IAuditLogger auditLogger) { diff --git a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Import/SqlImporter.cs b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Import/SqlImporter.cs index 2e2c5a3489..71f709a8b2 100644 --- a/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Import/SqlImporter.cs +++ b/src/Microsoft.Health.Fhir.SqlServer/Features/Operations/Import/SqlImporter.cs @@ -26,7 +26,7 @@ internal class SqlImporter : IImporter { private readonly SqlServerFhirDataStore _store; private readonly SqlServerFhirModel _model; - private readonly ImportTaskConfiguration _importTaskConfiguration; + private readonly ImportJobConfiguration _importTaskConfiguration; private readonly ILogger _logger; public SqlImporter( diff --git a/src/Microsoft.Health.TaskManagement/JobHosting.cs b/src/Microsoft.Health.TaskManagement/JobHosting.cs index ad4ee906ec..b091af5c0c 100644 --- a/src/Microsoft.Health.TaskManagement/JobHosting.cs +++ b/src/Microsoft.Health.TaskManagement/JobHosting.cs @@ -61,7 +61,7 @@ public async Task ExecuteAsync(byte queueType, short runningJobCount, string wor { try { - _logger.LogInformation("Dequeuing next job."); + _logger.LogInformation("Dequeuing next job for {QueueType}.", queueType); if (checkTimeoutJobStopwatch.Elapsed.TotalSeconds > 600) { @@ -73,7 +73,7 @@ public async Task ExecuteAsync(byte queueType, short runningJobCount, string wor } catch (Exception ex) { - _logger.LogError(ex, "Failed to dequeue new job."); + _logger.LogError(ex, "Failed to dequeue new job for {QueueType}.", queueType); } } @@ -102,12 +102,12 @@ public async Task ExecuteAsync(byte queueType, short runningJobCount, string wor { try { - _logger.LogInformation("Empty queue. Delaying until next iteration."); + _logger.LogInformation("Empty queue {QueueType}. Delaying until next iteration.", queueType); await Task.Delay(TimeSpan.FromSeconds(PollingFrequencyInSeconds), cancellationTokenSource.Token); } catch (TaskCanceledException) { - _logger.LogInformation("Queue is stopping, worker is shutting down."); + _logger.LogInformation("Queue {QueueType} is stopping, worker is shutting down.", queueType); } } }