diff --git a/src/Microsoft.Health.Fhir.Core.UnitTests/Config/OperationsConfigurationTests.cs b/src/Microsoft.Health.Fhir.Core.UnitTests/Config/OperationsConfigurationTests.cs new file mode 100644 index 0000000000..a34dc87e31 --- /dev/null +++ b/src/Microsoft.Health.Fhir.Core.UnitTests/Config/OperationsConfigurationTests.cs @@ -0,0 +1,170 @@ +// ------------------------------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License (MIT). See LICENSE in the repo root for license information. +// ------------------------------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using Microsoft.Health.Fhir.Core.Configs; +using Microsoft.Health.Fhir.Core.Features.Operations; +using Microsoft.Health.Fhir.Tests.Common; +using Microsoft.Health.Test.Utilities; +using Xunit; + +namespace Microsoft.Health.Fhir.Core.UnitTests.Config +{ + [Trait(Traits.OwningTeam, OwningTeam.Fhir)] + [Trait(Traits.Category, Categories.Operations)] + public sealed class OperationsConfigurationTests + { + [Fact] + public void GivenAnOperationsConfiguration_WhenQueuesAreDisabled_RemoveThemFromTheList() + { + // Arrange + var operationsConfig = new OperationsConfiguration + { + Export = new ExportJobConfiguration { Enabled = false }, + Import = new ImportTaskConfiguration { Enabled = false }, + }; + + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Export, MaxRunningTaskCount = 2 }); + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Import, MaxRunningTaskCount = 2 }); + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.BulkDelete, MaxRunningTaskCount = 1 }); + + // Act + operationsConfig.RemoveDisabledQueues(); + + // Assert + Assert.DoesNotContain(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Export); + Assert.DoesNotContain(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Import); + Assert.Contains(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.BulkDelete); + } + + [Fact] + public void GivenAnOperationsConfiguration_WhenQueuesAreEnabledWithConnectionString_KeepThemInTheList() + { + // Arrange + var operationsConfig = new OperationsConfiguration + { + Export = new ExportJobConfiguration { Enabled = true, StorageAccountConnection = "test-connection-string" }, + Import = new ImportTaskConfiguration { Enabled = true }, + IntegrationDataStore = new IntegrationDataStoreConfiguration { StorageAccountConnection = "test-connection-string" }, + }; + + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Export, MaxRunningTaskCount = 2 }); + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Import, MaxRunningTaskCount = 2 }); + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.BulkDelete, MaxRunningTaskCount = 1 }); + + // Act + operationsConfig.RemoveDisabledQueues(); + + // Assert + Assert.Contains(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Export); + Assert.Contains(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Import); + Assert.Contains(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.BulkDelete); + } + + [Fact] + public void GivenAnOperationsConfiguration_WhenQueuesAreEnabledWithUri_KeepThemInTheList() + { + // Arrange + var operationsConfig = new OperationsConfiguration + { + Export = new ExportJobConfiguration { Enabled = true, StorageAccountUri = "https://test-account.blob.core.windows.net" }, + Import = new ImportTaskConfiguration { Enabled = true }, + IntegrationDataStore = new IntegrationDataStoreConfiguration { StorageAccountUri = "https://test-account.blob.core.windows.net" }, + }; + + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Export, MaxRunningTaskCount = 2 }); + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Import, MaxRunningTaskCount = 2 }); + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.BulkDelete, MaxRunningTaskCount = 1 }); + + // Act + operationsConfig.RemoveDisabledQueues(); + + // Assert + Assert.Contains(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Export); + Assert.Contains(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Import); + Assert.Contains(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.BulkDelete); + } + + [Fact] + public void GivenAnOperationsConfiguration_WhenNoQueuesExist_NoExceptionIsThrown() + { + // Arrange + var operationsConfig = new OperationsConfiguration + { + Export = new ExportJobConfiguration { Enabled = false }, + Import = new ImportTaskConfiguration { Enabled = false }, + }; + + // Act & Assert + var exception = Record.Exception(() => operationsConfig.RemoveDisabledQueues()); + Assert.Null(exception); + } + + [Fact] + public void GivenAnOperationsConfiguration_WhenMixedQueuesAreEnabledOrDisabled_HandleCorrectly() + { + // Arrange + var operationsConfigWithExport = new OperationsConfiguration + { + Export = new ExportJobConfiguration { Enabled = true, StorageAccountUri = "https://test-account.blob.core.windows.net" }, + Import = new ImportTaskConfiguration { Enabled = false }, + }; + + var operationsConfigWithImport = new OperationsConfiguration + { + Export = new ExportJobConfiguration { Enabled = false }, + Import = new ImportTaskConfiguration { Enabled = true}, + IntegrationDataStore = new IntegrationDataStoreConfiguration { StorageAccountUri = "https://test-account.blob.core.windows.net" }, + }; + + operationsConfigWithExport.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Export, MaxRunningTaskCount = 2 }); + operationsConfigWithExport.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Import, MaxRunningTaskCount = 2 }); + operationsConfigWithExport.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.BulkDelete, MaxRunningTaskCount = 1 }); + + operationsConfigWithImport.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Export, MaxRunningTaskCount = 2 }); + operationsConfigWithImport.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Import, MaxRunningTaskCount = 2 }); + operationsConfigWithImport.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.BulkDelete, MaxRunningTaskCount = 1 }); + + // Act + operationsConfigWithExport.RemoveDisabledQueues(); + + operationsConfigWithImport.RemoveDisabledQueues(); + + // Assert + Assert.Contains(operationsConfigWithExport.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Export); + Assert.DoesNotContain(operationsConfigWithExport.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Import); + Assert.Contains(operationsConfigWithExport.HostingBackgroundServiceQueues, q => q.Queue == QueueType.BulkDelete); + + Assert.Contains(operationsConfigWithImport.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Import); + Assert.DoesNotContain(operationsConfigWithImport.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Export); + Assert.Contains(operationsConfigWithImport.HostingBackgroundServiceQueues, q => q.Queue == QueueType.BulkDelete); + } + + [Fact] + public void GivenAnOperationsConfiguration_WhenStorageStringsAreEmpty_RemoveQueues() + { + // Arrange + var operationsConfig = new OperationsConfiguration + { + Export = new ExportJobConfiguration { Enabled = true, StorageAccountConnection = string.Empty, StorageAccountUri = string.Empty }, + Import = new ImportTaskConfiguration { Enabled = true }, + IntegrationDataStore = new IntegrationDataStoreConfiguration { StorageAccountConnection = string.Empty, StorageAccountUri = string.Empty }, + }; + + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Export, MaxRunningTaskCount = 2 }); + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.Import, MaxRunningTaskCount = 2 }); + operationsConfig.HostingBackgroundServiceQueues.Add(new HostingBackgroundServiceQueueItem { Queue = QueueType.BulkDelete, MaxRunningTaskCount = 1 }); + + // Act + operationsConfig.RemoveDisabledQueues(); + + // Assert + Assert.DoesNotContain(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Export); + Assert.DoesNotContain(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.Import); + Assert.Contains(operationsConfig.HostingBackgroundServiceQueues, q => q.Queue == QueueType.BulkDelete); + } + } +} diff --git a/src/Microsoft.Health.Fhir.Core/Configs/OperationsConfiguration.cs b/src/Microsoft.Health.Fhir.Core/Configs/OperationsConfiguration.cs index 11b95b41e3..a92540288c 100644 --- a/src/Microsoft.Health.Fhir.Core/Configs/OperationsConfiguration.cs +++ b/src/Microsoft.Health.Fhir.Core/Configs/OperationsConfiguration.cs @@ -4,6 +4,8 @@ // ------------------------------------------------------------------------------------------------- using System.Collections.Generic; +using System.Linq; +using Microsoft.Health.Fhir.Core.Features.Operations; namespace Microsoft.Health.Fhir.Core.Configs { @@ -22,5 +24,27 @@ public class OperationsConfiguration public IntegrationDataStoreConfiguration IntegrationDataStore { get; set; } = new IntegrationDataStoreConfiguration(); public ImportTaskConfiguration Import { get; set; } = new ImportTaskConfiguration(); + + /// + /// Removes queues based on the enabled status of the operations. + /// + public void RemoveDisabledQueues() + { + if (!Export.Enabled || (string.IsNullOrEmpty(Export.StorageAccountConnection) && string.IsNullOrEmpty(Export.StorageAccountUri))) + { + HostingBackgroundServiceQueues + .Where(q => q.Queue == QueueType.Export) + .ToList() + .ForEach(q => HostingBackgroundServiceQueues.Remove(q)); + } + + if (!Import.Enabled || (string.IsNullOrEmpty(IntegrationDataStore.StorageAccountConnection) && string.IsNullOrEmpty(IntegrationDataStore.StorageAccountUri))) + { + HostingBackgroundServiceQueues + .Where(q => q.Queue == QueueType.Import) + .ToList() + .ForEach(q => HostingBackgroundServiceQueues.Remove(q)); + } + } } } diff --git a/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs b/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs index 64abbdcd71..e0554b15d5 100644 --- a/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs +++ b/src/Microsoft.Health.Fhir.Shared.Api/Registration/FhirServerServiceCollectionExtensions.cs @@ -71,6 +71,9 @@ public static IFhirServerBuilder AddFhirServer( configurationRoot?.GetSection(FhirServerConfigurationSectionName).Bind(fhirServerConfiguration); configureAction?.Invoke(fhirServerConfiguration); + // Remove any job queues that are disables or don't have a storage account configured. + fhirServerConfiguration.Operations.RemoveDisabledQueues(); + services.AddSingleton(Options.Options.Create(fhirServerConfiguration)); services.AddSingleton(Options.Options.Create(fhirServerConfiguration.Security)); services.AddSingleton(Options.Options.Create(fhirServerConfiguration.Features)); diff --git a/src/Microsoft.Health.TaskManagement/JobHosting.cs b/src/Microsoft.Health.TaskManagement/JobHosting.cs index 7a662d668f..de89e614ea 100644 --- a/src/Microsoft.Health.TaskManagement/JobHosting.cs +++ b/src/Microsoft.Health.TaskManagement/JobHosting.cs @@ -61,7 +61,7 @@ public async Task ExecuteAsync(byte queueType, short runningJobCount, string wor { try { - _logger.LogInformation("Dequeuing next job."); + _logger.LogInformation("Dequeuing next job for {QueueType}.", queueType); if (checkTimeoutJobStopwatch.Elapsed.TotalSeconds > 600) { @@ -73,7 +73,7 @@ public async Task ExecuteAsync(byte queueType, short runningJobCount, string wor } catch (Exception ex) { - _logger.LogError(ex, "Failed to dequeue new job."); + _logger.LogError(ex, "Failed to dequeue new job for {QueueType}.", queueType); } } @@ -102,12 +102,12 @@ public async Task ExecuteAsync(byte queueType, short runningJobCount, string wor { try { - _logger.LogInformation("Empty queue. Delaying until next iteration."); + _logger.LogInformation("Empty queue {QueueType}. Delaying until next iteration.", queueType); await Task.Delay(TimeSpan.FromSeconds(PollingFrequencyInSeconds), cancellationTokenSource.Token); } catch (TaskCanceledException) { - _logger.LogInformation("Queue is stopping, worker is shutting down."); + _logger.LogInformation("Queue {QueueType} is stopping, worker is shutting down.", queueType); } } }