From 2bb7035ba15487cc6d0c6b3f8a9cf7233fa53950 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Tue, 5 Apr 2022 16:42:20 -0700 Subject: [PATCH 01/45] Update release job --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e186736..da2af97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,7 +226,7 @@ jobs: run: dotnet nuget push ${{ steps.download.outputs.download-path }}/artifact/*.nupkg release: - if: ${{ contains(github.ref, 'refs/heads/main') ||contains(github.head_ref, 'release/') }} + if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.ref, 'refs/heads/release/') }} runs-on: ubuntu-latest needs: [build, unit-test] env: From 367882ee257cf0fe873fe0445c499baef650fe8f Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Tue, 5 Apr 2022 16:50:17 -0700 Subject: [PATCH 02/45] Skip publishing duplicate nuget packages --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da2af97..f5c33be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -223,7 +223,7 @@ jobs: source-url: https://nuget.pkg.github.com/Project-MONAI/index.json - name: Publish to GitHub - run: dotnet nuget push ${{ steps.download.outputs.download-path }}/artifact/*.nupkg + run: dotnet nuget push ${{ steps.download.outputs.download-path }}/artifact/*.nupkg --skip-duplicate release: if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.ref, 'refs/heads/release/') }} From 05376c066d014dbd1b7aa5a1255fba2c4e053945 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Tue, 5 Apr 2022 18:29:42 -0700 Subject: [PATCH 03/45] Requires manual trigger to deploy to NuGet.org --- .github/workflows/ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5c33be..5607bb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,11 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: + inputs: + nuget: + type: boolean + default: false + description: Publish to NuGet.org env: BUILD_CONFIG: "Release" @@ -206,6 +211,7 @@ jobs: retention-days: 30 publish: + name: Publish to GitHub Packages runs-on: ubuntu-latest needs: [build, unit-test] steps: @@ -226,7 +232,8 @@ jobs: run: dotnet nuget push ${{ steps.download.outputs.download-path }}/artifact/*.nupkg --skip-duplicate release: - if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.ref, 'refs/heads/release/') }} + name: Official Release to NuGet.org + if: ${{ github.event.inputs.nuget }} runs-on: ubuntu-latest needs: [build, unit-test] env: @@ -269,7 +276,6 @@ jobs: - name: Publish release with GitReleaseManager uses: gittools/actions/gitreleasemanager/publish@v0.9.13 - if: ${{ contains(github.ref, 'refs/heads/main') }} with: token: ${{ secrets.GITHUB_TOKEN }} owner: ${{ steps.repo.outputs._0 }} @@ -278,7 +284,6 @@ jobs: - name: Close release with GitReleaseManager uses: gittools/actions/gitreleasemanager/close@v0.9.13 - if: ${{ contains(github.ref, 'refs/heads/main') }} with: token: ${{ secrets.GITHUB_TOKEN }} owner: ${{ steps.repo.outputs._0 }} From a852ec721a41e95ad86ea6422149ee09824400e2 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Wed, 6 Apr 2022 08:44:22 -0700 Subject: [PATCH 04/45] Update ci.yml Publish & close release only on main branch. --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5607bb6..4671bb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -275,6 +275,7 @@ jobs: name: Release v${{ env.MAJORMINORPATCH }} - name: Publish release with GitReleaseManager + if: ${{ contains(github.ref, 'refs/heads/main') }} uses: gittools/actions/gitreleasemanager/publish@v0.9.13 with: token: ${{ secrets.GITHUB_TOKEN }} @@ -283,6 +284,7 @@ jobs: tagName: ${{ env.MAJORMINORPATCH }} - name: Close release with GitReleaseManager + if: ${{ contains(github.ref, 'refs/heads/main') }} uses: gittools/actions/gitreleasemanager/close@v0.9.13 with: token: ${{ secrets.GITHUB_TOKEN }} From da1c02b7f1483fd81e2e1484b18458a5ae18f218 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Tue, 5 Apr 2022 16:42:20 -0700 Subject: [PATCH 05/45] Update release job --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e186736..da2af97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -226,7 +226,7 @@ jobs: run: dotnet nuget push ${{ steps.download.outputs.download-path }}/artifact/*.nupkg release: - if: ${{ contains(github.ref, 'refs/heads/main') ||contains(github.head_ref, 'release/') }} + if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.ref, 'refs/heads/release/') }} runs-on: ubuntu-latest needs: [build, unit-test] env: From bfafff05e8f9a0ecd642623e91fefd71f3b420df Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Tue, 5 Apr 2022 16:50:17 -0700 Subject: [PATCH 06/45] Skip publishing duplicate nuget packages --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da2af97..f5c33be 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -223,7 +223,7 @@ jobs: source-url: https://nuget.pkg.github.com/Project-MONAI/index.json - name: Publish to GitHub - run: dotnet nuget push ${{ steps.download.outputs.download-path }}/artifact/*.nupkg + run: dotnet nuget push ${{ steps.download.outputs.download-path }}/artifact/*.nupkg --skip-duplicate release: if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.ref, 'refs/heads/release/') }} From 529d9c5d5ce2a8f2bf80dc10173fe7d35e001787 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Tue, 5 Apr 2022 18:29:42 -0700 Subject: [PATCH 07/45] Requires manual trigger to deploy to NuGet.org --- .github/workflows/ci.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f5c33be..5607bb6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,11 @@ on: # Allows you to run this workflow manually from the Actions tab workflow_dispatch: + inputs: + nuget: + type: boolean + default: false + description: Publish to NuGet.org env: BUILD_CONFIG: "Release" @@ -206,6 +211,7 @@ jobs: retention-days: 30 publish: + name: Publish to GitHub Packages runs-on: ubuntu-latest needs: [build, unit-test] steps: @@ -226,7 +232,8 @@ jobs: run: dotnet nuget push ${{ steps.download.outputs.download-path }}/artifact/*.nupkg --skip-duplicate release: - if: ${{ contains(github.ref, 'refs/heads/main') || contains(github.ref, 'refs/heads/release/') }} + name: Official Release to NuGet.org + if: ${{ github.event.inputs.nuget }} runs-on: ubuntu-latest needs: [build, unit-test] env: @@ -269,7 +276,6 @@ jobs: - name: Publish release with GitReleaseManager uses: gittools/actions/gitreleasemanager/publish@v0.9.13 - if: ${{ contains(github.ref, 'refs/heads/main') }} with: token: ${{ secrets.GITHUB_TOKEN }} owner: ${{ steps.repo.outputs._0 }} @@ -278,7 +284,6 @@ jobs: - name: Close release with GitReleaseManager uses: gittools/actions/gitreleasemanager/close@v0.9.13 - if: ${{ contains(github.ref, 'refs/heads/main') }} with: token: ${{ secrets.GITHUB_TOKEN }} owner: ${{ steps.repo.outputs._0 }} From 5fff0897ed8d18f1353b50f2f995975cf89a228f Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Wed, 6 Apr 2022 09:10:44 -0700 Subject: [PATCH 08/45] Change target framework to .net standard 2.1 --- src/Messaging/Messages/ExportRequestMessage.cs | 2 +- src/Messaging/Messages/JsonMessage.cs | 2 +- src/Messaging/Messages/Message.cs | 2 +- src/Messaging/Messages/MessageBase.cs | 14 +++++++------- src/Messaging/Monai.Deploy.Messaging.csproj | 3 ++- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Messaging/Messages/ExportRequestMessage.cs b/src/Messaging/Messages/ExportRequestMessage.cs index bff2b3d..2fd3881 100644 --- a/src/Messaging/Messages/ExportRequestMessage.cs +++ b/src/Messaging/Messages/ExportRequestMessage.cs @@ -63,7 +63,7 @@ public bool IsCompleted /// /// Gets or sets error messages related to this export task. /// - public List ErrorMessages { get; init; } + public List ErrorMessages { get; private set; } public ExportRequestMessage() { diff --git a/src/Messaging/Messages/JsonMessage.cs b/src/Messaging/Messages/JsonMessage.cs index 928b9a3..6c8cdb0 100644 --- a/src/Messaging/Messages/JsonMessage.cs +++ b/src/Messaging/Messages/JsonMessage.cs @@ -13,7 +13,7 @@ public sealed class JsonMessage : MessageBase /// /// Body of the message. /// - public T Body { get; init; } + public T Body { get; private set; } public JsonMessage(T body, string applicationId, diff --git a/src/Messaging/Messages/Message.cs b/src/Messaging/Messages/Message.cs index 4a8c996..1c6d2ca 100644 --- a/src/Messaging/Messages/Message.cs +++ b/src/Messaging/Messages/Message.cs @@ -11,7 +11,7 @@ public sealed class Message : MessageBase /// /// Body of the message. /// - public byte[] Body { get; init; } + public byte[] Body { get; private set; } public Message(byte[] body, string messageDescription, diff --git a/src/Messaging/Messages/MessageBase.cs b/src/Messaging/Messages/MessageBase.cs index 3cbefa7..1abc105 100644 --- a/src/Messaging/Messages/MessageBase.cs +++ b/src/Messaging/Messages/MessageBase.cs @@ -11,40 +11,40 @@ public abstract class MessageBase /// UUID for the message formatted with hyphens. /// xxxxxxxx-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx /// - public string MessageId { get; init; } + public string MessageId { get; private set; } /// /// A short description of the type serialized in the message body. /// - public string MessageDescription { get; init; } + public string MessageDescription { get; private set; } /// /// Content or MIME type of the message body. /// - public string ContentType { get; init; } + public string ContentType { get; private set; } /// /// UUID of the application, in this case, the Informatics Gateway. /// The UUID of Informatics Gateway is 16988a78-87b5-4168-a5c3-2cfc2bab8e54. /// - public string ApplicationId { get; init; } + public string ApplicationId { get; private set; } /// /// Correlation ID of the message. /// For DIMSE connections, the ID generated during association is used. /// For ACR inference requests, the Transaction ID provided in the request is used. /// - public string CorrelationId { get; init; } + public string CorrelationId { get; private set; } /// /// Datetime the message is created. /// - public DateTime CreationDateTime { get; init; } + public DateTime CreationDateTime { get; private set; } /// /// Gets or set the delivery tag/acknoweldge token for the message. /// - public string DeliveryTag { get; init; } + public string DeliveryTag { get; protected set; } protected MessageBase(string messageId, string messageDescription, diff --git a/src/Messaging/Monai.Deploy.Messaging.csproj b/src/Messaging/Monai.Deploy.Messaging.csproj index d6931a7..30716a8 100644 --- a/src/Messaging/Monai.Deploy.Messaging.csproj +++ b/src/Messaging/Monai.Deploy.Messaging.csproj @@ -5,7 +5,8 @@ SPDX-License-Identifier: Apache License 2.0 - net6.0 + netstandard2.1 + latest enable enable false From f90bf2e5755e28b7bd9c0971418390de1ea2a073 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Wed, 6 Apr 2022 10:04:30 -0700 Subject: [PATCH 09/45] Update GitVersion.yml --- GitVersion.yml | 42 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/GitVersion.yml b/GitVersion.yml index ab2a9a2..f3ca542 100644 --- a/GitVersion.yml +++ b/GitVersion.yml @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: � 2022 MONAI Consortium +# SPDX-FileCopyrightText: © 2022 MONAI Consortium # SPDX-License-Identifier: Apache License 2.0 assembly-versioning-scheme: MajorMinorPatchTag @@ -6,12 +6,52 @@ mode: ContinuousDelivery branches: main: tag: '' + mode: ContinuousDelivery + increment: Patch + prevent-increment-of-merged-branch-version: true + track-merge-target: false + source-branches: [ 'release' ] + tracks-release-branches: false + is-release-branch: false + is-mainline: true + pre-release-weight: 55000 release: tag: rc + regex: ^releases?[/-] + mode: ContinuousDelivery + increment: None + prevent-increment-of-merged-branch-version: true + track-merge-target: false + source-branches: [ 'main', 'release' ] + tracks-release-branches: false + is-release-branch: true + is-mainline: false + pre-release-weight: 30000 feature: tag: alpha.{BranchName} + regex: ^features?[/-] + mode: ContinuousDelivery + increment: Inherit + prevent-increment-of-merged-branch-version: false + track-merge-target: false + source-branches: [ 'main', 'release', 'feature' ] + tracks-release-branches: false + is-release-branch: false + is-mainline: false + pre-release-weight: 30000 pull-request: tag: pr + regex: ^(pull|pull\-requests|pr)[/-] + mode: ContinuousDelivery + increment: Inherit + prevent-increment-of-merged-branch-version: false + tag-number-pattern: '[/-](?\d+)[-/]' + track-merge-target: false + source-branches: [ 'main', 'release', 'feature' ] + tracks-release-branches: false + is-release-branch: false + is-mainline: false + pre-release-weight: 30000 ignore: sha: [] From 649c656d92b6334ed268101db9258845d3c7f635 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Wed, 6 Apr 2022 10:08:35 -0700 Subject: [PATCH 10/45] Update GitVersion.yml --- GitVersion.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GitVersion.yml b/GitVersion.yml index f3ca542..8e2e4b0 100644 --- a/GitVersion.yml +++ b/GitVersion.yml @@ -18,7 +18,7 @@ branches: release: tag: rc regex: ^releases?[/-] - mode: ContinuousDelivery + mode: ContinuousDeployment increment: None prevent-increment-of-merged-branch-version: true track-merge-target: false @@ -30,7 +30,7 @@ branches: feature: tag: alpha.{BranchName} regex: ^features?[/-] - mode: ContinuousDelivery + mode: ContinuousDeployment increment: Inherit prevent-increment-of-merged-branch-version: false track-merge-target: false @@ -42,7 +42,7 @@ branches: pull-request: tag: pr regex: ^(pull|pull\-requests|pr)[/-] - mode: ContinuousDelivery + mode: ContinuousDeployment increment: Inherit prevent-increment-of-merged-branch-version: false tag-number-pattern: '[/-](?\d+)[-/]' From fd4e2f696cea05c7431c78bec44465cd4b882162 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Wed, 6 Apr 2022 11:36:25 -0700 Subject: [PATCH 11/45] Update ci.yml --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4671bb1..4c1d0ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -233,7 +233,7 @@ jobs: release: name: Official Release to NuGet.org - if: ${{ github.event.inputs.nuget }} + if: ${{ github.event.inputs.nuget || contains(github.ref, 'refs/heads/release') }} runs-on: ubuntu-latest needs: [build, unit-test] env: From ffcf29a511c4aeacd3c609af2d2dbdf1b22231e9 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Thu, 7 Apr 2022 08:24:02 -0700 Subject: [PATCH 12/45] [Breaking] Unit tests for RabbitMQ plugin (#4) * Implement unit tests for RabbitMQ plugin * Fix sonar code smells * Update PR template Signed-off-by: Victor Chang --- .github/pull_request_template.md | 9 +- src/Messaging/AssemblyInfo.cs | 13 -- src/Messaging/InternalVisible.cs | 3 + src/Messaging/Monai.Deploy.Messaging.csproj | 4 + .../RabbitMq/IServiceCollectionExtension.cs | 17 ++ .../RabbitMq/RabbitMqConnectionFactory.cs | 47 +++++ .../RabbitMqMessagePublisherService.cs | 21 +-- .../RabbitMqMessageSubscriberService.cs | 14 +- src/Messaging/Test/DummyTest.cs | 13 -- .../Test/ExportCompleteMessageTest.cs | 58 ++++++ src/Messaging/Test/JsonMessageTest.cs | 32 ++++ .../Test/Monai.Deploy.Messaging.Test.csproj | 5 + .../RabbitMqMessagePublisherServiceTest.cs | 101 ++++++++++ .../RabbitMqMessageSubscriberServiceTest.cs | 177 ++++++++++++++++++ .../Test/WorkflowRequestMessageTest.cs | 40 ++++ 15 files changed, 498 insertions(+), 56 deletions(-) delete mode 100644 src/Messaging/AssemblyInfo.cs create mode 100644 src/Messaging/InternalVisible.cs create mode 100644 src/Messaging/RabbitMq/IServiceCollectionExtension.cs create mode 100644 src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs delete mode 100644 src/Messaging/Test/DummyTest.cs create mode 100644 src/Messaging/Test/ExportCompleteMessageTest.cs create mode 100644 src/Messaging/Test/JsonMessageTest.cs create mode 100644 src/Messaging/Test/RabbitMq/RabbitMqMessagePublisherServiceTest.cs create mode 100644 src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs create mode 100644 src/Messaging/Test/WorkflowRequestMessageTest.cs diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1e63bf0..1fd856d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,8 +1,3 @@ - - Fixes # . ### Description @@ -16,7 +11,5 @@ A few sentences describing the changes proposed in this pull request. - [ ] Non-breaking change (fix or new feature that would not break existing functionality). - [ ] Breaking change (fix or new feature that would cause existing functionality to change). - [ ] New tests added to cover the changes. -- [ ] All tests passed locally by running `./src/run-tests-in-docker.sh`. +- [ ] All tests passed locally. - [ ] [Documentation comments](https://docs.microsoft.com/en-us/dotnet/csharp/language-reference/language-specification/documentation-comments) included/updated. -- [ ] User guide updated. -- [ ] I have updated the changelog diff --git a/src/Messaging/AssemblyInfo.cs b/src/Messaging/AssemblyInfo.cs deleted file mode 100644 index f6fa0f8..0000000 --- a/src/Messaging/AssemblyInfo.cs +++ /dev/null @@ -1,13 +0,0 @@ -//------------------------------------------------------------------------------ -// -// This code was generated by GitVersion. -// -// You can modify this code as we will not overwrite it when re-executing GitVersion -// -//------------------------------------------------------------------------------ - -using System.Reflection; - -[assembly: AssemblyFileVersion("0.1.0.0")] -[assembly: AssemblyVersion("0.1.0.0")] -[assembly: AssemblyInformationalVersion("0.1.0+0.Branch.main.Sha.ae43f4c7111e09d2a34d881c6704a2dea81d9155")] diff --git a/src/Messaging/InternalVisible.cs b/src/Messaging/InternalVisible.cs new file mode 100644 index 0000000..9426699 --- /dev/null +++ b/src/Messaging/InternalVisible.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Monai.Deploy.Messaging.Test")] diff --git a/src/Messaging/Monai.Deploy.Messaging.csproj b/src/Messaging/Monai.Deploy.Messaging.csproj index 30716a8..e1be3ed 100644 --- a/src/Messaging/Monai.Deploy.Messaging.csproj +++ b/src/Messaging/Monai.Deploy.Messaging.csproj @@ -38,6 +38,10 @@ SPDX-License-Identifier: Apache License 2.0 + + + + True diff --git a/src/Messaging/RabbitMq/IServiceCollectionExtension.cs b/src/Messaging/RabbitMq/IServiceCollectionExtension.cs new file mode 100644 index 0000000..bd6f9f5 --- /dev/null +++ b/src/Messaging/RabbitMq/IServiceCollectionExtension.cs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using Microsoft.Extensions.DependencyInjection; + +namespace Monai.Deploy.Messaging.RabbitMq +{ + public static class IServiceCollectionExtension + { + public static IServiceCollection UseRabbitMq(this IServiceCollection services) + { + services.AddSingleton(); + + return services; + } + } +} diff --git a/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs b/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs new file mode 100644 index 0000000..f16af08 --- /dev/null +++ b/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using Ardalis.GuardClauses; +using RabbitMQ.Client; + +namespace Monai.Deploy.Messaging.RabbitMq +{ + public interface IRabbitMqConnectionFactory + { + /// + /// Creates a new connection for RabbitMQ client. + /// + /// Host name + /// User name + /// Password + /// Virtual host + /// Instance of IConnection. + IConnection CreateConnection(string hostName, string username, string password, string virtualHost); + } + + public class RabbitMqConnectionFactory : IRabbitMqConnectionFactory + { + private ConnectionFactory? _connectionFactory; + + public IConnection CreateConnection(string hostName, string username, string password, string virtualHost) + { + Guard.Against.NullOrWhiteSpace(hostName, nameof(hostName)); + Guard.Against.NullOrWhiteSpace(username, nameof(username)); + Guard.Against.NullOrWhiteSpace(password, nameof(password)); + Guard.Against.NullOrWhiteSpace(virtualHost, nameof(virtualHost)); + + if (_connectionFactory is null) + { + _connectionFactory = new ConnectionFactory() + { + HostName = hostName, + UserName = username, + Password = password, + VirtualHost = virtualHost + }; + } + + return _connectionFactory.CreateConnection(); + } + } +} diff --git a/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs b/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs index d22397e..3f606c0 100644 --- a/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs +++ b/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs @@ -14,6 +14,8 @@ namespace Monai.Deploy.Messaging.RabbitMq { public class RabbitMqMessagePublisherService : IMessageBrokerPublisherService, IDisposable { + private const int PersistentDeliveryMode = 2; + private readonly ILogger _logger; private readonly string _endpoint; private readonly string _virtualHost; @@ -24,9 +26,11 @@ public class RabbitMqMessagePublisherService : IMessageBrokerPublisherService, I public string Name => "Rabbit MQ Publisher"; public RabbitMqMessagePublisherService(IOptions options, - ILogger logger) + ILogger logger, + IRabbitMqConnectionFactory rabbitMqConnectionFactory) { Guard.Against.Null(options, nameof(options)); + Guard.Against.Null(rabbitMqConnectionFactory, nameof(rabbitMqConnectionFactory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -35,17 +39,10 @@ public RabbitMqMessagePublisherService(IOptions "Rabbit MQ Subscriber"; public RabbitMqMessageSubscriberService(IOptions options, - ILogger logger) + ILogger logger, + IRabbitMqConnectionFactory rabbitMqConnectionFactory) { Guard.Against.Null(options, nameof(options)); + Guard.Against.Null(rabbitMqConnectionFactory, nameof(rabbitMqConnectionFactory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -41,16 +43,8 @@ public RabbitMqMessageSubscriberService(IOptions() + { + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + }; + + var errors = new List() + { + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + Guid.NewGuid().ToString(), + }; + + exportRequestMessage.AddErrorMessages(errors); + + var exportCompleteMessage = new ExportCompleteMessage(exportRequestMessage); + + Assert.Equal(exportRequestMessage.WorkflowId, exportCompleteMessage.WorkflowId); + Assert.Equal(exportRequestMessage.ExportTaskId, exportCompleteMessage.ExportTaskId); + Assert.Equal(string.Join(System.Environment.NewLine, errors), exportCompleteMessage.Message); + Assert.Equal(status, exportCompleteMessage.Status); + } + } +} diff --git a/src/Messaging/Test/JsonMessageTest.cs b/src/Messaging/Test/JsonMessageTest.cs new file mode 100644 index 0000000..c7c2e1a --- /dev/null +++ b/src/Messaging/Test/JsonMessageTest.cs @@ -0,0 +1,32 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System; +using Monai.Deploy.Messaging.Messages; +using Xunit; + +namespace Monai.Deploy.Messaging.Test +{ + public class JsonMessageTest + { + [Fact(DisplayName = "Converts JSONMessage to Message")] + public void ConvertsJsonMessageToMessage() + { + var expected = "hello world"; + var jsonMessage = new JsonMessage(expected, Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + var message = jsonMessage.ToMessage(); + + Assert.Equal(jsonMessage.ApplicationId, message.ApplicationId); + Assert.Equal(jsonMessage.CreationDateTime, message.CreationDateTime); + Assert.Equal(jsonMessage.ContentType, message.ContentType); + Assert.Equal(jsonMessage.CorrelationId, message.CorrelationId); + Assert.Equal(jsonMessage.DeliveryTag, message.DeliveryTag); + Assert.Equal(jsonMessage.MessageDescription, message.MessageDescription); + Assert.Equal(jsonMessage.MessageId, message.MessageId); + + var result = message.ConvertTo(); + + Assert.Equal(expected, result); + } + } +} diff --git a/src/Messaging/Test/Monai.Deploy.Messaging.Test.csproj b/src/Messaging/Test/Monai.Deploy.Messaging.Test.csproj index 9f77f5f..d3833e5 100644 --- a/src/Messaging/Test/Monai.Deploy.Messaging.Test.csproj +++ b/src/Messaging/Test/Monai.Deploy.Messaging.Test.csproj @@ -9,6 +9,7 @@ + runtime; build; native; contentfiles; analyzers; buildtransitive @@ -20,4 +21,8 @@ + + + + diff --git a/src/Messaging/Test/RabbitMq/RabbitMqMessagePublisherServiceTest.cs b/src/Messaging/Test/RabbitMq/RabbitMqMessagePublisherServiceTest.cs new file mode 100644 index 0000000..40542ba --- /dev/null +++ b/src/Messaging/Test/RabbitMq/RabbitMqMessagePublisherServiceTest.cs @@ -0,0 +1,101 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Messaging.Configuration; +using Monai.Deploy.Messaging.Messages; +using Monai.Deploy.Messaging.RabbitMq; +using Moq; +using RabbitMQ.Client; +using Xunit; + +namespace Monai.Deploy.Messaging.Test.RabbitMq +{ + public class RabbitMqMessagePublisherServiceTest + { + private readonly IOptions _options; + private readonly Mock> _logger; + private readonly Mock _connectionFactory; + private readonly Mock _connection; + private readonly Mock _model; + + public RabbitMqMessagePublisherServiceTest() + { + _options = Options.Create(new MessageBrokerServiceConfiguration()); + _logger = new Mock>(); + _connectionFactory = new Mock(); + _connection = new Mock(); + _model = new Mock(); + + _connectionFactory.Setup(p => p.CreateConnection(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(_connection.Object); + + _connection.Setup(p => p.CreateModel()).Returns(_model.Object); + } + + [Fact(DisplayName = "Fails to validate when required keys are missing")] + public void FailsToValidateWhenRequiredKeysAreMissing() + { + Assert.Throws(() => new RabbitMqMessagePublisherService(_options, _logger.Object, _connectionFactory.Object)); + } + + [Fact(DisplayName = "Cleanup connections on Dispose")] + public void CleanupOnDispose() + { + _options.Value.PublisherSettings.Add(ConfigurationKeys.EndPoint, "endpoint"); + _options.Value.PublisherSettings.Add(ConfigurationKeys.Username, "username"); + _options.Value.PublisherSettings.Add(ConfigurationKeys.Password, "password"); + _options.Value.PublisherSettings.Add(ConfigurationKeys.VirtualHost, "virtual-host"); + _options.Value.PublisherSettings.Add(ConfigurationKeys.Exchange, "exchange"); + + var service = new RabbitMqMessagePublisherService(_options, _logger.Object, _connectionFactory.Object); + service.Dispose(); + + _connection.Verify(p => p.Close(), Times.Once()); + _connection.Verify(p => p.Dispose(), Times.Once()); + + } + + [Fact(DisplayName = "Publishes a message")] + public async Task PublishesAMessage() + { + _options.Value.PublisherSettings.Add(ConfigurationKeys.EndPoint, "endpoint"); + _options.Value.PublisherSettings.Add(ConfigurationKeys.Username, "username"); + _options.Value.PublisherSettings.Add(ConfigurationKeys.Password, "password"); + _options.Value.PublisherSettings.Add(ConfigurationKeys.VirtualHost, "virtual-host"); + _options.Value.PublisherSettings.Add(ConfigurationKeys.Exchange, "exchange"); + + var basicProperties = new Mock(); + _model.Setup(p => p.CreateBasicProperties()).Returns(basicProperties.Object); + _model.Setup(p => p.BasicPublish( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())); + + var service = new RabbitMqMessagePublisherService(_options, _logger.Object, _connectionFactory.Object); + + var jsonMessage = new JsonMessage("hello world", Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + var message = jsonMessage.ToMessage(); + await service.Publish("topic", message).ConfigureAwait(false); + + basicProperties.VerifySet(p => p.Persistent = true); + basicProperties.VerifySet(p => p.ContentType = jsonMessage.ContentType); + basicProperties.VerifySet(p => p.MessageId = jsonMessage.MessageId); + basicProperties.VerifySet(p => p.AppId = jsonMessage.ApplicationId); + basicProperties.VerifySet(p => p.CorrelationId = jsonMessage.CorrelationId); + basicProperties.VerifySet(p => p.DeliveryMode = 2); + + _model.Verify(p => p.BasicPublish( + It.Is(p => p.Equals("exchange")), + It.Is(p => p.Equals("topic")), + false, + It.Is(p => p.Equals(basicProperties.Object)), + It.IsAny>()), Times.Once()); + } + } +} diff --git a/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs b/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs new file mode 100644 index 0000000..7522f85 --- /dev/null +++ b/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Text; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Messaging.Configuration; +using Monai.Deploy.Messaging.Messages; +using Monai.Deploy.Messaging.RabbitMq; +using Moq; +using RabbitMQ.Client; +using Xunit; + +namespace Monai.Deploy.Messaging.Test.RabbitMq +{ + public class RabbitMqMessageSubscriberServiceTest + { + private readonly IOptions _options; + private readonly Mock> _logger; + private readonly Mock _connectionFactory; + private readonly Mock _connection; + private readonly Mock _model; + + public RabbitMqMessageSubscriberServiceTest() + { + _options = Options.Create(new MessageBrokerServiceConfiguration()); + _logger = new Mock>(); + _connectionFactory = new Mock(); + _connection = new Mock(); + _model = new Mock(); + + _connectionFactory.Setup(p => p.CreateConnection(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(_connection.Object); + + _connection.Setup(p => p.CreateModel()).Returns(_model.Object); + } + + [Fact(DisplayName = "Fails to validate when required keys are missing")] + public void FailsToValidateWhenRequiredKeysAreMissing() + { + Assert.Throws(() => new RabbitMqMessageSubscriberService(_options, _logger.Object, _connectionFactory.Object)); + } + + [Fact(DisplayName = "Cleanup connections on Dispose")] + public void CleanupOnDispose() + { + _options.Value.SubscriberSettings.Add(ConfigurationKeys.EndPoint, "endpoint"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Username, "username"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Password, "password"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.VirtualHost, "virtual-host"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Exchange, "exchange"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.ExportRequestQueue, "export-request-queue"); + + var service = new RabbitMqMessageSubscriberService(_options, _logger.Object, _connectionFactory.Object); + service.Dispose(); + + _connection.Verify(p => p.Close(), Times.Once()); + _connection.Verify(p => p.Dispose(), Times.Once()); + } + + [Fact(DisplayName = "Subscribes to a topic")] + public void SubscribesToATopic() + { + _options.Value.SubscriberSettings.Add(ConfigurationKeys.EndPoint, "endpoint"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Username, "username"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Password, "password"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.VirtualHost, "virtual-host"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Exchange, "exchange"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.ExportRequestQueue, "export-request-queue"); + + var jsonMessage = new JsonMessage("hello world", Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), "1"); + var message = jsonMessage.ToMessage(); + var basicProperties = new Mock(); + basicProperties.SetupGet(p => p.MessageId).Returns(jsonMessage.MessageId); + basicProperties.SetupGet(p => p.AppId).Returns(jsonMessage.ApplicationId); + basicProperties.SetupGet(p => p.ContentType).Returns(jsonMessage.ContentType); + basicProperties.SetupGet(p => p.CorrelationId).Returns(jsonMessage.CorrelationId); + basicProperties.SetupGet(p => p.Headers["CreationDateTime"]).Returns(Encoding.UTF8.GetBytes(jsonMessage.CreationDateTime.ToString("o", System.Globalization.CultureInfo.InvariantCulture))); + + _model.Setup(p => p.QueueDeclare( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())) + .Returns(new QueueDeclareOk("queue-name", 1, 1)); + _model.Setup(p => p.QueueBind( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>())); + _model.Setup(p => p.BasicQos( + It.IsAny(), + It.IsAny(), + It.IsAny())); + _model.Setup(p => p.BasicConsume( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny>(), + It.IsAny())) + .Callback, IBasicConsumer>( + (queue, autoAck, tag, noLocal, exclusive, args, consumer) => + { + consumer.HandleBasicDeliver(tag, Convert.ToUInt64(jsonMessage.DeliveryTag, CultureInfo.InvariantCulture), false, "exchange", "topic", basicProperties.Object, new ReadOnlyMemory(message.Body)); + }); + + var service = new RabbitMqMessageSubscriberService(_options, _logger.Object, _connectionFactory.Object); + + service.Subscribe("topic", "queue", (args) => + { + Assert.Equal(message.ApplicationId, args.Message.ApplicationId); + Assert.Equal(message.ContentType, args.Message.ContentType); + Assert.Equal(message.MessageId, args.Message.MessageId); + Assert.Equal(message.CreationDateTime.ToUniversalTime(), args.Message.CreationDateTime.ToUniversalTime()); + Assert.Equal(message.DeliveryTag, args.Message.DeliveryTag); + Assert.Equal("topic", args.Message.MessageDescription); + Assert.Equal(message.MessageId, args.Message.MessageId); + Assert.Equal(message.Body, args.Message.Body); + }); + } + + [Fact(DisplayName = "Acknowledge a message")] + public void AcknowledgeAMessage() + { + _options.Value.SubscriberSettings.Add(ConfigurationKeys.EndPoint, "endpoint"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Username, "username"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Password, "password"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.VirtualHost, "virtual-host"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Exchange, "exchange"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.ExportRequestQueue, "export-request-queue"); + + var jsonMessage = new JsonMessage("hello world", Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), "1"); + var message = jsonMessage.ToMessage(); + + _model.Setup(p => p.BasicAck( + It.IsAny(), + It.IsAny())); + + var service = new RabbitMqMessageSubscriberService(_options, _logger.Object, _connectionFactory.Object); + + service.Acknowledge(message); + + _model.Verify(p => p.BasicAck(1, false), Times.Once()); + } + + [Fact(DisplayName = "Reject a message")] + public void RejectAMessage() + { + _options.Value.SubscriberSettings.Add(ConfigurationKeys.EndPoint, "endpoint"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Username, "username"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Password, "password"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.VirtualHost, "virtual-host"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.Exchange, "exchange"); + _options.Value.SubscriberSettings.Add(ConfigurationKeys.ExportRequestQueue, "export-request-queue"); + + var jsonMessage = new JsonMessage("hello world", Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), "1"); + var message = jsonMessage.ToMessage(); + + _model.Setup(p => p.BasicNack( + It.IsAny(), + It.IsAny(), + It.IsAny())); + + var service = new RabbitMqMessageSubscriberService(_options, _logger.Object, _connectionFactory.Object); + + service.Reject(message); + + _model.Verify(p => p.BasicNack(1, false, true), Times.Once()); + } + } +} diff --git a/src/Messaging/Test/WorkflowRequestMessageTest.cs b/src/Messaging/Test/WorkflowRequestMessageTest.cs new file mode 100644 index 0000000..caef614 --- /dev/null +++ b/src/Messaging/Test/WorkflowRequestMessageTest.cs @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System; +using System.Collections.Generic; +using Monai.Deploy.Messaging.Common; +using Monai.Deploy.Messaging.Messages; +using Xunit; + +namespace Monai.Deploy.Messaging.Test +{ + public class WorkflowRequestMessageTest + { + [Fact(DisplayName = "Converts JSONMessage to Message")] + public void ConvertsJsonMessageToMessage() + { + var input = new WorkflowRequestMessage() + { + Bucket = Guid.NewGuid().ToString(), + CalledAeTitle = Guid.NewGuid().ToString(), + CallingAeTitle = Guid.NewGuid().ToString(), + CorrelationId = Guid.NewGuid().ToString(), + FileCount = 10, + PayloadId = Guid.NewGuid(), + Timestamp = DateTime.Now, + Workflows = new List { Guid.NewGuid().ToString() } + }; + + var files = new List() + { + new BlockStorageInfo{ Path =Guid.NewGuid().ToString(), Metadata=Guid.NewGuid().ToString() }, + new BlockStorageInfo{ Path =Guid.NewGuid().ToString(), Metadata=Guid.NewGuid().ToString() }, + }; + + input.AddFiles(files); + + Assert.Equal(files, input.Payload); + } + } +} From 367ce0fddd1b4b605c99fe226888f5b26520e7b9 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Mon, 11 Apr 2022 08:25:15 -0700 Subject: [PATCH 13/45] Event validation & add Task related events (#7) * Rename Message to Event used in the message body. * Add validation to events and validate event messages on all properties recursively. Signed-off-by: Victor Chang Signed-off-by: JP LEGER --- src/Messaging/Common/Credentials.cs | 26 ++++ .../Common/MessageValidationException.cs | 31 +++++ src/Messaging/Common/Storage.cs | 45 ++++++ .../Configuration/ConfigurationException.cs | 8 -- src/Messaging/Events/EventBase.cs | 129 ++++++++++++++++++ .../ExportCompleteEvent.cs} | 24 ++-- .../ExportRequestEvent.cs} | 19 ++- src/Messaging/Events/ExportStatus.cs | 13 ++ src/Messaging/Events/TaskCompleteEvent.cs | 41 ++++++ src/Messaging/Events/TaskDispatchEvent.cs | 60 ++++++++ src/Messaging/Events/TaskStatus.cs | 13 ++ .../WorkflowRequestEvent.cs} | 14 +- src/Messaging/Monai.Deploy.Messaging.csproj | 1 + src/Messaging/Test/EventBaseTest.cs | 81 +++++++++++ ...sageTest.cs => ExportCompleteEventTest.cs} | 17 ++- src/Messaging/Test/TaskCompleteEventTest.cs | 30 ++++ src/Messaging/Test/TaskDispatchEventTest.cs | 67 +++++++++ .../Test/WorkflowRequestMessageTest.cs | 4 +- 18 files changed, 591 insertions(+), 32 deletions(-) create mode 100644 src/Messaging/Common/Credentials.cs create mode 100644 src/Messaging/Common/MessageValidationException.cs create mode 100644 src/Messaging/Common/Storage.cs create mode 100644 src/Messaging/Events/EventBase.cs rename src/Messaging/{Messages/ExportCompleteMessage.cs => Events/ExportCompleteEvent.cs} (77%) rename src/Messaging/{Messages/ExportRequestMessage.cs => Events/ExportRequestEvent.cs} (81%) create mode 100644 src/Messaging/Events/ExportStatus.cs create mode 100644 src/Messaging/Events/TaskCompleteEvent.cs create mode 100644 src/Messaging/Events/TaskDispatchEvent.cs create mode 100644 src/Messaging/Events/TaskStatus.cs rename src/Messaging/{Messages/WorkflowRequestMessage.cs => Events/WorkflowRequestEvent.cs} (88%) create mode 100644 src/Messaging/Test/EventBaseTest.cs rename src/Messaging/Test/{ExportCompleteMessageTest.cs => ExportCompleteEventTest.cs} (78%) create mode 100644 src/Messaging/Test/TaskCompleteEventTest.cs create mode 100644 src/Messaging/Test/TaskDispatchEventTest.cs diff --git a/src/Messaging/Common/Credentials.cs b/src/Messaging/Common/Credentials.cs new file mode 100644 index 0000000..5a7b45a --- /dev/null +++ b/src/Messaging/Common/Credentials.cs @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.ComponentModel.DataAnnotations; +using Newtonsoft.Json; + +namespace Monai.Deploy.Messaging.Common +{ + public class Credentials + { + /// + /// Access key or username of the credentials pair. + /// + [JsonProperty(PropertyName = "access_key")] + [Required] + public string? AccessKey { get; set; } + + /// + /// Access token or password of the credentials pair. + /// + [JsonProperty(PropertyName = "access_token")] + [Required] + public string? AccessToken { get; set; } + + } +} diff --git a/src/Messaging/Common/MessageValidationException.cs b/src/Messaging/Common/MessageValidationException.cs new file mode 100644 index 0000000..fb055a9 --- /dev/null +++ b/src/Messaging/Common/MessageValidationException.cs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.ComponentModel.DataAnnotations; +using System.Runtime.Serialization; + +namespace Monai.Deploy.Messaging.Common +{ + [Serializable] + public class MessageValidationException : Exception + { + public MessageValidationException(List errors) + : base(FormatMessage(errors)) + { + } + + protected MessageValidationException(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + + private static string FormatMessage(List errors) + { + if (errors == null || errors.Count == 0) + { + return "Invalid message."; + } + + return $"Invalid message: {string.Join(',', errors.Select(p => $"{p.ErrorMessage} Path: {string.Join(',', p.MemberNames)}."))}"; + } + } +} diff --git a/src/Messaging/Common/Storage.cs b/src/Messaging/Common/Storage.cs new file mode 100644 index 0000000..e43dc3b --- /dev/null +++ b/src/Messaging/Common/Storage.cs @@ -0,0 +1,45 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.ComponentModel.DataAnnotations; +using Newtonsoft.Json; + +namespace Monai.Deploy.Messaging.Common +{ + public class Storage + { + /// + /// Gets or sets the endpoint of the storage service. + /// + [JsonProperty(PropertyName = "endpoint")] + [Required] + public string? Endpoint { get; set; } + + /// + /// Gets or sets credentials for accessing the storage service. + /// + [JsonProperty(PropertyName = "credentials")] + [Required] + public Credentials? Credentials { get; set; } + + /// + /// Gets or sets name of the bucket. + /// + [JsonProperty(PropertyName = "bucket")] + [Required] + public string? Bucket { get; set; } + + /// + /// Gets or sets whether the connection should be secured or not. + /// + [JsonProperty(PropertyName = "secured_connection")] + public bool SecuredConnection { get; set; } = false; + + /// + /// Gets or sets the optional relative root path to the data. + /// + [JsonProperty(PropertyName = "relative_root_path")] + [Required] + public string? RelativeRootPath { get; set; } + } +} diff --git a/src/Messaging/Configuration/ConfigurationException.cs b/src/Messaging/Configuration/ConfigurationException.cs index 1da9311..3fc53b6 100644 --- a/src/Messaging/Configuration/ConfigurationException.cs +++ b/src/Messaging/Configuration/ConfigurationException.cs @@ -8,18 +8,10 @@ namespace Monai.Deploy.Messaging.Configuration [Serializable] public class ConfigurationException : Exception { - public ConfigurationException() - { - } - public ConfigurationException(string? message) : base(message) { } - public ConfigurationException(string? message, Exception? innerException) : base(message, innerException) - { - } - protected ConfigurationException(SerializationInfo info, StreamingContext context) : base(info, context) { } diff --git a/src/Messaging/Events/EventBase.cs b/src/Messaging/Events/EventBase.cs new file mode 100644 index 0000000..91c9524 --- /dev/null +++ b/src/Messaging/Events/EventBase.cs @@ -0,0 +1,129 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.Collections; +using System.ComponentModel.DataAnnotations; +using Ardalis.GuardClauses; +using Monai.Deploy.Messaging.Common; + +namespace Monai.Deploy.Messaging.Events +{ + public class EventBase + { + /// + /// Validates the message with all properties recursively. + /// Throws on error. + /// + public void Validate() + { + var validationContextItems = new Dictionary(); + var validationResults = new List(); + if (!TryValidateRecusively(this, validationContextItems, validationResults, new HashSet(), GetType().Name)) + { + throw new MessageValidationException(validationResults); + } + } + + private bool TryValidateRecusively(T instance, + IDictionary validationContextItems, + List validationResults, + ISet validatedObjects, + string propertyPath) + { + Guard.Against.Null(instance, nameof(instance)); + Guard.Against.Null(validationContextItems, nameof(validationContextItems)); + Guard.Against.Null(validationResults, nameof(validationResults)); + Guard.Against.Null(validatedObjects, nameof(validatedObjects)); + Guard.Against.NullOrWhiteSpace(propertyPath, nameof(propertyPath)); + + if (validatedObjects.Contains(instance)) + { + return true; + } + validatedObjects.Add(instance); + + var result = Validator.TryValidateObject(instance, new ValidationContext(instance, null, validationContextItems), validationResults, true); + + var properties = instance.GetType().GetProperties().Where(prop => prop.CanRead && prop.GetIndexParameters().Length == 0).ToList(); + + foreach (var property in properties) + { + if (property.PropertyType == typeof(string) || property.PropertyType.IsValueType) continue; + + var value = instance.GetType().GetProperty(property.Name)?.GetValue(instance, null); + + if (value == null) + { + continue; + } + + result &= TryValidateProperty(validationContextItems, validationResults, validatedObjects, value, $"{propertyPath}.{property.Name}"); + } + + return result; + } + + private bool TryValidateProperty(IDictionary validationContextItems, + List validationResults, + ISet validatedObjects, + object? value, + string propertyPath) + { + Guard.Against.Null(validationContextItems, nameof(validationContextItems)); + Guard.Against.Null(validationResults, nameof(validationResults)); + Guard.Against.Null(validatedObjects, nameof(validatedObjects)); + Guard.Against.NullOrWhiteSpace(propertyPath, nameof(propertyPath)); + + var result = true; + + if (value is IEnumerable enumerable && + !TryValidateEnumerableRecursively(enumerable, validationContextItems, validationResults, validatedObjects, propertyPath)) + { + result = false; + } + + var nestedValidationResults = new List(); + if (!TryValidateRecusively(value, validationContextItems, nestedValidationResults, validatedObjects, propertyPath)) + { + result = false; + foreach (var validationResult in nestedValidationResults) + { + validationResults.Add(new ValidationResult(validationResult.ErrorMessage, validationResult.MemberNames.Select(p => propertyPath + '.' + p))); + } + } + + return result; + } + + private bool TryValidateEnumerableRecursively(IEnumerable enumerable, + IDictionary validationContextItems, + IList validationResults, + ISet validatedObjects, + string propertyPath) + { + Guard.Against.Null(enumerable, nameof(enumerable)); + Guard.Against.Null(validationContextItems, nameof(validationContextItems)); + Guard.Against.Null(validationResults, nameof(validationResults)); + Guard.Against.Null(validatedObjects, nameof(validatedObjects)); + Guard.Against.NullOrWhiteSpace(propertyPath, nameof(propertyPath)); + + var result = true; + foreach (var enumObj in enumerable) + { + if (enumObj is not null) + { + var nestedValidationResults = new List(); + if (!TryValidateRecusively(enumObj, validationContextItems, nestedValidationResults, validatedObjects, propertyPath)) + { + result = false; + foreach (var validationResult in nestedValidationResults) + { + validationResults.Add(new ValidationResult(validationResult.ErrorMessage, validationResult.MemberNames.Select(p => $"{propertyPath}.{p}"))); + } + } + } + } + return result; + } + } +} diff --git a/src/Messaging/Messages/ExportCompleteMessage.cs b/src/Messaging/Events/ExportCompleteEvent.cs similarity index 77% rename from src/Messaging/Messages/ExportCompleteMessage.cs rename to src/Messaging/Events/ExportCompleteEvent.cs index d907676..12859da 100644 --- a/src/Messaging/Messages/ExportCompleteMessage.cs +++ b/src/Messaging/Events/ExportCompleteEvent.cs @@ -1,50 +1,50 @@ // SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium // SPDX-License-Identifier: Apache License 2.0 +using System.ComponentModel.DataAnnotations; using Ardalis.GuardClauses; using Newtonsoft.Json; using Newtonsoft.Json.Converters; -namespace Monai.Deploy.Messaging.Messages +namespace Monai.Deploy.Messaging.Events { - public enum ExportStatus - { - Success = 0, - Failure, - PartialFailure, - Unknown - } - - public class ExportCompleteMessage + public class ExportCompleteEvent : EventBase { /// /// Gets or sets the workflow ID generated by the Workflow Manager. /// + [JsonProperty(PropertyName = "workflow_id")] + [Required] public string WorkflowId { get; set; } = default!; /// /// Gets or sets the export task ID generated by the Workflow Manager. /// + [JsonProperty(PropertyName = "export_task_id")] + [Required] public string ExportTaskId { get; set; } = default!; /// /// Gets or sets the state of the export task. /// + [JsonProperty(PropertyName = "status")] [JsonConverter(typeof(StringEnumConverter))] + [Required] public ExportStatus Status { get; set; } /// /// Gets or sets error messages, if any, when exporting. /// + [JsonProperty(PropertyName = "message")] public string Message { get; set; } = default!; [JsonConstructor] - public ExportCompleteMessage() + public ExportCompleteEvent() { Status = ExportStatus.Unknown; } - public ExportCompleteMessage(ExportRequestMessage exportRequest) + public ExportCompleteEvent(ExportRequestEvent exportRequest) { Guard.Against.Null(exportRequest, nameof(exportRequest)); diff --git a/src/Messaging/Messages/ExportRequestMessage.cs b/src/Messaging/Events/ExportRequestEvent.cs similarity index 81% rename from src/Messaging/Messages/ExportRequestMessage.cs rename to src/Messaging/Events/ExportRequestEvent.cs index 2fd3881..72207d5 100644 --- a/src/Messaging/Messages/ExportRequestMessage.cs +++ b/src/Messaging/Events/ExportRequestEvent.cs @@ -1,23 +1,32 @@ // SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium // SPDX-License-Identifier: Apache License 2.0 -namespace Monai.Deploy.Messaging.Messages +using System.ComponentModel.DataAnnotations; +using Newtonsoft.Json; + +namespace Monai.Deploy.Messaging.Events { - public class ExportRequestMessage + public class ExportRequestEvent : EventBase { /// /// Gets or sets the workflow ID generated by the Workflow Manager. /// + [JsonProperty(PropertyName = "workflow_id")] + [Required] public string WorkflowId { get; set; } = default!; /// /// Gets or sets the export task ID generated by the Workflow Manager. /// + [JsonProperty(PropertyName = "export_task_id")] + [Required] public string ExportTaskId { get; set; } = default!; /// /// Gets or sets a list of files to be exported. /// + [JsonProperty(PropertyName = "files")] + [Required, MinLength(1)] public IEnumerable Files { get; set; } = default!; /// @@ -25,6 +34,8 @@ public class ExportRequestMessage /// For DIMSE, the named DICOM destination. /// For ACR, the Transaction ID in the original inference request. /// + [JsonProperty(PropertyName = "destination")] + [Required] public string Destination { get; set; } = default!; /// @@ -32,6 +43,8 @@ public class ExportRequestMessage /// For DIMSE, the correlation ID is the UUID associated with the first DICOM association received. /// For ACR, use the Transaction ID in the original request. /// + [JsonProperty(PropertyName = "correlation_id")] + [Required] public string CorrelationId { get; set; } = default!; /// @@ -65,7 +78,7 @@ public bool IsCompleted /// public List ErrorMessages { get; private set; } - public ExportRequestMessage() + public ExportRequestEvent() { ErrorMessages = new List(); } diff --git a/src/Messaging/Events/ExportStatus.cs b/src/Messaging/Events/ExportStatus.cs new file mode 100644 index 0000000..3788574 --- /dev/null +++ b/src/Messaging/Events/ExportStatus.cs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +namespace Monai.Deploy.Messaging.Events +{ + public enum ExportStatus + { + Success = 0, + Failure, + PartialFailure, + Unknown + } +} diff --git a/src/Messaging/Events/TaskCompleteEvent.cs b/src/Messaging/Events/TaskCompleteEvent.cs new file mode 100644 index 0000000..3432a54 --- /dev/null +++ b/src/Messaging/Events/TaskCompleteEvent.cs @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.ComponentModel.DataAnnotations; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Monai.Deploy.Messaging.Events +{ + public class TaskCompleteEvent : EventBase + { + /// + /// Gets or sets the ID representing the instance of the workflow. + /// + [JsonProperty(PropertyName = "workflow_id")] + [Required] + public string? WorkflowId { get; set; } + + /// + /// Gets or sets the ID representing the instance of the Task. + /// + [Required] + [JsonProperty(PropertyName = "task_id")] + public string? TaskId { get; set; } + + /// + /// Gets or sets the correlation ID. + /// + [JsonProperty(PropertyName = "correlation_id")] + [Required] + public string? CorrelationId { get; set; } + + /// + /// Gets or set the status of the task. + /// + [JsonProperty(PropertyName = "status")] + [JsonConverter(typeof(StringEnumConverter))] + [Required] + public TaskStatus Status { get; set; } = TaskStatus.NotRun; + } +} diff --git a/src/Messaging/Events/TaskDispatchEvent.cs b/src/Messaging/Events/TaskDispatchEvent.cs new file mode 100644 index 0000000..a8ee848 --- /dev/null +++ b/src/Messaging/Events/TaskDispatchEvent.cs @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.ComponentModel.DataAnnotations; +using Monai.Deploy.Messaging.Common; +using Newtonsoft.Json; + +namespace Monai.Deploy.Messaging.Events +{ + public class TaskDispatchEvent : EventBase + { + /// + /// Gets or sets the ID representing the instance of the workflow. + /// + [JsonProperty(PropertyName = "workflow_id")] + [Required] + public string? WorkflowId { get; set; } + + /// + /// Gets or sets the ID representing the instance of the Task. + /// + [Required] + [JsonProperty(PropertyName = "task_id")] + public string? TaskId { get; set; } + + /// + /// Gets or sets the correlation ID. + /// + [JsonProperty(PropertyName = "correlation_id")] + [Required] + public string? CorrelationId { get; set; } + + /// + /// Gets or sets the fully qualified assembly name of the task plug-in for the task. + /// + [JsonProperty(PropertyName = "task_assembly_name")] + [Required] + public string? TaskAssemblyName { get; set; } + + /// + /// Gets or sets the task execution arguments. + /// + [JsonProperty(PropertyName = "arguments")] + public Dictionary? Arguments { get; set; } + + /// + /// Gets or sets the input storage information. + /// + [JsonProperty(PropertyName = "input")] + [Required] + public Storage? Input { get; set; } + + /// + /// Gets or sets the output storage information. + /// + [JsonProperty(PropertyName = "output")] + [Required] + public Storage? Output { get; set; } + } +} diff --git a/src/Messaging/Events/TaskStatus.cs b/src/Messaging/Events/TaskStatus.cs new file mode 100644 index 0000000..12cb638 --- /dev/null +++ b/src/Messaging/Events/TaskStatus.cs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +namespace Monai.Deploy.Messaging.Events +{ + public enum TaskStatus + { + NotRun, + Succeeded, + Failed, + Canceled, + } +} diff --git a/src/Messaging/Messages/WorkflowRequestMessage.cs b/src/Messaging/Events/WorkflowRequestEvent.cs similarity index 88% rename from src/Messaging/Messages/WorkflowRequestMessage.cs rename to src/Messaging/Events/WorkflowRequestEvent.cs index 9b96a1a..94d47a0 100644 --- a/src/Messaging/Messages/WorkflowRequestMessage.cs +++ b/src/Messaging/Events/WorkflowRequestEvent.cs @@ -1,12 +1,13 @@ // SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium // SPDX-License-Identifier: Apache License 2.0 +using System.ComponentModel.DataAnnotations; using Monai.Deploy.Messaging.Common; using Newtonsoft.Json; -namespace Monai.Deploy.Messaging.Messages +namespace Monai.Deploy.Messaging.Events { - public class WorkflowRequestMessage + public class WorkflowRequestEvent : EventBase { private readonly List _payload; @@ -14,6 +15,7 @@ public class WorkflowRequestMessage /// Gets or sets the ID of the payload which is also used as the root path of the payload. /// [JsonProperty(PropertyName = "payload_id")] + [Required] public Guid PayloadId { get; set; } /// @@ -26,6 +28,7 @@ public class WorkflowRequestMessage /// Gets or sets number of files in the payload. /// [JsonProperty(PropertyName = "file_count")] + [Required] public int FileCount { get; set; } /// @@ -33,12 +36,14 @@ public class WorkflowRequestMessage /// For an ACR inference request, the correlation ID is the Transaction ID in the original request. /// [JsonProperty(PropertyName = "correlation_id")] + [Required] public string CorrelationId { get; set; } = default!; /// /// Gets or set the name of the bucket where the files in are stored. /// [JsonProperty(PropertyName = "bucket")] + [Required] public string Bucket { get; set; } = default!; /// @@ -46,6 +51,7 @@ public class WorkflowRequestMessage /// For an ACR inference request, the transaction ID. /// [JsonProperty(PropertyName = "calling_aetitle")] + [Required] public string CallingAeTitle { get; set; } = default!; /// @@ -59,15 +65,17 @@ public class WorkflowRequestMessage /// Gets or sets the time the data was received. /// [JsonProperty(PropertyName = "timestamp")] + [Required] public DateTime Timestamp { get; set; } /// /// Gets or sets a list of files and metadata files in this request. /// [JsonProperty(PropertyName = "payload")] + [Required, MinLength(1, ErrorMessage = "At least one file is required.")] public IReadOnlyList Payload { get => _payload; } - public WorkflowRequestMessage() + public WorkflowRequestEvent() { _payload = new List(); Workflows = new List(); diff --git a/src/Messaging/Monai.Deploy.Messaging.csproj b/src/Messaging/Monai.Deploy.Messaging.csproj index e1be3ed..1ef2b45 100644 --- a/src/Messaging/Monai.Deploy.Messaging.csproj +++ b/src/Messaging/Monai.Deploy.Messaging.csproj @@ -55,5 +55,6 @@ SPDX-License-Identifier: Apache License 2.0 + \ No newline at end of file diff --git a/src/Messaging/Test/EventBaseTest.cs b/src/Messaging/Test/EventBaseTest.cs new file mode 100644 index 0000000..c082d01 --- /dev/null +++ b/src/Messaging/Test/EventBaseTest.cs @@ -0,0 +1,81 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using Monai.Deploy.Messaging.Common; +using Monai.Deploy.Messaging.Events; +using Xunit; + +namespace Monai.Deploy.Messaging.Test +{ + internal class StringClass : EventBase + { + [Required] + public string? StringField { get; set; } + } + + internal class NestedStringClass : EventBase + { + [Required] + public StringClass? NestedStringField { get; set; } + } + + internal class NestedStringCollectionClass : EventBase + { + [Required, MinLength(1)] + public IList? NestedStrings { get; set; } + } + + public class EventBaseTest + { + [Fact(DisplayName = "Validates StringClass")] + public void ValidatesStringField() + { + var obj = new StringClass(); + var validationException = Assert.Throws(() => obj.Validate()); + Assert.Equal($"Invalid message: The {nameof(obj.StringField)} field is required. Path: {nameof(obj.StringField)}.", validationException.Message); + + obj.StringField = "hello"; + var exception = Record.Exception(() => obj.Validate()); + Assert.Null(exception); + } + + [Fact(DisplayName = "Validates NestedStringClass")] + public void ValidatesNestedStringField() + { + var obj = new NestedStringClass(); + var validationException = Assert.Throws(() => obj.Validate()); + Assert.Equal($"Invalid message: The {nameof(obj.NestedStringField)} field is required. Path: {nameof(obj.NestedStringField)}.", validationException.Message); + + obj.NestedStringField = new StringClass(); + validationException = Assert.Throws(() => obj.Validate()); + Assert.Equal($"Invalid message: The {nameof(obj.NestedStringField.StringField)} field is required. Path: {nameof(NestedStringClass)}.{nameof(obj.NestedStringField)}.{nameof(obj.NestedStringField.StringField)}.", validationException.Message); + + obj.NestedStringField.StringField = "hello"; + var exception = Record.Exception(() => obj.Validate()); + Assert.Null(exception); + } + + [Fact(DisplayName = "Validates NestedStringCollectionClass")] + public void ValidatesNestedStringCollectionClassField() + { + var obj = new NestedStringCollectionClass(); + var validationException = Assert.Throws(() => obj.Validate()); + Assert.Equal($"Invalid message: The {nameof(obj.NestedStrings)} field is required. Path: {nameof(obj.NestedStrings)}.", validationException.Message); + + obj.NestedStrings = new List(); + validationException = Assert.Throws(() => obj.Validate()); + Assert.Equal($"Invalid message: The field {nameof(obj.NestedStrings)} must be a string or array type with a minimum length of '1'. Path: {nameof(obj.NestedStrings)}.", validationException.Message); + + var stringClass = new StringClass(); + obj.NestedStrings = new List() { stringClass }; + validationException = Assert.Throws(() => obj.Validate()); + Assert.Equal($"Invalid message: The {nameof(stringClass.StringField)} field is required. Path: {nameof(NestedStringCollectionClass)}.{nameof(obj.NestedStrings)}.{nameof(StringClass.StringField)}.", validationException.Message); + + stringClass.StringField = "Hello World!"; + var exception = Record.Exception(() => obj.Validate()); + Assert.Null(exception); + } + } +} diff --git a/src/Messaging/Test/ExportCompleteMessageTest.cs b/src/Messaging/Test/ExportCompleteEventTest.cs similarity index 78% rename from src/Messaging/Test/ExportCompleteMessageTest.cs rename to src/Messaging/Test/ExportCompleteEventTest.cs index c53d795..6dcae2f 100644 --- a/src/Messaging/Test/ExportCompleteMessageTest.cs +++ b/src/Messaging/Test/ExportCompleteEventTest.cs @@ -3,12 +3,13 @@ using System; using System.Collections.Generic; -using Monai.Deploy.Messaging.Messages; +using Monai.Deploy.Messaging.Common; +using Monai.Deploy.Messaging.Events; using Xunit; namespace Monai.Deploy.Messaging.Test { - public class ExportCompleteMessageTest + public class ExportCompleteEventTest { [Theory(DisplayName = "Shall generate ExportCompleteMessageTest from ExportRequestMessage")] [InlineData(1, 0, ExportStatus.Success)] @@ -16,7 +17,7 @@ public class ExportCompleteMessageTest [InlineData(3, 3, ExportStatus.PartialFailure)] public void ShallGenerateExportCompleteMessageTestFromExportRequestMessage(int successded, int fialure, ExportStatus status) { - var exportRequestMessage = new ExportRequestMessage + var exportRequestMessage = new ExportRequestEvent { CorrelationId = Guid.NewGuid().ToString(), DeliveryTag = Guid.NewGuid().ToString(), @@ -47,12 +48,20 @@ public void ShallGenerateExportCompleteMessageTestFromExportRequestMessage(int s exportRequestMessage.AddErrorMessages(errors); - var exportCompleteMessage = new ExportCompleteMessage(exportRequestMessage); + var exportCompleteMessage = new ExportCompleteEvent(exportRequestMessage); Assert.Equal(exportRequestMessage.WorkflowId, exportCompleteMessage.WorkflowId); Assert.Equal(exportRequestMessage.ExportTaskId, exportCompleteMessage.ExportTaskId); Assert.Equal(string.Join(System.Environment.NewLine, errors), exportCompleteMessage.Message); Assert.Equal(status, exportCompleteMessage.Status); } + + [Fact(DisplayName = "Validation shall throw on error")] + public void ValidationShallThrowOnError() + { + var exportCompleteEvent = new ExportCompleteEvent(); + + Assert.Throws(() => exportCompleteEvent.Validate()); + } } } diff --git a/src/Messaging/Test/TaskCompleteEventTest.cs b/src/Messaging/Test/TaskCompleteEventTest.cs new file mode 100644 index 0000000..088bfd2 --- /dev/null +++ b/src/Messaging/Test/TaskCompleteEventTest.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System; +using Monai.Deploy.Messaging.Common; +using Monai.Deploy.Messaging.Events; +using Xunit; + +namespace Monai.Deploy.Messaging.Test +{ + public class TaskCompleteEventTest + { + [Fact(DisplayName = "Validation throws on error")] + public void ValidationThrowsOnError() + { + var taskDispatchEvent = new TaskCompleteEvent(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.WorkflowId = Guid.NewGuid().ToString(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.TaskId = Guid.NewGuid().ToString(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.CorrelationId = Guid.NewGuid().ToString(); + var exception = Record.Exception(() => taskDispatchEvent.Validate()); + Assert.Null(exception); + } + } +} diff --git a/src/Messaging/Test/TaskDispatchEventTest.cs b/src/Messaging/Test/TaskDispatchEventTest.cs new file mode 100644 index 0000000..565fc97 --- /dev/null +++ b/src/Messaging/Test/TaskDispatchEventTest.cs @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System; +using Monai.Deploy.Messaging.Common; +using Monai.Deploy.Messaging.Events; +using Xunit; + +namespace Monai.Deploy.Messaging.Test +{ + public class TaskDispatchEventTest + { + [Fact(DisplayName = "Validation throws on error")] + public void ValidationThrowsOnError() + { + var taskDispatchEvent = new TaskDispatchEvent(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.WorkflowId = Guid.NewGuid().ToString(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.TaskId = Guid.NewGuid().ToString(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.CorrelationId = Guid.NewGuid().ToString(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.TaskAssemblyName = Guid.NewGuid().ToString(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.Input = new Storage(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.Output = new Storage(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.Output.Endpoint = "endpoint"; + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.Output.Credentials = new Credentials(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.Output.Credentials.AccessToken = "token"; + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.Output.Credentials.AccessKey = "key"; + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.Output.Bucket = "bucket"; + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.Output.RelativeRootPath = "path"; + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.Input.Endpoint = "endpoint"; + taskDispatchEvent.Input.Credentials = new Credentials + { + AccessToken = "token", + AccessKey = "key" + }; + taskDispatchEvent.Input.Bucket = "bucket"; + taskDispatchEvent.Input.RelativeRootPath = "path"; + var exception = Record.Exception(() => taskDispatchEvent.Validate()); + Assert.Null(exception); + } + } +} diff --git a/src/Messaging/Test/WorkflowRequestMessageTest.cs b/src/Messaging/Test/WorkflowRequestMessageTest.cs index caef614..faf911d 100644 --- a/src/Messaging/Test/WorkflowRequestMessageTest.cs +++ b/src/Messaging/Test/WorkflowRequestMessageTest.cs @@ -4,7 +4,7 @@ using System; using System.Collections.Generic; using Monai.Deploy.Messaging.Common; -using Monai.Deploy.Messaging.Messages; +using Monai.Deploy.Messaging.Events; using Xunit; namespace Monai.Deploy.Messaging.Test @@ -14,7 +14,7 @@ public class WorkflowRequestMessageTest [Fact(DisplayName = "Converts JSONMessage to Message")] public void ConvertsJsonMessageToMessage() { - var input = new WorkflowRequestMessage() + var input = new WorkflowRequestEvent() { Bucket = Guid.NewGuid().ToString(), CalledAeTitle = Guid.NewGuid().ToString(), From 5a314d32a3dff1860d525b47f4839ec52599baa6 Mon Sep 17 00:00:00 2001 From: jackschofield23 <56344499+jackschofield23@users.noreply.github.com> Date: Mon, 11 Apr 2022 23:10:33 +0100 Subject: [PATCH 14/45] Add subscription no requeue reject (#9) Signed-off-by: JP LEGER --- src/Messaging/API/IMessageBrokerSubscriberService.cs | 3 ++- src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Messaging/API/IMessageBrokerSubscriberService.cs b/src/Messaging/API/IMessageBrokerSubscriberService.cs index 6d14844..5d668c5 100644 --- a/src/Messaging/API/IMessageBrokerSubscriberService.cs +++ b/src/Messaging/API/IMessageBrokerSubscriberService.cs @@ -33,6 +33,7 @@ public interface IMessageBrokerSubscriberService /// Rejects a messags. /// /// Message to be rejected. - void Reject(MessageBase message); + /// Determines if the message should be requeued. + void Reject(MessageBase message, bool requeue = true); } } diff --git a/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs b/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs index 1302d62..b9ae5c6 100644 --- a/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs +++ b/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs @@ -107,12 +107,12 @@ public void Acknowledge(MessageBase message) _logger.AcknowledgementSent(message.MessageId); } - public void Reject(MessageBase message) + public void Reject(MessageBase message, bool requeue = true) { Guard.Against.Null(message, nameof(message)); _logger.SendingNAcknowledgement(message.MessageId); - _channel.BasicNack(ulong.Parse(message.DeliveryTag, CultureInfo.InvariantCulture), multiple: false, requeue: true); + _channel.BasicNack(ulong.Parse(message.DeliveryTag, CultureInfo.InvariantCulture), multiple: false, requeue: requeue); _logger.NAcknowledgementSent(message.MessageId); } From e3c4c91b28fd73dc3fd6d5b85109db35baf6d27e Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Thu, 14 Apr 2022 09:36:59 -0700 Subject: [PATCH 15/45] New RunnerCompleteEvent (#10) - Add RunnerCompleteEvent - Rename TaskCompleteEvent to TaskUpdateEvent with new fields and changes. - Allow multiple inputs/outputs for TaskDispatchEvent - Add Accepted TaskStatus Signed-off-by: Victor Chang Signed-off-by: JP LEGER --- GitVersion.yml | 2 +- src/Messaging/Common/Storage.cs | 8 ++ src/Messaging/Events/EventBase.cs | 4 +- src/Messaging/Events/FailureReason.cs | 13 ++++ src/Messaging/Events/RunnerCompleteEvent.cs | 55 ++++++++++++++ src/Messaging/Events/TaskCompleteEvent.cs | 41 ----------- src/Messaging/Events/TaskDispatchEvent.cs | 58 ++++++++++++--- src/Messaging/Events/TaskStatus.cs | 4 +- src/Messaging/Events/TaskUpdateEvent.cs | 73 +++++++++++++++++++ src/Messaging/Test/RunnerCompleteEventTest.cs | 39 ++++++++++ src/Messaging/Test/TaskDispatchEventTest.cs | 40 +++++++--- ...eteEventTest.cs => TaskUpdateEventTest.cs} | 7 +- 12 files changed, 274 insertions(+), 70 deletions(-) create mode 100644 src/Messaging/Events/FailureReason.cs create mode 100644 src/Messaging/Events/RunnerCompleteEvent.cs delete mode 100644 src/Messaging/Events/TaskCompleteEvent.cs create mode 100644 src/Messaging/Events/TaskUpdateEvent.cs create mode 100644 src/Messaging/Test/RunnerCompleteEventTest.cs rename src/Messaging/Test/{TaskCompleteEventTest.cs => TaskUpdateEventTest.cs} (79%) diff --git a/GitVersion.yml b/GitVersion.yml index 8e2e4b0..a35b3cd 100644 --- a/GitVersion.yml +++ b/GitVersion.yml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache License 2.0 assembly-versioning-scheme: MajorMinorPatchTag -mode: ContinuousDelivery +mode: ContinuousDeployment branches: main: tag: '' diff --git a/src/Messaging/Common/Storage.cs b/src/Messaging/Common/Storage.cs index e43dc3b..efe5050 100644 --- a/src/Messaging/Common/Storage.cs +++ b/src/Messaging/Common/Storage.cs @@ -8,6 +8,14 @@ namespace Monai.Deploy.Messaging.Common { public class Storage { + /// + /// Gets or sets the name of the artifact. + /// For Argo, name of the artifact used in the template. + /// + [JsonProperty(PropertyName = "name")] + [Required] + public string? Name { get; set; } + /// /// Gets or sets the endpoint of the storage service. /// diff --git a/src/Messaging/Events/EventBase.cs b/src/Messaging/Events/EventBase.cs index 91c9524..a108ba2 100644 --- a/src/Messaging/Events/EventBase.cs +++ b/src/Messaging/Events/EventBase.cs @@ -8,13 +8,13 @@ namespace Monai.Deploy.Messaging.Events { - public class EventBase + public abstract class EventBase { /// /// Validates the message with all properties recursively. /// Throws on error. /// - public void Validate() + public virtual void Validate() { var validationContextItems = new Dictionary(); var validationResults = new List(); diff --git a/src/Messaging/Events/FailureReason.cs b/src/Messaging/Events/FailureReason.cs new file mode 100644 index 0000000..e0ac06d --- /dev/null +++ b/src/Messaging/Events/FailureReason.cs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +namespace Monai.Deploy.Messaging.Events +{ + public enum FailureReason + { + None, + Unknown, + RunnerNotSupported, + InvalidMessage, + } +} diff --git a/src/Messaging/Events/RunnerCompleteEvent.cs b/src/Messaging/Events/RunnerCompleteEvent.cs new file mode 100644 index 0000000..5f65b8f --- /dev/null +++ b/src/Messaging/Events/RunnerCompleteEvent.cs @@ -0,0 +1,55 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.ComponentModel.DataAnnotations; +using Newtonsoft.Json; + +namespace Monai.Deploy.Messaging.Events +{ + public class RunnerCompleteEvent : EventBase + { + /// + /// Gets or sets the ID representing the instance of the workflow. + /// + [JsonProperty(PropertyName = "workflow_id")] + [Required] + public string WorkflowId { get; set; } + + /// + /// Gets or sets the ID representing the instance of the Task. + /// + [Required] + [JsonProperty(PropertyName = "task_id")] + public string TaskId { get; set; } + + /// + /// Gets or sets the execution ID representing the instance of the task. + /// + [JsonProperty(PropertyName = "execution_id")] + [Required] + public string ExecutionId { get; set; } + + /// + /// Gets or sets the correlation ID. + /// + [JsonProperty(PropertyName = "correlation_id")] + [Required] + public string CorrelationId { get; set; } + + /// + /// Gets or sets the identity provided by the external service. + /// + [JsonProperty(PropertyName = "identity")] + [Required, MaxLength(63)] + public string Identity { get; set; } + + public RunnerCompleteEvent() + { + WorkflowId = String.Empty; + TaskId = String.Empty; + ExecutionId = String.Empty; + CorrelationId = String.Empty; + Identity = String.Empty; + } + } +} diff --git a/src/Messaging/Events/TaskCompleteEvent.cs b/src/Messaging/Events/TaskCompleteEvent.cs deleted file mode 100644 index 3432a54..0000000 --- a/src/Messaging/Events/TaskCompleteEvent.cs +++ /dev/null @@ -1,41 +0,0 @@ -// SPDX-FileCopyrightText: © 2022 MONAI Consortium -// SPDX-License-Identifier: Apache License 2.0 - -using System.ComponentModel.DataAnnotations; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; - -namespace Monai.Deploy.Messaging.Events -{ - public class TaskCompleteEvent : EventBase - { - /// - /// Gets or sets the ID representing the instance of the workflow. - /// - [JsonProperty(PropertyName = "workflow_id")] - [Required] - public string? WorkflowId { get; set; } - - /// - /// Gets or sets the ID representing the instance of the Task. - /// - [Required] - [JsonProperty(PropertyName = "task_id")] - public string? TaskId { get; set; } - - /// - /// Gets or sets the correlation ID. - /// - [JsonProperty(PropertyName = "correlation_id")] - [Required] - public string? CorrelationId { get; set; } - - /// - /// Gets or set the status of the task. - /// - [JsonProperty(PropertyName = "status")] - [JsonConverter(typeof(StringEnumConverter))] - [Required] - public TaskStatus Status { get; set; } = TaskStatus.NotRun; - } -} diff --git a/src/Messaging/Events/TaskDispatchEvent.cs b/src/Messaging/Events/TaskDispatchEvent.cs index a8ee848..113ebc9 100644 --- a/src/Messaging/Events/TaskDispatchEvent.cs +++ b/src/Messaging/Events/TaskDispatchEvent.cs @@ -4,6 +4,7 @@ using System.ComponentModel.DataAnnotations; using Monai.Deploy.Messaging.Common; using Newtonsoft.Json; +using Newtonsoft.Json.Converters; namespace Monai.Deploy.Messaging.Events { @@ -14,47 +15,82 @@ public class TaskDispatchEvent : EventBase /// [JsonProperty(PropertyName = "workflow_id")] [Required] - public string? WorkflowId { get; set; } + public string WorkflowId { get; set; } /// /// Gets or sets the ID representing the instance of the Task. /// [Required] [JsonProperty(PropertyName = "task_id")] - public string? TaskId { get; set; } + public string TaskId { get; set; } + + /// + /// Gets or sets the execution ID representing the instance of the task. + /// + [JsonProperty(PropertyName = "execution_id")] + [Required] + public string ExecutionId { get; set; } /// /// Gets or sets the correlation ID. /// [JsonProperty(PropertyName = "correlation_id")] [Required] - public string? CorrelationId { get; set; } + public string CorrelationId { get; set; } /// /// Gets or sets the fully qualified assembly name of the task plug-in for the task. /// [JsonProperty(PropertyName = "task_assembly_name")] [Required] - public string? TaskAssemblyName { get; set; } + public string TaskAssemblyName { get; set; } /// /// Gets or sets the task execution arguments. /// - [JsonProperty(PropertyName = "arguments")] - public Dictionary? Arguments { get; set; } + [JsonProperty(PropertyName = "task_plugin_arguments")] + public Dictionary TaskPluginArguments { get; set; } /// - /// Gets or sets the input storage information. + /// Gets or set the status of the task. /// - [JsonProperty(PropertyName = "input")] + [JsonProperty(PropertyName = "status")] + [JsonConverter(typeof(StringEnumConverter))] [Required] - public Storage? Input { get; set; } + public TaskStatus Status { get; set; } + + /// + /// Gets or sets the input storage information. + /// + [JsonProperty(PropertyName = "inputs")] + [Required, MinLength(1, ErrorMessage = "At least input is required.")] + public List Inputs { get; set; } /// /// Gets or sets the output storage information. /// - [JsonProperty(PropertyName = "output")] + [JsonProperty(PropertyName = "outputs")] [Required] - public Storage? Output { get; set; } + public List Outputs { get; set; } + + /// + /// Gets or sets the task execution arguments. + /// + [JsonProperty(PropertyName = "metadata")] + public Dictionary Metadata { get; set; } + + public TaskDispatchEvent() + { + WorkflowId = String.Empty; + TaskId = String.Empty; + ExecutionId = String.Empty; + CorrelationId = String.Empty; + TaskAssemblyName = String.Empty; + TaskPluginArguments = new Dictionary(); + Status = TaskStatus.Unknown; + Inputs = new List(); + Outputs = new List(); + Metadata = new Dictionary(); + } } } diff --git a/src/Messaging/Events/TaskStatus.cs b/src/Messaging/Events/TaskStatus.cs index 12cb638..8d94c5d 100644 --- a/src/Messaging/Events/TaskStatus.cs +++ b/src/Messaging/Events/TaskStatus.cs @@ -5,7 +5,9 @@ namespace Monai.Deploy.Messaging.Events { public enum TaskStatus { - NotRun, + Unknown, + Created, + Accepted, Succeeded, Failed, Canceled, diff --git a/src/Messaging/Events/TaskUpdateEvent.cs b/src/Messaging/Events/TaskUpdateEvent.cs new file mode 100644 index 0000000..801cf0d --- /dev/null +++ b/src/Messaging/Events/TaskUpdateEvent.cs @@ -0,0 +1,73 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.ComponentModel.DataAnnotations; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace Monai.Deploy.Messaging.Events +{ + public class TaskUpdateEvent : EventBase + { + /// + /// Gets or sets the ID representing the instance of the workflow. + /// + [JsonProperty(PropertyName = "workflow_id")] + [Required] + public string WorkflowId { get; set; } + + /// + /// Gets or sets the ID representing the instance of the Task. + /// + [Required] + [JsonProperty(PropertyName = "task_id")] + public string TaskId { get; set; } + + /// + /// Gets or sets the execution ID representing the instance of the task. + /// + [JsonProperty(PropertyName = "execution_id")] + [Required] + public string ExecutionId { get; set; } + + /// + /// Gets or sets the correlation ID. + /// + [JsonProperty(PropertyName = "correlation_id")] + [Required] + public string CorrelationId { get; set; } + + /// + /// Gets or set the status of the task. + /// + [JsonProperty(PropertyName = "status")] + [JsonConverter(typeof(StringEnumConverter))] + [Required] + public TaskStatus Status { get; set; } + + /// + /// Gets or set the failure reason of the task. + /// + [JsonProperty(PropertyName = "message")] + [JsonConverter(typeof(StringEnumConverter))] + [Required] + public FailureReason Reason { get; set; } + + /// + /// Gets or set any additional (error) message related to the task. + /// + [JsonProperty(PropertyName = "reason")] + public string Message { get; set; } + + public TaskUpdateEvent() + { + WorkflowId = String.Empty; + TaskId = String.Empty; + ExecutionId = String.Empty; + CorrelationId = String.Empty; + Status = TaskStatus.Unknown; + Reason = FailureReason.None; + Message = String.Empty; + } + } +} diff --git a/src/Messaging/Test/RunnerCompleteEventTest.cs b/src/Messaging/Test/RunnerCompleteEventTest.cs new file mode 100644 index 0000000..36dabcb --- /dev/null +++ b/src/Messaging/Test/RunnerCompleteEventTest.cs @@ -0,0 +1,39 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System; +using Monai.Deploy.Messaging.Common; +using Monai.Deploy.Messaging.Events; +using Xunit; + +namespace Monai.Deploy.Messaging.Test +{ + public class RunnerCompleteEventTest + { + [Fact(DisplayName = "Validation throws on error")] + public void ValidationThrowsOnError() + { + var runnerComplete = new RunnerCompleteEvent(); + Assert.Throws(() => runnerComplete.Validate()); + + runnerComplete.WorkflowId = Guid.NewGuid().ToString(); + Assert.Throws(() => runnerComplete.Validate()); + + runnerComplete.TaskId = Guid.NewGuid().ToString(); + Assert.Throws(() => runnerComplete.Validate()); + + runnerComplete.ExecutionId = Guid.NewGuid().ToString(); + Assert.Throws(() => runnerComplete.Validate()); + + runnerComplete.CorrelationId = Guid.NewGuid().ToString(); + Assert.Throws(() => runnerComplete.Validate()); + + runnerComplete.Identity = "1234567890123456789012345678901234567890123456789012345678901234567890"; + Assert.Throws(() => runnerComplete.Validate()); + + runnerComplete.Identity = "123456789012345678901234567890123456789012345678901234567890123"; + var exception = Record.Exception(() => runnerComplete.Validate()); + Assert.Null(exception); + } + } +} diff --git a/src/Messaging/Test/TaskDispatchEventTest.cs b/src/Messaging/Test/TaskDispatchEventTest.cs index 565fc97..7a4ed3e 100644 --- a/src/Messaging/Test/TaskDispatchEventTest.cs +++ b/src/Messaging/Test/TaskDispatchEventTest.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache License 2.0 using System; +using System.Collections.Generic; using Monai.Deploy.Messaging.Common; using Monai.Deploy.Messaging.Events; using Xunit; @@ -19,6 +20,9 @@ public void ValidationThrowsOnError() taskDispatchEvent.WorkflowId = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); + taskDispatchEvent.ExecutionId = Guid.NewGuid().ToString(); + Assert.Throws(() => taskDispatchEvent.Validate()); + taskDispatchEvent.TaskId = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); @@ -28,38 +32,50 @@ public void ValidationThrowsOnError() taskDispatchEvent.TaskAssemblyName = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.Input = new Storage(); + taskDispatchEvent.Inputs = new List(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + var input = new Storage(); + taskDispatchEvent.Inputs.Add(input); + Assert.Throws(() => taskDispatchEvent.Validate()); + + taskDispatchEvent.Outputs = new List(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + var output = new Storage(); + taskDispatchEvent.Outputs.Add(output); Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.Output = new Storage(); + output.Name = "name"; Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.Output.Endpoint = "endpoint"; + output.Endpoint = "endpoint"; Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.Output.Credentials = new Credentials(); + output.Credentials = new Credentials(); Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.Output.Credentials.AccessToken = "token"; + output.Credentials.AccessToken = "token"; Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.Output.Credentials.AccessKey = "key"; + output.Credentials.AccessKey = "key"; Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.Output.Bucket = "bucket"; + output.Bucket = "bucket"; Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.Output.RelativeRootPath = "path"; + output.RelativeRootPath = "path"; Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.Input.Endpoint = "endpoint"; - taskDispatchEvent.Input.Credentials = new Credentials + input.Name = "name"; + input.Endpoint = "endpoint"; + input.Credentials = new Credentials { AccessToken = "token", AccessKey = "key" }; - taskDispatchEvent.Input.Bucket = "bucket"; - taskDispatchEvent.Input.RelativeRootPath = "path"; + input.Bucket = "bucket"; + input.RelativeRootPath = "path"; var exception = Record.Exception(() => taskDispatchEvent.Validate()); Assert.Null(exception); } diff --git a/src/Messaging/Test/TaskCompleteEventTest.cs b/src/Messaging/Test/TaskUpdateEventTest.cs similarity index 79% rename from src/Messaging/Test/TaskCompleteEventTest.cs rename to src/Messaging/Test/TaskUpdateEventTest.cs index 088bfd2..56fbfb9 100644 --- a/src/Messaging/Test/TaskCompleteEventTest.cs +++ b/src/Messaging/Test/TaskUpdateEventTest.cs @@ -8,17 +8,20 @@ namespace Monai.Deploy.Messaging.Test { - public class TaskCompleteEventTest + public class TaskUpdateEventTest { [Fact(DisplayName = "Validation throws on error")] public void ValidationThrowsOnError() { - var taskDispatchEvent = new TaskCompleteEvent(); + var taskDispatchEvent = new TaskUpdateEvent(); Assert.Throws(() => taskDispatchEvent.Validate()); taskDispatchEvent.WorkflowId = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); + taskDispatchEvent.ExecutionId = Guid.NewGuid().ToString(); + Assert.Throws(() => taskDispatchEvent.Validate()); + taskDispatchEvent.TaskId = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); From 7fe7ce05fafc36cf435ddb2407f193817559caed Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Thu, 14 Apr 2022 12:34:38 -0700 Subject: [PATCH 16/45] Add additional failure reasons (#11) Signed-off-by: Victor Chang Signed-off-by: JP LEGER --- src/Messaging/Events/FailureReason.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Messaging/Events/FailureReason.cs b/src/Messaging/Events/FailureReason.cs index e0ac06d..0509a07 100644 --- a/src/Messaging/Events/FailureReason.cs +++ b/src/Messaging/Events/FailureReason.cs @@ -9,5 +9,7 @@ public enum FailureReason Unknown, RunnerNotSupported, InvalidMessage, + PluginError, + ExternalServiceError, } } From 2402d20050b76432eaf1791bb01743422e87530d Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Fri, 15 Apr 2022 10:19:51 -0700 Subject: [PATCH 17/45] Update FailureReason.cs Signed-off-by: JP LEGER --- src/Messaging/Events/FailureReason.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Messaging/Events/FailureReason.cs b/src/Messaging/Events/FailureReason.cs index 0509a07..a34605b 100644 --- a/src/Messaging/Events/FailureReason.cs +++ b/src/Messaging/Events/FailureReason.cs @@ -11,5 +11,6 @@ public enum FailureReason InvalidMessage, PluginError, ExternalServiceError, + TimedOut, } } From bfc08d96499f3fb9fc6a05dc64640f8c933ceb7d Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Thu, 21 Apr 2022 06:18:02 -0700 Subject: [PATCH 18/45] Ability to convert Message to JsonMessage (#13) * Ability to convert Message to JsonMessage Signed-off-by: Victor Chang * Add SubscribeAsync with a Func<> callback Signed-off-by: Victor Chang * Update unit test for SubscribeAsync Signed-off-by: Victor Chang * Use existing RabbitMQ connection if available, otherwise, create a new one. Signed-off-by: Victor Chang * Fix code smells Signed-off-by: Victor Chang * Breaking: IRabbitMqConnectionFactory to CreateChannel and grantee a single connection is used Signed-off-by: Victor Chang * APIs to bind multiple topics to a single queue Signed-off-by: Victor Chang Signed-off-by: JP LEGER --- .../API/IMessageBrokerPublisherService.cs | 2 +- .../API/IMessageBrokerSubscriberService.cs | 44 +++++++- src/Messaging/Common/Log.cs | 4 +- src/Messaging/Messages/Message.cs | 30 +++++- .../RabbitMq/RabbitMqConnectionFactory.cs | 100 +++++++++++++++--- .../RabbitMqMessagePublisherService.cs | 22 ++-- .../RabbitMqMessageSubscriberService.cs | 98 ++++++++++++----- src/Messaging/Test/JsonMessageTest.cs | 53 +++++++++- .../RabbitMqMessagePublisherServiceTest.cs | 27 +---- .../RabbitMqMessageSubscriberServiceTest.cs | 26 +++-- 10 files changed, 314 insertions(+), 92 deletions(-) diff --git a/src/Messaging/API/IMessageBrokerPublisherService.cs b/src/Messaging/API/IMessageBrokerPublisherService.cs index 73bfb17..a36ac8f 100644 --- a/src/Messaging/API/IMessageBrokerPublisherService.cs +++ b/src/Messaging/API/IMessageBrokerPublisherService.cs @@ -5,7 +5,7 @@ namespace Monai.Deploy.Messaging { - public interface IMessageBrokerPublisherService + public interface IMessageBrokerPublisherService : IDisposable { /// /// Gets or sets the name of the storage service. diff --git a/src/Messaging/API/IMessageBrokerSubscriberService.cs b/src/Messaging/API/IMessageBrokerSubscriberService.cs index 5d668c5..240d867 100644 --- a/src/Messaging/API/IMessageBrokerSubscriberService.cs +++ b/src/Messaging/API/IMessageBrokerSubscriberService.cs @@ -6,7 +6,7 @@ namespace Monai.Deploy.Messaging { - public interface IMessageBrokerSubscriberService + public interface IMessageBrokerSubscriberService : IDisposable { /// /// Gets or sets the name of the storage service. @@ -14,15 +14,49 @@ public interface IMessageBrokerSubscriberService string Name { get; } /// - /// Subscribe to a message topic & queue. + /// Subscribe to a message topic & queue and executes messageReceivedCallback for every message that is received. /// Either provide a topic, a queue or both. + /// A queue is generated if the name of the queue is not provided. /// - /// Name of the topic to subscribe to + /// Topic/routing key to bind to /// Name of the queue to consume /// Action to be performed when message is received - /// Number of unacknowledged messages to receive at once. Defaults to 0. + /// Number of unacknowledged messages to receive at once. Defaults to 0. void Subscribe(string topic, string queue, Action messageReceivedCallback, ushort prefetchCount = 0); + /// + /// Subscribe to a message topic & queue and executes messageReceivedCallback for every message that is received. + /// Either provide a topic, a queue or both. + /// A queue is generated if the name of the queue is not provided. + /// + /// Topics/routing keys to bind to + /// Name of the queue to consume + /// Action to be performed when message is received + /// Number of unacknowledged messages to receive at once. Defaults to 0. + void Subscribe(string[] topics, string queue, Action messageReceivedCallback, ushort prefetchCount = 0); + + /// + /// Subscribe to a message topic & queue and executes messageReceivedCallback asynchronously for every message that is received. + /// Either provide a topic, a queue or both. + /// A queue is generated if the name of the queue is not provided. + /// + /// Topic/routing key to bind to + /// Name of the queue to consume + /// to be performed when message is received + /// Number of unacknowledged messages to receive at once. Defaults to 0. + void SubscribeAsync(string topic, string queue, Func messageReceivedCallback, ushort prefetchCount = 0); + + /// + /// Subscribe to a message topic & queue and executes messageReceivedCallback asynchronously for every message that is received. + /// Either provide a topic, a queue or both. + /// A queue is generated if the name of the queue is not provided. + /// + /// Topics/routing keys to bind to + /// Name of the queue to consume + /// to be performed when message is received + /// Number of unacknowledged messages to receive at once. Defaults to 0. + void SubscribeAsync(string[] topics, string queue, Func messageReceivedCallback, ushort prefetchCount = 0); + /// /// Acknowledge receiving of a message with the given token. /// @@ -30,7 +64,7 @@ public interface IMessageBrokerSubscriberService void Acknowledge(MessageBase message); /// - /// Rejects a messags. + /// Rejects a message. /// /// Message to be rejected. /// Determines if the message should be requeued. diff --git a/src/Messaging/Common/Log.cs b/src/Messaging/Common/Log.cs index dfb4098..bcbd549 100644 --- a/src/Messaging/Common/Log.cs +++ b/src/Messaging/Common/Log.cs @@ -33,7 +33,7 @@ public static partial class Log [LoggerMessage(EventId = 10007, Level = LogLevel.Information, Message = "Nack message sent for message {messageId}.")] public static partial void NAcknowledgementSent(this ILogger logger, string messageId); - [LoggerMessage(EventId = 10008, Level = LogLevel.Information, Message = "Closing connection.")] - public static partial void ClosingConnection(this ILogger logger); + [LoggerMessage(EventId = 10008, Level = LogLevel.Information, Message = "Closing connections.")] + public static partial void ClosingConnections(this ILogger logger); } } diff --git a/src/Messaging/Messages/Message.cs b/src/Messaging/Messages/Message.cs index 1c6d2ca..1e146c9 100644 --- a/src/Messaging/Messages/Message.cs +++ b/src/Messaging/Messages/Message.cs @@ -34,8 +34,34 @@ public Message(byte[] body, /// Instance of T or null if data cannot be deserialized. public T ConvertTo() { - var json = Encoding.UTF8.GetString(Body); - return JsonConvert.DeserializeObject(json)!; + try + { + var json = Encoding.UTF8.GetString(Body); + return JsonConvert.DeserializeObject(json)!; + } + catch + { + return default!; + } + } + + /// + /// Converts the Message into a JsonMessageT. + /// + /// Type to convert to + /// Instance of JsonMessageT or null if data cannot be deserialized. + public JsonMessage ConvertToJsonMessage() + { + try + { + var json = Encoding.UTF8.GetString(Body); + var body = JsonConvert.DeserializeObject(json)!; + return new JsonMessage(body, MessageDescription, MessageId, ApplicationId, CorrelationId, CreationDateTime, DeliveryTag); + } + catch + { + return null!; + } } } } diff --git a/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs b/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs index f16af08..9f646d6 100644 --- a/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs +++ b/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs @@ -1,7 +1,12 @@ // SPDX-FileCopyrightText: © 2022 MONAI Consortium // SPDX-License-Identifier: Apache License 2.0 +using System.Collections.Concurrent; +using System.Security.Cryptography; +using System.Text; using Ardalis.GuardClauses; +using Microsoft.Extensions.Logging; +using Monai.Deploy.Messaging.Common; using RabbitMQ.Client; namespace Monai.Deploy.Messaging.RabbitMq @@ -9,39 +14,106 @@ namespace Monai.Deploy.Messaging.RabbitMq public interface IRabbitMqConnectionFactory { /// - /// Creates a new connection for RabbitMQ client. + /// Creates a new channel for RabbitMQ client. + /// THe connection factory maintains a single connection to the specified + /// hostName, username, password, and virtualHost combination. /// /// Host name /// User name /// Password /// Virtual host - /// Instance of IConnection. - IConnection CreateConnection(string hostName, string username, string password, string virtualHost); + /// Instance of . + IModel CreateChannel(string hostName, string username, string password, string virtualHost); } - public class RabbitMqConnectionFactory : IRabbitMqConnectionFactory + public class RabbitMqConnectionFactory : IRabbitMqConnectionFactory, IDisposable { - private ConnectionFactory? _connectionFactory; + private readonly ConcurrentDictionary> _connectionFactoriess; + private readonly ConcurrentDictionary> _connections; + private readonly ILogger _logger; + private bool _disposedValue; - public IConnection CreateConnection(string hostName, string username, string password, string virtualHost) + public RabbitMqConnectionFactory(ILogger logger) + { + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _connectionFactoriess = new ConcurrentDictionary>(); + _connections = new ConcurrentDictionary>(); + } + + public IModel CreateChannel(string hostName, string username, string password, string virtualHost) { Guard.Against.NullOrWhiteSpace(hostName, nameof(hostName)); Guard.Against.NullOrWhiteSpace(username, nameof(username)); Guard.Against.NullOrWhiteSpace(password, nameof(password)); Guard.Against.NullOrWhiteSpace(virtualHost, nameof(virtualHost)); - if (_connectionFactory is null) + var key = $"{hostName}{username}{HashPassword(password)}{virtualHost}"; + + var connection = _connections.AddOrUpdate(key, + x => + { + return CreatConnection(hostName, username, password, virtualHost, key); + }, + (updateKey, updateConnection) => + { + if (updateConnection.Value.IsOpen) + { + return updateConnection; + } + else + { + return CreatConnection(hostName, username, password, virtualHost, key); + } + }); + + return connection.Value.CreateModel(); + } + + private Lazy CreatConnection(string hostName, string username, string password, string virtualHost, string key) + { + var connectionFactory = _connectionFactoriess.GetOrAdd(key, y => new Lazy(() => new ConnectionFactory() + { + HostName = hostName, + UserName = username, + Password = password, + VirtualHost = virtualHost + })); + + return new Lazy(() => connectionFactory.Value.CreateConnection()); + } + + private object HashPassword(string password) + { + Guard.Against.NullOrWhiteSpace(password, nameof(password)); + var sha256 = SHA256.Create(); + var hash = sha256.ComputeHash(Encoding.UTF8.GetBytes(password)); + return hash.Select(x => x.ToString("x2")); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) { - _connectionFactory = new ConnectionFactory() + if (disposing) { - HostName = hostName, - UserName = username, - Password = password, - VirtualHost = virtualHost - }; + _logger.ClosingConnections(); + foreach (var connection in _connections.Values) + { + connection.Value.Close(); + } + _connections.Clear(); + _connectionFactoriess.Clear(); + } + + _disposedValue = true; } + } - return _connectionFactory.CreateConnection(); + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); } } } diff --git a/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs b/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs index 3f606c0..cdf6cf8 100644 --- a/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs +++ b/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs @@ -12,15 +12,17 @@ namespace Monai.Deploy.Messaging.RabbitMq { - public class RabbitMqMessagePublisherService : IMessageBrokerPublisherService, IDisposable + public class RabbitMqMessagePublisherService : IMessageBrokerPublisherService { private const int PersistentDeliveryMode = 2; private readonly ILogger _logger; + private readonly IRabbitMqConnectionFactory _rabbitMqConnectionFactory; private readonly string _endpoint; + private readonly string _username; + private readonly string _password; private readonly string _virtualHost; private readonly string _exchange; - private readonly IConnection _connection; private bool _disposedValue; public string Name => "Rabbit MQ Publisher"; @@ -30,19 +32,19 @@ public RabbitMqMessagePublisherService(IOptions @@ -97,11 +99,9 @@ protected virtual void Dispose(bool disposing) { if (!_disposedValue) { - if (disposing && _connection != null) + if (disposing) { - _logger.ClosingConnection(); - _connection.Close(); - _connection.Dispose(); + // Dispose any managed objects } _disposedValue = true; diff --git a/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs b/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs index b9ae5c6..84420c6 100644 --- a/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs +++ b/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs @@ -14,13 +14,12 @@ namespace Monai.Deploy.Messaging.RabbitMq { - public class RabbitMqMessageSubscriberService : IMessageBrokerSubscriberService, IDisposable + public class RabbitMqMessageSubscriberService : IMessageBrokerSubscriberService { private readonly ILogger _logger; private readonly string _endpoint; private readonly string _virtualHost; private readonly string _exchange; - private readonly IConnection _connection; private readonly IModel _channel; private bool _disposedValue; @@ -28,10 +27,9 @@ public class RabbitMqMessageSubscriberService : IMessageBrokerSubscriberService, public RabbitMqMessageSubscriberService(IOptions options, ILogger logger, - IRabbitMqConnectionFactory rabbitMqConnectionFactory) + IRabbitMqConnectionFactory rabbitMqConnectionFactory) { Guard.Against.Null(options, nameof(options)); - Guard.Against.Null(rabbitMqConnectionFactory, nameof(rabbitMqConnectionFactory)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); @@ -44,8 +42,7 @@ public RabbitMqMessageSubscriberService(IOptions messageReceivedCallback, ushort prefetchCount = 0) + => Subscribe(new string[] { topic }, queue, messageReceivedCallback, prefetchCount); + + public void Subscribe(string[] topics, string queue, Action messageReceivedCallback, ushort prefetchCount = 0) { - Guard.Against.NullOrWhiteSpace(topic, nameof(topic)); + Guard.Against.Null(topics, nameof(topics)); Guard.Against.Null(messageReceivedCallback, nameof(messageReceivedCallback)); var queueDeclareResult = _channel.QueueDeclare(queue: queue, durable: true, exclusive: false, autoDelete: false); - _channel.QueueBind(queueDeclareResult.QueueName, _exchange, topic); + BindToRoutingKeys(topics, queueDeclareResult.QueueName); var consumer = new EventingBasicConsumer(_channel); consumer.Received += (model, eventArgs) => { using var loggerScope = _logger.BeginScope(string.Format(CultureInfo.InvariantCulture, Log.LoggingScopeMessageApplication, eventArgs.BasicProperties.MessageId, eventArgs.BasicProperties.AppId)); - _logger.MessageReceivedFromQueue(queueDeclareResult.QueueName, topic); - - var messageReceivedEventArgs = new MessageReceivedEventArgs( - new Message( - body: eventArgs.Body.ToArray(), - messageDescription: topic, - messageId: eventArgs.BasicProperties.MessageId, - applicationId: eventArgs.BasicProperties.AppId, - contentType: eventArgs.BasicProperties.ContentType, - correlationId: eventArgs.BasicProperties.CorrelationId, - creationDateTime: DateTime.Parse(Encoding.UTF8.GetString((byte[])eventArgs.BasicProperties.Headers["CreationDateTime"]), CultureInfo.InvariantCulture), - deliveryTag: eventArgs.DeliveryTag.ToString(CultureInfo.InvariantCulture)), - new CancellationToken()); + _logger.MessageReceivedFromQueue(queueDeclareResult.QueueName, eventArgs.RoutingKey); + var messageReceivedEventArgs = CreateMessage(eventArgs.RoutingKey, eventArgs); messageReceivedCallback(messageReceivedEventArgs); }; _channel.BasicQos(0, prefetchCount, false); _channel.BasicConsume(queueDeclareResult.QueueName, false, consumer); - _logger.SubscribeToRabbitMqQueue(_endpoint, _virtualHost, _exchange, queueDeclareResult.QueueName, topic); + _logger.SubscribeToRabbitMqQueue(_endpoint, _virtualHost, _exchange, queueDeclareResult.QueueName, string.Join(',', topics)); + } + + public void SubscribeAsync(string topic, string queue, Func messageReceivedCallback, ushort prefetchCount = 0) + => SubscribeAsync(new string[] { topic }, queue, messageReceivedCallback, prefetchCount); + + public void SubscribeAsync(string[] topics, string queue, Func messageReceivedCallback, ushort prefetchCount = 0) + { + Guard.Against.Null(topics, nameof(topics)); + Guard.Against.Null(messageReceivedCallback, nameof(messageReceivedCallback)); + + var queueDeclareResult = _channel.QueueDeclare(queue: queue, durable: true, exclusive: false, autoDelete: false); + BindToRoutingKeys(topics, queueDeclareResult.QueueName); + + var consumer = new EventingBasicConsumer(_channel); + consumer.Received += async (model, eventArgs) => + { + using var loggerScope = _logger.BeginScope(string.Format(CultureInfo.InvariantCulture, Log.LoggingScopeMessageApplication, eventArgs.BasicProperties.MessageId, eventArgs.BasicProperties.AppId)); + + _logger.MessageReceivedFromQueue(queueDeclareResult.QueueName, eventArgs.RoutingKey); + + var messageReceivedEventArgs = CreateMessage(eventArgs.RoutingKey, eventArgs); + await messageReceivedCallback(messageReceivedEventArgs); + }; + _channel.BasicQos(0, prefetchCount, false); + _channel.BasicConsume(queueDeclareResult.QueueName, false, consumer); + _logger.SubscribeToRabbitMqQueue(_endpoint, _virtualHost, _exchange, queueDeclareResult.QueueName, string.Join(',', topics)); } public void Acknowledge(MessageBase message) @@ -120,11 +135,10 @@ protected virtual void Dispose(bool disposing) { if (!_disposedValue) { - if (disposing && _connection is not null) + if (disposing) { - _logger.ClosingConnection(); - _connection.Close(); - _connection.Dispose(); + _channel.Close(); + _channel.Dispose(); } _disposedValue = true; @@ -137,5 +151,37 @@ public void Dispose() Dispose(disposing: true); GC.SuppressFinalize(this); } + + private void BindToRoutingKeys(string[] topics, string queue) + { + Guard.Against.Null(topics, nameof(topics)); + Guard.Against.NullOrWhiteSpace(queue, nameof(queue)); + + foreach (var topic in topics) + { + if (!string.IsNullOrEmpty(topic)) + { + _channel.QueueBind(queue, _exchange, topic); + } + } + } + + private static MessageReceivedEventArgs CreateMessage(string topic, BasicDeliverEventArgs eventArgs) + { + Guard.Against.NullOrWhiteSpace(topic, nameof(topic)); + Guard.Against.Null(eventArgs, nameof(eventArgs)); + + return new MessageReceivedEventArgs( + new Message( + body: eventArgs.Body.ToArray(), + messageDescription: topic, + messageId: eventArgs.BasicProperties.MessageId, + applicationId: eventArgs.BasicProperties.AppId, + contentType: eventArgs.BasicProperties.ContentType, + correlationId: eventArgs.BasicProperties.CorrelationId, + creationDateTime: DateTime.Parse(Encoding.UTF8.GetString((byte[])eventArgs.BasicProperties.Headers["CreationDateTime"]), CultureInfo.InvariantCulture), + deliveryTag: eventArgs.DeliveryTag.ToString(CultureInfo.InvariantCulture)), + new CancellationToken()); + } } } diff --git a/src/Messaging/Test/JsonMessageTest.cs b/src/Messaging/Test/JsonMessageTest.cs index c7c2e1a..f31635f 100644 --- a/src/Messaging/Test/JsonMessageTest.cs +++ b/src/Messaging/Test/JsonMessageTest.cs @@ -7,9 +7,33 @@ namespace Monai.Deploy.Messaging.Test { + public class DummyTypeOne + { + public string? MyProperty { get; set; } + } + + public class DummyTypeTwo + { + public int MyProperty { get; set; } + } + public class JsonMessageTest { - [Fact(DisplayName = "Converts JSONMessage to Message")] + [Fact(DisplayName = "Convert returns null on different type")] + public void ConvertsReturnsNull() + { + var data = new DummyTypeOne { MyProperty = "hello world" }; + var jsonMessage = new JsonMessage(data, Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + var message = jsonMessage.ToMessage(); + + + var result = message.ConvertTo(); + Assert.Null(result); + + var jsonMessageResult = message.ConvertToJsonMessage(); + Assert.Null(jsonMessageResult); + } + [Fact(DisplayName = "Converts JsonMessage to Message")] public void ConvertsJsonMessageToMessage() { var expected = "hello world"; @@ -28,5 +52,32 @@ public void ConvertsJsonMessageToMessage() Assert.Equal(expected, result); } + + [Fact(DisplayName = "Converts Message to JsonMessage")] + public void ConvertsMessageToJsonMessage() + { + var expected = "hello world"; + var jsonMessage = new JsonMessage(expected, Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + var message = jsonMessage.ToMessage(); + + Assert.Equal(jsonMessage.ApplicationId, message.ApplicationId); + Assert.Equal(jsonMessage.CreationDateTime, message.CreationDateTime); + Assert.Equal(jsonMessage.ContentType, message.ContentType); + Assert.Equal(jsonMessage.CorrelationId, message.CorrelationId); + Assert.Equal(jsonMessage.DeliveryTag, message.DeliveryTag); + Assert.Equal(jsonMessage.MessageDescription, message.MessageDescription); + Assert.Equal(jsonMessage.MessageId, message.MessageId); + + var result = message.ConvertToJsonMessage(); + + Assert.Equal(expected, result.Body); + Assert.Equal(jsonMessage.ApplicationId, result.ApplicationId); + Assert.Equal(jsonMessage.CreationDateTime, result.CreationDateTime); + Assert.Equal(jsonMessage.ContentType, result.ContentType); + Assert.Equal(jsonMessage.CorrelationId, result.CorrelationId); + Assert.Equal(jsonMessage.DeliveryTag, result.DeliveryTag); + Assert.Equal(jsonMessage.MessageDescription, result.MessageDescription); + Assert.Equal(jsonMessage.MessageId, result.MessageId); + } } } diff --git a/src/Messaging/Test/RabbitMq/RabbitMqMessagePublisherServiceTest.cs b/src/Messaging/Test/RabbitMq/RabbitMqMessagePublisherServiceTest.cs index 40542ba..e8f3284 100644 --- a/src/Messaging/Test/RabbitMq/RabbitMqMessagePublisherServiceTest.cs +++ b/src/Messaging/Test/RabbitMq/RabbitMqMessagePublisherServiceTest.cs @@ -19,7 +19,6 @@ public class RabbitMqMessagePublisherServiceTest private readonly IOptions _options; private readonly Mock> _logger; private readonly Mock _connectionFactory; - private readonly Mock _connection; private readonly Mock _model; public RabbitMqMessagePublisherServiceTest() @@ -27,13 +26,10 @@ public RabbitMqMessagePublisherServiceTest() _options = Options.Create(new MessageBrokerServiceConfiguration()); _logger = new Mock>(); _connectionFactory = new Mock(); - _connection = new Mock(); _model = new Mock(); - _connectionFactory.Setup(p => p.CreateConnection(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(_connection.Object); - - _connection.Setup(p => p.CreateModel()).Returns(_model.Object); + _connectionFactory.Setup(p => p.CreateChannel(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(_model.Object); } [Fact(DisplayName = "Fails to validate when required keys are missing")] @@ -42,23 +38,6 @@ public void FailsToValidateWhenRequiredKeysAreMissing() Assert.Throws(() => new RabbitMqMessagePublisherService(_options, _logger.Object, _connectionFactory.Object)); } - [Fact(DisplayName = "Cleanup connections on Dispose")] - public void CleanupOnDispose() - { - _options.Value.PublisherSettings.Add(ConfigurationKeys.EndPoint, "endpoint"); - _options.Value.PublisherSettings.Add(ConfigurationKeys.Username, "username"); - _options.Value.PublisherSettings.Add(ConfigurationKeys.Password, "password"); - _options.Value.PublisherSettings.Add(ConfigurationKeys.VirtualHost, "virtual-host"); - _options.Value.PublisherSettings.Add(ConfigurationKeys.Exchange, "exchange"); - - var service = new RabbitMqMessagePublisherService(_options, _logger.Object, _connectionFactory.Object); - service.Dispose(); - - _connection.Verify(p => p.Close(), Times.Once()); - _connection.Verify(p => p.Dispose(), Times.Once()); - - } - [Fact(DisplayName = "Publishes a message")] public async Task PublishesAMessage() { @@ -96,6 +75,8 @@ public async Task PublishesAMessage() false, It.Is(p => p.Equals(basicProperties.Object)), It.IsAny>()), Times.Once()); + + _model.Verify(p => p.Dispose(), Times.Once()); } } } diff --git a/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs b/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs index 7522f85..c887358 100644 --- a/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs +++ b/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs @@ -21,7 +21,6 @@ public class RabbitMqMessageSubscriberServiceTest private readonly IOptions _options; private readonly Mock> _logger; private readonly Mock _connectionFactory; - private readonly Mock _connection; private readonly Mock _model; public RabbitMqMessageSubscriberServiceTest() @@ -29,13 +28,11 @@ public RabbitMqMessageSubscriberServiceTest() _options = Options.Create(new MessageBrokerServiceConfiguration()); _logger = new Mock>(); _connectionFactory = new Mock(); - _connection = new Mock(); _model = new Mock(); - _connectionFactory.Setup(p => p.CreateConnection(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .Returns(_connection.Object); + _connectionFactory.Setup(p => p.CreateChannel(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(_model.Object); - _connection.Setup(p => p.CreateModel()).Returns(_model.Object); } [Fact(DisplayName = "Fails to validate when required keys are missing")] @@ -57,8 +54,8 @@ public void CleanupOnDispose() var service = new RabbitMqMessageSubscriberService(_options, _logger.Object, _connectionFactory.Object); service.Dispose(); - _connection.Verify(p => p.Close(), Times.Once()); - _connection.Verify(p => p.Dispose(), Times.Once()); + _model.Verify(p => p.Close(), Times.Once()); + _model.Verify(p => p.Dispose(), Times.Once()); } [Fact(DisplayName = "Subscribes to a topic")] @@ -123,6 +120,21 @@ public void SubscribesToATopic() Assert.Equal(message.MessageId, args.Message.MessageId); Assert.Equal(message.Body, args.Message.Body); }); + + service.SubscribeAsync("topic", "queue", async (args) => + { + await System.Threading.Tasks.Task.Run(() => + { + Assert.Equal(message.ApplicationId, args.Message.ApplicationId); + Assert.Equal(message.ContentType, args.Message.ContentType); + Assert.Equal(message.MessageId, args.Message.MessageId); + Assert.Equal(message.CreationDateTime.ToUniversalTime(), args.Message.CreationDateTime.ToUniversalTime()); + Assert.Equal(message.DeliveryTag, args.Message.DeliveryTag); + Assert.Equal("topic", args.Message.MessageDescription); + Assert.Equal(message.MessageId, args.Message.MessageId); + Assert.Equal(message.Body, args.Message.Body); + }).ConfigureAwait(false); + }); } [Fact(DisplayName = "Acknowledge a message")] From 7909d03049114ca1f5f9cc7bdc6628c45b1789ae Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Tue, 26 Apr 2022 08:23:35 -0700 Subject: [PATCH 19/45] Handle corrupted messages (#12) * Rename RunnerCompleteEvent to TaskCallbackEvent with additional Metadata property. * Handle corrupted messages * Check message properties * Use RabbitMQ Timestamp * Declare exchanges are durable & !auto delete * Fix JsonProperty mapping * Include metadata in the TaskUpdateEvent Signed-off-by: Victor Chang Signed-off-by: JP LEGER --- src/Messaging/Common/Log.cs | 10 ++- ...rCompleteEvent.cs => TaskCallbackEvent.cs} | 11 ++- src/Messaging/Events/TaskDispatchEvent.cs | 2 +- src/Messaging/Events/TaskStatus.cs | 1 + src/Messaging/Events/TaskUpdateEvent.cs | 10 ++- src/Messaging/Messages/JsonMessage.cs | 4 +- src/Messaging/Messages/Message.cs | 2 +- src/Messaging/Messages/MessageBase.cs | 4 +- .../RabbitMqMessagePublisherService.cs | 12 +--- .../RabbitMqMessageSubscriberService.cs | 69 ++++++++++++++++--- src/Messaging/Test/RunnerCompleteEventTest.cs | 2 +- 11 files changed, 95 insertions(+), 32 deletions(-) rename src/Messaging/Events/{RunnerCompleteEvent.cs => TaskCallbackEvent.cs} (80%) diff --git a/src/Messaging/Common/Log.cs b/src/Messaging/Common/Log.cs index bcbd549..0bc9795 100644 --- a/src/Messaging/Common/Log.cs +++ b/src/Messaging/Common/Log.cs @@ -30,10 +30,16 @@ public static partial class Log [LoggerMessage(EventId = 10006, Level = LogLevel.Information, Message = "Sending nack message {messageId} and requeuing.")] public static partial void SendingNAcknowledgement(this ILogger logger, string messageId); - [LoggerMessage(EventId = 10007, Level = LogLevel.Information, Message = "Nack message sent for message {messageId}.")] - public static partial void NAcknowledgementSent(this ILogger logger, string messageId); + [LoggerMessage(EventId = 10007, Level = LogLevel.Information, Message = "Nack message sent for message {messageId}, requeue={requeue}.")] + public static partial void NAcknowledgementSent(this ILogger logger, string messageId, bool requeue); [LoggerMessage(EventId = 10008, Level = LogLevel.Information, Message = "Closing connections.")] public static partial void ClosingConnections(this ILogger logger); + + [LoggerMessage(EventId = 10009, Level = LogLevel.Error, Message = "Invalid or corrupted message received: Queue={queueName}, Topic={topic}, Message ID={messageId}.")] + public static partial void InvalidMessage(this ILogger logger, string queueName, string topic, string messageId, Exception ex); + + [LoggerMessage(EventId = 10010, Level = LogLevel.Error, Message = "Exception not handled by the subscriber's callback function: Queue={queueName}, Topic={topic}, Message ID={messageId}.")] + public static partial void ErrorNotHandledByCallback(this ILogger logger, string queueName, string topic, string messageId, Exception ex); } } diff --git a/src/Messaging/Events/RunnerCompleteEvent.cs b/src/Messaging/Events/TaskCallbackEvent.cs similarity index 80% rename from src/Messaging/Events/RunnerCompleteEvent.cs rename to src/Messaging/Events/TaskCallbackEvent.cs index 5f65b8f..c6d2ba1 100644 --- a/src/Messaging/Events/RunnerCompleteEvent.cs +++ b/src/Messaging/Events/TaskCallbackEvent.cs @@ -6,7 +6,7 @@ namespace Monai.Deploy.Messaging.Events { - public class RunnerCompleteEvent : EventBase + public class TaskCallbackEvent : EventBase { /// /// Gets or sets the ID representing the instance of the workflow. @@ -43,13 +43,20 @@ public class RunnerCompleteEvent : EventBase [Required, MaxLength(63)] public string Identity { get; set; } - public RunnerCompleteEvent() + /// + /// Gets or sets any metadata generated by the task, including any output generated. + /// + [JsonProperty(PropertyName = "metadata")] + public Dictionary Metadata { get; set; } + + public TaskCallbackEvent() { WorkflowId = String.Empty; TaskId = String.Empty; ExecutionId = String.Empty; CorrelationId = String.Empty; Identity = String.Empty; + Metadata = new Dictionary(); } } } diff --git a/src/Messaging/Events/TaskDispatchEvent.cs b/src/Messaging/Events/TaskDispatchEvent.cs index 113ebc9..0a2252b 100644 --- a/src/Messaging/Events/TaskDispatchEvent.cs +++ b/src/Messaging/Events/TaskDispatchEvent.cs @@ -74,7 +74,7 @@ public class TaskDispatchEvent : EventBase public List Outputs { get; set; } /// - /// Gets or sets the task execution arguments. + /// Gets or sets any metadata relevant to the task. /// [JsonProperty(PropertyName = "metadata")] public Dictionary Metadata { get; set; } diff --git a/src/Messaging/Events/TaskStatus.cs b/src/Messaging/Events/TaskStatus.cs index 8d94c5d..314ce56 100644 --- a/src/Messaging/Events/TaskStatus.cs +++ b/src/Messaging/Events/TaskStatus.cs @@ -7,6 +7,7 @@ public enum TaskStatus { Unknown, Created, + Dispatched, Accepted, Succeeded, Failed, diff --git a/src/Messaging/Events/TaskUpdateEvent.cs b/src/Messaging/Events/TaskUpdateEvent.cs index 801cf0d..2a7706e 100644 --- a/src/Messaging/Events/TaskUpdateEvent.cs +++ b/src/Messaging/Events/TaskUpdateEvent.cs @@ -48,7 +48,7 @@ public class TaskUpdateEvent : EventBase /// /// Gets or set the failure reason of the task. /// - [JsonProperty(PropertyName = "message")] + [JsonProperty(PropertyName = "reason")] [JsonConverter(typeof(StringEnumConverter))] [Required] public FailureReason Reason { get; set; } @@ -56,9 +56,15 @@ public class TaskUpdateEvent : EventBase /// /// Gets or set any additional (error) message related to the task. /// - [JsonProperty(PropertyName = "reason")] + [JsonProperty(PropertyName = "message")] public string Message { get; set; } + /// + /// Gets or sets any metadata relevant to the output of the task. + /// + [JsonProperty(PropertyName = "metadata")] + public Dictionary Metadata { get; set; } + public TaskUpdateEvent() { WorkflowId = String.Empty; diff --git a/src/Messaging/Messages/JsonMessage.cs b/src/Messaging/Messages/JsonMessage.cs index 6c8cdb0..a84b743 100644 --- a/src/Messaging/Messages/JsonMessage.cs +++ b/src/Messaging/Messages/JsonMessage.cs @@ -24,7 +24,7 @@ public JsonMessage(T body, Guid.NewGuid().ToString(), applicationId, correlationId, - DateTime.UtcNow, + DateTimeOffset.UtcNow, deliveryTag) { } @@ -34,7 +34,7 @@ public JsonMessage(T body, string messageId, string applicationId, string correlationId, - DateTime creationDateTime, + DateTimeOffset creationDateTime, string deliveryTag) : base(messageId, messageDescription, MediaTypeNames.Application.Json, applicationId, correlationId, creationDateTime) { diff --git a/src/Messaging/Messages/Message.cs b/src/Messaging/Messages/Message.cs index 1e146c9..c2c32bf 100644 --- a/src/Messaging/Messages/Message.cs +++ b/src/Messaging/Messages/Message.cs @@ -19,7 +19,7 @@ public Message(byte[] body, string applicationId, string contentType, string correlationId, - DateTime creationDateTime, + DateTimeOffset creationDateTime, string deliveryTag = "") : base(messageId, messageDescription, contentType, applicationId, correlationId, creationDateTime) { diff --git a/src/Messaging/Messages/MessageBase.cs b/src/Messaging/Messages/MessageBase.cs index 1abc105..e206599 100644 --- a/src/Messaging/Messages/MessageBase.cs +++ b/src/Messaging/Messages/MessageBase.cs @@ -39,7 +39,7 @@ public abstract class MessageBase /// /// Datetime the message is created. /// - public DateTime CreationDateTime { get; private set; } + public DateTimeOffset CreationDateTime { get; private set; } /// /// Gets or set the delivery tag/acknoweldge token for the message. @@ -51,7 +51,7 @@ protected MessageBase(string messageId, string contentType, string applicationId, string correlationId, - DateTime creationDateTime) + DateTimeOffset creationDateTime) { Guard.Against.NullOrWhiteSpace(messageId, nameof(messageId)); Guard.Against.NullOrWhiteSpace(messageDescription, nameof(messageDescription)); diff --git a/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs b/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs index cdf6cf8..1c82c7b 100644 --- a/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs +++ b/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs @@ -33,7 +33,6 @@ public RabbitMqMessagePublisherService(IOptions - { - { "CreationDateTime", message.CreationDateTime.ToString("o") } - }; + channel.ExchangeDeclare(_exchange, ExchangeType.Topic, durable: true, autoDelete: false); var properties = channel.CreateBasicProperties(); properties.Persistent = true; @@ -85,8 +78,9 @@ public Task Publish(string topic, Message message) properties.AppId = message.ApplicationId; properties.CorrelationId = message.CorrelationId; properties.DeliveryMode = PersistentDeliveryMode; + properties.Type = message.MessageDescription; + properties.Timestamp = new AmqpTimestamp(message.CreationDateTime.ToUnixTimeSeconds()); - properties.Headers = propertiesDictionary; channel.BasicPublish(exchange: _exchange, routingKey: topic, basicProperties: properties, diff --git a/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs b/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs index 84420c6..5fcb162 100644 --- a/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs +++ b/src/Messaging/RabbitMq/RabbitMqMessageSubscriberService.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: Apache License 2.0 using System.Globalization; -using System.Text; using Ardalis.GuardClauses; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -43,7 +42,7 @@ public RabbitMqMessageSubscriberService(IOptions(() => runnerComplete.Validate()); runnerComplete.WorkflowId = Guid.NewGuid().ToString(); From 19b141d7a8f42c8e079776eac72520c5bae13654 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Tue, 26 Apr 2022 14:15:53 -0700 Subject: [PATCH 20/45] Update TaskUpdateEvent.cs Signed-off-by: JP LEGER --- src/Messaging/Events/TaskUpdateEvent.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Messaging/Events/TaskUpdateEvent.cs b/src/Messaging/Events/TaskUpdateEvent.cs index 2a7706e..d68b207 100644 --- a/src/Messaging/Events/TaskUpdateEvent.cs +++ b/src/Messaging/Events/TaskUpdateEvent.cs @@ -74,6 +74,7 @@ public TaskUpdateEvent() Status = TaskStatus.Unknown; Reason = FailureReason.None; Message = String.Empty; + Metadata = new Dictionary(); } } } From 6436c664aa686f239ff70007009678bd6c38c2d9 Mon Sep 17 00:00:00 2001 From: jackschofield23 Date: Thu, 28 Apr 2022 14:03:10 +0100 Subject: [PATCH 21/45] update task dispatch event Signed-off-by: jackschofield23 Signed-off-by: JP LEGER --- src/Messaging/Common/Credentials.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Messaging/Common/Credentials.cs b/src/Messaging/Common/Credentials.cs index 5a7b45a..ce3381a 100644 --- a/src/Messaging/Common/Credentials.cs +++ b/src/Messaging/Common/Credentials.cs @@ -22,5 +22,11 @@ public class Credentials [Required] public string? AccessToken { get; set; } + /// + /// Session token of the credentials pair. + /// + [JsonProperty(PropertyName = "session_token")] + public string? SessionToken { get; set; } + } } From 50c66fc3a40014dbf44452af9d716d4e7e44d099 Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Thu, 5 May 2022 12:22:45 -0700 Subject: [PATCH 22/45] Make credentials optional in TaskDispatchEvent.Storage (#17) Signed-off-by: Victor Chang Signed-off-by: JP LEGER --- src/Messaging/Common/Credentials.cs | 18 ++++++++----- .../Common/MessageValidationException.cs | 2 +- src/Messaging/Common/Storage.cs | 21 ++++++++++----- src/Messaging/Events/TaskDispatchEvent.cs | 16 ++++++------ src/Messaging/Test/TaskDispatchEventTest.cs | 26 +++++++++---------- 5 files changed, 48 insertions(+), 35 deletions(-) diff --git a/src/Messaging/Common/Credentials.cs b/src/Messaging/Common/Credentials.cs index ce3381a..f14c980 100644 --- a/src/Messaging/Common/Credentials.cs +++ b/src/Messaging/Common/Credentials.cs @@ -9,24 +9,30 @@ namespace Monai.Deploy.Messaging.Common public class Credentials { /// - /// Access key or username of the credentials pair. + /// Gets or sets the access key or user name of the credentials pair. /// [JsonProperty(PropertyName = "access_key")] [Required] - public string? AccessKey { get; set; } + public string AccessKey { get; set; } /// - /// Access token or password of the credentials pair. + /// Gets or sets the access token or password of the credentials pair. /// [JsonProperty(PropertyName = "access_token")] [Required] - public string? AccessToken { get; set; } + public string AccessToken { get; set; } /// - /// Session token of the credentials pair. + /// Gets or sets the session token of the credentials pair. /// [JsonProperty(PropertyName = "session_token")] - public string? SessionToken { get; set; } + public string SessionToken { get; set; } + public Credentials() + { + AccessKey = string.Empty; + AccessToken = string.Empty; + SessionToken = string.Empty; + } } } diff --git a/src/Messaging/Common/MessageValidationException.cs b/src/Messaging/Common/MessageValidationException.cs index fb055a9..9be16e2 100644 --- a/src/Messaging/Common/MessageValidationException.cs +++ b/src/Messaging/Common/MessageValidationException.cs @@ -20,7 +20,7 @@ protected MessageValidationException(SerializationInfo info, StreamingContext co private static string FormatMessage(List errors) { - if (errors == null || errors.Count == 0) + if (errors is null || errors.Count == 0) { return "Invalid message."; } diff --git a/src/Messaging/Common/Storage.cs b/src/Messaging/Common/Storage.cs index efe5050..3a6784e 100644 --- a/src/Messaging/Common/Storage.cs +++ b/src/Messaging/Common/Storage.cs @@ -14,20 +14,19 @@ public class Storage /// [JsonProperty(PropertyName = "name")] [Required] - public string? Name { get; set; } + public string Name { get; set; } /// /// Gets or sets the endpoint of the storage service. /// [JsonProperty(PropertyName = "endpoint")] [Required] - public string? Endpoint { get; set; } + public string Endpoint { get; set; } /// /// Gets or sets credentials for accessing the storage service. /// [JsonProperty(PropertyName = "credentials")] - [Required] public Credentials? Credentials { get; set; } /// @@ -35,19 +34,29 @@ public class Storage /// [JsonProperty(PropertyName = "bucket")] [Required] - public string? Bucket { get; set; } + public string Bucket { get; set; } /// /// Gets or sets whether the connection should be secured or not. /// [JsonProperty(PropertyName = "secured_connection")] - public bool SecuredConnection { get; set; } = false; + public bool SecuredConnection { get; set; } /// /// Gets or sets the optional relative root path to the data. /// [JsonProperty(PropertyName = "relative_root_path")] [Required] - public string? RelativeRootPath { get; set; } + public string RelativeRootPath { get; set; } + + public Storage() + { + Name = string.Empty; + Endpoint = string.Empty; + Credentials = null; + Bucket = string.Empty; + SecuredConnection = false; + RelativeRootPath = string.Empty; + } } } diff --git a/src/Messaging/Events/TaskDispatchEvent.cs b/src/Messaging/Events/TaskDispatchEvent.cs index 0a2252b..090ec37 100644 --- a/src/Messaging/Events/TaskDispatchEvent.cs +++ b/src/Messaging/Events/TaskDispatchEvent.cs @@ -39,11 +39,11 @@ public class TaskDispatchEvent : EventBase public string CorrelationId { get; set; } /// - /// Gets or sets the fully qualified assembly name of the task plug-in for the task. + /// Gets or sets the type of plug-in the task is associated with. /// - [JsonProperty(PropertyName = "task_assembly_name")] + [JsonProperty(PropertyName = "type")] [Required] - public string TaskAssemblyName { get; set; } + public string TaskPluginType { get; set; } /// /// Gets or sets the task execution arguments. @@ -81,11 +81,11 @@ public class TaskDispatchEvent : EventBase public TaskDispatchEvent() { - WorkflowId = String.Empty; - TaskId = String.Empty; - ExecutionId = String.Empty; - CorrelationId = String.Empty; - TaskAssemblyName = String.Empty; + WorkflowId = string.Empty; + TaskId = string.Empty; + ExecutionId = string.Empty; + CorrelationId = string.Empty; + TaskPluginType = string.Empty; TaskPluginArguments = new Dictionary(); Status = TaskStatus.Unknown; Inputs = new List(); diff --git a/src/Messaging/Test/TaskDispatchEventTest.cs b/src/Messaging/Test/TaskDispatchEventTest.cs index 7a4ed3e..5bce4bc 100644 --- a/src/Messaging/Test/TaskDispatchEventTest.cs +++ b/src/Messaging/Test/TaskDispatchEventTest.cs @@ -29,7 +29,7 @@ public void ValidationThrowsOnError() taskDispatchEvent.CorrelationId = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.TaskAssemblyName = Guid.NewGuid().ToString(); + taskDispatchEvent.TaskPluginType = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); taskDispatchEvent.Inputs = new List(); @@ -52,14 +52,7 @@ public void ValidationThrowsOnError() output.Endpoint = "endpoint"; Assert.Throws(() => taskDispatchEvent.Validate()); - output.Credentials = new Credentials(); - Assert.Throws(() => taskDispatchEvent.Validate()); - - output.Credentials.AccessToken = "token"; - Assert.Throws(() => taskDispatchEvent.Validate()); - - output.Credentials.AccessKey = "key"; - Assert.Throws(() => taskDispatchEvent.Validate()); + // Skip settings credentials for output, this shall not throw given that is not required output.Bucket = "bucket"; Assert.Throws(() => taskDispatchEvent.Validate()); @@ -69,15 +62,20 @@ public void ValidationThrowsOnError() input.Name = "name"; input.Endpoint = "endpoint"; - input.Credentials = new Credentials - { - AccessToken = "token", - AccessKey = "key" - }; input.Bucket = "bucket"; input.RelativeRootPath = "path"; + var exception = Record.Exception(() => taskDispatchEvent.Validate()); Assert.Null(exception); + + // Let's set the credentials for input, this should throw validation exception given that it's no longer null + input.Credentials = new Credentials(); + Assert.Throws(() => taskDispatchEvent.Validate()); + + input.Credentials.AccessKey = "key"; + input.Credentials.AccessToken = "token"; + exception = Record.Exception(() => taskDispatchEvent.Validate()); + Assert.Null(exception); } } } From bdb5be73f210d7850da2b43074db72c81d564fdc Mon Sep 17 00:00:00 2001 From: jackschofield23 Date: Fri, 6 May 2022 08:39:25 +0100 Subject: [PATCH 23/45] change task status name Signed-off-by: jackschofield23 Signed-off-by: JP LEGER --- src/Messaging/Events/TaskDispatchEvent.cs | 4 ++-- .../Events/{TaskStatus.cs => TaskExecutionStatus.cs} | 2 +- src/Messaging/Events/TaskUpdateEvent.cs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) rename src/Messaging/Events/{TaskStatus.cs => TaskExecutionStatus.cs} (88%) diff --git a/src/Messaging/Events/TaskDispatchEvent.cs b/src/Messaging/Events/TaskDispatchEvent.cs index 090ec37..bc30875 100644 --- a/src/Messaging/Events/TaskDispatchEvent.cs +++ b/src/Messaging/Events/TaskDispatchEvent.cs @@ -57,7 +57,7 @@ public class TaskDispatchEvent : EventBase [JsonProperty(PropertyName = "status")] [JsonConverter(typeof(StringEnumConverter))] [Required] - public TaskStatus Status { get; set; } + public TaskExecutionStatus Status { get; set; } /// /// Gets or sets the input storage information. @@ -87,7 +87,7 @@ public TaskDispatchEvent() CorrelationId = string.Empty; TaskPluginType = string.Empty; TaskPluginArguments = new Dictionary(); - Status = TaskStatus.Unknown; + Status = TaskExecutionStatus.Unknown; Inputs = new List(); Outputs = new List(); Metadata = new Dictionary(); diff --git a/src/Messaging/Events/TaskStatus.cs b/src/Messaging/Events/TaskExecutionStatus.cs similarity index 88% rename from src/Messaging/Events/TaskStatus.cs rename to src/Messaging/Events/TaskExecutionStatus.cs index 314ce56..790389b 100644 --- a/src/Messaging/Events/TaskStatus.cs +++ b/src/Messaging/Events/TaskExecutionStatus.cs @@ -3,7 +3,7 @@ namespace Monai.Deploy.Messaging.Events { - public enum TaskStatus + public enum TaskExecutionStatus { Unknown, Created, diff --git a/src/Messaging/Events/TaskUpdateEvent.cs b/src/Messaging/Events/TaskUpdateEvent.cs index d68b207..5d2f3e7 100644 --- a/src/Messaging/Events/TaskUpdateEvent.cs +++ b/src/Messaging/Events/TaskUpdateEvent.cs @@ -43,7 +43,7 @@ public class TaskUpdateEvent : EventBase [JsonProperty(PropertyName = "status")] [JsonConverter(typeof(StringEnumConverter))] [Required] - public TaskStatus Status { get; set; } + public TaskExecutionStatus Status { get; set; } /// /// Gets or set the failure reason of the task. @@ -71,7 +71,7 @@ public TaskUpdateEvent() TaskId = String.Empty; ExecutionId = String.Empty; CorrelationId = String.Empty; - Status = TaskStatus.Unknown; + Status = TaskExecutionStatus.Unknown; Reason = FailureReason.None; Message = String.Empty; Metadata = new Dictionary(); From f637151405ebf91e49e366c88eb6dcc72677e390 Mon Sep 17 00:00:00 2001 From: jackschofield23 <56344499+jackschofield23@users.noreply.github.com> Date: Tue, 10 May 2022 19:05:15 +0100 Subject: [PATCH 24/45] Rename workflow instance id (#20) Signed-off-by: jackschofield23 Signed-off-by: JP LEGER --- src/Messaging/Events/TaskCallbackEvent.cs | 6 +++--- src/Messaging/Events/TaskDispatchEvent.cs | 6 +++--- src/Messaging/Events/TaskUpdateEvent.cs | 6 +++--- src/Messaging/Test/RunnerCompleteEventTest.cs | 2 +- src/Messaging/Test/TaskDispatchEventTest.cs | 2 +- src/Messaging/Test/TaskUpdateEventTest.cs | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/Messaging/Events/TaskCallbackEvent.cs b/src/Messaging/Events/TaskCallbackEvent.cs index c6d2ba1..1866100 100644 --- a/src/Messaging/Events/TaskCallbackEvent.cs +++ b/src/Messaging/Events/TaskCallbackEvent.cs @@ -11,9 +11,9 @@ public class TaskCallbackEvent : EventBase /// /// Gets or sets the ID representing the instance of the workflow. /// - [JsonProperty(PropertyName = "workflow_id")] + [JsonProperty(PropertyName = "workflow_instance_id")] [Required] - public string WorkflowId { get; set; } + public string WorkflowInstanceId { get; set; } /// /// Gets or sets the ID representing the instance of the Task. @@ -51,7 +51,7 @@ public class TaskCallbackEvent : EventBase public TaskCallbackEvent() { - WorkflowId = String.Empty; + WorkflowInstanceId = String.Empty; TaskId = String.Empty; ExecutionId = String.Empty; CorrelationId = String.Empty; diff --git a/src/Messaging/Events/TaskDispatchEvent.cs b/src/Messaging/Events/TaskDispatchEvent.cs index bc30875..b2f189f 100644 --- a/src/Messaging/Events/TaskDispatchEvent.cs +++ b/src/Messaging/Events/TaskDispatchEvent.cs @@ -13,9 +13,9 @@ public class TaskDispatchEvent : EventBase /// /// Gets or sets the ID representing the instance of the workflow. /// - [JsonProperty(PropertyName = "workflow_id")] + [JsonProperty(PropertyName = "workflow_instance_id")] [Required] - public string WorkflowId { get; set; } + public string WorkflowInstanceId { get; set; } /// /// Gets or sets the ID representing the instance of the Task. @@ -81,7 +81,7 @@ public class TaskDispatchEvent : EventBase public TaskDispatchEvent() { - WorkflowId = string.Empty; + WorkflowInstanceId = string.Empty; TaskId = string.Empty; ExecutionId = string.Empty; CorrelationId = string.Empty; diff --git a/src/Messaging/Events/TaskUpdateEvent.cs b/src/Messaging/Events/TaskUpdateEvent.cs index 5d2f3e7..850a6b8 100644 --- a/src/Messaging/Events/TaskUpdateEvent.cs +++ b/src/Messaging/Events/TaskUpdateEvent.cs @@ -12,9 +12,9 @@ public class TaskUpdateEvent : EventBase /// /// Gets or sets the ID representing the instance of the workflow. /// - [JsonProperty(PropertyName = "workflow_id")] + [JsonProperty(PropertyName = "workflow_instance_id")] [Required] - public string WorkflowId { get; set; } + public string WorkflowInstanceId { get; set; } /// /// Gets or sets the ID representing the instance of the Task. @@ -67,7 +67,7 @@ public class TaskUpdateEvent : EventBase public TaskUpdateEvent() { - WorkflowId = String.Empty; + WorkflowInstanceId = String.Empty; TaskId = String.Empty; ExecutionId = String.Empty; CorrelationId = String.Empty; diff --git a/src/Messaging/Test/RunnerCompleteEventTest.cs b/src/Messaging/Test/RunnerCompleteEventTest.cs index da7be3f..5195f28 100644 --- a/src/Messaging/Test/RunnerCompleteEventTest.cs +++ b/src/Messaging/Test/RunnerCompleteEventTest.cs @@ -16,7 +16,7 @@ public void ValidationThrowsOnError() var runnerComplete = new TaskCallbackEvent(); Assert.Throws(() => runnerComplete.Validate()); - runnerComplete.WorkflowId = Guid.NewGuid().ToString(); + runnerComplete.WorkflowInstanceId = Guid.NewGuid().ToString(); Assert.Throws(() => runnerComplete.Validate()); runnerComplete.TaskId = Guid.NewGuid().ToString(); diff --git a/src/Messaging/Test/TaskDispatchEventTest.cs b/src/Messaging/Test/TaskDispatchEventTest.cs index 5bce4bc..6de17e1 100644 --- a/src/Messaging/Test/TaskDispatchEventTest.cs +++ b/src/Messaging/Test/TaskDispatchEventTest.cs @@ -17,7 +17,7 @@ public void ValidationThrowsOnError() var taskDispatchEvent = new TaskDispatchEvent(); Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.WorkflowId = Guid.NewGuid().ToString(); + taskDispatchEvent.WorkflowInstanceId = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); taskDispatchEvent.ExecutionId = Guid.NewGuid().ToString(); diff --git a/src/Messaging/Test/TaskUpdateEventTest.cs b/src/Messaging/Test/TaskUpdateEventTest.cs index 56fbfb9..4777d13 100644 --- a/src/Messaging/Test/TaskUpdateEventTest.cs +++ b/src/Messaging/Test/TaskUpdateEventTest.cs @@ -16,7 +16,7 @@ public void ValidationThrowsOnError() var taskDispatchEvent = new TaskUpdateEvent(); Assert.Throws(() => taskDispatchEvent.Validate()); - taskDispatchEvent.WorkflowId = Guid.NewGuid().ToString(); + taskDispatchEvent.WorkflowInstanceId = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); taskDispatchEvent.ExecutionId = Guid.NewGuid().ToString(); From 5f360a7f951da07933687119c13345e082363a6c Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Fri, 13 May 2022 09:28:27 -0700 Subject: [PATCH 25/45] Intermediate Storage for TaskDispatchEvent (#21) * Add intermediate storage for TaskDispatchEvent * Implement ICloneable for Storage Signed-off-by: Victor Chang Signed-off-by: JP LEGER --- src/Messaging/Common/Storage.cs | 7 ++++++- src/Messaging/Events/TaskDispatchEvent.cs | 8 ++++++++ ...leteEventTest.cs => TaskCallbackEventTest.cs} | 2 +- src/Messaging/Test/TaskDispatchEventTest.cs | 16 ++++++++++++++++ 4 files changed, 31 insertions(+), 2 deletions(-) rename src/Messaging/Test/{RunnerCompleteEventTest.cs => TaskCallbackEventTest.cs} (97%) diff --git a/src/Messaging/Common/Storage.cs b/src/Messaging/Common/Storage.cs index 3a6784e..424f4f3 100644 --- a/src/Messaging/Common/Storage.cs +++ b/src/Messaging/Common/Storage.cs @@ -6,7 +6,7 @@ namespace Monai.Deploy.Messaging.Common { - public class Storage + public class Storage : ICloneable { /// /// Gets or sets the name of the artifact. @@ -58,5 +58,10 @@ public Storage() SecuredConnection = false; RelativeRootPath = string.Empty; } + + public object Clone() + { + return MemberwiseClone(); + } } } diff --git a/src/Messaging/Events/TaskDispatchEvent.cs b/src/Messaging/Events/TaskDispatchEvent.cs index b2f189f..20d511f 100644 --- a/src/Messaging/Events/TaskDispatchEvent.cs +++ b/src/Messaging/Events/TaskDispatchEvent.cs @@ -73,6 +73,13 @@ public class TaskDispatchEvent : EventBase [Required] public List Outputs { get; set; } + /// + /// Gets or sets the intermediate storage information. + /// + [JsonProperty(PropertyName = "intermediate_storage")] + [Required] + public Storage IntermediateStorage { get; set; } + /// /// Gets or sets any metadata relevant to the task. /// @@ -90,6 +97,7 @@ public TaskDispatchEvent() Status = TaskExecutionStatus.Unknown; Inputs = new List(); Outputs = new List(); + IntermediateStorage = null!; Metadata = new Dictionary(); } } diff --git a/src/Messaging/Test/RunnerCompleteEventTest.cs b/src/Messaging/Test/TaskCallbackEventTest.cs similarity index 97% rename from src/Messaging/Test/RunnerCompleteEventTest.cs rename to src/Messaging/Test/TaskCallbackEventTest.cs index 5195f28..ca670a2 100644 --- a/src/Messaging/Test/RunnerCompleteEventTest.cs +++ b/src/Messaging/Test/TaskCallbackEventTest.cs @@ -8,7 +8,7 @@ namespace Monai.Deploy.Messaging.Test { - public class RunnerCompleteEventTest + public class TaskCallbackEventTest { [Fact(DisplayName = "Validation throws on error")] public void ValidationThrowsOnError() diff --git a/src/Messaging/Test/TaskDispatchEventTest.cs b/src/Messaging/Test/TaskDispatchEventTest.cs index 6de17e1..58088fd 100644 --- a/src/Messaging/Test/TaskDispatchEventTest.cs +++ b/src/Messaging/Test/TaskDispatchEventTest.cs @@ -60,6 +60,22 @@ public void ValidationThrowsOnError() output.RelativeRootPath = "path"; Assert.Throws(() => taskDispatchEvent.Validate()); + var intermediate = new Storage(); + taskDispatchEvent.IntermediateStorage = intermediate; + Assert.Throws(() => taskDispatchEvent.Validate()); + + intermediate.Name = "name"; + Assert.Throws(() => taskDispatchEvent.Validate()); + + intermediate.Endpoint = "endpoint"; + Assert.Throws(() => taskDispatchEvent.Validate()); + + intermediate.Bucket = "bucket"; + Assert.Throws(() => taskDispatchEvent.Validate()); + + intermediate.RelativeRootPath = "path"; + Assert.Throws(() => taskDispatchEvent.Validate()); + input.Name = "name"; input.Endpoint = "endpoint"; input.Bucket = "bucket"; From 64437ab0daaa8bb2cac71a0830452a1824fd5e1a Mon Sep 17 00:00:00 2001 From: Victor Chang Date: Tue, 17 May 2022 10:14:46 -0700 Subject: [PATCH 26/45] Throw MessageConversionException on error converting JSON event message (#22) Signed-off-by: Victor Chang Signed-off-by: JP LEGER --- .../Common/MessageConversionException.cs | 27 +++++++++++++++++++ src/Messaging/Messages/Message.cs | 12 ++++----- src/Messaging/Test/JsonMessageTest.cs | 12 ++++----- 3 files changed, 38 insertions(+), 13 deletions(-) create mode 100644 src/Messaging/Common/MessageConversionException.cs diff --git a/src/Messaging/Common/MessageConversionException.cs b/src/Messaging/Common/MessageConversionException.cs new file mode 100644 index 0000000..3f5dac6 --- /dev/null +++ b/src/Messaging/Common/MessageConversionException.cs @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: © 2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 +// +using System.Runtime.Serialization; + +namespace Monai.Deploy.Messaging.Common +{ + [Serializable] + public class MessageConversionException : Exception + { + public MessageConversionException() + { + } + + public MessageConversionException(string message) : base(message) + { + } + + public MessageConversionException(string message, Exception innerException) : base(message, innerException) + { + } + + protected MessageConversionException(SerializationInfo info, StreamingContext context) : base(info, context) + { + } + } +} diff --git a/src/Messaging/Messages/Message.cs b/src/Messaging/Messages/Message.cs index c2c32bf..144d0ba 100644 --- a/src/Messaging/Messages/Message.cs +++ b/src/Messaging/Messages/Message.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache License 2.0 using System.Text; +using Monai.Deploy.Messaging.Common; using Newtonsoft.Json; namespace Monai.Deploy.Messaging.Messages @@ -39,9 +40,9 @@ public T ConvertTo() var json = Encoding.UTF8.GetString(Body); return JsonConvert.DeserializeObject(json)!; } - catch + catch(Exception ex) { - return default!; + throw new MessageConversionException($"Error converting message to type {typeof(T)}", ex); } } @@ -54,13 +55,12 @@ public JsonMessage ConvertToJsonMessage() { try { - var json = Encoding.UTF8.GetString(Body); - var body = JsonConvert.DeserializeObject(json)!; + var body = ConvertTo(); return new JsonMessage(body, MessageDescription, MessageId, ApplicationId, CorrelationId, CreationDateTime, DeliveryTag); } - catch + catch (Exception ex) { - return null!; + throw new MessageConversionException($"Error converting message to type {typeof(T)}", ex); } } } diff --git a/src/Messaging/Test/JsonMessageTest.cs b/src/Messaging/Test/JsonMessageTest.cs index f31635f..1bab7b7 100644 --- a/src/Messaging/Test/JsonMessageTest.cs +++ b/src/Messaging/Test/JsonMessageTest.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache License 2.0 using System; +using Monai.Deploy.Messaging.Common; using Monai.Deploy.Messaging.Messages; using Xunit; @@ -19,19 +20,16 @@ public class DummyTypeTwo public class JsonMessageTest { - [Fact(DisplayName = "Convert returns null on different type")] - public void ConvertsReturnsNull() + [Fact(DisplayName = "Convert throws on different type")] + public void ConvertsThrowsError() { var data = new DummyTypeOne { MyProperty = "hello world" }; var jsonMessage = new JsonMessage(data, Guid.NewGuid().ToString(), Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); var message = jsonMessage.ToMessage(); - var result = message.ConvertTo(); - Assert.Null(result); - - var jsonMessageResult = message.ConvertToJsonMessage(); - Assert.Null(jsonMessageResult); + Assert.Throws(() => message.ConvertTo()); + Assert.Throws(() => message.ConvertToJsonMessage()); } [Fact(DisplayName = "Converts JsonMessage to Message")] public void ConvertsJsonMessageToMessage() From 310d005ece92c6acc5afc3ddceb9df00dff07d02 Mon Sep 17 00:00:00 2001 From: Jack Schofield Date: Thu, 19 May 2022 11:54:27 +0100 Subject: [PATCH 27/45] update taskupdate to include output artifacts Signed-off-by: Jack Schofield Signed-off-by: JP LEGER --- src/Messaging/Events/TaskUpdateEvent.cs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Messaging/Events/TaskUpdateEvent.cs b/src/Messaging/Events/TaskUpdateEvent.cs index 850a6b8..f0092c6 100644 --- a/src/Messaging/Events/TaskUpdateEvent.cs +++ b/src/Messaging/Events/TaskUpdateEvent.cs @@ -59,6 +59,12 @@ public class TaskUpdateEvent : EventBase [JsonProperty(PropertyName = "message")] public string Message { get; set; } + /// + /// Gets or sets any output artifacts relevent to the output of the task. + /// + [JsonProperty(PropertyName = "output_artifacts")] + public Dictionary OutputArtifacts { get; set; } + /// /// Gets or sets any metadata relevant to the output of the task. /// @@ -75,6 +81,7 @@ public TaskUpdateEvent() Reason = FailureReason.None; Message = String.Empty; Metadata = new Dictionary(); + OutputArtifacts = new Dictionary(); } } } From 1761aad03bc6bc8f9b0cdb33734a561da222a107 Mon Sep 17 00:00:00 2001 From: jackschofield23 <56344499+jackschofield23@users.noreply.github.com> Date: Mon, 30 May 2022 09:30:30 +0100 Subject: [PATCH 28/45] update task callback event (#24) Signed-off-by: Jack Schofield Signed-off-by: JP LEGER --- src/Messaging/Events/TaskCallbackEvent.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Messaging/Events/TaskCallbackEvent.cs b/src/Messaging/Events/TaskCallbackEvent.cs index 1866100..476a40a 100644 --- a/src/Messaging/Events/TaskCallbackEvent.cs +++ b/src/Messaging/Events/TaskCallbackEvent.cs @@ -49,7 +49,13 @@ public class TaskCallbackEvent : EventBase [JsonProperty(PropertyName = "metadata")] public Dictionary Metadata { get; set; } - public TaskCallbackEvent() + /// + /// Gets or sets any output artifacts generated by the task. + /// + [JsonProperty(PropertyName = "output_artifacts")] + public Dictionary OutputArtifacts { get; set; } + + public TaskCallbackEvent() { WorkflowInstanceId = String.Empty; TaskId = String.Empty; @@ -57,6 +63,7 @@ public TaskCallbackEvent() CorrelationId = String.Empty; Identity = String.Empty; Metadata = new Dictionary(); + OutputArtifacts = new Dictionary(); } } } From 8f63ccd7af2770ea3fd593ab7fdf4086f7fe8c13 Mon Sep 17 00:00:00 2001 From: jackschofield23 <56344499+jackschofield23@users.noreply.github.com> Date: Mon, 30 May 2022 14:10:41 +0100 Subject: [PATCH 29/45] fix task callback and task update event (#25) Signed-off-by: Jack Schofield Signed-off-by: JP LEGER --- src/Messaging/Events/TaskCallbackEvent.cs | 11 ++++++----- src/Messaging/Events/TaskUpdateEvent.cs | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/Messaging/Events/TaskCallbackEvent.cs b/src/Messaging/Events/TaskCallbackEvent.cs index 476a40a..5e0f3b8 100644 --- a/src/Messaging/Events/TaskCallbackEvent.cs +++ b/src/Messaging/Events/TaskCallbackEvent.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache License 2.0 using System.ComponentModel.DataAnnotations; +using Monai.Deploy.Messaging.Common; using Newtonsoft.Json; namespace Monai.Deploy.Messaging.Events @@ -50,12 +51,12 @@ public class TaskCallbackEvent : EventBase public Dictionary Metadata { get; set; } /// - /// Gets or sets any output artifacts generated by the task. + /// Gets or sets the output storage information. /// - [JsonProperty(PropertyName = "output_artifacts")] - public Dictionary OutputArtifacts { get; set; } + [JsonProperty(PropertyName = "outputs")] + public List Outputs { get; set; } - public TaskCallbackEvent() + public TaskCallbackEvent() { WorkflowInstanceId = String.Empty; TaskId = String.Empty; @@ -63,7 +64,7 @@ public TaskCallbackEvent() CorrelationId = String.Empty; Identity = String.Empty; Metadata = new Dictionary(); - OutputArtifacts = new Dictionary(); + Outputs = new List(); } } } diff --git a/src/Messaging/Events/TaskUpdateEvent.cs b/src/Messaging/Events/TaskUpdateEvent.cs index f0092c6..aeae7cf 100644 --- a/src/Messaging/Events/TaskUpdateEvent.cs +++ b/src/Messaging/Events/TaskUpdateEvent.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache License 2.0 using System.ComponentModel.DataAnnotations; +using Monai.Deploy.Messaging.Common; using Newtonsoft.Json; using Newtonsoft.Json.Converters; @@ -62,8 +63,8 @@ public class TaskUpdateEvent : EventBase /// /// Gets or sets any output artifacts relevent to the output of the task. /// - [JsonProperty(PropertyName = "output_artifacts")] - public Dictionary OutputArtifacts { get; set; } + [JsonProperty(PropertyName = "outputs")] + public List Outputs { get; set; } /// /// Gets or sets any metadata relevant to the output of the task. @@ -81,7 +82,7 @@ public TaskUpdateEvent() Reason = FailureReason.None; Message = String.Empty; Metadata = new Dictionary(); - OutputArtifacts = new Dictionary(); + Outputs = new List(); } } } From b0764910feb9167e3e4aad744af721a160396f6c Mon Sep 17 00:00:00 2001 From: jackschofield23 <56344499+jackschofield23@users.noreply.github.com> Date: Wed, 1 Jun 2022 11:14:34 +0100 Subject: [PATCH 30/45] add payload id to task dispatch (#26) Signed-off-by: Jack Schofield Signed-off-by: JP LEGER --- src/Messaging/Events/TaskDispatchEvent.cs | 8 ++++++++ src/Messaging/Test/TaskDispatchEventTest.cs | 3 +++ 2 files changed, 11 insertions(+) diff --git a/src/Messaging/Events/TaskDispatchEvent.cs b/src/Messaging/Events/TaskDispatchEvent.cs index 20d511f..c4cf450 100644 --- a/src/Messaging/Events/TaskDispatchEvent.cs +++ b/src/Messaging/Events/TaskDispatchEvent.cs @@ -31,6 +31,13 @@ public class TaskDispatchEvent : EventBase [Required] public string ExecutionId { get; set; } + /// + /// Gets or sets the payload ID of the current workflow instance. + /// + [JsonProperty(PropertyName = "payload_id")] + [Required] + public string PayloadId { get; set; } + /// /// Gets or sets the correlation ID. /// @@ -92,6 +99,7 @@ public TaskDispatchEvent() TaskId = string.Empty; ExecutionId = string.Empty; CorrelationId = string.Empty; + PayloadId = string.Empty; TaskPluginType = string.Empty; TaskPluginArguments = new Dictionary(); Status = TaskExecutionStatus.Unknown; diff --git a/src/Messaging/Test/TaskDispatchEventTest.cs b/src/Messaging/Test/TaskDispatchEventTest.cs index 58088fd..5841c69 100644 --- a/src/Messaging/Test/TaskDispatchEventTest.cs +++ b/src/Messaging/Test/TaskDispatchEventTest.cs @@ -23,6 +23,9 @@ public void ValidationThrowsOnError() taskDispatchEvent.ExecutionId = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); + taskDispatchEvent.PayloadId = Guid.NewGuid().ToString(); + Assert.Throws(() => taskDispatchEvent.Validate()); + taskDispatchEvent.TaskId = Guid.NewGuid().ToString(); Assert.Throws(() => taskDispatchEvent.Validate()); From c167be8b2f2f6977da67c2f294d5f71b53b048a8 Mon Sep 17 00:00:00 2001 From: jackschofield23 <56344499+jackschofield23@users.noreply.github.com> Date: Mon, 13 Jun 2022 09:19:04 +0100 Subject: [PATCH 31/45] add export status (#28) Signed-off-by: Jack Schofield Signed-off-by: JP LEGER --- src/Messaging/Events/TaskExecutionStatus.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Messaging/Events/TaskExecutionStatus.cs b/src/Messaging/Events/TaskExecutionStatus.cs index 790389b..b002187 100644 --- a/src/Messaging/Events/TaskExecutionStatus.cs +++ b/src/Messaging/Events/TaskExecutionStatus.cs @@ -12,5 +12,6 @@ public enum TaskExecutionStatus Succeeded, Failed, Canceled, + Exported } } From 16a53b7be1c2354b0c50a31956355103b4199076 Mon Sep 17 00:00:00 2001 From: jeanpatrickleger Date: Mon, 13 Jun 2022 11:34:09 -0500 Subject: [PATCH 32/45] Options to enable secure RabbitMQ (#27) * RMQ connection support for SSL/TLS (no certificate check). * Refactored RabbitMqConnectionFactory to support TLS connection and Port number config. * Update on IRabbitMqConnectionFactory Summary. Co-authored-by: JP Leger Signed-off-by: JP LEGER --- .../Configuration/ConfigurationKeys.cs | 3 +- .../RabbitMq/RabbitMqConnectionFactory.cs | 33 +++++++++++++++---- .../RabbitMqMessagePublisherService.cs | 16 +++++++-- .../RabbitMqMessageSubscriberService.cs | 16 +++++++-- .../RabbitMqMessagePublisherServiceTest.cs | 2 +- .../RabbitMqMessageSubscriberServiceTest.cs | 2 +- 6 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/Messaging/Configuration/ConfigurationKeys.cs b/src/Messaging/Configuration/ConfigurationKeys.cs index 367b25c..0bed43a 100644 --- a/src/Messaging/Configuration/ConfigurationKeys.cs +++ b/src/Messaging/Configuration/ConfigurationKeys.cs @@ -11,7 +11,8 @@ internal static class ConfigurationKeys public static readonly string VirtualHost = "virtualHost"; public static readonly string Exchange = "exchange"; public static readonly string ExportRequestQueue = "exportRequestQueue"; - + public static readonly string UseSSL = "useSSL"; + public static readonly string Port = "port"; public static readonly string[] PublisherRequiredKeys = new[] { EndPoint, Username, Password, VirtualHost, Exchange }; public static readonly string[] SubscriberRequiredKeys = new[] { EndPoint, Username, Password, VirtualHost, Exchange, ExportRequestQueue }; } diff --git a/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs b/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs index 9f646d6..9cdda71 100644 --- a/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs +++ b/src/Messaging/RabbitMq/RabbitMqConnectionFactory.cs @@ -8,6 +8,7 @@ using Microsoft.Extensions.Logging; using Monai.Deploy.Messaging.Common; using RabbitMQ.Client; +using System.Net.Security; namespace Monai.Deploy.Messaging.RabbitMq { @@ -22,8 +23,10 @@ public interface IRabbitMqConnectionFactory /// User name /// Password /// Virtual host + /// Encrypt communication + /// Port Number /// Instance of . - IModel CreateChannel(string hostName, string username, string password, string virtualHost); + IModel CreateChannel(string hostName, string username, string password, string virtualHost, string useSSL, string portnumber); } public class RabbitMqConnectionFactory : IRabbitMqConnectionFactory, IDisposable @@ -40,19 +43,20 @@ public RabbitMqConnectionFactory(ILogger logger) _connections = new ConcurrentDictionary>(); } - public IModel CreateChannel(string hostName, string username, string password, string virtualHost) + public IModel CreateChannel(string hostName, string username, string password, string virtualHost, string useSSL, string portnumber ) { Guard.Against.NullOrWhiteSpace(hostName, nameof(hostName)); Guard.Against.NullOrWhiteSpace(username, nameof(username)); Guard.Against.NullOrWhiteSpace(password, nameof(password)); Guard.Against.NullOrWhiteSpace(virtualHost, nameof(virtualHost)); + var key = $"{hostName}{username}{HashPassword(password)}{virtualHost}"; var connection = _connections.AddOrUpdate(key, x => { - return CreatConnection(hostName, username, password, virtualHost, key); + return CreatConnection(hostName, username, password, virtualHost, key, useSSL, portnumber); }, (updateKey, updateConnection) => { @@ -62,21 +66,38 @@ public IModel CreateChannel(string hostName, string username, string password, s } else { - return CreatConnection(hostName, username, password, virtualHost, key); + return CreatConnection(hostName, username, password, virtualHost, key, useSSL, portnumber); } }); return connection.Value.CreateModel(); } - private Lazy CreatConnection(string hostName, string username, string password, string virtualHost, string key) + private Lazy CreatConnection(string hostName, string username, string password, string virtualHost, string key, string useSSL, string portnumber) { + int port; + Boolean SslEnabled; + Boolean.TryParse(useSSL, out SslEnabled); + if (!Int32.TryParse(portnumber, out port)) + { + port = SslEnabled ? 5671 : 5672; // 5671 is default port for SSL/TLS , 5672 is default port for PLAIN. + } + + SslOption sslOptions = new SslOption + { + Enabled = SslEnabled, + ServerName = hostName, + AcceptablePolicyErrors = SslPolicyErrors.RemoteCertificateNameMismatch | SslPolicyErrors.RemoteCertificateChainErrors | SslPolicyErrors.RemoteCertificateNotAvailable + }; + var connectionFactory = _connectionFactoriess.GetOrAdd(key, y => new Lazy(() => new ConnectionFactory() { HostName = hostName, UserName = username, Password = password, - VirtualHost = virtualHost + VirtualHost = virtualHost, + Ssl = sslOptions, + Port = port })); return new Lazy(() => connectionFactory.Value.CreateConnection()); diff --git a/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs b/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs index 1c82c7b..0fdf5ec 100644 --- a/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs +++ b/src/Messaging/RabbitMq/RabbitMqMessagePublisherService.cs @@ -23,6 +23,8 @@ public class RabbitMqMessagePublisherService : IMessageBrokerPublisherService private readonly string _password; private readonly string _virtualHost; private readonly string _exchange; + private readonly string _useSSL = string.Empty; + private readonly string _portNumber = string.Empty; private bool _disposedValue; public string Name => "Rabbit MQ Publisher"; @@ -43,6 +45,16 @@ public RabbitMqMessagePublisherService(IOptions(); _model = new Mock(); - _connectionFactory.Setup(p => p.CreateChannel(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + _connectionFactory.Setup(p => p.CreateChannel(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(),It.IsAny(),It.IsAny())) .Returns(_model.Object); } diff --git a/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs b/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs index c887358..4c9c0d6 100644 --- a/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs +++ b/src/Messaging/Test/RabbitMq/RabbitMqMessageSubscriberServiceTest.cs @@ -30,7 +30,7 @@ public RabbitMqMessageSubscriberServiceTest() _connectionFactory = new Mock(); _model = new Mock(); - _connectionFactory.Setup(p => p.CreateChannel(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + _connectionFactory.Setup(p => p.CreateChannel(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) .Returns(_model.Object); } From d0ba49e9ef3a15c27f9bba6f88e33c8cb3600a21 Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Mon, 13 Jun 2022 11:51:57 -0500 Subject: [PATCH 33/45] Added SQS plug-in as RMQ alternative. Signed-off-by: JP LEGER --- src/Messaging/Monai.Deploy.Messaging.csproj | 3 + src/Messaging/SQS/ConfigurationKeys.cs | 19 ++ src/Messaging/SQS/Log.cs | 49 ++++ src/Messaging/SQS/QueueFormatter.cs | 28 ++ src/Messaging/SQS/README.MD | 24 ++ .../SQS/SQSMessagePublisherService.cs | 209 +++++++++++++++ .../SQS/SQSMessageSubscriberService.cs | 245 ++++++++++++++++++ 7 files changed, 577 insertions(+) create mode 100644 src/Messaging/SQS/ConfigurationKeys.cs create mode 100644 src/Messaging/SQS/Log.cs create mode 100644 src/Messaging/SQS/QueueFormatter.cs create mode 100644 src/Messaging/SQS/README.MD create mode 100644 src/Messaging/SQS/SQSMessagePublisherService.cs create mode 100644 src/Messaging/SQS/SQSMessageSubscriberService.cs diff --git a/src/Messaging/Monai.Deploy.Messaging.csproj b/src/Messaging/Monai.Deploy.Messaging.csproj index 1ef2b45..f1bf8cd 100644 --- a/src/Messaging/Monai.Deploy.Messaging.csproj +++ b/src/Messaging/Monai.Deploy.Messaging.csproj @@ -50,7 +50,10 @@ SPDX-License-Identifier: Apache License 2.0 + + + diff --git a/src/Messaging/SQS/ConfigurationKeys.cs b/src/Messaging/SQS/ConfigurationKeys.cs new file mode 100644 index 0000000..54d0e11 --- /dev/null +++ b/src/Messaging/SQS/ConfigurationKeys.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +namespace Monai.Deploy.Messaging.Configuration +{ + internal static class SQSConfigurationKeys + { + public static readonly string AccessKey = "accessKey"; + public static readonly string AccessToken = "accessToken"; + public static readonly string Region = "region"; + public static readonly string WorkflowRequestQueue = "workflowRequestQueue"; + public static readonly string ExportRequestQueue = "exportRequestQueue"; + public static readonly string BucketName = "bucketName"; + public static readonly string Envid = "environmentId"; + + public static readonly string[] PublisherRequiredKeys = new[] { WorkflowRequestQueue, BucketName }; + public static readonly string[] SubscriberRequiredKeys = new[] { ExportRequestQueue, BucketName }; + } +} diff --git a/src/Messaging/SQS/Log.cs b/src/Messaging/SQS/Log.cs new file mode 100644 index 0000000..9d15df2 --- /dev/null +++ b/src/Messaging/SQS/Log.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.Logging; + +namespace Monai.Deploy.Messaging.SQS +{ + public static partial class Log + { + internal static readonly string LoggingScopeMessageApplication = "Message ID={0}. Application ID={1}."; + + + [LoggerMessage(EventId = 10000, Level = LogLevel.Information, Message = "Publishing message {MessageId} to Queue={topic}.")] + public static partial void PublishingToSQS(this ILogger logger, string topic, string MessageId); + + [LoggerMessage(EventId = 10001, Level = LogLevel.Information, Message = "{ServiceName} connecting to SQS.")] + public static partial void ConnectingToSQS(this ILogger logger, string serviceName); + + [LoggerMessage(EventId = 10002, Level = LogLevel.Information, Message = "Message received from queue {queue}.")] + public static partial void MessageReceivedFromQueue(this ILogger logger, string queue, string topic); + + [LoggerMessage(EventId = 10003, Level = LogLevel.Information, Message = "Listening for messages from {endpoint}. Queue={queue}.")] + public static partial void SubscribeToSQSQueue(this ILogger logger, string endpoint, string virtualHost, string exchange, string queue, string topic); + + [LoggerMessage(EventId = 10004, Level = LogLevel.Information, Message = "Sending message acknowledgement for message {messageId}.")] + public static partial void SendingAcknowledgement(this ILogger logger, string messageId); + + [LoggerMessage(EventId = 10005, Level = LogLevel.Information, Message = "Ackowledge sent for message {messageId}.")] + public static partial void AcknowledgementSent(this ILogger logger, string messageId); + + [LoggerMessage(EventId = 10006, Level = LogLevel.Information, Message = "Sending nack message {messageId} and requeuing.")] + public static partial void SendingNAcknowledgement(this ILogger logger, string messageId); + + [LoggerMessage(EventId = 10007, Level = LogLevel.Information, Message = "Nack message sent for message {messageId}, requeue={requeue}.")] + public static partial void NAcknowledgementSent(this ILogger logger, string messageId, bool requeue); + + [LoggerMessage(EventId = 10008, Level = LogLevel.Information, Message = "Closing connections.")] + public static partial void ClosingConnections(this ILogger logger); + + [LoggerMessage(EventId = 10009, Level = LogLevel.Error, Message = "Invalid or corrupted message received: Queue={queueName}, Message ID={messageId}.")] + public static partial void InvalidMessage(this ILogger logger, string queueName, string topic, string messageId, Exception ex); + + [LoggerMessage(EventId = 10010, Level = LogLevel.Error, Message = "Exception not handled by the subscriber's callback function: Queue={queueName}, Message ID={messageId}.")] + public static partial void ErrorNotHandledByCallback(this ILogger logger, string queueName, string topic, string messageId, Exception ex); + + [LoggerMessage(EventId = 10011, Level = LogLevel.Error, Message = "Creating SQS client.")] + public static partial void CreateSQSClient(this ILogger logger); + + [LoggerMessage(EventId = 10012, Level = LogLevel.Error, Message = "{ServiceName} failed to connect to SQS.")] + public static partial void ConnectingToSQSError(this ILogger logger, string serviceName, Exception ex); + } +} diff --git a/src/Messaging/SQS/QueueFormatter.cs b/src/Messaging/SQS/QueueFormatter.cs new file mode 100644 index 0000000..7c3181a --- /dev/null +++ b/src/Messaging/SQS/QueueFormatter.cs @@ -0,0 +1,28 @@ +using System.Text.RegularExpressions; + +namespace Monai.Deploy.Messaging.SQS +{ + internal static class QueueFormatter + { + /// + /// Returns an aggregate of the the environmentId, queueBasename nd topic as the name of the queue defined in SQS. + /// The returned string is made compliant to SQS naming convention : It will replace non alphanumeric and other characters than "_" and "-", by an hyphen + /// + /// + /// + /// + /// string + public static string FormatQueueName(string environmentId, string? queuebasename, string topic) + { + + string queue = $"{queuebasename}_{topic}"; + + if (!environmentId.Equals(String.Empty)) + queue = $"{environmentId}_{queue}"; + queue = Regex.Replace(queue, "[^a-zA-Z0-9_]", "-"); + if (queue.Length > 80) + queue = queue.Substring(0, 80); + return queue; + } + } +} diff --git a/src/Messaging/SQS/README.MD b/src/Messaging/SQS/README.MD new file mode 100644 index 0000000..d265d47 --- /dev/null +++ b/src/Messaging/SQS/README.MD @@ -0,0 +1,24 @@ +

+project-monai +

+ +💡 If you want to know more about MONAI Deploy WG vision, overall structure, and guidelines, please read [MONAI Deploy](https://github.com/Project-MONAI/monai-deploy) first. + +# MONAI Deploy Messaging +Messaging layer for MONAI Deploy clinical data pipelines system. + +## Contributing + +For guidance on making a contribution to MONAI Deploy Workflow Manager, see the [contributing guidelines](https://github.com/Project-MONAI/monai-deploy/blob/main/CONTRIBUTING.md). + +Join the conversation on Twitter [@ProjectMONAI](https://twitter.com/ProjectMONAI) or join our [Slack channel](https://forms.gle/QTxJq3hFictp31UM9). + +Ask and answer questions over on [MONAI Deploy Workflow Manager's GitHub Discussions tab](https://github.com/Project-MONAI/monai-deploy-workflow-manager/discussions). + +## Links + +- Website: +- Code: +- Project tracker: +- Issue tracker: +- Test status: \ No newline at end of file diff --git a/src/Messaging/SQS/SQSMessagePublisherService.cs b/src/Messaging/SQS/SQSMessagePublisherService.cs new file mode 100644 index 0000000..d78ec6b --- /dev/null +++ b/src/Messaging/SQS/SQSMessagePublisherService.cs @@ -0,0 +1,209 @@ +// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.Globalization; +using System.Text; +using Amazon.S3; +using Amazon.SQS; +using Amazon.SQS.ExtendedClient; +using Amazon.SQS.Model; +using Ardalis.GuardClauses; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Messaging.Configuration; + +namespace Monai.Deploy.Messaging.SQS +{ + public class SQSMessagePublisherService : IMessageBrokerPublisherService + { + private const int PersistentDeliveryMode = 2; + + private readonly ILogger _logger; + private readonly string? _accessKey; + private readonly string? _accessToken; + private readonly string _environmentId = string.Empty; + private bool _disposedValue; + + + public string Name => "AWS SQS Publisher"; + private readonly string _queueName; + private readonly string _bucketName; + private readonly AmazonSQSClient? _sqsClient; + private readonly AmazonS3Client? _s3Client; + private readonly AmazonSQSExtendedClient? _sqSExtendedClient; + + public SQSMessagePublisherService(IOptions options, + ILogger logger) + { + Guard.Against.Null(options, nameof(options)); + + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var configuration = options.Value; + ValidateConfiguration(configuration); + + + //This 2 config entries are mandatory. + _queueName = configuration.PublisherSettings[SQSConfigurationKeys.WorkflowRequestQueue]; + _bucketName = configuration.PublisherSettings[SQSConfigurationKeys.BucketName]; + + + if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.AccessKey)) + { + _logger.LogInformation("accessKey found in configuration."); + _accessKey = configuration.PublisherSettings[SQSConfigurationKeys.AccessKey]; + } + + + if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.AccessToken)) + { + _logger.LogInformation("accessToken found in configuration."); + _accessToken = configuration.PublisherSettings[SQSConfigurationKeys.AccessToken]; + } + + if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.Envid)) + _environmentId = configuration.PublisherSettings[SQSConfigurationKeys.Envid]; + + try + { + _logger.ConnectingToSQS(Name); + + if (!(_accessKey is null) && !(_accessToken is null)) + { + _logger.LogInformation("Assuming IAM user as found in the configuration file."); + _sqsClient = new AmazonSQSClient(_accessKey, _accessToken); + _s3Client = new AmazonS3Client(_accessKey, _accessToken); + } + else + { + _logger.LogInformation("Attempting to assume local AWS credentials."); + _sqsClient = new AmazonSQSClient(); + _s3Client = new AmazonS3Client(); + } + + _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, + new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, _bucketName)); + + + + } + catch (Amazon.SQS.AmazonSQSException Ex) + { + _logger.ConnectingToSQSError(Name, Ex); + } + } + + private void ValidateConfiguration(MessageBrokerServiceConfiguration configuration) + { + Guard.Against.Null(configuration, nameof(configuration)); + Guard.Against.Null(configuration.PublisherSettings, nameof(configuration.PublisherSettings)); + + foreach (var key in ConfigurationKeys.PublisherRequiredKeys) + { + if (!configuration.PublisherSettings.ContainsKey(key)) + { + throw new ConfigurationException($"{Name} is missing configuration for {key}."); + } + } + } + + public Task Publish(string topic, Monai.Deploy.Messaging.Messages.Message message) + { + + Guard.Against.NullOrWhiteSpace(topic, nameof(topic)); + Guard.Against.Null(message, nameof(message)); + + + using var loggerScope = _logger.BeginScope(string.Format(CultureInfo.InvariantCulture, Log.LoggingScopeMessageApplication, message.MessageId, message.ApplicationId)); + _logger.PublishingToSQS(topic, message.MessageId); + var sendMessageRequest = new SendMessageRequest(); + + Dictionary MessageAttributes = new Dictionary(); + MessageAttributeValue messageIdAttribute = new MessageAttributeValue(); + messageIdAttribute.DataType = "String"; + messageIdAttribute.StringValue = message.MessageId; + MessageAttributes.Add("MessageId", messageIdAttribute); + + MessageAttributeValue ContentTypeAttribute = new MessageAttributeValue(); + ContentTypeAttribute.DataType = "String"; + ContentTypeAttribute.StringValue = message.ContentType; + MessageAttributes.Add("ContentType", ContentTypeAttribute); + + + MessageAttributeValue ApplicationIdAttribute = new MessageAttributeValue(); + ApplicationIdAttribute.DataType = "String"; + ApplicationIdAttribute.StringValue = message.MessageId; + MessageAttributes.Add("ApplicationId", ApplicationIdAttribute); + + sendMessageRequest.MessageAttributes = MessageAttributes; + + + Console.WriteLine("Message information : "); + Console.WriteLine(message); + Console.WriteLine(message.Body); + Console.WriteLine(message.Body.Length); + + + string queueName = QueueFormatter.FormatQueueName(_environmentId, _queueName, topic); + _logger.LogDebug($"Attempting to create or subscribe to {queueName}"); + + var queueAttributes = new Dictionary(); + + queueAttributes.Add("KmsMasterKeyId", "alias/aws/sqs"); + var request = new CreateQueueRequest + { + Attributes = queueAttributes, + QueueName = queueName + }; + + CreateQueueResponse createQueueResponse = new CreateQueueResponse(); + try + { + createQueueResponse = _sqSExtendedClient.CreateQueueAsync(request).Result; + } + catch (Exception ex) + { + _logger.LogDebug($"The queue could not be created or subscribed to: {ex.Message}"); + } + + sendMessageRequest.QueueUrl = createQueueResponse.QueueUrl; + + + + sendMessageRequest.MessageBody = Encoding.UTF8.GetString(message.Body, 0, message.Body.Length); + + try + { + SendMessageResponse sqsresp = _sqSExtendedClient.SendMessageAsync(sendMessageRequest).Result; + } + catch(Exception e) + { + _logger.LogError($"The message could not be posted to the queue {queueName} : \n {e.Message}"); + } + + + return Task.CompletedTask; + } + + + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + // Dispose any managed objects + } + + _disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/src/Messaging/SQS/SQSMessageSubscriberService.cs b/src/Messaging/SQS/SQSMessageSubscriberService.cs new file mode 100644 index 0000000..8af30df --- /dev/null +++ b/src/Messaging/SQS/SQSMessageSubscriberService.cs @@ -0,0 +1,245 @@ +// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.Text; +using Amazon.S3; +using Amazon.SQS; +using Amazon.SQS.ExtendedClient; +using Amazon.SQS.Model; +using Ardalis.GuardClauses; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Messaging.Common; +using Monai.Deploy.Messaging.Configuration; +using Monai.Deploy.Messaging.Messages; +using Newtonsoft.Json.Linq; + +namespace Monai.Deploy.Messaging.SQS +{ + public class SqsMessageSubscriberService : IMessageBrokerSubscriberService + { + private readonly ILogger _logger; + private bool _disposedValue; + private readonly string? _accessKey; + private readonly string? _accessToken; + private readonly string? _queueName; + private readonly string? _bucketName; + private readonly string _environmentId = string.Empty; + + private readonly AmazonSQSClient? _sqsClient; + private readonly AmazonS3Client? _s3Client; + private readonly AmazonSQSExtendedClient _sqSExtendedClient; + + public string Name => "AWS SQS Subscriber"; + + public SqsMessageSubscriberService(IOptions options, + ILogger logger) + { + Guard.Against.Null(options, nameof(options)); + + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var configuration = options.Value; + ValidateConfiguration(configuration); + _queueName = configuration.SubscriberSettings[SQSConfigurationKeys.ExportRequestQueue]; + _bucketName = configuration.SubscriberSettings[SQSConfigurationKeys.BucketName]; + + + if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.AccessKey)) + _accessKey = configuration.SubscriberSettings[SQSConfigurationKeys.AccessKey]; + + + if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.AccessToken)) + _accessToken = configuration.SubscriberSettings[SQSConfigurationKeys.AccessToken]; + + + if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.Envid)) + _environmentId = configuration.SubscriberSettings[SQSConfigurationKeys.Envid]; + + try + { + _logger.ConnectingToSQS(Name); + + if (!(_accessKey is null) && !(_accessToken is null)) + { + _logger.LogInformation("Assuming IAM user as found in the configuration file."); + _sqsClient = new AmazonSQSClient(_accessKey, _accessToken); + _s3Client = new AmazonS3Client(_accessKey, _accessToken); + } + else + { + _logger.LogInformation("Attempting to assume local AWS credentials."); + _sqsClient = new AmazonSQSClient(); + _s3Client = new AmazonS3Client(); + } + + _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, + new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, _bucketName)); + + } + catch (Amazon.SQS.AmazonSQSException Ex) + { + _logger.ConnectingToSQSError(Name, Ex); + Guard.Against.Null(_sqSExtendedClient, nameof(_sqSExtendedClient)); + } + } + + + + private void ValidateConfiguration(MessageBrokerServiceConfiguration configuration) + { + Guard.Against.Null(configuration, nameof(configuration)); + Guard.Against.Null(configuration.SubscriberSettings, nameof(configuration.SubscriberSettings)); + + foreach (var key in SQSConfigurationKeys.SubscriberRequiredKeys) + { + if (!configuration.SubscriberSettings.ContainsKey(key)) + { + throw new ConfigurationException($"{Name} is missing configuration for {key}."); + } + } + } + + public void Subscribe(string topic, string queue, Action messageReceivedCallback, ushort prefetchCount = 0) + => Subscribe(new string[] { topic }, queue, messageReceivedCallback, prefetchCount); + + public void Subscribe(string[] topics, string queue, Action messageReceivedCallback, ushort prefetchCount = 0) + { + Guard.Against.Null(topics, nameof(topics)); + Guard.Against.Null(messageReceivedCallback, nameof(messageReceivedCallback)); + + foreach (string topic in topics) + { + Task.Run(() => + { + + + string queueName = QueueFormatter.FormatQueueName(_environmentId, _queueName, topic); + _logger.LogDebug($"Attempting to create or subscribe to {queueName}"); + + var queueAttributes = new Dictionary(); + + queueAttributes.Add("KmsMasterKeyId", "alias/aws/sqs"); + var request = new CreateQueueRequest + { + Attributes = queueAttributes, + QueueName = queueName + }; + + CreateQueueResponse createQueueResponse = new CreateQueueResponse(); + try + { + createQueueResponse = _sqSExtendedClient.CreateQueueAsync(request).Result; + } + catch (Exception ex) + { + _logger.LogDebug($"The queue could not be created or subscribed to: {ex.Message}"); + } + + + while (true) + { + List AttributesList = new List(); + AttributesList.Add("*"); + + var messageResponse = _sqSExtendedClient.ReceiveMessageAsync(new ReceiveMessageRequest + { + QueueUrl = createQueueResponse.QueueUrl, + WaitTimeSeconds = 2, + AttributeNames = new List { "All" }, + MessageAttributeNames = new List { "All" } + }).Result; + var messages = messageResponse.Messages; + + if (messages.Any()) + { + foreach (var msg in messages) + { + _logger.Log(LogLevel.Debug, $"Message {msg.MessageId} received from SQS."); + MessageReceivedEventArgs messageReceivedEventArgs = CreateMessage(msg); + try + { + _logger.AcknowledgementSent(msg.MessageId); + _sqSExtendedClient.DeleteMessageAsync(new DeleteMessageRequest { QueueUrl = createQueueResponse.QueueUrl, ReceiptHandle = msg.ReceiptHandle }).Wait(); + messageReceivedCallback(messageReceivedEventArgs); + } + catch (Exception ex) + { + _logger.Log(LogLevel.Error, ex.Message); + } + } + } + } + }); + } + + + + } + + public void SubscribeAsync(string topic, string queue, Func messageReceivedCallback, ushort prefetchCount = 0) + => SubscribeAsync(new string[] { topic }, queue, messageReceivedCallback, prefetchCount); + + public void SubscribeAsync(string[] topics, string queue, Func messageReceivedCallback, ushort prefetchCount = 0) + { + throw new NotImplementedException(); + + } + + public void Acknowledge(MessageBase message) + { + //We will use this to delelete the message from the SQS qeue. + } + + public void Reject(MessageBase message, bool requeue = true) + { + Guard.Against.Null(message, nameof(message)); + + + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + + } + + _disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + + private static MessageReceivedEventArgs CreateMessage(Amazon.SQS.Model.Message msg) + { + + Guard.Against.Null(msg, nameof(msg)); + Guard.Against.Null(msg.Body, nameof(msg.Body)); + Guard.Against.Null(msg.Attributes, nameof(msg.Attributes)); + Guard.Against.Null(msg.MessageId, nameof(msg.MessageId)); + + JObject bodyobj = JObject.Parse(msg.Body); + + return new MessageReceivedEventArgs( + new Monai.Deploy.Messaging.Messages.Message( + body: Encoding.UTF8.GetBytes(msg.Body), + messageDescription: "desc1", + messageId: bodyobj["MessageId"].ToString(), + applicationId: msg.Attributes["SenderId"], + contentType: msg.MessageAttributes["ContentType"].ToString(), + correlationId: bodyobj["correlation_id"].ToString(), + creationDateTime: DateTimeOffset.FromUnixTimeMilliseconds(Int64.Parse(msg.Attributes["SentTimestamp"])), + deliveryTag: msg.ReceiptHandle) + , CancellationToken.None); + } + } +} From bc71ebd1a77d4dada49d0759d6f3ea63ff542fd7 Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 08:20:27 -0500 Subject: [PATCH 34/45] SQS Support adde to messaging lib Signed-off-by: JP LEGER --- SQS/ConfigurationKeys.cs | 19 +++ SQS/Log.cs | 49 ++++++ SQS/QueueFormatter.cs | 28 ++++ SQS/README.MD | 200 +++++++++++++++++++++++ SQS/SQSMessagePublisherService.cs | 209 ++++++++++++++++++++++++ SQS/SQSMessageSubscriberService.cs | 245 +++++++++++++++++++++++++++++ src/Messaging/SQS/README.MD | 182 ++++++++++++++++++++- 7 files changed, 929 insertions(+), 3 deletions(-) create mode 100644 SQS/ConfigurationKeys.cs create mode 100644 SQS/Log.cs create mode 100644 SQS/QueueFormatter.cs create mode 100644 SQS/README.MD create mode 100644 SQS/SQSMessagePublisherService.cs create mode 100644 SQS/SQSMessageSubscriberService.cs diff --git a/SQS/ConfigurationKeys.cs b/SQS/ConfigurationKeys.cs new file mode 100644 index 0000000..54d0e11 --- /dev/null +++ b/SQS/ConfigurationKeys.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +namespace Monai.Deploy.Messaging.Configuration +{ + internal static class SQSConfigurationKeys + { + public static readonly string AccessKey = "accessKey"; + public static readonly string AccessToken = "accessToken"; + public static readonly string Region = "region"; + public static readonly string WorkflowRequestQueue = "workflowRequestQueue"; + public static readonly string ExportRequestQueue = "exportRequestQueue"; + public static readonly string BucketName = "bucketName"; + public static readonly string Envid = "environmentId"; + + public static readonly string[] PublisherRequiredKeys = new[] { WorkflowRequestQueue, BucketName }; + public static readonly string[] SubscriberRequiredKeys = new[] { ExportRequestQueue, BucketName }; + } +} diff --git a/SQS/Log.cs b/SQS/Log.cs new file mode 100644 index 0000000..9d15df2 --- /dev/null +++ b/SQS/Log.cs @@ -0,0 +1,49 @@ +using Microsoft.Extensions.Logging; + +namespace Monai.Deploy.Messaging.SQS +{ + public static partial class Log + { + internal static readonly string LoggingScopeMessageApplication = "Message ID={0}. Application ID={1}."; + + + [LoggerMessage(EventId = 10000, Level = LogLevel.Information, Message = "Publishing message {MessageId} to Queue={topic}.")] + public static partial void PublishingToSQS(this ILogger logger, string topic, string MessageId); + + [LoggerMessage(EventId = 10001, Level = LogLevel.Information, Message = "{ServiceName} connecting to SQS.")] + public static partial void ConnectingToSQS(this ILogger logger, string serviceName); + + [LoggerMessage(EventId = 10002, Level = LogLevel.Information, Message = "Message received from queue {queue}.")] + public static partial void MessageReceivedFromQueue(this ILogger logger, string queue, string topic); + + [LoggerMessage(EventId = 10003, Level = LogLevel.Information, Message = "Listening for messages from {endpoint}. Queue={queue}.")] + public static partial void SubscribeToSQSQueue(this ILogger logger, string endpoint, string virtualHost, string exchange, string queue, string topic); + + [LoggerMessage(EventId = 10004, Level = LogLevel.Information, Message = "Sending message acknowledgement for message {messageId}.")] + public static partial void SendingAcknowledgement(this ILogger logger, string messageId); + + [LoggerMessage(EventId = 10005, Level = LogLevel.Information, Message = "Ackowledge sent for message {messageId}.")] + public static partial void AcknowledgementSent(this ILogger logger, string messageId); + + [LoggerMessage(EventId = 10006, Level = LogLevel.Information, Message = "Sending nack message {messageId} and requeuing.")] + public static partial void SendingNAcknowledgement(this ILogger logger, string messageId); + + [LoggerMessage(EventId = 10007, Level = LogLevel.Information, Message = "Nack message sent for message {messageId}, requeue={requeue}.")] + public static partial void NAcknowledgementSent(this ILogger logger, string messageId, bool requeue); + + [LoggerMessage(EventId = 10008, Level = LogLevel.Information, Message = "Closing connections.")] + public static partial void ClosingConnections(this ILogger logger); + + [LoggerMessage(EventId = 10009, Level = LogLevel.Error, Message = "Invalid or corrupted message received: Queue={queueName}, Message ID={messageId}.")] + public static partial void InvalidMessage(this ILogger logger, string queueName, string topic, string messageId, Exception ex); + + [LoggerMessage(EventId = 10010, Level = LogLevel.Error, Message = "Exception not handled by the subscriber's callback function: Queue={queueName}, Message ID={messageId}.")] + public static partial void ErrorNotHandledByCallback(this ILogger logger, string queueName, string topic, string messageId, Exception ex); + + [LoggerMessage(EventId = 10011, Level = LogLevel.Error, Message = "Creating SQS client.")] + public static partial void CreateSQSClient(this ILogger logger); + + [LoggerMessage(EventId = 10012, Level = LogLevel.Error, Message = "{ServiceName} failed to connect to SQS.")] + public static partial void ConnectingToSQSError(this ILogger logger, string serviceName, Exception ex); + } +} diff --git a/SQS/QueueFormatter.cs b/SQS/QueueFormatter.cs new file mode 100644 index 0000000..7c3181a --- /dev/null +++ b/SQS/QueueFormatter.cs @@ -0,0 +1,28 @@ +using System.Text.RegularExpressions; + +namespace Monai.Deploy.Messaging.SQS +{ + internal static class QueueFormatter + { + /// + /// Returns an aggregate of the the environmentId, queueBasename nd topic as the name of the queue defined in SQS. + /// The returned string is made compliant to SQS naming convention : It will replace non alphanumeric and other characters than "_" and "-", by an hyphen + /// + /// + /// + /// + /// string + public static string FormatQueueName(string environmentId, string? queuebasename, string topic) + { + + string queue = $"{queuebasename}_{topic}"; + + if (!environmentId.Equals(String.Empty)) + queue = $"{environmentId}_{queue}"; + queue = Regex.Replace(queue, "[^a-zA-Z0-9_]", "-"); + if (queue.Length > 80) + queue = queue.Substring(0, 80); + return queue; + } + } +} diff --git a/SQS/README.MD b/SQS/README.MD new file mode 100644 index 0000000..09d24f1 --- /dev/null +++ b/SQS/README.MD @@ -0,0 +1,200 @@ +

+project-monai +

+ +💡 If you want to know more about MONAI Deploy WG vision, overall structure, and guidelines, please read [MONAI Deploy](https://github.com/Project-MONAI/monai-deploy) first. + +# MONAI Deploy Messaging SQS Plug-In +AWS SQS plugin for the messaging layer of MONAI Deploy. + +## Information: +This plugin can be used as a messaging mechanism alternative to RabbitMQ, and allows MONAI Deploy to integrate with AWS Simple Queue Service. Unlike RabbitMQ, SQS does not have a concept of vhost, exchange and routing key; instead the plugin will create one specific queue for each vhost, exchange and routing key combibation. + + + + + + + + + + + + + + + + + +
+ RMQ plugin + + SQS plugin + Description +
vhostenvironmentIdThe vhost namespace is replaced be the parameter environmentId. This parmater is used as a prefix for each queue name, allowing mutliple MONAI deploy installations on the same AWS account with not queue name conflict.
exchange and routing keyworkflowRequestQueue and exportRequestQueueThese parmaeters control the name of the queues for each purpose. Each queue name will be suffixed with the name routing key.
+ +* Queue names generated by the plugin have the following name convention : [environmentId]_[RequestQueue]_[routingKey]. +* Queue names will be less or equal to 80 characters long. The name will be truncated if the combinatino of environmentId, RequestQueue and routingKey exceeds this limit. +* Any non-alphanumerical character other than "-" and "\_" found in the environmentId, RequestQueue, parameters or routing key variables will be replaced by "\_". +* The queues are created automatically upon the 1st message publication/subscription if they do not already exist. +* Because MONAI Deploy can generate payloads greater than 256KBytes, the plugin leverages the SQS extended client, allowing payload up to 2GB in size. The use of this specific client mandates the usage of an S3 bucket as transient store for the messages to be held until their delivery to the queue subscriber. More information is available below in this documentation about IAM privileges, SQS and S3 bucket requirements. + + + +## Plugin activation + +Because MONAI Deploy plugin management work is still in progress (as of 06/13/2022), plugins cannot be loaded at run-time. Instead this SQS pluging can be activated by doing small code changes to the MONAI Informatic Gateway project https://github.com/Project-MONAI/monai-deploy-informatics-gateway, in the `Program.cs` file and recompiling. + +In the declaration for `CreateHostBuilder`, locate the following code block: + + services.UseRabbitMq(); + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.PublisherServiceAssemblyName); + }); + + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.SubscriberServiceAssemblyName); + }); + + + +and alter it by commenting the line `services.UseRabbitMq();`, replace `services.AddSingleton();` by `services.AddSingleton();` and the line `services.AddSingleton();` by `services.AddSingleton();`. The code block should now look like this : + + + //services.UseRabbitMq(); + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.PublisherServiceAssemblyName); + }); + + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.SubscriberServiceAssemblyName); + }); + +## MONAI Informatic Gateway Configuration + +The plugin is configured in the Messaging section of `appsettings.json` / `appsettings.Development.json` : + +```json + "messaging": { + "publisherServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SQSMessagePublisherService, Monai.Deploy.Messaging", + "subscriberServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SqsMessageSubscriberService, Monai.Deploy.Messaging", + "publisherSettings": { + "bucketName": "monai-minio", + "workflowRequestQueue": "workflow_tasks", + "environmentId": "monai-1", + "accessKey": "ASDFGHJKLADF123456789", + "accessToken": "QwErTyUiOpAsDonMB88W1mcCCwQdePe8X27SEu1S" + }, + "subscriberSettings": { + "exportRequestQueue": "export_tasks", + "bucketName": "monai-minio", + "environmentId": "monai-1", + "accessKey": "ASDFGHJKLADF123456789", + "accessToken": "QwErTyUiOpAsDonMB88W1mcCCwQdePe8X27SEu1S" + } + }, +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
bucketNameS3 bucket used to store the messages temporarily until the subscriber gets it.
workflowRequestQueueQueue prefix for the workflow requests ( MIG -> Workflow Manager ). This parameter is only useful in the PublisherSettings section.
exportRequestQueueQueue prefix for the export requests ( Workflow Manager -> MIG ). This parameter is only useful in the SubscriberSettings section.
bucketNameS3 bucket used to store the messages temporarily until the subscriber gets it.
environmentIdA prefix used to identify the MONAI environment. This allows to create multiple environments within a single AWS account without queue name conflict. This parameter allows for a configuration comparable to the vhost concept in RabbitMq.
accessKeyAWS IAM user access key. This parameter is optional. If not present the plugin will fallback to local credentials, then EC2 role. If this parameter is used the parameter accessToken is also required. Refer to the section IAM privileges for IAM configuration.
accessTokenAWS IAM user access token. This parameter is only required when the parameter accesskey is provided.
+ +## IAM Privileges + +For the plugin to function a set of specific privileges need to be provided. The permissions can be associated either with an IAM user ( by using accessKey and accessToken in the configuratio file ) or by running MONAI on an EC2 instance associated with an IAM Role granted for the following privileges: + +Replace the tags below in the below policy as follow : + +[AWS_Account] : The AWS Account ID ( numeric ) +[EnvironmentId] : The Environment Id use int Subscriber and Publisher settings of the Messaging seciton of `appsettings.json` / `appsettings.Development.json`. +[BucketName] : the name of the S3 bucket used to store the messages temporarily. The same bucket as the one used to store incoming DICOM objects can be used if desired. ( see the AWS S3 plugin to use S3 natively with MONAI Deploy ) + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "sqs:DeleteMessage", + "s3:PutObject", + "s3:GetObject", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:CreateQueue", + "s3:DeleteObject", + "sqs:SetQueueAttributes" + ], + "Resource": [ + "arn:aws:sqs:*:[AWS_Account]:[EnvironmentId]", + "arn:aws:s3:::[BucketName]/*" + ] + } + ] +} +``` + +## Contributing + +For guidance on making a contribution to MONAI Deploy Workflow Manager, see the [contributing guidelines](https://github.com/Project-MONAI/monai-deploy/blob/main/CONTRIBUTING.md). + +Join the conversation on Twitter [@ProjectMONAI](https://twitter.com/ProjectMONAI) or join our [Slack channel](https://forms.gle/QTxJq3hFictp31UM9). + +Ask and answer questions over on [MONAI Deploy Workflow Manager's GitHub Discussions tab](https://github.com/Project-MONAI/monai-deploy-workflow-manager/discussions). + +## Links + +- Website: +- Code: +- Project tracker: +- Issue tracker: +- Test status: diff --git a/SQS/SQSMessagePublisherService.cs b/SQS/SQSMessagePublisherService.cs new file mode 100644 index 0000000..d78ec6b --- /dev/null +++ b/SQS/SQSMessagePublisherService.cs @@ -0,0 +1,209 @@ +// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.Globalization; +using System.Text; +using Amazon.S3; +using Amazon.SQS; +using Amazon.SQS.ExtendedClient; +using Amazon.SQS.Model; +using Ardalis.GuardClauses; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Messaging.Configuration; + +namespace Monai.Deploy.Messaging.SQS +{ + public class SQSMessagePublisherService : IMessageBrokerPublisherService + { + private const int PersistentDeliveryMode = 2; + + private readonly ILogger _logger; + private readonly string? _accessKey; + private readonly string? _accessToken; + private readonly string _environmentId = string.Empty; + private bool _disposedValue; + + + public string Name => "AWS SQS Publisher"; + private readonly string _queueName; + private readonly string _bucketName; + private readonly AmazonSQSClient? _sqsClient; + private readonly AmazonS3Client? _s3Client; + private readonly AmazonSQSExtendedClient? _sqSExtendedClient; + + public SQSMessagePublisherService(IOptions options, + ILogger logger) + { + Guard.Against.Null(options, nameof(options)); + + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var configuration = options.Value; + ValidateConfiguration(configuration); + + + //This 2 config entries are mandatory. + _queueName = configuration.PublisherSettings[SQSConfigurationKeys.WorkflowRequestQueue]; + _bucketName = configuration.PublisherSettings[SQSConfigurationKeys.BucketName]; + + + if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.AccessKey)) + { + _logger.LogInformation("accessKey found in configuration."); + _accessKey = configuration.PublisherSettings[SQSConfigurationKeys.AccessKey]; + } + + + if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.AccessToken)) + { + _logger.LogInformation("accessToken found in configuration."); + _accessToken = configuration.PublisherSettings[SQSConfigurationKeys.AccessToken]; + } + + if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.Envid)) + _environmentId = configuration.PublisherSettings[SQSConfigurationKeys.Envid]; + + try + { + _logger.ConnectingToSQS(Name); + + if (!(_accessKey is null) && !(_accessToken is null)) + { + _logger.LogInformation("Assuming IAM user as found in the configuration file."); + _sqsClient = new AmazonSQSClient(_accessKey, _accessToken); + _s3Client = new AmazonS3Client(_accessKey, _accessToken); + } + else + { + _logger.LogInformation("Attempting to assume local AWS credentials."); + _sqsClient = new AmazonSQSClient(); + _s3Client = new AmazonS3Client(); + } + + _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, + new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, _bucketName)); + + + + } + catch (Amazon.SQS.AmazonSQSException Ex) + { + _logger.ConnectingToSQSError(Name, Ex); + } + } + + private void ValidateConfiguration(MessageBrokerServiceConfiguration configuration) + { + Guard.Against.Null(configuration, nameof(configuration)); + Guard.Against.Null(configuration.PublisherSettings, nameof(configuration.PublisherSettings)); + + foreach (var key in ConfigurationKeys.PublisherRequiredKeys) + { + if (!configuration.PublisherSettings.ContainsKey(key)) + { + throw new ConfigurationException($"{Name} is missing configuration for {key}."); + } + } + } + + public Task Publish(string topic, Monai.Deploy.Messaging.Messages.Message message) + { + + Guard.Against.NullOrWhiteSpace(topic, nameof(topic)); + Guard.Against.Null(message, nameof(message)); + + + using var loggerScope = _logger.BeginScope(string.Format(CultureInfo.InvariantCulture, Log.LoggingScopeMessageApplication, message.MessageId, message.ApplicationId)); + _logger.PublishingToSQS(topic, message.MessageId); + var sendMessageRequest = new SendMessageRequest(); + + Dictionary MessageAttributes = new Dictionary(); + MessageAttributeValue messageIdAttribute = new MessageAttributeValue(); + messageIdAttribute.DataType = "String"; + messageIdAttribute.StringValue = message.MessageId; + MessageAttributes.Add("MessageId", messageIdAttribute); + + MessageAttributeValue ContentTypeAttribute = new MessageAttributeValue(); + ContentTypeAttribute.DataType = "String"; + ContentTypeAttribute.StringValue = message.ContentType; + MessageAttributes.Add("ContentType", ContentTypeAttribute); + + + MessageAttributeValue ApplicationIdAttribute = new MessageAttributeValue(); + ApplicationIdAttribute.DataType = "String"; + ApplicationIdAttribute.StringValue = message.MessageId; + MessageAttributes.Add("ApplicationId", ApplicationIdAttribute); + + sendMessageRequest.MessageAttributes = MessageAttributes; + + + Console.WriteLine("Message information : "); + Console.WriteLine(message); + Console.WriteLine(message.Body); + Console.WriteLine(message.Body.Length); + + + string queueName = QueueFormatter.FormatQueueName(_environmentId, _queueName, topic); + _logger.LogDebug($"Attempting to create or subscribe to {queueName}"); + + var queueAttributes = new Dictionary(); + + queueAttributes.Add("KmsMasterKeyId", "alias/aws/sqs"); + var request = new CreateQueueRequest + { + Attributes = queueAttributes, + QueueName = queueName + }; + + CreateQueueResponse createQueueResponse = new CreateQueueResponse(); + try + { + createQueueResponse = _sqSExtendedClient.CreateQueueAsync(request).Result; + } + catch (Exception ex) + { + _logger.LogDebug($"The queue could not be created or subscribed to: {ex.Message}"); + } + + sendMessageRequest.QueueUrl = createQueueResponse.QueueUrl; + + + + sendMessageRequest.MessageBody = Encoding.UTF8.GetString(message.Body, 0, message.Body.Length); + + try + { + SendMessageResponse sqsresp = _sqSExtendedClient.SendMessageAsync(sendMessageRequest).Result; + } + catch(Exception e) + { + _logger.LogError($"The message could not be posted to the queue {queueName} : \n {e.Message}"); + } + + + return Task.CompletedTask; + } + + + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + // Dispose any managed objects + } + + _disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + } +} diff --git a/SQS/SQSMessageSubscriberService.cs b/SQS/SQSMessageSubscriberService.cs new file mode 100644 index 0000000..8af30df --- /dev/null +++ b/SQS/SQSMessageSubscriberService.cs @@ -0,0 +1,245 @@ +// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium +// SPDX-License-Identifier: Apache License 2.0 + +using System.Text; +using Amazon.S3; +using Amazon.SQS; +using Amazon.SQS.ExtendedClient; +using Amazon.SQS.Model; +using Ardalis.GuardClauses; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Monai.Deploy.Messaging.Common; +using Monai.Deploy.Messaging.Configuration; +using Monai.Deploy.Messaging.Messages; +using Newtonsoft.Json.Linq; + +namespace Monai.Deploy.Messaging.SQS +{ + public class SqsMessageSubscriberService : IMessageBrokerSubscriberService + { + private readonly ILogger _logger; + private bool _disposedValue; + private readonly string? _accessKey; + private readonly string? _accessToken; + private readonly string? _queueName; + private readonly string? _bucketName; + private readonly string _environmentId = string.Empty; + + private readonly AmazonSQSClient? _sqsClient; + private readonly AmazonS3Client? _s3Client; + private readonly AmazonSQSExtendedClient _sqSExtendedClient; + + public string Name => "AWS SQS Subscriber"; + + public SqsMessageSubscriberService(IOptions options, + ILogger logger) + { + Guard.Against.Null(options, nameof(options)); + + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + var configuration = options.Value; + ValidateConfiguration(configuration); + _queueName = configuration.SubscriberSettings[SQSConfigurationKeys.ExportRequestQueue]; + _bucketName = configuration.SubscriberSettings[SQSConfigurationKeys.BucketName]; + + + if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.AccessKey)) + _accessKey = configuration.SubscriberSettings[SQSConfigurationKeys.AccessKey]; + + + if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.AccessToken)) + _accessToken = configuration.SubscriberSettings[SQSConfigurationKeys.AccessToken]; + + + if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.Envid)) + _environmentId = configuration.SubscriberSettings[SQSConfigurationKeys.Envid]; + + try + { + _logger.ConnectingToSQS(Name); + + if (!(_accessKey is null) && !(_accessToken is null)) + { + _logger.LogInformation("Assuming IAM user as found in the configuration file."); + _sqsClient = new AmazonSQSClient(_accessKey, _accessToken); + _s3Client = new AmazonS3Client(_accessKey, _accessToken); + } + else + { + _logger.LogInformation("Attempting to assume local AWS credentials."); + _sqsClient = new AmazonSQSClient(); + _s3Client = new AmazonS3Client(); + } + + _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, + new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, _bucketName)); + + } + catch (Amazon.SQS.AmazonSQSException Ex) + { + _logger.ConnectingToSQSError(Name, Ex); + Guard.Against.Null(_sqSExtendedClient, nameof(_sqSExtendedClient)); + } + } + + + + private void ValidateConfiguration(MessageBrokerServiceConfiguration configuration) + { + Guard.Against.Null(configuration, nameof(configuration)); + Guard.Against.Null(configuration.SubscriberSettings, nameof(configuration.SubscriberSettings)); + + foreach (var key in SQSConfigurationKeys.SubscriberRequiredKeys) + { + if (!configuration.SubscriberSettings.ContainsKey(key)) + { + throw new ConfigurationException($"{Name} is missing configuration for {key}."); + } + } + } + + public void Subscribe(string topic, string queue, Action messageReceivedCallback, ushort prefetchCount = 0) + => Subscribe(new string[] { topic }, queue, messageReceivedCallback, prefetchCount); + + public void Subscribe(string[] topics, string queue, Action messageReceivedCallback, ushort prefetchCount = 0) + { + Guard.Against.Null(topics, nameof(topics)); + Guard.Against.Null(messageReceivedCallback, nameof(messageReceivedCallback)); + + foreach (string topic in topics) + { + Task.Run(() => + { + + + string queueName = QueueFormatter.FormatQueueName(_environmentId, _queueName, topic); + _logger.LogDebug($"Attempting to create or subscribe to {queueName}"); + + var queueAttributes = new Dictionary(); + + queueAttributes.Add("KmsMasterKeyId", "alias/aws/sqs"); + var request = new CreateQueueRequest + { + Attributes = queueAttributes, + QueueName = queueName + }; + + CreateQueueResponse createQueueResponse = new CreateQueueResponse(); + try + { + createQueueResponse = _sqSExtendedClient.CreateQueueAsync(request).Result; + } + catch (Exception ex) + { + _logger.LogDebug($"The queue could not be created or subscribed to: {ex.Message}"); + } + + + while (true) + { + List AttributesList = new List(); + AttributesList.Add("*"); + + var messageResponse = _sqSExtendedClient.ReceiveMessageAsync(new ReceiveMessageRequest + { + QueueUrl = createQueueResponse.QueueUrl, + WaitTimeSeconds = 2, + AttributeNames = new List { "All" }, + MessageAttributeNames = new List { "All" } + }).Result; + var messages = messageResponse.Messages; + + if (messages.Any()) + { + foreach (var msg in messages) + { + _logger.Log(LogLevel.Debug, $"Message {msg.MessageId} received from SQS."); + MessageReceivedEventArgs messageReceivedEventArgs = CreateMessage(msg); + try + { + _logger.AcknowledgementSent(msg.MessageId); + _sqSExtendedClient.DeleteMessageAsync(new DeleteMessageRequest { QueueUrl = createQueueResponse.QueueUrl, ReceiptHandle = msg.ReceiptHandle }).Wait(); + messageReceivedCallback(messageReceivedEventArgs); + } + catch (Exception ex) + { + _logger.Log(LogLevel.Error, ex.Message); + } + } + } + } + }); + } + + + + } + + public void SubscribeAsync(string topic, string queue, Func messageReceivedCallback, ushort prefetchCount = 0) + => SubscribeAsync(new string[] { topic }, queue, messageReceivedCallback, prefetchCount); + + public void SubscribeAsync(string[] topics, string queue, Func messageReceivedCallback, ushort prefetchCount = 0) + { + throw new NotImplementedException(); + + } + + public void Acknowledge(MessageBase message) + { + //We will use this to delelete the message from the SQS qeue. + } + + public void Reject(MessageBase message, bool requeue = true) + { + Guard.Against.Null(message, nameof(message)); + + + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + + } + + _disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + + private static MessageReceivedEventArgs CreateMessage(Amazon.SQS.Model.Message msg) + { + + Guard.Against.Null(msg, nameof(msg)); + Guard.Against.Null(msg.Body, nameof(msg.Body)); + Guard.Against.Null(msg.Attributes, nameof(msg.Attributes)); + Guard.Against.Null(msg.MessageId, nameof(msg.MessageId)); + + JObject bodyobj = JObject.Parse(msg.Body); + + return new MessageReceivedEventArgs( + new Monai.Deploy.Messaging.Messages.Message( + body: Encoding.UTF8.GetBytes(msg.Body), + messageDescription: "desc1", + messageId: bodyobj["MessageId"].ToString(), + applicationId: msg.Attributes["SenderId"], + contentType: msg.MessageAttributes["ContentType"].ToString(), + correlationId: bodyobj["correlation_id"].ToString(), + creationDateTime: DateTimeOffset.FromUnixTimeMilliseconds(Int64.Parse(msg.Attributes["SentTimestamp"])), + deliveryTag: msg.ReceiptHandle) + , CancellationToken.None); + } + } +} diff --git a/src/Messaging/SQS/README.MD b/src/Messaging/SQS/README.MD index d265d47..09d24f1 100644 --- a/src/Messaging/SQS/README.MD +++ b/src/Messaging/SQS/README.MD @@ -4,8 +4,184 @@ 💡 If you want to know more about MONAI Deploy WG vision, overall structure, and guidelines, please read [MONAI Deploy](https://github.com/Project-MONAI/monai-deploy) first. -# MONAI Deploy Messaging -Messaging layer for MONAI Deploy clinical data pipelines system. +# MONAI Deploy Messaging SQS Plug-In +AWS SQS plugin for the messaging layer of MONAI Deploy. + +## Information: +This plugin can be used as a messaging mechanism alternative to RabbitMQ, and allows MONAI Deploy to integrate with AWS Simple Queue Service. Unlike RabbitMQ, SQS does not have a concept of vhost, exchange and routing key; instead the plugin will create one specific queue for each vhost, exchange and routing key combibation. + + + + + + + + + + + + + + + + + +
+ RMQ plugin + + SQS plugin + Description +
vhostenvironmentIdThe vhost namespace is replaced be the parameter environmentId. This parmater is used as a prefix for each queue name, allowing mutliple MONAI deploy installations on the same AWS account with not queue name conflict.
exchange and routing keyworkflowRequestQueue and exportRequestQueueThese parmaeters control the name of the queues for each purpose. Each queue name will be suffixed with the name routing key.
+ +* Queue names generated by the plugin have the following name convention : [environmentId]_[RequestQueue]_[routingKey]. +* Queue names will be less or equal to 80 characters long. The name will be truncated if the combinatino of environmentId, RequestQueue and routingKey exceeds this limit. +* Any non-alphanumerical character other than "-" and "\_" found in the environmentId, RequestQueue, parameters or routing key variables will be replaced by "\_". +* The queues are created automatically upon the 1st message publication/subscription if they do not already exist. +* Because MONAI Deploy can generate payloads greater than 256KBytes, the plugin leverages the SQS extended client, allowing payload up to 2GB in size. The use of this specific client mandates the usage of an S3 bucket as transient store for the messages to be held until their delivery to the queue subscriber. More information is available below in this documentation about IAM privileges, SQS and S3 bucket requirements. + + + +## Plugin activation + +Because MONAI Deploy plugin management work is still in progress (as of 06/13/2022), plugins cannot be loaded at run-time. Instead this SQS pluging can be activated by doing small code changes to the MONAI Informatic Gateway project https://github.com/Project-MONAI/monai-deploy-informatics-gateway, in the `Program.cs` file and recompiling. + +In the declaration for `CreateHostBuilder`, locate the following code block: + + services.UseRabbitMq(); + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.PublisherServiceAssemblyName); + }); + + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.SubscriberServiceAssemblyName); + }); + + + +and alter it by commenting the line `services.UseRabbitMq();`, replace `services.AddSingleton();` by `services.AddSingleton();` and the line `services.AddSingleton();` by `services.AddSingleton();`. The code block should now look like this : + + + //services.UseRabbitMq(); + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.PublisherServiceAssemblyName); + }); + + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.SubscriberServiceAssemblyName); + }); + +## MONAI Informatic Gateway Configuration + +The plugin is configured in the Messaging section of `appsettings.json` / `appsettings.Development.json` : + +```json + "messaging": { + "publisherServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SQSMessagePublisherService, Monai.Deploy.Messaging", + "subscriberServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SqsMessageSubscriberService, Monai.Deploy.Messaging", + "publisherSettings": { + "bucketName": "monai-minio", + "workflowRequestQueue": "workflow_tasks", + "environmentId": "monai-1", + "accessKey": "ASDFGHJKLADF123456789", + "accessToken": "QwErTyUiOpAsDonMB88W1mcCCwQdePe8X27SEu1S" + }, + "subscriberSettings": { + "exportRequestQueue": "export_tasks", + "bucketName": "monai-minio", + "environmentId": "monai-1", + "accessKey": "ASDFGHJKLADF123456789", + "accessToken": "QwErTyUiOpAsDonMB88W1mcCCwQdePe8X27SEu1S" + } + }, +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
bucketNameS3 bucket used to store the messages temporarily until the subscriber gets it.
workflowRequestQueueQueue prefix for the workflow requests ( MIG -> Workflow Manager ). This parameter is only useful in the PublisherSettings section.
exportRequestQueueQueue prefix for the export requests ( Workflow Manager -> MIG ). This parameter is only useful in the SubscriberSettings section.
bucketNameS3 bucket used to store the messages temporarily until the subscriber gets it.
environmentIdA prefix used to identify the MONAI environment. This allows to create multiple environments within a single AWS account without queue name conflict. This parameter allows for a configuration comparable to the vhost concept in RabbitMq.
accessKeyAWS IAM user access key. This parameter is optional. If not present the plugin will fallback to local credentials, then EC2 role. If this parameter is used the parameter accessToken is also required. Refer to the section IAM privileges for IAM configuration.
accessTokenAWS IAM user access token. This parameter is only required when the parameter accesskey is provided.
+ +## IAM Privileges + +For the plugin to function a set of specific privileges need to be provided. The permissions can be associated either with an IAM user ( by using accessKey and accessToken in the configuratio file ) or by running MONAI on an EC2 instance associated with an IAM Role granted for the following privileges: + +Replace the tags below in the below policy as follow : + +[AWS_Account] : The AWS Account ID ( numeric ) +[EnvironmentId] : The Environment Id use int Subscriber and Publisher settings of the Messaging seciton of `appsettings.json` / `appsettings.Development.json`. +[BucketName] : the name of the S3 bucket used to store the messages temporarily. The same bucket as the one used to store incoming DICOM objects can be used if desired. ( see the AWS S3 plugin to use S3 natively with MONAI Deploy ) + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "sqs:DeleteMessage", + "s3:PutObject", + "s3:GetObject", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:CreateQueue", + "s3:DeleteObject", + "sqs:SetQueueAttributes" + ], + "Resource": [ + "arn:aws:sqs:*:[AWS_Account]:[EnvironmentId]", + "arn:aws:s3:::[BucketName]/*" + ] + } + ] +} +``` ## Contributing @@ -21,4 +197,4 @@ Ask and answer questions over on [MONAI Deploy Workflow Manager's GitHub Discuss - Code: - Project tracker: - Issue tracker: -- Test status: \ No newline at end of file +- Test status: From 4edabf0dde3e4977739fd985d740e23a0d361d5d Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 08:37:26 -0500 Subject: [PATCH 35/45] Code analysis fixes. Signed-off-by: JP LEGER --- src/Messaging/SQS/README.MD | 2 +- src/Messaging/SQS/SQSMessagePublisherService.cs | 12 +++++------- src/Messaging/SQS/SQSMessageSubscriberService.cs | 6 ++---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/Messaging/SQS/README.MD b/src/Messaging/SQS/README.MD index 09d24f1..10fbd72 100644 --- a/src/Messaging/SQS/README.MD +++ b/src/Messaging/SQS/README.MD @@ -96,7 +96,7 @@ The plugin is configured in the Messaging section of `appsettings.json` / `appse ```json "messaging": { - "publisherServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SQSMessagePublisherService, Monai.Deploy.Messaging", + "publisherServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SqsMessagePublisherService, Monai.Deploy.Messaging", "subscriberServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SqsMessageSubscriberService, Monai.Deploy.Messaging", "publisherSettings": { "bucketName": "monai-minio", diff --git a/src/Messaging/SQS/SQSMessagePublisherService.cs b/src/Messaging/SQS/SQSMessagePublisherService.cs index d78ec6b..e6839a0 100644 --- a/src/Messaging/SQS/SQSMessagePublisherService.cs +++ b/src/Messaging/SQS/SQSMessagePublisherService.cs @@ -14,11 +14,11 @@ namespace Monai.Deploy.Messaging.SQS { - public class SQSMessagePublisherService : IMessageBrokerPublisherService + public class SqsMessagePublisherService : IMessageBrokerPublisherService { private const int PersistentDeliveryMode = 2; - private readonly ILogger _logger; + private readonly ILogger _logger; private readonly string? _accessKey; private readonly string? _accessToken; private readonly string _environmentId = string.Empty; @@ -32,8 +32,8 @@ public class SQSMessagePublisherService : IMessageBrokerPublisherService private readonly AmazonS3Client? _s3Client; private readonly AmazonSQSExtendedClient? _sqSExtendedClient; - public SQSMessagePublisherService(IOptions options, - ILogger logger) + public SqsMessagePublisherService(IOptions options, + ILogger logger) { Guard.Against.Null(options, nameof(options)); @@ -42,8 +42,6 @@ public SQSMessagePublisherService(IOptions op var configuration = options.Value; ValidateConfiguration(configuration); - - //This 2 config entries are mandatory. _queueName = configuration.PublisherSettings[SQSConfigurationKeys.WorkflowRequestQueue]; _bucketName = configuration.PublisherSettings[SQSConfigurationKeys.BucketName]; @@ -176,7 +174,7 @@ public Task Publish(string topic, Monai.Deploy.Messaging.Messages.Message messag { SendMessageResponse sqsresp = _sqSExtendedClient.SendMessageAsync(sendMessageRequest).Result; } - catch(Exception e) + catch (Exception e) { _logger.LogError($"The message could not be posted to the queue {queueName} : \n {e.Message}"); } diff --git a/src/Messaging/SQS/SQSMessageSubscriberService.cs b/src/Messaging/SQS/SQSMessageSubscriberService.cs index 8af30df..1f273c5 100644 --- a/src/Messaging/SQS/SQSMessageSubscriberService.cs +++ b/src/Messaging/SQS/SQSMessageSubscriberService.cs @@ -188,14 +188,13 @@ public void SubscribeAsync(string[] topics, string queue, Func Date: Tue, 14 Jun 2022 09:05:02 -0500 Subject: [PATCH 36/45] remove unecessary folder. Signed-off-by: JP Leger Signed-off-by: JP LEGER --- SQS/ConfigurationKeys.cs | 19 --- SQS/Log.cs | 49 ------ SQS/QueueFormatter.cs | 28 ---- SQS/README.MD | 200 ----------------------- SQS/SQSMessagePublisherService.cs | 209 ------------------------ SQS/SQSMessageSubscriberService.cs | 245 ----------------------------- 6 files changed, 750 deletions(-) delete mode 100644 SQS/ConfigurationKeys.cs delete mode 100644 SQS/Log.cs delete mode 100644 SQS/QueueFormatter.cs delete mode 100644 SQS/README.MD delete mode 100644 SQS/SQSMessagePublisherService.cs delete mode 100644 SQS/SQSMessageSubscriberService.cs diff --git a/SQS/ConfigurationKeys.cs b/SQS/ConfigurationKeys.cs deleted file mode 100644 index 54d0e11..0000000 --- a/SQS/ConfigurationKeys.cs +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium -// SPDX-License-Identifier: Apache License 2.0 - -namespace Monai.Deploy.Messaging.Configuration -{ - internal static class SQSConfigurationKeys - { - public static readonly string AccessKey = "accessKey"; - public static readonly string AccessToken = "accessToken"; - public static readonly string Region = "region"; - public static readonly string WorkflowRequestQueue = "workflowRequestQueue"; - public static readonly string ExportRequestQueue = "exportRequestQueue"; - public static readonly string BucketName = "bucketName"; - public static readonly string Envid = "environmentId"; - - public static readonly string[] PublisherRequiredKeys = new[] { WorkflowRequestQueue, BucketName }; - public static readonly string[] SubscriberRequiredKeys = new[] { ExportRequestQueue, BucketName }; - } -} diff --git a/SQS/Log.cs b/SQS/Log.cs deleted file mode 100644 index 9d15df2..0000000 --- a/SQS/Log.cs +++ /dev/null @@ -1,49 +0,0 @@ -using Microsoft.Extensions.Logging; - -namespace Monai.Deploy.Messaging.SQS -{ - public static partial class Log - { - internal static readonly string LoggingScopeMessageApplication = "Message ID={0}. Application ID={1}."; - - - [LoggerMessage(EventId = 10000, Level = LogLevel.Information, Message = "Publishing message {MessageId} to Queue={topic}.")] - public static partial void PublishingToSQS(this ILogger logger, string topic, string MessageId); - - [LoggerMessage(EventId = 10001, Level = LogLevel.Information, Message = "{ServiceName} connecting to SQS.")] - public static partial void ConnectingToSQS(this ILogger logger, string serviceName); - - [LoggerMessage(EventId = 10002, Level = LogLevel.Information, Message = "Message received from queue {queue}.")] - public static partial void MessageReceivedFromQueue(this ILogger logger, string queue, string topic); - - [LoggerMessage(EventId = 10003, Level = LogLevel.Information, Message = "Listening for messages from {endpoint}. Queue={queue}.")] - public static partial void SubscribeToSQSQueue(this ILogger logger, string endpoint, string virtualHost, string exchange, string queue, string topic); - - [LoggerMessage(EventId = 10004, Level = LogLevel.Information, Message = "Sending message acknowledgement for message {messageId}.")] - public static partial void SendingAcknowledgement(this ILogger logger, string messageId); - - [LoggerMessage(EventId = 10005, Level = LogLevel.Information, Message = "Ackowledge sent for message {messageId}.")] - public static partial void AcknowledgementSent(this ILogger logger, string messageId); - - [LoggerMessage(EventId = 10006, Level = LogLevel.Information, Message = "Sending nack message {messageId} and requeuing.")] - public static partial void SendingNAcknowledgement(this ILogger logger, string messageId); - - [LoggerMessage(EventId = 10007, Level = LogLevel.Information, Message = "Nack message sent for message {messageId}, requeue={requeue}.")] - public static partial void NAcknowledgementSent(this ILogger logger, string messageId, bool requeue); - - [LoggerMessage(EventId = 10008, Level = LogLevel.Information, Message = "Closing connections.")] - public static partial void ClosingConnections(this ILogger logger); - - [LoggerMessage(EventId = 10009, Level = LogLevel.Error, Message = "Invalid or corrupted message received: Queue={queueName}, Message ID={messageId}.")] - public static partial void InvalidMessage(this ILogger logger, string queueName, string topic, string messageId, Exception ex); - - [LoggerMessage(EventId = 10010, Level = LogLevel.Error, Message = "Exception not handled by the subscriber's callback function: Queue={queueName}, Message ID={messageId}.")] - public static partial void ErrorNotHandledByCallback(this ILogger logger, string queueName, string topic, string messageId, Exception ex); - - [LoggerMessage(EventId = 10011, Level = LogLevel.Error, Message = "Creating SQS client.")] - public static partial void CreateSQSClient(this ILogger logger); - - [LoggerMessage(EventId = 10012, Level = LogLevel.Error, Message = "{ServiceName} failed to connect to SQS.")] - public static partial void ConnectingToSQSError(this ILogger logger, string serviceName, Exception ex); - } -} diff --git a/SQS/QueueFormatter.cs b/SQS/QueueFormatter.cs deleted file mode 100644 index 7c3181a..0000000 --- a/SQS/QueueFormatter.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Text.RegularExpressions; - -namespace Monai.Deploy.Messaging.SQS -{ - internal static class QueueFormatter - { - /// - /// Returns an aggregate of the the environmentId, queueBasename nd topic as the name of the queue defined in SQS. - /// The returned string is made compliant to SQS naming convention : It will replace non alphanumeric and other characters than "_" and "-", by an hyphen - /// - /// - /// - /// - /// string - public static string FormatQueueName(string environmentId, string? queuebasename, string topic) - { - - string queue = $"{queuebasename}_{topic}"; - - if (!environmentId.Equals(String.Empty)) - queue = $"{environmentId}_{queue}"; - queue = Regex.Replace(queue, "[^a-zA-Z0-9_]", "-"); - if (queue.Length > 80) - queue = queue.Substring(0, 80); - return queue; - } - } -} diff --git a/SQS/README.MD b/SQS/README.MD deleted file mode 100644 index 09d24f1..0000000 --- a/SQS/README.MD +++ /dev/null @@ -1,200 +0,0 @@ -

-project-monai -

- -💡 If you want to know more about MONAI Deploy WG vision, overall structure, and guidelines, please read [MONAI Deploy](https://github.com/Project-MONAI/monai-deploy) first. - -# MONAI Deploy Messaging SQS Plug-In -AWS SQS plugin for the messaging layer of MONAI Deploy. - -## Information: -This plugin can be used as a messaging mechanism alternative to RabbitMQ, and allows MONAI Deploy to integrate with AWS Simple Queue Service. Unlike RabbitMQ, SQS does not have a concept of vhost, exchange and routing key; instead the plugin will create one specific queue for each vhost, exchange and routing key combibation. - - - - - - - - - - - - - - - - - -
- RMQ plugin - - SQS plugin - Description -
vhostenvironmentIdThe vhost namespace is replaced be the parameter environmentId. This parmater is used as a prefix for each queue name, allowing mutliple MONAI deploy installations on the same AWS account with not queue name conflict.
exchange and routing keyworkflowRequestQueue and exportRequestQueueThese parmaeters control the name of the queues for each purpose. Each queue name will be suffixed with the name routing key.
- -* Queue names generated by the plugin have the following name convention : [environmentId]_[RequestQueue]_[routingKey]. -* Queue names will be less or equal to 80 characters long. The name will be truncated if the combinatino of environmentId, RequestQueue and routingKey exceeds this limit. -* Any non-alphanumerical character other than "-" and "\_" found in the environmentId, RequestQueue, parameters or routing key variables will be replaced by "\_". -* The queues are created automatically upon the 1st message publication/subscription if they do not already exist. -* Because MONAI Deploy can generate payloads greater than 256KBytes, the plugin leverages the SQS extended client, allowing payload up to 2GB in size. The use of this specific client mandates the usage of an S3 bucket as transient store for the messages to be held until their delivery to the queue subscriber. More information is available below in this documentation about IAM privileges, SQS and S3 bucket requirements. - - - -## Plugin activation - -Because MONAI Deploy plugin management work is still in progress (as of 06/13/2022), plugins cannot be loaded at run-time. Instead this SQS pluging can be activated by doing small code changes to the MONAI Informatic Gateway project https://github.com/Project-MONAI/monai-deploy-informatics-gateway, in the `Program.cs` file and recompiling. - -In the declaration for `CreateHostBuilder`, locate the following code block: - - services.UseRabbitMq(); - services.AddSingleton(); - services.AddSingleton(implementationFactory => - { - var options = implementationFactory.GetService>(); - var serviceProvider = implementationFactory.GetService(); - var logger = implementationFactory.GetService>(); - return serviceProvider.LocateService(logger, options.Value.Messaging.PublisherServiceAssemblyName); - }); - - services.AddSingleton(); - services.AddSingleton(implementationFactory => - { - var options = implementationFactory.GetService>(); - var serviceProvider = implementationFactory.GetService(); - var logger = implementationFactory.GetService>(); - return serviceProvider.LocateService(logger, options.Value.Messaging.SubscriberServiceAssemblyName); - }); - - - -and alter it by commenting the line `services.UseRabbitMq();`, replace `services.AddSingleton();` by `services.AddSingleton();` and the line `services.AddSingleton();` by `services.AddSingleton();`. The code block should now look like this : - - - //services.UseRabbitMq(); - services.AddSingleton(); - services.AddSingleton(implementationFactory => - { - var options = implementationFactory.GetService>(); - var serviceProvider = implementationFactory.GetService(); - var logger = implementationFactory.GetService>(); - return serviceProvider.LocateService(logger, options.Value.Messaging.PublisherServiceAssemblyName); - }); - - services.AddSingleton(); - services.AddSingleton(implementationFactory => - { - var options = implementationFactory.GetService>(); - var serviceProvider = implementationFactory.GetService(); - var logger = implementationFactory.GetService>(); - return serviceProvider.LocateService(logger, options.Value.Messaging.SubscriberServiceAssemblyName); - }); - -## MONAI Informatic Gateway Configuration - -The plugin is configured in the Messaging section of `appsettings.json` / `appsettings.Development.json` : - -```json - "messaging": { - "publisherServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SQSMessagePublisherService, Monai.Deploy.Messaging", - "subscriberServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SqsMessageSubscriberService, Monai.Deploy.Messaging", - "publisherSettings": { - "bucketName": "monai-minio", - "workflowRequestQueue": "workflow_tasks", - "environmentId": "monai-1", - "accessKey": "ASDFGHJKLADF123456789", - "accessToken": "QwErTyUiOpAsDonMB88W1mcCCwQdePe8X27SEu1S" - }, - "subscriberSettings": { - "exportRequestQueue": "export_tasks", - "bucketName": "monai-minio", - "environmentId": "monai-1", - "accessKey": "ASDFGHJKLADF123456789", - "accessToken": "QwErTyUiOpAsDonMB88W1mcCCwQdePe8X27SEu1S" - } - }, -``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
bucketNameS3 bucket used to store the messages temporarily until the subscriber gets it.
workflowRequestQueueQueue prefix for the workflow requests ( MIG -> Workflow Manager ). This parameter is only useful in the PublisherSettings section.
exportRequestQueueQueue prefix for the export requests ( Workflow Manager -> MIG ). This parameter is only useful in the SubscriberSettings section.
bucketNameS3 bucket used to store the messages temporarily until the subscriber gets it.
environmentIdA prefix used to identify the MONAI environment. This allows to create multiple environments within a single AWS account without queue name conflict. This parameter allows for a configuration comparable to the vhost concept in RabbitMq.
accessKeyAWS IAM user access key. This parameter is optional. If not present the plugin will fallback to local credentials, then EC2 role. If this parameter is used the parameter accessToken is also required. Refer to the section IAM privileges for IAM configuration.
accessTokenAWS IAM user access token. This parameter is only required when the parameter accesskey is provided.
- -## IAM Privileges - -For the plugin to function a set of specific privileges need to be provided. The permissions can be associated either with an IAM user ( by using accessKey and accessToken in the configuratio file ) or by running MONAI on an EC2 instance associated with an IAM Role granted for the following privileges: - -Replace the tags below in the below policy as follow : - -[AWS_Account] : The AWS Account ID ( numeric ) -[EnvironmentId] : The Environment Id use int Subscriber and Publisher settings of the Messaging seciton of `appsettings.json` / `appsettings.Development.json`. -[BucketName] : the name of the S3 bucket used to store the messages temporarily. The same bucket as the one used to store incoming DICOM objects can be used if desired. ( see the AWS S3 plugin to use S3 natively with MONAI Deploy ) - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "sqs:DeleteMessage", - "s3:PutObject", - "s3:GetObject", - "sqs:GetQueueUrl", - "sqs:ReceiveMessage", - "sqs:SendMessage", - "sqs:GetQueueAttributes", - "sqs:CreateQueue", - "s3:DeleteObject", - "sqs:SetQueueAttributes" - ], - "Resource": [ - "arn:aws:sqs:*:[AWS_Account]:[EnvironmentId]", - "arn:aws:s3:::[BucketName]/*" - ] - } - ] -} -``` - -## Contributing - -For guidance on making a contribution to MONAI Deploy Workflow Manager, see the [contributing guidelines](https://github.com/Project-MONAI/monai-deploy/blob/main/CONTRIBUTING.md). - -Join the conversation on Twitter [@ProjectMONAI](https://twitter.com/ProjectMONAI) or join our [Slack channel](https://forms.gle/QTxJq3hFictp31UM9). - -Ask and answer questions over on [MONAI Deploy Workflow Manager's GitHub Discussions tab](https://github.com/Project-MONAI/monai-deploy-workflow-manager/discussions). - -## Links - -- Website: -- Code: -- Project tracker: -- Issue tracker: -- Test status: diff --git a/SQS/SQSMessagePublisherService.cs b/SQS/SQSMessagePublisherService.cs deleted file mode 100644 index d78ec6b..0000000 --- a/SQS/SQSMessagePublisherService.cs +++ /dev/null @@ -1,209 +0,0 @@ -// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium -// SPDX-License-Identifier: Apache License 2.0 - -using System.Globalization; -using System.Text; -using Amazon.S3; -using Amazon.SQS; -using Amazon.SQS.ExtendedClient; -using Amazon.SQS.Model; -using Ardalis.GuardClauses; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Monai.Deploy.Messaging.Configuration; - -namespace Monai.Deploy.Messaging.SQS -{ - public class SQSMessagePublisherService : IMessageBrokerPublisherService - { - private const int PersistentDeliveryMode = 2; - - private readonly ILogger _logger; - private readonly string? _accessKey; - private readonly string? _accessToken; - private readonly string _environmentId = string.Empty; - private bool _disposedValue; - - - public string Name => "AWS SQS Publisher"; - private readonly string _queueName; - private readonly string _bucketName; - private readonly AmazonSQSClient? _sqsClient; - private readonly AmazonS3Client? _s3Client; - private readonly AmazonSQSExtendedClient? _sqSExtendedClient; - - public SQSMessagePublisherService(IOptions options, - ILogger logger) - { - Guard.Against.Null(options, nameof(options)); - - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - - var configuration = options.Value; - ValidateConfiguration(configuration); - - - //This 2 config entries are mandatory. - _queueName = configuration.PublisherSettings[SQSConfigurationKeys.WorkflowRequestQueue]; - _bucketName = configuration.PublisherSettings[SQSConfigurationKeys.BucketName]; - - - if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.AccessKey)) - { - _logger.LogInformation("accessKey found in configuration."); - _accessKey = configuration.PublisherSettings[SQSConfigurationKeys.AccessKey]; - } - - - if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.AccessToken)) - { - _logger.LogInformation("accessToken found in configuration."); - _accessToken = configuration.PublisherSettings[SQSConfigurationKeys.AccessToken]; - } - - if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.Envid)) - _environmentId = configuration.PublisherSettings[SQSConfigurationKeys.Envid]; - - try - { - _logger.ConnectingToSQS(Name); - - if (!(_accessKey is null) && !(_accessToken is null)) - { - _logger.LogInformation("Assuming IAM user as found in the configuration file."); - _sqsClient = new AmazonSQSClient(_accessKey, _accessToken); - _s3Client = new AmazonS3Client(_accessKey, _accessToken); - } - else - { - _logger.LogInformation("Attempting to assume local AWS credentials."); - _sqsClient = new AmazonSQSClient(); - _s3Client = new AmazonS3Client(); - } - - _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, - new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, _bucketName)); - - - - } - catch (Amazon.SQS.AmazonSQSException Ex) - { - _logger.ConnectingToSQSError(Name, Ex); - } - } - - private void ValidateConfiguration(MessageBrokerServiceConfiguration configuration) - { - Guard.Against.Null(configuration, nameof(configuration)); - Guard.Against.Null(configuration.PublisherSettings, nameof(configuration.PublisherSettings)); - - foreach (var key in ConfigurationKeys.PublisherRequiredKeys) - { - if (!configuration.PublisherSettings.ContainsKey(key)) - { - throw new ConfigurationException($"{Name} is missing configuration for {key}."); - } - } - } - - public Task Publish(string topic, Monai.Deploy.Messaging.Messages.Message message) - { - - Guard.Against.NullOrWhiteSpace(topic, nameof(topic)); - Guard.Against.Null(message, nameof(message)); - - - using var loggerScope = _logger.BeginScope(string.Format(CultureInfo.InvariantCulture, Log.LoggingScopeMessageApplication, message.MessageId, message.ApplicationId)); - _logger.PublishingToSQS(topic, message.MessageId); - var sendMessageRequest = new SendMessageRequest(); - - Dictionary MessageAttributes = new Dictionary(); - MessageAttributeValue messageIdAttribute = new MessageAttributeValue(); - messageIdAttribute.DataType = "String"; - messageIdAttribute.StringValue = message.MessageId; - MessageAttributes.Add("MessageId", messageIdAttribute); - - MessageAttributeValue ContentTypeAttribute = new MessageAttributeValue(); - ContentTypeAttribute.DataType = "String"; - ContentTypeAttribute.StringValue = message.ContentType; - MessageAttributes.Add("ContentType", ContentTypeAttribute); - - - MessageAttributeValue ApplicationIdAttribute = new MessageAttributeValue(); - ApplicationIdAttribute.DataType = "String"; - ApplicationIdAttribute.StringValue = message.MessageId; - MessageAttributes.Add("ApplicationId", ApplicationIdAttribute); - - sendMessageRequest.MessageAttributes = MessageAttributes; - - - Console.WriteLine("Message information : "); - Console.WriteLine(message); - Console.WriteLine(message.Body); - Console.WriteLine(message.Body.Length); - - - string queueName = QueueFormatter.FormatQueueName(_environmentId, _queueName, topic); - _logger.LogDebug($"Attempting to create or subscribe to {queueName}"); - - var queueAttributes = new Dictionary(); - - queueAttributes.Add("KmsMasterKeyId", "alias/aws/sqs"); - var request = new CreateQueueRequest - { - Attributes = queueAttributes, - QueueName = queueName - }; - - CreateQueueResponse createQueueResponse = new CreateQueueResponse(); - try - { - createQueueResponse = _sqSExtendedClient.CreateQueueAsync(request).Result; - } - catch (Exception ex) - { - _logger.LogDebug($"The queue could not be created or subscribed to: {ex.Message}"); - } - - sendMessageRequest.QueueUrl = createQueueResponse.QueueUrl; - - - - sendMessageRequest.MessageBody = Encoding.UTF8.GetString(message.Body, 0, message.Body.Length); - - try - { - SendMessageResponse sqsresp = _sqSExtendedClient.SendMessageAsync(sendMessageRequest).Result; - } - catch(Exception e) - { - _logger.LogError($"The message could not be posted to the queue {queueName} : \n {e.Message}"); - } - - - return Task.CompletedTask; - } - - - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) - { - if (disposing) - { - // Dispose any managed objects - } - - _disposedValue = true; - } - } - - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - } -} diff --git a/SQS/SQSMessageSubscriberService.cs b/SQS/SQSMessageSubscriberService.cs deleted file mode 100644 index 8af30df..0000000 --- a/SQS/SQSMessageSubscriberService.cs +++ /dev/null @@ -1,245 +0,0 @@ -// SPDX-FileCopyrightText: © 2021-2022 MONAI Consortium -// SPDX-License-Identifier: Apache License 2.0 - -using System.Text; -using Amazon.S3; -using Amazon.SQS; -using Amazon.SQS.ExtendedClient; -using Amazon.SQS.Model; -using Ardalis.GuardClauses; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Options; -using Monai.Deploy.Messaging.Common; -using Monai.Deploy.Messaging.Configuration; -using Monai.Deploy.Messaging.Messages; -using Newtonsoft.Json.Linq; - -namespace Monai.Deploy.Messaging.SQS -{ - public class SqsMessageSubscriberService : IMessageBrokerSubscriberService - { - private readonly ILogger _logger; - private bool _disposedValue; - private readonly string? _accessKey; - private readonly string? _accessToken; - private readonly string? _queueName; - private readonly string? _bucketName; - private readonly string _environmentId = string.Empty; - - private readonly AmazonSQSClient? _sqsClient; - private readonly AmazonS3Client? _s3Client; - private readonly AmazonSQSExtendedClient _sqSExtendedClient; - - public string Name => "AWS SQS Subscriber"; - - public SqsMessageSubscriberService(IOptions options, - ILogger logger) - { - Guard.Against.Null(options, nameof(options)); - - _logger = logger ?? throw new ArgumentNullException(nameof(logger)); - - var configuration = options.Value; - ValidateConfiguration(configuration); - _queueName = configuration.SubscriberSettings[SQSConfigurationKeys.ExportRequestQueue]; - _bucketName = configuration.SubscriberSettings[SQSConfigurationKeys.BucketName]; - - - if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.AccessKey)) - _accessKey = configuration.SubscriberSettings[SQSConfigurationKeys.AccessKey]; - - - if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.AccessToken)) - _accessToken = configuration.SubscriberSettings[SQSConfigurationKeys.AccessToken]; - - - if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.Envid)) - _environmentId = configuration.SubscriberSettings[SQSConfigurationKeys.Envid]; - - try - { - _logger.ConnectingToSQS(Name); - - if (!(_accessKey is null) && !(_accessToken is null)) - { - _logger.LogInformation("Assuming IAM user as found in the configuration file."); - _sqsClient = new AmazonSQSClient(_accessKey, _accessToken); - _s3Client = new AmazonS3Client(_accessKey, _accessToken); - } - else - { - _logger.LogInformation("Attempting to assume local AWS credentials."); - _sqsClient = new AmazonSQSClient(); - _s3Client = new AmazonS3Client(); - } - - _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, - new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, _bucketName)); - - } - catch (Amazon.SQS.AmazonSQSException Ex) - { - _logger.ConnectingToSQSError(Name, Ex); - Guard.Against.Null(_sqSExtendedClient, nameof(_sqSExtendedClient)); - } - } - - - - private void ValidateConfiguration(MessageBrokerServiceConfiguration configuration) - { - Guard.Against.Null(configuration, nameof(configuration)); - Guard.Against.Null(configuration.SubscriberSettings, nameof(configuration.SubscriberSettings)); - - foreach (var key in SQSConfigurationKeys.SubscriberRequiredKeys) - { - if (!configuration.SubscriberSettings.ContainsKey(key)) - { - throw new ConfigurationException($"{Name} is missing configuration for {key}."); - } - } - } - - public void Subscribe(string topic, string queue, Action messageReceivedCallback, ushort prefetchCount = 0) - => Subscribe(new string[] { topic }, queue, messageReceivedCallback, prefetchCount); - - public void Subscribe(string[] topics, string queue, Action messageReceivedCallback, ushort prefetchCount = 0) - { - Guard.Against.Null(topics, nameof(topics)); - Guard.Against.Null(messageReceivedCallback, nameof(messageReceivedCallback)); - - foreach (string topic in topics) - { - Task.Run(() => - { - - - string queueName = QueueFormatter.FormatQueueName(_environmentId, _queueName, topic); - _logger.LogDebug($"Attempting to create or subscribe to {queueName}"); - - var queueAttributes = new Dictionary(); - - queueAttributes.Add("KmsMasterKeyId", "alias/aws/sqs"); - var request = new CreateQueueRequest - { - Attributes = queueAttributes, - QueueName = queueName - }; - - CreateQueueResponse createQueueResponse = new CreateQueueResponse(); - try - { - createQueueResponse = _sqSExtendedClient.CreateQueueAsync(request).Result; - } - catch (Exception ex) - { - _logger.LogDebug($"The queue could not be created or subscribed to: {ex.Message}"); - } - - - while (true) - { - List AttributesList = new List(); - AttributesList.Add("*"); - - var messageResponse = _sqSExtendedClient.ReceiveMessageAsync(new ReceiveMessageRequest - { - QueueUrl = createQueueResponse.QueueUrl, - WaitTimeSeconds = 2, - AttributeNames = new List { "All" }, - MessageAttributeNames = new List { "All" } - }).Result; - var messages = messageResponse.Messages; - - if (messages.Any()) - { - foreach (var msg in messages) - { - _logger.Log(LogLevel.Debug, $"Message {msg.MessageId} received from SQS."); - MessageReceivedEventArgs messageReceivedEventArgs = CreateMessage(msg); - try - { - _logger.AcknowledgementSent(msg.MessageId); - _sqSExtendedClient.DeleteMessageAsync(new DeleteMessageRequest { QueueUrl = createQueueResponse.QueueUrl, ReceiptHandle = msg.ReceiptHandle }).Wait(); - messageReceivedCallback(messageReceivedEventArgs); - } - catch (Exception ex) - { - _logger.Log(LogLevel.Error, ex.Message); - } - } - } - } - }); - } - - - - } - - public void SubscribeAsync(string topic, string queue, Func messageReceivedCallback, ushort prefetchCount = 0) - => SubscribeAsync(new string[] { topic }, queue, messageReceivedCallback, prefetchCount); - - public void SubscribeAsync(string[] topics, string queue, Func messageReceivedCallback, ushort prefetchCount = 0) - { - throw new NotImplementedException(); - - } - - public void Acknowledge(MessageBase message) - { - //We will use this to delelete the message from the SQS qeue. - } - - public void Reject(MessageBase message, bool requeue = true) - { - Guard.Against.Null(message, nameof(message)); - - - } - - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) - { - if (disposing) - { - - } - - _disposedValue = true; - } - } - - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); - } - - - private static MessageReceivedEventArgs CreateMessage(Amazon.SQS.Model.Message msg) - { - - Guard.Against.Null(msg, nameof(msg)); - Guard.Against.Null(msg.Body, nameof(msg.Body)); - Guard.Against.Null(msg.Attributes, nameof(msg.Attributes)); - Guard.Against.Null(msg.MessageId, nameof(msg.MessageId)); - - JObject bodyobj = JObject.Parse(msg.Body); - - return new MessageReceivedEventArgs( - new Monai.Deploy.Messaging.Messages.Message( - body: Encoding.UTF8.GetBytes(msg.Body), - messageDescription: "desc1", - messageId: bodyobj["MessageId"].ToString(), - applicationId: msg.Attributes["SenderId"], - contentType: msg.MessageAttributes["ContentType"].ToString(), - correlationId: bodyobj["correlation_id"].ToString(), - creationDateTime: DateTimeOffset.FromUnixTimeMilliseconds(Int64.Parse(msg.Attributes["SentTimestamp"])), - deliveryTag: msg.ReceiptHandle) - , CancellationToken.None); - } - } -} From 75973f046ff8532438e29b8aa5d621d6efc49ebb Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 09:39:25 -0500 Subject: [PATCH 37/45] problematic README.MD removed Signed-off-by: JP LEGER --- src/Messaging/SQS/README.MD | 200 ------------------------------------ 1 file changed, 200 deletions(-) delete mode 100644 src/Messaging/SQS/README.MD diff --git a/src/Messaging/SQS/README.MD b/src/Messaging/SQS/README.MD deleted file mode 100644 index 10fbd72..0000000 --- a/src/Messaging/SQS/README.MD +++ /dev/null @@ -1,200 +0,0 @@ -

-project-monai -

- -💡 If you want to know more about MONAI Deploy WG vision, overall structure, and guidelines, please read [MONAI Deploy](https://github.com/Project-MONAI/monai-deploy) first. - -# MONAI Deploy Messaging SQS Plug-In -AWS SQS plugin for the messaging layer of MONAI Deploy. - -## Information: -This plugin can be used as a messaging mechanism alternative to RabbitMQ, and allows MONAI Deploy to integrate with AWS Simple Queue Service. Unlike RabbitMQ, SQS does not have a concept of vhost, exchange and routing key; instead the plugin will create one specific queue for each vhost, exchange and routing key combibation. - - - - - - - - - - - - - - - - - -
- RMQ plugin - - SQS plugin - Description -
vhostenvironmentIdThe vhost namespace is replaced be the parameter environmentId. This parmater is used as a prefix for each queue name, allowing mutliple MONAI deploy installations on the same AWS account with not queue name conflict.
exchange and routing keyworkflowRequestQueue and exportRequestQueueThese parmaeters control the name of the queues for each purpose. Each queue name will be suffixed with the name routing key.
- -* Queue names generated by the plugin have the following name convention : [environmentId]_[RequestQueue]_[routingKey]. -* Queue names will be less or equal to 80 characters long. The name will be truncated if the combinatino of environmentId, RequestQueue and routingKey exceeds this limit. -* Any non-alphanumerical character other than "-" and "\_" found in the environmentId, RequestQueue, parameters or routing key variables will be replaced by "\_". -* The queues are created automatically upon the 1st message publication/subscription if they do not already exist. -* Because MONAI Deploy can generate payloads greater than 256KBytes, the plugin leverages the SQS extended client, allowing payload up to 2GB in size. The use of this specific client mandates the usage of an S3 bucket as transient store for the messages to be held until their delivery to the queue subscriber. More information is available below in this documentation about IAM privileges, SQS and S3 bucket requirements. - - - -## Plugin activation - -Because MONAI Deploy plugin management work is still in progress (as of 06/13/2022), plugins cannot be loaded at run-time. Instead this SQS pluging can be activated by doing small code changes to the MONAI Informatic Gateway project https://github.com/Project-MONAI/monai-deploy-informatics-gateway, in the `Program.cs` file and recompiling. - -In the declaration for `CreateHostBuilder`, locate the following code block: - - services.UseRabbitMq(); - services.AddSingleton(); - services.AddSingleton(implementationFactory => - { - var options = implementationFactory.GetService>(); - var serviceProvider = implementationFactory.GetService(); - var logger = implementationFactory.GetService>(); - return serviceProvider.LocateService(logger, options.Value.Messaging.PublisherServiceAssemblyName); - }); - - services.AddSingleton(); - services.AddSingleton(implementationFactory => - { - var options = implementationFactory.GetService>(); - var serviceProvider = implementationFactory.GetService(); - var logger = implementationFactory.GetService>(); - return serviceProvider.LocateService(logger, options.Value.Messaging.SubscriberServiceAssemblyName); - }); - - - -and alter it by commenting the line `services.UseRabbitMq();`, replace `services.AddSingleton();` by `services.AddSingleton();` and the line `services.AddSingleton();` by `services.AddSingleton();`. The code block should now look like this : - - - //services.UseRabbitMq(); - services.AddSingleton(); - services.AddSingleton(implementationFactory => - { - var options = implementationFactory.GetService>(); - var serviceProvider = implementationFactory.GetService(); - var logger = implementationFactory.GetService>(); - return serviceProvider.LocateService(logger, options.Value.Messaging.PublisherServiceAssemblyName); - }); - - services.AddSingleton(); - services.AddSingleton(implementationFactory => - { - var options = implementationFactory.GetService>(); - var serviceProvider = implementationFactory.GetService(); - var logger = implementationFactory.GetService>(); - return serviceProvider.LocateService(logger, options.Value.Messaging.SubscriberServiceAssemblyName); - }); - -## MONAI Informatic Gateway Configuration - -The plugin is configured in the Messaging section of `appsettings.json` / `appsettings.Development.json` : - -```json - "messaging": { - "publisherServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SqsMessagePublisherService, Monai.Deploy.Messaging", - "subscriberServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SqsMessageSubscriberService, Monai.Deploy.Messaging", - "publisherSettings": { - "bucketName": "monai-minio", - "workflowRequestQueue": "workflow_tasks", - "environmentId": "monai-1", - "accessKey": "ASDFGHJKLADF123456789", - "accessToken": "QwErTyUiOpAsDonMB88W1mcCCwQdePe8X27SEu1S" - }, - "subscriberSettings": { - "exportRequestQueue": "export_tasks", - "bucketName": "monai-minio", - "environmentId": "monai-1", - "accessKey": "ASDFGHJKLADF123456789", - "accessToken": "QwErTyUiOpAsDonMB88W1mcCCwQdePe8X27SEu1S" - } - }, -``` - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
bucketNameS3 bucket used to store the messages temporarily until the subscriber gets it.
workflowRequestQueueQueue prefix for the workflow requests ( MIG -> Workflow Manager ). This parameter is only useful in the PublisherSettings section.
exportRequestQueueQueue prefix for the export requests ( Workflow Manager -> MIG ). This parameter is only useful in the SubscriberSettings section.
bucketNameS3 bucket used to store the messages temporarily until the subscriber gets it.
environmentIdA prefix used to identify the MONAI environment. This allows to create multiple environments within a single AWS account without queue name conflict. This parameter allows for a configuration comparable to the vhost concept in RabbitMq.
accessKeyAWS IAM user access key. This parameter is optional. If not present the plugin will fallback to local credentials, then EC2 role. If this parameter is used the parameter accessToken is also required. Refer to the section IAM privileges for IAM configuration.
accessTokenAWS IAM user access token. This parameter is only required when the parameter accesskey is provided.
- -## IAM Privileges - -For the plugin to function a set of specific privileges need to be provided. The permissions can be associated either with an IAM user ( by using accessKey and accessToken in the configuratio file ) or by running MONAI on an EC2 instance associated with an IAM Role granted for the following privileges: - -Replace the tags below in the below policy as follow : - -[AWS_Account] : The AWS Account ID ( numeric ) -[EnvironmentId] : The Environment Id use int Subscriber and Publisher settings of the Messaging seciton of `appsettings.json` / `appsettings.Development.json`. -[BucketName] : the name of the S3 bucket used to store the messages temporarily. The same bucket as the one used to store incoming DICOM objects can be used if desired. ( see the AWS S3 plugin to use S3 natively with MONAI Deploy ) - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "VisualEditor0", - "Effect": "Allow", - "Action": [ - "sqs:DeleteMessage", - "s3:PutObject", - "s3:GetObject", - "sqs:GetQueueUrl", - "sqs:ReceiveMessage", - "sqs:SendMessage", - "sqs:GetQueueAttributes", - "sqs:CreateQueue", - "s3:DeleteObject", - "sqs:SetQueueAttributes" - ], - "Resource": [ - "arn:aws:sqs:*:[AWS_Account]:[EnvironmentId]", - "arn:aws:s3:::[BucketName]/*" - ] - } - ] -} -``` - -## Contributing - -For guidance on making a contribution to MONAI Deploy Workflow Manager, see the [contributing guidelines](https://github.com/Project-MONAI/monai-deploy/blob/main/CONTRIBUTING.md). - -Join the conversation on Twitter [@ProjectMONAI](https://twitter.com/ProjectMONAI) or join our [Slack channel](https://forms.gle/QTxJq3hFictp31UM9). - -Ask and answer questions over on [MONAI Deploy Workflow Manager's GitHub Discussions tab](https://github.com/Project-MONAI/monai-deploy-workflow-manager/discussions). - -## Links - -- Website: -- Code: -- Project tracker: -- Issue tracker: -- Test status: From fad266c58d2f99e0b6222983770c18d27a6cefb5 Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 09:43:06 -0500 Subject: [PATCH 38/45] README.MD added back. Signed-off-by: JP LEGER --- src/Messaging/SQS/README.MD | 200 ++++++++++++++++++++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 src/Messaging/SQS/README.MD diff --git a/src/Messaging/SQS/README.MD b/src/Messaging/SQS/README.MD new file mode 100644 index 0000000..09d24f1 --- /dev/null +++ b/src/Messaging/SQS/README.MD @@ -0,0 +1,200 @@ +

+project-monai +

+ +💡 If you want to know more about MONAI Deploy WG vision, overall structure, and guidelines, please read [MONAI Deploy](https://github.com/Project-MONAI/monai-deploy) first. + +# MONAI Deploy Messaging SQS Plug-In +AWS SQS plugin for the messaging layer of MONAI Deploy. + +## Information: +This plugin can be used as a messaging mechanism alternative to RabbitMQ, and allows MONAI Deploy to integrate with AWS Simple Queue Service. Unlike RabbitMQ, SQS does not have a concept of vhost, exchange and routing key; instead the plugin will create one specific queue for each vhost, exchange and routing key combibation. + + + + + + + + + + + + + + + + + +
+ RMQ plugin + + SQS plugin + Description +
vhostenvironmentIdThe vhost namespace is replaced be the parameter environmentId. This parmater is used as a prefix for each queue name, allowing mutliple MONAI deploy installations on the same AWS account with not queue name conflict.
exchange and routing keyworkflowRequestQueue and exportRequestQueueThese parmaeters control the name of the queues for each purpose. Each queue name will be suffixed with the name routing key.
+ +* Queue names generated by the plugin have the following name convention : [environmentId]_[RequestQueue]_[routingKey]. +* Queue names will be less or equal to 80 characters long. The name will be truncated if the combinatino of environmentId, RequestQueue and routingKey exceeds this limit. +* Any non-alphanumerical character other than "-" and "\_" found in the environmentId, RequestQueue, parameters or routing key variables will be replaced by "\_". +* The queues are created automatically upon the 1st message publication/subscription if they do not already exist. +* Because MONAI Deploy can generate payloads greater than 256KBytes, the plugin leverages the SQS extended client, allowing payload up to 2GB in size. The use of this specific client mandates the usage of an S3 bucket as transient store for the messages to be held until their delivery to the queue subscriber. More information is available below in this documentation about IAM privileges, SQS and S3 bucket requirements. + + + +## Plugin activation + +Because MONAI Deploy plugin management work is still in progress (as of 06/13/2022), plugins cannot be loaded at run-time. Instead this SQS pluging can be activated by doing small code changes to the MONAI Informatic Gateway project https://github.com/Project-MONAI/monai-deploy-informatics-gateway, in the `Program.cs` file and recompiling. + +In the declaration for `CreateHostBuilder`, locate the following code block: + + services.UseRabbitMq(); + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.PublisherServiceAssemblyName); + }); + + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.SubscriberServiceAssemblyName); + }); + + + +and alter it by commenting the line `services.UseRabbitMq();`, replace `services.AddSingleton();` by `services.AddSingleton();` and the line `services.AddSingleton();` by `services.AddSingleton();`. The code block should now look like this : + + + //services.UseRabbitMq(); + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.PublisherServiceAssemblyName); + }); + + services.AddSingleton(); + services.AddSingleton(implementationFactory => + { + var options = implementationFactory.GetService>(); + var serviceProvider = implementationFactory.GetService(); + var logger = implementationFactory.GetService>(); + return serviceProvider.LocateService(logger, options.Value.Messaging.SubscriberServiceAssemblyName); + }); + +## MONAI Informatic Gateway Configuration + +The plugin is configured in the Messaging section of `appsettings.json` / `appsettings.Development.json` : + +```json + "messaging": { + "publisherServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SQSMessagePublisherService, Monai.Deploy.Messaging", + "subscriberServiceAssemblyName": "Monai.Deploy.Messaging.SQS.SqsMessageSubscriberService, Monai.Deploy.Messaging", + "publisherSettings": { + "bucketName": "monai-minio", + "workflowRequestQueue": "workflow_tasks", + "environmentId": "monai-1", + "accessKey": "ASDFGHJKLADF123456789", + "accessToken": "QwErTyUiOpAsDonMB88W1mcCCwQdePe8X27SEu1S" + }, + "subscriberSettings": { + "exportRequestQueue": "export_tasks", + "bucketName": "monai-minio", + "environmentId": "monai-1", + "accessKey": "ASDFGHJKLADF123456789", + "accessToken": "QwErTyUiOpAsDonMB88W1mcCCwQdePe8X27SEu1S" + } + }, +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
bucketNameS3 bucket used to store the messages temporarily until the subscriber gets it.
workflowRequestQueueQueue prefix for the workflow requests ( MIG -> Workflow Manager ). This parameter is only useful in the PublisherSettings section.
exportRequestQueueQueue prefix for the export requests ( Workflow Manager -> MIG ). This parameter is only useful in the SubscriberSettings section.
bucketNameS3 bucket used to store the messages temporarily until the subscriber gets it.
environmentIdA prefix used to identify the MONAI environment. This allows to create multiple environments within a single AWS account without queue name conflict. This parameter allows for a configuration comparable to the vhost concept in RabbitMq.
accessKeyAWS IAM user access key. This parameter is optional. If not present the plugin will fallback to local credentials, then EC2 role. If this parameter is used the parameter accessToken is also required. Refer to the section IAM privileges for IAM configuration.
accessTokenAWS IAM user access token. This parameter is only required when the parameter accesskey is provided.
+ +## IAM Privileges + +For the plugin to function a set of specific privileges need to be provided. The permissions can be associated either with an IAM user ( by using accessKey and accessToken in the configuratio file ) or by running MONAI on an EC2 instance associated with an IAM Role granted for the following privileges: + +Replace the tags below in the below policy as follow : + +[AWS_Account] : The AWS Account ID ( numeric ) +[EnvironmentId] : The Environment Id use int Subscriber and Publisher settings of the Messaging seciton of `appsettings.json` / `appsettings.Development.json`. +[BucketName] : the name of the S3 bucket used to store the messages temporarily. The same bucket as the one used to store incoming DICOM objects can be used if desired. ( see the AWS S3 plugin to use S3 natively with MONAI Deploy ) + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "sqs:DeleteMessage", + "s3:PutObject", + "s3:GetObject", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage", + "sqs:SendMessage", + "sqs:GetQueueAttributes", + "sqs:CreateQueue", + "s3:DeleteObject", + "sqs:SetQueueAttributes" + ], + "Resource": [ + "arn:aws:sqs:*:[AWS_Account]:[EnvironmentId]", + "arn:aws:s3:::[BucketName]/*" + ] + } + ] +} +``` + +## Contributing + +For guidance on making a contribution to MONAI Deploy Workflow Manager, see the [contributing guidelines](https://github.com/Project-MONAI/monai-deploy/blob/main/CONTRIBUTING.md). + +Join the conversation on Twitter [@ProjectMONAI](https://twitter.com/ProjectMONAI) or join our [Slack channel](https://forms.gle/QTxJq3hFictp31UM9). + +Ask and answer questions over on [MONAI Deploy Workflow Manager's GitHub Discussions tab](https://github.com/Project-MONAI/monai-deploy-workflow-manager/discussions). + +## Links + +- Website: +- Code: +- Project tracker: +- Issue tracker: +- Test status: From 4d930e81a649a09796f4f592e59f2a4f5535738f Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 10:07:22 -0500 Subject: [PATCH 39/45] Code smells corrections. Signed-off-by: JP LEGER --- src/Messaging/SQS/QueueFormatter.cs | 2 +- src/Messaging/SQS/SQSMessagePublisherService.cs | 16 ++++++++-------- src/Messaging/SQS/SQSMessageSubscriberService.cs | 13 +++++++------ 3 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/Messaging/SQS/QueueFormatter.cs b/src/Messaging/SQS/QueueFormatter.cs index 7c3181a..4c70821 100644 --- a/src/Messaging/SQS/QueueFormatter.cs +++ b/src/Messaging/SQS/QueueFormatter.cs @@ -17,7 +17,7 @@ public static string FormatQueueName(string environmentId, string? queuebasename string queue = $"{queuebasename}_{topic}"; - if (!environmentId.Equals(String.Empty)) + if (!string.IsNullOrEmpty(environmentId)) queue = $"{environmentId}_{queue}"; queue = Regex.Replace(queue, "[^a-zA-Z0-9_]", "-"); if (queue.Length > 80) diff --git a/src/Messaging/SQS/SQSMessagePublisherService.cs b/src/Messaging/SQS/SQSMessagePublisherService.cs index e6839a0..f695116 100644 --- a/src/Messaging/SQS/SQSMessagePublisherService.cs +++ b/src/Messaging/SQS/SQSMessagePublisherService.cs @@ -16,7 +16,6 @@ namespace Monai.Deploy.Messaging.SQS { public class SqsMessagePublisherService : IMessageBrokerPublisherService { - private const int PersistentDeliveryMode = 2; private readonly ILogger _logger; private readonly string? _accessKey; @@ -27,10 +26,9 @@ public class SqsMessagePublisherService : IMessageBrokerPublisherService public string Name => "AWS SQS Publisher"; private readonly string _queueName; - private readonly string _bucketName; - private readonly AmazonSQSClient? _sqsClient; - private readonly AmazonS3Client? _s3Client; - private readonly AmazonSQSExtendedClient? _sqSExtendedClient; + private readonly AmazonSQSClient _sqsClient; + private readonly AmazonS3Client _s3Client; + private readonly AmazonSQSExtendedClient _sqSExtendedClient; public SqsMessagePublisherService(IOptions options, ILogger logger) @@ -43,7 +41,7 @@ public SqsMessagePublisherService(IOptions op ValidateConfiguration(configuration); _queueName = configuration.PublisherSettings[SQSConfigurationKeys.WorkflowRequestQueue]; - _bucketName = configuration.PublisherSettings[SQSConfigurationKeys.BucketName]; + string bucketName = configuration.PublisherSettings[SQSConfigurationKeys.BucketName]; if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.AccessKey)) @@ -80,7 +78,7 @@ public SqsMessagePublisherService(IOptions op } _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, - new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, _bucketName)); + new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, bucketName)); @@ -190,7 +188,9 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - // Dispose any managed objects + _sqSExtendedClient.Dispose(); + _s3Client.Dispose(); + _sqsClient.Dispose(); } _disposedValue = true; diff --git a/src/Messaging/SQS/SQSMessageSubscriberService.cs b/src/Messaging/SQS/SQSMessageSubscriberService.cs index 1f273c5..1f40bde 100644 --- a/src/Messaging/SQS/SQSMessageSubscriberService.cs +++ b/src/Messaging/SQS/SQSMessageSubscriberService.cs @@ -23,11 +23,10 @@ public class SqsMessageSubscriberService : IMessageBrokerSubscriberService private readonly string? _accessKey; private readonly string? _accessToken; private readonly string? _queueName; - private readonly string? _bucketName; private readonly string _environmentId = string.Empty; - private readonly AmazonSQSClient? _sqsClient; - private readonly AmazonS3Client? _s3Client; + private readonly AmazonSQSClient _sqsClient; + private readonly AmazonS3Client _s3Client; private readonly AmazonSQSExtendedClient _sqSExtendedClient; public string Name => "AWS SQS Subscriber"; @@ -42,7 +41,7 @@ public SqsMessageSubscriberService(IOptions o var configuration = options.Value; ValidateConfiguration(configuration); _queueName = configuration.SubscriberSettings[SQSConfigurationKeys.ExportRequestQueue]; - _bucketName = configuration.SubscriberSettings[SQSConfigurationKeys.BucketName]; + string bucketName = configuration.SubscriberSettings[SQSConfigurationKeys.BucketName]; if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.AccessKey)) @@ -74,7 +73,7 @@ public SqsMessageSubscriberService(IOptions o } _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, - new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, _bucketName)); + new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, bucketName)); } catch (Amazon.SQS.AmazonSQSException Ex) @@ -203,7 +202,9 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - + _sqSExtendedClient.Dispose(); + _s3Client.Dispose(); + _sqsClient.Dispose(); } _disposedValue = true; } From fa737a0ec23ff09790930bf654def529168a6386 Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 10:52:54 -0500 Subject: [PATCH 40/45] Code smells corrections. Signed-off-by: JP LEGER --- src/Messaging/SQS/Log.cs | 8 ++-- .../SQS/SQSMessagePublisherService.cs | 39 ++++++++----------- .../SQS/SQSMessageSubscriberService.cs | 36 +++++++---------- 3 files changed, 35 insertions(+), 48 deletions(-) diff --git a/src/Messaging/SQS/Log.cs b/src/Messaging/SQS/Log.cs index 9d15df2..9a989c8 100644 --- a/src/Messaging/SQS/Log.cs +++ b/src/Messaging/SQS/Log.cs @@ -14,10 +14,10 @@ public static partial class Log public static partial void ConnectingToSQS(this ILogger logger, string serviceName); [LoggerMessage(EventId = 10002, Level = LogLevel.Information, Message = "Message received from queue {queue}.")] - public static partial void MessageReceivedFromQueue(this ILogger logger, string queue, string topic); + public static partial void MessageReceivedFromQueue(this ILogger logger, string queue); [LoggerMessage(EventId = 10003, Level = LogLevel.Information, Message = "Listening for messages from {endpoint}. Queue={queue}.")] - public static partial void SubscribeToSQSQueue(this ILogger logger, string endpoint, string virtualHost, string exchange, string queue, string topic); + public static partial void SubscribeToSQSQueue(this ILogger logger, string endpoint, string queue); [LoggerMessage(EventId = 10004, Level = LogLevel.Information, Message = "Sending message acknowledgement for message {messageId}.")] public static partial void SendingAcknowledgement(this ILogger logger, string messageId); @@ -35,10 +35,10 @@ public static partial class Log public static partial void ClosingConnections(this ILogger logger); [LoggerMessage(EventId = 10009, Level = LogLevel.Error, Message = "Invalid or corrupted message received: Queue={queueName}, Message ID={messageId}.")] - public static partial void InvalidMessage(this ILogger logger, string queueName, string topic, string messageId, Exception ex); + public static partial void InvalidMessage(this ILogger logger, string queueName, string messageId, Exception ex); [LoggerMessage(EventId = 10010, Level = LogLevel.Error, Message = "Exception not handled by the subscriber's callback function: Queue={queueName}, Message ID={messageId}.")] - public static partial void ErrorNotHandledByCallback(this ILogger logger, string queueName, string topic, string messageId, Exception ex); + public static partial void ErrorNotHandledByCallback(this ILogger logger, string queueName, string messageId, Exception ex); [LoggerMessage(EventId = 10011, Level = LogLevel.Error, Message = "Creating SQS client.")] public static partial void CreateSQSClient(this ILogger logger); diff --git a/src/Messaging/SQS/SQSMessagePublisherService.cs b/src/Messaging/SQS/SQSMessagePublisherService.cs index f695116..77ee3f6 100644 --- a/src/Messaging/SQS/SQSMessagePublisherService.cs +++ b/src/Messaging/SQS/SQSMessagePublisherService.cs @@ -60,33 +60,26 @@ public SqsMessagePublisherService(IOptions op if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.Envid)) _environmentId = configuration.PublisherSettings[SQSConfigurationKeys.Envid]; - try - { - _logger.ConnectingToSQS(Name); - - if (!(_accessKey is null) && !(_accessToken is null)) - { - _logger.LogInformation("Assuming IAM user as found in the configuration file."); - _sqsClient = new AmazonSQSClient(_accessKey, _accessToken); - _s3Client = new AmazonS3Client(_accessKey, _accessToken); - } - else - { - _logger.LogInformation("Attempting to assume local AWS credentials."); - _sqsClient = new AmazonSQSClient(); - _s3Client = new AmazonS3Client(); - } - - _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, - new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, bucketName)); - + _logger.ConnectingToSQS(Name); + if (!(_accessKey is null) && !(_accessToken is null)) + { + _logger.LogInformation("Assuming IAM user as found in the configuration file."); + _sqsClient = new AmazonSQSClient(_accessKey, _accessToken); + _s3Client = new AmazonS3Client(_accessKey, _accessToken); } - catch (Amazon.SQS.AmazonSQSException Ex) + else { - _logger.ConnectingToSQSError(Name, Ex); + _logger.LogInformation("Attempting to assume local AWS credentials."); + _sqsClient = new AmazonSQSClient(); + _s3Client = new AmazonS3Client(); } + + _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, + new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, bucketName)); + + } private void ValidateConfiguration(MessageBrokerServiceConfiguration configuration) @@ -170,7 +163,7 @@ public Task Publish(string topic, Monai.Deploy.Messaging.Messages.Message messag try { - SendMessageResponse sqsresp = _sqSExtendedClient.SendMessageAsync(sendMessageRequest).Result; + _sqSExtendedClient.SendMessageAsync(sendMessageRequest).Wait(); } catch (Exception e) { diff --git a/src/Messaging/SQS/SQSMessageSubscriberService.cs b/src/Messaging/SQS/SQSMessageSubscriberService.cs index 1f40bde..508c7cd 100644 --- a/src/Messaging/SQS/SQSMessageSubscriberService.cs +++ b/src/Messaging/SQS/SQSMessageSubscriberService.cs @@ -55,32 +55,26 @@ public SqsMessageSubscriberService(IOptions o if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.Envid)) _environmentId = configuration.SubscriberSettings[SQSConfigurationKeys.Envid]; - try - { - _logger.ConnectingToSQS(Name); - - if (!(_accessKey is null) && !(_accessToken is null)) - { - _logger.LogInformation("Assuming IAM user as found in the configuration file."); - _sqsClient = new AmazonSQSClient(_accessKey, _accessToken); - _s3Client = new AmazonS3Client(_accessKey, _accessToken); - } - else - { - _logger.LogInformation("Attempting to assume local AWS credentials."); - _sqsClient = new AmazonSQSClient(); - _s3Client = new AmazonS3Client(); - } - _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, - new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, bucketName)); + _logger.ConnectingToSQS(Name); + if (!(_accessKey is null) && !(_accessToken is null)) + { + _logger.LogInformation("Assuming IAM user as found in the configuration file."); + _sqsClient = new AmazonSQSClient(_accessKey, _accessToken); + _s3Client = new AmazonS3Client(_accessKey, _accessToken); } - catch (Amazon.SQS.AmazonSQSException Ex) + else { - _logger.ConnectingToSQSError(Name, Ex); - Guard.Against.Null(_sqSExtendedClient, nameof(_sqSExtendedClient)); + _logger.LogInformation("Attempting to assume local AWS credentials."); + _sqsClient = new AmazonSQSClient(); + _s3Client = new AmazonS3Client(); } + + _sqSExtendedClient = new AmazonSQSExtendedClient(_sqsClient, + new ExtendedClientConfiguration().WithLargePayloadSupportEnabled(_s3Client, bucketName)); + + } From 9fe3d1b56d54ea1f454d67b502034619c5683076 Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 11:00:14 -0500 Subject: [PATCH 41/45] Code smells corrections. Signed-off-by: JP LEGER --- src/Messaging/SQS/ConfigurationKeys.cs | 2 +- .../SQS/SQSMessagePublisherService.cs | 16 +-- .../SQS/SQSMessageSubscriberService.cs | 103 +++++++++--------- 3 files changed, 61 insertions(+), 60 deletions(-) diff --git a/src/Messaging/SQS/ConfigurationKeys.cs b/src/Messaging/SQS/ConfigurationKeys.cs index 54d0e11..e759b05 100644 --- a/src/Messaging/SQS/ConfigurationKeys.cs +++ b/src/Messaging/SQS/ConfigurationKeys.cs @@ -3,7 +3,7 @@ namespace Monai.Deploy.Messaging.Configuration { - internal static class SQSConfigurationKeys + internal static class SqsConfigurationKeys { public static readonly string AccessKey = "accessKey"; public static readonly string AccessToken = "accessToken"; diff --git a/src/Messaging/SQS/SQSMessagePublisherService.cs b/src/Messaging/SQS/SQSMessagePublisherService.cs index 77ee3f6..071431e 100644 --- a/src/Messaging/SQS/SQSMessagePublisherService.cs +++ b/src/Messaging/SQS/SQSMessagePublisherService.cs @@ -40,25 +40,25 @@ public SqsMessagePublisherService(IOptions op var configuration = options.Value; ValidateConfiguration(configuration); - _queueName = configuration.PublisherSettings[SQSConfigurationKeys.WorkflowRequestQueue]; - string bucketName = configuration.PublisherSettings[SQSConfigurationKeys.BucketName]; + _queueName = configuration.PublisherSettings[SqsConfigurationKeys.WorkflowRequestQueue]; + string bucketName = configuration.PublisherSettings[SqsConfigurationKeys.BucketName]; - if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.AccessKey)) + if (configuration.PublisherSettings.ContainsKey(SqsConfigurationKeys.AccessKey)) { _logger.LogInformation("accessKey found in configuration."); - _accessKey = configuration.PublisherSettings[SQSConfigurationKeys.AccessKey]; + _accessKey = configuration.PublisherSettings[SqsConfigurationKeys.AccessKey]; } - if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.AccessToken)) + if (configuration.PublisherSettings.ContainsKey(SqsConfigurationKeys.AccessToken)) { _logger.LogInformation("accessToken found in configuration."); - _accessToken = configuration.PublisherSettings[SQSConfigurationKeys.AccessToken]; + _accessToken = configuration.PublisherSettings[SqsConfigurationKeys.AccessToken]; } - if (configuration.PublisherSettings.ContainsKey(SQSConfigurationKeys.Envid)) - _environmentId = configuration.PublisherSettings[SQSConfigurationKeys.Envid]; + if (configuration.PublisherSettings.ContainsKey(SqsConfigurationKeys.Envid)) + _environmentId = configuration.PublisherSettings[SqsConfigurationKeys.Envid]; _logger.ConnectingToSQS(Name); diff --git a/src/Messaging/SQS/SQSMessageSubscriberService.cs b/src/Messaging/SQS/SQSMessageSubscriberService.cs index 508c7cd..6c81f9c 100644 --- a/src/Messaging/SQS/SQSMessageSubscriberService.cs +++ b/src/Messaging/SQS/SQSMessageSubscriberService.cs @@ -40,20 +40,20 @@ public SqsMessageSubscriberService(IOptions o var configuration = options.Value; ValidateConfiguration(configuration); - _queueName = configuration.SubscriberSettings[SQSConfigurationKeys.ExportRequestQueue]; - string bucketName = configuration.SubscriberSettings[SQSConfigurationKeys.BucketName]; + _queueName = configuration.SubscriberSettings[SqsConfigurationKeys.ExportRequestQueue]; + string bucketName = configuration.SubscriberSettings[SqsConfigurationKeys.BucketName]; - if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.AccessKey)) - _accessKey = configuration.SubscriberSettings[SQSConfigurationKeys.AccessKey]; + if (configuration.SubscriberSettings.ContainsKey(SqsConfigurationKeys.AccessKey)) + _accessKey = configuration.SubscriberSettings[SqsConfigurationKeys.AccessKey]; - if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.AccessToken)) - _accessToken = configuration.SubscriberSettings[SQSConfigurationKeys.AccessToken]; + if (configuration.SubscriberSettings.ContainsKey(SqsConfigurationKeys.AccessToken)) + _accessToken = configuration.SubscriberSettings[SqsConfigurationKeys.AccessToken]; - if (configuration.SubscriberSettings.ContainsKey(SQSConfigurationKeys.Envid)) - _environmentId = configuration.SubscriberSettings[SQSConfigurationKeys.Envid]; + if (configuration.SubscriberSettings.ContainsKey(SqsConfigurationKeys.Envid)) + _environmentId = configuration.SubscriberSettings[SqsConfigurationKeys.Envid]; _logger.ConnectingToSQS(Name); @@ -84,7 +84,7 @@ private void ValidateConfiguration(MessageBrokerServiceConfiguration configurati Guard.Against.Null(configuration, nameof(configuration)); Guard.Against.Null(configuration.SubscriberSettings, nameof(configuration.SubscriberSettings)); - foreach (var key in SQSConfigurationKeys.SubscriberRequiredKeys) + foreach (var key in SqsConfigurationKeys.SubscriberRequiredKeys) { if (!configuration.SubscriberSettings.ContainsKey(key)) { @@ -106,63 +106,64 @@ public void Subscribe(string[] topics, string queue, Action { + try + { + string queueName = QueueFormatter.FormatQueueName(_environmentId, _queueName, topic); + _logger.LogDebug($"Attempting to create or subscribe to {queueName}"); - string queueName = QueueFormatter.FormatQueueName(_environmentId, _queueName, topic); - _logger.LogDebug($"Attempting to create or subscribe to {queueName}"); + var queueAttributes = new Dictionary(); - var queueAttributes = new Dictionary(); + queueAttributes.Add("KmsMasterKeyId", "alias/aws/sqs"); + var request = new CreateQueueRequest + { + Attributes = queueAttributes, + QueueName = queueName + }; - queueAttributes.Add("KmsMasterKeyId", "alias/aws/sqs"); - var request = new CreateQueueRequest - { - Attributes = queueAttributes, - QueueName = queueName - }; + CreateQueueResponse createQueueResponse = new CreateQueueResponse(); - CreateQueueResponse createQueueResponse = new CreateQueueResponse(); - try - { createQueueResponse = _sqSExtendedClient.CreateQueueAsync(request).Result; - } - catch (Exception ex) - { - _logger.LogDebug($"The queue could not be created or subscribed to: {ex.Message}"); - } - while (true) - { - List AttributesList = new List(); - AttributesList.Add("*"); - var messageResponse = _sqSExtendedClient.ReceiveMessageAsync(new ReceiveMessageRequest - { - QueueUrl = createQueueResponse.QueueUrl, - WaitTimeSeconds = 2, - AttributeNames = new List { "All" }, - MessageAttributeNames = new List { "All" } - }).Result; - var messages = messageResponse.Messages; - - if (messages.Any()) + while (true) { - foreach (var msg in messages) + List AttributesList = new List(); + AttributesList.Add("*"); + + var messageResponse = _sqSExtendedClient.ReceiveMessageAsync(new ReceiveMessageRequest { - _logger.Log(LogLevel.Debug, $"Message {msg.MessageId} received from SQS."); - MessageReceivedEventArgs messageReceivedEventArgs = CreateMessage(msg); - try - { - _logger.AcknowledgementSent(msg.MessageId); - _sqSExtendedClient.DeleteMessageAsync(new DeleteMessageRequest { QueueUrl = createQueueResponse.QueueUrl, ReceiptHandle = msg.ReceiptHandle }).Wait(); - messageReceivedCallback(messageReceivedEventArgs); - } - catch (Exception ex) + QueueUrl = createQueueResponse.QueueUrl, + WaitTimeSeconds = 2, + AttributeNames = new List { "All" }, + MessageAttributeNames = new List { "All" } + }).Result; + var messages = messageResponse.Messages; + + if (messages.Any()) + { + foreach (var msg in messages) { - _logger.Log(LogLevel.Error, ex.Message); + _logger.Log(LogLevel.Debug, $"Message {msg.MessageId} received from SQS."); + MessageReceivedEventArgs messageReceivedEventArgs = CreateMessage(msg); + try + { + _logger.AcknowledgementSent(msg.MessageId); + _sqSExtendedClient.DeleteMessageAsync(new DeleteMessageRequest { QueueUrl = createQueueResponse.QueueUrl, ReceiptHandle = msg.ReceiptHandle }).Wait(); + messageReceivedCallback(messageReceivedEventArgs); + } + catch (Exception ex) + { + _logger.Log(LogLevel.Error, ex.Message); + } } } } } + catch (Exception ex) + { + _logger.LogDebug(ex.Message); + } }); } From 674029f0b7677f1c2cf37baa2d68807ed24451d6 Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 11:10:23 -0500 Subject: [PATCH 42/45] Code smells corrections. Signed-off-by: JP LEGER --- .../SQS/SQSMessageSubscriberService.cs | 97 +++++++++---------- 1 file changed, 48 insertions(+), 49 deletions(-) diff --git a/src/Messaging/SQS/SQSMessageSubscriberService.cs b/src/Messaging/SQS/SQSMessageSubscriberService.cs index 6c81f9c..1f1b79a 100644 --- a/src/Messaging/SQS/SQSMessageSubscriberService.cs +++ b/src/Messaging/SQS/SQSMessageSubscriberService.cs @@ -105,70 +105,69 @@ public void Subscribe(string[] topics, string queue, Action { + QueueRunner(topic); + }); + } - try - { - string queueName = QueueFormatter.FormatQueueName(_environmentId, _queueName, topic); - _logger.LogDebug($"Attempting to create or subscribe to {queueName}"); + } - var queueAttributes = new Dictionary(); - queueAttributes.Add("KmsMasterKeyId", "alias/aws/sqs"); - var request = new CreateQueueRequest - { - Attributes = queueAttributes, - QueueName = queueName - }; + private void QueueRunner(string topic) + { + try + { + string queueName = QueueFormatter.FormatQueueName(_environmentId, _queueName, topic); + _logger.LogDebug($"Attempting to create or subscribe to {queueName}"); - CreateQueueResponse createQueueResponse = new CreateQueueResponse(); + var queueAttributes = new Dictionary(); - createQueueResponse = _sqSExtendedClient.CreateQueueAsync(request).Result; + queueAttributes.Add("KmsMasterKeyId", "alias/aws/sqs"); + var request = new CreateQueueRequest + { + Attributes = queueAttributes, + QueueName = queueName + }; + CreateQueueResponse createQueueResponse = _sqSExtendedClient.CreateQueueAsync(request).Result; + while (true) + { + List AttributesList = new List(); + AttributesList.Add("*"); - while (true) + var messageResponse = _sqSExtendedClient.ReceiveMessageAsync(new ReceiveMessageRequest + { + QueueUrl = createQueueResponse.QueueUrl, + WaitTimeSeconds = 2, + AttributeNames = new List { "All" }, + MessageAttributeNames = new List { "All" } + }).Result; + var messages = messageResponse.Messages; + + if (messages.Any()) + { + foreach (var msg in messages) { - List AttributesList = new List(); - AttributesList.Add("*"); - - var messageResponse = _sqSExtendedClient.ReceiveMessageAsync(new ReceiveMessageRequest + _logger.Log(LogLevel.Debug, $"Message {msg.MessageId} received from SQS."); + MessageReceivedEventArgs messageReceivedEventArgs = CreateMessage(msg); + try { - QueueUrl = createQueueResponse.QueueUrl, - WaitTimeSeconds = 2, - AttributeNames = new List { "All" }, - MessageAttributeNames = new List { "All" } - }).Result; - var messages = messageResponse.Messages; - - if (messages.Any()) + _logger.AcknowledgementSent(msg.MessageId); + _sqSExtendedClient.DeleteMessageAsync(new DeleteMessageRequest { QueueUrl = createQueueResponse.QueueUrl, ReceiptHandle = msg.ReceiptHandle }).Wait(); + messageReceivedCallback(messageReceivedEventArgs); + } + catch (Exception ex) { - foreach (var msg in messages) - { - _logger.Log(LogLevel.Debug, $"Message {msg.MessageId} received from SQS."); - MessageReceivedEventArgs messageReceivedEventArgs = CreateMessage(msg); - try - { - _logger.AcknowledgementSent(msg.MessageId); - _sqSExtendedClient.DeleteMessageAsync(new DeleteMessageRequest { QueueUrl = createQueueResponse.QueueUrl, ReceiptHandle = msg.ReceiptHandle }).Wait(); - messageReceivedCallback(messageReceivedEventArgs); - } - catch (Exception ex) - { - _logger.Log(LogLevel.Error, ex.Message); - } - } + _logger.Log(LogLevel.Error, ex.Message); } } } - catch (Exception ex) - { - _logger.LogDebug(ex.Message); - } - }); + } + } + catch (Exception ex) + { + _logger.LogDebug(ex.Message); } - - - } public void SubscribeAsync(string topic, string queue, Func messageReceivedCallback, ushort prefetchCount = 0) From 47781e1fb276046816ce3b3785f48bf2ad700571 Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 11:13:30 -0500 Subject: [PATCH 43/45] Code smells corrections. Signed-off-by: JP LEGER --- src/Messaging/SQS/SQSMessageSubscriberService.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Messaging/SQS/SQSMessageSubscriberService.cs b/src/Messaging/SQS/SQSMessageSubscriberService.cs index 1f1b79a..7fad5b0 100644 --- a/src/Messaging/SQS/SQSMessageSubscriberService.cs +++ b/src/Messaging/SQS/SQSMessageSubscriberService.cs @@ -105,14 +105,14 @@ public void Subscribe(string[] topics, string queue, Action { - QueueRunner(topic); + QueueRunner(topic, messageReceivedCallback); }); } } - private void QueueRunner(string topic) + private void QueueRunner(string topic, Action messageReceivedCallback) { try { From 2e5ab53451b316ef6d9ad3845b7577334853ff26 Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 11:20:54 -0500 Subject: [PATCH 44/45] Code smells corrections. Signed-off-by: JP LEGER --- src/Messaging/SQS/SQSMessageSubscriberService.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Messaging/SQS/SQSMessageSubscriberService.cs b/src/Messaging/SQS/SQSMessageSubscriberService.cs index 7fad5b0..77d70dd 100644 --- a/src/Messaging/SQS/SQSMessageSubscriberService.cs +++ b/src/Messaging/SQS/SQSMessageSubscriberService.cs @@ -221,15 +221,22 @@ private static MessageReceivedEventArgs CreateMessage(Amazon.SQS.Model.Message m Guard.Against.Null(msg.MessageId, nameof(msg.MessageId)); JObject bodyobj = JObject.Parse(msg.Body); + string messageid = bodyobj["MessageId"].ToString(); + string contentype = msg.MessageAttributes["ContentType"].ToString(); + string correlationId = bodyobj["correlation_id"].ToString(); + + Guard.Against.Null(messageid, nameof(messageid)); + Guard.Against.Null(contentype, nameof(contentype)); + Guard.Against.Null(correlationId, nameof(correlationId)); return new MessageReceivedEventArgs( new Monai.Deploy.Messaging.Messages.Message( body: Encoding.UTF8.GetBytes(msg.Body), messageDescription: msg.MessageAttributes["ContentType"].ToString(), - messageId: bodyobj["MessageId"].ToString(), + messageId: messageid, applicationId: msg.Attributes["SenderId"], - contentType: msg.MessageAttributes["ContentType"].ToString(), - correlationId: bodyobj["correlation_id"].ToString(), + contentType: contentype, + correlationId: correlationId, creationDateTime: DateTimeOffset.FromUnixTimeMilliseconds(Int64.Parse(msg.Attributes["SentTimestamp"])), deliveryTag: msg.ReceiptHandle) , CancellationToken.None); From ed2f2b5ac97b402d0be8673a9b7151ba72681a29 Mon Sep 17 00:00:00 2001 From: JP LEGER Date: Tue, 14 Jun 2022 11:46:36 -0500 Subject: [PATCH 45/45] Code smells corrections. Signed-off-by: JP LEGER --- .../SQS/SQSMessageSubscriberService.cs | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/src/Messaging/SQS/SQSMessageSubscriberService.cs b/src/Messaging/SQS/SQSMessageSubscriberService.cs index 77d70dd..a20e0ed 100644 --- a/src/Messaging/SQS/SQSMessageSubscriberService.cs +++ b/src/Messaging/SQS/SQSMessageSubscriberService.cs @@ -221,23 +221,31 @@ private static MessageReceivedEventArgs CreateMessage(Amazon.SQS.Model.Message m Guard.Against.Null(msg.MessageId, nameof(msg.MessageId)); JObject bodyobj = JObject.Parse(msg.Body); - string messageid = bodyobj["MessageId"].ToString(); - string contentype = msg.MessageAttributes["ContentType"].ToString(); - string correlationId = bodyobj["correlation_id"].ToString(); + string messageId = string.Empty; + string correlationId = string.Empty; + + + JToken? messageIdtoken = bodyobj["MessageId"]; + if (messageIdtoken != null) + messageId = messageIdtoken.ToString(); + + JToken? correlationIdtoken = bodyobj["correlation_id"]; + if (correlationIdtoken != null) + correlationId = correlationIdtoken.ToString(); + + string contentType = msg.MessageAttributes["ContentType"].ToString(); + DateTimeOffset SentTimestamp = DateTimeOffset.FromUnixTimeMilliseconds(Int64.Parse(msg.Attributes["SentTimestamp"])); - Guard.Against.Null(messageid, nameof(messageid)); - Guard.Against.Null(contentype, nameof(contentype)); - Guard.Against.Null(correlationId, nameof(correlationId)); return new MessageReceivedEventArgs( new Monai.Deploy.Messaging.Messages.Message( body: Encoding.UTF8.GetBytes(msg.Body), - messageDescription: msg.MessageAttributes["ContentType"].ToString(), - messageId: messageid, + messageDescription: contentType, + messageId: messageId, applicationId: msg.Attributes["SenderId"], - contentType: contentype, + contentType: contentType, correlationId: correlationId, - creationDateTime: DateTimeOffset.FromUnixTimeMilliseconds(Int64.Parse(msg.Attributes["SentTimestamp"])), + creationDateTime: SentTimestamp, deliveryTag: msg.ReceiptHandle) , CancellationToken.None); }