diff --git a/.aspire/settings.json b/.aspire/settings.json new file mode 100644 index 0000000..5e504e9 --- /dev/null +++ b/.aspire/settings.json @@ -0,0 +1,3 @@ +{ + "appHostPath": "../src/Aspire/Aspire.AppHost/Aspire.AppHost.csproj" +} \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json index 59fffd2..698fded 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -5,10 +5,10 @@ "version": "0.2.0", "configurations": [ { - "name": "C#: Aspire.AppHost [Default Configuration]", - "type": "dotnet", + "type": "aspire", "request": "launch", - "projectPath": "${workspaceFolder}/src/Aspire/Aspire.AppHost/Aspire.AppHost.csproj" + "name": "Aspire: Launch AppHost", + "program": "" // Uses the default AppHost defined in .aspire/settings.json } ] } \ No newline at end of file diff --git a/Directory.Build.props b/Directory.Build.props index 802a1f7..78603a7 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -1,12 +1,4 @@ - - $(MSBuildThisFileDirectory)development.local.env - - - - - - net10.0 diff --git a/Directory.Packages.props b/Directory.Packages.props index cbe3af7..d4273b9 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -3,11 +3,11 @@ true - - - - - + + + + + @@ -22,7 +22,6 @@ - @@ -61,12 +60,14 @@ - - + + + + all diff --git a/README.md b/README.md index 379db16..e7c36fb 100644 --- a/README.md +++ b/README.md @@ -2,17 +2,39 @@ HMPPS CFO DMS ============= - +[![Run Tests](https://github.com/ministryofjustice/CFO-ExternalDataIntegrationSystem/actions/workflows/run-tests.yml/badge.svg)](https://github.com/ministryofjustice/CFO-ExternalDataIntegrationSystem/actions/workflows/run-tests.yml) +[![.NET](https://img.shields.io/badge/.NET-10.0-512BD4)](https://dotnet.microsoft.com/) +[![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) +[![Issues](https://img.shields.io/github/issues/ministryofjustice/CFO-ExternalDataIntegrationSystem)](https://github.com/ministryofjustice/CFO-ExternalDataIntegrationSystem/issues) +[![Pull Requests](https://img.shields.io/github/issues-pr/ministryofjustice/CFO-ExternalDataIntegrationSystem)](https://github.com/ministryofjustice/CFO-ExternalDataIntegrationSystem/pulls) +## Overview HMPPS Creating Future Opportunities (CFO) - Data Management System (DMS). It is intended for internal use only and is used to process PNOMIS and NDelius offender data to supply CATS (Case Assessment and Tracking System - also used by HMPPS CFO) with accurate offender movements and updates. -# Queries +## Architecture +CFO DMS is built as a microservices architecture using .NET Aspire for orchestration. Data flows through the following pipeline: + +**File Ingestion → Parsing/Cleaning → Staging → Import → Running Picture → Blocking/Matching → Clustering** + +1. **FileSync** monitors MinIO/S3/FileSystem storage and syncs incoming files +2. **Parsers/Cleaners** (Offloc, Delius) transform raw PNOMIS and NDelius files into structured records in staging databases +3. **Import** validates and migrates data from staging to running picture databases +4. **Matching Engine** identifies and links related offender records across systems +5. **Cluster database** maintains grouped offender data +6. **API** exposes the processed data via REST endpoints for downstream consumers (e.g., CATS) +7. **Visualiser** provides a web UI for exploring and visualising relationships between offender data -Any queries, please contact andrew.grocott@justice.gov.uk or visit our slack channel. https://app.slack.com/client/T02DYEB3A/C011Z8PGWCU/details/ +Supporting services include **DbInteractions** (complex database operations), **Blocking** (matching rules), **Cleanup** (data maintenance), and **Logging**. Services communicate asynchronously via RabbitMQ message queues. # Development Setup and Execution Guide +## Prerequisites +- [.NET 10 SDK](https://dotnet.microsoft.com/download/dotnet/10.0) +- **Visual Studio Code users**: + - [C# Dev Kit Extension](https://marketplace.visualstudio.com/items?itemName=ms-dotnettools.csdevkit) + - [Aspire Extension](https://marketplace.visualstudio.com/items?itemName=microsoft-aspire.aspire-vscode) + ## Setup (development) -1. To use the Visualiser app, you must configure secret(s) for applications in the *src* directory: +1. **(Optional)** To use the Visualiser app, you must configure secret(s) for applications in the *src* directory: - *Visualiser.csproj* → Manage User Secrets ```json { @@ -24,3 +46,250 @@ Any queries, please contact andrew.grocott@justice.gov.uk or visit our slack cha The recommended way to run and debug these apps is using .NET Aspire. - **Using Visual Studio Code**: open the project and press `F5`, selecting the *Default Configuration*. - **Using Visual Studio or other IDEs**: From the debug configuration dropdown, select `Aspire.AppHost` and start the application. + +### Services and Credentials +When running via Aspire, the following services are available: + +| Service | Purpose | Access | Credentials | +|---------|---------|--------|-------------| +| **API** | REST endpoints for querying offender data, searches, and clustering operations | https://localhost:7013/swagger | API Key: `password` | +| **MinIO** | S3-compatible file storage | *random port* (check Aspire) | Username: `minioadmin`
Password: `minioadmin` | +| **MSSQL** | Application databases (staging, running picture, matching, cluster) | `127.0.0.1,61749` | Username: `sa`
Password: `P@ssword123!` | +| **RabbitMQ** | Message broker for inter-service communication | http://localhost:15672 | Username: `guest`
Password: `guest` | + +## Message Flow Diagram + +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ INITIAL FILE DETECTION │ +└─────────────────────────────────────────────────────────────────────────────┘ + + ┌──────────────┐ + │ FileSync │ (Monitors storage for new files) + └──────┬───────┘ + │ + ├─────────────────────────────────────────┐ + │ │ + ▼ ▼ + DeliusDownloadFinishedMessage OfflocDownloadFinished + │ │ + │ │ + ▼ ▼ + +┌─────────────────────────────────────────────────────────────────────────────┐ +│ PARSING & CLEANING STAGE │ +└─────────────────────────────────────────────────────────────────────────────┘ + + ┌──────────────────┐ ┌─────────────────┐ + │ Delius.Parser │ │ Offloc.Cleaner │ + │ │ │ │ + │ (Parses Delius │ │ (Cleans Offloc │ + │ files into │ │ files, removes │ + │ structured │ │ redundant │ + │ records) │ │ fields) │ + └────────┬─────────┘ └────────┬────────┘ + │ │ + │ Sends DB requests: │ + │ - StartDeliusFileProcessingRequest │ + │ │ + ▼ ▼ + DeliusParserFinishedMessage OfflocCleanerFinishedMessage + │ │ + │ │ + │ ▼ + │ ┌─────────────────┐ + │ │ Offloc.Parser │ + │ │ │ + │ │ (Parses cleaned │ + │ │ Offloc files │ + │ │ into structured│ + │ │ records) │ + │ └────────┬────────┘ + │ │ + │ │ Sends DB requests: + │ │ - StartOfflocFileProcessingRequest + │ │ + │ ▼ + │ OfflocParserFinishedMessage + │ │ + └───────────────┬───────────────────────┘ + │ + ▼ + +┌─────────────────────────────────────────────────────────────────────────────┐ +│ STAGING & IMPORT STAGE │ +└─────────────────────────────────────────────────────────────────────────────┘ + + ┌────────────────┐ + │ Import │ + │ │ + │ (Coordinates │ + │ staging and │ + │ merging of │ + │ both data │ + │ sources) │ + └───────┬────────┘ + │ + │ Sends DB requests: + │ - StageDeliusRequest + │ - MergeDeliusRequest + │ - StageOfflocRequest + │ - MergeOfflocRequest + │ + ▼ + ┌────────────────┐ + │ DbInteractions │ + │ │ + │ (Stages data │ + │ from parsers, │ + │ merges into │ + │ running │ + │ picture DB) │ + └───────┬────────┘ + │ + │ Sends responses: + │ - StageDeliusResponse + │ - MergeDeliusResponse + │ - StageOfflocResponse + │ - MergeOfflocResponse + │ - DeliusFilesCleanupMessage + │ - OfflocFilesCleanupMessage + │ + ▼ + ImportFinishedMessage + │ + │ + ▼ + +┌─────────────────────────────────────────────────────────────────────────────┐ +│ MATCHING & BLOCKING STAGE │ +└─────────────────────────────────────────────────────────────────────────────┘ + + ┌────────────────┐ + │ Blocking │ + │ │ + │ (Generates │ + │ candidate │ + │ pairs of │ + │ records that │ + │ may match) │ + └───────┬────────┘ + │ + ▼ + BlockingFinishedMessage + │ + │ + ▼ + ┌───────────────────────────────┐ + │ Matching.Engine │ + │ (ComparatorService) │ + │ │ + │ (Compares candidate pairs │ + │ using matching rules to │ + │ identify potential matches) │ + └───────────────┬───────────────┘ + │ + ▼ + MatchingScoreCandidatesMessage + │ + │ + ▼ + ┌───────────────────────────────┐ + │ Matching.Engine │ + │ (ScorerService) │ + │ │ + │ (Scores comparisons using │ + │ Bayesian probability to │ + │ determine match likelihood) │ + └───────────────┬───────────────┘ + │ + ▼ + MatchingScoreCandidatesFinishedMessage + │ + │ + ▼ + +┌─────────────────────────────────────────────────────────────────────────────┐ +│ CLUSTERING STAGE │ +└─────────────────────────────────────────────────────────────────────────────┘ + + ┌───────────────────────────────┐ + │ Matching.Engine │ + │ (ClusteringService) │ + │ │ + │ (Pre-processes clustering: │ + │ prepares data for grouping) │ + └───────────────┬───────────────┘ + │ + ▼ + ClusteringPreProcessingStartedMessage + │ + │ + ▼ + ┌───────────────────────────────┐ + │ Matching.Engine │ + │ (ComparatorService) │ + │ │ + │ (Compares outstanding edges │ + │ for clustering) │ + └───────────────┬───────────────┘ + │ + ▼ + MatchingScoreOutstandingEdgesMessage + │ + │ + ▼ + ┌───────────────────────────────┐ + │ Matching.Engine │ + │ (ScorerService) │ + │ │ + │ (Scores outstanding edges) │ + └───────────────┬───────────────┘ + │ + ▼ + ClusteringPreProcessingFinishedMessage + │ + │ + ▼ + ┌───────────────────────────────┐ + │ Matching.Engine │ + │ (ClusteringService) │ + │ │ + │ (Post-processes clustering: │ + │ groups related records into │ + │ clusters representing │ + │ unique individuals) │ + └───────────────┬───────────────┘ + │ + ▼ + ClusteringPostProcessingFinishedMessage + │ + │ + ▼ + ┌────────────────┐ + │ FileSync │ + │ │ + │ (Triggers next │ + │ processing │ + │ cycle if │ + │ configured) │ + └────────────────┘ + +┌─────────────────────────────────────────────────────────────────────────────┐ +│ DATA CONSUMPTION │ +└─────────────────────────────────────────────────────────────────────────────┘ + + ┌────────────────┐ + │ API │ + │ │ + │ (Exposes REST │ + │ endpoints for │ + │ querying │ + │ processed │ + │ data) │ + └───────┬────────┘ + │ + ▼ + External Consumers + (e.g., CATS system) +``` \ No newline at end of file diff --git a/development.local.env b/development.local.env deleted file mode 100644 index a639dd8..0000000 --- a/development.local.env +++ /dev/null @@ -1,54 +0,0 @@ -DOTNET_ENVIRONMENT=Development - -RedundantOfflocFields= Age, Occupation Description, Check Hold Governor, Check Hold General (to be left blank), Check Hold Discipline (to be left blank), Check Hold Allocation, Check Hold Security, Check Hold Medical, Check Hold Parole, ACCT Status (F2052), Status Rank (to be left blank), Pending Transfers (Full Establishment Name), Received From, Vulnerable Prisoner Alert, Height (metres), Complexion, Hair Colour, Left Eye, Right Eye, Build, Facial Shape, Facial Hair, Physical Mark Head, Physical Mark Body, Rule 45/YOI Rule 49, ACCT (Self Harm) Status, ACCT (Self Harm) Start Date, Remark Type Allocation, Remarks Allocation, Remark Type Security, Remarks Security, Remark Type Medical, Remarks Medical, Remark Type Parole, Remarks Parole, Remark Type Discipline, Remarks Discipline, Remark Type General, Remarks General, Remark Type Reception, Remarks Reception, Remark Type Labour, Remarks Labour, Date Of First Movement, Diary Details - -# Rabbit -RABBIT_HOSTING=localhost -RABBIT_USERNAME=dms -RABBIT_PASSWORD=R@bbit - -# Connection Strings - uncomment in production -#USER_ID=sa -#SQL_HOST=localhost,1433 -#SA_PASSWORD=Pass@word -#ConnectionStrings__ClusterDb=Data Source=${SQL_HOST};Initial Catalog=ClusterDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true -#ConnectionStrings__DeliusRunningPictureDb=Data Source=${SQL_HOST};Initial Catalog=DeliusRunningPictureDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true -#ConnectionStrings__DeliusStagingDb=Data Source=${SQL_HOST};Initial Catalog=DeliusStagingDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true -#ConnectionStrings__MatchingDb=Data Source=${SQL_HOST};Initial Catalog=MatchingDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true -#ConnectionStrings__OfflocRunningPictureDb=Data Source=${SQL_HOST};Initial Catalog=OfflocRunningPictureDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true -#ConnectionStrings__OfflocStagingDb=Data Source=${SQL_HOST};Initial Catalog=OfflocStagingDb;User Id=${USER_ID};Password=${SA_PASSWORD};TrustServerCertificate=true - -# Sentry SDK -Sentry_Dsn=https:// -Sentry_Environment=${DOTNET_ENVIRONMENT} - -# Serilog -Serilog__MinimumLevel__Default=Information -Serilog__MinimumLevel__Override__Microsoft=Information -Serilog__MinimumLevel__Override__System=Warning - -# Serilog: Console Sink -Serilog__WriteTo__0__Name=Console - -# Serilog: File Sink -Serilog__WriteTo__1__Name=File -Serilog__WriteTo__1__Args__path=.\logs\log-.txt #Overrides the default appsettings.json value -Serilog__WriteTo__1__Args__rollingInterval=Day - -# Serilog: Sentry Sink -#Serilog__WriteTo__2__Name=Sentry -#Serilog__WriteTo__2__Args__Dsn=${Sentry_Dsn} -#Serilog__WriteTo__2__Args__Environment=${Sentry_Environment} -#Serilog__WriteTo__2__Args__MinimumEventLevel=Information -#Serilog__WriteTo__2__Args__MinimumBreadcrumbLevel=Information -#Serilog__WriteTo__2__Args__AttachStacktrace=true -#Serilog__WriteTo__2__Args__SendDefaultPii=true - -# Serilog: Enrichers -Serilog__Enrich__0=FromLogContext -Serilog__Enrich__1=WithMachineName -Serilog__Enrich__2=WithProcessId -Serilog__Enrich__3=WithThreadId - -# e.g. /app/ -#DMSFilesBasePath=~/DMS/ \ No newline at end of file diff --git a/dms.sln b/dms.sln index 3eeeeb9..8e1d812 100644 --- a/dms.sln +++ b/dms.sln @@ -1,5 +1,4 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 +Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 18 VisualStudioVersion = 18.0.11116.177 MinimumVisualStudioVersion = 15.0.26124.0 @@ -38,9 +37,6 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{D3DAF78A-03D7-42EC-8B73-4B4622E61EF8}" ProjectSection(SolutionItems) = preProject .gitignore = .gitignore - base.development.env = base.development.env - development.docker.env = development.docker.env - development.local.env = development.local.env Directory.Build.props = Directory.Build.props Directory.Packages.props = Directory.Packages.props NuGet.config = NuGet.config @@ -100,6 +96,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Offloc.Cleaner.Tests", "tes EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Delius.Parser.Tests", "tests\Delius.Parser.Tests\Delius.Parser.Tests.csproj", "{841B9B78-5596-46EF-B38D-0E1C18321CDF}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Messaging.Tests", "tests\Messaging.Tests\Messaging.Tests.csproj", "{16431464-ACDD-4DE0-8AFF-D8926CF8C14E}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -548,6 +546,18 @@ Global {841B9B78-5596-46EF-B38D-0E1C18321CDF}.Release|x64.Build.0 = Release|Any CPU {841B9B78-5596-46EF-B38D-0E1C18321CDF}.Release|x86.ActiveCfg = Release|Any CPU {841B9B78-5596-46EF-B38D-0E1C18321CDF}.Release|x86.Build.0 = Release|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Debug|Any CPU.Build.0 = Debug|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Debug|x64.ActiveCfg = Debug|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Debug|x64.Build.0 = Debug|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Debug|x86.ActiveCfg = Debug|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Debug|x86.Build.0 = Debug|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Release|Any CPU.ActiveCfg = Release|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Release|Any CPU.Build.0 = Release|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Release|x64.ActiveCfg = Release|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Release|x64.Build.0 = Release|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Release|x86.ActiveCfg = Release|Any CPU + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -593,6 +603,7 @@ Global {10A645A2-039D-4906-BCB7-DBE12B0EF34D} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {199066EA-192B-4126-A1BC-85900CDCA1DD} = {0AB3BF05-4346-4AA6-1389-037BE0695223} {841B9B78-5596-46EF-B38D-0E1C18321CDF} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + {16431464-ACDD-4DE0-8AFF-D8926CF8C14E} = {0AB3BF05-4346-4AA6-1389-037BE0695223} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {39E62751-A441-4DEF-AD24-20CDC6D59FF3} diff --git a/src/API/API.csproj b/src/API/API.csproj index 0f6eb6c..84aa7f8 100644 --- a/src/API/API.csproj +++ b/src/API/API.csproj @@ -16,6 +16,10 @@ + + + + diff --git a/src/API/Program.cs b/src/API/Program.cs index 90163e0..6b293f9 100644 --- a/src/API/Program.cs +++ b/src/API/Program.cs @@ -4,15 +4,13 @@ using Infrastructure.Middlewares; using Microsoft.AspNetCore.Authentication.JwtBearer; using Microsoft.Identity.Web; -using Namotion.Reflection; using OpenApiContact = NSwag.OpenApiContact; using OpenApiInfo = NSwag.OpenApiInfo; using OpenApiSecurityScheme = NSwag.OpenApiSecurityScheme; - var builder = WebApplication.CreateBuilder(args); -IConfiguration configuration = builder.Configuration; +builder.UseDmsSerilog(); // Add service defaults & Aspire components. builder.AddServiceDefaults(); @@ -96,7 +94,7 @@ // Configure the HTTP request pipeline. app.UseExceptionHandler(); -if (configuration["IsDevelopment"] is not null && configuration.GetValue("IsDevelopment")) +if (builder.Configuration["IsDevelopment"] is not null && builder.Configuration.GetValue("IsDevelopment")) { app.RegisterDevelopmentEndpoints(); diff --git a/src/API/appsettings.Development.json b/src/API/appsettings.Development.json index f972c41..c091d62 100644 --- a/src/API/appsettings.Development.json +++ b/src/API/appsettings.Development.json @@ -5,7 +5,11 @@ "Microsoft.AspNetCore": "Warning" } }, + "Auhentication": { + "ApiKey": "password" + }, "AzureAd": { "ClientId": "916ace49-a3db-4b11-84c5-6c4bd20260ef" - } + }, + "IsDevelopment": "True" } diff --git a/src/API/appsettings.json b/src/API/appsettings.json index 2c2d4ce..ce86b1d 100644 --- a/src/API/appsettings.json +++ b/src/API/appsettings.json @@ -1,60 +1,42 @@ { - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, - "AllowedHosts": "*", - "Authentication": { - "ApiKey": "3A27A442D05942689AF3800E0FE8B242" - }, - "AzureAd": { - "TenantId": "c6874728-71e6-41fe-a9e1-2e8c36776ad8", - "Instance": "https://login.microsoftonline.com/", - "ClientId": "", - "Scopes": { - "Read": "dms.read", - "Write": "dms.write" - } - }, "ConnectionStrings": { - "AuditDb": "Data Source=localhost;Initial Catalog=AuditDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "ClusterDb": "Data Source=localhost;Initial Catalog=ClusterDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "DeliusRunningPictureDb": "Data Source=localhost;Initial Catalog=DeliusRunningPictureDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "OfflocRunningPictureDb": "Data Source=localhost;Initial Catalog=OfflocRunningPictureDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true" + "AuditDb": "", + "DeliusRunningPictureDb": "", + "OfflocRunningPictureDb": "", + "ClusterDb": "" }, "Serilog": { - "Using": [ - "Serilog.Sinks.Console" - ], "MinimumLevel": { - "Default": "Debug", + "Default": "Information", "Override": { + "Microsoft.AspNetCore": "Warning", "Microsoft": "Information", - "System": "Information" + "System": "Warning" } }, "WriteTo": [ - { - "Name": "Console" - }, - { + { "Name": "Console" }, + { "Name": "File", "Args": { - "path": "../../logs/log-.txt", - "rollingInterval": "Day", - "restrictedToMinimumLevel": "Debug" + "path": "../logs/API-.txt", + "rollingInterval": "Day" } } ], - "Enrich": [ - "FromLogContext", - "WithMachineName", - "WithThreadId" - ], - "Properties": { - "Application": "DMS" + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, + "AllowedHosts": "*", + "Authentication": { + "ApiKey": "" + }, + "AzureAd": { + "TenantId": "c6874728-71e6-41fe-a9e1-2e8c36776ad8", + "Instance": "https://login.microsoftonline.com/", + "ClientId": "", + "Scopes": { + "Read": "dms.read", + "Write": "dms.write" } }, "IsDevelopment": "False" diff --git a/src/Aspire/Aspire.AppHost/Extensions/AppExtensions.cs b/src/Aspire/Aspire.AppHost/Extensions/AppExtensions.cs index c05217e..eea2791 100644 --- a/src/Aspire/Aspire.AppHost/Extensions/AppExtensions.cs +++ b/src/Aspire/Aspire.AppHost/Extensions/AppExtensions.cs @@ -10,7 +10,7 @@ public static IResourceBuilder AddDmsApi( IResourceBuilder apiKey, IResourceBuilder isDevelopment) { - return builder.AddProject("api") + return builder.AddProject("Api") .WithDmsDatabaseReference(databases.OfflocRunningPicture) .WithDmsDatabaseReference(databases.DeliusRunningPicture) .WithDmsDatabaseReference(databases.Cluster) @@ -23,7 +23,7 @@ public static IResourceBuilder AddDmsVisualiser( this IDistributedApplicationBuilder builder, IResourceBuilder apiService) { - return builder.AddProject("visualiser") + return builder.AddProject("Visualiser") .WithReference(apiService).WaitFor(apiService); } @@ -48,10 +48,9 @@ public static IDistributedApplicationBuilder AddDmsServices( builder.AddDmsService("Meow", rabbit, databases, hostMount).WithExplicitStart(); builder.AddDmsService("Offloc-Cleaner", rabbit, databases, hostMount); builder.AddDmsService("Offloc-Parser", rabbit, databases, hostMount); - + builder.AddDmsService("FileSync", rabbit, databases, hostMount) .WithReference(minio).WaitFor(minio) - .WithExplicitStart() .WithEnvironment("MinIO:BucketName", "cfo-dms-files"); return builder; diff --git a/src/Aspire/Aspire.AppHost/Extensions/DatabaseExtensions.cs b/src/Aspire/Aspire.AppHost/Extensions/DatabaseExtensions.cs index c8d3aac..fea3a1a 100644 --- a/src/Aspire/Aspire.AppHost/Extensions/DatabaseExtensions.cs +++ b/src/Aspire/Aspire.AppHost/Extensions/DatabaseExtensions.cs @@ -8,8 +8,6 @@ public static IResourceBuilder AddDmsSqlServer( this IDistributedApplicationBuilder builder, IResourceBuilder password) { - var home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); - #pragma warning disable ASPIREPROXYENDPOINTS001 return builder.AddSqlServer("sql", password, 61749) .WithDataVolume("dms-data") @@ -79,9 +77,7 @@ public static DmsDatabaseResources AddDmsDatabases( { builder.AddProject("FakeDataSeeder") .WithReference(cluster) - .WaitForCompletion(clusterSqlProj) - .WaitForCompletion(offlocRunningPictureSqlProj) - .WaitForCompletion(deliusRunningPictureSqlProj); + .WaitForCompletion(clusterSqlProj); } return new DmsDatabaseResources( diff --git a/src/Aspire/Aspire.AppHost/Program.cs b/src/Aspire/Aspire.AppHost/Program.cs index f2bb85f..dc69670 100644 --- a/src/Aspire/Aspire.AppHost/Program.cs +++ b/src/Aspire/Aspire.AppHost/Program.cs @@ -1,4 +1,5 @@ using Aspire.AppHost.Extensions; +using Microsoft.Extensions.Configuration; var builder = DistributedApplication.CreateBuilder(args); @@ -8,6 +9,8 @@ var rabbitPassword = builder.AddParameter("rabbitPassword", true); var sqlPassword = builder.AddParameter("sqlPassword", true); var minioPassword = builder.AddParameter("minioPassword", true); +var seedData = builder.Configuration.GetValue("Parameters:seedData"); +var startCoreServices = builder.Configuration.GetValue("Parameters:startCoreServices"); var hostMount = HostExtensions.Create(Path.Combine(builder.AppHostDirectory, "DMS_STAGING")); var targetMount = "/app/"; @@ -16,7 +19,7 @@ var sql = builder.AddDmsSqlServer(sqlPassword) .WithBindMount(hostMount, targetMount); -var databases = builder.AddDmsDatabases(sql, seedData: false); +var databases = builder.AddDmsDatabases(sql, seedData); // API setup var apiService = builder.AddDmsApi(databases, apiKey, isDevelopment); @@ -24,19 +27,23 @@ // Visualiser setup builder.AddDmsVisualiser(apiService); -var rabbit = builder +if (startCoreServices) +{ + var rabbit = builder .AddRabbitMQ("RabbitMQ", password: rabbitPassword) - .WithManagementPlugin(); + .WithManagementPlugin(port: 15672); -// MinIO (s3 emulation) -var minio = builder.AddMinioContainer("minio", rootPassword: minioPassword) + // MinIO (s3 emulation) + var minio = builder + .AddMinioContainer("minio", rootPassword: minioPassword) .WithDataVolume("dms-minio-data"); -builder.AddDmsServices( - minio, - rabbit, - databases, - hostMount, - targetMount); + builder.AddDmsServices( + minio, + rabbit, + databases, + hostMount, + targetMount); +} builder.Build().Run(); \ No newline at end of file diff --git a/src/Aspire/Aspire.AppHost/appsettings.json b/src/Aspire/Aspire.AppHost/appsettings.json index 168b646..a72dd1b 100644 --- a/src/Aspire/Aspire.AppHost/appsettings.json +++ b/src/Aspire/Aspire.AppHost/appsettings.json @@ -8,9 +8,11 @@ }, "Parameters": { "apiKey": "password", - "isDevelopment": "True", + "isDevelopment": true, "rabbitPassword": "guest", + "seedData": true, "sqlPassword": "P@ssword123!", - "minioPassword": "minioadmin" + "minioPassword": "minioadmin", + "startCoreServices": true } } diff --git a/src/Blocking/Blocking.csproj b/src/Blocking/Blocking.csproj index 3ed3060..8c7a711 100644 --- a/src/Blocking/Blocking.csproj +++ b/src/Blocking/Blocking.csproj @@ -4,23 +4,10 @@ Exe enable enable - Linux - ..\..\.. - - - - - - - - - - - - + diff --git a/src/Blocking/BlockingBackgroundService.cs b/src/Blocking/BlockingBackgroundService.cs index 923bcc8..3f82089 100644 --- a/src/Blocking/BlockingBackgroundService.cs +++ b/src/Blocking/BlockingBackgroundService.cs @@ -7,37 +7,22 @@ namespace Blocking; -public class BlockingBackgroundService : BackgroundService +public class BlockingBackgroundService( + IMessageService messageService, + DatabaseInsert database) : BackgroundService { - private readonly IStatusMessagingService statusMessagingService; - private readonly IImportMessagingService importMessageService; - private readonly IBlockingMessagingService blockingMessageService; - private readonly DatabaseInsert matchingDbInsert; - - public BlockingBackgroundService(IStatusMessagingService statusMessagingService, IImportMessagingService importMessageService, - DatabaseInsert matchingDbInsert, IBlockingMessagingService blockingMessageService) - { - this.statusMessagingService = statusMessagingService; - this.matchingDbInsert = matchingDbInsert; - this.importMessageService = importMessageService; - this.blockingMessageService = blockingMessageService; - } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await Task.Run(() => + await messageService.SubscribeAsync(async (message) => { - importMessageService.ImportSubscribeAsync(async (message) => - { - await statusMessagingService.StatusPublishAsync(new StatusUpdateMessage("Blocking candidates...")); - await CallBlocking(); - }, TImportQueue.ImportFinished); - }, stoppingToken); + await messageService.PublishAsync(new StatusUpdateMessage("Blocking candidates...")); + await CallBlocking(); + }, TImportQueue.ImportFinished); } private async Task CallBlocking() { - await matchingDbInsert.InsertCandidates(); - await blockingMessageService.BlockingPublishAsync(new BlockingFinishedMessage()); + await database.InsertCandidates(); + await messageService.PublishAsync(new BlockingFinishedMessage()); } } diff --git a/src/Blocking/ConfigurationModels/BlockingConfigurationDefaults.cs b/src/Blocking/ConfigurationModels/BlockingConfigurationDefaults.cs deleted file mode 100644 index 7563656..0000000 --- a/src/Blocking/ConfigurationModels/BlockingConfigurationDefaults.cs +++ /dev/null @@ -1,17 +0,0 @@ - -namespace Blocking.ConfigurationModels; - -public class BlockingConfigurationDefaults -{ - public DatabaseTable offlocTable = new DatabaseTable - { - DatabaseName = "OfflocRunningPictureDb", - TableName = "[OfflocRunningPicture].[RecordMatchingView]" - }; - - public DatabaseTable deliusTable = new DatabaseTable - { - DatabaseName = "DeliusRunningPictureDb", - TableName = "[DeliusRunningPicture].[RecordMatchingView]" - }; -} diff --git a/src/Blocking/ConfigurationModels/BlockingField.cs b/src/Blocking/ConfigurationModels/BlockingField.cs deleted file mode 100644 index c50119c..0000000 --- a/src/Blocking/ConfigurationModels/BlockingField.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Blocking.ConfigurationModels.Enums; - -namespace Blocking.ConfigurationModels; - -public class BlockingField -{ - public TFieldType FieldType { get; init; } - public TableField OfflocField { get; init; } = new(); - public TableField DeliusField { get; init; } = new(); -} diff --git a/src/Blocking/ConfigurationModels/BlockingFieldGroup.cs b/src/Blocking/ConfigurationModels/BlockingFieldGroup.cs deleted file mode 100644 index af9b4b8..0000000 --- a/src/Blocking/ConfigurationModels/BlockingFieldGroup.cs +++ /dev/null @@ -1,19 +0,0 @@ - -namespace Blocking.ConfigurationModels; - -//Necessary to be able to match between different tables. -public class BlockingFieldsGroup -{ - public BlockingField[] BlockingFields { get; init; } = Array.Empty(); - //These could also be views - public DatabaseTable OfflocTable { get; init; } = new DatabaseTable //Default. - { - DatabaseName = "OfflocRunningPictureDb", - TableName = "[OfflocRunningPicture].[RecordMatchingView]" - }; - public DatabaseTable DeliusTable { get; init; } = new DatabaseTable //Default. - { - DatabaseName = "DeliusRunningPictureDb", - TableName = "[DeliusRunningPicture].[RecordMatchingView]" - }; -} diff --git a/src/Blocking/ConfigurationModels/BlockingFieldsConfig.cs b/src/Blocking/ConfigurationModels/BlockingFieldsConfig.cs deleted file mode 100644 index 740b4fe..0000000 --- a/src/Blocking/ConfigurationModels/BlockingFieldsConfig.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace Blocking.ConfigurationModels; - -//For making configuration type-safe. -public class BlockingFieldsConfig -{ - //Blocking fields in the same inner array will be blocked on together (they will all need to match to be considered a candidate). - public BlockingFieldsGroup[] BlockingFieldsGroups { get; init; } = []; -} diff --git a/src/Blocking/ConfigurationModels/CleaningConfig.cs b/src/Blocking/ConfigurationModels/CleaningConfig.cs deleted file mode 100644 index 65903fa..0000000 --- a/src/Blocking/ConfigurationModels/CleaningConfig.cs +++ /dev/null @@ -1,9 +0,0 @@ -using Blocking.ConfigurationModels.Enums; - -namespace Blocking.ConfigurationModels; - -public class CleaningConfiguration -{ - public TCleaningType CleaningType { get; init; } - public int? TrimAmount { get; init; } -} diff --git a/src/Blocking/ConfigurationModels/DatabaseTable.cs b/src/Blocking/ConfigurationModels/DatabaseTable.cs deleted file mode 100644 index 082003e..0000000 --- a/src/Blocking/ConfigurationModels/DatabaseTable.cs +++ /dev/null @@ -1,34 +0,0 @@ - -namespace Blocking.ConfigurationModels; - -public class DatabaseTable -{ - public string DatabaseName { get; init; } = string.Empty; - public string TableName { get ; init; } = string.Empty; - public string? Columns { get; init; } - - public string FormulateExpression => $"{DatabaseName}.{TableName}"; - - public override bool Equals(object? obj) - { - DatabaseTable castObj; - try - { - castObj = (DatabaseTable)obj; - } - catch (InvalidCastException ex) - { - return false; - } - - if(obj == null) - { - return false; - } - else if(this.FormulateExpression == castObj.FormulateExpression && this.Columns == castObj.Columns) - { - return true; - } - return false; - } -} diff --git a/src/Blocking/ConfigurationModels/Enums/TCleaningType.cs b/src/Blocking/ConfigurationModels/Enums/TCleaningType.cs deleted file mode 100644 index 7c8c56c..0000000 --- a/src/Blocking/ConfigurationModels/Enums/TCleaningType.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Blocking.ConfigurationModels.Enums; - -public enum TCleaningType -{ - Normalization, - TrimStart, - TrimEnd, - PNC -} diff --git a/src/Blocking/ConfigurationModels/Enums/TFieldType.cs b/src/Blocking/ConfigurationModels/Enums/TFieldType.cs deleted file mode 100644 index 1ecf88e..0000000 --- a/src/Blocking/ConfigurationModels/Enums/TFieldType.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Blocking.ConfigurationModels.Enums; - -public enum TFieldType -{ - String, - Integer, - SpecialIdentifier, //Idenifier with a /. Either CRO or PNC - Date -} diff --git a/src/Blocking/ConfigurationModels/TableField.cs b/src/Blocking/ConfigurationModels/TableField.cs deleted file mode 100644 index 6794553..0000000 --- a/src/Blocking/ConfigurationModels/TableField.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace Blocking.ConfigurationModels; - -public class TableField -{ - public string FieldName { get; set; } = string.Empty; - public CleaningConfiguration? CleaningConfig { get; init; } -} diff --git a/src/Blocking/DatabaseInsert.cs b/src/Blocking/DatabaseInsert.cs index 1d2c184..65d6281 100644 --- a/src/Blocking/DatabaseInsert.cs +++ b/src/Blocking/DatabaseInsert.cs @@ -1,42 +1,29 @@ - using Blocking.ConfigurationModels; using Microsoft.Data.SqlClient; -using System.ComponentModel.DataAnnotations; using System.Data; -using System.Data.SqlTypes; -using System.Runtime.InteropServices.JavaScript; using Newtonsoft.Json; -using System.Linq.Expressions; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; namespace Blocking; -public class DatabaseInsert +public class DatabaseInsert( + IConfiguration configuration, + IOptions storedProceduresOptions, + IOptions blockingQueriesOptions) { - private readonly string connString; - - private readonly string truncateProcedure; - private readonly string insertCandidatesProcedure; - private readonly BlockingQueriesConfig blockingQueriesConfig; - - public DatabaseInsert(ServerConfiguration config) - { - connString = config.connectionString; - - (truncateProcedure, insertCandidatesProcedure, blockingQueriesConfig) = - (config.storedProceduresConfig.TruncateStoredProcedure, - config.storedProceduresConfig.InsertCandidatesStoredProcedure, - config.blockingQueriesConfig); - } - public async Task ClearTables() { - SqlConnection conn = new SqlConnection(connString); + var connectionString = configuration.GetConnectionString("MatchingDb") + ?? throw new InvalidOperationException("Connection string 'MatchingDb' is not configured."); + + SqlConnection conn = new SqlConnection(connectionString); await conn.OpenAsync(); using (conn) { - SqlCommand command = new SqlCommand(truncateProcedure, conn); + SqlCommand command = new SqlCommand(storedProceduresOptions.Value.TruncateStoredProcedure, conn); command.CommandTimeout = 120; command.CommandType = CommandType.StoredProcedure; @@ -47,28 +34,32 @@ public async Task ClearTables() catch (SqlException ex) { Console.WriteLine(ex.ToString()); - throw ex; + throw; } } } public async Task InsertCandidates() { - var blockingQueriesGroups = blockingQueriesConfig.BlockingQueriesGroups; + var blockingQueriesGroups = blockingQueriesOptions.Value.BlockingQueriesGroups; if (blockingQueriesGroups == null || blockingQueriesGroups.Length == 0) { //throw exception? or publish a different message? or may be next phase return; } + + var connectionString = configuration.GetConnectionString("MatchingDb") + ?? throw new InvalidOperationException("Connection string 'MatchingDb' is not configured."); + var jsonQueries = JsonConvert.SerializeObject(blockingQueriesGroups); - SqlConnection conn = new SqlConnection(connString); + SqlConnection conn = new SqlConnection(connectionString); await conn.OpenAsync(); using (conn) { - SqlCommand command = new SqlCommand(insertCandidatesProcedure, conn); + SqlCommand command = new SqlCommand(storedProceduresOptions.Value.InsertCandidatesStoredProcedure, conn); command.CommandTimeout = 600; command.CommandType = CommandType.StoredProcedure; command.Parameters.AddWithValue("@jsonQueries", jsonQueries); @@ -80,7 +71,7 @@ public async Task InsertCandidates() catch (SqlException ex) { Console.WriteLine(ex.ToString()); - throw ex; + throw; } } } diff --git a/src/Blocking/Program.cs b/src/Blocking/Program.cs index 277a9c7..1c9fcf7 100644 --- a/src/Blocking/Program.cs +++ b/src/Blocking/Program.cs @@ -1,61 +1,29 @@ -using Messaging.Interfaces; -using Messaging.Services; +using Messaging.Extensions; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using EnvironmentSetup; using Blocking; using Blocking.ConfigurationModels; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); - - builder.Services.ConfigureServices(builder.Configuration); - - var storedProceduresConfig = builder.Configuration.GetSection("StoredProceduresConfig")!; - var blockingQueriesGroups = builder.Configuration.GetSection("BlockingQueriesConfig")!; + var builder = Host.CreateApplicationBuilder(args); - builder.Services.AddSingleton( - new ServerConfiguration - { - connectionString = builder.Configuration.GetConnectionString("MatchingDb")!, - storedProceduresConfig = builder.Configuration.GetRequiredSection("StoredProceduresConfig").Get()!, - blockingQueriesConfig = builder.Configuration.GetSection("BlockingQueriesConfig").Get()! - } - ); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); + builder.Services.Configure(builder.Configuration.GetRequiredSection("StoredProceduresConfig")); + builder.Services.Configure(builder.Configuration.GetRequiredSection("BlockingQueriesConfig")); builder.Services.AddSingleton(); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - - builder.Services.AddHostedService(); var app = builder.Build(); Log.Information("Starting application"); app.Run(); + return 0; } catch (Exception ex) diff --git a/src/Blocking/Properties/launchSettings.json b/src/Blocking/Properties/launchSettings.json deleted file mode 100644 index 8646c87..0000000 --- a/src/Blocking/Properties/launchSettings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "profiles": { - "Blocking": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Docker" - } - } -} \ No newline at end of file diff --git a/src/Blocking/ServerConfiguration.cs b/src/Blocking/ServerConfiguration.cs deleted file mode 100644 index 5df237d..0000000 --- a/src/Blocking/ServerConfiguration.cs +++ /dev/null @@ -1,11 +0,0 @@ - -using Blocking.ConfigurationModels; - -namespace Blocking; - -public class ServerConfiguration -{ - public string connectionString = string.Empty; - public StoredProceduresConfig storedProceduresConfig = new(); - public BlockingQueriesConfig blockingQueriesConfig = new(); -} diff --git a/src/Blocking/StoredProceduresConfig.cs b/src/Blocking/StoredProceduresConfig.cs index 0d0b222..a390955 100644 --- a/src/Blocking/StoredProceduresConfig.cs +++ b/src/Blocking/StoredProceduresConfig.cs @@ -1,11 +1,7 @@ - -using Blocking.ConfigurationModels; - namespace Blocking; public class StoredProceduresConfig { public string TruncateStoredProcedure { get; set; } = string.Empty; public string InsertCandidatesStoredProcedure { get; set; } = string.Empty; - public BlockingQueriesConfig blockingQueriesConfig { get; set; } = new BlockingQueriesConfig(); } diff --git a/src/Blocking/appsettings.json b/src/Blocking/appsettings.json index 4696206..7e0568c 100644 --- a/src/Blocking/appsettings.json +++ b/src/Blocking/appsettings.json @@ -1,5 +1,28 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Blocking-.txt", + "ConnectionStrings": { + "MatchingDb": "", + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/Blocking-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, "StoredProceduresConfig": { "TruncateStoredProcedure": "[Matching].[TruncateTables]", "InsertCandidatesStoredProcedure": "[Matching].[InsertCandidates]", diff --git a/src/Cleanup/Cleanup.csproj b/src/Cleanup/Cleanup.csproj index 7f81d8a..4e16161 100644 --- a/src/Cleanup/Cleanup.csproj +++ b/src/Cleanup/Cleanup.csproj @@ -5,20 +5,8 @@ enable enable 32b35ac4-a44b-4a1e-bdfd-d7ab13dbdc5f - Linux - ..\..\.. - - - - - - - - - - diff --git a/src/Cleanup/CleanupBackgroundService.cs b/src/Cleanup/CleanupBackgroundService.cs index 49ef867..3c0f49b 100644 --- a/src/Cleanup/CleanupBackgroundService.cs +++ b/src/Cleanup/CleanupBackgroundService.cs @@ -1,70 +1,38 @@ using Cleanup.CleanupServices.LiveCleanup; - using Messaging.Interfaces; using Messaging.Messages.MergingMessages.CleanupMessages; using Messaging.Queues; using Microsoft.Extensions.Hosting; -using System.Globalization; using Messaging.Messages.StagingMessages.Offloc; using Messaging.Messages.StagingMessages.Delius; namespace Cleanup; -public class CleanupBackgroundService : BackgroundService +public class CleanupBackgroundService( + IMessageService messageService, + DeliusCleanupService deliusCleanup, + OfflocCleanupService offlocCleanup) : BackgroundService { - private readonly IStagingMessagingService stagingMessagingService; - private readonly IMergingMessagingService mergingMessagingService; - private readonly IDbMessagingService dbMessagingService; - - private DeliusCleanupService deliusCleanup; - private OfflocCleanupService offlocCleanup; - - private CultureInfo cultureInfo = new CultureInfo("en-GB"); - - public CleanupBackgroundService(IMergingMessagingService mergingService, - IDbMessagingService dbMessagingService, IStagingMessagingService stagingService, - OfflocCleanupService offlocCleanup, DeliusCleanupService deliusCleanup) - { - this.stagingMessagingService = stagingService; - this.mergingMessagingService = mergingService; - this.deliusCleanup = deliusCleanup; - this.offlocCleanup = offlocCleanup; - this.dbMessagingService = dbMessagingService; - } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await Task.CompletedTask; + await messageService.SubscribeAsync(async message => + { + deliusCleanup.Cleanup(message.FileName); + }, TMergingQueue.DeliusFilesCleanupQueue); - mergingMessagingService.MergingSubscribeAsync(async(message) => - await CleanDelius(message), TMergingQueue.DeliusFilesCleanupQueue); - mergingMessagingService.MergingSubscribeAsync(async(message) => - await CleanOffloc(message), TMergingQueue.OfflocFilesCleanupQueue); + await messageService.SubscribeAsync(async message => + { + offlocCleanup.Cleanup(message.FileName); + }, TMergingQueue.OfflocFilesCleanupQueue); - stagingMessagingService.StagingSubscribeAsync(async (message) => + await messageService.SubscribeAsync(async _ => { - await Task.CompletedTask; deliusCleanup.ClearIllegalFiles(); }, TStagingQueue.DeliusFilesClear); - stagingMessagingService.StagingSubscribeAsync(async (message) => + await messageService.SubscribeAsync(async _ => { - await Task.CompletedTask; offlocCleanup.ClearIllegalFiles(); }, TStagingQueue.OfflocFilesClear); } - - private async Task CleanDelius(DeliusFilesCleanupMessage message) - { - await Task.CompletedTask; - - deliusCleanup.Cleanup(message.fileName); - } - - private async Task CleanOffloc(OfflocFilesCleanupMessage message) - { - await Task.CompletedTask; - - offlocCleanup.Cleanup(message.fileName); - } } diff --git a/src/Cleanup/CleanupServices/LiveCleanup/OfflocCleanupService.cs b/src/Cleanup/CleanupServices/LiveCleanup/OfflocCleanupService.cs index 45ef0a3..c30fea2 100644 --- a/src/Cleanup/CleanupServices/LiveCleanup/OfflocCleanupService.cs +++ b/src/Cleanup/CleanupServices/LiveCleanup/OfflocCleanupService.cs @@ -1,22 +1,11 @@ using FileStorage; -using Messaging.Interfaces; -using Messaging.Messages.DbMessages.Receiving; -using Messaging.Messages.DbMessages.Sending; namespace Cleanup.CleanupServices.LiveCleanup; -public class OfflocCleanupService : CleanupServiceBase +public class OfflocCleanupService(IFileLocations fileLocations) : CleanupServiceBase(fileLocations.offlocInput, fileLocations.offlocOutput) { - private readonly IDbMessagingService dbMessagingService; - - public OfflocCleanupService(IFileLocations fileLocations, IDbMessagingService dbMessagingService) - : base(fileLocations.offlocInput, fileLocations.offlocOutput) - { - this.dbMessagingService = dbMessagingService; - } - - //A method to make DMS more resilient (by stopping it from trying to process half-cleaned files). - public override void ClearIllegalFiles() + //A method to make DMS more resilient (by stopping it from trying to process half-cleaned files). + public override void ClearIllegalFiles() { DirectoryInfo di = new DirectoryInfo(inputFolderPath); var illegalFiles = di.GetFiles("*_clean*"); diff --git a/src/Cleanup/Program.cs b/src/Cleanup/Program.cs index c169cde..4ab96fc 100644 --- a/src/Cleanup/Program.cs +++ b/src/Cleanup/Program.cs @@ -1,42 +1,21 @@ - +using Messaging.Extensions; using Cleanup; using Cleanup.CleanupServices.LiveCleanup; using EnvironmentSetup; -using Messaging.Interfaces; -using Messaging.Services; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); + var builder = Host.CreateApplicationBuilder(args); - builder.Services.ConfigureServices(builder.Configuration); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); builder.Services.AddSingleton(); builder.Services.AddSingleton(); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddHostedService(); var app = builder.Build(); diff --git a/src/Cleanup/appsettings.json b/src/Cleanup/appsettings.json index 906e0ae..d1db885 100644 --- a/src/Cleanup/appsettings.json +++ b/src/Cleanup/appsettings.json @@ -1,6 +1,27 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Cleanup-.txt", - "EnvFilePath": "../", + "ConnectionStrings": { + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/Cleanup-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, "exclude": [ "**/bin", "**/bower_components", diff --git a/src/DbInteractions/ConnectionStrings.cs b/src/DbInteractions/ConnectionStrings.cs deleted file mode 100644 index 93e42b1..0000000 --- a/src/DbInteractions/ConnectionStrings.cs +++ /dev/null @@ -1,10 +0,0 @@ - -namespace DbInteractions; - -public class ConnectionStrings -{ - public string deliusStagingConnectionString { get; set; } = string.Empty; - public string offlocStagingConnectionString { get; set; } = string.Empty; - public string deliusPictureConnectionString { get; set; } = string.Empty; - public string offlocPictureConnectionString { get; set; } = string.Empty; -} diff --git a/src/DbInteractions/DbBackgroundService.cs b/src/DbInteractions/DbBackgroundService.cs index 57f8b13..0fb9541 100644 --- a/src/DbInteractions/DbBackgroundService.cs +++ b/src/DbInteractions/DbBackgroundService.cs @@ -1,7 +1,6 @@  using DbInteractions.Services; using Messaging.Interfaces; -using Messaging.Messages.DbMessages; using Messaging.Messages.DbMessages.Receiving; using Messaging.Messages.DbMessages.Sending; using Messaging.Messages.MergingMessages.CleanupMessages; @@ -11,129 +10,105 @@ namespace DbInteractions; //This class links the messaging service and dbInteractionService- the dbinteraction service does all of the work. -public class DbBackgroundService : BackgroundService +public class DbBackgroundService( + IDbInteractionService dbInteractionService, + IMessageService messageService) : BackgroundService { - private readonly IDbInteractionService dbInteractionService; - private readonly IDbMessagingService dbMessagingService; - private readonly IMergingMessagingService mergingService; - - public DbBackgroundService(IDbInteractionService dbInteractionService, IDbMessagingService dbMessagingService, - IMergingMessagingService mergingService) - { - this.dbInteractionService = dbInteractionService; - this.dbMessagingService = dbMessagingService; - this.mergingService = mergingService; - } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await Task.CompletedTask; - - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { string[] results = await dbInteractionService.GetProcessedDeliusFileNames(); - DeliusFilesReturnMessage msg = new DeliusFilesReturnMessage(results); - await dbMessagingService.DbPublishResponseAsync(msg); + GetDeliusFilesResponse msg = new GetDeliusFilesResponse { FileNames = results }; + await messageService.PublishAsync(msg); }, TDbQueue.GetProcessedDeliusFiles); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { string[] results = await dbInteractionService.GetProcessedOfflocFileNames(); - await dbMessagingService.DbPublishResponseAsync(new OfflocFilesReturnMessage(results)); + await messageService.PublishAsync(new GetOfflocFilesResponse { OfflocFiles = results }); }, TDbQueue.GetProcessedOfflocFiles); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { - await dbInteractionService.StageDelius(message.fileName, message.filePath); - await dbMessagingService.DbPublishResponseAsync(new StageDeliusReturnMessage()); - await mergingService.MergingPublishAsync(new DeliusFilesCleanupMessage(message.fileName)); + await dbInteractionService.StageDelius(message.FileName, message.FilePath); + await messageService.PublishAsync(new StageDeliusResponse()); + await messageService.PublishAsync(new DeliusFilesCleanupMessage(message.FileName)); }, TDbQueue.StageDelius); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { - //await dbInteractionService.StandardiseDeliusStaging(); - await dbInteractionService.MergeDeliusPicture(message.fileName); + await dbInteractionService.MergeDeliusPicture(message.FileName); await dbInteractionService.ClearDeliusStaging(); - - await dbMessagingService.DbPublishResponseAsync(new MergeDeliusReturnMessage()); + await messageService.PublishAsync(new MergeDeliusResponse()); }, TDbQueue.MergeDelius); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { await dbInteractionService.ClearDeliusStaging(); - await dbMessagingService.DbPublishResponseAsync(new ResultClearDeliusStaging()); + await messageService.PublishAsync(new ClearDeliusStagingResponse()); }, TDbQueue.ClearDeliusStaging); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { - await dbInteractionService.StageOffloc(message.fileName); - await dbMessagingService.DbPublishResponseAsync(new StageOfflocReturnMessage()); - - await mergingService.MergingPublishAsync(new OfflocFilesCleanupMessage(message.fileName)); + await dbInteractionService.StageOffloc(message.FileName); + await messageService.PublishAsync(new StageOfflocResponse()); + await messageService.PublishAsync(new OfflocFilesCleanupMessage(message.FileName)); }, TDbQueue.StageOffloc); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { - await dbInteractionService.MergeOfflocPicture(message.fileName); + await dbInteractionService.MergeOfflocPicture(message.FileName); await dbInteractionService.ClearOfflocStaging(); - - await dbMessagingService.DbPublishResponseAsync(new MergeOfflocReturnMessage()); - + await messageService.PublishAsync(new MergeOfflocResponse()); }, TDbQueue.MergeOffloc); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { await dbInteractionService.ClearOfflocStaging(); - await dbMessagingService.DbPublishResponseAsync(new ResultClearOfflocStaging()); + await messageService.PublishAsync(new ClearOfflocStagingResponse()); }, TDbQueue.ClearOfflocStaging); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { - int res = await dbInteractionService.DeliusGetFileIdLastFull(); - await dbMessagingService.DbPublishResponseAsync(new ResultDeliusGetLastFullMessage(res)); - }, TDbQueue.DeliusGetLastFullId); - - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => - { - await dbInteractionService.CreateOfflocProcessedFileEntry(message.fileName, message.fileId, message.archiveName); - await dbMessagingService.DbPublishResponseAsync(new ResultOfflocFileProcessingStarted()); + await dbInteractionService.CreateOfflocProcessedFileEntry(message.FileName, message.FileId, message.ArchiveName); + await messageService.PublishAsync(new StartOfflocFileProcessingResponse()); }, TDbQueue.OfflocFileProcessingStarted); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { - await dbInteractionService.CreateDeliusProcessedFileEntry(message.fileName, message.fileId); - await dbMessagingService.DbPublishResponseAsync(new ResultDeliusFileProcessingStarted()); + await dbInteractionService.CreateDeliusProcessedFileEntry(message.FileName, message.FileId); + await messageService.PublishAsync(new StartDeliusFileProcessingResponse()); }, TDbQueue.DeliusFileProcessingStarted); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { - await dbInteractionService.AssociateOfflocFileWithArchive(message.fileName, message.archiveName); - await dbMessagingService.DbPublishResponseAsync(new ResultAssociateOfflocFileWithArchiveMessage()); + await dbInteractionService.AssociateOfflocFileWithArchive(message.FileName, message.ArchiveName); + await messageService.PublishAsync(new AssociateOfflocFileWithArchiveResponse()); }, TDbQueue.AssociateOfflocFileWithArchive); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { bool result = await dbInteractionService.IsDeliusReadyForProcessing(); - await dbMessagingService.DbPublishResponseAsync(new IsDeliusReadyForProcessingReturnMessage(result)); + await messageService.PublishAsync(new CheckDeliusReadyResponse { IsReady = result }); }, TDbQueue.IsDeliusReadyForProcessing); - - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { bool result = await dbInteractionService.IsOfflocReadyForProcessing(); - await dbMessagingService.DbPublishResponseAsync(new IsOfflocReadyForProcessingReturnMessage(result)); + await messageService.PublishAsync(new CheckOfflocReadyResponse { IsReady = result }); }, TDbQueue.IsOfflocReadyForProcessing); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { string? result = await dbInteractionService.GetLastProcessedOfflocFileName(); - await dbMessagingService.DbPublishResponseAsync(new ResultGetLastProcessedOfflocFileMessage(result)); + await messageService.PublishAsync(new GetLastProcessedOfflocFileResponse { FileName = result }); }, TDbQueue.GetLastProcessedOfflocFile); - await dbMessagingService.SubscribeToDbRequestAsync(async (message) => + await messageService.SubscribeAsync(async (message) => { string? result = await dbInteractionService.GetLastProcessedDeliusFileName(); - await dbMessagingService.DbPublishResponseAsync(new ResultGetLastProcessedDeliusFileMessage(result)); + await messageService.PublishAsync(new GetLastProcessedDeliusFileResponse { FileName = result }); }, TDbQueue.GetLastProcessedDeliusFile); } diff --git a/src/DbInteractions/DbInteractions.csproj b/src/DbInteractions/DbInteractions.csproj index f0648c3..4b38dda 100644 --- a/src/DbInteractions/DbInteractions.csproj +++ b/src/DbInteractions/DbInteractions.csproj @@ -4,8 +4,6 @@ enable enable 2a7a51b3-c06a-45a4-a32f-29f1d000aa24 - Linux - ..\..\.. diff --git a/src/DbInteractions/Program.cs b/src/DbInteractions/Program.cs index 17125a5..51512bf 100644 --- a/src/DbInteractions/Program.cs +++ b/src/DbInteractions/Program.cs @@ -1,58 +1,22 @@ -//This project is so all database interactions occur through a single service and so there's +//This project is so all database interactions occur through a single service and so there's //only 1 set of connection strings to maintan. It will also make the examining of logs if //something goes wrong with the database easier in debugging. - using DbInteractions; using DbInteractions.Services; -using Messaging.Interfaces; -using Messaging.Services; -using Microsoft.Extensions.Configuration; +using Messaging.Extensions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Options; using EnvironmentSetup; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); - - builder.Services.ConfigureServices(builder.Configuration); - - builder.Services.Configure( - builder.Configuration.GetSection("ServerConfiguration") - ); - - builder.Services.AddSingleton(s => s.GetRequiredService>().Value); - - builder.Services.AddSingleton( - new ConnectionStrings { - deliusStagingConnectionString = builder.Configuration.GetConnectionString("DeliusStagingDb")!, - offlocStagingConnectionString = builder.Configuration.GetConnectionString("OfflocStagingDb")!, - deliusPictureConnectionString = builder.Configuration.GetConnectionString("DeliusRunningPictureDb")!, - offlocPictureConnectionString = builder.Configuration.GetConnectionString("OfflocRunningPictureDb")! - } - ); + var builder = Host.CreateApplicationBuilder(args); - builder.Services.AddSingleton(builder.Configuration); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + builder.Services.AddOptions().BindConfiguration("ServerConfiguration"); builder.Services.AddSingleton(); builder.Services.AddHostedService(); diff --git a/src/DbInteractions/Properties/launchSettings.json b/src/DbInteractions/Properties/launchSettings.json deleted file mode 100644 index 33d63a6..0000000 --- a/src/DbInteractions/Properties/launchSettings.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "profiles": { - "DbInteractions": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development", - "SYSTEM_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development", - "SYSTEM_ENVIRONMENT": "Development" - } - } - } -} \ No newline at end of file diff --git a/src/DbInteractions/Services/DbInteractionService.cs b/src/DbInteractions/Services/DbInteractionService.cs index a0d26db..6416fdd 100644 --- a/src/DbInteractions/Services/DbInteractionService.cs +++ b/src/DbInteractions/Services/DbInteractionService.cs @@ -3,44 +3,35 @@ using Messaging.Messages.StatusMessages; using Microsoft.Data.SqlClient; using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; using System.Data; namespace DbInteractions.Services; public class DbInteractionService : IDbInteractionService { - private IStatusMessagingService statusService; - private IFileLocations fileLocations; - - private ServerConfiguration serverConfig; - - private string deliusStagingConnString; - private string offlocStagingConnString; - private string deliusPictureConnString; - private string offlocPictureConnString; - - private bool inContainer; - - public DbInteractionService(IStatusMessagingService messageService, - ConnectionStrings connStrings, ServerConfiguration serverConfig, - IConfiguration config, IFileLocations fileLocations) + private readonly IFileLocations fileLocations; + private readonly ServerConfiguration serverConfig; + private readonly IConfiguration configuration; + private readonly bool inContainer; + private readonly IMessageService messageService; + + public DbInteractionService( + IOptions serverConfig, + IMessageService messageService, + IConfiguration config, + IFileLocations fileLocations) { - this.statusService = messageService; - this.serverConfig = serverConfig; + this.serverConfig = serverConfig.Value; + this.configuration = config; this.fileLocations = fileLocations; - - deliusStagingConnString = connStrings.deliusStagingConnectionString; - offlocStagingConnString = connStrings.offlocStagingConnectionString; - deliusPictureConnString = connStrings.deliusPictureConnectionString; - offlocPictureConnString = connStrings.offlocPictureConnectionString; - - //Make tidier. - inContainer = config.GetValue("RUNNING_IN_CONTAINER"); + this.messageService = messageService; + this.inContainer = config.GetValue("RUNNING_IN_CONTAINER"); } public async Task GetProcessedDeliusFileNames() { - SqlConnection conn = new SqlConnection(deliusPictureConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("DeliusRunningPictureDb")!); await conn.OpenAsync(); @@ -69,7 +60,7 @@ public async Task DeliusGetFileIdLastFull() { int fileId = 0; - SqlConnection conn = new SqlConnection(deliusPictureConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("DeliusRunningPictureDb")!); using (conn) { @@ -90,7 +81,7 @@ public async Task DeliusGetFileIdLastFull() } catch (SqlException e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return 0; } } @@ -101,7 +92,7 @@ public async Task DeliusGetFileIdLastFull() //Returning primitive types instead of strings should make DMS work better over time. public async Task GetProcessedOfflocIds() { - SqlConnection conn = new SqlConnection(offlocPictureConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (conn) { @@ -133,7 +124,7 @@ public async Task GetProcessedOfflocIds() public async Task GetProcessedOfflocFileNames() { - SqlConnection conn = new SqlConnection(offlocPictureConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (conn) { @@ -160,7 +151,7 @@ public async Task StageDelius(string fileName, string filePath) { string folderName = filePath.Split('/').Last(); - await statusService.StatusPublishAsync(new StatusUpdateMessage($"Delius staging started for file number {fileName}")); + await messageService.PublishAsync(new StatusUpdateMessage($"Delius staging started for file number {fileName}")); string containerFlag = string.Empty; if (inContainer) @@ -172,7 +163,7 @@ public async Task StageDelius(string fileName, string filePath) containerFlag = "N"; } - var deliusConn = new SqlConnection(deliusStagingConnString); + var deliusConn = new SqlConnection(configuration.GetConnectionString("DeliusStagingDb")!); using (deliusConn) { await deliusConn.OpenAsync(); @@ -190,20 +181,20 @@ public async Task StageDelius(string fileName, string filePath) } catch (Exception e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return; } } - await statusService.StatusPublishAsync(new StatusUpdateMessage($"Delius staging finished for file {fileName}.")); + await messageService.PublishAsync(new StatusUpdateMessage($"Delius staging finished for file {fileName}.")); } public async Task StageOffloc(string fileName) { string folderName = fileName.Split('.').First(); - await statusService.StatusPublishAsync(new StatusUpdateMessage($"Offloc staging started for file {fileName}.")); + await messageService.PublishAsync(new StatusUpdateMessage($"Offloc staging started for file {fileName}.")); - var offlocConn = new SqlConnection(offlocStagingConnString); + var offlocConn = new SqlConnection(configuration.GetConnectionString("OfflocStagingDb")!); using (offlocConn) { await offlocConn.OpenAsync(); @@ -219,17 +210,17 @@ public async Task StageOffloc(string fileName) } catch (Exception e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return; } } - await statusService.StatusPublishAsync(new StatusUpdateMessage($"Offloc staging finished for file {fileName}.")); + await messageService.PublishAsync(new StatusUpdateMessage($"Offloc staging finished for file {fileName}.")); } //Calls merge and then on completion public async Task StandardiseDeliusStaging() { - SqlConnection conn = new SqlConnection(deliusStagingConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("DeliusStagingDb")!); using (conn) { @@ -245,16 +236,16 @@ public async Task StandardiseDeliusStaging() } catch (SqlException exception) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(exception.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(exception.Message)); return; } } - await statusService.StatusPublishAsync(new StatusUpdateMessage("Delius staging database standardisation complete.")); + await messageService.PublishAsync(new StatusUpdateMessage("Delius staging database standardisation complete.")); } //Calls merge and then on completion public async Task MergeDeliusPicture(string fileName) { - SqlConnection conn = new SqlConnection(deliusPictureConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("DeliusRunningPictureDb")!); using (conn) { @@ -272,16 +263,16 @@ public async Task MergeDeliusPicture(string fileName) } catch (SqlException exception) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(exception.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(exception.Message)); return; } } - await statusService.StatusPublishAsync(new StatusUpdateMessage("Delius merging complete.")); + await messageService.PublishAsync(new StatusUpdateMessage("Delius merging complete.")); } public async Task ClearDeliusStaging() { - SqlConnection conn = new SqlConnection(deliusStagingConnString); + SqlConnection conn = new SqlConnection(configuration.GetConnectionString("DeliusStagingDb")!); using (conn) { @@ -297,16 +288,16 @@ public async Task ClearDeliusStaging() } catch (SqlException exception) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(exception.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(exception.Message)); return; } } - await statusService.StatusPublishAsync(new StatusUpdateMessage("Delius staging database cleared.")); + await messageService.PublishAsync(new StatusUpdateMessage("Delius staging database cleared.")); } public async Task MergeOfflocPicture(string fileName) { - SqlConnection offlocConn = new SqlConnection(offlocPictureConnString); + SqlConnection offlocConn = new SqlConnection(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (offlocConn) { @@ -324,16 +315,16 @@ public async Task MergeOfflocPicture(string fileName) } catch (SqlException e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return; } } - await statusService.StatusPublishAsync(new StatusUpdateMessage("Offloc merging complete.")); + await messageService.PublishAsync(new StatusUpdateMessage("Offloc merging complete.")); } public async Task ClearOfflocStaging() { - SqlConnection offlocConn = new SqlConnection(offlocStagingConnString); + SqlConnection offlocConn = new SqlConnection(configuration.GetConnectionString("OfflocStagingDb")!); using (offlocConn) { await offlocConn.OpenAsync(); @@ -348,17 +339,17 @@ public async Task ClearOfflocStaging() } catch (SqlException e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return; } } - await statusService.StatusPublishAsync(new StatusUpdateMessage("Offloc staging database cleared.")); + await messageService.PublishAsync(new StatusUpdateMessage("Offloc staging database cleared.")); } public async Task CreateOfflocProcessedFileEntry(string fileName, int fileId, string? archiveName = null) { - SqlConnection offlocConn = new(offlocPictureConnString); + SqlConnection offlocConn = new(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (offlocConn) { @@ -380,7 +371,7 @@ public async Task CreateOfflocProcessedFileEntry(string fileName, int fileId, st } catch (SqlException e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return; } } @@ -388,7 +379,7 @@ public async Task CreateOfflocProcessedFileEntry(string fileName, int fileId, st public async Task CreateDeliusProcessedFileEntry(string fileName, string fileId) { - SqlConnection deliusConn = new(deliusPictureConnString); + SqlConnection deliusConn = new(configuration.GetConnectionString("DeliusRunningPictureDb")!); using (deliusConn) { @@ -409,7 +400,7 @@ public async Task CreateDeliusProcessedFileEntry(string fileName, string fileId) } catch (SqlException e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return; } } @@ -417,7 +408,7 @@ public async Task CreateDeliusProcessedFileEntry(string fileName, string fileId) public async Task AssociateOfflocFileWithArchive(string fileName, string archiveName) { - SqlConnection offlocConn = new(offlocPictureConnString); + SqlConnection offlocConn = new(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (offlocConn) { @@ -441,7 +432,7 @@ UPDATE [OfflocRunningPicture].[ProcessedFiles] } catch (SqlException e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return; } } @@ -449,7 +440,7 @@ UPDATE [OfflocRunningPicture].[ProcessedFiles] public async Task IsDeliusReadyForProcessing() { - SqlConnection deliusConn = new(deliusPictureConnString); + SqlConnection deliusConn = new(configuration.GetConnectionString("DeliusRunningPictureDb")!); using (deliusConn) { @@ -479,7 +470,7 @@ WHERE Status <> 'Merged' } catch (SqlException e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return false; } } @@ -487,7 +478,7 @@ WHERE Status <> 'Merged' public async Task IsOfflocReadyForProcessing() { - SqlConnection offlocConn = new(offlocPictureConnString); + SqlConnection offlocConn = new(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (offlocConn) { @@ -517,7 +508,7 @@ WHERE Status <> 'Merged' } catch (SqlException e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return false; } } @@ -525,7 +516,7 @@ WHERE Status <> 'Merged' public async Task GetLastProcessedOfflocFileName() { - SqlConnection offlocConn = new(offlocPictureConnString); + SqlConnection offlocConn = new(configuration.GetConnectionString("OfflocRunningPictureDb")!); using (offlocConn) { @@ -549,7 +540,7 @@ FROM OfflocRunningPicture.ProcessedFiles } catch (SqlException e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return null; } } @@ -557,7 +548,7 @@ FROM OfflocRunningPicture.ProcessedFiles public async Task GetLastProcessedDeliusFileName() { - SqlConnection deliusConn = new(deliusPictureConnString); + SqlConnection deliusConn = new(configuration.GetConnectionString("DeliusRunningPictureDb")!); using (deliusConn) { @@ -581,7 +572,7 @@ FROM DeliusRunningPicture.ProcessedFiles } catch (SqlException e) { - await statusService.StatusPublishAsync(new StatusUpdateMessage(e.Message)); + await messageService.PublishAsync(new StatusUpdateMessage(e.Message)); return null; } } diff --git a/src/DbInteractions/appsettings.json b/src/DbInteractions/appsettings.json index 20ab3ff..094d57f 100644 --- a/src/DbInteractions/appsettings.json +++ b/src/DbInteractions/appsettings.json @@ -1,11 +1,30 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/DbInterations-.txt", - "EnvFilePath": "../", "ConnectionStrings": { - "deliusStagingConnectionString": "", - "offlocStagingConnectionString": "", - "deliusPictureConnectionString": "", - "offlocPictureConnectionString": "" + "DeliusRunningPictureDb": "", + "OfflocRunningPictureDb": "", + "DeliusStagingDb": "", + "OfflocStagingDb": "", + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/DbInteractions-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, "ServerConfiguration": { "DeliusStagingProcedure": "DeliusStaging.StageDelius", diff --git a/src/Delius.Parser/AppConfig/ConfigurationParser.cs b/src/Delius.Parser/AppConfig/ConfigurationParser.cs deleted file mode 100644 index 816cb8d..0000000 --- a/src/Delius.Parser/AppConfig/ConfigurationParser.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System.Xml.Serialization; -using Delius.Parser.Configuration.Models; - -namespace Delius.Parser.Config; - -//Parses DeliusConfiguration.xml in same directory to construct a list of lines that can then be used by the processor. -//Assumes structure and type of xml file is correct and simply copies lines in xml file to Line and Field instances. -public static class ConfigurationParser -{ - internal static Line[] GetLines() - { - var ser = new XmlSerializer(typeof(Line[])); - - var streamReader = new StreamReader(AppContext.BaseDirectory + "\\Config\\DeliusConfiguration.xml"); - - var lines = ser.Deserialize(streamReader) as Line[] ?? throw new ApplicationException("Cannot deserialize the file :("); - return lines; - - } -} diff --git a/src/Delius.Parser/AppConfig/Models/NextCloudCredentials.cs b/src/Delius.Parser/AppConfig/Models/NextCloudCredentials.cs deleted file mode 100644 index a3c8031..0000000 --- a/src/Delius.Parser/AppConfig/Models/NextCloudCredentials.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace Delius.Parser.AppConfig.Models; - -public class NextCloudCredentials -{ - public const string Key = nameof(NextCloudCredentials); - - public string Username { get; set; } = string.Empty; - public string Password { get; set; } = string.Empty; -} diff --git a/src/Delius.Parser/Core/TextFileProcessor.cs b/src/Delius.Parser/Core/TextFileProcessor.cs index c3308e9..3a0e564 100644 --- a/src/Delius.Parser/Core/TextFileProcessor.cs +++ b/src/Delius.Parser/Core/TextFileProcessor.cs @@ -8,10 +8,10 @@ namespace Delius.Parser.Core; public class TextFileProcessor : IFileProcessor { - private readonly IStagingMessagingService messageService; + private readonly IMessageService messageService; private DeliusProcessor deliusProcessor; - public TextFileProcessor(IStagingMessagingService messageService, DeliusProcessor deliusProcessor) + public TextFileProcessor(IMessageService messageService, DeliusProcessor deliusProcessor) { this.messageService = messageService; this.deliusProcessor = deliusProcessor; diff --git a/src/Delius.Parser/Delius.Parser.csproj b/src/Delius.Parser/Delius.Parser.csproj index 8731d4e..1a6cb4d 100644 --- a/src/Delius.Parser/Delius.Parser.csproj +++ b/src/Delius.Parser/Delius.Parser.csproj @@ -4,8 +4,6 @@ Exe enable enable - Linux - ..\..\.. bed042b3-1d10-47f8-9b97-b516f3ae5895 diff --git a/src/Delius.Parser/DeliusParserBackgroundService.cs b/src/Delius.Parser/DeliusParserBackgroundService.cs index e7ac374..a701c38 100644 --- a/src/Delius.Parser/DeliusParserBackgroundService.cs +++ b/src/Delius.Parser/DeliusParserBackgroundService.cs @@ -1,6 +1,4 @@ using Delius.Parser.Services; -using EnvironmentSetup; -using FileStorage; using Messaging.Interfaces; using Messaging.Messages.DbMessages.Receiving; using Messaging.Messages.DbMessages.Sending; @@ -12,27 +10,25 @@ namespace Delius.Parser; public class DeliusParserBackgroundService( - IStagingMessagingService messageService, - IDbMessagingService dbService, - IStatusMessagingService statusService, + IMessageService messageService, IParsingStrategy parseService) : BackgroundService { protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await Task.Run(() => - { - messageService.StagingSubscribeAsync(async (message) => await ParseFileAsync(message), TStagingQueue.DeliusParser); - }, stoppingToken); + await messageService.SubscribeAsync(async (message) => + { + await ParseFileAsync(message); + }, TStagingQueue.DeliusParser); } private async Task ParseFileAsync(DeliusDownloadFinishedMessage message) { - var file = message.fileName; + var file = message.FileName; if (await HasAlreadyBeenProcessedAsync(file)) { - await statusService.StatusPublishAsync(new StatusUpdateMessage($"File {file} has already been processed")); - await messageService.StagingPublishAsync(new DeliusParserFinishedMessage("File already processed", "No Path", true)); + await messageService.PublishAsync(new StatusUpdateMessage($"File {file} has already been processed")); + await messageService.PublishAsync(new DeliusParserFinishedMessage("File already processed", "No Path", emptyFile: true)); } else { @@ -42,14 +38,18 @@ private async Task ParseFileAsync(DeliusDownloadFinishedMessage message) private async Task BeginProcessing(string fileName, string fileId) { - var request = new DeliusFileProcessingStarted(fileName, fileId); - await dbService.SendDbRequestAndWaitForResponseAsync(request); + var request = new StartDeliusFileProcessingRequest + { + FileName = fileName, + FileId = fileId + }; + await messageService.SendDbRequestAndWaitForResponseAsync(request); await parseService.ParseFileAsync(fileName); } private async Task HasAlreadyBeenProcessedAsync(string file) { - var res = await dbService.SendDbRequestAndWaitForResponseAsync(new GetDeliusFilesMessage()); - return res.fileNames.Contains(file); + var res = await messageService.SendDbRequestAndWaitForResponseAsync(new GetDeliusFilesRequest()); + return res.FileNames.Contains(file); } } \ No newline at end of file diff --git a/src/Delius.Parser/Program.cs b/src/Delius.Parser/Program.cs index 332945b..f85a4f0 100644 --- a/src/Delius.Parser/Program.cs +++ b/src/Delius.Parser/Program.cs @@ -1,41 +1,18 @@ - +using Messaging.Extensions; using Delius.Parser; using Delius.Parser.Core; using Delius.Parser.Services; using EnvironmentSetup; -using Messaging.Interfaces; -using Messaging.Services; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); - - builder.Services.AddSingleton(builder.Configuration); - - builder.Services.ConfigureServices(builder.Configuration); - - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + var builder = Host.CreateApplicationBuilder(args); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); builder.Services.AddSingleton(); diff --git a/src/Delius.Parser/Properties/launchSettings.json b/src/Delius.Parser/Properties/launchSettings.json deleted file mode 100644 index 26674c5..0000000 --- a/src/Delius.Parser/Properties/launchSettings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "profiles": { - "Delius.Parser": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Project" - } - } -} \ No newline at end of file diff --git a/src/Delius.Parser/Services/ParsingStrategyBase.cs b/src/Delius.Parser/Services/ParsingStrategyBase.cs index 5303827..1926a5b 100644 --- a/src/Delius.Parser/Services/ParsingStrategyBase.cs +++ b/src/Delius.Parser/Services/ParsingStrategyBase.cs @@ -10,24 +10,21 @@ public abstract class ParsingStrategyBase { protected readonly IFileLocations fileLocations; protected readonly IFileProcessor fileProcessor; - protected readonly IStatusMessagingService statusService; - protected readonly IStagingMessagingService stagingService; + protected readonly IMessageService messageService; - public ParsingStrategyBase(IFileProcessor fp, IStatusMessagingService statusService, - IStagingMessagingService stagingService, IFileLocations fileLocations) + public ParsingStrategyBase(IFileProcessor fp, IMessageService messageService, IFileLocations fileLocations) { - this.statusService = statusService; - this.stagingService = stagingService; + this.messageService = messageService; fileProcessor = fp; this.fileLocations = fileLocations; } - public async Task ProcessFile(string file) + public async Task ProcessFileAsync(string file) { - await statusService.StatusPublishAsync(new StatusUpdateMessage($"Delius parser started on file {file}.")); + await messageService.PublishAsync(new StatusUpdateMessage($"Delius parser started on file {file}.")); await fileProcessor.Process(fileLocations.deliusInput + '/' + file, $"{fileLocations.deliusOutput}/{file.Split('.').First()}"); - await stagingService.StagingPublishAsync(new DeliusParserFinishedMessage(file, fileLocations.deliusInput + '/' + file.Split('.').First(), false)); + await messageService.PublishAsync(new DeliusParserFinishedMessage(file, fileLocations.deliusInput + '/' + file.Split('.').First(), false)); } } diff --git a/src/Delius.Parser/Services/SequentialParsingStrategy.cs b/src/Delius.Parser/Services/SequentialParsingStrategy.cs index 0ca9372..8f697b8 100644 --- a/src/Delius.Parser/Services/SequentialParsingStrategy.cs +++ b/src/Delius.Parser/Services/SequentialParsingStrategy.cs @@ -1,17 +1,13 @@ - -using System.Diagnostics; -using Delius.Parser.Core; +using Delius.Parser.Core; using FileStorage; using Messaging.Interfaces; namespace Delius.Parser.Services; -public class SequentialParsingStrategy : ParsingStrategyBase, IParsingStrategy +public class SequentialParsingStrategy( + IFileProcessor fileProcessor, + IMessageService stagingMessagingService, + IFileLocations fileLocations) : ParsingStrategyBase(fileProcessor, stagingMessagingService, fileLocations), IParsingStrategy { - public SequentialParsingStrategy(IFileProcessor fileProcessor, - IStagingMessagingService stagingMessagingService, IStatusMessagingService statusMessagingService, - IFileLocations fileLocations) - : base(fileProcessor, statusMessagingService, stagingMessagingService, fileLocations) { } - - public Task ParseFileAsync(string file) => ProcessFile(file); + public Task ParseFileAsync(string file) => ProcessFileAsync(file); } diff --git a/src/Delius.Parser/appsettings.development.json b/src/Delius.Parser/appsettings.development.json deleted file mode 100644 index 9fd8aee..0000000 --- a/src/Delius.Parser/appsettings.development.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "exclude": [ - "**/bin", - "**/bower_components", - "**/jspm_packages", - "**/node_modules", - "**/obj", - "**/platforms" - ] - -} \ No newline at end of file diff --git a/src/Delius.Parser/appsettings.json b/src/Delius.Parser/appsettings.json index 888ff7d..98e8030 100644 --- a/src/Delius.Parser/appsettings.json +++ b/src/Delius.Parser/appsettings.json @@ -1,4 +1,25 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Delius.Parser-.txt", - "EnvFilePath": "..\\" + "ConnectionStrings": { + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/Delius.Parser-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + } } diff --git a/src/FakeDataSeeder/FakeDataSeeder.csproj b/src/FakeDataSeeder/FakeDataSeeder.csproj index e1382c0..f254e9f 100644 --- a/src/FakeDataSeeder/FakeDataSeeder.csproj +++ b/src/FakeDataSeeder/FakeDataSeeder.csproj @@ -17,5 +17,11 @@ + + + Always + + + \ No newline at end of file diff --git a/src/FakeDataSeeder/Scripts/0001_Create_CreateDummyCandidate.sql b/src/FakeDataSeeder/Scripts/0001_Create_CreateDummyCandidate.sql index e0448ef..5b7c9f0 100644 --- a/src/FakeDataSeeder/Scripts/0001_Create_CreateDummyCandidate.sql +++ b/src/FakeDataSeeder/Scripts/0001_Create_CreateDummyCandidate.sql @@ -36,6 +36,17 @@ BEGIN RETURN; END + IF EXISTS(SELECT TOP(1) NOMSnumber FROM OfflocRunningPictureDb.OfflocRunningPicture.PersonalDetails where NOMSnumber = @NomisNumber) + BEGIN + PRINT 'Nomis already exists'; + RETURN; + END + + IF EXISTS(SELECT TOP(1) CRN FROM DeliusRunningPictureDb.DeliusRunningPicture.Offenders where CRN = @Crn) + BEGIN + PRINT 'Delius already exists'; + RETURN; + END SELECT TOP 1 @OffenderToOffenderManagerMappingId = Id FROM [DeliusRunningPictureDb].[DeliusRunningPicture].OffenderToOffenderManagerMappings ORDER BY Id DESC; SET @OffenderToOffenderManagerMappingId = @OffenderToOffenderManagerMappingId +1; @@ -67,11 +78,11 @@ BEGIN IF @ClusterId IS NULL BEGIN - INSERT INTO [reference].[UPCI2] (UPCI2) VALUES(@Identifier); - SELECT @ClusterId = SCOPE_IDENTITY() + -- Find the next available ClusterId + SELECT @ClusterId = (SELECT COALESCE(MAX(ClusterId), 0) + 1 FROM [output].[Clusters]); + UPDATE [reference].[UPCI2] SET UPCI2 = @Identifier WHERE ClusterId = @ClusterId END - IF LEN(@NomisNumber)>0 and LEN(@Crn) > 0 BEGIN SET @RecordCount=2; diff --git a/src/FakeDataSeeder/Scripts/0002_Populate_DummyCandidate.sql b/src/FakeDataSeeder/Scripts/0002_Populate_DummyCandidate.sql index 7befef9..816f3ad 100644 --- a/src/FakeDataSeeder/Scripts/0002_Populate_DummyCandidate.sql +++ b/src/FakeDataSeeder/Scripts/0002_Populate_DummyCandidate.sql @@ -6,56 +6,48 @@ PRINT 'Executing CreateDummyCandidate to populate dummy candidates...'; -- Check if the stored procedure exists before calling it (good practice for idempotency) IF OBJECT_ID(N'[dbo].[CreateDummyCandidate]', N'P') IS NOT NULL BEGIN - - IF EXISTS (SELECT 1 FROM [output].[ClusterMembership]) - BEGIN - PRINT 'Skipping, candidates already populated' - END - ELSE - BEGIN - exec CreateDummyCandidate '1CFG1109X','Barry','Stone','18/01/1979 00:00','A7022BA','B008622','TRUE','Male','British','White','DELIUS','ACI','C06'; - exec CreateDummyCandidate '1CFG3305M','Clark','Queen','16/07/1985 00:00','A7373XA','B004786','TRUE','Female','British','White','DELIUS','HII','N28'; - exec CreateDummyCandidate '1CFG3647Z','Hal','Parker','31/08/2000 00:00','A1118OA','B006754','TRUE','Male','British','Black','DELIUS','KMI','LCS'; - exec CreateDummyCandidate '1CFG4819M','Arthur','Stone','08/03/1961 00:00','A6972XA','B008866','TRUE','Female','British','White','NOMIS','LPI','MRS'; - exec CreateDummyCandidate '1CFG3977A','Oliver','Kent','16/08/1966 00:00','A3749HA','B004529','FALSE','Male','British','White','NOMIS','MRI','MCG'; - exec CreateDummyCandidate '1CFG4580T','Barry','Jordan','18/06/1979 00:00','A6102NA','B007498','FALSE','Male','British','Black','NOMIS','RSI','N50'; - exec CreateDummyCandidate '1CFG7754K','Victor','Kent','21/10/1956 00:00','A2817BA','B003913','FALSE','Male','British','White','NOMIS','PNI','CMB'; - exec CreateDummyCandidate '1CFG1873Q','Arthur','Wayne','15/08/1988 00:00','A8654XA','B004044','TRUE','Male','British','White','NOMIS','BAI','C17'; - exec CreateDummyCandidate '1CFG5221C','Victor','Parker','14/09/1977 00:00','A3576SA','B003506','TRUE','Male','British','White','DELIUS','BXI','LDN'; - exec CreateDummyCandidate '1CFG5006P','Clark','Kent','09/09/1988 00:00','A8065EA','B001407','TRUE','Female','British','White','DELIUS','FMI','N07'; - exec CreateDummyCandidate '1CFG2001T','Oliver','Jordan','30/09/2004 00:00','A6587YA','B003237','TRUE','Female','British','Black','NOMIS','ISI','N21'; - exec CreateDummyCandidate '1CFG5437L','Hal','Parker','25/09/1956 00:00','A6952ZA','B003171','TRUE','Male','British','White','DELIUS','HLI','N55'; - exec CreateDummyCandidate '1CFG2883L','Oliver','Prince','29/08/1961 00:00','A3402IA','B007887','TRUE','Male','British','White','DELIUS','HMI','HBS'; - exec CreateDummyCandidate '1CFG2167R','Bruce','Allen','27/08/1969 00:00','A7364GA','B009234','TRUE','Female','British','White','NOMIS','LEI','YSW'; - exec CreateDummyCandidate '1CFG9064R','Bruce','Jordan','19/07/1982 00:00','A6222SA','B004752','TRUE','Male','British','White','NOMIS','NHI','C05'; - exec CreateDummyCandidate '1CFG1900I','Oliver','Curry','30/06/1983 00:00','A9293CA','B005270','TRUE','Male','British','White','NOMIS','LHI','C09'; - exec CreateDummyCandidate '1CFG2847D','Bruce','Jordan','17/09/1991 00:00','A5884JA','B005459','TRUE','Female','British','Black','DELIUS','BMI','N52'; - exec CreateDummyCandidate '1CFG7892B','Arthur','Wayne','22/08/1952 00:00','A9162QA','B006933','TRUE','Male','British','White','DELIUS','DGI','N30'; - exec CreateDummyCandidate '1CFG6906H','Diana','Curry','09/08/1980 00:00','A6150IA','B005260','TRUE','Male','British','White','NOMIS','DHI','SWM'; - exec CreateDummyCandidate '1CFG1479H','Peter','Prince','18/07/1956 00:00','A6948JA','B004442','TRUE','Female','British','White','NOMIS','HEI','WMP'; - exec CreateDummyCandidate '1CFG7788V','Diana','Lance','27/10/1953 00:00','A9649AA','B004219','TRUE','Male','British','White','NOMIS','SFI','C11'; - exec CreateDummyCandidate '1CFG1553F','Victor','Jordan','08/12/1978 00:00','A9860RA','B008576','TRUE','Female','British','White','DELIUS','FHI','DBS'; - exec CreateDummyCandidate '1CFG4867Q','Oliver','Jordan','03/02/2000 00:00','A1567LA','B008340','TRUE','Male','British','Black','NOMIS','FWI','N31'; - exec CreateDummyCandidate '1CFG5469L','Peter','Queen','18/11/1953 00:00','A4692DA','B001257','TRUE','Male','British','White','NOMIS','LII','LNS'; - exec CreateDummyCandidate '1CFG5502H','Bruce','Kent','11/02/1978 00:00','A1411YA','B009194','TRUE','Male','British','Asian','DELIUS','WLI','NSP'; - exec CreateDummyCandidate '1CFG2825E','Peter','Prince','23/12/1975 00:00','A7907DA','B009268','TRUE','Female','British','White','DELIUS','PBI','CBS'; - exec CreateDummyCandidate '1CFG8516T','Dinah','Kent','19/06/1954 00:00','A7400NA','B009761','TRUE','Female','British','White','NOMIS','NWI','N56'; - exec CreateDummyCandidate '1CFG5298O','Victor','Jordan','28/07/1961 00:00','A8430TA','B007256','TRUE','Male','British','White','NOMIS','MTI','HFS'; - exec CreateDummyCandidate '1CFG9775P','Barry','Stone','14/08/1960 00:00','A1067SA','B008155','TRUE','Female','British','White','DELIUS','LNI','N02'; - exec CreateDummyCandidate '1CFG6849L','Bruce','Parker','20/02/1977 00:00','A7483AA','B009597','TRUE','Male','British','White','DELIUS','NLI','NBR'; - exec CreateDummyCandidate '1CFG8347B','Clark','Queen','06/10/2005 00:00','A1056JA','B005305','TRUE','Female','British','White','NOMIS','KVI','N23'; - exec CreateDummyCandidate '1CFG4189E','Bruce','Curry','14/04/1956 00:00','A3064CA','B006480','TRUE','Female','British','Black','NOMIS','HHI','N32'; - exec CreateDummyCandidate '1CFG5514M','Clark','Curry','28/05/1962 00:00','A7236EA','B005528','TRUE','Female','British','White','NOMIS','DMI','N54'; - exec CreateDummyCandidate '1CFG5252T','Peter','Queen','23/09/1964 00:00','A4570WA','B008719','TRUE','Female','British','White','DELIUS','SDI','SSP'; - exec CreateDummyCandidate '1CFG6789D','Clark','Parker','24/10/1968 00:00','A6440GA','B002584','TRUE','Male','British','White','DELIUS','SLI','N35'; - exec CreateDummyCandidate '1CFG1848U','Barry','Jordan','09/06/1981 00:00','A7126QA','B004852','TRUE','Female','British','White','DELIUS','SPI','N37'; - exec CreateDummyCandidate '1CFG4892S','Arthur','Allen','09/10/1960 00:00','A6249NA','B008270','TRUE','Male','British','White','NOMIS','WHI','N59'; - exec CreateDummyCandidate '1CFG5866O','Peter','Jordan','07/10/2001 00:00','A9186BA','B008561','TRUE','Female','British','Black','NOMIS','CWI','C19'; - exec CreateDummyCandidate '1CFG7842D','Diana','Queen','15/07/1982 00:00','A2699ZA','B005033','TRUE','Female','British','White','NOMIS','DAI','DCP'; - exec CreateDummyCandidate '1CFG8262O','Peter','Curry','02/07/1984 00:00','A2865UA','B001022','TRUE','Male','British','White','NOMIS','BLI','C15'; - exec CreateDummyCandidate '1CFG6353O','Hal','Stone','15/01/1952 00:00','A4973LA','B002433','TRUE','Male','British','White','NOMIS','GMI','DRS'; - exec CreateDummyCandidate '1CFG2308N','Clark','Parker','26/06/2004 00:00','A3961UA','B006501','TRUE','Male','British','White','NOMIS','EWI','ASP'; - END + exec CreateDummyCandidate '1CFG1109X','Barry','Stone','18/01/1979 00:00','A7022BA','B008622','TRUE','Male','British','White','DELIUS','ACI','C06'; + exec CreateDummyCandidate '1CFG3305M','Clark','Queen','16/07/1985 00:00','A7373XA','B004786','TRUE','Female','British','White','DELIUS','HII','N28'; + exec CreateDummyCandidate '1CFG3647Z','Hal','Parker','31/08/2000 00:00','A1118OA','B006754','TRUE','Male','British','Black','DELIUS','KMI','LCS'; + exec CreateDummyCandidate '1CFG4819M','Arthur','Stone','08/03/1961 00:00','A6972XA','B008866','TRUE','Female','British','White','NOMIS','LPI','MRS'; + exec CreateDummyCandidate '1CFG3977A','Oliver','Kent','16/08/1966 00:00','A3749HA','B004529','FALSE','Male','British','White','NOMIS','MRI','MCG'; + exec CreateDummyCandidate '1CFG4580T','Barry','Jordan','18/06/1979 00:00','A6102NA','B007498','FALSE','Male','British','Black','NOMIS','RSI','N50'; + exec CreateDummyCandidate '1CFG7754K','Victor','Kent','21/10/1956 00:00','A2817BA','B003913','FALSE','Male','British','White','NOMIS','PNI','CMB'; + exec CreateDummyCandidate '1CFG1873Q','Arthur','Wayne','15/08/1988 00:00','A8654XA','B004044','TRUE','Male','British','White','NOMIS','BAI','C17'; + exec CreateDummyCandidate '1CFG5221C','Victor','Parker','14/09/1977 00:00','A3576SA','B003506','TRUE','Male','British','White','DELIUS','BXI','LDN'; + exec CreateDummyCandidate '1CFG5006P','Clark','Kent','09/09/1988 00:00','A8065EA','B001407','TRUE','Female','British','White','DELIUS','FMI','N07'; + exec CreateDummyCandidate '1CFG2001T','Oliver','Jordan','30/09/2004 00:00','A6587YA','B003237','TRUE','Female','British','Black','NOMIS','ISI','N21'; + exec CreateDummyCandidate '1CFG5437L','Hal','Parker','25/09/1956 00:00','A6952ZA','B003171','TRUE','Male','British','White','DELIUS','HLI','N55'; + exec CreateDummyCandidate '1CFG2883L','Oliver','Prince','29/08/1961 00:00','A3402IA','B007887','TRUE','Male','British','White','DELIUS','HMI','HBS'; + exec CreateDummyCandidate '1CFG2167R','Bruce','Allen','27/08/1969 00:00','A7364GA','B009234','TRUE','Female','British','White','NOMIS','LEI','YSW'; + exec CreateDummyCandidate '1CFG9064R','Bruce','Jordan','19/07/1982 00:00','A6222SA','B004752','TRUE','Male','British','White','NOMIS','NHI','C05'; + exec CreateDummyCandidate '1CFG1900I','Oliver','Curry','30/06/1983 00:00','A9293CA','B005270','TRUE','Male','British','White','NOMIS','LHI','C09'; + exec CreateDummyCandidate '1CFG2847D','Bruce','Jordan','17/09/1991 00:00','A5884JA','B005459','TRUE','Female','British','Black','DELIUS','BMI','N52'; + exec CreateDummyCandidate '1CFG7892B','Arthur','Wayne','22/08/1952 00:00','A9162QA','B006933','TRUE','Male','British','White','DELIUS','DGI','N30'; + exec CreateDummyCandidate '1CFG6906H','Diana','Curry','09/08/1980 00:00','A6150IA','B005260','TRUE','Male','British','White','NOMIS','DHI','SWM'; + exec CreateDummyCandidate '1CFG1479H','Peter','Prince','18/07/1956 00:00','A6948JA','B004442','TRUE','Female','British','White','NOMIS','HEI','WMP'; + exec CreateDummyCandidate '1CFG7788V','Diana','Lance','27/10/1953 00:00','A9649AB','B004219','TRUE','Male','British','White','NOMIS','SFI','C11'; + exec CreateDummyCandidate '1CFG1553F','Victor','Jordan','08/12/1978 00:00','A9860RA','B008576','TRUE','Female','British','White','DELIUS','FHI','DBS'; + exec CreateDummyCandidate '1CFG4867Q','Oliver','Jordan','03/02/2000 00:00','A1567LA','B008340','TRUE','Male','British','Black','NOMIS','FWI','N31'; + exec CreateDummyCandidate '1CFG5469L','Peter','Queen','18/11/1953 00:00','A4692DA','B001257','TRUE','Male','British','White','NOMIS','LII','LNS'; + exec CreateDummyCandidate '1CFG5502H','Bruce','Kent','11/02/1978 00:00','A1411YA','B009194','TRUE','Male','British','Asian','DELIUS','WLI','NSP'; + exec CreateDummyCandidate '1CFG2825E','Peter','Prince','23/12/1975 00:00','A7907DA','B009268','TRUE','Female','British','White','DELIUS','PBI','CBS'; + exec CreateDummyCandidate '1CFG8516T','Dinah','Kent','19/06/1954 00:00','A7400NA','B009761','TRUE','Female','British','White','NOMIS','NWI','N56'; + exec CreateDummyCandidate '1CFG5298O','Victor','Jordan','28/07/1961 00:00','A8430TA','B007256','TRUE','Male','British','White','NOMIS','MTI','HFS'; + exec CreateDummyCandidate '1CFG9775P','Barry','Stone','14/08/1960 00:00','A1067SA','B008155','TRUE','Female','British','White','DELIUS','LNI','N02'; + exec CreateDummyCandidate '1CFG6849L','Bruce','Parker','20/02/1977 00:00','A7483AB','B009597','TRUE','Male','British','White','DELIUS','NLI','NBR'; + exec CreateDummyCandidate '1CFG8347A','Clark','Queen','06/10/2005 00:00','A1056JA','B005305','TRUE','Female','British','White','NOMIS','KVI','N23'; + exec CreateDummyCandidate '1CFG4189E','Bruce','Curry','14/04/1956 00:00','A3064CA','B006480','TRUE','Female','British','Black','NOMIS','HHI','N32'; + exec CreateDummyCandidate '1CFG5514M','Clark','Curry','28/05/1962 00:00','A7236EA','B005528','TRUE','Female','British','White','NOMIS','DMI','N54'; + exec CreateDummyCandidate '1CFG5252T','Peter','Queen','23/09/1964 00:00','A4570WA','B008719','TRUE','Female','British','White','DELIUS','SDI','SSP'; + exec CreateDummyCandidate '1CFG6789E','Clark','Parker','24/10/1968 00:00','A6440GA','B002584','TRUE','Male','British','White','DELIUS','SLI','N35'; + exec CreateDummyCandidate '1CFG1848U','Barry','Jordan','09/06/1981 00:00','A7126QA','B004852','TRUE','Female','British','White','DELIUS','SPI','N37'; + exec CreateDummyCandidate '1CFG4892S','Arthur','Allen','09/10/1960 00:00','A6249NA','B008270','TRUE','Male','British','White','NOMIS','WHI','N59'; + exec CreateDummyCandidate '1CFG5866O','Peter','Jordan','07/10/2001 00:00','A9186BA','B008561','TRUE','Female','British','Black','NOMIS','CWI','C19'; + exec CreateDummyCandidate '1CFG7842D','Diana','Queen','15/07/1982 00:00','A2699ZA','B005033','TRUE','Female','British','White','NOMIS','DAI','DCP'; + exec CreateDummyCandidate '1CFG8262O','Peter','Curry','02/07/1984 00:00','A2865UA','B001022','TRUE','Male','British','White','NOMIS','BLI','C15'; + exec CreateDummyCandidate '1CFG6353O','Hal','Stone','15/01/1952 00:00','A4973LA','B002433','TRUE','Male','British','White','NOMIS','GMI','DRS'; + exec CreateDummyCandidate '1CFG2308N','Clark','Parker','26/06/2004 00:00','A3961UA','B006501','TRUE','Male','British','White','NOMIS','EWI','ASP'; END ELSE BEGIN diff --git a/src/FakeDataSeeder/appsettings.json b/src/FakeDataSeeder/appsettings.json index e0c297d..6dcc6b4 100644 --- a/src/FakeDataSeeder/appsettings.json +++ b/src/FakeDataSeeder/appsettings.json @@ -1,3 +1,25 @@ { - "ConnectionStrings:ClusterDb": "" + "ConnectionStrings": { + "ClusterDb": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/FakeDataSeeder-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + } } diff --git a/src/FileSync/FileSync.csproj b/src/FileSync/FileSync.csproj index 951977c..205d6da 100644 --- a/src/FileSync/FileSync.csproj +++ b/src/FileSync/FileSync.csproj @@ -7,12 +7,6 @@ 8317f847-53e8-4625-a75d-5aab845b6f17 - - - Always - - - @@ -25,4 +19,13 @@ + + + Always + + + Always + + + diff --git a/src/FileSync/FileSyncBackgroundService.cs b/src/FileSync/FileSyncBackgroundService.cs index 8488800..4686966 100644 --- a/src/FileSync/FileSyncBackgroundService.cs +++ b/src/FileSync/FileSyncBackgroundService.cs @@ -1,4 +1,3 @@ -using System.Globalization; using System.IO.Compression; using System.Text.RegularExpressions; using FileStorage; @@ -19,16 +18,12 @@ namespace FileSync; public class FileSyncBackgroundService( ILogger logger, - IMatchingMessagingService matchingMessagingService, - IDbMessagingService dbMessagingService, - IStagingMessagingService stagingMessagingService, - IStatusMessagingService statusMessagingService, + IMessageService messageService, IFileLocations fileLocations, FileSource fileSource, IOptions syncOptions) : BackgroundService, IDisposable { private Timer? timer = null; - private readonly SemaphoreSlim _lock = new(1, 1); protected override async Task ExecuteAsync(CancellationToken stoppingToken) @@ -39,11 +34,9 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { if (syncOptions.Value.ProcessOnCompletion) { - matchingMessagingService.MatchingSubscribeAsync( - async msg => - { - await ProcessMessageAsync(msg, stoppingToken); - }, TMatchingQueue.ClusteringPostProcessingFinished); + await messageService.SubscribeAsync( + async msg => await ProcessMessageAsync(msg, stoppingToken), + TMatchingQueue.ClusteringPostProcessingFinished); } if (syncOptions.Value is { ProcessOnTimer: true, ProcessTimerIntervalSeconds: > 0 }) @@ -153,8 +146,8 @@ async Task ProcessAsync(CancellationToken cancellationToken = default) logger.LogInformation($"Targeting: Delius file ({unprocessedDeliusFile.Name}), Offloc file ({unprocessedOfflocFile.Name})"); - await stagingMessagingService.StagingPublishAsync(new DeliusDownloadFinishedMessage(unprocessedDeliusFile.Name, unprocessedDeliusFile.GetFileId())); - await stagingMessagingService.StagingPublishAsync(new OfflocDownloadFinished(unprocessedOfflocFile.Name, unprocessedOfflocFile.GetFileId()!.Value, unprocessedOfflocFile.ParentArchiveName)); + await messageService.PublishAsync(new DeliusDownloadFinishedMessage(unprocessedDeliusFile.Name, unprocessedDeliusFile.GetFileId())); + await messageService.PublishAsync(new OfflocDownloadFinished(unprocessedOfflocFile.Name, unprocessedOfflocFile.GetFileId()!.Value, unprocessedOfflocFile.ParentArchiveName)); } finally { @@ -165,14 +158,14 @@ async Task ProcessAsync(CancellationToken cancellationToken = default) private async Task IsOfflocReady() { - var response = await dbMessagingService.SendDbRequestAndWaitForResponseAsync(new()); - return response.isReady; + var response = await messageService.SendDbRequestAndWaitForResponseAsync(new CheckOfflocReadyRequest()); + return response.IsReady; } private async Task IsDeliusReady() { - var response = await dbMessagingService.SendDbRequestAndWaitForResponseAsync(new()); - return response.isReady; + var response = await messageService.SendDbRequestAndWaitForResponseAsync(new CheckDeliusReadyRequest()); + return response.IsReady; } private async Task GetNextUnprocessedOfflocFileAsync(CancellationToken cancellationToken = default) @@ -184,12 +177,12 @@ private async Task IsDeliusReady() // Get already processed files logger.LogInformation("Retrieving processed Offloc files..."); - var response = await dbMessagingService.SendDbRequestAndWaitForResponseAsync(new GetOfflocFilesMessage()); - logger.LogInformation($"Retrieved {response.offlocFiles.Length} processed Offloc file(s)."); + var response = await messageService.SendDbRequestAndWaitForResponseAsync(new GetOfflocFilesRequest()); + logger.LogInformation($"Retrieved {response.OfflocFiles.Length} processed Offloc file(s)."); // Find latest unprocessed file var unprocessedOfflocFiles = offlocFileStore - .ExceptBy(response.offlocFiles, file => Path.GetFileName(file.Name)) // Excludes processed + .ExceptBy(response.OfflocFiles, file => Path.GetFileName(file.Name)) // Excludes processed .OrderBy(f => f.GetDatestamp()) .ToList(); @@ -230,11 +223,15 @@ private async Task IsDeliusReady() logger.LogInformation("Extracted Offloc file: " + file); // We have already processed the contents of the archive, get the next unprocessed file. - if(response.offlocFiles.Contains(file)) + if(response.OfflocFiles.Contains(file)) { // Associate the zip with the file name logger.LogWarning("Already processed contents of archive: " + file); - await dbMessagingService.SendDbRequestAndWaitForResponseAsync(new AssociateOfflocFileWithArchiveMessage(file, Path.GetFileName(downloadedFile))); + await messageService.SendDbRequestAndWaitForResponseAsync(new AssociateOfflocFileWithArchiveRequest + { + FileName = file, + ArchiveName = Path.GetFileName(downloadedFile) + }); // Remove the extracted file from the input directory - we have already processed it! File.Delete(filePath); @@ -252,12 +249,12 @@ private async Task IsDeliusReady() // Get already processed files logger.LogInformation("Retrieving processed Delius files..."); - var response = await dbMessagingService.SendDbRequestAndWaitForResponseAsync(new GetDeliusFilesMessage()); - logger.LogInformation($"Retrieved {response.fileNames.Length} processed Delius file(s)."); + var response = await messageService.SendDbRequestAndWaitForResponseAsync(new GetDeliusFilesRequest()); + logger.LogInformation($"Retrieved {response.FileNames.Length} processed Delius file(s)."); // Return unprocessed files var unprocessedDeliusFiles = deliusFileStore - .ExceptBy(response.fileNames, file => Path.GetFileName(file.Name)) + .ExceptBy(response.FileNames, file => Path.GetFileName(file.Name)) .OrderBy(file => file.GetDatestamp()) .ToList(); @@ -279,17 +276,17 @@ private async Task IsDeliusReady() private async Task PreKickoffTasks() { - LogStatus("Publishing pre-kickoff messages..."); - await stagingMessagingService.StagingPublishAsync(new ClearHalfCleanedOfflocFiles()); - await stagingMessagingService.StagingPublishAsync(new ClearTemporaryDeliusFiles()); + await LogStatus("Publishing pre-kickoff messages..."); + await messageService.PublishAsync(new ClearHalfCleanedOfflocFiles()); + await messageService.PublishAsync(new ClearTemporaryDeliusFiles()); - LogStatus("Pre-kickoff messages published. Beginning staging database tear down..."); - await dbMessagingService.SendDbRequestAndWaitForResponseAsync(new ClearDeliusStaging()); - await dbMessagingService.SendDbRequestAndWaitForResponseAsync(new ClearOfflocStaging()); - LogStatus("Staging database tear down complete."); + await LogStatus("Pre-kickoff messages published. Beginning staging database tear down..."); + await messageService.SendDbRequestAndWaitForResponseAsync(new ClearDeliusStagingRequest()); + await messageService.SendDbRequestAndWaitForResponseAsync(new ClearOfflocStagingRequest()); + await LogStatus("Staging database tear down complete."); // Delete any files in input directories - LogStatus("Clearing input directories..."); + await LogStatus("Clearing input directories..."); int counter = 0; foreach (var file in Directory.GetFiles(fileLocations.deliusInput)) @@ -304,13 +301,13 @@ private async Task PreKickoffTasks() counter++; } - LogStatus("Input directories cleared. Deleted " + counter + " file(s)."); + await LogStatus("Input directories cleared. Deleted " + counter + " file(s)."); } async Task LogStatus(string message) { logger.LogInformation(message); - await statusMessagingService.StatusPublishAsync(new StatusUpdateMessage(message)); + await messageService.PublishAsync(new StatusUpdateMessage(message)); } public override void Dispose() @@ -320,11 +317,11 @@ public override void Dispose() public async Task IsDeliusFileNewerThanLastProcessed(DeliusFile unprocessedDeliusFile, CancellationToken cancellationToken = default) { - var lastProcessedDeliusFileName = await dbMessagingService.SendDbRequestAndWaitForResponseAsync(new()); + var lastProcessedDeliusFileName = await messageService.SendDbRequestAndWaitForResponseAsync(new GetLastProcessedDeliusFileRequest()); - if(!string.IsNullOrEmpty(lastProcessedDeliusFileName.fileName)) + if(!string.IsNullOrEmpty(lastProcessedDeliusFileName.FileName)) { - var file = new DeliusFile(lastProcessedDeliusFileName.fileName); + var file = new DeliusFile(lastProcessedDeliusFileName.FileName); if(file.GetDatestamp() >= unprocessedDeliusFile.GetDatestamp()) { @@ -338,11 +335,11 @@ public async Task IsDeliusFileNewerThanLastProcessed(DeliusFile unprocesse public async Task IsOfflocFileNewerThanLastProcessed(OfflocFile unprocessedOfflocFile, CancellationToken cancellationToken = default) { - var lastProcessedOfflocFileName = await dbMessagingService.SendDbRequestAndWaitForResponseAsync(new()); + var lastProcessedOfflocFileName = await messageService.SendDbRequestAndWaitForResponseAsync(new GetLastProcessedOfflocFileRequest()); - if(!string.IsNullOrEmpty(lastProcessedOfflocFileName.fileName)) + if(!string.IsNullOrEmpty(lastProcessedOfflocFileName.FileName)) { - var file = new OfflocFile(lastProcessedOfflocFileName.fileName); + var file = new OfflocFile(lastProcessedOfflocFileName.FileName); if(file.GetDatestamp() >= unprocessedOfflocFile.GetDatestamp()) { diff --git a/src/FileSync/Program.cs b/src/FileSync/Program.cs index 8540cf8..47e2c3d 100644 --- a/src/FileSync/Program.cs +++ b/src/FileSync/Program.cs @@ -1,26 +1,11 @@ -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@"./logs/fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); +using Messaging.Extensions; try { var builder = Host.CreateApplicationBuilder(args); - builder.Configuration.ConfigureByEnvironment(); - - builder.Services.ConfigureServices(builder.Configuration); - - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); builder.Services.AddOptions().BindConfiguration("SyncOptions"); diff --git a/src/FileSync/appsettings.Development.json b/src/FileSync/appsettings.Development.json new file mode 100644 index 0000000..bd71a73 --- /dev/null +++ b/src/FileSync/appsettings.Development.json @@ -0,0 +1,6 @@ +{ + "FileSourceProvider": "MinIO", + "SyncOptions": { + "ProcessTimerIntervalSeconds": 30 + } +} \ No newline at end of file diff --git a/src/FileSync/appsettings.json b/src/FileSync/appsettings.json index 3ee8180..4cdb7bd 100644 --- a/src/FileSync/appsettings.json +++ b/src/FileSync/appsettings.json @@ -1,12 +1,35 @@ { + "ConnectionStrings": { + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/FileSync-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, "SyncOptions": { - "ProcessOnStartup": true, + "ProcessOnStartup": false, "ProcessOnCompletion": true, "ProcessOnTimer": true, - "ProcessTimerIntervalSeconds": 30, + "ProcessTimerIntervalSeconds": 900, "AllowProcessingOlderFiles": false }, - "FileSourceProvider": "MinIO", // S3, FileSystem, MinIO + "FileSourceProvider": "S3", // S3, FileSystem, MinIO // Required if using 'S3' file source provider "AWS": { "Region": "eu-west-2", diff --git a/src/Import/Import.csproj b/src/Import/Import.csproj index dfbab5e..71c2290 100644 --- a/src/Import/Import.csproj +++ b/src/Import/Import.csproj @@ -4,8 +4,6 @@ Exe enable enable - Linux - ..\..\.. diff --git a/src/Import/ImportBackgroundService.cs b/src/Import/ImportBackgroundService.cs index 407aa09..f7a1082 100644 --- a/src/Import/ImportBackgroundService.cs +++ b/src/Import/ImportBackgroundService.cs @@ -3,76 +3,52 @@ using Messaging.Messages.DbMessages.Sending; using Messaging.Messages.ImportMessages; using Messaging.Messages.StagingMessages; -using Messaging.Messages.StatusMessages; using Messaging.Queues; using Microsoft.Extensions.Hosting; namespace Import; -public class ImportBackgroundService : BackgroundService +public class ImportBackgroundService(IMessageService messageService) : BackgroundService { - private readonly IStagingMessagingService messageService; - private readonly IStatusMessagingService statusService; - private readonly IDbMessagingService dbService; - private readonly IImportMessagingService importMessagingService; - - //Uses semaphore like a lock/monitor. This is because locks/monitors - //don't work in an asynchronous context. - private static SemaphoreSlim offlocSem = new SemaphoreSlim(1, 1); - private static SemaphoreSlim deliusSem = new SemaphoreSlim(1, 1); + private static readonly SemaphoreSlim offlocSem = new(1, 1); + private static readonly SemaphoreSlim deliusSem = new(1, 1); private bool deliusParserCompleted; private bool offlocParserCompleted; - private bool deliusFileEmpty; private bool offlocFileEmpty; - private bool[] parserStates => [deliusParserCompleted, offlocParserCompleted]; - private bool[] filesEmpty => [deliusFileEmpty, offlocFileEmpty]; - - public ImportBackgroundService(IStagingMessagingService messageService, IStatusMessagingService statusService, IDbMessagingService dbService, IImportMessagingService importMessagingService) - { - this.dbService = dbService; - this.messageService = messageService; - this.statusService = statusService; - this.importMessagingService=importMessagingService; -} + private bool[] ParserStates => [deliusParserCompleted, offlocParserCompleted]; + private bool[] FilesEmpty => [deliusFileEmpty, offlocFileEmpty]; protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await Task.Run(() => - { - messageService.StagingSubscribeAsync(async (message) => - { - await OnOfflocMessageReceived(message); - }, TStagingQueue.OfflocImport); + await messageService.SubscribeAsync( + async message => await OnOfflocMessageReceived(message), + TStagingQueue.OfflocImport); - messageService.StagingSubscribeAsync(async (message) => - { - await OnDeliusMessageReceived(message); - }, TStagingQueue.DeliusImport); - - }, stoppingToken); + await messageService.SubscribeAsync( + async message => await OnDeliusMessageReceived(message), + TStagingQueue.DeliusImport); } private async Task OnDeliusMessageReceived(DeliusParserFinishedMessage message) { await deliusSem.WaitAsync(); - if (message.emptyFile) + if (message.EmptyFile) { deliusFileEmpty = true; - await statusService.StatusPublishAsync(new StatusUpdateMessage($"No Delius File to import (staging or merging)")); } else { - await statusService.StatusPublishAsync(new StatusUpdateMessage($"Importing Delius File.....")); - var res = await dbService.SendDbRequestAndWaitForResponseAsync(new StageDeliusMessage(message.fileName, message.filePath)); - var msg = await dbService.SendDbRequestAndWaitForResponseAsync(new MergeDeliusRunningPictureMessage(message.fileName)); + await messageService.SendDbRequestAndWaitForResponseAsync( + new StageDeliusRequest { FileName = message.FileName, FilePath = message.FilePath }); + await messageService.SendDbRequestAndWaitForResponseAsync( + new MergeDeliusRequest { FileName = message.FileName }); } deliusSem.Release(); - deliusParserCompleted = true; await CheckStates(); } @@ -81,43 +57,33 @@ private async Task OnOfflocMessageReceived(OfflocParserFinishedMessage message) { await offlocSem.WaitAsync(); - if (message.emptyFile) + if (message.EmptyFile) { offlocFileEmpty = true; - await statusService.StatusPublishAsync(new StatusUpdateMessage($"No Offloc File to import (staging or merging)")); } else { - await statusService.StatusPublishAsync(new StatusUpdateMessage($"Importing Offloc File.....")); - var res = await dbService.SendDbRequestAndWaitForResponseAsync(new StageOfflocMessage(message.filePath)); - var msg = await dbService.SendDbRequestAndWaitForResponseAsync(new MergeOfflocRunningPictureMessage(Path.GetFileName(message.filePath))); + await messageService.SendDbRequestAndWaitForResponseAsync( + new StageOfflocRequest { FileName = message.FilePath }); + await messageService.SendDbRequestAndWaitForResponseAsync( + new MergeOfflocRequest { FileName = Path.GetFileName(message.FilePath) }); } + offlocSem.Release(); offlocParserCompleted = true; - await CheckStates(); } - //Purely for status updates. private async Task CheckStates() { - await Task.CompletedTask; - - if (parserStates.All(b => b == true)) + if (ParserStates.All(b => b)) { deliusParserCompleted = false; offlocParserCompleted = false; - if (filesEmpty.All(b => b == true)) - { - await statusService.StatusPublishAsync(new StatusUpdateMessage($"no Files imported")); - } - else + if (!FilesEmpty.All(b => b)) { - //Clear staging Db after 10 minutes for testing purposes. - //await Task.Delay(new TimeSpan(0, 10, 0)); - await statusService.StatusPublishAsync(new StatusUpdateMessage($"Files imported")); - await importMessagingService.ImportPublishAsync(new ImportFinishedMessage()); + await messageService.PublishAsync(new ImportFinishedMessage()); } } } diff --git a/src/Import/Program.cs b/src/Import/Program.cs index 9c98f74..795bb20 100644 --- a/src/Import/Program.cs +++ b/src/Import/Program.cs @@ -1,35 +1,18 @@ -using Import; -using Messaging.Interfaces; -using Messaging.Services; +using Messaging.Extensions; +using Import; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using EnvironmentSetup; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); + var builder = Host.CreateApplicationBuilder(args); - builder.Configuration.AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); - builder.Services.ConfigureServices(builder.Configuration); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); var app = builder.Build(); diff --git a/src/Import/Properties/launchSettings.json b/src/Import/Properties/launchSettings.json deleted file mode 100644 index 3369a71..0000000 --- a/src/Import/Properties/launchSettings.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "profiles": { - "Import": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development", - "SYSTEM_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Project", - "environmentVariables": { - "SYSTEM_ENVIRONMENT": "Development", - "DOTNET_ENVIRONMENT": "Development" - } - } - } -} \ No newline at end of file diff --git a/src/Import/appsettings.json b/src/Import/appsettings.json index cd0c9a1..91ab3ed 100644 --- a/src/Import/appsettings.json +++ b/src/Import/appsettings.json @@ -1,6 +1,27 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Import-.txt", - "EnvFilePath": "../", + "ConnectionStrings": { + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/Import-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, "ServerConfiguration": { "deliusStagingProcedure": "dbo.StageDelius", "offlocStagingProcedure": "dbo.Import" diff --git a/src/Libraries/EnvironmentSetup/ConfigureEnvironment.cs b/src/Libraries/EnvironmentSetup/ConfigureEnvironment.cs deleted file mode 100644 index 865f6ca..0000000 --- a/src/Libraries/EnvironmentSetup/ConfigureEnvironment.cs +++ /dev/null @@ -1,108 +0,0 @@ -using DotNetEnv.Configuration; -using FileStorage; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.DependencyInjection; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; -using Serilog; - -namespace EnvironmentSetup; - -public static class ConfigureEnvironment -{ - private static string envFileBasePath = $"{AppContext.BaseDirectory}.."; - - public static ConfigurationManager ConfigureByEnvironment(this ConfigurationManager configManager) - { - Directory.SetCurrentDirectory(AppContext.BaseDirectory); - - configManager.SetDotnetEnvironment(); - - if (!configManager.GetValue("RUNNING_IN_CONTAINER")) - { - ConfigureEnv(configManager); - } - - return configManager; - } - - public static ConfigurationManager ConfigureEnv(this ConfigurationManager configManager) - { - string envFilePath = Path.Combine(envFileBasePath, "development.local.env"); - - if (File.Exists(envFilePath)) - { - configManager.AddDotNetEnv(envFilePath); - } - - return configManager; - } - - public static void SetDotnetEnvironment(this ConfigurationManager config) - { - if (config.GetValue("DOTNET_ENVIRONMENT") == null) - { - config["DOTNET_ENVIRONMENT"] = "Production"; - } - } -} - -public static class ServiceConfiguration -{ - public static IServiceCollection ConfigureServices(this IServiceCollection services, ConfigurationManager configManager) - { - services.AddSingleton( - f => new FileLocations(configManager.GetValue("DMSFilesBasePath")!) - ); - - services.AddWindowsService(); - - services.ConfigureRabbit(configManager); - services.ConfigureLogging(configManager); - - return services; - } - - private static IServiceCollection ConfigureRabbit(this IServiceCollection services, IConfiguration config) - { - RabbitHostingContextWrapper rabbit; - - // Prefer connection string if available - if (config.GetConnectionString("RabbitMQ") is { Length: > 0 } connectionString) - { - rabbit = new RabbitHostingContextWrapper(new Uri(connectionString)); - } - else - { - string? hosting = config.GetValue("RABBIT_HOSTING"); - string? username = config.GetValue("RABBIT_USERNAME"); - string? password = config.GetValue("RABBIT_PASSWORD"); - - rabbit = new RabbitHostingContextWrapper(hosting, username, password); - } - - services.AddSingleton(rabbit); - - return services; - } - - - private static IServiceCollection ConfigureLogging(this IServiceCollection services, IConfiguration config) - { - Log.Logger = new LoggerConfiguration() - .ReadFrom.Configuration(config) - .WriteTo.File(Path.Combine("logs", "fatal.txt"), Serilog.Events.LogEventLevel.Fatal) - .CreateLogger(); - - var loggerFactory = LoggerFactory.Create(builder => - { - builder.AddSerilog(dispose: true); - }); - - services.AddSingleton(loggerFactory); - services.AddLogging(); - - return services; - } - -} diff --git a/src/Libraries/EnvironmentSetup/DmsServiceExtensions.cs b/src/Libraries/EnvironmentSetup/DmsServiceExtensions.cs new file mode 100644 index 0000000..f669726 --- /dev/null +++ b/src/Libraries/EnvironmentSetup/DmsServiceExtensions.cs @@ -0,0 +1,93 @@ +using FileStorage; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Hosting.WindowsServices; +using Serilog; +using Serilog.Events; + +namespace EnvironmentSetup; + +public static class DmsServiceExtensions +{ + /// + /// Configures Serilog logging for all DMS applications. + /// Works for both ASP.NET Core apps and Worker Services. + /// Reads configuration from appsettings.json Serilog section. + /// Integrates with Sentry for error tracking. + /// + public static IHostApplicationBuilder UseDmsSerilog(this IHostApplicationBuilder builder) + { + builder.Services.AddSerilog(config => + { + config.ReadFrom.Configuration(builder.Configuration); + + if (builder.Configuration["SENTRY_DSN"] is { Length: > 0 } dsn) + { + config.WriteTo.Sentry(options => + { + options.Dsn = dsn; + options.Environment = builder.Environment.EnvironmentName; + options.TracesSampleRate = 0.1; + options.AttachStacktrace = true; + options.SendDefaultPii = true; + options.AutoSessionTracking = true; + options.MinimumEventLevel = LogEventLevel.Information; + options.MinimumBreadcrumbLevel = LogEventLevel.Information; + }); + } + }); + + return builder; + } + + /// + /// Configures the application to run as a Windows Service. + /// Required for all DMS worker services running on Windows Server EC2. + /// + public static IServiceCollection AddDmsWindowsService(this IServiceCollection services) + { + services.AddWindowsService(); + return services; + } + + /// + /// Configures file location paths for services that process files. + /// Required by: FileSync, Offloc.Parser, Offloc.Cleaner, DbInteractions, Delius.Parser, Cleanup. + /// + public static IServiceCollection AddDmsFileLocations(this IServiceCollection services, IConfiguration config) + { + services.AddSingleton( + _ => new FileLocations(config.GetValue("DMSFilesBasePath")!) + ); + + return services; + } + + /// + /// Configures common DMS worker service infrastructure: Serilog logging, Windows Service hosting, + /// and file locations. + /// NOTE: RabbitMQ must be configured separately using AddDmsRabbitMQ() from Messaging library. + /// This is a convenience method for the core DMS worker services. + /// + /// The host application builder + /// + /// Suitable for: FileSync, Offloc.Parser, Offloc.Cleaner, DbInteractions, Delius.Parser, Cleanup, Import, Blocking, Matching.Engine, Logging. + /// NOT suitable for: Meow (uses CATS RabbitMQ), API (ASP.NET Core), Visualiser (container). + /// + public static IHostApplicationBuilder AddDmsCoreWorkerService(this IHostApplicationBuilder builder) + { + // Ensure the current directory is set correctly when running as a Windows Service + // This ensures relative paths in appsettings.json work as expected (i.e. for Serilog log files) + if (WindowsServiceHelpers.IsWindowsService()) + { + Directory.SetCurrentDirectory(AppContext.BaseDirectory); + } + + builder.UseDmsSerilog(); + builder.Services.AddDmsWindowsService(); + builder.Services.AddDmsFileLocations(builder.Configuration); + + return builder; + } +} diff --git a/src/Libraries/EnvironmentSetup/EnvironmentSetup.csproj b/src/Libraries/EnvironmentSetup/EnvironmentSetup.csproj index 12a1f3e..2770135 100644 --- a/src/Libraries/EnvironmentSetup/EnvironmentSetup.csproj +++ b/src/Libraries/EnvironmentSetup/EnvironmentSetup.csproj @@ -6,7 +6,6 @@ - diff --git a/src/Libraries/EnvironmentSetup/RabbitHostingContextWrapper.cs b/src/Libraries/EnvironmentSetup/RabbitHostingContextWrapper.cs deleted file mode 100644 index 3c81bbe..0000000 --- a/src/Libraries/EnvironmentSetup/RabbitHostingContextWrapper.cs +++ /dev/null @@ -1,25 +0,0 @@ - -namespace EnvironmentSetup; - -public class RabbitHostingContextWrapper -{ - private const string defaultContext = "host.docker.internal"; - //Default value assumes docker. - public string Context { get; } = defaultContext; - public string Username { get; } - public string Password { get; } - - public Uri? Uri { get; } - - public RabbitHostingContextWrapper(Uri uri) : this(defaultContext) - { - Uri = uri; - } - - public RabbitHostingContextWrapper(string? hostingContext, string username = "guest", string password = "guest") - { - Context = hostingContext ?? defaultContext; - Username = username; - Password = password; - } -} diff --git a/src/Libraries/Infrastructure/DatabaseExtensions.cs b/src/Libraries/Infrastructure/DatabaseExtensions.cs index 8bcf7f0..af818c8 100644 --- a/src/Libraries/Infrastructure/DatabaseExtensions.cs +++ b/src/Libraries/Infrastructure/DatabaseExtensions.cs @@ -14,26 +14,33 @@ public static class DatabaseExtensions { public static IHostApplicationBuilder AddDatabaseServices(this IHostApplicationBuilder builder) { + if (!builder.Services.Any(x => x.ServiceType == typeof(ICurrentUserService))) + { + throw new InvalidOperationException( + "ICurrentUserService must be registered before calling AddDatabaseServices. " + + "Register an implementation using builder.Services.AddScoped()"); + } + builder.Services.AddScoped(); builder.Services.AddDbContext((sp, options) => { - options.UseSqlServer(builder.Configuration.GetConnectionString("AuditDb")); + options.UseSqlServer(GetRequiredConnectionString(builder.Configuration, "AuditDb")); }); builder.Services.AddDbContext((sp, options) => { - options.UseSqlServer(builder.Configuration.GetConnectionString("DeliusRunningPictureDb")); + options.UseSqlServer(GetRequiredConnectionString(builder.Configuration, "DeliusRunningPictureDb")); }); builder.Services.AddDbContext((sp, options) => { - options.UseSqlServer(builder.Configuration.GetConnectionString("OfflocRunningPictureDb")); + options.UseSqlServer(GetRequiredConnectionString(builder.Configuration, "OfflocRunningPictureDb")); }); builder.Services.AddDbContext((sp, options) => { - options.UseSqlServer(builder.Configuration.GetConnectionString("ClusterDb")); + options.UseSqlServer(GetRequiredConnectionString(builder.Configuration, "ClusterDb")); options.AddInterceptors(sp.GetRequiredService()); }); @@ -44,4 +51,7 @@ public static IHostApplicationBuilder AddDatabaseServices(this IHostApplicationB return builder; } + + private static string GetRequiredConnectionString(IConfiguration configuration, string name) => + configuration.GetConnectionString(name) ?? throw new InvalidOperationException($"Connection string '{name}' is not configured."); } diff --git a/src/Libraries/Messaging/Extensions/DmsRabbitMQExtensions.cs b/src/Libraries/Messaging/Extensions/DmsRabbitMQExtensions.cs new file mode 100644 index 0000000..72ee381 --- /dev/null +++ b/src/Libraries/Messaging/Extensions/DmsRabbitMQExtensions.cs @@ -0,0 +1,26 @@ +using Messaging.Interfaces; +using Messaging.Services; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace Messaging.Extensions; + +public static class DmsRabbitMQExtensions +{ + /// + /// Configures DMS RabbitMQ messaging for worker services. + /// Registers RabbitService and all messaging interfaces it implements. + /// + public static IServiceCollection AddDmsRabbitMQ(this IServiceCollection services, IConfiguration config) + { + services.AddSingleton(sp => + { + var connectionString = config.GetConnectionString("RabbitMQ")!; + return RabbitService.CreateAsync(new Uri(connectionString)).GetAwaiter().GetResult(); + }); + + services.AddSingleton(sp => sp.GetRequiredService()); + + return services; + } +} diff --git a/src/Libraries/Messaging/Interfaces/IBlockingMessagingService.cs b/src/Libraries/Messaging/Interfaces/IBlockingMessagingService.cs deleted file mode 100644 index d101734..0000000 --- a/src/Libraries/Messaging/Interfaces/IBlockingMessagingService.cs +++ /dev/null @@ -1,11 +0,0 @@ -using Messaging.Messages.BlockingMessages; -using Messaging.Queues; - -namespace Messaging.Interfaces -{ - public interface IBlockingMessagingService - { - Task BlockingPublishAsync(T message) where T : BlockingMessage; - Task BlockingSubscribeAsync(Action handler, TBlockingQueue queue) where T : BlockingMessage; - } -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Interfaces/IDbMessagingService.cs b/src/Libraries/Messaging/Interfaces/IDbMessagingService.cs deleted file mode 100644 index 4116bd4..0000000 --- a/src/Libraries/Messaging/Interfaces/IDbMessagingService.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Messaging.Messages.DbMessages.Receiving; -using Messaging.Messages.DbMessages.Sending; -using Messaging.Queues; - -namespace Messaging.Interfaces; - -public interface IDbMessagingService -{ - Task DbPublishResponseAsync(T message) where T : DbResponseMessage; - Task SubscribeToDbRequestAsync(Action handler, TDbQueue queue) where T : DbRequestMessage; - Task SendDbRequestAndWaitForResponseAsync(T message) where T : DbRequestMessage where T2 : DbResponseMessage; -} diff --git a/src/Libraries/Messaging/Interfaces/IImportMessagingService.cs b/src/Libraries/Messaging/Interfaces/IImportMessagingService.cs deleted file mode 100644 index 27f15d0..0000000 --- a/src/Libraries/Messaging/Interfaces/IImportMessagingService.cs +++ /dev/null @@ -1,11 +0,0 @@ -using Messaging.Messages.ImportMessages; -using Messaging.Queues; - -namespace Messaging.Interfaces -{ - public interface IImportMessagingService - { - Task ImportPublishAsync(T message) where T : ImportMessage; - Task ImportSubscribeAsync(Action handler, TImportQueue queue) where T : ImportMessage; - } -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Interfaces/IMatchingMessagingService.cs b/src/Libraries/Messaging/Interfaces/IMatchingMessagingService.cs deleted file mode 100644 index 3da1c9a..0000000 --- a/src/Libraries/Messaging/Interfaces/IMatchingMessagingService.cs +++ /dev/null @@ -1,11 +0,0 @@ -using Messaging.Messages.MatchingMessages; -using Messaging.Queues; - -namespace Messaging.Interfaces -{ - public interface IMatchingMessagingService - { - Task MatchingPublishAsync(T message) where T : MatchingMessage; - Task MatchingSubscribeAsync(Action handler, TMatchingQueue queue) where T : MatchingMessage; - } -} diff --git a/src/Libraries/Messaging/Interfaces/IMergingMessagingService.cs b/src/Libraries/Messaging/Interfaces/IMergingMessagingService.cs deleted file mode 100644 index de24319..0000000 --- a/src/Libraries/Messaging/Interfaces/IMergingMessagingService.cs +++ /dev/null @@ -1,11 +0,0 @@ - -using Messaging.Messages.MergingMessages; -using Messaging.Queues; - -namespace Messaging.Interfaces; - -public interface IMergingMessagingService -{ - Task MergingPublishAsync(T message) where T : MergingMessage; - Task MergingSubscribeAsync(Action handler, TMergingQueue queue) where T : MergingMessage; -} diff --git a/src/Libraries/Messaging/Interfaces/IMessageService.cs b/src/Libraries/Messaging/Interfaces/IMessageService.cs index 05adabf..b6bb5fa 100644 --- a/src/Libraries/Messaging/Interfaces/IMessageService.cs +++ b/src/Libraries/Messaging/Interfaces/IMessageService.cs @@ -1,7 +1,12 @@ -namespace Messaging.Interfaces; +using Messaging.Messages; +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.DbMessages.Sending; -//This interface is defined assuming each queue will only have 1 type of messages. -//Perhaps should be split up into mutliple pub/sub interfaces. -public interface IMessageService : IStagingMessagingService, IMergingMessagingService, IStatusMessagingService, - IDbMessagingService, IImportMessagingService, IBlockingMessagingService, IMatchingMessagingService -{ } +namespace Messaging.Interfaces; + +public interface IMessageService +{ + Task PublishAsync(T message) where T : IMessage; + Task SubscribeAsync(Action handler, Enum queue) where T : IMessage; + Task SendDbRequestAndWaitForResponseAsync(DbRequestMessage message) where TResponse : DbResponseMessage, new(); +} \ No newline at end of file diff --git a/src/Libraries/Messaging/Interfaces/IStagingMessagingService.cs b/src/Libraries/Messaging/Interfaces/IStagingMessagingService.cs deleted file mode 100644 index bf6c29c..0000000 --- a/src/Libraries/Messaging/Interfaces/IStagingMessagingService.cs +++ /dev/null @@ -1,11 +0,0 @@ - -using Messaging.Messages.StagingMessages; -using Messaging.Queues; - -namespace Messaging.Interfaces; - -public interface IStagingMessagingService -{ - Task StagingPublishAsync(T message) where T : StagingMessage; - Task StagingSubscribeAsync(Action handler, TStagingQueue queue) where T : StagingMessage; -} diff --git a/src/Libraries/Messaging/Interfaces/IStatusMessagingService.cs b/src/Libraries/Messaging/Interfaces/IStatusMessagingService.cs deleted file mode 100644 index 0de901e..0000000 --- a/src/Libraries/Messaging/Interfaces/IStatusMessagingService.cs +++ /dev/null @@ -1,11 +0,0 @@ - -using Messaging.Messages.StatusMessages; -using Messaging.Queues; - -namespace Messaging.Interfaces; - -public interface IStatusMessagingService -{ - Task StatusPublishAsync(T Message) where T : StatusUpdateMessage; - Task StatusSubscribeAsync(Action handler, TStatusQueue queue) where T : StatusUpdateMessage; -} diff --git a/src/Libraries/Messaging/Messages/BlockingMessages/BlockingFinishedMessage.cs b/src/Libraries/Messaging/Messages/BlockingMessages/BlockingFinishedMessage.cs index a70a4de..9558500 100644 --- a/src/Libraries/Messaging/Messages/BlockingMessages/BlockingFinishedMessage.cs +++ b/src/Libraries/Messaging/Messages/BlockingMessages/BlockingFinishedMessage.cs @@ -13,6 +13,6 @@ public class BlockingFinishedMessage : BlockingMessage [JsonConstructor] public BlockingFinishedMessage() { - routingKey = TBlockingQueue.BlockingFinished; + Queue = TBlockingQueue.BlockingFinished; } } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/BlockingMessages/BlockingMessage.cs b/src/Libraries/Messaging/Messages/BlockingMessages/BlockingMessage.cs index fa91315..3924037 100644 --- a/src/Libraries/Messaging/Messages/BlockingMessages/BlockingMessage.cs +++ b/src/Libraries/Messaging/Messages/BlockingMessages/BlockingMessage.cs @@ -4,7 +4,10 @@ namespace Messaging.Messages.BlockingMessages { public abstract class BlockingMessage : Message { - public TBlockingQueue routingKey { get; set; } - public string fileName = string.Empty; //To handle processing of multiple files. + public TBlockingQueue Queue { get; set; } + public string FileName { get; set; } = string.Empty; + + public override string Exchange => Exchanges.blocking; + public override string RoutingKey => Queue.ToString(); } } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/DbMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/DbMessage.cs index cac4ff2..007e744 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/DbMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/DbMessage.cs @@ -5,5 +5,8 @@ namespace Messaging.Messages.DbMessages; public abstract class DbMessage : Message { - public TDbQueue Queue { get; set; } + public virtual TDbQueue Queue { get; set; } + + public override string Exchange => Exchanges.database; + public override string RoutingKey => Queue.ToString(); } diff --git a/src/Libraries/Messaging/Messages/DbMessages/OfflocFileIdReturnMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/OfflocFileIdReturnMessage.cs deleted file mode 100644 index 85d499f..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/OfflocFileIdReturnMessage.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Messaging.Messages.DbMessages.Receiving; -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages; - -public class OfflocFileIdReturnMessage : DbResponseMessage -{ - public DateOnly[] fileDates = Array.Empty(); - - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage("Processed Offloc files returned."); - - [JsonConstructor] - public OfflocFileIdReturnMessage() - { - Queue = TDbQueue.ReturnedOfflocFileDates; - } - - public OfflocFileIdReturnMessage(DateOnly[] fileIds) : this() - { - this.fileDates = fileIds; - } -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultAssociateOfflocFileWithArchiveMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/AssociateOfflocFileWithArchiveResponse.cs similarity index 51% rename from src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultAssociateOfflocFileWithArchiveMessage.cs rename to src/Libraries/Messaging/Messages/DbMessages/Receiving/AssociateOfflocFileWithArchiveResponse.cs index 550dc1d..e31df4a 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultAssociateOfflocFileWithArchiveMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/AssociateOfflocFileWithArchiveResponse.cs @@ -3,12 +3,8 @@ namespace Messaging.Messages.DbMessages.Receiving; -public class ResultAssociateOfflocFileWithArchiveMessage : DbResponseMessage +public class AssociateOfflocFileWithArchiveResponse : DbResponseMessage { public override StatusUpdateMessage StatusMessage => new("Offloc file associated with archive"); - - public ResultAssociateOfflocFileWithArchiveMessage() - { - Queue = TDbQueue.ResultAssociateOfflocFileWithArchive; - } + public override TDbQueue Queue { get; set; } = TDbQueue.ResultAssociateOfflocFileWithArchive; } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/CheckDeliusReadyResponse.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/CheckDeliusReadyResponse.cs new file mode 100644 index 0000000..f594ac4 --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/CheckDeliusReadyResponse.cs @@ -0,0 +1,12 @@ +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using System.Text.Json.Serialization; + +namespace Messaging.Messages.DbMessages.Receiving; + +public class CheckDeliusReadyResponse : DbResponseMessage +{ + public bool IsReady { get; set; } + public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage("Received Delius readiness: " + IsReady); + public override TDbQueue Queue { get; set; } = TDbQueue.IsDeliusReadyForProcessingResult; +} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/CheckOfflocReadyResponse.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/CheckOfflocReadyResponse.cs new file mode 100644 index 0000000..10b87b2 --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/CheckOfflocReadyResponse.cs @@ -0,0 +1,12 @@ +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using System.Text.Json.Serialization; + +namespace Messaging.Messages.DbMessages.Receiving; + +public class CheckOfflocReadyResponse : DbResponseMessage +{ + public bool IsReady { get; set; } + public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage("Received Offloc readiness: " + IsReady); + public override TDbQueue Queue { get; set; } = TDbQueue.IsOfflocReadyForProcessingResult; +} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultClearDeliusStaging.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/ClearDeliusStagingResponse.cs similarity index 60% rename from src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultClearDeliusStaging.cs rename to src/Libraries/Messaging/Messages/DbMessages/Receiving/ClearDeliusStagingResponse.cs index d09cbc9..fd11ba7 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultClearDeliusStaging.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/ClearDeliusStagingResponse.cs @@ -4,13 +4,9 @@ namespace Messaging.Messages.DbMessages.Receiving; -public class ResultClearDeliusStaging : DbResponseMessage +public class ClearDeliusStagingResponse : DbResponseMessage { public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage("Delius staging cleared."); - - public ResultClearDeliusStaging() - { - Queue = TDbQueue.ResultClearDelius; - } + public override TDbQueue Queue { get; set; } = TDbQueue.ResultClearDelius; } diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultClearOfflocStaging.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/ClearOfflocStagingResponse.cs similarity index 60% rename from src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultClearOfflocStaging.cs rename to src/Libraries/Messaging/Messages/DbMessages/Receiving/ClearOfflocStagingResponse.cs index 3fc8002..6724a73 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultClearOfflocStaging.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/ClearOfflocStagingResponse.cs @@ -3,13 +3,9 @@ namespace Messaging.Messages.DbMessages.Receiving; -public class ResultClearOfflocStaging : DbResponseMessage +public class ClearOfflocStagingResponse : DbResponseMessage { public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage("Offloc staging cleared."); - - public ResultClearOfflocStaging() - { - Queue = TDbQueue.ResultClearOffloc; - } + public override TDbQueue Queue { get; set; } = TDbQueue.ResultClearOffloc; } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/DbResponseMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/DbResponseMessage.cs index e1a1ea1..5733854 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/DbResponseMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/DbResponseMessage.cs @@ -1,5 +1,4 @@ - -namespace Messaging.Messages.DbMessages.Receiving; +namespace Messaging.Messages.DbMessages.Receiving; public abstract class DbResponseMessage : DbMessage { } diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/DeliusFilesReturnMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/DeliusFilesReturnMessage.cs deleted file mode 100644 index 8b16c91..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/DeliusFilesReturnMessage.cs +++ /dev/null @@ -1,22 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Receiving; - -public class DeliusFilesReturnMessage : DbResponseMessage -{ - public string[] fileNames = { }; - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage("Processed Delius files returned."); - - [JsonConstructor] - public DeliusFilesReturnMessage() - { } - - public DeliusFilesReturnMessage(string[] fileNames) - { - this.fileNames = fileNames; - Queue = TDbQueue.ReturnedDeliusFiles; - } -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetDeliusFilesResponse.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetDeliusFilesResponse.cs new file mode 100644 index 0000000..b010c2b --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetDeliusFilesResponse.cs @@ -0,0 +1,13 @@ +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using System.Text.Json.Serialization; + +namespace Messaging.Messages.DbMessages.Receiving; + +public class GetDeliusFilesResponse : DbResponseMessage +{ + public string[] FileNames { get; set; } = []; + public override StatusUpdateMessage StatusMessage => + new StatusUpdateMessage("Processed Delius files returned."); + public override TDbQueue Queue { get; set; } = TDbQueue.ReturnedDeliusFiles; +} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetLastProcessedDeliusFileResponse.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetLastProcessedDeliusFileResponse.cs new file mode 100644 index 0000000..1f773f9 --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetLastProcessedDeliusFileResponse.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; + +namespace Messaging.Messages.DbMessages.Receiving; + +public class GetLastProcessedDeliusFileResponse : DbResponseMessage +{ + public string? FileName { get; set; } + public override StatusUpdateMessage StatusMessage => new(); + public override TDbQueue Queue { get; set; } = TDbQueue.ResultLastProcessedDeliusFile; +} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetLastProcessedOfflocFileResponse.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetLastProcessedOfflocFileResponse.cs new file mode 100644 index 0000000..6b0a1bf --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetLastProcessedOfflocFileResponse.cs @@ -0,0 +1,12 @@ +using System.Text.Json.Serialization; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; + +namespace Messaging.Messages.DbMessages.Receiving; + +public class GetLastProcessedOfflocFileResponse : DbResponseMessage +{ + public string? FileName { get; set; } + public override StatusUpdateMessage StatusMessage => new(); + public override TDbQueue Queue { get; set; } = TDbQueue.ResultLastProcessedOfflocFile; +} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetOfflocFilesResponse.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetOfflocFilesResponse.cs new file mode 100644 index 0000000..5ce2e62 --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/GetOfflocFilesResponse.cs @@ -0,0 +1,14 @@ + +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using System.Text.Json.Serialization; + +namespace Messaging.Messages.DbMessages.Receiving; + +public class GetOfflocFilesResponse : DbResponseMessage +{ + public string[] OfflocFiles { get; set; } = []; + public override StatusUpdateMessage StatusMessage => + new StatusUpdateMessage("Processed Offloc files returned."); + public override TDbQueue Queue { get; set; } = TDbQueue.ReturnedOfflocFiles; +} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/IsDeliusReadyForProcessingReturnMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/IsDeliusReadyForProcessingReturnMessage.cs deleted file mode 100644 index 8674737..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/IsDeliusReadyForProcessingReturnMessage.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Receiving; - -public class IsDeliusReadyForProcessingReturnMessage : DbResponseMessage -{ - public bool isReady; - public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage("Received Delius readiness: " + isReady); - - [JsonConstructor] - public IsDeliusReadyForProcessingReturnMessage() - { } - - public IsDeliusReadyForProcessingReturnMessage(bool isReady) - { - Queue = TDbQueue.IsDeliusReadyForProcessingResult; - this.isReady = isReady; - } -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/IsOfflocReadyForProcessingReturnMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/IsOfflocReadyForProcessingReturnMessage.cs deleted file mode 100644 index ee03513..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/IsOfflocReadyForProcessingReturnMessage.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Receiving; - -public class IsOfflocReadyForProcessingReturnMessage : DbResponseMessage -{ - public bool isReady; - public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage("Received Offloc readiness: " + isReady); - - [JsonConstructor] - public IsOfflocReadyForProcessingReturnMessage() - { } - - public IsOfflocReadyForProcessingReturnMessage(bool isReady) - { - Queue = TDbQueue.IsOfflocReadyForProcessingResult; - this.isReady = isReady; - } -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeDeliusReturnMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeDeliusResponse.cs similarity index 66% rename from src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeDeliusReturnMessage.cs rename to src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeDeliusResponse.cs index 7e5042f..1356a14 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeDeliusReturnMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeDeliusResponse.cs @@ -4,15 +4,10 @@ namespace Messaging.Messages.DbMessages.Receiving; -public class MergeDeliusReturnMessage : DbResponseMessage +public class MergeDeliusResponse : DbResponseMessage { public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage($"Delius staging database successfully " + $"merged into Delius running picture database."); - - [JsonConstructor] - public MergeDeliusReturnMessage() - { - Queue = TDbQueue.ResultMergeDelius; - } + public override TDbQueue Queue { get; set; } = TDbQueue.ResultMergeDelius; } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeOfflocReturnMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeOfflocResponse.cs similarity index 67% rename from src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeOfflocReturnMessage.cs rename to src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeOfflocResponse.cs index 590f23d..59f5fde 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeOfflocReturnMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/MergeOfflocResponse.cs @@ -4,14 +4,10 @@ namespace Messaging.Messages.DbMessages.Receiving; -public class MergeOfflocReturnMessage : DbResponseMessage +public class MergeOfflocResponse : DbResponseMessage { public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage($"Offloc staging database successfully " + $"merged into offloc running picture database."); - - public MergeOfflocReturnMessage() - { - Queue = TDbQueue.ResultMergeOffloc; - } + public override TDbQueue Queue { get; set; } = TDbQueue.ResultMergeOffloc; } diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/OfflocFilesReturnMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/OfflocFilesReturnMessage.cs deleted file mode 100644 index a3b9bec..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/OfflocFilesReturnMessage.cs +++ /dev/null @@ -1,22 +0,0 @@ - -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Receiving; - -public class OfflocFilesReturnMessage : DbResponseMessage -{ - public string[] offlocFiles = new string[] { }; - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage("Processed Offloc files returned."); - - [JsonConstructor] - public OfflocFilesReturnMessage() - { } - public OfflocFilesReturnMessage(string[] offlocFiles) - { - this.offlocFiles = offlocFiles; - Queue = TDbQueue.ReturnedOfflocFiles; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultDeliusGetLastFullMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultDeliusGetLastFullMessage.cs deleted file mode 100644 index f07908d..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultDeliusGetLastFullMessage.cs +++ /dev/null @@ -1,25 +0,0 @@ - -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Receiving; - -public class ResultDeliusGetLastFullMessage : DbResponseMessage -{ - public int fileId; - - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage(); - - [JsonConstructor] - public ResultDeliusGetLastFullMessage() - { - Queue = TDbQueue.ReturnDeliusGetLastFull; - } - - public ResultDeliusGetLastFullMessage(int fileId) : this() - { - this.fileId = fileId; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultGetLastProcessedDeliusFileMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultGetLastProcessedDeliusFileMessage.cs deleted file mode 100644 index e200fc2..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultGetLastProcessedDeliusFileMessage.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Text.Json.Serialization; -using Messaging.Messages.StatusMessages; -using Messaging.Queues; - -namespace Messaging.Messages.DbMessages.Receiving; - -public class ResultGetLastProcessedDeliusFileMessage : DbResponseMessage -{ - public string? fileName; - - public override StatusUpdateMessage StatusMessage => new(); - - [JsonConstructor] - public ResultGetLastProcessedDeliusFileMessage() - { - Queue = TDbQueue.ResultLastProcessedDeliusFile; - } - - public ResultGetLastProcessedDeliusFileMessage(string? fileName) : this() - { - this.fileName = fileName; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultGetLastProcessedOfflocFileMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultGetLastProcessedOfflocFileMessage.cs deleted file mode 100644 index 975ad1f..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultGetLastProcessedOfflocFileMessage.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Text.Json.Serialization; -using Messaging.Messages.StatusMessages; -using Messaging.Queues; - -namespace Messaging.Messages.DbMessages.Receiving; - -public class ResultGetLastProcessedOfflocFileMessage : DbResponseMessage -{ - public string? fileName; - - public override StatusUpdateMessage StatusMessage => new(); - - [JsonConstructor] - public ResultGetLastProcessedOfflocFileMessage() - { - Queue = TDbQueue.ResultLastProcessedOfflocFile; - } - - public ResultGetLastProcessedOfflocFileMessage(string? fileName) : this() - { - this.fileName = fileName; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultStandardiseDeliusStaging.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultStandardiseDeliusStaging.cs deleted file mode 100644 index cb34a80..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultStandardiseDeliusStaging.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Messaging.Messages.DbMessages.Receiving -{ - internal class ResultStandardiseDeliusStaging : DbResponseMessage - { - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage("Delius staging standardisation complete."); - - public ResultStandardiseDeliusStaging() - { - Queue = TDbQueue.ResultStandardiseDelius; - } - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/StageDeliusReturnMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/StageDeliusResponse.cs similarity index 61% rename from src/Libraries/Messaging/Messages/DbMessages/Receiving/StageDeliusReturnMessage.cs rename to src/Libraries/Messaging/Messages/DbMessages/Receiving/StageDeliusResponse.cs index aa791cf..e9200e0 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/StageDeliusReturnMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/StageDeliusResponse.cs @@ -4,14 +4,9 @@ namespace Messaging.Messages.DbMessages.Receiving; -public class StageDeliusReturnMessage : DbResponseMessage +public class StageDeliusResponse : DbResponseMessage { public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage("Delius Staging completed."); - - [JsonConstructor] - public StageDeliusReturnMessage() - { - Queue = TDbQueue.ResultStageDelius; - } + public override TDbQueue Queue { get; set; } = TDbQueue.ResultStageDelius; } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/StageOfflocReturnMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/StageOfflocResponse.cs similarity index 61% rename from src/Libraries/Messaging/Messages/DbMessages/Receiving/StageOfflocReturnMessage.cs rename to src/Libraries/Messaging/Messages/DbMessages/Receiving/StageOfflocResponse.cs index c1c90cb..15d3b38 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/StageOfflocReturnMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/StageOfflocResponse.cs @@ -4,14 +4,9 @@ namespace Messaging.Messages.DbMessages.Receiving; -public class StageOfflocReturnMessage : DbResponseMessage +public class StageOfflocResponse : DbResponseMessage { public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage("Offloc Staging completed."); - - [JsonConstructor] - public StageOfflocReturnMessage() - { - Queue = TDbQueue.ResultStageOffloc; - } + public override TDbQueue Queue { get; set; } = TDbQueue.ResultStageOffloc; } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultDeliusFileProcessingStarted.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/StartDeliusFileProcessingResponse.cs similarity index 54% rename from src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultDeliusFileProcessingStarted.cs rename to src/Libraries/Messaging/Messages/DbMessages/Receiving/StartDeliusFileProcessingResponse.cs index 80b541f..8495d38 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultDeliusFileProcessingStarted.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/StartDeliusFileProcessingResponse.cs @@ -3,12 +3,8 @@ namespace Messaging.Messages.DbMessages.Receiving; -public class ResultDeliusFileProcessingStarted : DbResponseMessage +public class StartDeliusFileProcessingResponse : DbResponseMessage { public override StatusUpdateMessage StatusMessage => new("Delius file processing started"); - - public ResultDeliusFileProcessingStarted() - { - Queue = TDbQueue.ResultDeliusFileProcessingStarted; - } + public override TDbQueue Queue { get; set; } = TDbQueue.ResultDeliusFileProcessingStarted; } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultOfflocFileProcessingStarted.cs b/src/Libraries/Messaging/Messages/DbMessages/Receiving/StartOfflocFileProcessingResponse.cs similarity index 55% rename from src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultOfflocFileProcessingStarted.cs rename to src/Libraries/Messaging/Messages/DbMessages/Receiving/StartOfflocFileProcessingResponse.cs index ca4c348..3840ac3 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Receiving/ResultOfflocFileProcessingStarted.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Receiving/StartOfflocFileProcessingResponse.cs @@ -4,13 +4,8 @@ namespace Messaging.Messages.DbMessages.Receiving; -public class ResultOfflocFileProcessingStarted : DbResponseMessage +public class StartOfflocFileProcessingResponse : DbResponseMessage { public override StatusUpdateMessage StatusMessage => new("Offloc file processing started"); - - [JsonConstructor] - public ResultOfflocFileProcessingStarted() - { - Queue = TDbQueue.ResultOfflocFileProcessingStarted; - } + public override TDbQueue Queue { get; set; } = TDbQueue.ResultOfflocFileProcessingStarted; } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/AssociateOfflocFileWithArchiveMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/AssociateOfflocFileWithArchiveMessage.cs deleted file mode 100644 index 1c7b9f0..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/AssociateOfflocFileWithArchiveMessage.cs +++ /dev/null @@ -1,22 +0,0 @@ -using System.Text.Json.Serialization; -using Messaging.Messages.StatusMessages; -using Messaging.Queues; - -namespace Messaging.Messages.DbMessages.Sending; - -public class AssociateOfflocFileWithArchiveMessage : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => new(); - - public string fileName { get; set; } - public string archiveName { get; set; } - - [JsonConstructor] - public AssociateOfflocFileWithArchiveMessage(string fileName, string archiveName) - { - Queue = TDbQueue.AssociateOfflocFileWithArchive; - ReplyQueue = TDbQueue.ResultAssociateOfflocFileWithArchive; - this.fileName = fileName; - this.archiveName = archiveName; - } -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/AssociateOfflocFileWithArchiveRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/AssociateOfflocFileWithArchiveRequest.cs new file mode 100644 index 0000000..50ef023 --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/AssociateOfflocFileWithArchiveRequest.cs @@ -0,0 +1,14 @@ +using System.Text.Json.Serialization; +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; + +namespace Messaging.Messages.DbMessages.Sending; + +public class AssociateOfflocFileWithArchiveRequest : DbRequestMessage +{ + public override StatusUpdateMessage StatusMessage => new(); + public required string FileName { get; set; } + public required string ArchiveName { get; set; } + public override TDbQueue Queue { get; set; } = TDbQueue.AssociateOfflocFileWithArchive; +} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/IsDeliusReadyForProcessingMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/CheckDeliusReadyRequest.cs similarity index 51% rename from src/Libraries/Messaging/Messages/DbMessages/Sending/IsDeliusReadyForProcessingMessage.cs rename to src/Libraries/Messaging/Messages/DbMessages/Sending/CheckDeliusReadyRequest.cs index 5fcd057..97ce83f 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/IsDeliusReadyForProcessingMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/CheckDeliusReadyRequest.cs @@ -1,17 +1,12 @@ using System.Text.Json.Serialization; +using Messaging.Messages.DbMessages.Receiving; using Messaging.Messages.StatusMessages; using Messaging.Queues; namespace Messaging.Messages.DbMessages.Sending; -public class IsDeliusReadyForProcessingMessage : DbRequestMessage +public class CheckDeliusReadyRequest : DbRequestMessage { public override StatusUpdateMessage StatusMessage => new("Sent request to check if delius is ready for processing."); - - [JsonConstructor] - public IsDeliusReadyForProcessingMessage() - { - Queue = TDbQueue.IsDeliusReadyForProcessing; - ReplyQueue = TDbQueue.IsDeliusReadyForProcessingResult; - } + public override TDbQueue Queue { get; set; } = TDbQueue.IsDeliusReadyForProcessing; } diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/IsOfflocReadyForProcessingMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/CheckOfflocReadyRequest.cs similarity index 51% rename from src/Libraries/Messaging/Messages/DbMessages/Sending/IsOfflocReadyForProcessingMessage.cs rename to src/Libraries/Messaging/Messages/DbMessages/Sending/CheckOfflocReadyRequest.cs index c3f938d..6f3e509 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/IsOfflocReadyForProcessingMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/CheckOfflocReadyRequest.cs @@ -1,17 +1,12 @@ using System.Text.Json.Serialization; +using Messaging.Messages.DbMessages.Receiving; using Messaging.Messages.StatusMessages; using Messaging.Queues; namespace Messaging.Messages.DbMessages.Sending; -public class IsOfflocReadyForProcessingMessage : DbRequestMessage +public class CheckOfflocReadyRequest : DbRequestMessage { public override StatusUpdateMessage StatusMessage => new("Sent request to check if offloc is ready for processing."); - - [JsonConstructor] - public IsOfflocReadyForProcessingMessage() - { - Queue = TDbQueue.IsOfflocReadyForProcessing; - ReplyQueue = TDbQueue.IsOfflocReadyForProcessingResult; - } + public override TDbQueue Queue { get; set; } = TDbQueue.IsOfflocReadyForProcessing; } diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/ClearDeliusStaging.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/ClearDeliusStagingRequest.cs similarity index 57% rename from src/Libraries/Messaging/Messages/DbMessages/Sending/ClearDeliusStaging.cs rename to src/Libraries/Messaging/Messages/DbMessages/Sending/ClearDeliusStagingRequest.cs index 2ed7d36..045f4db 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/ClearDeliusStaging.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/ClearDeliusStagingRequest.cs @@ -1,4 +1,5 @@  +using Messaging.Messages.DbMessages.Receiving; using Messaging.Messages.StagingMessages; using Messaging.Messages.StatusMessages; using Messaging.Queues; @@ -6,16 +7,10 @@ namespace Messaging.Messages.DbMessages.Sending; -public class ClearDeliusStaging : DbRequestMessage +public class ClearDeliusStagingRequest : DbRequestMessage { public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage(); - - [JsonConstructor] - public ClearDeliusStaging() - { - Queue = TDbQueue.ClearDeliusStaging; - ReplyQueue = TDbQueue.ResultClearDelius; - } + public override TDbQueue Queue { get; set; } = TDbQueue.ClearDeliusStaging; } diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/ClearOfflocStaging.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/ClearOfflocStaging.cs deleted file mode 100644 index 5c414de..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/ClearOfflocStaging.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; - -namespace Messaging.Messages.DbMessages.Sending; - -public class ClearOfflocStaging : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage("Offloc staging database being cleared."); - - public ClearOfflocStaging() - { - Queue = TDbQueue.ClearOfflocStaging; - ReplyQueue = TDbQueue.ResultClearOffloc; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/ClearOfflocStagingRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/ClearOfflocStagingRequest.cs new file mode 100644 index 0000000..28edfee --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/ClearOfflocStagingRequest.cs @@ -0,0 +1,12 @@ +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; + +namespace Messaging.Messages.DbMessages.Sending; + +public class ClearOfflocStagingRequest : DbRequestMessage +{ + public override StatusUpdateMessage StatusMessage => + new StatusUpdateMessage("Offloc staging database being cleared."); + public override TDbQueue Queue { get; set; } = TDbQueue.ClearOfflocStaging; +} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/DbRequestMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/DbRequestMessage.cs index 1e39434..9968796 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/DbRequestMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/DbRequestMessage.cs @@ -1,8 +1,12 @@ -using Messaging.Queues; +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Queues; namespace Messaging.Messages.DbMessages.Sending; -public abstract class DbRequestMessage : DbMessage +public abstract class DbRequestMessage : DbMessage + where TResponse : DbResponseMessage, new() { - public TDbQueue ReplyQueue { get; set; } + protected DbRequestMessage() + { + } } diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/DeliusFileProcessingStarted.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/DeliusFileProcessingStarted.cs deleted file mode 100644 index 0dcbd37..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/DeliusFileProcessingStarted.cs +++ /dev/null @@ -1,26 +0,0 @@ -using System.Text.Json.Serialization; -using Messaging.Messages.StatusMessages; -using Messaging.Queues; - -namespace Messaging.Messages.DbMessages.Sending; - -public class DeliusFileProcessingStarted : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => new(); - - public string fileName = string.Empty; - public string fileId = string.Empty; - - [JsonConstructor] - public DeliusFileProcessingStarted() - { - Queue = TDbQueue.DeliusFileProcessingStarted; - ReplyQueue = TDbQueue.ResultDeliusFileProcessingStarted; - } - - public DeliusFileProcessingStarted(string fileName, string fileId) : this() - { - this.fileName = fileName; - this.fileId = fileId; - } -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/DeliusGetLastFullMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/DeliusGetLastFullMessage.cs deleted file mode 100644 index 41bedac..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/DeliusGetLastFullMessage.cs +++ /dev/null @@ -1,19 +0,0 @@ - -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Sending; - -public class DeliusGetLastFullMessage : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage(); - - [JsonConstructor] - public DeliusGetLastFullMessage() - { - Queue = TDbQueue.DeliusGetLastFullId; - ReplyQueue = TDbQueue.ReturnDeliusGetLastFull; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetDeliusFilesMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetDeliusFilesMessage.cs deleted file mode 100644 index ddc0986..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetDeliusFilesMessage.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Sending; - -public class GetDeliusFilesMessage : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage("Request sent to retrieve processed delius files."); - - [JsonConstructor] - public GetDeliusFilesMessage() - { - Queue = TDbQueue.GetProcessedDeliusFiles; - ReplyQueue = TDbQueue.ReturnedDeliusFiles; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetDeliusFilesRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetDeliusFilesRequest.cs new file mode 100644 index 0000000..85d531c --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetDeliusFilesRequest.cs @@ -0,0 +1,13 @@ +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using System.Text.Json.Serialization; + +namespace Messaging.Messages.DbMessages.Sending; + +public class GetDeliusFilesRequest : DbRequestMessage +{ + public override StatusUpdateMessage StatusMessage => + new StatusUpdateMessage("Request sent to retrieve processed delius files."); + public override TDbQueue Queue { get; set; } = TDbQueue.GetProcessedDeliusFiles; +} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedDeliusFile.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedDeliusFile.cs deleted file mode 100644 index 3781c47..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedDeliusFile.cs +++ /dev/null @@ -1,15 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; - -namespace Messaging.Messages.DbMessages.Sending; - -public class GetLastProcessedDeliusFile : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => new("Getting last processed delius file."); - - public GetLastProcessedDeliusFile() - { - Queue = TDbQueue.GetLastProcessedDeliusFile; - ReplyQueue = TDbQueue.ResultLastProcessedDeliusFile; - } -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedDeliusFileRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedDeliusFileRequest.cs new file mode 100644 index 0000000..0d3ec2e --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedDeliusFileRequest.cs @@ -0,0 +1,11 @@ +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; + +namespace Messaging.Messages.DbMessages.Sending; + +public class GetLastProcessedDeliusFileRequest : DbRequestMessage +{ + public override StatusUpdateMessage StatusMessage => new("Getting last processed delius file."); + public override TDbQueue Queue { get; set; } = TDbQueue.GetLastProcessedDeliusFile; +} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedOfflocFile.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedOfflocFile.cs deleted file mode 100644 index 8dfab26..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedOfflocFile.cs +++ /dev/null @@ -1,16 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; - -namespace Messaging.Messages.DbMessages.Sending; - -public class GetLastProcessedOfflocFile : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => new("Getting last processed offloc file."); - - public GetLastProcessedOfflocFile() - { - Queue = TDbQueue.GetLastProcessedOfflocFile; - ReplyQueue = TDbQueue.ResultLastProcessedOfflocFile; - } - -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedOfflocFileRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedOfflocFileRequest.cs new file mode 100644 index 0000000..e881298 --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetLastProcessedOfflocFileRequest.cs @@ -0,0 +1,11 @@ +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; + +namespace Messaging.Messages.DbMessages.Sending; + +public class GetLastProcessedOfflocFileRequest : DbRequestMessage +{ + public override StatusUpdateMessage StatusMessage => new("Getting last processed offloc file."); + public override TDbQueue Queue { get; set; } = TDbQueue.GetLastProcessedOfflocFile; +} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetOfflocFileDatesMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetOfflocFileDatesMessage.cs deleted file mode 100644 index e91985a..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetOfflocFileDatesMessage.cs +++ /dev/null @@ -1,19 +0,0 @@ - -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Sending; - -public class GetOfflocFileDatesMessage : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage("Request for processed offloc files sent."); - - [JsonConstructor] - public GetOfflocFileDatesMessage() - { - Queue = TDbQueue.GetOfflocFileDates; - ReplyQueue = TDbQueue.ReturnedOfflocFileDates; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetOfflocFilesMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetOfflocFilesMessage.cs deleted file mode 100644 index 8182b26..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetOfflocFilesMessage.cs +++ /dev/null @@ -1,18 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Sending; - -public class GetOfflocFilesMessage : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage("Sent request to get processed offloc files."); - - [JsonConstructor] - public GetOfflocFilesMessage() - { - Queue = TDbQueue.GetProcessedOfflocFiles; - ReplyQueue = TDbQueue.ReturnedOfflocFiles; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/GetOfflocFilesRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetOfflocFilesRequest.cs new file mode 100644 index 0000000..df4479a --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/GetOfflocFilesRequest.cs @@ -0,0 +1,13 @@ +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using System.Text.Json.Serialization; + +namespace Messaging.Messages.DbMessages.Sending; + +public class GetOfflocFilesRequest : DbRequestMessage +{ + public override StatusUpdateMessage StatusMessage => + new StatusUpdateMessage("Sent request to get processed offloc files."); + public override TDbQueue Queue { get; set; } = TDbQueue.GetProcessedOfflocFiles; +} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/MergeDeliusRunningPictureMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/MergeDeliusRequest.cs similarity index 50% rename from src/Libraries/Messaging/Messages/DbMessages/Sending/MergeDeliusRunningPictureMessage.cs rename to src/Libraries/Messaging/Messages/DbMessages/Sending/MergeDeliusRequest.cs index a74395c..69f4927 100644 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/MergeDeliusRunningPictureMessage.cs +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/MergeDeliusRequest.cs @@ -1,22 +1,15 @@  +using Messaging.Messages.DbMessages.Receiving; using Messaging.Messages.StatusMessages; using Messaging.Queues; using System.Text.Json.Serialization; namespace Messaging.Messages.DbMessages.Sending; -public class MergeDeliusRunningPictureMessage : DbRequestMessage +public class MergeDeliusRequest : DbRequestMessage { public override StatusUpdateMessage StatusMessage => new StatusUpdateMessage("Merging into delius running picture started."); - - public string fileName { get; set; } - - [JsonConstructor] - public MergeDeliusRunningPictureMessage(string fileName) - { - Queue = TDbQueue.MergeDelius; - ReplyQueue = TDbQueue.ResultMergeDelius; - this.fileName = fileName; - } + public required string FileName { get; set; } + public override TDbQueue Queue { get; set; } = TDbQueue.MergeDelius; } diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/MergeOfflocRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/MergeOfflocRequest.cs new file mode 100644 index 0000000..d868915 --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/MergeOfflocRequest.cs @@ -0,0 +1,15 @@ +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using System.Text.Json.Serialization; +using Messaging.Queues; +using Messaging.Messages.MergingMessages; + +namespace Messaging.Messages.DbMessages.Sending; + +public class MergeOfflocRequest : DbRequestMessage +{ + public override StatusUpdateMessage StatusMessage => + new StatusUpdateMessage("Merging into Offloc running picture started."); + public required string FileName { get; set; } + public override TDbQueue Queue { get; set; } = TDbQueue.MergeOffloc; +} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/MergeOfflocRunningPictureMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/MergeOfflocRunningPictureMessage.cs deleted file mode 100644 index 001b33e..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/MergeOfflocRunningPictureMessage.cs +++ /dev/null @@ -1,22 +0,0 @@ -using Messaging.Messages.StatusMessages; -using System.Text.Json.Serialization; -using Messaging.Queues; -using Messaging.Messages.MergingMessages; - -namespace Messaging.Messages.DbMessages.Sending; - -public class MergeOfflocRunningPictureMessage : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage("Merging into Offloc running picture started."); - - public string fileName { get; set; } - - [JsonConstructor] - public MergeOfflocRunningPictureMessage(string fileName) - { - Queue = TDbQueue.MergeOffloc; - ReplyQueue = TDbQueue.ResultMergeOffloc; - this.fileName = fileName; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/OfflocFileProcessingStarted.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/OfflocFileProcessingStarted.cs deleted file mode 100644 index fa59141..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/OfflocFileProcessingStarted.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System.Text.Json.Serialization; -using Messaging.Messages.StatusMessages; -using Messaging.Queues; - -namespace Messaging.Messages.DbMessages.Sending; - -public class OfflocFileProcessingStarted : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => new(); - - public string fileName = string.Empty; - public int fileId; - public string? archiveName; - - [JsonConstructor] - public OfflocFileProcessingStarted() - { - Queue = TDbQueue.OfflocFileProcessingStarted; - ReplyQueue = TDbQueue.ResultOfflocFileProcessingStarted; - } - - public OfflocFileProcessingStarted(string fileName, int fileId, string? archiveName = null) : this() - { - this.fileName = fileName; - this.fileId = fileId; - this.archiveName = archiveName; - } -} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/StageDeliusMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/StageDeliusMessage.cs deleted file mode 100644 index fd768e0..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/StageDeliusMessage.cs +++ /dev/null @@ -1,28 +0,0 @@ - -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Sending; - -public class StageDeliusMessage : DbRequestMessage -{ - public string fileName = string.Empty; - public string filePath = string.Empty; - - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage("Delius Staging started."); - - [JsonConstructor] - public StageDeliusMessage() - { - Queue = TDbQueue.StageDelius; - ReplyQueue = TDbQueue.ResultStageDelius; - } - - public StageDeliusMessage(string fileName, string filePath) : this() - { - this.fileName = fileName; - this.filePath = filePath; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/StageDeliusRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/StageDeliusRequest.cs new file mode 100644 index 0000000..5b673d6 --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/StageDeliusRequest.cs @@ -0,0 +1,16 @@ + +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using System.Text.Json.Serialization; + +namespace Messaging.Messages.DbMessages.Sending; + +public class StageDeliusRequest : DbRequestMessage +{ + public required string FileName { get; set; } + public required string FilePath { get; set; } + public override StatusUpdateMessage StatusMessage => + new StatusUpdateMessage("Delius Staging started."); + public override TDbQueue Queue { get; set; } = TDbQueue.StageDelius; +} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/StageOfflocMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/StageOfflocMessage.cs deleted file mode 100644 index 3337eed..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/StageOfflocMessage.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System.Text.Json.Serialization; - -namespace Messaging.Messages.DbMessages.Sending; - -public class StageOfflocMessage : DbRequestMessage -{ - public string fileName = string.Empty; - - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage(); - - [JsonConstructor] - public StageOfflocMessage() - { - Queue = TDbQueue.StageOffloc; - ReplyQueue = TDbQueue.ResultStageOffloc; - } - - public StageOfflocMessage(string fileName) : this() - { - this.fileName = fileName; - } -} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/StageOfflocRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/StageOfflocRequest.cs new file mode 100644 index 0000000..db3e9ca --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/StageOfflocRequest.cs @@ -0,0 +1,14 @@ +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using System.Text.Json.Serialization; + +namespace Messaging.Messages.DbMessages.Sending; + +public class StageOfflocRequest : DbRequestMessage +{ + public required string FileName { get; set; } + public override StatusUpdateMessage StatusMessage => + new StatusUpdateMessage(); + public override TDbQueue Queue { get; set; } = TDbQueue.StageOffloc; +} diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/StandardiseDeliusMessage.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/StandardiseDeliusMessage.cs deleted file mode 100644 index 9296593..0000000 --- a/src/Libraries/Messaging/Messages/DbMessages/Sending/StandardiseDeliusMessage.cs +++ /dev/null @@ -1,25 +0,0 @@ -using Messaging.Messages.StatusMessages; -using Messaging.Queues; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.Json.Serialization; -using System.Threading.Tasks; - -namespace Messaging.Messages.DbMessages.Sending; - -public class StandardiseDeliusMessage : DbRequestMessage -{ - public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage(); - - [JsonConstructor] - public StandardiseDeliusMessage() - { - Queue = TDbQueue.StandardiseDelius; - ReplyQueue = TDbQueue.ResultStandardiseDelius; - } -} - - diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/StartDeliusFileProcessingRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/StartDeliusFileProcessingRequest.cs new file mode 100644 index 0000000..160d12d --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/StartDeliusFileProcessingRequest.cs @@ -0,0 +1,14 @@ +using System.Text.Json.Serialization; +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; + +namespace Messaging.Messages.DbMessages.Sending; + +public class StartDeliusFileProcessingRequest : DbRequestMessage +{ + public override StatusUpdateMessage StatusMessage => new(); + public required string FileName { get; set; } + public required string FileId { get; set; } + public override TDbQueue Queue { get; set; } = TDbQueue.DeliusFileProcessingStarted; +} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/DbMessages/Sending/StartOfflocFileProcessingRequest.cs b/src/Libraries/Messaging/Messages/DbMessages/Sending/StartOfflocFileProcessingRequest.cs new file mode 100644 index 0000000..950064a --- /dev/null +++ b/src/Libraries/Messaging/Messages/DbMessages/Sending/StartOfflocFileProcessingRequest.cs @@ -0,0 +1,15 @@ +using System.Text.Json.Serialization; +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; + +namespace Messaging.Messages.DbMessages.Sending; + +public class StartOfflocFileProcessingRequest : DbRequestMessage +{ + public override StatusUpdateMessage StatusMessage => new(); + public required string FileName { get; set; } + public required int FileId { get; set; } + public string? ArchiveName { get; set; } + public override TDbQueue Queue { get; set; } = TDbQueue.OfflocFileProcessingStarted; +} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/IMessage.cs b/src/Libraries/Messaging/Messages/IMessage.cs new file mode 100644 index 0000000..dbaf225 --- /dev/null +++ b/src/Libraries/Messaging/Messages/IMessage.cs @@ -0,0 +1,7 @@ +namespace Messaging.Messages; + +public interface IMessage +{ + public abstract string RoutingKey { get; } + public abstract string Exchange { get; } +} \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/ImportMessages/ImportFinishedMessage.cs b/src/Libraries/Messaging/Messages/ImportMessages/ImportFinishedMessage.cs index 87c6590..accc4ea 100644 --- a/src/Libraries/Messaging/Messages/ImportMessages/ImportFinishedMessage.cs +++ b/src/Libraries/Messaging/Messages/ImportMessages/ImportFinishedMessage.cs @@ -12,6 +12,6 @@ public class ImportFinishedMessage : ImportMessage [JsonConstructor] public ImportFinishedMessage() { - routingKey = TImportQueue.ImportFinished; + Queue = TImportQueue.ImportFinished; } } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/ImportMessages/ImportMessage.cs b/src/Libraries/Messaging/Messages/ImportMessages/ImportMessage.cs index a26f9fa..affecd5 100644 --- a/src/Libraries/Messaging/Messages/ImportMessages/ImportMessage.cs +++ b/src/Libraries/Messaging/Messages/ImportMessages/ImportMessage.cs @@ -4,7 +4,10 @@ namespace Messaging.Messages.ImportMessages { public abstract class ImportMessage : Message { - public TImportQueue routingKey { get; set; } - public string fileName = string.Empty; //To handle processing of multiple files. + public TImportQueue Queue { get; set; } + public string FileName { get; set; } = string.Empty; + + public override string Exchange => Exchanges.import; + public override string RoutingKey => Queue.ToString(); } } \ No newline at end of file diff --git a/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPostProcessingFinishedMessage.cs b/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPostProcessingFinishedMessage.cs index bac929e..9feabf4 100644 --- a/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPostProcessingFinishedMessage.cs +++ b/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPostProcessingFinishedMessage.cs @@ -12,6 +12,6 @@ public class ClusteringPostProcessingFinishedMessage : MatchingMessage [JsonConstructor] public ClusteringPostProcessingFinishedMessage() { - routingKey = TMatchingQueue.ClusteringPostProcessingFinished; + Queue = TMatchingQueue.ClusteringPostProcessingFinished; } } diff --git a/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPreProcessingFinishedMessage.cs b/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPreProcessingFinishedMessage.cs index 59843f6..6896ea2 100644 --- a/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPreProcessingFinishedMessage.cs +++ b/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPreProcessingFinishedMessage.cs @@ -12,6 +12,6 @@ public class ClusteringPreProcessingFinishedMessage : MatchingMessage [JsonConstructor] public ClusteringPreProcessingFinishedMessage() { - routingKey = TMatchingQueue.ClusteringPreProcessingFinished; + Queue = TMatchingQueue.ClusteringPreProcessingFinished; } } diff --git a/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPreProcessingStartedMessage.cs b/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPreProcessingStartedMessage.cs index ce45552..5ee2614 100644 --- a/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPreProcessingStartedMessage.cs +++ b/src/Libraries/Messaging/Messages/MatchingMessages/Clustering/ClusteringPreProcessingStartedMessage.cs @@ -12,6 +12,6 @@ public class ClusteringPreProcessingStartedMessage : MatchingMessage [JsonConstructor] public ClusteringPreProcessingStartedMessage() { - routingKey = TMatchingQueue.ClusteringPreProcessingStarted; + Queue = TMatchingQueue.ClusteringPreProcessingStarted; } } diff --git a/src/Libraries/Messaging/Messages/MatchingMessages/MatchingMessage.cs b/src/Libraries/Messaging/Messages/MatchingMessages/MatchingMessage.cs index 5ce46ef..4e78489 100644 --- a/src/Libraries/Messaging/Messages/MatchingMessages/MatchingMessage.cs +++ b/src/Libraries/Messaging/Messages/MatchingMessages/MatchingMessage.cs @@ -4,5 +4,8 @@ namespace Messaging.Messages.MatchingMessages; public abstract class MatchingMessage : Message { - public TMatchingQueue routingKey { get; set; } + public TMatchingQueue Queue { get; set; } + + public override string Exchange => Exchanges.matching; + public override string RoutingKey => Queue.ToString(); } diff --git a/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreCandidatesFinishedMessage.cs b/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreCandidatesFinishedMessage.cs index 7a43341..c0bfae9 100644 --- a/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreCandidatesFinishedMessage.cs +++ b/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreCandidatesFinishedMessage.cs @@ -12,6 +12,6 @@ public class MatchingScoreCandidatesFinishedMessage : MatchingMessage [JsonConstructor] public MatchingScoreCandidatesFinishedMessage() { - routingKey = TMatchingQueue.MatchingScoreCandidatesFinished; + Queue = TMatchingQueue.MatchingScoreCandidatesFinished; } } diff --git a/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreCandidatesMessage.cs b/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreCandidatesMessage.cs index dc8162f..341db25 100644 --- a/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreCandidatesMessage.cs +++ b/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreCandidatesMessage.cs @@ -9,7 +9,7 @@ public class MatchingScoreCandidatesMessage : MatchingMessage [JsonConstructor] public MatchingScoreCandidatesMessage() { - routingKey = TMatchingQueue.MatchingScoreCandidatesMessage; + Queue = TMatchingQueue.MatchingScoreCandidatesMessage; } public override StatusUpdateMessage StatusMessage => diff --git a/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreOustandingEdgesMessage.cs b/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreOustandingEdgesMessage.cs index f098df9..d28579f 100644 --- a/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreOustandingEdgesMessage.cs +++ b/src/Libraries/Messaging/Messages/MatchingMessages/MatchingScoreOustandingEdgesMessage.cs @@ -9,7 +9,7 @@ public class MatchingScoreOutstandingEdgesMessage : MatchingMessage [JsonConstructor] public MatchingScoreOutstandingEdgesMessage() { - routingKey = TMatchingQueue.MatchingScoreOutstandingEdgesMessage; + Queue = TMatchingQueue.MatchingScoreOutstandingEdgesMessage; } public override StatusUpdateMessage StatusMessage => diff --git a/src/Libraries/Messaging/Messages/MergingMessages/CleanupMessages/DeliusFilesCleanupMessage.cs b/src/Libraries/Messaging/Messages/MergingMessages/CleanupMessages/DeliusFilesCleanupMessage.cs index 216e3d4..3d87442 100644 --- a/src/Libraries/Messaging/Messages/MergingMessages/CleanupMessages/DeliusFilesCleanupMessage.cs +++ b/src/Libraries/Messaging/Messages/MergingMessages/CleanupMessages/DeliusFilesCleanupMessage.cs @@ -8,16 +8,16 @@ namespace Messaging.Messages.MergingMessages.CleanupMessages; public class DeliusFilesCleanupMessage : MergingMessage { public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage($"File cleanup started for delius file {fileName}"); + new StatusUpdateMessage($"File cleanup started for delius file {FileName}"); [JsonConstructor] public DeliusFilesCleanupMessage() { - routingKey = TMergingQueue.DeliusFilesCleanupQueue; + Queue = TMergingQueue.DeliusFilesCleanupQueue; } public DeliusFilesCleanupMessage(string fileName) : this() { - this.fileName = fileName; + FileName = fileName; } } diff --git a/src/Libraries/Messaging/Messages/MergingMessages/CleanupMessages/OfflocFilesCleanupMessage.cs b/src/Libraries/Messaging/Messages/MergingMessages/CleanupMessages/OfflocFilesCleanupMessage.cs index ce88c50..6d8b1f8 100644 --- a/src/Libraries/Messaging/Messages/MergingMessages/CleanupMessages/OfflocFilesCleanupMessage.cs +++ b/src/Libraries/Messaging/Messages/MergingMessages/CleanupMessages/OfflocFilesCleanupMessage.cs @@ -8,16 +8,16 @@ namespace Messaging.Messages.MergingMessages.CleanupMessages; public class OfflocFilesCleanupMessage : MergingMessage { public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage($"Cleanup started for offloc file {fileName}"); + new StatusUpdateMessage($"Cleanup started for offloc file {FileName}"); [JsonConstructor] public OfflocFilesCleanupMessage() { - routingKey = TMergingQueue.OfflocFilesCleanupQueue; + Queue = TMergingQueue.OfflocFilesCleanupQueue; } public OfflocFilesCleanupMessage(string fileName) : this() { - this.fileName = fileName; + FileName = fileName; } } diff --git a/src/Libraries/Messaging/Messages/MergingMessages/MergingMessage.cs b/src/Libraries/Messaging/Messages/MergingMessages/MergingMessage.cs index 4d0391a..b5e0e05 100644 --- a/src/Libraries/Messaging/Messages/MergingMessages/MergingMessage.cs +++ b/src/Libraries/Messaging/Messages/MergingMessages/MergingMessage.cs @@ -5,6 +5,9 @@ namespace Messaging.Messages.MergingMessages; public abstract class MergingMessage : Message { - public TMergingQueue routingKey; - public string fileName = string.Empty; + public TMergingQueue Queue { get; set; } + public string FileName { get; set; } = string.Empty; + + public override string Exchange => Exchanges.merging; + public override string RoutingKey => Queue.ToString(); } diff --git a/src/Libraries/Messaging/Messages/Message.cs b/src/Libraries/Messaging/Messages/Message.cs index 463ac95..aad0bfc 100644 --- a/src/Libraries/Messaging/Messages/Message.cs +++ b/src/Libraries/Messaging/Messages/Message.cs @@ -3,9 +3,12 @@ namespace Messaging.Messages; -public abstract class Message +public abstract class Message : IMessage { //Tightly couples each message to a status update as every message signals //the completion of the stage. public abstract StatusUpdateMessage StatusMessage { get; } + + public abstract string Exchange { get; } + public abstract string RoutingKey { get; } } diff --git a/src/Libraries/Messaging/Messages/StagingMessages/Delius/ClearTemporaryDeliusFiles.cs b/src/Libraries/Messaging/Messages/StagingMessages/Delius/ClearTemporaryDeliusFiles.cs index 6dbc9c3..38f9efe 100644 --- a/src/Libraries/Messaging/Messages/StagingMessages/Delius/ClearTemporaryDeliusFiles.cs +++ b/src/Libraries/Messaging/Messages/StagingMessages/Delius/ClearTemporaryDeliusFiles.cs @@ -10,6 +10,6 @@ public class ClearTemporaryDeliusFiles : StagingMessage public ClearTemporaryDeliusFiles() { - routingKey = TStagingQueue.DeliusFilesClear; + Queue = TStagingQueue.DeliusFilesClear; } } diff --git a/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusDownloadFinishedMessage.cs b/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusDownloadFinishedMessage.cs index ad1512c..ca48259 100644 --- a/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusDownloadFinishedMessage.cs +++ b/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusDownloadFinishedMessage.cs @@ -13,8 +13,8 @@ public class DeliusDownloadFinishedMessage : StagingMessage [JsonConstructor] public DeliusDownloadFinishedMessage(string fileName, string fileId) { - routingKey = TStagingQueue.DeliusParser; - base.fileName = fileName; + Queue = TStagingQueue.DeliusParser; + base.FileName = fileName; FileId = fileId; } diff --git a/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusKickoffMessage.cs b/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusKickoffMessage.cs index a01e726..09dd801 100644 --- a/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusKickoffMessage.cs +++ b/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusKickoffMessage.cs @@ -10,6 +10,6 @@ public class DeliusKickoffMessage : StagingMessage public DeliusKickoffMessage() { - routingKey = TStagingQueue.DeliusFileDownload; + Queue = TStagingQueue.DeliusFileDownload; } } diff --git a/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusParserFinishedMessage.cs b/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusParserFinishedMessage.cs index a2d0a41..ccf41e6 100644 --- a/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusParserFinishedMessage.cs +++ b/src/Libraries/Messaging/Messages/StagingMessages/Delius/DeliusParserFinishedMessage.cs @@ -6,20 +6,21 @@ namespace Messaging.Messages.StagingMessages; public class DeliusParserFinishedMessage : StagingMessage { - public string filePath = string.Empty; + public string FilePath { get; set; } = string.Empty; public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage($"Delius parser finished for file {fileName}"); - public bool emptyFile; + new StatusUpdateMessage($"Delius parser finished for file {FileName}"); + public bool EmptyFile { get; set; } [JsonConstructor] public DeliusParserFinishedMessage() - { } + { + Queue = TStagingQueue.DeliusImport; + } - public DeliusParserFinishedMessage(string fileName, string filePath,bool emptyFile) + public DeliusParserFinishedMessage(string fileName, string filePath, bool emptyFile) : this() { - routingKey = TStagingQueue.DeliusImport; - this.fileName = fileName; - this.filePath = filePath; - this.emptyFile = emptyFile; + FileName = fileName; + FilePath = filePath; + EmptyFile = emptyFile; } } diff --git a/src/Libraries/Messaging/Messages/StagingMessages/KickoffMessage.cs b/src/Libraries/Messaging/Messages/StagingMessages/KickoffMessage.cs index 8c70910..bf488f1 100644 --- a/src/Libraries/Messaging/Messages/StagingMessages/KickoffMessage.cs +++ b/src/Libraries/Messaging/Messages/StagingMessages/KickoffMessage.cs @@ -12,6 +12,6 @@ public class KickoffMessage : StagingMessage [JsonConstructor] public KickoffMessage() { - routingKey = TStagingQueue.Kickoff; + Queue = TStagingQueue.Kickoff; } } diff --git a/src/Libraries/Messaging/Messages/StagingMessages/Offloc/ClearHalfCleanedOfflocFiles.cs b/src/Libraries/Messaging/Messages/StagingMessages/Offloc/ClearHalfCleanedOfflocFiles.cs index f4162d8..d97cd1b 100644 --- a/src/Libraries/Messaging/Messages/StagingMessages/Offloc/ClearHalfCleanedOfflocFiles.cs +++ b/src/Libraries/Messaging/Messages/StagingMessages/Offloc/ClearHalfCleanedOfflocFiles.cs @@ -13,6 +13,6 @@ public class ClearHalfCleanedOfflocFiles : StagingMessage [JsonConstructor] public ClearHalfCleanedOfflocFiles() { - routingKey = TStagingQueue.OfflocFilesClear; + Queue = TStagingQueue.OfflocFilesClear; } } diff --git a/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocCleanerFinishedMessage.cs b/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocCleanerFinishedMessage.cs index 7128cd2..66e066d 100644 --- a/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocCleanerFinishedMessage.cs +++ b/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocCleanerFinishedMessage.cs @@ -16,7 +16,7 @@ public class OfflocCleanerFinishedMessage : StagingMessage [JsonConstructor] public OfflocCleanerFinishedMessage() { - routingKey = TStagingQueue.OfflocParser; + Queue = TStagingQueue.OfflocParser; } //For parallel processing. public OfflocCleanerFinishedMessage(string[] filesToParse, int[] redundantFields) @@ -29,6 +29,6 @@ public OfflocCleanerFinishedMessage(string[] filesToParse, int[] redundantFields public OfflocCleanerFinishedMessage(string fileToParse, int[] redundantFields, string fileName) : this(new string[] { fileToParse }, redundantFields) { - this.fileName = fileName; + FileName = fileName; } } diff --git a/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocDownloadFinished.cs b/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocDownloadFinished.cs index 3f691cf..4bb77d3 100644 --- a/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocDownloadFinished.cs +++ b/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocDownloadFinished.cs @@ -15,8 +15,8 @@ public class OfflocDownloadFinished : StagingMessage [JsonConstructor] public OfflocDownloadFinished(string fileName, int fileId, string? archiveFileName = null) { - routingKey = TStagingQueue.OfflocCleaner; - base.fileName = fileName; + Queue = TStagingQueue.OfflocCleaner; + base.FileName = fileName; FileId = fileId; ArchiveFileName = archiveFileName; } diff --git a/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocParserFinishedMessage.cs b/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocParserFinishedMessage.cs index cdb971b..7d13f06 100644 --- a/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocParserFinishedMessage.cs +++ b/src/Libraries/Messaging/Messages/StagingMessages/Offloc/OfflocParserFinishedMessage.cs @@ -5,19 +5,20 @@ namespace Messaging.Messages.StagingMessages; public class OfflocParserFinishedMessage : StagingMessage { - public string filePath = string.Empty; + public string FilePath { get; set; } = string.Empty; public override StatusUpdateMessage StatusMessage => - new StatusUpdateMessage($"Offloc Parser finished for file {filePath.Split('/').Last()}."); - public bool emptyFile; + new StatusUpdateMessage($"Offloc Parser finished for file {FilePath.Split('/').Last()}."); + public bool EmptyFile { get; set; } [JsonConstructor] public OfflocParserFinishedMessage() - { } + { + Queue = TStagingQueue.OfflocImport; + } - public OfflocParserFinishedMessage(string filePath,bool emptyFile) + public OfflocParserFinishedMessage(string filePath, bool emptyFile) : this() { - routingKey = TStagingQueue.OfflocImport; - this.filePath = filePath; - this.emptyFile = emptyFile; + FilePath = filePath; + EmptyFile = emptyFile; } } diff --git a/src/Libraries/Messaging/Messages/StagingMessages/StagingMessage.cs b/src/Libraries/Messaging/Messages/StagingMessages/StagingMessage.cs index a40c393..37b44d3 100644 --- a/src/Libraries/Messaging/Messages/StagingMessages/StagingMessage.cs +++ b/src/Libraries/Messaging/Messages/StagingMessages/StagingMessage.cs @@ -4,6 +4,9 @@ namespace Messaging.Messages.StagingMessages; public abstract class StagingMessage : Message { - public TStagingQueue routingKey { get; set; } - public string fileName = string.Empty; //To handle processing of multiple files. + public TStagingQueue Queue { get; set; } + public string FileName { get; set; } = string.Empty; + + public override string Exchange => Exchanges.staging; + public override string RoutingKey => Queue.ToString(); } diff --git a/src/Libraries/Messaging/Messages/StatusMessages/StagingFinishedMessage.cs b/src/Libraries/Messaging/Messages/StatusMessages/StagingFinishedMessage.cs index 68bda23..a35152e 100644 --- a/src/Libraries/Messaging/Messages/StatusMessages/StagingFinishedMessage.cs +++ b/src/Libraries/Messaging/Messages/StatusMessages/StagingFinishedMessage.cs @@ -5,8 +5,10 @@ namespace Messaging.Messages.StatusMessages; public class StagingFinishedMessage : StatusUpdateMessage { + public override string RoutingKey { get; } + public StagingFinishedMessage() { - RoutingKey = TStatusQueue.StagingFinished; + RoutingKey = TStatusQueue.StagingFinished.ToString(); } } diff --git a/src/Libraries/Messaging/Messages/StatusMessages/StatusUpdateMessage.cs b/src/Libraries/Messaging/Messages/StatusMessages/StatusUpdateMessage.cs index 7111ce9..79b2485 100644 --- a/src/Libraries/Messaging/Messages/StatusMessages/StatusUpdateMessage.cs +++ b/src/Libraries/Messaging/Messages/StatusMessages/StatusUpdateMessage.cs @@ -5,18 +5,19 @@ namespace Messaging.Messages.StatusMessages; -public class StatusUpdateMessage +public class StatusUpdateMessage : IMessage { - public TStatusQueue RoutingKey { get; set; } + public virtual string RoutingKey { get; } = TStatusQueue.StatusUpdate.ToString(); public string Message { get; set; } public string Caller { get; set; } + public string Exchange => Exchanges.status; + [JsonConstructor] public StatusUpdateMessage() : this(string.Empty) { } public StatusUpdateMessage(string message) { - RoutingKey = TStatusQueue.StatusUpdate; Message = message; Caller = Assembly.GetCallingAssembly().GetName().Name!; } diff --git a/src/Libraries/Messaging/Queues/TDbQueue.cs b/src/Libraries/Messaging/Queues/TDbQueue.cs index ccd511a..2bc24fa 100644 --- a/src/Libraries/Messaging/Queues/TDbQueue.cs +++ b/src/Libraries/Messaging/Queues/TDbQueue.cs @@ -8,14 +8,11 @@ public enum TDbQueue //Outgoing requests. AssociateOfflocFileWithArchive, GetProcessedOfflocFiles, - GetOfflocFileDates, - DeliusGetLastFullId, GetProcessedDeliusFiles, StageOffloc, StageDelius, MergeOffloc, MergeDelius, - StandardiseDelius, ClearOfflocStaging, ClearDeliusStaging, OfflocFileProcessingStarted, @@ -24,16 +21,13 @@ public enum TDbQueue IsOfflocReadyForProcessing, GetLastProcessedDeliusFile, GetLastProcessedOfflocFile, - //Incoming requests. + //Incoming responses. ReturnedOfflocFiles, - ReturnedOfflocFileDates, - ReturnDeliusGetLastFull, ReturnedDeliusFiles, ResultStageOffloc, ResultStageDelius, ResultMergeOffloc, ResultMergeDelius, - ResultStandardiseDelius, ResultClearDelius, ResultClearOffloc, ResultOfflocFileProcessingStarted, diff --git a/src/Libraries/Messaging/Services/RabbitService.cs b/src/Libraries/Messaging/Services/RabbitService.cs index 2fa08c0..a3c024f 100644 --- a/src/Libraries/Messaging/Services/RabbitService.cs +++ b/src/Libraries/Messaging/Services/RabbitService.cs @@ -3,17 +3,10 @@ using RabbitMQ.Client.Events; using System.Text.Json; using Messaging.Queues; -using Messaging.Messages.StagingMessages; -using Messaging.Messages.MergingMessages; -using Messaging.Messages.StatusMessages; using Messaging.Interfaces; using Messaging.Messages.DbMessages.Sending; using Messaging.Messages.DbMessages.Receiving; -using EnvironmentSetup; using Messaging.Messages; -using Messaging.Messages.ImportMessages; -using Messaging.Messages.BlockingMessages; -using Messaging.Messages.MatchingMessages; namespace Messaging.Services; @@ -23,65 +16,41 @@ public class RabbitService : IMessageService private IConnection connection; private SemaphoreSlim dbSemaphore = new SemaphoreSlim(1, 1); - private JsonSerializerOptions serializerOpts = new JsonSerializerOptions { IncludeFields = true }; + private JsonSerializerOptions serializerOpts = new JsonSerializerOptions() { IncludeFields = true }; private RabbitService() { } - public static async Task CreateAsync(RabbitHostingContextWrapper hostingContext) + public static async Task CreateAsync(Uri connectionUri) { var service = new RabbitService(); - await service.InitializeAsync(hostingContext); + await service.InitializeAsync(connectionUri); return service; } - private async Task InitializeAsync(RabbitHostingContextWrapper hostingContext) + private async Task InitializeAsync(Uri connectionUri) { - ConnectionFactory factory; - - // Configures connection factory based on whether a full URI is provided or individual components. - if (hostingContext.Uri is not null) - { - factory = new ConnectionFactory() { Uri = hostingContext.Uri }; - } - else - { - factory = new ConnectionFactory() - { - HostName = hostingContext.Context, - UserName = hostingContext.Username, - Password = hostingContext.Password - }; - } + var factory = new ConnectionFactory() { Uri = connectionUri }; connection = await factory.CreateConnectionAsync(); channel = await connection.CreateChannelAsync(); - //Declares separate exchanges for each group of messages and binds queue. - TStagingQueue[] stagingQueues = Enum.GetValues(); - TMergingQueue[] mergingQueues = Enum.GetValues(); - TDbQueue[] dbQueues = Enum.GetValues(); - TStatusQueue[] statusQueues = Enum.GetValues(); - TImportQueue[] importQueues = Enum.GetValues(); - TBlockingQueue[] blockingQueues = Enum.GetValues(); - TMatchingQueue[] matchingQueues = Enum.GetValues(); + string[] exchanges = [Exchanges.staging, Exchanges.merging, Exchanges.database, + Exchanges.status, Exchanges.import, Exchanges.blocking, Exchanges.matching]; - await channel.ExchangeDeclareAsync(Exchanges.staging, ExchangeType.Direct); - await channel.ExchangeDeclareAsync(Exchanges.merging, ExchangeType.Direct); - await channel.ExchangeDeclareAsync(Exchanges.database, ExchangeType.Direct); - await channel.ExchangeDeclareAsync(Exchanges.status, ExchangeType.Direct); - await channel.ExchangeDeclareAsync(Exchanges.import, ExchangeType.Direct); - await channel.ExchangeDeclareAsync(Exchanges.blocking, ExchangeType.Direct); - await channel.ExchangeDeclareAsync(Exchanges.matching, ExchangeType.Direct); + foreach (var exchange in exchanges) + { + await channel.ExchangeDeclareAsync(exchange, ExchangeType.Direct); + } - await InitializeQueueAsync(stagingQueues, Exchanges.staging); - await InitializeQueueAsync(mergingQueues, Exchanges.merging); - await InitializeQueueAsync(dbQueues, Exchanges.database); - await InitializeQueueAsync(statusQueues, Exchanges.status); - await InitializeQueueAsync(importQueues, Exchanges.import); - await InitializeQueueAsync(blockingQueues, Exchanges.blocking); - await InitializeQueueAsync(matchingQueues, Exchanges.matching); + await InitializeQueueAsync(Enum.GetValues(), Exchanges.staging); + await InitializeQueueAsync(Enum.GetValues(), Exchanges.merging); + await InitializeQueueAsync(Enum.GetValues(), Exchanges.database); + await InitializeQueueAsync(Enum.GetValues(), Exchanges.status); + await InitializeQueueAsync(Enum.GetValues(), Exchanges.import); + await InitializeQueueAsync(Enum.GetValues(), Exchanges.blocking); + await InitializeQueueAsync(Enum.GetValues(), Exchanges.matching); } private async Task InitializeQueueAsync(T[] queues, string exchange) where T : Enum @@ -99,118 +68,56 @@ await channel.QueueDeclareAsync( } } - public async Task StatusPublishAsync(T message) where T : StatusUpdateMessage + public async Task PublishAsync(T message) where T : IMessage { await channel.BasicPublishAsync( - exchange: Exchanges.status, - routingKey: message.RoutingKey.ToString(), + exchange: message.Exchange, + routingKey: message.RoutingKey, mandatory: false, - body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, serializerOpts)) - ); - } - - public async Task StatusSubscribeAsync(Action handler, TStatusQueue queue) where T : StatusUpdateMessage - { - var consumer = new AsyncEventingBasicConsumer(channel); - await channel.BasicConsumeAsync(queue.ToString(), true, consumer); - - consumer.ReceivedAsync += (model, ea) => { handler.Invoke(JsonSerializer.Deserialize(Encoding.UTF8.GetString(ea.Body.ToArray()), new JsonSerializerOptions { IncludeFields = true })!); return Task.CompletedTask; }; - } + body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, message!.GetType(), serializerOpts)) + ); - public async Task StagingPublishAsync(T message) where T : StagingMessage - { - await channel.BasicPublishAsync( - exchange: Exchanges.staging, - routingKey: message.routingKey.ToString(), - mandatory: false, - body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, serializerOpts)) - ); - await AssociatedMessagePublishAsync(message); + await PublishStatusAsync(message); } - - public async Task StagingSubscribeAsync(Action handler, TStagingQueue queue) where T : StagingMessage + + public async Task SubscribeAsync(Action handler, Enum queue) where T : IMessage { var consumer = new AsyncEventingBasicConsumer(channel); await channel.BasicConsumeAsync(queue.ToString(), true, consumer); - consumer.ReceivedAsync += (model, ea) => { handler.Invoke(JsonSerializer.Deserialize(Encoding.UTF8.GetString(ea.Body.ToArray()), new JsonSerializerOptions { IncludeFields = true })!); return Task.CompletedTask; }; - } - - private async Task DbPublishRequestAsync(T message) where T : DbRequestMessage - { - var props = new BasicProperties - { - ReplyTo = message.ReplyQueue.ToString() - }; - - await channel.BasicPublishAsync( - exchange: Exchanges.database, - routingKey: message.Queue.ToString(), - mandatory: false, - basicProperties: props, - body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, serializerOpts)) - ); - - await AssociatedMessagePublishAsync(message); - } - - public async Task DbPublishResponseAsync(T message) where T : DbResponseMessage - { - await channel.BasicPublishAsync( - exchange: Exchanges.database, - routingKey: message.Queue.ToString(), - mandatory: false, - body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, serializerOpts)) - ); - - await AssociatedMessagePublishAsync(message); - } - - public async Task SubscribeToDbRequestAsync(Action handler, TDbQueue queue) where T : DbRequestMessage - { - var consumer = new AsyncEventingBasicConsumer(channel); - await channel.BasicConsumeAsync(queue.ToString(), true, consumer); - consumer.ReceivedAsync += (model, ea) => { - var msg = JsonSerializer.Deserialize(Encoding.UTF8.GetString(ea.Body.ToArray()), serializerOpts)!; - handler.Invoke(msg); + handler.Invoke(JsonSerializer.Deserialize(Encoding.UTF8.GetString(ea.Body.ToArray()), serializerOpts)!); return Task.CompletedTask; }; } - public async Task SendDbRequestAndWaitForResponseAsync(TRequest message) - where TRequest : DbRequestMessage - where TResponse : DbResponseMessage + public async Task SendDbRequestAndWaitForResponseAsync(DbRequestMessage message) + where TResponse : DbResponseMessage, new() { await dbSemaphore.WaitAsync(); - - var consumer = new AsyncEventingBasicConsumer(channel); - string tempConsumer = await channel.BasicConsumeAsync(message.ReplyQueue.ToString(), true, consumer); - + var responseQueue = new TResponse().Queue.ToString(); var taskCompletion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); - AsyncEventHandler eventHandler = (model, ea) => - { - DbMessageReceivedCallback(model, ea); - return Task.CompletedTask; - }; - - consumer.ReceivedAsync += eventHandler; + // Subscribe to response queue temporarily + var consumer = new AsyncEventingBasicConsumer(channel); + + consumer.ReceivedAsync += DbMessageReceivedCallback; + string consumerTag = await channel.BasicConsumeAsync(responseQueue, true, consumer); try { - await DbPublishRequestAsync(message); + await DbPublishRequestAsync(message, responseQueue); return await taskCompletion.Task; } finally { - consumer.ReceivedAsync -= eventHandler; - await channel.BasicCancelAsync(tempConsumer); + consumer.ReceivedAsync -= DbMessageReceivedCallback; + await channel.BasicCancelAsync(consumerTag); dbSemaphore.Release(); } - void DbMessageReceivedCallback(object? model, BasicDeliverEventArgs ea) + Task DbMessageReceivedCallback(object? model, BasicDeliverEventArgs ea) { try { @@ -221,107 +128,35 @@ void DbMessageReceivedCallback(object? model, BasicDeliverEventArgs ea) { taskCompletion.SetException(ex); } - } - } - - public async Task MergingPublishAsync(T message) where T : MergingMessage - { - await channel.BasicPublishAsync( - exchange: Exchanges.merging, - routingKey: message.routingKey.ToString(), - mandatory: false, - body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, serializerOpts)) - ); - - await AssociatedMessagePublishAsync(message); - } - - public async Task MergingSubscribeAsync(Action handler, TMergingQueue queue) where T : MergingMessage - { - var consumer = new AsyncEventingBasicConsumer(channel); - await channel.BasicConsumeAsync(queue.ToString(), true, consumer); - - consumer.ReceivedAsync += (model, ea) => - { - handler.Invoke(JsonSerializer.Deserialize(Encoding.UTF8.GetString(ea.Body.ToArray()), new JsonSerializerOptions { IncludeFields = true })!); return Task.CompletedTask; - }; - } - - public async Task ImportPublishAsync(T message) where T : ImportMessage - { - await channel.BasicPublishAsync( - exchange: Exchanges.import, - routingKey: message.routingKey.ToString(), - mandatory: false, - body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, serializerOpts)) - ); - await AssociatedMessagePublishAsync(message); + } } - public async Task ImportSubscribeAsync(Action handler, TImportQueue queue) where T : ImportMessage + private async Task DbPublishRequestAsync(DbRequestMessage message, string replyQueue) where TResponse : DbResponseMessage, new() { - var consumer = new AsyncEventingBasicConsumer(channel); - await channel.BasicConsumeAsync(queue.ToString(), true, consumer); - - consumer.ReceivedAsync += (model, ea) => + var props = new BasicProperties { - handler.Invoke(JsonSerializer.Deserialize(Encoding.UTF8.GetString(ea.Body.ToArray()), new JsonSerializerOptions { IncludeFields = true })!); - return Task.CompletedTask; + ReplyTo = replyQueue }; - } - public async Task BlockingPublishAsync(T message) where T : BlockingMessage - { + var json = JsonSerializer.Serialize(message, message.GetType(), serializerOpts); + await channel.BasicPublishAsync( - exchange: Exchanges.blocking, - routingKey: message.routingKey.ToString(), + exchange: Exchanges.database, + routingKey: message.Queue.ToString(), mandatory: false, - body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, serializerOpts)) - ); - await AssociatedMessagePublishAsync(message); - } - - public async Task BlockingSubscribeAsync(Action handler, TBlockingQueue queue) where T : BlockingMessage - { - var consumer = new AsyncEventingBasicConsumer(channel); - await channel.BasicConsumeAsync(queue.ToString(), true, consumer); + basicProperties: props, + body: Encoding.UTF8.GetBytes(json)); - consumer.ReceivedAsync += (model, ea) => - { - handler.Invoke(JsonSerializer.Deserialize(Encoding.UTF8.GetString(ea.Body.ToArray()), new JsonSerializerOptions { IncludeFields = true })!); - return Task.CompletedTask; - }; - } + await PublishStatusAsync(message); + } - private async Task AssociatedMessagePublishAsync(T message) where T : Message + private async Task PublishStatusAsync(T message) where T : IMessage { - if (message.StatusMessage.Message != string.Empty) + if (message is Message { StatusMessage.Message: not "" } status) { - await StatusPublishAsync(message.StatusMessage); + await PublishAsync(status.StatusMessage); } } - public async Task MatchingPublishAsync(T message) where T : MatchingMessage - { - await channel.BasicPublishAsync( - exchange: Exchanges.matching, - routingKey: message.routingKey.ToString(), - mandatory: false, - body: Encoding.UTF8.GetBytes(JsonSerializer.Serialize(message, serializerOpts)) - ); - await AssociatedMessagePublishAsync(message); - } - - public async Task MatchingSubscribeAsync(Action handler, TMatchingQueue queue) where T : MatchingMessage - { - var consumer = new AsyncEventingBasicConsumer(channel); - await channel.BasicConsumeAsync(queue.ToString(), true, consumer); - - consumer.ReceivedAsync += (model, ea) => - { - handler.Invoke(JsonSerializer.Deserialize(Encoding.UTF8.GetString(ea.Body.ToArray()), new JsonSerializerOptions { IncludeFields = true })!); - return Task.CompletedTask; - }; - } } \ No newline at end of file diff --git a/src/Logging/LoggingBackgroundService.cs b/src/Logging/LoggingBackgroundService.cs index 86046c5..0828c8a 100644 --- a/src/Logging/LoggingBackgroundService.cs +++ b/src/Logging/LoggingBackgroundService.cs @@ -6,14 +6,13 @@ namespace Logging; public class LoggingBackgroundService( ILogger logger, - IStatusMessagingService statusService) : BackgroundService + IMessageService messageService) : BackgroundService { protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await statusService.StatusSubscribeAsync(Log, TStatusQueue.StatusUpdate); - await statusService.StatusSubscribeAsync(Log, TStatusQueue.StagingFinished); - - await statusService.StatusPublishAsync(new StatusUpdateMessage("Logger configured.")); + await messageService.SubscribeAsync(Log, TStatusQueue.StatusUpdate); + await messageService.SubscribeAsync(Log, TStatusQueue.StagingFinished); + await messageService.PublishAsync(new StatusUpdateMessage("Logger configured.")); } private void Log(StatusUpdateMessage statusUpdate) diff --git a/src/Logging/Program.cs b/src/Logging/Program.cs index dafdd73..691ad19 100644 --- a/src/Logging/Program.cs +++ b/src/Logging/Program.cs @@ -1,25 +1,11 @@ -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - -try +using Messaging.Extensions; +try { var builder = Host.CreateApplicationBuilder(args); - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - - builder.Configuration.ConfigureByEnvironment(); - - builder.Services.ConfigureServices(builder.Configuration); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); var app = builder.Build(); diff --git a/src/Logging/Properties/launchSettings.json b/src/Logging/Properties/launchSettings.json deleted file mode 100644 index 9851d14..0000000 --- a/src/Logging/Properties/launchSettings.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "profiles": { - "Logging": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - } - } -} \ No newline at end of file diff --git a/src/Logging/appsettings.json b/src/Logging/appsettings.json index c2ca43d..a003c45 100644 --- a/src/Logging/appsettings.json +++ b/src/Logging/appsettings.json @@ -1,3 +1,25 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Logging-.txt" + "ConnectionStrings": { + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/Logging-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + } } \ No newline at end of file diff --git a/src/Matching.Engine/Extensions/DmsExtensions.cs b/src/Matching.Engine/Extensions/DmsExtensions.cs deleted file mode 100644 index 232dd26..0000000 --- a/src/Matching.Engine/Extensions/DmsExtensions.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Messaging.Services; -using EnvironmentSetup; - -namespace Matching.Engine.Extensions; - -public static class DmsExtensions -{ - public static void AddMessagingServices(this IHostApplicationBuilder builder) - { - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - } - -} diff --git a/src/Matching.Engine/Matching.Engine.csproj b/src/Matching.Engine/Matching.Engine.csproj index 7580da1..0e48e1c 100644 --- a/src/Matching.Engine/Matching.Engine.csproj +++ b/src/Matching.Engine/Matching.Engine.csproj @@ -5,21 +5,7 @@ enable enable - - - - - - - - - Always - - - Always - - - + @@ -31,8 +17,14 @@ - - + + + + + + + Always + diff --git a/src/Matching.Engine/Program.cs b/src/Matching.Engine/Program.cs index 14f52a5..ece050e 100644 --- a/src/Matching.Engine/Program.cs +++ b/src/Matching.Engine/Program.cs @@ -1,8 +1,4 @@ -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); +using Messaging.Extensions; try { @@ -14,9 +10,8 @@ container.RegisterScorers(builder.Configuration); }); - builder.Configuration.ConfigureByEnvironment(); - builder.Services.ConfigureServices(builder.Configuration); - builder.AddMessagingServices(); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); builder.AddApplicationServices(); diff --git a/src/Matching.Engine/Properties/launchSettings.json b/src/Matching.Engine/Properties/launchSettings.json deleted file mode 100644 index 04cea13..0000000 --- a/src/Matching.Engine/Properties/launchSettings.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "profiles": { - "Matching.Comparer": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - } - } -} diff --git a/src/Matching.Engine/Services/ClusteringService.cs b/src/Matching.Engine/Services/ClusteringService.cs index 2fc0ad0..efcd051 100644 --- a/src/Matching.Engine/Services/ClusteringService.cs +++ b/src/Matching.Engine/Services/ClusteringService.cs @@ -7,8 +7,7 @@ namespace Matching.Engine.Services; public class ClusteringService( - IStatusMessagingService statusMessagingService, - IMatchingMessagingService matchingMessagingService, + IMessageService messageService, IClusteringRepository clusteringRepository, ILogger logger) : BackgroundService { @@ -16,20 +15,15 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { - await Task.Run(async () => + await messageService.SubscribeAsync(async (message) => { - await matchingMessagingService.MatchingSubscribeAsync(async (message) => - { - await PreProcessAsync(stoppingToken); + await PreProcessAsync(stoppingToken); + }, TMatchingQueue.MatchingScoreCandidatesFinished); - }, TMatchingQueue.MatchingScoreCandidatesFinished); - - await matchingMessagingService.MatchingSubscribeAsync(async (message) => - { - await PostProcessAsync(stoppingToken); - }, TMatchingQueue.ClusteringPreProcessingFinished); - - }, stoppingToken); + await messageService.SubscribeAsync(async (message) => + { + await PostProcessAsync(stoppingToken); + }, TMatchingQueue.ClusteringPreProcessingFinished); } catch (Exception ex) { @@ -40,15 +34,15 @@ await matchingMessagingService.MatchingSubscribeAsync logger, IOptions> options, - IStatusMessagingService statusMessagingService, - IBlockingMessagingService blockingMessagingService, - IMatchingMessagingService matchingMessagingService, + IMessageService messageService, IMatchingRepository matchingRepository, IClusteringRepository clusteringRepository, MatcherCache cache, @@ -25,42 +23,39 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { - await Task.Run(async () => - { - matchingOptions = options.Value.ToDictionary(o => o.MatchingKey); + matchingOptions = options.Value.ToDictionary(o => o.MatchingKey); - await blockingMessagingService.BlockingSubscribeAsync(async (message) => - { - var items = await matchingRepository.GetAllAsync(); + await messageService.SubscribeAsync(async (message) => + { + var items = await matchingRepository.GetAllAsync(); - var records = items - .Cast>() - .ToImmutableArray(); + var records = items + .Cast>() + .ToImmutableArray(); - await ProcessAsync(records, stoppingToken); + await ProcessAsync(records, stoppingToken); - records.Clear(); + records.Clear(); - await matchingMessagingService.MatchingPublishAsync(new MatchingScoreCandidatesMessage()); + await messageService.PublishAsync(new MatchingScoreCandidatesMessage()); - }, TBlockingQueue.BlockingFinished); + }, TBlockingQueue.BlockingFinished); - await matchingMessagingService.MatchingSubscribeAsync(async (message) => - { - var items = await clusteringRepository.GetAllAsync(); + await messageService.SubscribeAsync(async (message) => + { + var items = await clusteringRepository.GetAllAsync(); - var records = items - .Cast>() - .ToImmutableArray(); + var records = items + .Cast>() + .ToImmutableArray(); - await ProcessAsync(records, stoppingToken); + await ProcessAsync(records, stoppingToken); - records.Clear(); + records.Clear(); - await matchingMessagingService.MatchingPublishAsync(new MatchingScoreOutstandingEdgesMessage()); + await messageService.PublishAsync(new MatchingScoreOutstandingEdgesMessage()); - }, TMatchingQueue.ClusteringPreProcessingStarted); - }, stoppingToken); + }, TMatchingQueue.ClusteringPreProcessingStarted); } catch(Exception ex) { @@ -70,7 +65,7 @@ await matchingMessagingService.MatchingSubscribeAsync> records, CancellationToken stoppingToken) { - await statusMessagingService.StatusPublishAsync(new StatusUpdateMessage("Comparing candidates...")); + await messageService.PublishAsync(new StatusUpdateMessage("Comparing candidates...")); queue.Results.Clear(); @@ -103,7 +98,6 @@ private async Task ProcessAsync(IEnumerable> records queue.Results.Add(result); } - await Task.CompletedTask; } } diff --git a/src/Matching.Engine/Services/ScorerService.cs b/src/Matching.Engine/Services/ScorerService.cs index d783f21..6c7c133 100644 --- a/src/Matching.Engine/Services/ScorerService.cs +++ b/src/Matching.Engine/Services/ScorerService.cs @@ -6,7 +6,7 @@ namespace Matching.Engine.Services; public class ScorerService( - IMatchingMessagingService matchingMessagingService, + IMessageService matchingMessagingService, IMatchingRepository matchingRepository, IClusteringRepository clusteringRepository, ILogger logger, @@ -20,29 +20,25 @@ protected override async Task ExecuteAsync(CancellationToken stoppingToken) { try { - await Task.Run(async () => - { - matchingOptions = options.Value.ToDictionary(o => o.MatchingKey); - - await matchingMessagingService.MatchingSubscribeAsync(async (message) => - { - var results = await ScoreAsync(stoppingToken); - await matchingRepository.BulkInsertAsync(results); - results = null; + matchingOptions = options.Value.ToDictionary(o => o.MatchingKey); - await matchingMessagingService.MatchingPublishAsync(new MatchingScoreCandidatesFinishedMessage()); - }, TMatchingQueue.MatchingScoreCandidatesMessage); + await matchingMessagingService.SubscribeAsync(async (message) => + { + var results = await ScoreAsync(stoppingToken); + await matchingRepository.BulkInsertAsync(results); + results = null; - await matchingMessagingService.MatchingSubscribeAsync(async (message) => - { - var results = await ScoreAsync(stoppingToken); - await clusteringRepository.BulkInsertAsync(results); - results = null; + await matchingMessagingService.PublishAsync(new MatchingScoreCandidatesFinishedMessage()); + }, TMatchingQueue.MatchingScoreCandidatesMessage); - await matchingMessagingService.MatchingPublishAsync(new ClusteringPreProcessingFinishedMessage()); - }, TMatchingQueue.MatchingScoreOutstandingEdgesMessage); + await matchingMessagingService.SubscribeAsync(async (message) => + { + var results = await ScoreAsync(stoppingToken); + await clusteringRepository.BulkInsertAsync(results); + results = null; - }, stoppingToken); + await matchingMessagingService.PublishAsync(new ClusteringPreProcessingFinishedMessage()); + }, TMatchingQueue.MatchingScoreOutstandingEdgesMessage); } catch (Exception ex) { diff --git a/src/Matching.Engine/appsettings.Development.json b/src/Matching.Engine/appsettings.Development.json deleted file mode 100644 index 0db3279..0000000 --- a/src/Matching.Engine/appsettings.Development.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - -} diff --git a/src/Matching.Engine/appsettings.json b/src/Matching.Engine/appsettings.json index c6243ed..9ac0695 100644 --- a/src/Matching.Engine/appsettings.json +++ b/src/Matching.Engine/appsettings.json @@ -1,8 +1,28 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Matching.Engine-.txt", "ConnectionStrings": { - "MatchingDb": "Server=localhost,32771;User ID=sa;Password=Pass@word;TrustServerCertificate=true;Database=MatchingDb", - "ClusterDb": "Server=localhost,32771;User ID=sa;Password=Pass@word;TrustServerCertificate=true;Database=ClusterDb" + "ClusterDb": "", + "MatchingDb": "", + "RabbitMQ": "" + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/Matching.Engine-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, "ProcessorConfig": { /* diff --git a/src/Meow/Program.cs b/src/Meow/Program.cs index b00f06a..fa27d6b 100644 --- a/src/Meow/Program.cs +++ b/src/Meow/Program.cs @@ -3,29 +3,20 @@ using Meow.Features.Participants.Handlers; using Rebus.Serialization; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { var builder = Host.CreateApplicationBuilder(args); - var services = builder.Services; - var configuration = builder.Configuration; + builder.AddDmsCoreWorkerService(); builder.Services.AddScoped(); builder.AddDatabaseServices(); - services.ConfigureServices(configuration); - builder.Services.AddRebus(configure => { - var connectionString = configuration.GetConnectionString("RabbitMQ"); - var rabbitSettings = configuration.GetRequiredSection("RabbitSettings"); + var connectionString = builder.Configuration.GetConnectionString("CatsRabbitMQ"); + var rabbitSettings = builder.Configuration.GetRequiredSection("RabbitSettings"); string queueName = rabbitSettings["DmsService"]!, exchangeName = rabbitSettings["DirectExchange"]!, @@ -44,9 +35,10 @@ builder.Services.AddHostedService(); - var host = builder.Build(); - - await host.RunAsync(); + var app = builder.Build(); + + Log.Information("Starting application"); + app.Run(); return 0; } diff --git a/src/Meow/appsettings.json b/src/Meow/appsettings.json index f80dfae..1c83825 100644 --- a/src/Meow/appsettings.json +++ b/src/Meow/appsettings.json @@ -1,9 +1,30 @@ { "ConnectionStrings": { - "ClusterDb": "Data Source=localhost;Initial Catalog=ClusterDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "DeliusRunningPictureDb": "Data Source=localhost;Initial Catalog=DeliusRunningPictureDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "OfflocRunningPictureDb": "Data Source=localhost;Initial Catalog=OfflocRunningPictureDb;User Id=sa;Password=YourStrong@Passw0rd;TrustServerCertificate=true", - "RabbitMQ": "amqp://guest:guest@localhost:5672" + "AuditDb": "", + "DeliusRunningPictureDb": "", + "OfflocRunningPictureDb": "", + "ClusterDb": "", + "CatsRabbitMQ": "" // Meow uses the CATS RabbitMQ instance + }, + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/Meow-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, "RabbitSettings": { "TopicExchange": "CatsTopics", diff --git a/src/Offloc.Cleaner/Offloc.Cleaner.csproj b/src/Offloc.Cleaner/Offloc.Cleaner.csproj index faecf36..21d2dfa 100644 --- a/src/Offloc.Cleaner/Offloc.Cleaner.csproj +++ b/src/Offloc.Cleaner/Offloc.Cleaner.csproj @@ -4,8 +4,6 @@ Exe enable enable - Linux - ..\..\.. bd191539-371a-49fd-a9f4-f6c77878fc6a @@ -32,7 +30,5 @@ Always - - - + diff --git a/src/Offloc.Cleaner/OfflocCleanerBackgroundService.cs b/src/Offloc.Cleaner/OfflocCleanerBackgroundService.cs index b391457..7878bf2 100644 --- a/src/Offloc.Cleaner/OfflocCleanerBackgroundService.cs +++ b/src/Offloc.Cleaner/OfflocCleanerBackgroundService.cs @@ -13,38 +13,41 @@ namespace Offloc.Cleaner; public class OfflocCleanerBackgroundService( - IStagingMessagingService stagingService, - IDbMessagingService dbService, - ICleaningStrategy cleaningService, - IStatusMessagingService statusService) : BackgroundService + IMessageService messageService, + ICleaningStrategy cleaningService) : BackgroundService { protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await Task.Run(() => + await messageService.SubscribeAsync(async (message) => { - stagingService.StagingSubscribeAsync(async (message) => await ParseFileAsync(message), TStagingQueue.OfflocCleaner); - }, stoppingToken); + await ParseFileAsync(message); + }, TStagingQueue.OfflocCleaner); } private async Task ParseFileAsync(OfflocDownloadFinished message) { - string file = message.fileName; + string file = message.FileName; if (await HasAlreadyBeenProcessedAsync(file)) { - await statusService.StatusPublishAsync(new StatusUpdateMessage($"File {file} has already been processed")); - await stagingService.StagingPublishAsync(new OfflocParserFinishedMessage("File already processed", true)); + await messageService.PublishAsync(new StatusUpdateMessage($"File {file} has already been processed")); + await messageService.PublishAsync(new OfflocParserFinishedMessage("File already processed", emptyFile: true)); } else { - var request = new OfflocFileProcessingStarted(message.fileName, message.FileId, message.ArchiveFileName); - await dbService.SendDbRequestAndWaitForResponseAsync(request); + var request = new StartOfflocFileProcessingRequest + { + FileName = message.FileName, + FileId = message.FileId, + ArchiveName = message.ArchiveFileName + }; + await messageService.SendDbRequestAndWaitForResponseAsync(request); await cleaningService.CleanFile(file); } } private async Task HasAlreadyBeenProcessedAsync(string file) { - var res = await dbService.SendDbRequestAndWaitForResponseAsync(new GetOfflocFilesMessage()); - return res.offlocFiles.Contains(file); + var res = await messageService.SendDbRequestAndWaitForResponseAsync(new GetOfflocFilesRequest()); + return res.OfflocFiles.Contains(file); } } diff --git a/src/Offloc.Cleaner/Program.cs b/src/Offloc.Cleaner/Program.cs index ede2a27..071d084 100644 --- a/src/Offloc.Cleaner/Program.cs +++ b/src/Offloc.Cleaner/Program.cs @@ -1,44 +1,25 @@ - +using Messaging.Extensions; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Offloc.Cleaner; -using Messaging.Services; -using Messaging.Interfaces; using Microsoft.Extensions.Configuration; using Offloc.Cleaner.Services; using EnvironmentSetup; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { + var builder = Host.CreateApplicationBuilder(args); - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); - + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); + builder.Services.AddSingleton( new RedundantFieldsWrapper( builder.Configuration.GetValue("RedundantOfflocFields")! )); builder.Services.AddSingleton(); - - builder.Services.ConfigureServices(builder.Configuration); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddHostedService(); diff --git a/src/Offloc.Cleaner/Properties/launchSettings.json b/src/Offloc.Cleaner/Properties/launchSettings.json deleted file mode 100644 index 374e514..0000000 --- a/src/Offloc.Cleaner/Properties/launchSettings.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "profiles": { - "Offloc.Cleaner": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Project" - } - } -} \ No newline at end of file diff --git a/src/Offloc.Cleaner/Services/CleaningStrategyBase.cs b/src/Offloc.Cleaner/Services/CleaningStrategyBase.cs index 64a32c8..6c03b45 100644 --- a/src/Offloc.Cleaner/Services/CleaningStrategyBase.cs +++ b/src/Offloc.Cleaner/Services/CleaningStrategyBase.cs @@ -1,32 +1,12 @@ - -using FileStorage; -using Messaging.Interfaces; -using Offloc.Cleaner.Cleaners; +using Offloc.Cleaner.Cleaners; using Serilog; namespace Offloc.Cleaner.Services; -public abstract class CleaningStrategyBase +public abstract class CleaningStrategyBase(int[] redundantFieldIndexes) { - protected IStagingMessagingService stagingService; - protected IStatusMessagingService statusService; - protected IFileLocations fileLocations; - - protected int[] redundantFieldIndexes; - - public CleaningStrategyBase(IStagingMessagingService stagingService, IStatusMessagingService statusService, - IFileLocations fileLocations, int[] redundantFieldIndexes) + public void ProcessFile(string file) { - this.stagingService = stagingService; - this.statusService = statusService; - this.redundantFieldIndexes = redundantFieldIndexes; - this.fileLocations = fileLocations; - } - - public virtual async Task ProcessFile(string file) - { - await Task.CompletedTask; - FileCleaner fc = new FileCleaner(file, redundantFieldIndexes); // We are here. Lets try to clean the line var clean = fc.Clean(); diff --git a/src/Offloc.Cleaner/Services/SequentialCleaningStrategy.cs b/src/Offloc.Cleaner/Services/SequentialCleaningStrategy.cs index add9f8a..afdccaf 100644 --- a/src/Offloc.Cleaner/Services/SequentialCleaningStrategy.cs +++ b/src/Offloc.Cleaner/Services/SequentialCleaningStrategy.cs @@ -3,22 +3,19 @@ using Messaging.Interfaces; using Messaging.Messages.StagingMessages; using Messaging.Messages.StatusMessages; -using System.Globalization; namespace Offloc.Cleaner.Services; -public class SequentialCleaningStrategy : CleaningStrategyBase, ICleaningStrategy -{ public SequentialCleaningStrategy(IStagingMessagingService stagingService, - IStatusMessagingService statusService, IFileLocations fileLocations, - RedundantFieldsWrapper redundantWrapper) - : base(stagingService, statusService, fileLocations, - redundantWrapper.redundantFieldIndexes){ } - +public class SequentialCleaningStrategy( + IMessageService messageService, + IFileLocations fileLocations, + RedundantFieldsWrapper redundantWrapper) : CleaningStrategyBase(redundantWrapper.redundantFieldIndexes), ICleaningStrategy +{ public async Task CleanFile(string file) { - await statusService.StatusPublishAsync(new StatusUpdateMessage($"Cleaning file: {file}")); - await base.ProcessFile(Path.Combine(fileLocations.offlocInput, file)); - await stagingService.StagingPublishAsync(new OfflocCleanerFinishedMessage([file], redundantFieldIndexes)); + await messageService.PublishAsync(new StatusUpdateMessage($"Cleaning file: {file}")); + ProcessFile(Path.Combine(fileLocations.offlocInput, file)); + await messageService.PublishAsync(new OfflocCleanerFinishedMessage([file], redundantWrapper.redundantFieldIndexes)); } } \ No newline at end of file diff --git a/src/Offloc.Cleaner/appsettings.json b/src/Offloc.Cleaner/appsettings.json index 9686cab..ac236aa 100644 --- a/src/Offloc.Cleaner/appsettings.json +++ b/src/Offloc.Cleaner/appsettings.json @@ -1,6 +1,28 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Offloc.Cleaner-.txt", - "EnvFilePath": "../", + "ConnectionStrings": { + "RabbitMQ": "" + }, + "RedundantOfflocFields": "Age, Occupation Description, Check Hold Governor, Check Hold General (to be left blank), Check Hold Discipline (to be left blank), Check Hold Allocation, Check Hold Security, Check Hold Medical, Check Hold Parole, ACCT Status (F2052), Status Rank (to be left blank), Pending Transfers (Full Establishment Name), Received From, Vulnerable Prisoner Alert, Height (metres), Complexion, Hair Colour, Left Eye, Right Eye, Build, Facial Shape, Facial Hair, Physical Mark Head, Physical Mark Body, Rule 45/YOI Rule 49, ACCT (Self Harm) Status, ACCT (Self Harm) Start Date, Remark Type Allocation, Remarks Allocation, Remark Type Security, Remarks Security, Remark Type Medical, Remarks Medical, Remark Type Parole, Remarks Parole, Remark Type Discipline, Remarks Discipline, Remark Type General, Remarks General, Remark Type Reception, Remarks Reception, Remark Type Labour, Remarks Labour, Date Of First Movement, Diary Details", + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/Offloc.Cleaner-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] + }, //Indexing here assumes 0-indexing. Corresponding Id in Offloc definition is 1 greater. "fieldsToIgnore": [ 11, diff --git a/src/Offloc.Parser/Offloc.Parser.csproj b/src/Offloc.Parser/Offloc.Parser.csproj index 4c0d382..0b8386a 100644 --- a/src/Offloc.Parser/Offloc.Parser.csproj +++ b/src/Offloc.Parser/Offloc.Parser.csproj @@ -4,8 +4,6 @@ Exe enable enable - Linux - ..\..\.. diff --git a/src/Offloc.Parser/OfflocParserBackgroundService.cs b/src/Offloc.Parser/OfflocParserBackgroundService.cs index 32c9931..d219056 100644 --- a/src/Offloc.Parser/OfflocParserBackgroundService.cs +++ b/src/Offloc.Parser/OfflocParserBackgroundService.cs @@ -1,5 +1,4 @@ - -using Messaging.Interfaces; +using Messaging.Interfaces; using Messaging.Messages.StagingMessages; using Messaging.Queues; using Microsoft.Extensions.Hosting; @@ -7,28 +6,14 @@ namespace Offloc.Parser; -public class OfflocParserBackgroundService : BackgroundService +public class OfflocParserBackgroundService( + IMessageService messageService, + IParsingStrategy parsingService) : BackgroundService { - private readonly IStagingMessagingService messageService; - private readonly IParsingStrategy parsingService; - private readonly IStatusMessagingService statusService; - - public OfflocParserBackgroundService(IStagingMessagingService messageService, - IParsingStrategy parsingService, IStatusMessagingService statusService) - { - this.messageService = messageService; - this.parsingService = parsingService; - this.statusService = statusService; - } - protected override async Task ExecuteAsync(CancellationToken stoppingToken) { - await Task.CompletedTask; - - messageService.StagingSubscribeAsync(async(message) => + await messageService.SubscribeAsync(async message => { - //For parallel processing, the file id is insignificant. - //For sequential processing, filesToParse is always of length 1. await parsingService.ParseFiles(message.filesToParse); }, TStagingQueue.OfflocParser); } diff --git a/src/Offloc.Parser/Program.cs b/src/Offloc.Parser/Program.cs index ee40367..5eff30d 100644 --- a/src/Offloc.Parser/Program.cs +++ b/src/Offloc.Parser/Program.cs @@ -1,49 +1,22 @@ - +using Messaging.Extensions; using EnvironmentSetup; -using Messaging.Interfaces; -using Messaging.Services; -using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Offloc.Parser; -using Offloc.Parser.Configuration; -using Offloc.Parser.Processor; using Offloc.Parser.Services; using Offloc.Parser.Services.TrimmerContext; -using Offloc.Parser.Writers.Factory; using Serilog; -Log.Logger = new LoggerConfiguration() - .Enrich.FromLogContext() - .WriteTo.Console() - .WriteTo.File(@".\logs\fatal.txt", Serilog.Events.LogEventLevel.Fatal) - .CreateBootstrapLogger(); - try { - HostApplicationBuilder builder = Host.CreateApplicationBuilder(args); - - builder.Configuration.AddJsonFile("appsettings.json").AddEnvironmentVariables(); - builder.Configuration.ConfigureByEnvironment(); + var builder = Host.CreateApplicationBuilder(args); - builder.Services.AddSingleton(); + builder.AddDmsCoreWorkerService(); + builder.Services.AddDmsRabbitMQ(builder.Configuration); - //Extremely bad practice but just - builder.Services.AddSingleton(builder.Configuration); - - builder.Services.ConfigureServices(builder.Configuration); - builder.Services.AddSingleton(sp => - { - var rabbitContext = sp.GetRequiredService(); - return RabbitService.CreateAsync(rabbitContext).GetAwaiter().GetResult(); - }); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); - builder.Services.AddSingleton(sp => sp.GetRequiredService()); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); - //builder.Services.AddSingleton(); - //builder.Services.AddSingleton(); - //builder.Services.AddSingleton(); builder.Services.AddHostedService(); diff --git a/src/Offloc.Parser/Properties/launchSettings.json b/src/Offloc.Parser/Properties/launchSettings.json deleted file mode 100644 index f0489ae..0000000 --- a/src/Offloc.Parser/Properties/launchSettings.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "profiles": { - "Offloc.Parser": { - "commandName": "Project", - "environmentVariables": { - "DOTNET_ENVIRONMENT": "Development", - "SYSTEM_ENVIRONMENT": "Development" - } - }, - "Docker": { - "commandName": "Project" - } - } -} \ No newline at end of file diff --git a/src/Offloc.Parser/Services/ParsingStrategy/ParsingStrategyBase.cs b/src/Offloc.Parser/Services/ParsingStrategy/ParsingStrategyBase.cs index b216414..6fedf7b 100644 --- a/src/Offloc.Parser/Services/ParsingStrategy/ParsingStrategyBase.cs +++ b/src/Offloc.Parser/Services/ParsingStrategy/ParsingStrategyBase.cs @@ -7,26 +7,14 @@ namespace Offloc.Parser.Services; -public class ParsingStrategyBase +public class ParsingStrategyBase(IMessageService messageService, IFileLocations fileLocations, FieldTrimmerContext trimmerContext) { - protected IStagingMessagingService stagingService; - protected IStatusMessagingService statusService; - protected IFileLocations fileLocations; - - private FieldTrimmerContext trimmerContext; - - public ParsingStrategyBase(IStagingMessagingService stagingService, IStatusMessagingService statusService, - IFileLocations fileLocations, FieldTrimmerContext trimmerContext) - { - this.stagingService = stagingService; - this.statusService = statusService; - this.trimmerContext = trimmerContext; - this.fileLocations = fileLocations; - } + protected IMessageService messageService = messageService; + protected IFileLocations fileLocations = fileLocations; protected async Task ParseFile(string fileName) { - await statusService.StatusPublishAsync(new StatusUpdateMessage($"Offloc parser started for file {fileName.Split('/').Last()}.")); + await messageService.PublishAsync(new StatusUpdateMessage($"Offloc parser started for file {fileName.Split('/').Last()}.")); OfflocProcessor op = new OfflocProcessor(fileName, $"{fileLocations.offlocOutput}/{fileName.Split('/').Last()}", trimmerContext); await op.Process(); diff --git a/src/Offloc.Parser/Services/ParsingStrategy/SequentialParsingStrategy.cs b/src/Offloc.Parser/Services/ParsingStrategy/SequentialParsingStrategy.cs index a619025..74674c8 100644 --- a/src/Offloc.Parser/Services/ParsingStrategy/SequentialParsingStrategy.cs +++ b/src/Offloc.Parser/Services/ParsingStrategy/SequentialParsingStrategy.cs @@ -5,15 +5,13 @@ namespace Offloc.Parser.Services; -public class SequentialParsingStrategy : ParsingStrategyBase, IParsingStrategy +public class SequentialParsingStrategy( + IMessageService messageService, + IFileLocations fileLocations, + FieldTrimmerContext trimmerContext) : ParsingStrategyBase(messageService, fileLocations, trimmerContext), IParsingStrategy { private static SemaphoreSlim sem = new SemaphoreSlim(1, 1); - public SequentialParsingStrategy(IStagingMessagingService stagingService, - IStatusMessagingService statusService, IFileLocations fileLocations, - FieldTrimmerContext trimmerContext) - : base(stagingService, statusService, fileLocations, trimmerContext) { } - public async Task ParseFiles(string[] files) { if (files.Length != 1) @@ -25,7 +23,7 @@ public async Task ParseFiles(string[] files) await sem.WaitAsync(); await ParseFile(fileLocations.offlocInput + '/' + files[0]); - await stagingService.StagingPublishAsync(new OfflocParserFinishedMessage(files[0].Split('/').Last(), false)); + await messageService.PublishAsync(new OfflocParserFinishedMessage(files[0].Split('/').Last(), false)); sem.Release(); } diff --git a/src/Offloc.Parser/appsettings.json b/src/Offloc.Parser/appsettings.json index 64a3d25..0266e8c 100644 --- a/src/Offloc.Parser/appsettings.json +++ b/src/Offloc.Parser/appsettings.json @@ -1,11 +1,27 @@ { - "Serilog:WriteTo:1:Args:path": "%USERPROFILE%/DMS/logs/Offloc.Parser-.txt", - "EnvFilePath": "../", - "Logging": { - "LogLevel": { - "Default": "Warning", - "Microsoft.Hosting.Lifetime": "Warning" - } + "ConnectionStrings": { + "RabbitMQ": "" + }, + "RedundantOfflocFields": "Age, Occupation Description, Check Hold Governor, Check Hold General (to be left blank), Check Hold Discipline (to be left blank), Check Hold Allocation, Check Hold Security, Check Hold Medical, Check Hold Parole, ACCT Status (F2052), Status Rank (to be left blank), Pending Transfers (Full Establishment Name), Received From, Vulnerable Prisoner Alert, Height (metres), Complexion, Hair Colour, Left Eye, Right Eye, Build, Facial Shape, Facial Hair, Physical Mark Head, Physical Mark Body, Rule 45/YOI Rule 49, ACCT (Self Harm) Status, ACCT (Self Harm) Start Date, Remark Type Allocation, Remarks Allocation, Remark Type Security, Remarks Security, Remark Type Medical, Remarks Medical, Remark Type Parole, Remarks Parole, Remark Type Discipline, Remarks Discipline, Remark Type General, Remarks General, Remark Type Reception, Remarks Reception, Remark Type Labour, Remarks Labour, Date Of First Movement, Diary Details", + "Serilog": { + "MinimumLevel": { + "Default": "Information", + "Override": { + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/Offloc.Parser-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, //Indexing here assumes 0-indexing. Corresponding Id in Offloc definition is 1 greater. "fieldsToIgnore": [ diff --git a/src/Visualiser/.dockerignore b/src/Visualiser/.dockerignore deleted file mode 100644 index fe1152b..0000000 --- a/src/Visualiser/.dockerignore +++ /dev/null @@ -1,30 +0,0 @@ -**/.classpath -**/.dockerignore -**/.env -**/.git -**/.gitignore -**/.project -**/.settings -**/.toolstarget -**/.vs -**/.vscode -**/*.*proj.user -**/*.dbmdl -**/*.jfm -**/azds.yaml -**/bin -**/charts -**/docker-compose* -**/Dockerfile* -**/node_modules -**/npm-debug.log -**/obj -**/secrets.dev.yaml -**/values.dev.yaml -LICENSE -README.md -!**/.gitignore -!.git/HEAD -!.git/config -!.git/packed-refs -!.git/refs/heads/** \ No newline at end of file diff --git a/src/Visualiser/Dockerfile b/src/Visualiser/Dockerfile deleted file mode 100644 index ef68649..0000000 --- a/src/Visualiser/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -# See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging. - -# This stage is used when running from VS in fast mode (Default for Debug configuration) -FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS base -USER $APP_UID -WORKDIR /app -EXPOSE 8080 -EXPOSE 8081 - - -# This stage is used to build the service project -FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build -ARG BUILD_CONFIGURATION=Release -WORKDIR /src -COPY ["Visualiser.csproj", "."] -RUN dotnet restore "./Visualiser.csproj" -COPY . . -WORKDIR "/src/." -RUN dotnet build "./Visualiser.csproj" -c $BUILD_CONFIGURATION -o /app/build - -# This stage is used to publish the service project to be copied to the final stage -FROM build AS publish -ARG BUILD_CONFIGURATION=Release -RUN dotnet publish "./Visualiser.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false - -# This stage is used in production or when running from VS in regular mode (Default when not using the Debug configuration) -FROM base AS final -WORKDIR /app -COPY --from=publish /app/publish . -ENTRYPOINT ["dotnet", "Visualiser.dll"] \ No newline at end of file diff --git a/src/Visualiser/Program.cs b/src/Visualiser/Program.cs index 1382cb9..b325355 100644 --- a/src/Visualiser/Program.cs +++ b/src/Visualiser/Program.cs @@ -1,3 +1,4 @@ +using EnvironmentSetup; using Microsoft.AspNetCore.Authentication.OpenIdConnect; using Microsoft.AspNetCore.HttpOverrides; using Microsoft.Identity.Web; @@ -5,6 +6,8 @@ var builder = WebApplication.CreateBuilder(args); +builder.UseDmsSerilog(); + builder.Services.AddHealthChecks(); builder.Services.AddAuthentication(OpenIdConnectDefaults.AuthenticationScheme) diff --git a/src/Visualiser/Properties/launchSettings.json b/src/Visualiser/Properties/launchSettings.json index 14eadb8..8735697 100644 --- a/src/Visualiser/Properties/launchSettings.json +++ b/src/Visualiser/Properties/launchSettings.json @@ -8,17 +8,6 @@ }, "dotnetRunMessages": true, "applicationUrl": "https://localhost:7123;http://localhost:5015" - }, - "Container (Dockerfile)": { - "commandName": "Docker", - "launchBrowser": true, - "launchUrl": "{Scheme}://{ServiceHost}:{ServicePort}", - "environmentVariables": { - "ASPNETCORE_HTTPS_PORTS": "8081", - "ASPNETCORE_HTTP_PORTS": "8080" - }, - "publishAllPorts": true, - "useSSL": true } }, "$schema": "https://json.schemastore.org/launchsettings.json" diff --git a/src/Visualiser/Visualiser.csproj b/src/Visualiser/Visualiser.csproj index 623fd3c..4671b49 100644 --- a/src/Visualiser/Visualiser.csproj +++ b/src/Visualiser/Visualiser.csproj @@ -5,8 +5,6 @@ enable Visualiser 6b76fc04-e0a8-4265-b2ac-d8600fd901dc - Linux - . @@ -15,6 +13,14 @@ + + + + + + + + diff --git a/src/Visualiser/appsettings.json b/src/Visualiser/appsettings.json index 2753e63..52a047e 100644 --- a/src/Visualiser/appsettings.json +++ b/src/Visualiser/appsettings.json @@ -20,11 +20,26 @@ For more info see https://aka.ms/dotnet-template-ms-identity-platform // E.g. "api://{api_client_id}/dms.read" and "api://{api_client_id}/dms.write" ] }, - "Logging": { - "LogLevel": { + "Serilog": { + "MinimumLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } + "Override": { + "Microsoft.AspNetCore": "Warning", + "Microsoft": "Information", + "System": "Warning" + } + }, + "WriteTo": [ + { "Name": "Console" }, + { + "Name": "File", + "Args": { + "path": "../logs/Visualiser-.txt", + "rollingInterval": "Day" + } + } + ], + "Enrich": ["FromLogContext", "WithMachineName", "WithProcessId", "WithThreadId"] }, "AllowedHosts": "*" } diff --git a/tests/Messaging.Tests/Messaging.Tests.csproj b/tests/Messaging.Tests/Messaging.Tests.csproj new file mode 100644 index 0000000..3d34681 --- /dev/null +++ b/tests/Messaging.Tests/Messaging.Tests.csproj @@ -0,0 +1,27 @@ + + + + enable + enable + false + true + + + + + + + + + + + + + + + + + + + + diff --git a/tests/Messaging.Tests/RabbitContainerFixture.cs b/tests/Messaging.Tests/RabbitContainerFixture.cs new file mode 100644 index 0000000..d1cad3e --- /dev/null +++ b/tests/Messaging.Tests/RabbitContainerFixture.cs @@ -0,0 +1,11 @@ +using Testcontainers.RabbitMq; +using Testcontainers.Xunit; +using Xunit.Abstractions; + +namespace Messaging.Tests; + +public sealed class RabbitContainerFixture(IMessageSink messageSink) + : ContainerFixture(messageSink) +{ + protected override RabbitMqBuilder Configure(RabbitMqBuilder builder) => builder.WithImage("rabbitmq:3"); +} \ No newline at end of file diff --git a/tests/Messaging.Tests/RabbitServiceIntegrationTests.cs b/tests/Messaging.Tests/RabbitServiceIntegrationTests.cs new file mode 100644 index 0000000..246b3ef --- /dev/null +++ b/tests/Messaging.Tests/RabbitServiceIntegrationTests.cs @@ -0,0 +1,149 @@ +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using Messaging.Services; +using Messaging.Tests.TestMessages; + +namespace Messaging.Tests; + +public class RabbitServiceIntegrationTests(RabbitContainerFixture fixture) : IAsyncLifetime, IClassFixture +{ + private RabbitService? _rabbitService; + + public async Task InitializeAsync() + { + var connectionString = new Uri(fixture.Container.GetConnectionString()); + _rabbitService = await RabbitService.CreateAsync(connectionString); + } + + [Fact] + public async Task PublishAndSubscribe_SerializesAndDeserializesMessage_Correctly() + { + // Arrange + var completionSource = new TaskCompletionSource(); + + await _rabbitService!.SubscribeAsync( + completionSource.SetResult, + TMatchingQueue.ClusteringPostProcessingFinished + ); + + var publishedMessage = new TestMessage(TMatchingQueue.ClusteringPostProcessingFinished.ToString(), Exchanges.matching) + { + CustomProperty = "Test Value" + }; + + // Act + await _rabbitService.PublishAsync(publishedMessage); + + var receivedMessage = await completionSource.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // Assert + Assert.NotNull(receivedMessage); + Assert.Equal(publishedMessage.CustomProperty, receivedMessage.CustomProperty); + Assert.Equal(publishedMessage.RoutingKey, receivedMessage.RoutingKey); + } + + [Fact] + public async Task PublishAsync_WithStatusMessageSet_PublishesStatusMessage() + { + // Arrange + var messageCompletionSource = new TaskCompletionSource(); + var statusCompletionSource = new TaskCompletionSource(); + + await _rabbitService!.SubscribeAsync( + messageCompletionSource.SetResult, + TStagingQueue.DeliusCleanup + ); + + await _rabbitService.SubscribeAsync( + statusCompletionSource.SetResult, + TStatusQueue.StatusUpdate + ); + + var publishedMessage = new TestMessage(TStagingQueue.DeliusCleanup.ToString(), Exchanges.staging) + { + CustomProperty = "Test Value", + Status = "Test completed successfully" + }; + + // Act + await _rabbitService.PublishAsync(publishedMessage); + + var receivedMessage = await messageCompletionSource.Task.WaitAsync(TimeSpan.FromSeconds(5)); + var receivedStatus = await statusCompletionSource.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // Assert + Assert.NotNull(receivedMessage); + Assert.Equal("Test Value", receivedMessage.CustomProperty); + + Assert.NotNull(receivedStatus); + Assert.Equal("Test completed successfully", receivedStatus.Message); + } + + [Fact] + public async Task PublishAsync_WithoutStatusMessageSet_DoesNotPublishStatusMessage() + { + // Arrange + var messageCompletionSource = new TaskCompletionSource(); + var statusCompletionSource = new TaskCompletionSource(); + + await _rabbitService!.SubscribeAsync( + messageCompletionSource.SetResult, + TImportQueue.ImportFinished + ); + + await _rabbitService.SubscribeAsync( + statusCompletionSource.SetResult, + TStatusQueue.StatusUpdate + ); + + var publishedMessage = new TestMessage(TImportQueue.ImportFinished.ToString(), Exchanges.import) + { + CustomProperty = "Test Value" + // Status is not set (null or empty string) + }; + + // Act + await _rabbitService.PublishAsync(publishedMessage); + + var receivedMessage = await messageCompletionSource.Task.WaitAsync(TimeSpan.FromSeconds(5)); + + // Assert - message should be received + Assert.NotNull(receivedMessage); + Assert.Equal("Test Value", receivedMessage.CustomProperty); + + // Assert - status message should NOT be received (timeout should occur) + await Assert.ThrowsAsync(async () => + { + await statusCompletionSource.Task.WaitAsync(TimeSpan.FromSeconds(2)); + }); + } + + + [Fact] + public async Task SendDbRequestAndWaitForResponseAsync_SendsRequestAndReceivesResponse_Successfully() + { + // Arrange + var requestData = "RequestData"; + var expectedResponseData = $"Echo: {requestData}"; + await _rabbitService!.SubscribeAsync( + msg => + { + Task.Run(async () => + { + await Task.Delay(250); // Simulate processing delay + var response = new TestDbResponseMessage($"Echo: {msg.RequestData}", true); + await _rabbitService.PublishAsync(response); + }); + }, + TDbQueue.GetProcessedDeliusFiles + ); + // Act + var response = await _rabbitService.SendDbRequestAndWaitForResponseAsync(new TestDbRequestMessage(requestData)); + + // Assert + Assert.Equal(expectedResponseData, response.ResponseData); + Assert.True(response.Success); + } + + public Task DisposeAsync() => Task.CompletedTask; +} \ No newline at end of file diff --git a/tests/Messaging.Tests/TestMessages/TestDbRequestMessage.cs b/tests/Messaging.Tests/TestMessages/TestDbRequestMessage.cs new file mode 100644 index 0000000..24ef826 --- /dev/null +++ b/tests/Messaging.Tests/TestMessages/TestDbRequestMessage.cs @@ -0,0 +1,24 @@ +using Messaging.Messages.DbMessages.Sending; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using System.Text.Json.Serialization; + +namespace Messaging.Tests.TestMessages; + +public class TestDbRequestMessage : DbRequestMessage +{ + public string RequestData { get; set; } = string.Empty; + + public override StatusUpdateMessage StatusMessage => new(); + + [JsonConstructor] + public TestDbRequestMessage() + { + Queue = TDbQueue.GetProcessedDeliusFiles; + } + + public TestDbRequestMessage(string requestData) : this() + { + RequestData = requestData; + } +} diff --git a/tests/Messaging.Tests/TestMessages/TestDbResponseMessage.cs b/tests/Messaging.Tests/TestMessages/TestDbResponseMessage.cs new file mode 100644 index 0000000..ce7b2cb --- /dev/null +++ b/tests/Messaging.Tests/TestMessages/TestDbResponseMessage.cs @@ -0,0 +1,26 @@ +using Messaging.Messages.DbMessages.Receiving; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; +using System.Text.Json.Serialization; + +namespace Messaging.Tests.TestMessages; + +public class TestDbResponseMessage : DbResponseMessage +{ + public string ResponseData { get; set; } = string.Empty; + public bool Success { get; set; } + + public override StatusUpdateMessage StatusMessage => new(); + + [JsonConstructor] + public TestDbResponseMessage() + { + Queue = TDbQueue.ReturnedDeliusFiles; + } + + public TestDbResponseMessage(string responseData, bool success) : this() + { + ResponseData = responseData; + Success = success; + } +} diff --git a/tests/Messaging.Tests/TestMessages/TestMessage.cs b/tests/Messaging.Tests/TestMessages/TestMessage.cs new file mode 100644 index 0000000..4eb4f1e --- /dev/null +++ b/tests/Messaging.Tests/TestMessages/TestMessage.cs @@ -0,0 +1,18 @@ + +using Messaging.Messages; +using Messaging.Messages.StatusMessages; +using Messaging.Queues; + +namespace Messaging.Tests.TestMessages; + +public class TestMessage(string routingKey, string exchange) : Message +{ + public required string CustomProperty { get; set; } + public string? Status { get; set; } + + public override StatusUpdateMessage StatusMessage => new(Status ?? string.Empty); + + public override string Exchange => exchange; + + public override string RoutingKey => routingKey; +} \ No newline at end of file diff --git a/tests/Messaging.Tests/TestMessages/TestStatusMessage.cs b/tests/Messaging.Tests/TestMessages/TestStatusMessage.cs new file mode 100644 index 0000000..4515ad0 --- /dev/null +++ b/tests/Messaging.Tests/TestMessages/TestStatusMessage.cs @@ -0,0 +1,9 @@ +using Messaging.Messages.StatusMessages; + +namespace Messaging.Tests.TestMessages; + +public class TestStatusMessage : StatusUpdateMessage +{ + public required string CustomProperty { get; set; } + public required DateTime Timestamp { get; set; } +}